structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,420 @@
1
+ # Copyright (c) 2025, Soda team @ Inria
2
+ # Licensed under the BSD 3-Clause License; see LICENSE.
3
+
4
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5
+ # SPDX-License-Identifier: Apache-2.0
6
+
7
+ # ruff: noqa: D101, D102
8
+
9
+ import math
10
+ from typing import Any, TypeAlias, cast
11
+
12
+ import torch
13
+ from torch import Tensor
14
+ from torch.nn import GELU, Embedding, LayerNorm, Linear, ModuleList, Sequential
15
+
16
+ from sdm.cache import Cache, KVCacheEntry
17
+ from sdm.models.tabiclv2.block import TabICLv2TransformerBlock
18
+
19
+ _Node: TypeAlias = dict[str, Tensor | list["_Node"]]
20
+
21
+
22
+ class ICLBlock(torch.nn.Module):
23
+ def __init__(
24
+ self,
25
+ num_classes: int,
26
+ out_channels: int,
27
+ channels: int,
28
+ num_layers: int,
29
+ num_heads: int,
30
+ norm_bias: bool,
31
+ temperature: float = 1.0,
32
+ device: torch.device | str | None = None,
33
+ dtype: torch.dtype | None = None,
34
+ ) -> None:
35
+ super().__init__()
36
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
37
+
38
+ self.num_classes = num_classes
39
+ self.temperature = temperature
40
+
41
+ self.y_emb: torch.nn.Module | None = None
42
+ self.y_lin: torch.nn.Module | None = None
43
+ if num_classes > 0:
44
+ self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
45
+ else:
46
+ self.y_lin = Linear(1, channels, **factory_kwargs)
47
+
48
+ self.layers = ModuleList(
49
+ TabICLv2TransformerBlock(
50
+ channels=channels,
51
+ num_heads=num_heads,
52
+ norm_bias=norm_bias,
53
+ qassmax=True,
54
+ **factory_kwargs,
55
+ )
56
+ for _ in range(num_layers)
57
+ )
58
+
59
+ self.norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs)
60
+ self.head = Sequential(
61
+ Linear(channels, 2 * channels, **factory_kwargs),
62
+ GELU(),
63
+ Linear(2 * channels, out_channels, **factory_kwargs),
64
+ )
65
+
66
+ def forward(
67
+ self,
68
+ x: Tensor, # [..., R, D]
69
+ y: Tensor, # [..., R_train]
70
+ *,
71
+ num_classes: int | None = None,
72
+ cache: Cache | None = None,
73
+ ) -> Tensor: # [..., R_test, out_channels or num_classes]
74
+ if num_classes is None or num_classes <= self.num_classes:
75
+ return self._forward(
76
+ x=x,
77
+ y=y,
78
+ cache=cache,
79
+ cache_prefix="icl_block",
80
+ )
81
+
82
+ if self.num_classes < 2:
83
+ raise ValueError(
84
+ "Hierarchical classification requires 'num_classes' to be "
85
+ "at least two"
86
+ )
87
+
88
+ return self._forward_hierarchical(
89
+ x=x,
90
+ y=y,
91
+ num_classes=num_classes,
92
+ cache=cache,
93
+ )
94
+
95
+ def _forward(
96
+ self,
97
+ x: Tensor, # [..., R, D]
98
+ y: Tensor, # [..., R_train]
99
+ *,
100
+ cache: Cache | None,
101
+ cache_prefix: str,
102
+ ) -> Tensor: # [..., R_test, out_channels]
103
+ R_train = y.size(-1)
104
+
105
+ if y.numel() > 0:
106
+ if self.y_emb is not None:
107
+ y_emb = self.y_emb(y) # [..., R_train, D]
108
+ else:
109
+ assert self.y_lin is not None
110
+ y_emb = self.y_lin(y.unsqueeze(-1)) # [..., R_train, D]
111
+
112
+ x[..., :R_train, :] += y_emb.to(x.dtype)
113
+
114
+ for i, layer in enumerate(self.layers):
115
+ key = f"{cache_prefix}.layer{i}"
116
+ result = layer(
117
+ query=x[..., R_train:, :] if i == len(self.layers) - 1 else x,
118
+ key_value=(
119
+ cast(KVCacheEntry, cache[key])
120
+ if cache is not None and cache.is_replaying
121
+ else x[..., :R_train, :]
122
+ ),
123
+ return_key_value=cache is not None and cache.is_recording,
124
+ # `x` is still the caller's tensor at i == 0; don't mutate.
125
+ out=None
126
+ if torch.is_grad_enabled() or i == 0
127
+ else x[..., R_train:, :]
128
+ if i == len(self.layers) - 1
129
+ else x,
130
+ )
131
+
132
+ if cache is not None and cache.is_recording:
133
+ x, cache[key] = result
134
+ else:
135
+ x = result
136
+ del result
137
+
138
+ return self.head(self.norm(x)) # [..., R_test, out_channels]
139
+
140
+ def _forward_hierarchical(
141
+ self,
142
+ x: Tensor, # [..., R, D]
143
+ y: Tensor, # [..., R_train]
144
+ *,
145
+ num_classes: int,
146
+ cache: Cache | None,
147
+ ) -> Tensor: # [..., R_test, C]
148
+ *batch_shape, num_rows, channels = x.size()
149
+ train_size = y.size(-1)
150
+ test_size = num_rows - train_size
151
+
152
+ if 0 in batch_shape:
153
+ empty = x[..., train_size:, :].sum(dim=-1, keepdim=True)
154
+ return empty.expand(*batch_shape, test_size, num_classes)
155
+
156
+ num_tables = math.prod(batch_shape)
157
+ flat_rows = x.reshape(num_tables, num_rows, channels)
158
+
159
+ # Tree nodes have different row counts, so tables and recursive model
160
+ # calls cannot be represented by one dense tensor operation.
161
+ if cache is not None and cache.is_replaying:
162
+ trees = cast("list[_Node]", cache["icl_block.trees"])
163
+ if len(trees) != num_tables:
164
+ raise RuntimeError(
165
+ f"Expected {len(trees)} cached tables (got {num_tables})"
166
+ )
167
+ table_outputs = [
168
+ self._replay_table(
169
+ test_rows=rows[train_size:],
170
+ node=tree,
171
+ num_classes=num_classes,
172
+ cache=cache,
173
+ cache_prefix=f"icl_block.table{table_idx}.node",
174
+ )
175
+ for table_idx, (rows, tree) in enumerate(zip(flat_rows, trees))
176
+ ]
177
+ else:
178
+ flat_y = y.reshape(num_tables, train_size)
179
+ trees = []
180
+ table_outputs = []
181
+ for table_idx, (rows, labels) in enumerate(zip(flat_rows, flat_y)):
182
+ class_ids, local_log_probs, tree = self._process_node(
183
+ train_rows=rows[:train_size],
184
+ train_labels=labels,
185
+ test_rows=rows[train_size:],
186
+ cache=cache,
187
+ cache_prefix=f"icl_block.table{table_idx}.node",
188
+ )
189
+ trees.append(tree)
190
+ table_outputs.append(
191
+ self._expand_log_probs(
192
+ class_ids=class_ids,
193
+ local_log_probs=local_log_probs,
194
+ num_classes=num_classes,
195
+ )
196
+ )
197
+
198
+ if cache is not None and cache.is_recording:
199
+ cache["icl_block.trees"] = trees
200
+
201
+ log_probs = torch.stack(table_outputs)
202
+ # The output Softmax divides by temperature, so scale the combined
203
+ # log-probabilities to preserve their normalized probabilities.
204
+ return log_probs.reshape(*batch_shape, test_size, num_classes).mul(
205
+ self.temperature
206
+ )
207
+
208
+ def _replay_table(
209
+ self,
210
+ test_rows: Tensor, # [R_test, D]
211
+ node: _Node,
212
+ *,
213
+ num_classes: int,
214
+ cache: Cache,
215
+ cache_prefix: str,
216
+ ) -> Tensor: # [R_test, C]
217
+ class_ids, local_log_probs = self._replay_node(
218
+ test_rows=test_rows,
219
+ node=node,
220
+ cache=cache,
221
+ cache_prefix=cache_prefix,
222
+ )
223
+ return self._expand_log_probs(
224
+ class_ids=class_ids,
225
+ local_log_probs=local_log_probs,
226
+ num_classes=num_classes,
227
+ )
228
+
229
+ def _process_node(
230
+ self,
231
+ train_rows: Tensor, # [R_node, D]
232
+ train_labels: Tensor, # [R_node]
233
+ test_rows: Tensor, # [R_test, D]
234
+ *,
235
+ cache: Cache | None,
236
+ cache_prefix: str,
237
+ ) -> tuple[Tensor, Tensor, _Node]: # [C_node], [R_test, C_node]
238
+ class_ids, local_labels = train_labels.unique(
239
+ sorted=True,
240
+ return_inverse=True,
241
+ )
242
+ node_num_classes = class_ids.numel()
243
+ node: _Node = {"class_ids": class_ids, "children": []}
244
+
245
+ if node_num_classes <= self.num_classes:
246
+ if node_num_classes == 1:
247
+ local_log_probs = test_rows.sum(
248
+ dim=-1,
249
+ keepdim=True,
250
+ ).mul(0)
251
+ else:
252
+ local_log_probs = self._predict_log_probs(
253
+ x=torch.cat((train_rows, test_rows), dim=0),
254
+ y=local_labels,
255
+ num_classes=node_num_classes,
256
+ cache=cache,
257
+ cache_prefix=cache_prefix,
258
+ )
259
+
260
+ return class_ids, local_log_probs, node
261
+
262
+ class_groups, num_groups = self._grouping(
263
+ num_classes=node_num_classes,
264
+ device=train_labels.device,
265
+ )
266
+ group_labels = class_groups[local_labels]
267
+ group_masks = group_labels == torch.arange(
268
+ num_groups,
269
+ device=group_labels.device,
270
+ ).unsqueeze(-1) # [G, R_node]
271
+ group_log_probs = self._predict_log_probs(
272
+ x=torch.cat((train_rows, test_rows), dim=0),
273
+ y=group_labels,
274
+ num_classes=num_groups,
275
+ cache=cache,
276
+ cache_prefix=cache_prefix,
277
+ )
278
+
279
+ child_class_ids: list[Tensor] = []
280
+ children_log_probs: list[Tensor] = []
281
+ children = cast("list[_Node]", node["children"])
282
+ for group_idx in range(num_groups):
283
+ mask = group_masks[group_idx]
284
+ child_ids, child_log_probs, child = self._process_node(
285
+ train_rows=train_rows[mask],
286
+ train_labels=train_labels[mask],
287
+ test_rows=test_rows,
288
+ cache=cache,
289
+ cache_prefix=f"{cache_prefix}.child{group_idx}",
290
+ )
291
+ child_class_ids.append(child_ids)
292
+ children_log_probs.append(
293
+ child_log_probs + group_log_probs[:, group_idx : group_idx + 1]
294
+ )
295
+ children.append(child)
296
+
297
+ # Balanced groups are contiguous in sorted class order, so concatenated
298
+ # child outputs retain the node's sorted class order.
299
+ return (
300
+ torch.cat(child_class_ids),
301
+ torch.cat(children_log_probs, dim=-1),
302
+ node,
303
+ )
304
+
305
+ def _replay_node(
306
+ self,
307
+ test_rows: Tensor, # [R_test, D]
308
+ node: _Node,
309
+ *,
310
+ cache: Cache,
311
+ cache_prefix: str,
312
+ ) -> tuple[Tensor, Tensor]: # [C_node], [R_test, C_node]
313
+ class_ids = cast(Tensor, node["class_ids"])
314
+ children = cast("list[_Node]", node["children"])
315
+
316
+ if not children:
317
+ if class_ids.numel() == 1:
318
+ local_log_probs = test_rows.sum(
319
+ dim=-1,
320
+ keepdim=True,
321
+ ).mul(0)
322
+ else:
323
+ local_log_probs = self._predict_log_probs(
324
+ x=test_rows,
325
+ y=class_ids.new_empty((0,)),
326
+ num_classes=class_ids.numel(),
327
+ cache=cache,
328
+ cache_prefix=cache_prefix,
329
+ )
330
+ return class_ids, local_log_probs
331
+
332
+ group_log_probs = self._predict_log_probs(
333
+ x=test_rows,
334
+ y=class_ids.new_empty((0,)),
335
+ num_classes=len(children),
336
+ cache=cache,
337
+ cache_prefix=cache_prefix,
338
+ )
339
+ child_class_ids: list[Tensor] = []
340
+ children_log_probs: list[Tensor] = []
341
+ for group_idx, child in enumerate(children):
342
+ child_ids, child_log_probs = self._replay_node(
343
+ test_rows=test_rows,
344
+ node=child,
345
+ cache=cache,
346
+ cache_prefix=f"{cache_prefix}.child{group_idx}",
347
+ )
348
+ child_class_ids.append(child_ids)
349
+ children_log_probs.append(
350
+ child_log_probs + group_log_probs[:, group_idx : group_idx + 1]
351
+ )
352
+
353
+ return (
354
+ torch.cat(child_class_ids),
355
+ torch.cat(children_log_probs, dim=-1),
356
+ )
357
+
358
+ def _predict_log_probs(
359
+ self,
360
+ x: Tensor, # [R_node + R_test, D] or [R_test, D] with cache
361
+ y: Tensor, # [R_node] or [0] with cache
362
+ *,
363
+ num_classes: int,
364
+ cache: Cache | None,
365
+ cache_prefix: str,
366
+ ) -> Tensor: # [R_test, C_node]
367
+ logits = self._forward(
368
+ x=x,
369
+ y=y,
370
+ cache=cache,
371
+ cache_prefix=cache_prefix,
372
+ )
373
+ return (logits[:, :num_classes] / self.temperature).log_softmax(dim=-1)
374
+
375
+ @staticmethod
376
+ def _expand_log_probs(
377
+ class_ids: Tensor, # [C_node]
378
+ local_log_probs: Tensor, # [R_test, C_node]
379
+ *,
380
+ num_classes: int,
381
+ ) -> Tensor: # [R_test, C]
382
+ log_probs = local_log_probs.new_full(
383
+ (local_log_probs.size(-2), num_classes),
384
+ -torch.inf,
385
+ )
386
+ return log_probs.index_copy(
387
+ -1,
388
+ class_ids.to(torch.long),
389
+ local_log_probs,
390
+ )
391
+
392
+ def _grouping(
393
+ self,
394
+ num_classes: int,
395
+ device: torch.device,
396
+ ) -> tuple[Tensor, int]:
397
+ if num_classes <= self.num_classes:
398
+ assignments = torch.zeros(
399
+ num_classes,
400
+ dtype=torch.long,
401
+ device=device,
402
+ )
403
+ return assignments, 1
404
+ num_groups = min(
405
+ (num_classes + self.num_classes - 1) // self.num_classes,
406
+ self.num_classes,
407
+ )
408
+ group_sizes = torch.full(
409
+ (num_groups,),
410
+ num_classes // num_groups,
411
+ dtype=torch.long,
412
+ device=device,
413
+ )
414
+ group_sizes[: num_classes % num_groups] += 1
415
+ group_indices = torch.arange(num_groups, device=device)
416
+ assignments = group_indices.repeat_interleave(
417
+ group_sizes,
418
+ output_size=num_classes,
419
+ )
420
+ return assignments, num_groups
@@ -0,0 +1,293 @@
1
+ # Copyright (c) 2025, Soda team @ Inria
2
+ # Licensed under the BSD 3-Clause License; see LICENSE.
3
+
4
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5
+ # SPDX-License-Identifier: Apache-2.0
6
+
7
+ # ruff: noqa: D205
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Iterable
12
+ from typing import Any, ClassVar, cast
13
+
14
+ import torch
15
+ from torch import Tensor
16
+ from torch.nn import ModuleDict
17
+
18
+ from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
19
+ from sdm.cache import Cache
20
+ from sdm.models import ICLModel
21
+ from sdm.models._huggingface import download_checkpoint
22
+ from sdm.models.tabiclv2.ckpt import remap_ckpt
23
+ from sdm.models.tabiclv2.icl import ICLBlock
24
+ from sdm.models.tabiclv2.recipe import default_recipe
25
+ from sdm.models.tabiclv2.row_embedding import RowEmbedding
26
+
27
+
28
+ class TabICLv2(ICLModel):
29
+ r"""The tabular foundation model from the `"TabICLv2: A Better, Faster,
30
+ Scalable, and Open Tabular Foundation Model"
31
+ <https://arxiv.org/abs/2602.11139>`_ paper.
32
+
33
+ .. figure:: /images/tabicl_light.svg
34
+ :figclass: light-only
35
+ :align: center
36
+ :width: 600px
37
+
38
+ .. figure:: /images/tabicl_dark.svg
39
+ :figclass: dark-only
40
+ :align: center
41
+ :width: 600px
42
+
43
+ :class:`TabICLv2` treats a table as an in-context learning problem: a set
44
+ of labeled training rows provides context, and the model predicts targets
45
+ for held-out test rows from the same table.
46
+
47
+ Feature columns are encoded via repeated feature grouping, where each
48
+ feature participates in multiple shifted feature groups.
49
+ This breaks symmetries between similarly distributed columns while
50
+ preserving fine-grained feature information.
51
+ Target-aware embeddings are then added to the training-row feature
52
+ representations, injecting label information early without giving test rows
53
+ access to their own targets.
54
+
55
+ Afterwards, the module applies three attention stages in sequence:
56
+
57
+ * **Column-wise:** Each grouped feature is processed as a set of row tokens
58
+ with induced set attention.
59
+ The inducing tokens summarize information from the in-context training
60
+ rows, then pass it back to the row tokens, giving each feature group a
61
+ target-aware representation of its values across examples.
62
+ Query-Aware Scalable SoftMax (:class:`~sdm.nn.QASSMax`) sharpens
63
+ attention over long contexts and reduces attention fading as the number
64
+ of rows grows.
65
+ * **Row-wise:** For each row, the feature-group embeddings are processed
66
+ together with learnable readout tokens.
67
+ Attention across the grouped features lets the model combine column
68
+ evidence and feature interactions within that row.
69
+ The readout token outputs are concatenated to form a fixed-size row
70
+ embedding.
71
+ * **Dataset-wise:** The row embeddings are processed across the dataset
72
+ for in-context prediction.
73
+ Training-row embeddings are combined with target embeddings, and test
74
+ rows attend to the labeled training rows. The resulting test-row states
75
+ are mapped to task outputs, such as class probabilities for
76
+ classification or quantile predictions for regression.
77
+
78
+ .. testcode::
79
+
80
+ from sdm import TableTensor
81
+ from sdm.models import TabICLv2
82
+
83
+ table = TableTensor.from_columns(
84
+ {
85
+ "col0": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0],
86
+ "col1": [1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0],
87
+ "target": ["t", "f", "t", "f", "t", None, None, None],
88
+ },
89
+ stypes={
90
+ "col0": "numerical",
91
+ "col1": "numerical",
92
+ "target": "categorical",
93
+ },
94
+ device="cuda",
95
+ )
96
+ model = TabICLv2(device="cuda")
97
+
98
+ # Default in-context learning forward pass:
99
+ out = model(
100
+ x_context=table[:5].drop_columns("target"),
101
+ y_context=table[:5, "target"],
102
+ x_query=table[5:].drop_columns("target"),
103
+ )
104
+ assert out.size() == (3, 2)
105
+
106
+ # Fit+Predict forward pass via key/value caching:
107
+ model.fit(
108
+ x=table[:5].drop_columns("target"),
109
+ y=table[:5, "target"],
110
+ )
111
+ out = model.predict(table[5:].drop_columns("target"))
112
+ assert out.size() == (3, 2)
113
+
114
+ Args:
115
+ task: The tasks to initialize. If ``None``, all tasks supported by this
116
+ model are initialized.
117
+ pretrained: Whether to load the pretrained checkpoint.
118
+ device: The device.
119
+ """
120
+
121
+ supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
122
+ {Stype.numerical}
123
+ )
124
+ supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
125
+ {Stype.numerical, Stype.categorical}
126
+ )
127
+ supports_multi_target: ClassVar[bool] = False
128
+ supports_related_tables: ClassVar[bool] = False
129
+
130
+ def __init__(
131
+ self,
132
+ task: TaskLike | Iterable[TaskLike] | None = None,
133
+ pretrained: bool = True,
134
+ device: torch.device | str | None = None,
135
+ ) -> None:
136
+ super().__init__(task=task)
137
+
138
+ self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
139
+ for task in self.tasks:
140
+ self.models[task] = _TabICLv2(
141
+ num_classes=10 if task == Task.classification else 0,
142
+ num_quantiles=999 if task == Task.regression else 0,
143
+ norm_bias=task == Task.classification,
144
+ device="meta" if pretrained else device,
145
+ )
146
+
147
+ if pretrained:
148
+ self._load_from_pretrained(device=device)
149
+
150
+ self.eval()
151
+
152
+ @classmethod
153
+ def default_recipe(cls) -> Recipe:
154
+ r""":meta private:""" # noqa: D415
155
+ return default_recipe()
156
+
157
+ def _load_from_pretrained(
158
+ self,
159
+ device: torch.device | str | None,
160
+ ) -> TabICLv2:
161
+ device = torch.get_default_device() if device is None else device
162
+
163
+ for task, model in self.models.items():
164
+ if task == Task.classification:
165
+ filename = "tabicl-classifier-v2-20260212.ckpt"
166
+ else:
167
+ assert task == Task.regression
168
+ filename = "tabicl-regressor-v2-20260212.ckpt"
169
+
170
+ ckpt = torch.load(
171
+ download_checkpoint("jingang/TabICL", filename),
172
+ map_location=device,
173
+ weights_only=True,
174
+ )["state_dict"]
175
+ ckpt = remap_ckpt(ckpt, is_classifier=task == Task.classification)
176
+ model.load_state_dict(ckpt, assign=True)
177
+
178
+ return self
179
+
180
+ def _forward(
181
+ self,
182
+ x_context: TableTensor | None, # [..., R_context, D]
183
+ y_context: TableTensor | None, # [..., R_context, 1]
184
+ x_query: TableTensor | None, # [..., R_query, D]
185
+ related_context_tables: RelatedTables[TableTensor] | None,
186
+ related_query_tables: RelatedTables[TableTensor] | None,
187
+ cache: Cache | None,
188
+ generator: torch.Generator | None,
189
+ **kwargs: Any,
190
+ ) -> TableTensor: # [..., R_query, num_classes or 999]
191
+
192
+ if x_query is None and x_context is not None:
193
+ x = x_context.numerical
194
+ elif x_context is None and x_query is not None:
195
+ x = x_query.numerical
196
+ else:
197
+ assert x_context is not None
198
+ assert x_query is not None
199
+ x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
200
+
201
+ y: Tensor | None = None
202
+ classes: Tensor | None = None
203
+ if y_context is not None and y_context.categorical.size(-1) > 0:
204
+ y = y_context.categorical.code.squeeze(-1)
205
+ classes = y_context.categorical.categories[0]
206
+ elif y_context is not None and y_context.numerical.size(-1) > 0:
207
+ y = y_context.numerical.squeeze(-1)
208
+ else:
209
+ assert cache is not None
210
+ classes = cast(Tensor | None, cache["classes"])
211
+ y = x.new_empty(
212
+ (*x.size()[:-2], 0),
213
+ dtype=torch.int64 if classes is not None else x.dtype,
214
+ )
215
+
216
+ if classes is None:
217
+ out = self.models[Task.regression](x, y, cache=cache)
218
+ return TableTensor(
219
+ columns={
220
+ Stype.numerical: [f"q{i:03d}" for i in range(1, 1000)]
221
+ },
222
+ numerical=out,
223
+ )
224
+
225
+ out = self.models[Task.classification](
226
+ x, y, cache=cache, num_classes=len(classes)
227
+ )
228
+ return TableTensor(
229
+ columns={Stype.numerical: [str(i) for i in classes.tolist()]},
230
+ numerical=out[..., : len(classes)],
231
+ )
232
+
233
+
234
+ class _TabICLv2(torch.nn.Module):
235
+ def __init__(
236
+ self,
237
+ num_classes: int,
238
+ num_quantiles: int,
239
+ channels: int = 128,
240
+ num_embedding_layers: int = 3,
241
+ num_embedding_heads: int = 8,
242
+ num_inducing_points: int = 128,
243
+ group_size: int = 3,
244
+ num_readout_tokens: int = 4,
245
+ num_icl_layers: int = 12,
246
+ num_icl_heads: int = 8,
247
+ norm_bias: bool = True,
248
+ device: torch.device | str | None = None,
249
+ dtype: torch.dtype | None = None,
250
+ ) -> None:
251
+ super().__init__()
252
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
253
+
254
+ self.row_embedding = RowEmbedding(
255
+ num_classes=num_classes,
256
+ channels=channels,
257
+ num_layers=num_embedding_layers,
258
+ num_heads=num_embedding_heads,
259
+ group_size=group_size,
260
+ num_inducing_points=num_inducing_points,
261
+ num_readout_tokens=num_readout_tokens,
262
+ norm_bias=norm_bias,
263
+ **factory_kwargs,
264
+ )
265
+ self.icl_block = ICLBlock(
266
+ num_classes=num_classes,
267
+ out_channels=num_classes or num_quantiles,
268
+ channels=num_readout_tokens * channels,
269
+ num_layers=num_icl_layers,
270
+ num_heads=num_icl_heads,
271
+ norm_bias=norm_bias,
272
+ temperature=0.9,
273
+ **factory_kwargs,
274
+ )
275
+
276
+ def forward(
277
+ self,
278
+ x: Tensor, # [..., R, C]
279
+ y: Tensor, # [..., R_train]
280
+ *,
281
+ cache: Cache | None = None,
282
+ num_classes: int | None = None,
283
+ ) -> Tensor: # [..., R_test, out_channels or num_classes]
284
+ if not y.is_floating_point():
285
+ assert num_classes is not None
286
+
287
+ x = self.row_embedding(x, y, num_classes=num_classes, cache=cache)
288
+ return self.icl_block(
289
+ x=x,
290
+ y=y,
291
+ num_classes=num_classes,
292
+ cache=cache,
293
+ )