structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
# Copyright (c) 2025, Soda team @ Inria
|
|
2
|
+
# Licensed under the BSD 3-Clause License; see LICENSE.
|
|
3
|
+
|
|
4
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
6
|
+
|
|
7
|
+
# ruff: noqa: D101, D102
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from typing import Any, TypeAlias, cast
|
|
11
|
+
|
|
12
|
+
import torch
|
|
13
|
+
from torch import Tensor
|
|
14
|
+
from torch.nn import GELU, Embedding, LayerNorm, Linear, ModuleList, Sequential
|
|
15
|
+
|
|
16
|
+
from sdm.cache import Cache, KVCacheEntry
|
|
17
|
+
from sdm.models.tabiclv2.block import TabICLv2TransformerBlock
|
|
18
|
+
|
|
19
|
+
_Node: TypeAlias = dict[str, Tensor | list["_Node"]]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ICLBlock(torch.nn.Module):
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
num_classes: int,
|
|
26
|
+
out_channels: int,
|
|
27
|
+
channels: int,
|
|
28
|
+
num_layers: int,
|
|
29
|
+
num_heads: int,
|
|
30
|
+
norm_bias: bool,
|
|
31
|
+
temperature: float = 1.0,
|
|
32
|
+
device: torch.device | str | None = None,
|
|
33
|
+
dtype: torch.dtype | None = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
super().__init__()
|
|
36
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
37
|
+
|
|
38
|
+
self.num_classes = num_classes
|
|
39
|
+
self.temperature = temperature
|
|
40
|
+
|
|
41
|
+
self.y_emb: torch.nn.Module | None = None
|
|
42
|
+
self.y_lin: torch.nn.Module | None = None
|
|
43
|
+
if num_classes > 0:
|
|
44
|
+
self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
|
|
45
|
+
else:
|
|
46
|
+
self.y_lin = Linear(1, channels, **factory_kwargs)
|
|
47
|
+
|
|
48
|
+
self.layers = ModuleList(
|
|
49
|
+
TabICLv2TransformerBlock(
|
|
50
|
+
channels=channels,
|
|
51
|
+
num_heads=num_heads,
|
|
52
|
+
norm_bias=norm_bias,
|
|
53
|
+
qassmax=True,
|
|
54
|
+
**factory_kwargs,
|
|
55
|
+
)
|
|
56
|
+
for _ in range(num_layers)
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
self.norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs)
|
|
60
|
+
self.head = Sequential(
|
|
61
|
+
Linear(channels, 2 * channels, **factory_kwargs),
|
|
62
|
+
GELU(),
|
|
63
|
+
Linear(2 * channels, out_channels, **factory_kwargs),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def forward(
|
|
67
|
+
self,
|
|
68
|
+
x: Tensor, # [..., R, D]
|
|
69
|
+
y: Tensor, # [..., R_train]
|
|
70
|
+
*,
|
|
71
|
+
num_classes: int | None = None,
|
|
72
|
+
cache: Cache | None = None,
|
|
73
|
+
) -> Tensor: # [..., R_test, out_channels or num_classes]
|
|
74
|
+
if num_classes is None or num_classes <= self.num_classes:
|
|
75
|
+
return self._forward(
|
|
76
|
+
x=x,
|
|
77
|
+
y=y,
|
|
78
|
+
cache=cache,
|
|
79
|
+
cache_prefix="icl_block",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if self.num_classes < 2:
|
|
83
|
+
raise ValueError(
|
|
84
|
+
"Hierarchical classification requires 'num_classes' to be "
|
|
85
|
+
"at least two"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return self._forward_hierarchical(
|
|
89
|
+
x=x,
|
|
90
|
+
y=y,
|
|
91
|
+
num_classes=num_classes,
|
|
92
|
+
cache=cache,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def _forward(
|
|
96
|
+
self,
|
|
97
|
+
x: Tensor, # [..., R, D]
|
|
98
|
+
y: Tensor, # [..., R_train]
|
|
99
|
+
*,
|
|
100
|
+
cache: Cache | None,
|
|
101
|
+
cache_prefix: str,
|
|
102
|
+
) -> Tensor: # [..., R_test, out_channels]
|
|
103
|
+
R_train = y.size(-1)
|
|
104
|
+
|
|
105
|
+
if y.numel() > 0:
|
|
106
|
+
if self.y_emb is not None:
|
|
107
|
+
y_emb = self.y_emb(y) # [..., R_train, D]
|
|
108
|
+
else:
|
|
109
|
+
assert self.y_lin is not None
|
|
110
|
+
y_emb = self.y_lin(y.unsqueeze(-1)) # [..., R_train, D]
|
|
111
|
+
|
|
112
|
+
x[..., :R_train, :] += y_emb.to(x.dtype)
|
|
113
|
+
|
|
114
|
+
for i, layer in enumerate(self.layers):
|
|
115
|
+
key = f"{cache_prefix}.layer{i}"
|
|
116
|
+
result = layer(
|
|
117
|
+
query=x[..., R_train:, :] if i == len(self.layers) - 1 else x,
|
|
118
|
+
key_value=(
|
|
119
|
+
cast(KVCacheEntry, cache[key])
|
|
120
|
+
if cache is not None and cache.is_replaying
|
|
121
|
+
else x[..., :R_train, :]
|
|
122
|
+
),
|
|
123
|
+
return_key_value=cache is not None and cache.is_recording,
|
|
124
|
+
# `x` is still the caller's tensor at i == 0; don't mutate.
|
|
125
|
+
out=None
|
|
126
|
+
if torch.is_grad_enabled() or i == 0
|
|
127
|
+
else x[..., R_train:, :]
|
|
128
|
+
if i == len(self.layers) - 1
|
|
129
|
+
else x,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
if cache is not None and cache.is_recording:
|
|
133
|
+
x, cache[key] = result
|
|
134
|
+
else:
|
|
135
|
+
x = result
|
|
136
|
+
del result
|
|
137
|
+
|
|
138
|
+
return self.head(self.norm(x)) # [..., R_test, out_channels]
|
|
139
|
+
|
|
140
|
+
def _forward_hierarchical(
|
|
141
|
+
self,
|
|
142
|
+
x: Tensor, # [..., R, D]
|
|
143
|
+
y: Tensor, # [..., R_train]
|
|
144
|
+
*,
|
|
145
|
+
num_classes: int,
|
|
146
|
+
cache: Cache | None,
|
|
147
|
+
) -> Tensor: # [..., R_test, C]
|
|
148
|
+
*batch_shape, num_rows, channels = x.size()
|
|
149
|
+
train_size = y.size(-1)
|
|
150
|
+
test_size = num_rows - train_size
|
|
151
|
+
|
|
152
|
+
if 0 in batch_shape:
|
|
153
|
+
empty = x[..., train_size:, :].sum(dim=-1, keepdim=True)
|
|
154
|
+
return empty.expand(*batch_shape, test_size, num_classes)
|
|
155
|
+
|
|
156
|
+
num_tables = math.prod(batch_shape)
|
|
157
|
+
flat_rows = x.reshape(num_tables, num_rows, channels)
|
|
158
|
+
|
|
159
|
+
# Tree nodes have different row counts, so tables and recursive model
|
|
160
|
+
# calls cannot be represented by one dense tensor operation.
|
|
161
|
+
if cache is not None and cache.is_replaying:
|
|
162
|
+
trees = cast("list[_Node]", cache["icl_block.trees"])
|
|
163
|
+
if len(trees) != num_tables:
|
|
164
|
+
raise RuntimeError(
|
|
165
|
+
f"Expected {len(trees)} cached tables (got {num_tables})"
|
|
166
|
+
)
|
|
167
|
+
table_outputs = [
|
|
168
|
+
self._replay_table(
|
|
169
|
+
test_rows=rows[train_size:],
|
|
170
|
+
node=tree,
|
|
171
|
+
num_classes=num_classes,
|
|
172
|
+
cache=cache,
|
|
173
|
+
cache_prefix=f"icl_block.table{table_idx}.node",
|
|
174
|
+
)
|
|
175
|
+
for table_idx, (rows, tree) in enumerate(zip(flat_rows, trees))
|
|
176
|
+
]
|
|
177
|
+
else:
|
|
178
|
+
flat_y = y.reshape(num_tables, train_size)
|
|
179
|
+
trees = []
|
|
180
|
+
table_outputs = []
|
|
181
|
+
for table_idx, (rows, labels) in enumerate(zip(flat_rows, flat_y)):
|
|
182
|
+
class_ids, local_log_probs, tree = self._process_node(
|
|
183
|
+
train_rows=rows[:train_size],
|
|
184
|
+
train_labels=labels,
|
|
185
|
+
test_rows=rows[train_size:],
|
|
186
|
+
cache=cache,
|
|
187
|
+
cache_prefix=f"icl_block.table{table_idx}.node",
|
|
188
|
+
)
|
|
189
|
+
trees.append(tree)
|
|
190
|
+
table_outputs.append(
|
|
191
|
+
self._expand_log_probs(
|
|
192
|
+
class_ids=class_ids,
|
|
193
|
+
local_log_probs=local_log_probs,
|
|
194
|
+
num_classes=num_classes,
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
if cache is not None and cache.is_recording:
|
|
199
|
+
cache["icl_block.trees"] = trees
|
|
200
|
+
|
|
201
|
+
log_probs = torch.stack(table_outputs)
|
|
202
|
+
# The output Softmax divides by temperature, so scale the combined
|
|
203
|
+
# log-probabilities to preserve their normalized probabilities.
|
|
204
|
+
return log_probs.reshape(*batch_shape, test_size, num_classes).mul(
|
|
205
|
+
self.temperature
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
def _replay_table(
|
|
209
|
+
self,
|
|
210
|
+
test_rows: Tensor, # [R_test, D]
|
|
211
|
+
node: _Node,
|
|
212
|
+
*,
|
|
213
|
+
num_classes: int,
|
|
214
|
+
cache: Cache,
|
|
215
|
+
cache_prefix: str,
|
|
216
|
+
) -> Tensor: # [R_test, C]
|
|
217
|
+
class_ids, local_log_probs = self._replay_node(
|
|
218
|
+
test_rows=test_rows,
|
|
219
|
+
node=node,
|
|
220
|
+
cache=cache,
|
|
221
|
+
cache_prefix=cache_prefix,
|
|
222
|
+
)
|
|
223
|
+
return self._expand_log_probs(
|
|
224
|
+
class_ids=class_ids,
|
|
225
|
+
local_log_probs=local_log_probs,
|
|
226
|
+
num_classes=num_classes,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
def _process_node(
|
|
230
|
+
self,
|
|
231
|
+
train_rows: Tensor, # [R_node, D]
|
|
232
|
+
train_labels: Tensor, # [R_node]
|
|
233
|
+
test_rows: Tensor, # [R_test, D]
|
|
234
|
+
*,
|
|
235
|
+
cache: Cache | None,
|
|
236
|
+
cache_prefix: str,
|
|
237
|
+
) -> tuple[Tensor, Tensor, _Node]: # [C_node], [R_test, C_node]
|
|
238
|
+
class_ids, local_labels = train_labels.unique(
|
|
239
|
+
sorted=True,
|
|
240
|
+
return_inverse=True,
|
|
241
|
+
)
|
|
242
|
+
node_num_classes = class_ids.numel()
|
|
243
|
+
node: _Node = {"class_ids": class_ids, "children": []}
|
|
244
|
+
|
|
245
|
+
if node_num_classes <= self.num_classes:
|
|
246
|
+
if node_num_classes == 1:
|
|
247
|
+
local_log_probs = test_rows.sum(
|
|
248
|
+
dim=-1,
|
|
249
|
+
keepdim=True,
|
|
250
|
+
).mul(0)
|
|
251
|
+
else:
|
|
252
|
+
local_log_probs = self._predict_log_probs(
|
|
253
|
+
x=torch.cat((train_rows, test_rows), dim=0),
|
|
254
|
+
y=local_labels,
|
|
255
|
+
num_classes=node_num_classes,
|
|
256
|
+
cache=cache,
|
|
257
|
+
cache_prefix=cache_prefix,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
return class_ids, local_log_probs, node
|
|
261
|
+
|
|
262
|
+
class_groups, num_groups = self._grouping(
|
|
263
|
+
num_classes=node_num_classes,
|
|
264
|
+
device=train_labels.device,
|
|
265
|
+
)
|
|
266
|
+
group_labels = class_groups[local_labels]
|
|
267
|
+
group_masks = group_labels == torch.arange(
|
|
268
|
+
num_groups,
|
|
269
|
+
device=group_labels.device,
|
|
270
|
+
).unsqueeze(-1) # [G, R_node]
|
|
271
|
+
group_log_probs = self._predict_log_probs(
|
|
272
|
+
x=torch.cat((train_rows, test_rows), dim=0),
|
|
273
|
+
y=group_labels,
|
|
274
|
+
num_classes=num_groups,
|
|
275
|
+
cache=cache,
|
|
276
|
+
cache_prefix=cache_prefix,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
child_class_ids: list[Tensor] = []
|
|
280
|
+
children_log_probs: list[Tensor] = []
|
|
281
|
+
children = cast("list[_Node]", node["children"])
|
|
282
|
+
for group_idx in range(num_groups):
|
|
283
|
+
mask = group_masks[group_idx]
|
|
284
|
+
child_ids, child_log_probs, child = self._process_node(
|
|
285
|
+
train_rows=train_rows[mask],
|
|
286
|
+
train_labels=train_labels[mask],
|
|
287
|
+
test_rows=test_rows,
|
|
288
|
+
cache=cache,
|
|
289
|
+
cache_prefix=f"{cache_prefix}.child{group_idx}",
|
|
290
|
+
)
|
|
291
|
+
child_class_ids.append(child_ids)
|
|
292
|
+
children_log_probs.append(
|
|
293
|
+
child_log_probs + group_log_probs[:, group_idx : group_idx + 1]
|
|
294
|
+
)
|
|
295
|
+
children.append(child)
|
|
296
|
+
|
|
297
|
+
# Balanced groups are contiguous in sorted class order, so concatenated
|
|
298
|
+
# child outputs retain the node's sorted class order.
|
|
299
|
+
return (
|
|
300
|
+
torch.cat(child_class_ids),
|
|
301
|
+
torch.cat(children_log_probs, dim=-1),
|
|
302
|
+
node,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
def _replay_node(
|
|
306
|
+
self,
|
|
307
|
+
test_rows: Tensor, # [R_test, D]
|
|
308
|
+
node: _Node,
|
|
309
|
+
*,
|
|
310
|
+
cache: Cache,
|
|
311
|
+
cache_prefix: str,
|
|
312
|
+
) -> tuple[Tensor, Tensor]: # [C_node], [R_test, C_node]
|
|
313
|
+
class_ids = cast(Tensor, node["class_ids"])
|
|
314
|
+
children = cast("list[_Node]", node["children"])
|
|
315
|
+
|
|
316
|
+
if not children:
|
|
317
|
+
if class_ids.numel() == 1:
|
|
318
|
+
local_log_probs = test_rows.sum(
|
|
319
|
+
dim=-1,
|
|
320
|
+
keepdim=True,
|
|
321
|
+
).mul(0)
|
|
322
|
+
else:
|
|
323
|
+
local_log_probs = self._predict_log_probs(
|
|
324
|
+
x=test_rows,
|
|
325
|
+
y=class_ids.new_empty((0,)),
|
|
326
|
+
num_classes=class_ids.numel(),
|
|
327
|
+
cache=cache,
|
|
328
|
+
cache_prefix=cache_prefix,
|
|
329
|
+
)
|
|
330
|
+
return class_ids, local_log_probs
|
|
331
|
+
|
|
332
|
+
group_log_probs = self._predict_log_probs(
|
|
333
|
+
x=test_rows,
|
|
334
|
+
y=class_ids.new_empty((0,)),
|
|
335
|
+
num_classes=len(children),
|
|
336
|
+
cache=cache,
|
|
337
|
+
cache_prefix=cache_prefix,
|
|
338
|
+
)
|
|
339
|
+
child_class_ids: list[Tensor] = []
|
|
340
|
+
children_log_probs: list[Tensor] = []
|
|
341
|
+
for group_idx, child in enumerate(children):
|
|
342
|
+
child_ids, child_log_probs = self._replay_node(
|
|
343
|
+
test_rows=test_rows,
|
|
344
|
+
node=child,
|
|
345
|
+
cache=cache,
|
|
346
|
+
cache_prefix=f"{cache_prefix}.child{group_idx}",
|
|
347
|
+
)
|
|
348
|
+
child_class_ids.append(child_ids)
|
|
349
|
+
children_log_probs.append(
|
|
350
|
+
child_log_probs + group_log_probs[:, group_idx : group_idx + 1]
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
return (
|
|
354
|
+
torch.cat(child_class_ids),
|
|
355
|
+
torch.cat(children_log_probs, dim=-1),
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
def _predict_log_probs(
|
|
359
|
+
self,
|
|
360
|
+
x: Tensor, # [R_node + R_test, D] or [R_test, D] with cache
|
|
361
|
+
y: Tensor, # [R_node] or [0] with cache
|
|
362
|
+
*,
|
|
363
|
+
num_classes: int,
|
|
364
|
+
cache: Cache | None,
|
|
365
|
+
cache_prefix: str,
|
|
366
|
+
) -> Tensor: # [R_test, C_node]
|
|
367
|
+
logits = self._forward(
|
|
368
|
+
x=x,
|
|
369
|
+
y=y,
|
|
370
|
+
cache=cache,
|
|
371
|
+
cache_prefix=cache_prefix,
|
|
372
|
+
)
|
|
373
|
+
return (logits[:, :num_classes] / self.temperature).log_softmax(dim=-1)
|
|
374
|
+
|
|
375
|
+
@staticmethod
|
|
376
|
+
def _expand_log_probs(
|
|
377
|
+
class_ids: Tensor, # [C_node]
|
|
378
|
+
local_log_probs: Tensor, # [R_test, C_node]
|
|
379
|
+
*,
|
|
380
|
+
num_classes: int,
|
|
381
|
+
) -> Tensor: # [R_test, C]
|
|
382
|
+
log_probs = local_log_probs.new_full(
|
|
383
|
+
(local_log_probs.size(-2), num_classes),
|
|
384
|
+
-torch.inf,
|
|
385
|
+
)
|
|
386
|
+
return log_probs.index_copy(
|
|
387
|
+
-1,
|
|
388
|
+
class_ids.to(torch.long),
|
|
389
|
+
local_log_probs,
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
def _grouping(
|
|
393
|
+
self,
|
|
394
|
+
num_classes: int,
|
|
395
|
+
device: torch.device,
|
|
396
|
+
) -> tuple[Tensor, int]:
|
|
397
|
+
if num_classes <= self.num_classes:
|
|
398
|
+
assignments = torch.zeros(
|
|
399
|
+
num_classes,
|
|
400
|
+
dtype=torch.long,
|
|
401
|
+
device=device,
|
|
402
|
+
)
|
|
403
|
+
return assignments, 1
|
|
404
|
+
num_groups = min(
|
|
405
|
+
(num_classes + self.num_classes - 1) // self.num_classes,
|
|
406
|
+
self.num_classes,
|
|
407
|
+
)
|
|
408
|
+
group_sizes = torch.full(
|
|
409
|
+
(num_groups,),
|
|
410
|
+
num_classes // num_groups,
|
|
411
|
+
dtype=torch.long,
|
|
412
|
+
device=device,
|
|
413
|
+
)
|
|
414
|
+
group_sizes[: num_classes % num_groups] += 1
|
|
415
|
+
group_indices = torch.arange(num_groups, device=device)
|
|
416
|
+
assignments = group_indices.repeat_interleave(
|
|
417
|
+
group_sizes,
|
|
418
|
+
output_size=num_classes,
|
|
419
|
+
)
|
|
420
|
+
return assignments, num_groups
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# Copyright (c) 2025, Soda team @ Inria
|
|
2
|
+
# Licensed under the BSD 3-Clause License; see LICENSE.
|
|
3
|
+
|
|
4
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
6
|
+
|
|
7
|
+
# ruff: noqa: D205
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Iterable
|
|
12
|
+
from typing import Any, ClassVar, cast
|
|
13
|
+
|
|
14
|
+
import torch
|
|
15
|
+
from torch import Tensor
|
|
16
|
+
from torch.nn import ModuleDict
|
|
17
|
+
|
|
18
|
+
from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
|
|
19
|
+
from sdm.cache import Cache
|
|
20
|
+
from sdm.models import ICLModel
|
|
21
|
+
from sdm.models._huggingface import download_checkpoint
|
|
22
|
+
from sdm.models.tabiclv2.ckpt import remap_ckpt
|
|
23
|
+
from sdm.models.tabiclv2.icl import ICLBlock
|
|
24
|
+
from sdm.models.tabiclv2.recipe import default_recipe
|
|
25
|
+
from sdm.models.tabiclv2.row_embedding import RowEmbedding
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TabICLv2(ICLModel):
|
|
29
|
+
r"""The tabular foundation model from the `"TabICLv2: A Better, Faster,
|
|
30
|
+
Scalable, and Open Tabular Foundation Model"
|
|
31
|
+
<https://arxiv.org/abs/2602.11139>`_ paper.
|
|
32
|
+
|
|
33
|
+
.. figure:: /images/tabicl_light.svg
|
|
34
|
+
:figclass: light-only
|
|
35
|
+
:align: center
|
|
36
|
+
:width: 600px
|
|
37
|
+
|
|
38
|
+
.. figure:: /images/tabicl_dark.svg
|
|
39
|
+
:figclass: dark-only
|
|
40
|
+
:align: center
|
|
41
|
+
:width: 600px
|
|
42
|
+
|
|
43
|
+
:class:`TabICLv2` treats a table as an in-context learning problem: a set
|
|
44
|
+
of labeled training rows provides context, and the model predicts targets
|
|
45
|
+
for held-out test rows from the same table.
|
|
46
|
+
|
|
47
|
+
Feature columns are encoded via repeated feature grouping, where each
|
|
48
|
+
feature participates in multiple shifted feature groups.
|
|
49
|
+
This breaks symmetries between similarly distributed columns while
|
|
50
|
+
preserving fine-grained feature information.
|
|
51
|
+
Target-aware embeddings are then added to the training-row feature
|
|
52
|
+
representations, injecting label information early without giving test rows
|
|
53
|
+
access to their own targets.
|
|
54
|
+
|
|
55
|
+
Afterwards, the module applies three attention stages in sequence:
|
|
56
|
+
|
|
57
|
+
* **Column-wise:** Each grouped feature is processed as a set of row tokens
|
|
58
|
+
with induced set attention.
|
|
59
|
+
The inducing tokens summarize information from the in-context training
|
|
60
|
+
rows, then pass it back to the row tokens, giving each feature group a
|
|
61
|
+
target-aware representation of its values across examples.
|
|
62
|
+
Query-Aware Scalable SoftMax (:class:`~sdm.nn.QASSMax`) sharpens
|
|
63
|
+
attention over long contexts and reduces attention fading as the number
|
|
64
|
+
of rows grows.
|
|
65
|
+
* **Row-wise:** For each row, the feature-group embeddings are processed
|
|
66
|
+
together with learnable readout tokens.
|
|
67
|
+
Attention across the grouped features lets the model combine column
|
|
68
|
+
evidence and feature interactions within that row.
|
|
69
|
+
The readout token outputs are concatenated to form a fixed-size row
|
|
70
|
+
embedding.
|
|
71
|
+
* **Dataset-wise:** The row embeddings are processed across the dataset
|
|
72
|
+
for in-context prediction.
|
|
73
|
+
Training-row embeddings are combined with target embeddings, and test
|
|
74
|
+
rows attend to the labeled training rows. The resulting test-row states
|
|
75
|
+
are mapped to task outputs, such as class probabilities for
|
|
76
|
+
classification or quantile predictions for regression.
|
|
77
|
+
|
|
78
|
+
.. testcode::
|
|
79
|
+
|
|
80
|
+
from sdm import TableTensor
|
|
81
|
+
from sdm.models import TabICLv2
|
|
82
|
+
|
|
83
|
+
table = TableTensor.from_columns(
|
|
84
|
+
{
|
|
85
|
+
"col0": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0],
|
|
86
|
+
"col1": [1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0],
|
|
87
|
+
"target": ["t", "f", "t", "f", "t", None, None, None],
|
|
88
|
+
},
|
|
89
|
+
stypes={
|
|
90
|
+
"col0": "numerical",
|
|
91
|
+
"col1": "numerical",
|
|
92
|
+
"target": "categorical",
|
|
93
|
+
},
|
|
94
|
+
device="cuda",
|
|
95
|
+
)
|
|
96
|
+
model = TabICLv2(device="cuda")
|
|
97
|
+
|
|
98
|
+
# Default in-context learning forward pass:
|
|
99
|
+
out = model(
|
|
100
|
+
x_context=table[:5].drop_columns("target"),
|
|
101
|
+
y_context=table[:5, "target"],
|
|
102
|
+
x_query=table[5:].drop_columns("target"),
|
|
103
|
+
)
|
|
104
|
+
assert out.size() == (3, 2)
|
|
105
|
+
|
|
106
|
+
# Fit+Predict forward pass via key/value caching:
|
|
107
|
+
model.fit(
|
|
108
|
+
x=table[:5].drop_columns("target"),
|
|
109
|
+
y=table[:5, "target"],
|
|
110
|
+
)
|
|
111
|
+
out = model.predict(table[5:].drop_columns("target"))
|
|
112
|
+
assert out.size() == (3, 2)
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
task: The tasks to initialize. If ``None``, all tasks supported by this
|
|
116
|
+
model are initialized.
|
|
117
|
+
pretrained: Whether to load the pretrained checkpoint.
|
|
118
|
+
device: The device.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
122
|
+
{Stype.numerical}
|
|
123
|
+
)
|
|
124
|
+
supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
125
|
+
{Stype.numerical, Stype.categorical}
|
|
126
|
+
)
|
|
127
|
+
supports_multi_target: ClassVar[bool] = False
|
|
128
|
+
supports_related_tables: ClassVar[bool] = False
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
task: TaskLike | Iterable[TaskLike] | None = None,
|
|
133
|
+
pretrained: bool = True,
|
|
134
|
+
device: torch.device | str | None = None,
|
|
135
|
+
) -> None:
|
|
136
|
+
super().__init__(task=task)
|
|
137
|
+
|
|
138
|
+
self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
|
|
139
|
+
for task in self.tasks:
|
|
140
|
+
self.models[task] = _TabICLv2(
|
|
141
|
+
num_classes=10 if task == Task.classification else 0,
|
|
142
|
+
num_quantiles=999 if task == Task.regression else 0,
|
|
143
|
+
norm_bias=task == Task.classification,
|
|
144
|
+
device="meta" if pretrained else device,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if pretrained:
|
|
148
|
+
self._load_from_pretrained(device=device)
|
|
149
|
+
|
|
150
|
+
self.eval()
|
|
151
|
+
|
|
152
|
+
@classmethod
|
|
153
|
+
def default_recipe(cls) -> Recipe:
|
|
154
|
+
r""":meta private:""" # noqa: D415
|
|
155
|
+
return default_recipe()
|
|
156
|
+
|
|
157
|
+
def _load_from_pretrained(
|
|
158
|
+
self,
|
|
159
|
+
device: torch.device | str | None,
|
|
160
|
+
) -> TabICLv2:
|
|
161
|
+
device = torch.get_default_device() if device is None else device
|
|
162
|
+
|
|
163
|
+
for task, model in self.models.items():
|
|
164
|
+
if task == Task.classification:
|
|
165
|
+
filename = "tabicl-classifier-v2-20260212.ckpt"
|
|
166
|
+
else:
|
|
167
|
+
assert task == Task.regression
|
|
168
|
+
filename = "tabicl-regressor-v2-20260212.ckpt"
|
|
169
|
+
|
|
170
|
+
ckpt = torch.load(
|
|
171
|
+
download_checkpoint("jingang/TabICL", filename),
|
|
172
|
+
map_location=device,
|
|
173
|
+
weights_only=True,
|
|
174
|
+
)["state_dict"]
|
|
175
|
+
ckpt = remap_ckpt(ckpt, is_classifier=task == Task.classification)
|
|
176
|
+
model.load_state_dict(ckpt, assign=True)
|
|
177
|
+
|
|
178
|
+
return self
|
|
179
|
+
|
|
180
|
+
def _forward(
|
|
181
|
+
self,
|
|
182
|
+
x_context: TableTensor | None, # [..., R_context, D]
|
|
183
|
+
y_context: TableTensor | None, # [..., R_context, 1]
|
|
184
|
+
x_query: TableTensor | None, # [..., R_query, D]
|
|
185
|
+
related_context_tables: RelatedTables[TableTensor] | None,
|
|
186
|
+
related_query_tables: RelatedTables[TableTensor] | None,
|
|
187
|
+
cache: Cache | None,
|
|
188
|
+
generator: torch.Generator | None,
|
|
189
|
+
**kwargs: Any,
|
|
190
|
+
) -> TableTensor: # [..., R_query, num_classes or 999]
|
|
191
|
+
|
|
192
|
+
if x_query is None and x_context is not None:
|
|
193
|
+
x = x_context.numerical
|
|
194
|
+
elif x_context is None and x_query is not None:
|
|
195
|
+
x = x_query.numerical
|
|
196
|
+
else:
|
|
197
|
+
assert x_context is not None
|
|
198
|
+
assert x_query is not None
|
|
199
|
+
x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
|
|
200
|
+
|
|
201
|
+
y: Tensor | None = None
|
|
202
|
+
classes: Tensor | None = None
|
|
203
|
+
if y_context is not None and y_context.categorical.size(-1) > 0:
|
|
204
|
+
y = y_context.categorical.code.squeeze(-1)
|
|
205
|
+
classes = y_context.categorical.categories[0]
|
|
206
|
+
elif y_context is not None and y_context.numerical.size(-1) > 0:
|
|
207
|
+
y = y_context.numerical.squeeze(-1)
|
|
208
|
+
else:
|
|
209
|
+
assert cache is not None
|
|
210
|
+
classes = cast(Tensor | None, cache["classes"])
|
|
211
|
+
y = x.new_empty(
|
|
212
|
+
(*x.size()[:-2], 0),
|
|
213
|
+
dtype=torch.int64 if classes is not None else x.dtype,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
if classes is None:
|
|
217
|
+
out = self.models[Task.regression](x, y, cache=cache)
|
|
218
|
+
return TableTensor(
|
|
219
|
+
columns={
|
|
220
|
+
Stype.numerical: [f"q{i:03d}" for i in range(1, 1000)]
|
|
221
|
+
},
|
|
222
|
+
numerical=out,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
out = self.models[Task.classification](
|
|
226
|
+
x, y, cache=cache, num_classes=len(classes)
|
|
227
|
+
)
|
|
228
|
+
return TableTensor(
|
|
229
|
+
columns={Stype.numerical: [str(i) for i in classes.tolist()]},
|
|
230
|
+
numerical=out[..., : len(classes)],
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class _TabICLv2(torch.nn.Module):
|
|
235
|
+
def __init__(
|
|
236
|
+
self,
|
|
237
|
+
num_classes: int,
|
|
238
|
+
num_quantiles: int,
|
|
239
|
+
channels: int = 128,
|
|
240
|
+
num_embedding_layers: int = 3,
|
|
241
|
+
num_embedding_heads: int = 8,
|
|
242
|
+
num_inducing_points: int = 128,
|
|
243
|
+
group_size: int = 3,
|
|
244
|
+
num_readout_tokens: int = 4,
|
|
245
|
+
num_icl_layers: int = 12,
|
|
246
|
+
num_icl_heads: int = 8,
|
|
247
|
+
norm_bias: bool = True,
|
|
248
|
+
device: torch.device | str | None = None,
|
|
249
|
+
dtype: torch.dtype | None = None,
|
|
250
|
+
) -> None:
|
|
251
|
+
super().__init__()
|
|
252
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
253
|
+
|
|
254
|
+
self.row_embedding = RowEmbedding(
|
|
255
|
+
num_classes=num_classes,
|
|
256
|
+
channels=channels,
|
|
257
|
+
num_layers=num_embedding_layers,
|
|
258
|
+
num_heads=num_embedding_heads,
|
|
259
|
+
group_size=group_size,
|
|
260
|
+
num_inducing_points=num_inducing_points,
|
|
261
|
+
num_readout_tokens=num_readout_tokens,
|
|
262
|
+
norm_bias=norm_bias,
|
|
263
|
+
**factory_kwargs,
|
|
264
|
+
)
|
|
265
|
+
self.icl_block = ICLBlock(
|
|
266
|
+
num_classes=num_classes,
|
|
267
|
+
out_channels=num_classes or num_quantiles,
|
|
268
|
+
channels=num_readout_tokens * channels,
|
|
269
|
+
num_layers=num_icl_layers,
|
|
270
|
+
num_heads=num_icl_heads,
|
|
271
|
+
norm_bias=norm_bias,
|
|
272
|
+
temperature=0.9,
|
|
273
|
+
**factory_kwargs,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
def forward(
|
|
277
|
+
self,
|
|
278
|
+
x: Tensor, # [..., R, C]
|
|
279
|
+
y: Tensor, # [..., R_train]
|
|
280
|
+
*,
|
|
281
|
+
cache: Cache | None = None,
|
|
282
|
+
num_classes: int | None = None,
|
|
283
|
+
) -> Tensor: # [..., R_test, out_channels or num_classes]
|
|
284
|
+
if not y.is_floating_point():
|
|
285
|
+
assert num_classes is not None
|
|
286
|
+
|
|
287
|
+
x = self.row_embedding(x, y, num_classes=num_classes, cache=cache)
|
|
288
|
+
return self.icl_block(
|
|
289
|
+
x=x,
|
|
290
|
+
y=y,
|
|
291
|
+
num_classes=num_classes,
|
|
292
|
+
cache=cache,
|
|
293
|
+
)
|