structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D205
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from typing import Any, ClassVar, Literal, cast
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
from torch.nn import Identity, Linear, ModuleDict
|
|
13
|
+
|
|
14
|
+
from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
|
|
15
|
+
from sdm.cache import Cache
|
|
16
|
+
from sdm.models import ECOC, ICLModel
|
|
17
|
+
from sdm.models._huggingface import download_checkpoint
|
|
18
|
+
from sdm.models.kumo.tabular.icl import ICLBlock
|
|
19
|
+
from sdm.models.kumo.tabular.recipe import default_recipe
|
|
20
|
+
from sdm.models.kumo.tabular.row_embedding import RowEmbedding
|
|
21
|
+
|
|
22
|
+
MODEL_KWARGS: dict[str, dict[str, Any]] = {
|
|
23
|
+
"small": {
|
|
24
|
+
"cell_channels": 128,
|
|
25
|
+
"num_embedding_layers": 4,
|
|
26
|
+
"num_embedding_heads": 4,
|
|
27
|
+
"num_inducing_points": 128,
|
|
28
|
+
"group_size": 3,
|
|
29
|
+
"num_frequencies": 32,
|
|
30
|
+
"num_readout_tokens": 4,
|
|
31
|
+
"icl_channels": 512,
|
|
32
|
+
"num_icl_layers": 12,
|
|
33
|
+
"num_icl_heads": 8,
|
|
34
|
+
"num_icl_key_value_heads_for_query": None,
|
|
35
|
+
},
|
|
36
|
+
"medium": {
|
|
37
|
+
"cell_channels": 256,
|
|
38
|
+
"num_embedding_layers": 6,
|
|
39
|
+
"num_embedding_heads": 4,
|
|
40
|
+
"num_inducing_points": 256,
|
|
41
|
+
"group_size": 3,
|
|
42
|
+
"num_frequencies": 32,
|
|
43
|
+
"num_readout_tokens": 4,
|
|
44
|
+
"icl_channels": 512,
|
|
45
|
+
"num_icl_layers": 24,
|
|
46
|
+
"num_icl_heads": 8,
|
|
47
|
+
"num_icl_key_value_heads_for_query": 2,
|
|
48
|
+
},
|
|
49
|
+
"large": {
|
|
50
|
+
"cell_channels": 256,
|
|
51
|
+
"num_embedding_layers": 6,
|
|
52
|
+
"num_embedding_heads": 4,
|
|
53
|
+
"num_inducing_points": 256,
|
|
54
|
+
"group_size": 3,
|
|
55
|
+
"num_frequencies": 32,
|
|
56
|
+
"num_readout_tokens": 4,
|
|
57
|
+
"icl_channels": 1024,
|
|
58
|
+
"num_icl_layers": 24,
|
|
59
|
+
"num_icl_heads": 16,
|
|
60
|
+
"num_icl_key_value_heads_for_query": 2,
|
|
61
|
+
},
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class KumoTabular(ICLModel):
|
|
66
|
+
r"""The tabular foundation model from `"NVIDIA Kumo Tabular Sets a New
|
|
67
|
+
Accuracy-Efficiency Frontier for Tabular Prediction"
|
|
68
|
+
<https://huggingface.co/blog/nvidia/kumo-tabular>`__.
|
|
69
|
+
|
|
70
|
+
.. figure:: /images/kumo_tabular.svg
|
|
71
|
+
:width: 100%
|
|
72
|
+
|
|
73
|
+
:class:`KumoTabular` processes a table in two stages. First, an interleaved
|
|
74
|
+
row/column encoder transforms raw table cells into fixed-size row
|
|
75
|
+
representations. Numerical and categorical cells use separate learned
|
|
76
|
+
Fourier frequencies and projections, together with a learned missingness
|
|
77
|
+
projection. Each encoder stage consists of:
|
|
78
|
+
|
|
79
|
+
* **Column-wise:** Each feature group is processed across rows using
|
|
80
|
+
induced set attention. Both context and query cells attend only to
|
|
81
|
+
context-row keys and values.
|
|
82
|
+
* **Row-wise:** Each row's feature groups and learnable readout tokens
|
|
83
|
+
attend to one another, combining feature interactions into a fixed-size
|
|
84
|
+
row representation.
|
|
85
|
+
|
|
86
|
+
Second, a dataset-wise in-context learning transformer processes the row
|
|
87
|
+
representations and predicts each query target from the labeled context
|
|
88
|
+
rows. The ``"large"`` model widens this transformer to 1024 channels and
|
|
89
|
+
16 attention heads. Its four 256-channel readout tokens concatenate
|
|
90
|
+
directly to that width without a projection layer.
|
|
91
|
+
|
|
92
|
+
For regression tasks, :class:`KumoTabular` predicts 999 quantiles named
|
|
93
|
+
``"q001"`` through ``"q999"``.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
task: The tasks to initialize. If ``None``, all tasks supported by this
|
|
97
|
+
model are initialized.
|
|
98
|
+
size: The model size, one of ``"small"``, ``"medium"``, or
|
|
99
|
+
``"large"``. Defaults to ``"large"``.
|
|
100
|
+
pretrained: Whether to load pretrained checkpoints.
|
|
101
|
+
device: The device for model parameters. If ``None``, uses PyTorch's
|
|
102
|
+
default device.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
106
|
+
{Stype.numerical}
|
|
107
|
+
)
|
|
108
|
+
supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
109
|
+
{Stype.numerical, Stype.categorical}
|
|
110
|
+
)
|
|
111
|
+
supports_multi_target: ClassVar[bool] = False
|
|
112
|
+
supports_related_tables: ClassVar[bool] = False
|
|
113
|
+
|
|
114
|
+
def __init__(
|
|
115
|
+
self,
|
|
116
|
+
task: TaskLike | Iterable[TaskLike] | None = None,
|
|
117
|
+
size: Literal["small", "medium", "large"] = "large",
|
|
118
|
+
pretrained: bool = True,
|
|
119
|
+
device: torch.device | str | None = None,
|
|
120
|
+
) -> None:
|
|
121
|
+
super().__init__(task=task)
|
|
122
|
+
if Task.classification in self.tasks:
|
|
123
|
+
self.ecoc = ECOC(max_classes=10)
|
|
124
|
+
|
|
125
|
+
self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
|
|
126
|
+
for task in self.tasks:
|
|
127
|
+
self.models[task] = _KumoTabular(
|
|
128
|
+
num_classes=10 if task == Task.classification else 0,
|
|
129
|
+
num_quantiles=999 if task == Task.regression else 0,
|
|
130
|
+
device="meta" if pretrained else device,
|
|
131
|
+
**MODEL_KWARGS[size],
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if pretrained:
|
|
135
|
+
self.models[task] = self._load_from_pretrained(size, device=device)
|
|
136
|
+
|
|
137
|
+
self.eval()
|
|
138
|
+
|
|
139
|
+
@classmethod
|
|
140
|
+
def default_recipe(cls) -> Recipe:
|
|
141
|
+
r""":meta private:""" # noqa: D415
|
|
142
|
+
return default_recipe()
|
|
143
|
+
|
|
144
|
+
def _load_from_pretrained(
|
|
145
|
+
self,
|
|
146
|
+
size: Literal["small", "medium", "large"],
|
|
147
|
+
device: torch.device | str | None,
|
|
148
|
+
) -> _KumoTabular:
|
|
149
|
+
device = torch.get_default_device() if device is None else device
|
|
150
|
+
|
|
151
|
+
for task, model in self.models.items():
|
|
152
|
+
if task == Task.classification:
|
|
153
|
+
filename = f"{size}/classifier.pt"
|
|
154
|
+
else:
|
|
155
|
+
assert task == Task.regression
|
|
156
|
+
filename = f"{size}/regressor.pt"
|
|
157
|
+
|
|
158
|
+
path = download_checkpoint(
|
|
159
|
+
repo_id="nvidia/Kumo-Tabular",
|
|
160
|
+
filename=filename,
|
|
161
|
+
revision="v1.0.1",
|
|
162
|
+
config_filename="config.json",
|
|
163
|
+
)
|
|
164
|
+
ckpt = torch.load(path, map_location=device, weights_only=True)
|
|
165
|
+
model.load_state_dict(ckpt, assign=True)
|
|
166
|
+
|
|
167
|
+
return model
|
|
168
|
+
|
|
169
|
+
def _forward(
|
|
170
|
+
self,
|
|
171
|
+
x_context: TableTensor | None, # [..., R_context, D]
|
|
172
|
+
y_context: TableTensor | None, # [..., R_context, 1]
|
|
173
|
+
x_query: TableTensor | None, # [..., R_query, D]
|
|
174
|
+
related_context_tables: RelatedTables[TableTensor] | None,
|
|
175
|
+
related_query_tables: RelatedTables[TableTensor] | None,
|
|
176
|
+
cache: Cache | None,
|
|
177
|
+
generator: torch.Generator | None,
|
|
178
|
+
*,
|
|
179
|
+
categorical_mask: Tensor,
|
|
180
|
+
**kwargs: Any,
|
|
181
|
+
) -> TableTensor: # [..., R_query, num_classes or 999]
|
|
182
|
+
|
|
183
|
+
if x_query is None and x_context is not None:
|
|
184
|
+
x = x_context.numerical
|
|
185
|
+
elif x_context is None and x_query is not None:
|
|
186
|
+
x = x_query.numerical
|
|
187
|
+
else:
|
|
188
|
+
assert x_context is not None
|
|
189
|
+
assert x_query is not None
|
|
190
|
+
x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
|
|
191
|
+
|
|
192
|
+
classes: Tensor | None = None
|
|
193
|
+
if y_context is not None and y_context.categorical.size(-1) > 0:
|
|
194
|
+
y = y_context.categorical.code.squeeze(-1)
|
|
195
|
+
classes = y_context.categorical.categories[0]
|
|
196
|
+
elif y_context is not None and y_context.numerical.size(-1) > 0:
|
|
197
|
+
y = y_context.numerical.squeeze(-1)
|
|
198
|
+
else:
|
|
199
|
+
assert cache is not None
|
|
200
|
+
classes = cast(Tensor | None, cache["classes"])
|
|
201
|
+
y = x.new_empty(
|
|
202
|
+
(*x.size()[:-2], 0),
|
|
203
|
+
dtype=torch.int64 if classes is not None else x.dtype,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
categorical_mask = categorical_mask.expand(*x.size()[:-2], -1)
|
|
207
|
+
|
|
208
|
+
if classes is None:
|
|
209
|
+
out = self.models[Task.regression](
|
|
210
|
+
x=x,
|
|
211
|
+
y=y,
|
|
212
|
+
categorical_mask=categorical_mask,
|
|
213
|
+
cache=cache,
|
|
214
|
+
)
|
|
215
|
+
return TableTensor(
|
|
216
|
+
columns={
|
|
217
|
+
Stype.numerical: [f"q{i:03d}" for i in range(1, 1000)]
|
|
218
|
+
},
|
|
219
|
+
numerical=out,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
out = self.ecoc(
|
|
223
|
+
model=self.models[Task.classification],
|
|
224
|
+
x=x,
|
|
225
|
+
y=y,
|
|
226
|
+
num_classes=len(classes),
|
|
227
|
+
num_members=x.size(0) if x.dim() > 2 else 1,
|
|
228
|
+
cache=cache,
|
|
229
|
+
generator=generator,
|
|
230
|
+
categorical_mask=categorical_mask,
|
|
231
|
+
)
|
|
232
|
+
return TableTensor(
|
|
233
|
+
columns={Stype.numerical: [str(i) for i in classes.tolist()]},
|
|
234
|
+
numerical=out,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class _KumoTabular(torch.nn.Module):
|
|
239
|
+
def __init__(
|
|
240
|
+
self,
|
|
241
|
+
num_classes: int,
|
|
242
|
+
num_quantiles: int,
|
|
243
|
+
cell_channels: int = 128,
|
|
244
|
+
num_embedding_layers: int = 4,
|
|
245
|
+
num_embedding_heads: int = 4,
|
|
246
|
+
num_inducing_points: int = 128,
|
|
247
|
+
group_size: int = 3,
|
|
248
|
+
num_frequencies: int = 32,
|
|
249
|
+
num_readout_tokens: int = 4,
|
|
250
|
+
icl_channels: int = 512,
|
|
251
|
+
num_icl_layers: int = 12,
|
|
252
|
+
num_icl_heads: int = 8,
|
|
253
|
+
num_icl_key_value_heads_for_query: int | None = None,
|
|
254
|
+
device: torch.device | str | None = None,
|
|
255
|
+
dtype: torch.dtype | None = None,
|
|
256
|
+
) -> None:
|
|
257
|
+
super().__init__()
|
|
258
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
259
|
+
|
|
260
|
+
self.row_embedding = RowEmbedding(
|
|
261
|
+
num_classes=num_classes,
|
|
262
|
+
channels=cell_channels,
|
|
263
|
+
num_layers=num_embedding_layers,
|
|
264
|
+
num_heads=num_embedding_heads,
|
|
265
|
+
group_size=group_size,
|
|
266
|
+
num_frequencies=num_frequencies,
|
|
267
|
+
num_inducing_points=num_inducing_points,
|
|
268
|
+
num_readout_tokens=num_readout_tokens,
|
|
269
|
+
**factory_kwargs,
|
|
270
|
+
)
|
|
271
|
+
if cell_channels * num_readout_tokens != icl_channels:
|
|
272
|
+
self.row_project = Linear(
|
|
273
|
+
cell_channels * num_readout_tokens,
|
|
274
|
+
icl_channels,
|
|
275
|
+
**factory_kwargs,
|
|
276
|
+
)
|
|
277
|
+
else:
|
|
278
|
+
self.row_project = Identity()
|
|
279
|
+
self.icl_block = ICLBlock(
|
|
280
|
+
num_classes=num_classes,
|
|
281
|
+
out_channels=num_classes or num_quantiles,
|
|
282
|
+
channels=icl_channels,
|
|
283
|
+
num_layers=num_icl_layers,
|
|
284
|
+
num_heads=num_icl_heads,
|
|
285
|
+
num_key_value_heads_for_query=num_icl_key_value_heads_for_query,
|
|
286
|
+
**factory_kwargs,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
def forward(
|
|
290
|
+
self,
|
|
291
|
+
x: Tensor, # [..., R, C]
|
|
292
|
+
y: Tensor, # [..., R_train]
|
|
293
|
+
categorical_mask: Tensor, # [..., C]
|
|
294
|
+
*,
|
|
295
|
+
cache: Cache | None = None,
|
|
296
|
+
) -> Tensor: # [..., R_test, num_classes or num_quantiles]
|
|
297
|
+
x = self.row_embedding(x, y, categorical_mask, cache=cache)
|
|
298
|
+
x = self.row_project(x)
|
|
299
|
+
return self.icl_block(x=x, y=y, cache=cache)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
import sdm.processing as sp
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def default_recipe() -> sp.Recipe: # noqa: D103
|
|
10
|
+
def numerical_processor() -> sp.Sequential:
|
|
11
|
+
return sp.Sequential(
|
|
12
|
+
# In single precision, a few huge outliers collapse the
|
|
13
|
+
# standardized values of all other rows onto one value.
|
|
14
|
+
sp.Cast(torch.float64),
|
|
15
|
+
sp.DropConstantColumns(),
|
|
16
|
+
sp.Standardize(eps=1e-6),
|
|
17
|
+
sp.Clip(-100.0, 100.0),
|
|
18
|
+
sp.Choice(
|
|
19
|
+
sp.Identity(),
|
|
20
|
+
sp.PowerTransform(),
|
|
21
|
+
[
|
|
22
|
+
sp.RobustScale(),
|
|
23
|
+
sp.ClipSoft(3.0),
|
|
24
|
+
],
|
|
25
|
+
sp.RankGaussian(max_knots=8192),
|
|
26
|
+
method="round_robin",
|
|
27
|
+
),
|
|
28
|
+
sp.ClipSigma(threshold=4.0),
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
return sp.Recipe(
|
|
32
|
+
features=[
|
|
33
|
+
sp.StypeDispatch(
|
|
34
|
+
numerical=[
|
|
35
|
+
numerical_processor(),
|
|
36
|
+
sp.FlipSign(),
|
|
37
|
+
],
|
|
38
|
+
categorical=[
|
|
39
|
+
sp.AlignCategories(sort_by="value"),
|
|
40
|
+
sp.AddCategoryCounts(min_cardinality=50),
|
|
41
|
+
sp.ToNumerical(),
|
|
42
|
+
numerical_processor(),
|
|
43
|
+
],
|
|
44
|
+
),
|
|
45
|
+
sp.ShuffleColumns(method="latin"),
|
|
46
|
+
sp.SelectColumns(500, method="first"),
|
|
47
|
+
sp.Cast(torch.float32),
|
|
48
|
+
],
|
|
49
|
+
target=[
|
|
50
|
+
sp.StypeDispatch(
|
|
51
|
+
categorical=[
|
|
52
|
+
sp.AlignCategories(sort_by="value"),
|
|
53
|
+
sp.ShuffleCategories(method="shift"),
|
|
54
|
+
],
|
|
55
|
+
numerical=[
|
|
56
|
+
sp.Standardize(),
|
|
57
|
+
sp.FlipSign(),
|
|
58
|
+
],
|
|
59
|
+
),
|
|
60
|
+
],
|
|
61
|
+
output=sp.TaskDispatch(
|
|
62
|
+
classification=[
|
|
63
|
+
sp.AverageEstimators(),
|
|
64
|
+
sp.Softmax(),
|
|
65
|
+
],
|
|
66
|
+
regression=[
|
|
67
|
+
sp.SortQuantiles(),
|
|
68
|
+
sp.AverageEstimators(trim_fraction=0.2),
|
|
69
|
+
],
|
|
70
|
+
),
|
|
71
|
+
)
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D101, D102
|
|
5
|
+
|
|
6
|
+
from typing import Any, cast
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
from torch.nn import Embedding, Linear, ModuleList, Parameter
|
|
11
|
+
|
|
12
|
+
from sdm.cache import Cache, KVCacheEntry
|
|
13
|
+
from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
|
|
14
|
+
from sdm.models.kumo.tabular.cell_embedding import CellEmbedding
|
|
15
|
+
from sdm.nn import (
|
|
16
|
+
GatedLogScale,
|
|
17
|
+
InducedTransformerBlock,
|
|
18
|
+
LogScale,
|
|
19
|
+
RMSNorm,
|
|
20
|
+
RotaryEmbedding,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RowEmbedding(torch.nn.Module):
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
num_classes: int,
|
|
28
|
+
channels: int,
|
|
29
|
+
num_layers: int,
|
|
30
|
+
num_heads: int,
|
|
31
|
+
group_size: int,
|
|
32
|
+
num_frequencies: int,
|
|
33
|
+
num_inducing_points: int,
|
|
34
|
+
num_readout_tokens: int,
|
|
35
|
+
device: torch.device | str | None = None,
|
|
36
|
+
dtype: torch.dtype | None = None,
|
|
37
|
+
) -> None:
|
|
38
|
+
super().__init__()
|
|
39
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
40
|
+
self.channels = channels
|
|
41
|
+
|
|
42
|
+
self.cell_embedding = CellEmbedding(
|
|
43
|
+
channels=channels,
|
|
44
|
+
group_size=group_size,
|
|
45
|
+
num_frequencies=num_frequencies,
|
|
46
|
+
**factory_kwargs,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
self.y_emb: torch.nn.Module | None = None
|
|
50
|
+
self.y_lin: torch.nn.Module | None = None
|
|
51
|
+
if num_classes > 0:
|
|
52
|
+
self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
|
|
53
|
+
else:
|
|
54
|
+
self.y_lin = Linear(1, channels, bias=False, **factory_kwargs)
|
|
55
|
+
|
|
56
|
+
self.readout_token = Parameter(
|
|
57
|
+
torch.empty(num_readout_tokens, channels, **factory_kwargs)
|
|
58
|
+
)
|
|
59
|
+
torch.nn.init.trunc_normal_(self.readout_token, std=0.02)
|
|
60
|
+
|
|
61
|
+
rope = RotaryEmbedding(
|
|
62
|
+
channels=channels // num_heads,
|
|
63
|
+
layout="split_half",
|
|
64
|
+
theta=100_000,
|
|
65
|
+
requires_grad=False,
|
|
66
|
+
**factory_kwargs,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
self.col_blocks = ModuleList(
|
|
70
|
+
InducedTransformerBlock(
|
|
71
|
+
channels=channels,
|
|
72
|
+
num_inducing_points=num_inducing_points,
|
|
73
|
+
inducing_block=KumoTabularTransformerBlock(
|
|
74
|
+
channels=channels,
|
|
75
|
+
num_heads=num_heads,
|
|
76
|
+
query_scaling=LogScale(
|
|
77
|
+
num_heads=num_heads,
|
|
78
|
+
**factory_kwargs,
|
|
79
|
+
),
|
|
80
|
+
**factory_kwargs,
|
|
81
|
+
),
|
|
82
|
+
output_block=KumoTabularTransformerBlock(
|
|
83
|
+
channels=channels,
|
|
84
|
+
num_heads=num_heads,
|
|
85
|
+
query_scaling=None,
|
|
86
|
+
**factory_kwargs,
|
|
87
|
+
),
|
|
88
|
+
**factory_kwargs,
|
|
89
|
+
)
|
|
90
|
+
for _ in range(num_layers)
|
|
91
|
+
)
|
|
92
|
+
self.row_blocks = ModuleList(
|
|
93
|
+
KumoTabularTransformerBlock(
|
|
94
|
+
channels=channels,
|
|
95
|
+
num_heads=num_heads,
|
|
96
|
+
query_scaling=GatedLogScale(
|
|
97
|
+
channels=channels // num_heads,
|
|
98
|
+
num_heads=num_heads,
|
|
99
|
+
hidden_channels=64,
|
|
100
|
+
**factory_kwargs,
|
|
101
|
+
),
|
|
102
|
+
rope=rope,
|
|
103
|
+
**factory_kwargs,
|
|
104
|
+
)
|
|
105
|
+
for _ in range(num_layers)
|
|
106
|
+
)
|
|
107
|
+
self.norm = RMSNorm(channels, **factory_kwargs)
|
|
108
|
+
|
|
109
|
+
def forward(
|
|
110
|
+
self,
|
|
111
|
+
x: Tensor, # [..., R, C]
|
|
112
|
+
y: Tensor, # [..., R_train]
|
|
113
|
+
categorical_mask: Tensor, # [..., C]
|
|
114
|
+
*,
|
|
115
|
+
cache: Cache | None = None,
|
|
116
|
+
) -> Tensor: # [..., R, K * D]
|
|
117
|
+
|
|
118
|
+
*B, R, C = x.size()
|
|
119
|
+
R_train = y.size(-1)
|
|
120
|
+
K = self.readout_token.size(-2)
|
|
121
|
+
D = self.channels
|
|
122
|
+
|
|
123
|
+
buffer: Tensor | None = None
|
|
124
|
+
if torch.is_grad_enabled():
|
|
125
|
+
x = self.cell_embedding(
|
|
126
|
+
x=x,
|
|
127
|
+
categorical_mask=categorical_mask,
|
|
128
|
+
train_size=R_train,
|
|
129
|
+
cache=cache,
|
|
130
|
+
) # [..., R, C, D]
|
|
131
|
+
else:
|
|
132
|
+
buffer = torch.empty(
|
|
133
|
+
(*B, R, K + C, D),
|
|
134
|
+
device=x.device,
|
|
135
|
+
dtype=torch.get_autocast_dtype(x.device.type)
|
|
136
|
+
if torch.is_autocast_enabled(x.device.type)
|
|
137
|
+
else x.dtype,
|
|
138
|
+
)
|
|
139
|
+
buffer[..., :K, :] = self.readout_token.to(buffer.dtype)
|
|
140
|
+
x = self.cell_embedding(
|
|
141
|
+
x=x,
|
|
142
|
+
categorical_mask=categorical_mask,
|
|
143
|
+
train_size=R_train,
|
|
144
|
+
cache=cache,
|
|
145
|
+
batch_size_limit="auto",
|
|
146
|
+
out=buffer[..., K:, :],
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
if y.numel() > 0:
|
|
150
|
+
if self.y_emb is not None:
|
|
151
|
+
y_emb = self.y_emb(y).unsqueeze(-2) # [..., R_train, 1, D]
|
|
152
|
+
else:
|
|
153
|
+
assert self.y_lin is not None
|
|
154
|
+
y_emb = self.y_lin(y.unsqueeze(-1)).unsqueeze(-2)
|
|
155
|
+
x[..., :R_train, :, :] += y_emb.to(x.dtype)
|
|
156
|
+
|
|
157
|
+
if buffer is not None:
|
|
158
|
+
x = x.transpose(-2, -3) # [..., C, R, D]
|
|
159
|
+
|
|
160
|
+
for i, (col_block, row_block) in enumerate(
|
|
161
|
+
zip(self.col_blocks, self.row_blocks)
|
|
162
|
+
):
|
|
163
|
+
if buffer is None:
|
|
164
|
+
if i > 0:
|
|
165
|
+
readout_token, x = x.split([K, x.size(-2) - K], dim=-2)
|
|
166
|
+
readout_token = readout_token.clone()
|
|
167
|
+
else:
|
|
168
|
+
readout_token = self.readout_token
|
|
169
|
+
readout_token = readout_token.view(*(1,) * len(B), 1, K, D)
|
|
170
|
+
readout_token = readout_token.expand(*B, R, K, D)
|
|
171
|
+
x = x.transpose(-2, -3) # [..., C, R, D]
|
|
172
|
+
|
|
173
|
+
key = f"row_embedding.col_block{i}"
|
|
174
|
+
result = col_block(
|
|
175
|
+
query=x,
|
|
176
|
+
key_value=cast(KVCacheEntry, cache[key])
|
|
177
|
+
if cache is not None and cache.is_replaying
|
|
178
|
+
else x[..., :R_train, :],
|
|
179
|
+
return_key_value=cache is not None and cache.is_recording,
|
|
180
|
+
batch_size_limit="auto",
|
|
181
|
+
out=None if buffer is None else x,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
if cache is not None and cache.is_recording:
|
|
185
|
+
x, cache[key] = result
|
|
186
|
+
else:
|
|
187
|
+
x = result
|
|
188
|
+
del result
|
|
189
|
+
|
|
190
|
+
if buffer is None:
|
|
191
|
+
x = torch.cat(
|
|
192
|
+
[readout_token.to(x.dtype), x.transpose(-2, -3)],
|
|
193
|
+
dim=-2,
|
|
194
|
+
)
|
|
195
|
+
x = row_block(
|
|
196
|
+
query=x[..., :K, :]
|
|
197
|
+
if i == len(self.row_blocks) - 1
|
|
198
|
+
else x,
|
|
199
|
+
key_value=x,
|
|
200
|
+
batch_size_limit="auto",
|
|
201
|
+
)
|
|
202
|
+
else:
|
|
203
|
+
buffer = row_block(
|
|
204
|
+
query=buffer[..., :K, :]
|
|
205
|
+
if i == len(self.row_blocks) - 1
|
|
206
|
+
else buffer,
|
|
207
|
+
key_value=buffer,
|
|
208
|
+
batch_size_limit="auto",
|
|
209
|
+
out=buffer[..., :K, :]
|
|
210
|
+
if i == len(self.row_blocks) - 1
|
|
211
|
+
else buffer,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
return self.norm(buffer if buffer is not None else x).flatten(-2)
|