structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/models/tabfm/ckpt.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
16
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from collections import defaultdict
|
|
20
|
+
from itertools import product
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
from torch import Tensor
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _map_transformer(prefix: str, tail: str, *, has_rope: bool) -> list[str]:
|
|
27
|
+
if tail.startswith("pre_attn_ln."):
|
|
28
|
+
tail = tail.removeprefix("pre_attn_ln.")
|
|
29
|
+
return [
|
|
30
|
+
f"{prefix}query_norm.{tail}",
|
|
31
|
+
f"{prefix}key_value_norm.{tail}",
|
|
32
|
+
]
|
|
33
|
+
if tail.startswith("post_attn_ln."):
|
|
34
|
+
tail = tail.replace("post_attn_ln.", "post_attn_norm.", 1)
|
|
35
|
+
elif tail.startswith("pre_ff_ln."):
|
|
36
|
+
tail = tail.replace("pre_ff_ln.", "mlp.0.", 1)
|
|
37
|
+
elif tail.startswith("linear1."):
|
|
38
|
+
tail = tail.replace("linear1.", "mlp.1.up_lin.", 1)
|
|
39
|
+
elif tail.startswith("linear1_gate."):
|
|
40
|
+
tail = tail.replace("linear1_gate.", "mlp.1.gate_lin.", 1)
|
|
41
|
+
elif tail.startswith("linear2."):
|
|
42
|
+
tail = tail.replace("linear2.", "mlp.1.down_lin.", 1)
|
|
43
|
+
elif tail.startswith("post_ff_ln."):
|
|
44
|
+
tail = tail.replace("post_ff_ln.", "mlp.2.", 1)
|
|
45
|
+
elif tail.startswith("attn.out_proj."):
|
|
46
|
+
tail = tail.replace("attn.out_proj.", "attn.out_lin.", 1)
|
|
47
|
+
elif tail == "attn.query_ln.weight":
|
|
48
|
+
index = int(has_rope)
|
|
49
|
+
tail = f"attn.query_transform.{index}.weight"
|
|
50
|
+
elif tail == "attn.key_ln.weight":
|
|
51
|
+
index = int(has_rope)
|
|
52
|
+
tail = f"attn.key_transform.{index}.weight"
|
|
53
|
+
elif tail == "attn.per_dim_scale":
|
|
54
|
+
index = int(has_rope) + 1
|
|
55
|
+
tail = f"attn.query_transform.{index}.weight"
|
|
56
|
+
|
|
57
|
+
return [prefix + tail]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _add_transformer(
|
|
61
|
+
out: dict[str, Tensor],
|
|
62
|
+
qkv: dict[tuple[str, str], dict[str, Tensor]],
|
|
63
|
+
prefix: str,
|
|
64
|
+
tail: str,
|
|
65
|
+
value: Tensor,
|
|
66
|
+
*,
|
|
67
|
+
has_rope: bool,
|
|
68
|
+
) -> None:
|
|
69
|
+
match = re.match(r"attn\.([qkv])_proj\.(weight|bias)", tail)
|
|
70
|
+
if match is not None:
|
|
71
|
+
projection, parameter = match.groups()
|
|
72
|
+
qkv[(prefix, parameter)][projection] = value
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
for key in _map_transformer(prefix, tail, has_rope=has_rope):
|
|
76
|
+
out[key] = value
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _add_col_embedding(
|
|
80
|
+
out: dict[str, Tensor],
|
|
81
|
+
qkv: dict[tuple[str, str], dict[str, Tensor]],
|
|
82
|
+
repeat: int,
|
|
83
|
+
tail: str,
|
|
84
|
+
value: Tensor,
|
|
85
|
+
) -> None:
|
|
86
|
+
if tail.startswith("tf_col.blocks."):
|
|
87
|
+
layer, tail = tail.removeprefix("tf_col.blocks.").split(".", 1)
|
|
88
|
+
if tail == "ind_vectors":
|
|
89
|
+
out[
|
|
90
|
+
f"row_embedding.col_blocks.{repeat}.{layer}.inducing_points"
|
|
91
|
+
] = value
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
block, tail = tail.split(".", 1)
|
|
95
|
+
block = {"mab1": "inducing_block", "mab2": "output_block"}[block]
|
|
96
|
+
_add_transformer(
|
|
97
|
+
out,
|
|
98
|
+
qkv,
|
|
99
|
+
f"row_embedding.col_blocks.{repeat}.{layer}.{block}.",
|
|
100
|
+
tail,
|
|
101
|
+
value,
|
|
102
|
+
has_rope=False,
|
|
103
|
+
)
|
|
104
|
+
return
|
|
105
|
+
|
|
106
|
+
if tail.startswith("out_w."):
|
|
107
|
+
out[
|
|
108
|
+
f"row_embedding.col_projections.{repeat}.0."
|
|
109
|
+
f"{tail.removeprefix('out_w.')}"
|
|
110
|
+
] = value
|
|
111
|
+
elif tail.startswith("ln_w."):
|
|
112
|
+
out[
|
|
113
|
+
f"row_embedding.col_projections.{repeat}.1."
|
|
114
|
+
f"{tail.removeprefix('ln_w.')}"
|
|
115
|
+
] = value
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _add_row_embedding(
|
|
119
|
+
out: dict[str, Tensor],
|
|
120
|
+
qkv: dict[tuple[str, str], dict[str, Tensor]],
|
|
121
|
+
repeat: int,
|
|
122
|
+
tail: str,
|
|
123
|
+
value: Tensor,
|
|
124
|
+
) -> None:
|
|
125
|
+
if tail.startswith("tf_row.blocks."):
|
|
126
|
+
layer, tail = tail.removeprefix("tf_row.blocks.").split(".", 1)
|
|
127
|
+
_add_transformer(
|
|
128
|
+
out,
|
|
129
|
+
qkv,
|
|
130
|
+
f"row_embedding.row_blocks.{repeat}.{layer}.",
|
|
131
|
+
tail,
|
|
132
|
+
value,
|
|
133
|
+
has_rope=True,
|
|
134
|
+
)
|
|
135
|
+
elif tail.startswith("out_ln."):
|
|
136
|
+
out[
|
|
137
|
+
f"row_embedding.row_norms.{repeat}.{tail.removeprefix('out_ln.')}"
|
|
138
|
+
] = value
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def remap_ckpt( # noqa: D103
|
|
142
|
+
ckpt: dict[str, Tensor],
|
|
143
|
+
is_classifier: bool,
|
|
144
|
+
) -> dict[str, Tensor]:
|
|
145
|
+
out: dict[str, Tensor] = {}
|
|
146
|
+
qkv: dict[tuple[str, str], dict[str, Tensor]] = defaultdict(dict)
|
|
147
|
+
|
|
148
|
+
if is_classifier:
|
|
149
|
+
class_weight = ckpt["icl_predictor.y_encoder.projection.weight"]
|
|
150
|
+
class_bias = ckpt["icl_predictor.y_encoder.projection.bias"]
|
|
151
|
+
out["icl_block.y_emb.weight"] = class_weight.t() + class_bias
|
|
152
|
+
|
|
153
|
+
for repeat, layer, side in product(range(2), range(3), ("query", "key")):
|
|
154
|
+
out[
|
|
155
|
+
f"row_embedding.row_blocks.{repeat}.{layer}.attn."
|
|
156
|
+
f"{side}_transform.0.inv_freq"
|
|
157
|
+
] = ckpt["row_interactor.tf_row.rope.freqs"]
|
|
158
|
+
|
|
159
|
+
for key, value in ckpt.items():
|
|
160
|
+
if key == "cell_embedder.fourier_frequencies":
|
|
161
|
+
out["row_embedding.cell_embedding.num_freq"] = value
|
|
162
|
+
|
|
163
|
+
elif key == "cell_embedder.fourier_frequencies_cat":
|
|
164
|
+
out["row_embedding.cell_embedding.cat_freq"] = value
|
|
165
|
+
|
|
166
|
+
elif key.startswith("cell_embedder.in_linear."):
|
|
167
|
+
out[
|
|
168
|
+
key.replace(
|
|
169
|
+
"cell_embedder.in_linear",
|
|
170
|
+
"row_embedding.cell_embedding.num_lin",
|
|
171
|
+
)
|
|
172
|
+
] = value
|
|
173
|
+
|
|
174
|
+
elif key.startswith("cell_embedder.in_linear_cat."):
|
|
175
|
+
out[
|
|
176
|
+
key.replace(
|
|
177
|
+
"cell_embedder.in_linear_cat",
|
|
178
|
+
"row_embedding.cell_embedding.cat_lin",
|
|
179
|
+
)
|
|
180
|
+
] = value
|
|
181
|
+
|
|
182
|
+
elif key == "cell_embedder.y_embedder_lookup.weight":
|
|
183
|
+
if is_classifier:
|
|
184
|
+
out["row_embedding.y_emb.weight"] = value
|
|
185
|
+
|
|
186
|
+
elif key.startswith("cell_embedder.y_embedder_lookup.layers."):
|
|
187
|
+
if not is_classifier:
|
|
188
|
+
layer, tail = key.removeprefix(
|
|
189
|
+
"cell_embedder.y_embedder_lookup.layers."
|
|
190
|
+
).split(".", 1)
|
|
191
|
+
out[f"row_embedding.y_mlp.{2 * int(layer)}.{tail}"] = value
|
|
192
|
+
|
|
193
|
+
elif key == "cls_tokens":
|
|
194
|
+
out["row_embedding.readout_token"] = value
|
|
195
|
+
|
|
196
|
+
elif key.startswith(("col_embedder.", "col_embedder_2.")):
|
|
197
|
+
root, tail = key.split(".", 1)
|
|
198
|
+
repeat = int(root == "col_embedder_2")
|
|
199
|
+
_add_col_embedding(out, qkv, repeat, tail, value)
|
|
200
|
+
|
|
201
|
+
elif key.startswith(("row_interactor.", "row_interactor_2.")):
|
|
202
|
+
root, tail = key.split(".", 1)
|
|
203
|
+
repeat = int(root == "row_interactor_2")
|
|
204
|
+
_add_row_embedding(out, qkv, repeat, tail, value)
|
|
205
|
+
|
|
206
|
+
elif key.startswith("icl_predictor.y_encoder.layers."):
|
|
207
|
+
if not is_classifier:
|
|
208
|
+
layer, tail = key.removeprefix(
|
|
209
|
+
"icl_predictor.y_encoder.layers."
|
|
210
|
+
).split(".", 1)
|
|
211
|
+
out[f"icl_block.y_mlp.{2 * int(layer)}.{tail}"] = value
|
|
212
|
+
|
|
213
|
+
elif key.startswith("icl_predictor.tf_icl.blocks."):
|
|
214
|
+
layer, tail = key.removeprefix(
|
|
215
|
+
"icl_predictor.tf_icl.blocks."
|
|
216
|
+
).split(".", 1)
|
|
217
|
+
_add_transformer(
|
|
218
|
+
out,
|
|
219
|
+
qkv,
|
|
220
|
+
f"icl_block.layers.{layer}.",
|
|
221
|
+
tail,
|
|
222
|
+
value,
|
|
223
|
+
has_rope=False,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
elif key.startswith("icl_predictor.ln."):
|
|
227
|
+
out[key.replace("icl_predictor.ln", "icl_block.norm", 1)] = value
|
|
228
|
+
|
|
229
|
+
elif key.startswith("icl_predictor.decoder.layers."):
|
|
230
|
+
layer, tail = key.removeprefix(
|
|
231
|
+
"icl_predictor.decoder.layers."
|
|
232
|
+
).split(".", 1)
|
|
233
|
+
out[f"icl_block.head.{2 * int(layer)}.{tail}"] = value
|
|
234
|
+
|
|
235
|
+
for (prefix, parameter), parts in qkv.items():
|
|
236
|
+
if all(projection in parts for projection in ("q", "k", "v")):
|
|
237
|
+
out[f"{prefix}attn.qkv_lin.{parameter}"] = torch.cat(
|
|
238
|
+
[parts[projection] for projection in ("q", "k", "v")],
|
|
239
|
+
dim=0,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return out
|
sdm/models/tabfm/icl.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
16
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
17
|
+
|
|
18
|
+
# ruff: noqa: D101, D102
|
|
19
|
+
|
|
20
|
+
from typing import Any, cast
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
from torch import Tensor
|
|
24
|
+
from torch.nn import GELU, Embedding, Linear, ModuleList, RMSNorm, Sequential
|
|
25
|
+
|
|
26
|
+
from sdm.cache import Cache, KVCacheEntry
|
|
27
|
+
from sdm.models.tabfm.block import TabFMTransformerBlock
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ICLBlock(torch.nn.Module):
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
num_classes: int,
|
|
34
|
+
out_channels: int,
|
|
35
|
+
channels: int,
|
|
36
|
+
num_layers: int,
|
|
37
|
+
num_heads: int,
|
|
38
|
+
device: torch.device | str | None = None,
|
|
39
|
+
dtype: torch.dtype | None = None,
|
|
40
|
+
) -> None:
|
|
41
|
+
super().__init__()
|
|
42
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
43
|
+
|
|
44
|
+
self.y_emb: torch.nn.Module | None = None
|
|
45
|
+
self.y_mlp: torch.nn.Module | None = None
|
|
46
|
+
if num_classes > 0:
|
|
47
|
+
self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
|
|
48
|
+
else:
|
|
49
|
+
self.y_mlp = Sequential(
|
|
50
|
+
Linear(1, 2 * channels, **factory_kwargs),
|
|
51
|
+
GELU(approximate="tanh"),
|
|
52
|
+
Linear(2 * channels, channels, **factory_kwargs),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
self.layers = ModuleList(
|
|
56
|
+
TabFMTransformerBlock(channels, num_heads, **factory_kwargs)
|
|
57
|
+
for _ in range(num_layers)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
self.norm = RMSNorm(channels, eps=1e-6, **factory_kwargs)
|
|
61
|
+
self.head = Sequential(
|
|
62
|
+
Linear(channels, 2 * channels, **factory_kwargs),
|
|
63
|
+
GELU(approximate="tanh"),
|
|
64
|
+
Linear(2 * channels, out_channels, **factory_kwargs),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def forward(
|
|
68
|
+
self,
|
|
69
|
+
x: Tensor, # [..., R, D]
|
|
70
|
+
y: Tensor, # [..., R_train]
|
|
71
|
+
*,
|
|
72
|
+
cache: Cache | None = None,
|
|
73
|
+
) -> Tensor: # [..., R_test, out_channels]
|
|
74
|
+
R_train = y.size(-1)
|
|
75
|
+
|
|
76
|
+
if y.numel() > 0:
|
|
77
|
+
if self.y_emb is not None:
|
|
78
|
+
y_emb = self.y_emb(y) # [..., R_train, D]
|
|
79
|
+
else:
|
|
80
|
+
assert self.y_mlp is not None
|
|
81
|
+
y_emb = self.y_mlp(y.unsqueeze(-1)) # [..., R_train, D]
|
|
82
|
+
x[..., :R_train, :] += y_emb.to(x.dtype)
|
|
83
|
+
|
|
84
|
+
for i, layer in enumerate(self.layers):
|
|
85
|
+
key = f"icl_block.layer{i}"
|
|
86
|
+
result = layer(
|
|
87
|
+
query=x[..., R_train:, :] if i == len(self.layers) - 1 else x,
|
|
88
|
+
key_value=(
|
|
89
|
+
cast(KVCacheEntry, cache[key])
|
|
90
|
+
if cache is not None and cache.is_replaying
|
|
91
|
+
else x[..., :R_train, :]
|
|
92
|
+
),
|
|
93
|
+
return_key_value=cache is not None and cache.is_recording,
|
|
94
|
+
out=None
|
|
95
|
+
if torch.is_grad_enabled()
|
|
96
|
+
else x[..., R_train:, :]
|
|
97
|
+
if i == len(self.layers) - 1
|
|
98
|
+
else x,
|
|
99
|
+
) # [..., R, D] or [..., R_test, D]
|
|
100
|
+
|
|
101
|
+
if cache is not None and cache.is_recording:
|
|
102
|
+
x, cache[key] = result
|
|
103
|
+
else:
|
|
104
|
+
x = result
|
|
105
|
+
del result
|
|
106
|
+
|
|
107
|
+
return self.head(self.norm(x)) # [..., R_test, out_channels]
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
16
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
17
|
+
|
|
18
|
+
# ruff: noqa: D205
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections.abc import Iterable
|
|
23
|
+
from typing import Any, ClassVar, cast
|
|
24
|
+
|
|
25
|
+
import torch
|
|
26
|
+
from torch import Tensor
|
|
27
|
+
from torch.nn import ModuleDict
|
|
28
|
+
|
|
29
|
+
from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
|
|
30
|
+
from sdm.cache import Cache
|
|
31
|
+
from sdm.models._huggingface import download_checkpoint
|
|
32
|
+
from sdm.models.base import ICLModel
|
|
33
|
+
from sdm.models.tabfm.ckpt import remap_ckpt
|
|
34
|
+
from sdm.models.tabfm.icl import ICLBlock
|
|
35
|
+
from sdm.models.tabfm.recipe import default_recipe
|
|
36
|
+
from sdm.models.tabfm.row_embedding import RowEmbedding
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class TabFM(ICLModel):
|
|
40
|
+
r"""The tabular foundation model from `"Introducing TabFM: A Zero-shot
|
|
41
|
+
Foundation Model for Tabular Data" <https://research.google/blog/
|
|
42
|
+
introducing-tabfm-a-zero-shot-foundation-model-for-tabular-data>`__.
|
|
43
|
+
|
|
44
|
+
.. figure:: /images/tabfm_light.png
|
|
45
|
+
:figclass: light-only
|
|
46
|
+
:width: 100%
|
|
47
|
+
|
|
48
|
+
.. figure:: /images/tabfm_dark.png
|
|
49
|
+
:figclass: dark-only
|
|
50
|
+
:width: 100%
|
|
51
|
+
|
|
52
|
+
Architecturally, :class:`TabFM` can be viewed as a scaled-up
|
|
53
|
+
:class:`TabICLv2`-style model with Fourier cell embeddings, separate
|
|
54
|
+
numerical and categorical cell projections, two repeated column/row
|
|
55
|
+
interaction stages, :class:`~torch.nn.RMSNorm`-based transformer blocks and
|
|
56
|
+
:class:`~sdm.nn.SwiGLU` feed-forward blocks.
|
|
57
|
+
|
|
58
|
+
.. note::
|
|
59
|
+
:class:`TabFM` model weights are distributed under the
|
|
60
|
+
`TabFM Non-Commercial License v1.0 <https://huggingface.co/google/
|
|
61
|
+
tabfm-1.0.0-pytorch/blob/main/LICENSE>`__.
|
|
62
|
+
Before downloading pretrained weights, users must accept the license
|
|
63
|
+
either interactively when prompted or explicitly via
|
|
64
|
+
``accept_license=True``.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
task: The tasks to initialize. If ``None``, all tasks supported by this
|
|
68
|
+
model are initialized. Pass a single task to avoid initializing
|
|
69
|
+
separate ~1.6B parameter models.
|
|
70
|
+
pretrained: Whether to load the pretrained checkpoint.
|
|
71
|
+
accept_license: Whether to accept the `TabFM Non-Commercial License
|
|
72
|
+
v1.0 <https://huggingface.co/google/tabfm-1.0.0-pytorch/blob/main/
|
|
73
|
+
LICENSE>`__ without showing the interactive license prompt.
|
|
74
|
+
device: The device.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
78
|
+
{Stype.numerical}
|
|
79
|
+
)
|
|
80
|
+
supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
|
|
81
|
+
{Stype.numerical, Stype.categorical}
|
|
82
|
+
)
|
|
83
|
+
supports_multi_target: ClassVar[bool] = False
|
|
84
|
+
supports_related_tables: ClassVar[bool] = False
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
task: TaskLike | Iterable[TaskLike] | None,
|
|
89
|
+
pretrained: bool = True,
|
|
90
|
+
accept_license: bool = False,
|
|
91
|
+
device: torch.device | str | None = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
super().__init__(task=task)
|
|
94
|
+
|
|
95
|
+
self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
|
|
96
|
+
for task in self.tasks:
|
|
97
|
+
self.models[task] = _TabFM(
|
|
98
|
+
num_classes=10 if task == Task.classification else 0,
|
|
99
|
+
device="meta" if pretrained else device,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
if pretrained:
|
|
103
|
+
self._load_from_pretrained(accept_license, device=device)
|
|
104
|
+
|
|
105
|
+
self.eval()
|
|
106
|
+
|
|
107
|
+
@classmethod
|
|
108
|
+
def default_recipe(cls) -> Recipe:
|
|
109
|
+
r""":meta private:""" # noqa: D415
|
|
110
|
+
return default_recipe()
|
|
111
|
+
|
|
112
|
+
def _load_from_pretrained(
|
|
113
|
+
self,
|
|
114
|
+
accept_license: bool,
|
|
115
|
+
device: torch.device | str | None,
|
|
116
|
+
) -> TabFM:
|
|
117
|
+
from safetensors.torch import load_file # noqa: PLC0415
|
|
118
|
+
|
|
119
|
+
TABFM_LICENSE_PROMPT = (
|
|
120
|
+
"TabFM pretrained weights are distributed under the TabFM "
|
|
121
|
+
"Non-Commercial License v1.0 and may be used only for "
|
|
122
|
+
"non-commercial, non-production purposes. Review the license at "
|
|
123
|
+
"'https://huggingface.co/google/tabfm-1.0.0-pytorch/blob/main/"
|
|
124
|
+
"LICENSE' before downloading."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
device = torch.get_default_device() if device is None else device
|
|
128
|
+
|
|
129
|
+
for task, model in self.models.items():
|
|
130
|
+
if task == Task.classification:
|
|
131
|
+
filename = "classification/model.safetensors"
|
|
132
|
+
else:
|
|
133
|
+
assert task == Task.regression
|
|
134
|
+
filename = "regression/model.safetensors"
|
|
135
|
+
|
|
136
|
+
path = download_checkpoint(
|
|
137
|
+
"google/tabfm-1.0.0-pytorch",
|
|
138
|
+
filename,
|
|
139
|
+
license_prompt=None
|
|
140
|
+
if accept_license
|
|
141
|
+
else TABFM_LICENSE_PROMPT,
|
|
142
|
+
)
|
|
143
|
+
ckpt = load_file(path, device=str(device))
|
|
144
|
+
ckpt = remap_ckpt(ckpt, is_classifier=task == Task.classification)
|
|
145
|
+
model.load_state_dict(ckpt, assign=True)
|
|
146
|
+
|
|
147
|
+
return self
|
|
148
|
+
|
|
149
|
+
def _forward(
|
|
150
|
+
self,
|
|
151
|
+
x_context: TableTensor | None, # [..., R_context, D]
|
|
152
|
+
y_context: TableTensor | None, # [..., R_context, 1]
|
|
153
|
+
x_query: TableTensor | None, # [..., R_query, D]
|
|
154
|
+
related_context_tables: RelatedTables[TableTensor] | None,
|
|
155
|
+
related_query_tables: RelatedTables[TableTensor] | None,
|
|
156
|
+
cache: Cache | None,
|
|
157
|
+
generator: torch.Generator | None,
|
|
158
|
+
*,
|
|
159
|
+
categorical_mask: Tensor,
|
|
160
|
+
**kwargs: Any,
|
|
161
|
+
) -> TableTensor: # [..., R_query, num_classes or 1]
|
|
162
|
+
|
|
163
|
+
if x_query is None and x_context is not None:
|
|
164
|
+
x = x_context.numerical
|
|
165
|
+
elif x_context is None and x_query is not None:
|
|
166
|
+
x = x_query.numerical
|
|
167
|
+
else:
|
|
168
|
+
assert x_context is not None
|
|
169
|
+
assert x_query is not None
|
|
170
|
+
x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
|
|
171
|
+
|
|
172
|
+
classes: Tensor | None = None
|
|
173
|
+
if y_context is not None and y_context.categorical.size(-1) > 0:
|
|
174
|
+
y = y_context.categorical.code.squeeze(-1)
|
|
175
|
+
classes = y_context.categorical.categories[0]
|
|
176
|
+
elif y_context is not None and y_context.numerical.size(-1) > 0:
|
|
177
|
+
y = y_context.numerical.squeeze(-1)
|
|
178
|
+
else:
|
|
179
|
+
assert cache is not None
|
|
180
|
+
classes = cast(Tensor | None, cache["classes"])
|
|
181
|
+
y = x.new_empty(
|
|
182
|
+
(*x.size()[:-2], 0),
|
|
183
|
+
dtype=torch.int64 if classes is not None else x.dtype,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
if classes is not None and len(classes) > 10:
|
|
187
|
+
raise ValueError(
|
|
188
|
+
f"{self.__class__.__name__!r} only supports up to 10 classes "
|
|
189
|
+
f"(got {len(classes)})"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
categorical_mask = categorical_mask.expand(*x.size()[:-2], -1)
|
|
193
|
+
|
|
194
|
+
task = Task.classification if classes is not None else Task.regression
|
|
195
|
+
out = self.models[task](x, y, categorical_mask, cache=cache)
|
|
196
|
+
|
|
197
|
+
if classes is None:
|
|
198
|
+
return TableTensor(
|
|
199
|
+
columns={Stype.numerical: ["pred"]},
|
|
200
|
+
numerical=out,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
return TableTensor(
|
|
204
|
+
columns={Stype.numerical: [str(i) for i in classes.tolist()]},
|
|
205
|
+
numerical=out[..., : len(classes)],
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class _TabFM(torch.nn.Module):
|
|
210
|
+
def __init__(
|
|
211
|
+
self,
|
|
212
|
+
num_classes: int,
|
|
213
|
+
channels: int = 256,
|
|
214
|
+
num_embedding_layers: int = 3,
|
|
215
|
+
num_embedding_repeats: int = 2,
|
|
216
|
+
num_embedding_col_heads: int = 4,
|
|
217
|
+
num_embedding_row_heads: int = 8,
|
|
218
|
+
group_size: int = 3,
|
|
219
|
+
num_frequencies: int = 32,
|
|
220
|
+
num_inducing_points: int = 256,
|
|
221
|
+
num_readout_tokens: int = 8,
|
|
222
|
+
num_icl_layers: int = 24,
|
|
223
|
+
num_icl_heads: int = 8,
|
|
224
|
+
device: torch.device | str | None = None,
|
|
225
|
+
dtype: torch.dtype | None = None,
|
|
226
|
+
) -> None:
|
|
227
|
+
super().__init__()
|
|
228
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
229
|
+
|
|
230
|
+
self.row_embedding = RowEmbedding(
|
|
231
|
+
num_classes=num_classes,
|
|
232
|
+
channels=channels,
|
|
233
|
+
num_layers=num_embedding_layers,
|
|
234
|
+
num_repeats=num_embedding_repeats,
|
|
235
|
+
num_col_heads=num_embedding_col_heads,
|
|
236
|
+
num_row_heads=num_embedding_row_heads,
|
|
237
|
+
group_size=group_size,
|
|
238
|
+
num_frequencies=num_frequencies,
|
|
239
|
+
num_inducing_points=num_inducing_points,
|
|
240
|
+
num_readout_tokens=num_readout_tokens,
|
|
241
|
+
**factory_kwargs,
|
|
242
|
+
)
|
|
243
|
+
self.icl_block = ICLBlock(
|
|
244
|
+
num_classes=num_classes,
|
|
245
|
+
out_channels=max(1, num_classes),
|
|
246
|
+
channels=num_readout_tokens * channels,
|
|
247
|
+
num_layers=num_icl_layers,
|
|
248
|
+
num_heads=num_icl_heads,
|
|
249
|
+
**factory_kwargs,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def forward(
|
|
253
|
+
self,
|
|
254
|
+
x: Tensor, # [..., R, C]
|
|
255
|
+
y: Tensor, # [..., R_train]
|
|
256
|
+
categorical_mask: Tensor, # [..., C]
|
|
257
|
+
*,
|
|
258
|
+
cache: Cache | None = None,
|
|
259
|
+
) -> Tensor: # [..., R_test, 1 or num_classes]
|
|
260
|
+
x = self.row_embedding(x, y, categorical_mask, cache=cache)
|
|
261
|
+
return self.icl_block(x, y, cache=cache)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import sdm.processing as sp
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def default_recipe() -> sp.Recipe: # noqa: D103
|
|
8
|
+
return sp.Recipe(
|
|
9
|
+
features=[
|
|
10
|
+
sp.StypeDispatch(
|
|
11
|
+
categorical=[
|
|
12
|
+
sp.AlignCategories(min_frequency=2),
|
|
13
|
+
sp.ToNumerical(),
|
|
14
|
+
],
|
|
15
|
+
),
|
|
16
|
+
sp.StypeDispatch(
|
|
17
|
+
numerical=[
|
|
18
|
+
sp.ImputeMean(),
|
|
19
|
+
sp.DropConstantColumns(),
|
|
20
|
+
sp.Standardize(eps=1e-6),
|
|
21
|
+
sp.Clip(-100.0, 100.0),
|
|
22
|
+
sp.Choice(
|
|
23
|
+
sp.Identity(),
|
|
24
|
+
sp.PowerTransform(),
|
|
25
|
+
method="round_robin",
|
|
26
|
+
),
|
|
27
|
+
sp.ClipSigma(threshold=4.0),
|
|
28
|
+
sp.ShuffleColumns(method="random"),
|
|
29
|
+
sp.SelectColumns(500, method="first"),
|
|
30
|
+
],
|
|
31
|
+
),
|
|
32
|
+
],
|
|
33
|
+
target=sp.StypeDispatch(
|
|
34
|
+
categorical=[
|
|
35
|
+
sp.AlignCategories(sort_by="value"),
|
|
36
|
+
sp.ShuffleCategories(method="shift"),
|
|
37
|
+
],
|
|
38
|
+
numerical=sp.Standardize(),
|
|
39
|
+
),
|
|
40
|
+
output=[
|
|
41
|
+
sp.AverageEstimators(),
|
|
42
|
+
sp.TaskDispatch(
|
|
43
|
+
classification=sp.Softmax(temperature=0.9),
|
|
44
|
+
),
|
|
45
|
+
],
|
|
46
|
+
)
|