structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,242 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
16
+ # SPDX-License-Identifier: Apache-2.0
17
+
18
+ import re
19
+ from collections import defaultdict
20
+ from itertools import product
21
+
22
+ import torch
23
+ from torch import Tensor
24
+
25
+
26
+ def _map_transformer(prefix: str, tail: str, *, has_rope: bool) -> list[str]:
27
+ if tail.startswith("pre_attn_ln."):
28
+ tail = tail.removeprefix("pre_attn_ln.")
29
+ return [
30
+ f"{prefix}query_norm.{tail}",
31
+ f"{prefix}key_value_norm.{tail}",
32
+ ]
33
+ if tail.startswith("post_attn_ln."):
34
+ tail = tail.replace("post_attn_ln.", "post_attn_norm.", 1)
35
+ elif tail.startswith("pre_ff_ln."):
36
+ tail = tail.replace("pre_ff_ln.", "mlp.0.", 1)
37
+ elif tail.startswith("linear1."):
38
+ tail = tail.replace("linear1.", "mlp.1.up_lin.", 1)
39
+ elif tail.startswith("linear1_gate."):
40
+ tail = tail.replace("linear1_gate.", "mlp.1.gate_lin.", 1)
41
+ elif tail.startswith("linear2."):
42
+ tail = tail.replace("linear2.", "mlp.1.down_lin.", 1)
43
+ elif tail.startswith("post_ff_ln."):
44
+ tail = tail.replace("post_ff_ln.", "mlp.2.", 1)
45
+ elif tail.startswith("attn.out_proj."):
46
+ tail = tail.replace("attn.out_proj.", "attn.out_lin.", 1)
47
+ elif tail == "attn.query_ln.weight":
48
+ index = int(has_rope)
49
+ tail = f"attn.query_transform.{index}.weight"
50
+ elif tail == "attn.key_ln.weight":
51
+ index = int(has_rope)
52
+ tail = f"attn.key_transform.{index}.weight"
53
+ elif tail == "attn.per_dim_scale":
54
+ index = int(has_rope) + 1
55
+ tail = f"attn.query_transform.{index}.weight"
56
+
57
+ return [prefix + tail]
58
+
59
+
60
+ def _add_transformer(
61
+ out: dict[str, Tensor],
62
+ qkv: dict[tuple[str, str], dict[str, Tensor]],
63
+ prefix: str,
64
+ tail: str,
65
+ value: Tensor,
66
+ *,
67
+ has_rope: bool,
68
+ ) -> None:
69
+ match = re.match(r"attn\.([qkv])_proj\.(weight|bias)", tail)
70
+ if match is not None:
71
+ projection, parameter = match.groups()
72
+ qkv[(prefix, parameter)][projection] = value
73
+ return
74
+
75
+ for key in _map_transformer(prefix, tail, has_rope=has_rope):
76
+ out[key] = value
77
+
78
+
79
+ def _add_col_embedding(
80
+ out: dict[str, Tensor],
81
+ qkv: dict[tuple[str, str], dict[str, Tensor]],
82
+ repeat: int,
83
+ tail: str,
84
+ value: Tensor,
85
+ ) -> None:
86
+ if tail.startswith("tf_col.blocks."):
87
+ layer, tail = tail.removeprefix("tf_col.blocks.").split(".", 1)
88
+ if tail == "ind_vectors":
89
+ out[
90
+ f"row_embedding.col_blocks.{repeat}.{layer}.inducing_points"
91
+ ] = value
92
+ return
93
+
94
+ block, tail = tail.split(".", 1)
95
+ block = {"mab1": "inducing_block", "mab2": "output_block"}[block]
96
+ _add_transformer(
97
+ out,
98
+ qkv,
99
+ f"row_embedding.col_blocks.{repeat}.{layer}.{block}.",
100
+ tail,
101
+ value,
102
+ has_rope=False,
103
+ )
104
+ return
105
+
106
+ if tail.startswith("out_w."):
107
+ out[
108
+ f"row_embedding.col_projections.{repeat}.0."
109
+ f"{tail.removeprefix('out_w.')}"
110
+ ] = value
111
+ elif tail.startswith("ln_w."):
112
+ out[
113
+ f"row_embedding.col_projections.{repeat}.1."
114
+ f"{tail.removeprefix('ln_w.')}"
115
+ ] = value
116
+
117
+
118
+ def _add_row_embedding(
119
+ out: dict[str, Tensor],
120
+ qkv: dict[tuple[str, str], dict[str, Tensor]],
121
+ repeat: int,
122
+ tail: str,
123
+ value: Tensor,
124
+ ) -> None:
125
+ if tail.startswith("tf_row.blocks."):
126
+ layer, tail = tail.removeprefix("tf_row.blocks.").split(".", 1)
127
+ _add_transformer(
128
+ out,
129
+ qkv,
130
+ f"row_embedding.row_blocks.{repeat}.{layer}.",
131
+ tail,
132
+ value,
133
+ has_rope=True,
134
+ )
135
+ elif tail.startswith("out_ln."):
136
+ out[
137
+ f"row_embedding.row_norms.{repeat}.{tail.removeprefix('out_ln.')}"
138
+ ] = value
139
+
140
+
141
+ def remap_ckpt( # noqa: D103
142
+ ckpt: dict[str, Tensor],
143
+ is_classifier: bool,
144
+ ) -> dict[str, Tensor]:
145
+ out: dict[str, Tensor] = {}
146
+ qkv: dict[tuple[str, str], dict[str, Tensor]] = defaultdict(dict)
147
+
148
+ if is_classifier:
149
+ class_weight = ckpt["icl_predictor.y_encoder.projection.weight"]
150
+ class_bias = ckpt["icl_predictor.y_encoder.projection.bias"]
151
+ out["icl_block.y_emb.weight"] = class_weight.t() + class_bias
152
+
153
+ for repeat, layer, side in product(range(2), range(3), ("query", "key")):
154
+ out[
155
+ f"row_embedding.row_blocks.{repeat}.{layer}.attn."
156
+ f"{side}_transform.0.inv_freq"
157
+ ] = ckpt["row_interactor.tf_row.rope.freqs"]
158
+
159
+ for key, value in ckpt.items():
160
+ if key == "cell_embedder.fourier_frequencies":
161
+ out["row_embedding.cell_embedding.num_freq"] = value
162
+
163
+ elif key == "cell_embedder.fourier_frequencies_cat":
164
+ out["row_embedding.cell_embedding.cat_freq"] = value
165
+
166
+ elif key.startswith("cell_embedder.in_linear."):
167
+ out[
168
+ key.replace(
169
+ "cell_embedder.in_linear",
170
+ "row_embedding.cell_embedding.num_lin",
171
+ )
172
+ ] = value
173
+
174
+ elif key.startswith("cell_embedder.in_linear_cat."):
175
+ out[
176
+ key.replace(
177
+ "cell_embedder.in_linear_cat",
178
+ "row_embedding.cell_embedding.cat_lin",
179
+ )
180
+ ] = value
181
+
182
+ elif key == "cell_embedder.y_embedder_lookup.weight":
183
+ if is_classifier:
184
+ out["row_embedding.y_emb.weight"] = value
185
+
186
+ elif key.startswith("cell_embedder.y_embedder_lookup.layers."):
187
+ if not is_classifier:
188
+ layer, tail = key.removeprefix(
189
+ "cell_embedder.y_embedder_lookup.layers."
190
+ ).split(".", 1)
191
+ out[f"row_embedding.y_mlp.{2 * int(layer)}.{tail}"] = value
192
+
193
+ elif key == "cls_tokens":
194
+ out["row_embedding.readout_token"] = value
195
+
196
+ elif key.startswith(("col_embedder.", "col_embedder_2.")):
197
+ root, tail = key.split(".", 1)
198
+ repeat = int(root == "col_embedder_2")
199
+ _add_col_embedding(out, qkv, repeat, tail, value)
200
+
201
+ elif key.startswith(("row_interactor.", "row_interactor_2.")):
202
+ root, tail = key.split(".", 1)
203
+ repeat = int(root == "row_interactor_2")
204
+ _add_row_embedding(out, qkv, repeat, tail, value)
205
+
206
+ elif key.startswith("icl_predictor.y_encoder.layers."):
207
+ if not is_classifier:
208
+ layer, tail = key.removeprefix(
209
+ "icl_predictor.y_encoder.layers."
210
+ ).split(".", 1)
211
+ out[f"icl_block.y_mlp.{2 * int(layer)}.{tail}"] = value
212
+
213
+ elif key.startswith("icl_predictor.tf_icl.blocks."):
214
+ layer, tail = key.removeprefix(
215
+ "icl_predictor.tf_icl.blocks."
216
+ ).split(".", 1)
217
+ _add_transformer(
218
+ out,
219
+ qkv,
220
+ f"icl_block.layers.{layer}.",
221
+ tail,
222
+ value,
223
+ has_rope=False,
224
+ )
225
+
226
+ elif key.startswith("icl_predictor.ln."):
227
+ out[key.replace("icl_predictor.ln", "icl_block.norm", 1)] = value
228
+
229
+ elif key.startswith("icl_predictor.decoder.layers."):
230
+ layer, tail = key.removeprefix(
231
+ "icl_predictor.decoder.layers."
232
+ ).split(".", 1)
233
+ out[f"icl_block.head.{2 * int(layer)}.{tail}"] = value
234
+
235
+ for (prefix, parameter), parts in qkv.items():
236
+ if all(projection in parts for projection in ("q", "k", "v")):
237
+ out[f"{prefix}attn.qkv_lin.{parameter}"] = torch.cat(
238
+ [parts[projection] for projection in ("q", "k", "v")],
239
+ dim=0,
240
+ )
241
+
242
+ return out
@@ -0,0 +1,107 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
16
+ # SPDX-License-Identifier: Apache-2.0
17
+
18
+ # ruff: noqa: D101, D102
19
+
20
+ from typing import Any, cast
21
+
22
+ import torch
23
+ from torch import Tensor
24
+ from torch.nn import GELU, Embedding, Linear, ModuleList, RMSNorm, Sequential
25
+
26
+ from sdm.cache import Cache, KVCacheEntry
27
+ from sdm.models.tabfm.block import TabFMTransformerBlock
28
+
29
+
30
+ class ICLBlock(torch.nn.Module):
31
+ def __init__(
32
+ self,
33
+ num_classes: int,
34
+ out_channels: int,
35
+ channels: int,
36
+ num_layers: int,
37
+ num_heads: int,
38
+ device: torch.device | str | None = None,
39
+ dtype: torch.dtype | None = None,
40
+ ) -> None:
41
+ super().__init__()
42
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
43
+
44
+ self.y_emb: torch.nn.Module | None = None
45
+ self.y_mlp: torch.nn.Module | None = None
46
+ if num_classes > 0:
47
+ self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
48
+ else:
49
+ self.y_mlp = Sequential(
50
+ Linear(1, 2 * channels, **factory_kwargs),
51
+ GELU(approximate="tanh"),
52
+ Linear(2 * channels, channels, **factory_kwargs),
53
+ )
54
+
55
+ self.layers = ModuleList(
56
+ TabFMTransformerBlock(channels, num_heads, **factory_kwargs)
57
+ for _ in range(num_layers)
58
+ )
59
+
60
+ self.norm = RMSNorm(channels, eps=1e-6, **factory_kwargs)
61
+ self.head = Sequential(
62
+ Linear(channels, 2 * channels, **factory_kwargs),
63
+ GELU(approximate="tanh"),
64
+ Linear(2 * channels, out_channels, **factory_kwargs),
65
+ )
66
+
67
+ def forward(
68
+ self,
69
+ x: Tensor, # [..., R, D]
70
+ y: Tensor, # [..., R_train]
71
+ *,
72
+ cache: Cache | None = None,
73
+ ) -> Tensor: # [..., R_test, out_channels]
74
+ R_train = y.size(-1)
75
+
76
+ if y.numel() > 0:
77
+ if self.y_emb is not None:
78
+ y_emb = self.y_emb(y) # [..., R_train, D]
79
+ else:
80
+ assert self.y_mlp is not None
81
+ y_emb = self.y_mlp(y.unsqueeze(-1)) # [..., R_train, D]
82
+ x[..., :R_train, :] += y_emb.to(x.dtype)
83
+
84
+ for i, layer in enumerate(self.layers):
85
+ key = f"icl_block.layer{i}"
86
+ result = layer(
87
+ query=x[..., R_train:, :] if i == len(self.layers) - 1 else x,
88
+ key_value=(
89
+ cast(KVCacheEntry, cache[key])
90
+ if cache is not None and cache.is_replaying
91
+ else x[..., :R_train, :]
92
+ ),
93
+ return_key_value=cache is not None and cache.is_recording,
94
+ out=None
95
+ if torch.is_grad_enabled()
96
+ else x[..., R_train:, :]
97
+ if i == len(self.layers) - 1
98
+ else x,
99
+ ) # [..., R, D] or [..., R_test, D]
100
+
101
+ if cache is not None and cache.is_recording:
102
+ x, cache[key] = result
103
+ else:
104
+ x = result
105
+ del result
106
+
107
+ return self.head(self.norm(x)) # [..., R_test, out_channels]
@@ -0,0 +1,261 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
16
+ # SPDX-License-Identifier: Apache-2.0
17
+
18
+ # ruff: noqa: D205
19
+
20
+ from __future__ import annotations
21
+
22
+ from collections.abc import Iterable
23
+ from typing import Any, ClassVar, cast
24
+
25
+ import torch
26
+ from torch import Tensor
27
+ from torch.nn import ModuleDict
28
+
29
+ from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
30
+ from sdm.cache import Cache
31
+ from sdm.models._huggingface import download_checkpoint
32
+ from sdm.models.base import ICLModel
33
+ from sdm.models.tabfm.ckpt import remap_ckpt
34
+ from sdm.models.tabfm.icl import ICLBlock
35
+ from sdm.models.tabfm.recipe import default_recipe
36
+ from sdm.models.tabfm.row_embedding import RowEmbedding
37
+
38
+
39
+ class TabFM(ICLModel):
40
+ r"""The tabular foundation model from `"Introducing TabFM: A Zero-shot
41
+ Foundation Model for Tabular Data" <https://research.google/blog/
42
+ introducing-tabfm-a-zero-shot-foundation-model-for-tabular-data>`__.
43
+
44
+ .. figure:: /images/tabfm_light.png
45
+ :figclass: light-only
46
+ :width: 100%
47
+
48
+ .. figure:: /images/tabfm_dark.png
49
+ :figclass: dark-only
50
+ :width: 100%
51
+
52
+ Architecturally, :class:`TabFM` can be viewed as a scaled-up
53
+ :class:`TabICLv2`-style model with Fourier cell embeddings, separate
54
+ numerical and categorical cell projections, two repeated column/row
55
+ interaction stages, :class:`~torch.nn.RMSNorm`-based transformer blocks and
56
+ :class:`~sdm.nn.SwiGLU` feed-forward blocks.
57
+
58
+ .. note::
59
+ :class:`TabFM` model weights are distributed under the
60
+ `TabFM Non-Commercial License v1.0 <https://huggingface.co/google/
61
+ tabfm-1.0.0-pytorch/blob/main/LICENSE>`__.
62
+ Before downloading pretrained weights, users must accept the license
63
+ either interactively when prompted or explicitly via
64
+ ``accept_license=True``.
65
+
66
+ Args:
67
+ task: The tasks to initialize. If ``None``, all tasks supported by this
68
+ model are initialized. Pass a single task to avoid initializing
69
+ separate ~1.6B parameter models.
70
+ pretrained: Whether to load the pretrained checkpoint.
71
+ accept_license: Whether to accept the `TabFM Non-Commercial License
72
+ v1.0 <https://huggingface.co/google/tabfm-1.0.0-pytorch/blob/main/
73
+ LICENSE>`__ without showing the interactive license prompt.
74
+ device: The device.
75
+ """
76
+
77
+ supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
78
+ {Stype.numerical}
79
+ )
80
+ supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
81
+ {Stype.numerical, Stype.categorical}
82
+ )
83
+ supports_multi_target: ClassVar[bool] = False
84
+ supports_related_tables: ClassVar[bool] = False
85
+
86
+ def __init__(
87
+ self,
88
+ task: TaskLike | Iterable[TaskLike] | None,
89
+ pretrained: bool = True,
90
+ accept_license: bool = False,
91
+ device: torch.device | str | None = None,
92
+ ) -> None:
93
+ super().__init__(task=task)
94
+
95
+ self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
96
+ for task in self.tasks:
97
+ self.models[task] = _TabFM(
98
+ num_classes=10 if task == Task.classification else 0,
99
+ device="meta" if pretrained else device,
100
+ )
101
+
102
+ if pretrained:
103
+ self._load_from_pretrained(accept_license, device=device)
104
+
105
+ self.eval()
106
+
107
+ @classmethod
108
+ def default_recipe(cls) -> Recipe:
109
+ r""":meta private:""" # noqa: D415
110
+ return default_recipe()
111
+
112
+ def _load_from_pretrained(
113
+ self,
114
+ accept_license: bool,
115
+ device: torch.device | str | None,
116
+ ) -> TabFM:
117
+ from safetensors.torch import load_file # noqa: PLC0415
118
+
119
+ TABFM_LICENSE_PROMPT = (
120
+ "TabFM pretrained weights are distributed under the TabFM "
121
+ "Non-Commercial License v1.0 and may be used only for "
122
+ "non-commercial, non-production purposes. Review the license at "
123
+ "'https://huggingface.co/google/tabfm-1.0.0-pytorch/blob/main/"
124
+ "LICENSE' before downloading."
125
+ )
126
+
127
+ device = torch.get_default_device() if device is None else device
128
+
129
+ for task, model in self.models.items():
130
+ if task == Task.classification:
131
+ filename = "classification/model.safetensors"
132
+ else:
133
+ assert task == Task.regression
134
+ filename = "regression/model.safetensors"
135
+
136
+ path = download_checkpoint(
137
+ "google/tabfm-1.0.0-pytorch",
138
+ filename,
139
+ license_prompt=None
140
+ if accept_license
141
+ else TABFM_LICENSE_PROMPT,
142
+ )
143
+ ckpt = load_file(path, device=str(device))
144
+ ckpt = remap_ckpt(ckpt, is_classifier=task == Task.classification)
145
+ model.load_state_dict(ckpt, assign=True)
146
+
147
+ return self
148
+
149
+ def _forward(
150
+ self,
151
+ x_context: TableTensor | None, # [..., R_context, D]
152
+ y_context: TableTensor | None, # [..., R_context, 1]
153
+ x_query: TableTensor | None, # [..., R_query, D]
154
+ related_context_tables: RelatedTables[TableTensor] | None,
155
+ related_query_tables: RelatedTables[TableTensor] | None,
156
+ cache: Cache | None,
157
+ generator: torch.Generator | None,
158
+ *,
159
+ categorical_mask: Tensor,
160
+ **kwargs: Any,
161
+ ) -> TableTensor: # [..., R_query, num_classes or 1]
162
+
163
+ if x_query is None and x_context is not None:
164
+ x = x_context.numerical
165
+ elif x_context is None and x_query is not None:
166
+ x = x_query.numerical
167
+ else:
168
+ assert x_context is not None
169
+ assert x_query is not None
170
+ x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
171
+
172
+ classes: Tensor | None = None
173
+ if y_context is not None and y_context.categorical.size(-1) > 0:
174
+ y = y_context.categorical.code.squeeze(-1)
175
+ classes = y_context.categorical.categories[0]
176
+ elif y_context is not None and y_context.numerical.size(-1) > 0:
177
+ y = y_context.numerical.squeeze(-1)
178
+ else:
179
+ assert cache is not None
180
+ classes = cast(Tensor | None, cache["classes"])
181
+ y = x.new_empty(
182
+ (*x.size()[:-2], 0),
183
+ dtype=torch.int64 if classes is not None else x.dtype,
184
+ )
185
+
186
+ if classes is not None and len(classes) > 10:
187
+ raise ValueError(
188
+ f"{self.__class__.__name__!r} only supports up to 10 classes "
189
+ f"(got {len(classes)})"
190
+ )
191
+
192
+ categorical_mask = categorical_mask.expand(*x.size()[:-2], -1)
193
+
194
+ task = Task.classification if classes is not None else Task.regression
195
+ out = self.models[task](x, y, categorical_mask, cache=cache)
196
+
197
+ if classes is None:
198
+ return TableTensor(
199
+ columns={Stype.numerical: ["pred"]},
200
+ numerical=out,
201
+ )
202
+
203
+ return TableTensor(
204
+ columns={Stype.numerical: [str(i) for i in classes.tolist()]},
205
+ numerical=out[..., : len(classes)],
206
+ )
207
+
208
+
209
+ class _TabFM(torch.nn.Module):
210
+ def __init__(
211
+ self,
212
+ num_classes: int,
213
+ channels: int = 256,
214
+ num_embedding_layers: int = 3,
215
+ num_embedding_repeats: int = 2,
216
+ num_embedding_col_heads: int = 4,
217
+ num_embedding_row_heads: int = 8,
218
+ group_size: int = 3,
219
+ num_frequencies: int = 32,
220
+ num_inducing_points: int = 256,
221
+ num_readout_tokens: int = 8,
222
+ num_icl_layers: int = 24,
223
+ num_icl_heads: int = 8,
224
+ device: torch.device | str | None = None,
225
+ dtype: torch.dtype | None = None,
226
+ ) -> None:
227
+ super().__init__()
228
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
229
+
230
+ self.row_embedding = RowEmbedding(
231
+ num_classes=num_classes,
232
+ channels=channels,
233
+ num_layers=num_embedding_layers,
234
+ num_repeats=num_embedding_repeats,
235
+ num_col_heads=num_embedding_col_heads,
236
+ num_row_heads=num_embedding_row_heads,
237
+ group_size=group_size,
238
+ num_frequencies=num_frequencies,
239
+ num_inducing_points=num_inducing_points,
240
+ num_readout_tokens=num_readout_tokens,
241
+ **factory_kwargs,
242
+ )
243
+ self.icl_block = ICLBlock(
244
+ num_classes=num_classes,
245
+ out_channels=max(1, num_classes),
246
+ channels=num_readout_tokens * channels,
247
+ num_layers=num_icl_layers,
248
+ num_heads=num_icl_heads,
249
+ **factory_kwargs,
250
+ )
251
+
252
+ def forward(
253
+ self,
254
+ x: Tensor, # [..., R, C]
255
+ y: Tensor, # [..., R_train]
256
+ categorical_mask: Tensor, # [..., C]
257
+ *,
258
+ cache: Cache | None = None,
259
+ ) -> Tensor: # [..., R_test, 1 or num_classes]
260
+ x = self.row_embedding(x, y, categorical_mask, cache=cache)
261
+ return self.icl_block(x, y, cache=cache)
@@ -0,0 +1,46 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import sdm.processing as sp
5
+
6
+
7
+ def default_recipe() -> sp.Recipe: # noqa: D103
8
+ return sp.Recipe(
9
+ features=[
10
+ sp.StypeDispatch(
11
+ categorical=[
12
+ sp.AlignCategories(min_frequency=2),
13
+ sp.ToNumerical(),
14
+ ],
15
+ ),
16
+ sp.StypeDispatch(
17
+ numerical=[
18
+ sp.ImputeMean(),
19
+ sp.DropConstantColumns(),
20
+ sp.Standardize(eps=1e-6),
21
+ sp.Clip(-100.0, 100.0),
22
+ sp.Choice(
23
+ sp.Identity(),
24
+ sp.PowerTransform(),
25
+ method="round_robin",
26
+ ),
27
+ sp.ClipSigma(threshold=4.0),
28
+ sp.ShuffleColumns(method="random"),
29
+ sp.SelectColumns(500, method="first"),
30
+ ],
31
+ ),
32
+ ],
33
+ target=sp.StypeDispatch(
34
+ categorical=[
35
+ sp.AlignCategories(sort_by="value"),
36
+ sp.ShuffleCategories(method="shift"),
37
+ ],
38
+ numerical=sp.Standardize(),
39
+ ),
40
+ output=[
41
+ sp.AverageEstimators(),
42
+ sp.TaskDispatch(
43
+ classification=sp.Softmax(temperature=0.9),
44
+ ),
45
+ ],
46
+ )