structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,241 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
16
+ # SPDX-License-Identifier: Apache-2.0
17
+
18
+ # ruff: noqa: D101, D102
19
+
20
+ from typing import Any, cast
21
+
22
+ import torch
23
+ from torch import Tensor
24
+ from torch.nn import (
25
+ GELU,
26
+ Embedding,
27
+ Linear,
28
+ ModuleList,
29
+ Parameter,
30
+ RMSNorm,
31
+ Sequential,
32
+ )
33
+
34
+ from sdm.cache import Cache, KVCacheEntry
35
+ from sdm.models.tabfm.block import TabFMTransformerBlock
36
+ from sdm.models.tabfm.cell_embedding import CellEmbedding
37
+ from sdm.nn import InducedTransformerBlock, RotaryEmbedding
38
+
39
+
40
+ class RowEmbedding(torch.nn.Module):
41
+ def __init__(
42
+ self,
43
+ num_classes: int,
44
+ channels: int,
45
+ num_layers: int,
46
+ num_repeats: int,
47
+ num_col_heads: int,
48
+ num_row_heads: int,
49
+ group_size: int,
50
+ num_frequencies: int,
51
+ num_inducing_points: int,
52
+ num_readout_tokens: int,
53
+ device: torch.device | str | None = None,
54
+ dtype: torch.dtype | None = None,
55
+ ) -> None:
56
+ super().__init__()
57
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
58
+ self.channels = channels
59
+
60
+ self.cell_embedding = CellEmbedding(
61
+ channels=channels,
62
+ group_size=group_size,
63
+ num_frequencies=num_frequencies,
64
+ **factory_kwargs,
65
+ )
66
+
67
+ self.y_emb: torch.nn.Module | None = None
68
+ self.y_mlp: torch.nn.Module | None = None
69
+ if num_classes > 0:
70
+ self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
71
+ else:
72
+ self.y_mlp = Sequential(
73
+ Linear(1, 6, **factory_kwargs),
74
+ GELU(approximate="tanh"),
75
+ Linear(6, channels, **factory_kwargs),
76
+ )
77
+
78
+ rope = RotaryEmbedding(
79
+ channels=channels // num_row_heads,
80
+ layout="interleaved",
81
+ theta=100_000,
82
+ requires_grad=False,
83
+ **factory_kwargs,
84
+ )
85
+
86
+ self.col_blocks: ModuleList[ModuleList] = ModuleList()
87
+ self.row_blocks: ModuleList[ModuleList] = ModuleList()
88
+ for _ in range(num_repeats):
89
+ col_layers = ModuleList(
90
+ InducedTransformerBlock(
91
+ channels=channels,
92
+ num_inducing_points=num_inducing_points,
93
+ inducing_block=TabFMTransformerBlock(
94
+ channels=channels,
95
+ num_heads=num_col_heads,
96
+ **factory_kwargs,
97
+ ),
98
+ output_block=TabFMTransformerBlock(
99
+ channels=channels,
100
+ num_heads=num_col_heads,
101
+ **factory_kwargs,
102
+ ),
103
+ **factory_kwargs,
104
+ )
105
+ for _ in range(num_layers)
106
+ )
107
+ self.col_blocks.append(col_layers)
108
+
109
+ row_layers = ModuleList(
110
+ TabFMTransformerBlock(
111
+ channels=channels,
112
+ num_heads=num_row_heads,
113
+ rope=rope,
114
+ **factory_kwargs,
115
+ )
116
+ for _ in range(num_layers)
117
+ )
118
+ self.row_blocks.append(row_layers)
119
+
120
+ self.col_projections = ModuleList(
121
+ Sequential(
122
+ Linear(channels, channels, **factory_kwargs),
123
+ RMSNorm(channels, eps=1e-6, **factory_kwargs),
124
+ )
125
+ for _ in range(num_repeats)
126
+ )
127
+
128
+ self.row_norms = ModuleList(
129
+ RMSNorm(channels, eps=1e-6, **factory_kwargs)
130
+ for _ in range(num_repeats)
131
+ )
132
+
133
+ self.readout_token = Parameter(
134
+ torch.empty((num_readout_tokens, channels), **factory_kwargs)
135
+ )
136
+ torch.nn.init.trunc_normal_(self.readout_token, std=0.02)
137
+
138
+ def forward(
139
+ self,
140
+ x: Tensor, # [..., R, C]
141
+ y: Tensor, # [..., R_train]
142
+ categorical_mask: Tensor, # [..., C]
143
+ *,
144
+ cache: Cache | None = None,
145
+ ) -> Tensor: # [..., R, K * D]
146
+
147
+ *B, R, C = x.size()
148
+ R_train = y.size(-1)
149
+ K = self.readout_token.size(-2)
150
+ D = self.channels
151
+
152
+ buffer: Tensor | None = None
153
+ if torch.is_grad_enabled():
154
+ x = self.cell_embedding(x, categorical_mask) # [..., R, C, D]
155
+ else:
156
+ buffer = torch.empty(
157
+ (*B, R, K + C, D),
158
+ device=x.device,
159
+ dtype=torch.get_autocast_dtype(x.device.type)
160
+ if torch.is_autocast_enabled(x.device.type)
161
+ else x.dtype,
162
+ )
163
+ buffer[..., :K, :] = self.readout_token.to(buffer.dtype)
164
+ x = self.cell_embedding(
165
+ x,
166
+ categorical_mask,
167
+ batch_size_limit="auto",
168
+ out=buffer[..., K:, :],
169
+ )
170
+
171
+ if y.numel() > 0:
172
+ if self.y_emb is not None:
173
+ y_emb = self.y_emb(y).unsqueeze(-2) # [..., R_train, 1, D]
174
+ else:
175
+ assert self.y_mlp is not None
176
+ y_emb = self.y_mlp(y.unsqueeze(-1)).unsqueeze(-2)
177
+ x[..., :R_train, :, :] += y_emb.to(x.dtype)
178
+
179
+ for i, (col_layers, col_proj, row_layers, row_norm) in enumerate(
180
+ zip(
181
+ self.col_blocks,
182
+ self.col_projections,
183
+ self.row_blocks,
184
+ self.row_norms,
185
+ )
186
+ ):
187
+ # Column-wise induced set attention (B * C as the batch axis).
188
+ x = x.transpose(-2, -3)
189
+ for j, col_layer in enumerate(col_layers):
190
+ key = f"row_embedding.col_layer{i}.{j}"
191
+ result = col_layer(
192
+ query=x, # [..., C, R, D]
193
+ key_value=cast(KVCacheEntry, cache[key])
194
+ if cache is not None and cache.is_replaying
195
+ else x[..., :R_train, :],
196
+ return_key_value=cache is not None and cache.is_recording,
197
+ batch_size_limit="auto",
198
+ out=None if buffer is None else x,
199
+ ) # [..., C, R, D]
200
+
201
+ if cache is not None and cache.is_recording:
202
+ x, cache[key] = result
203
+ else:
204
+ x = result
205
+ del result
206
+
207
+ if buffer is None:
208
+ x = col_proj(x.transpose(-2, -3)) # [..., R, C, D]
209
+ if i == 0: # Prepend readout tokens before row-wise attention.
210
+ x = torch.cat(
211
+ [
212
+ self.readout_token.to(x.dtype)
213
+ .view(*(1,) * len(B), 1, K, D)
214
+ .expand(*B, R, K, D),
215
+ x, # [..., R, C, D]
216
+ ],
217
+ dim=-2,
218
+ ) # [..., R, K + C, D]
219
+ else:
220
+ x.copy_(col_proj(x)) # NOTE Double peak memory.
221
+ x = buffer # [..., R, K + C, D]
222
+
223
+ # Row-wise attention (B * R as the batch axis).
224
+ for j, row_layer in enumerate(row_layers):
225
+ query = x
226
+ if i == len(self.row_blocks) - 1 and j == len(row_layers) - 1:
227
+ query = x[..., :K, :]
228
+
229
+ x = row_layer(
230
+ query=query, # [..., R, K + C, D] or [..., R, K, D]
231
+ key_value=x, # [..., R, K + C, D]
232
+ batch_size_limit="auto",
233
+ out=None if buffer is None else query,
234
+ ) # [..., R, K + C, D] or [..., R, K, D]
235
+
236
+ if buffer is None:
237
+ x = row_norm(x)
238
+ else:
239
+ x.copy_(row_norm(x)) # NOTE Double peak memory.
240
+
241
+ return x.flatten(-2)
@@ -0,0 +1,33 @@
1
+ This package includes code derived from the TabICLv2 implementation and may
2
+ load pretrained TabICLv2 weights:
3
+ https://github.com/soda-inria/tabicl
4
+ https://huggingface.co/jingang/TabICL
5
+
6
+ BSD 3-Clause License
7
+
8
+ Copyright (c) 2025, Soda team @ Inria
9
+
10
+ Redistribution and use in source and binary forms, with or without
11
+ modification, are permitted provided that the following conditions are met:
12
+
13
+ 1. Redistributions of source code must retain the above copyright notice, this
14
+ list of conditions and the following disclaimer.
15
+
16
+ 2. Redistributions in binary form must reproduce the above copyright notice,
17
+ this list of conditions and the following disclaimer in the documentation
18
+ and/or other materials provided with the distribution.
19
+
20
+ 3. Neither the name of the copyright holder nor the names of its
21
+ contributors may be used to endorse or promote products derived from
22
+ this software without specific prior written permission.
23
+
24
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
27
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
28
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
30
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,11 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """TabICLv2."""
5
+
6
+ from sdm.models.tabiclv2.model import TabICLv2
7
+
8
+
9
+ __all__ = [
10
+ "TabICLv2",
11
+ ]
@@ -0,0 +1,81 @@
1
+ # Copyright (c) 2025, Soda team @ Inria
2
+ # Licensed under the BSD 3-Clause License; see LICENSE.
3
+
4
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5
+ # SPDX-License-Identifier: Apache-2.0
6
+
7
+ # ruff: noqa: D101
8
+
9
+ from typing import Any, cast
10
+
11
+ import torch
12
+ from torch.nn import GELU, LayerNorm, Linear, Sequential
13
+
14
+ from sdm.nn import QASSMax, RotaryEmbedding, TransformerBlock
15
+
16
+
17
+ class TabICLv2TransformerBlock(TransformerBlock):
18
+ def __init__(
19
+ self,
20
+ channels: int,
21
+ num_heads: int,
22
+ norm_bias: bool,
23
+ qassmax: bool,
24
+ rope: RotaryEmbedding | None = None,
25
+ device: torch.device | str | None = None,
26
+ dtype: torch.dtype | None = None,
27
+ ) -> None:
28
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
29
+
30
+ norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs)
31
+ mlp = Sequential(
32
+ LayerNorm(channels, bias=norm_bias, **factory_kwargs),
33
+ Linear(channels, 2 * channels, **factory_kwargs),
34
+ GELU(),
35
+ Linear(2 * channels, channels, **factory_kwargs),
36
+ )
37
+ torch.nn.init.zeros_(cast(Linear, mlp[-1]).weight)
38
+ torch.nn.init.zeros_(cast(Linear, mlp[-1]).bias)
39
+
40
+ super().__init__(
41
+ channels=channels,
42
+ num_query_heads=num_heads,
43
+ mlp=mlp,
44
+ query_norm=norm,
45
+ key_value_norm=norm,
46
+ query_scaling=QASSMax(
47
+ channels=channels // num_heads,
48
+ num_heads=num_heads,
49
+ **factory_kwargs,
50
+ )
51
+ if qassmax
52
+ else None,
53
+ query_transform=rope,
54
+ key_transform=rope,
55
+ **factory_kwargs,
56
+ )
57
+
58
+ def peak_bytes_per_example(
59
+ self,
60
+ element_size: int,
61
+ query_length: int,
62
+ key_value_length: int | None = None,
63
+ ) -> int:
64
+ r""":meta private:""" # noqa: D415
65
+ length = max(query_length, key_value_length or 0)
66
+ factor = 10 if element_size <= 2 else 8
67
+ return factor * length * element_size * self.attn.q_dim
68
+
69
+
70
+ if __name__ == "__main__":
71
+ from sdm.testing.memory import benchmark_transformer_block_memory_peak
72
+
73
+ benchmark_transformer_block_memory_peak(
74
+ block=lambda channels, num_heads: TabICLv2TransformerBlock(
75
+ channels=channels,
76
+ num_heads=num_heads,
77
+ norm_bias=True,
78
+ qassmax=True,
79
+ ),
80
+ channels_and_heads=[(128, 8), (512, 8)],
81
+ )
@@ -0,0 +1,134 @@
1
+ # Copyright (c) 2025, Soda team @ Inria
2
+ # Licensed under the BSD 3-Clause License; see LICENSE.
3
+
4
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5
+ # SPDX-License-Identifier: Apache-2.0
6
+
7
+ from itertools import product
8
+
9
+ from torch import Tensor
10
+
11
+
12
+ def _map_transformer(prefix: str, tail: str) -> list[str]:
13
+ tail = tail.replace("attn.in_proj_weight", "attn.qkv_lin.weight")
14
+ tail = tail.replace("attn.in_proj_bias", "attn.qkv_lin.bias")
15
+ tail = tail.replace("attn.out_proj.", "attn.out_lin.")
16
+ tail = tail.replace(
17
+ "attn.ssmax_layer.base_mlp.",
18
+ "attn.sdpa.query_scaling.scale.",
19
+ )
20
+ tail = tail.replace(
21
+ "attn.ssmax_layer.query_mlp.",
22
+ "attn.sdpa.query_scaling.gate.",
23
+ )
24
+
25
+ if tail.startswith("norm1."):
26
+ tail = tail.removeprefix("norm1.")
27
+ return [
28
+ prefix + "query_norm." + tail,
29
+ prefix + "key_value_norm." + tail,
30
+ ]
31
+ if tail.startswith("norm2."):
32
+ tail = tail.replace("norm2.", "mlp.0.", 1)
33
+ elif tail.startswith("linear1."):
34
+ tail = tail.replace("linear1.", "mlp.1.", 1)
35
+ elif tail.startswith("linear2."):
36
+ tail = tail.replace("linear2.", "mlp.3.", 1)
37
+
38
+ return [prefix + tail]
39
+
40
+
41
+ def remap_ckpt( # noqa: D103
42
+ ckpt: dict[str, Tensor],
43
+ is_classifier: bool,
44
+ ) -> dict[str, Tensor]:
45
+
46
+ out: dict[str, Tensor] = {}
47
+
48
+ if is_classifier:
49
+ out["row_embedding.y_emb.weight"] = (
50
+ ckpt["col_embedder.y_encoder.weight"].t()
51
+ + ckpt["col_embedder.y_encoder.bias"]
52
+ )
53
+ out["icl_block.y_emb.weight"] = (
54
+ ckpt["icl_predictor.y_encoder.weight"].t()
55
+ + ckpt["icl_predictor.y_encoder.bias"]
56
+ )
57
+ else:
58
+ out["row_embedding.y_lin.weight"] = ckpt[
59
+ "col_embedder.y_encoder.weight"
60
+ ]
61
+ out["row_embedding.y_lin.bias"] = ckpt["col_embedder.y_encoder.bias"]
62
+ out["icl_block.y_lin.weight"] = ckpt["icl_predictor.y_encoder.weight"]
63
+ out["icl_block.y_lin.bias"] = ckpt["icl_predictor.y_encoder.bias"]
64
+
65
+ for key, value in ckpt.items():
66
+ if key.startswith("col_embedder.in_linear."):
67
+ new_key = key.replace(
68
+ "col_embedder.in_linear",
69
+ "row_embedding.lin",
70
+ )
71
+ out[new_key] = value
72
+
73
+ elif key.startswith("col_embedder.tf_col.blocks."):
74
+ layer, tail = key.removeprefix(
75
+ "col_embedder.tf_col.blocks."
76
+ ).split(".", 1)
77
+
78
+ if tail == "ind_vectors":
79
+ out[f"row_embedding.col_layers.{layer}.inducing_points"] = (
80
+ value
81
+ )
82
+ elif tail.startswith("multihead_attn1."):
83
+ prefix = f"row_embedding.col_layers.{layer}.inducing_block."
84
+ for new_key in _map_transformer(
85
+ prefix,
86
+ tail.removeprefix("multihead_attn1."),
87
+ ):
88
+ out[new_key] = value
89
+ elif tail.startswith("multihead_attn2."):
90
+ prefix = f"row_embedding.col_layers.{layer}.output_block."
91
+ for new_key in _map_transformer(
92
+ prefix,
93
+ tail.removeprefix("multihead_attn2."),
94
+ ):
95
+ out[new_key] = value
96
+
97
+ elif key == "row_interactor.cls_tokens":
98
+ out["row_embedding.readout_token"] = value
99
+
100
+ elif key.startswith("row_interactor.tf_row.blocks."):
101
+ layer, tail = key.removeprefix(
102
+ "row_interactor.tf_row.blocks."
103
+ ).split(".", 1)
104
+ prefix = f"row_embedding.row_layers.{layer}."
105
+ for new_key in _map_transformer(prefix, tail):
106
+ out[new_key] = value
107
+
108
+ elif key == "row_interactor.tf_row.rope.freqs":
109
+ for layer, side in product(range(3), ("query", "key")):
110
+ out[
111
+ f"row_embedding.row_layers.{layer}.attn."
112
+ f"{side}_transform.inv_freq"
113
+ ] = value
114
+
115
+ elif key.startswith("row_interactor.out_ln."):
116
+ out[key.replace("row_interactor.out_ln", "row_embedding.norm")] = (
117
+ value
118
+ )
119
+
120
+ elif key.startswith("icl_predictor.tf_icl.blocks."):
121
+ layer, tail = key.removeprefix(
122
+ "icl_predictor.tf_icl.blocks."
123
+ ).split(".", 1)
124
+ prefix = f"icl_block.layers.{layer}."
125
+ for new_key in _map_transformer(prefix, tail):
126
+ out[new_key] = value
127
+
128
+ elif key.startswith("icl_predictor.ln."):
129
+ out[key.replace("icl_predictor.ln", "icl_block.norm")] = value
130
+
131
+ elif key.startswith("icl_predictor.decoder."):
132
+ out[key.replace("icl_predictor.decoder", "icl_block.head")] = value
133
+
134
+ return out