structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,299 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D205
5
+ from __future__ import annotations
6
+
7
+ from collections.abc import Iterable
8
+ from typing import Any, ClassVar, Literal, cast
9
+
10
+ import torch
11
+ from torch import Tensor
12
+ from torch.nn import Identity, Linear, ModuleDict
13
+
14
+ from sdm import Recipe, RelatedTables, Stype, TableTensor, Task, TaskLike
15
+ from sdm.cache import Cache
16
+ from sdm.models import ECOC, ICLModel
17
+ from sdm.models._huggingface import download_checkpoint
18
+ from sdm.models.kumo.tabular.icl import ICLBlock
19
+ from sdm.models.kumo.tabular.recipe import default_recipe
20
+ from sdm.models.kumo.tabular.row_embedding import RowEmbedding
21
+
22
+ MODEL_KWARGS: dict[str, dict[str, Any]] = {
23
+ "small": {
24
+ "cell_channels": 128,
25
+ "num_embedding_layers": 4,
26
+ "num_embedding_heads": 4,
27
+ "num_inducing_points": 128,
28
+ "group_size": 3,
29
+ "num_frequencies": 32,
30
+ "num_readout_tokens": 4,
31
+ "icl_channels": 512,
32
+ "num_icl_layers": 12,
33
+ "num_icl_heads": 8,
34
+ "num_icl_key_value_heads_for_query": None,
35
+ },
36
+ "medium": {
37
+ "cell_channels": 256,
38
+ "num_embedding_layers": 6,
39
+ "num_embedding_heads": 4,
40
+ "num_inducing_points": 256,
41
+ "group_size": 3,
42
+ "num_frequencies": 32,
43
+ "num_readout_tokens": 4,
44
+ "icl_channels": 512,
45
+ "num_icl_layers": 24,
46
+ "num_icl_heads": 8,
47
+ "num_icl_key_value_heads_for_query": 2,
48
+ },
49
+ "large": {
50
+ "cell_channels": 256,
51
+ "num_embedding_layers": 6,
52
+ "num_embedding_heads": 4,
53
+ "num_inducing_points": 256,
54
+ "group_size": 3,
55
+ "num_frequencies": 32,
56
+ "num_readout_tokens": 4,
57
+ "icl_channels": 1024,
58
+ "num_icl_layers": 24,
59
+ "num_icl_heads": 16,
60
+ "num_icl_key_value_heads_for_query": 2,
61
+ },
62
+ }
63
+
64
+
65
+ class KumoTabular(ICLModel):
66
+ r"""The tabular foundation model from `"NVIDIA Kumo Tabular Sets a New
67
+ Accuracy-Efficiency Frontier for Tabular Prediction"
68
+ <https://huggingface.co/blog/nvidia/kumo-tabular>`__.
69
+
70
+ .. figure:: /images/kumo_tabular.svg
71
+ :width: 100%
72
+
73
+ :class:`KumoTabular` processes a table in two stages. First, an interleaved
74
+ row/column encoder transforms raw table cells into fixed-size row
75
+ representations. Numerical and categorical cells use separate learned
76
+ Fourier frequencies and projections, together with a learned missingness
77
+ projection. Each encoder stage consists of:
78
+
79
+ * **Column-wise:** Each feature group is processed across rows using
80
+ induced set attention. Both context and query cells attend only to
81
+ context-row keys and values.
82
+ * **Row-wise:** Each row's feature groups and learnable readout tokens
83
+ attend to one another, combining feature interactions into a fixed-size
84
+ row representation.
85
+
86
+ Second, a dataset-wise in-context learning transformer processes the row
87
+ representations and predicts each query target from the labeled context
88
+ rows. The ``"large"`` model widens this transformer to 1024 channels and
89
+ 16 attention heads. Its four 256-channel readout tokens concatenate
90
+ directly to that width without a projection layer.
91
+
92
+ For regression tasks, :class:`KumoTabular` predicts 999 quantiles named
93
+ ``"q001"`` through ``"q999"``.
94
+
95
+ Args:
96
+ task: The tasks to initialize. If ``None``, all tasks supported by this
97
+ model are initialized.
98
+ size: The model size, one of ``"small"``, ``"medium"``, or
99
+ ``"large"``. Defaults to ``"large"``.
100
+ pretrained: Whether to load pretrained checkpoints.
101
+ device: The device for model parameters. If ``None``, uses PyTorch's
102
+ default device.
103
+ """
104
+
105
+ supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset(
106
+ {Stype.numerical}
107
+ )
108
+ supported_target_stypes: ClassVar[frozenset[Stype]] = frozenset(
109
+ {Stype.numerical, Stype.categorical}
110
+ )
111
+ supports_multi_target: ClassVar[bool] = False
112
+ supports_related_tables: ClassVar[bool] = False
113
+
114
+ def __init__(
115
+ self,
116
+ task: TaskLike | Iterable[TaskLike] | None = None,
117
+ size: Literal["small", "medium", "large"] = "large",
118
+ pretrained: bool = True,
119
+ device: torch.device | str | None = None,
120
+ ) -> None:
121
+ super().__init__(task=task)
122
+ if Task.classification in self.tasks:
123
+ self.ecoc = ECOC(max_classes=10)
124
+
125
+ self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict()
126
+ for task in self.tasks:
127
+ self.models[task] = _KumoTabular(
128
+ num_classes=10 if task == Task.classification else 0,
129
+ num_quantiles=999 if task == Task.regression else 0,
130
+ device="meta" if pretrained else device,
131
+ **MODEL_KWARGS[size],
132
+ )
133
+
134
+ if pretrained:
135
+ self.models[task] = self._load_from_pretrained(size, device=device)
136
+
137
+ self.eval()
138
+
139
+ @classmethod
140
+ def default_recipe(cls) -> Recipe:
141
+ r""":meta private:""" # noqa: D415
142
+ return default_recipe()
143
+
144
+ def _load_from_pretrained(
145
+ self,
146
+ size: Literal["small", "medium", "large"],
147
+ device: torch.device | str | None,
148
+ ) -> _KumoTabular:
149
+ device = torch.get_default_device() if device is None else device
150
+
151
+ for task, model in self.models.items():
152
+ if task == Task.classification:
153
+ filename = f"{size}/classifier.pt"
154
+ else:
155
+ assert task == Task.regression
156
+ filename = f"{size}/regressor.pt"
157
+
158
+ path = download_checkpoint(
159
+ repo_id="nvidia/Kumo-Tabular",
160
+ filename=filename,
161
+ revision="v1.0.1",
162
+ config_filename="config.json",
163
+ )
164
+ ckpt = torch.load(path, map_location=device, weights_only=True)
165
+ model.load_state_dict(ckpt, assign=True)
166
+
167
+ return model
168
+
169
+ def _forward(
170
+ self,
171
+ x_context: TableTensor | None, # [..., R_context, D]
172
+ y_context: TableTensor | None, # [..., R_context, 1]
173
+ x_query: TableTensor | None, # [..., R_query, D]
174
+ related_context_tables: RelatedTables[TableTensor] | None,
175
+ related_query_tables: RelatedTables[TableTensor] | None,
176
+ cache: Cache | None,
177
+ generator: torch.Generator | None,
178
+ *,
179
+ categorical_mask: Tensor,
180
+ **kwargs: Any,
181
+ ) -> TableTensor: # [..., R_query, num_classes or 999]
182
+
183
+ if x_query is None and x_context is not None:
184
+ x = x_context.numerical
185
+ elif x_context is None and x_query is not None:
186
+ x = x_query.numerical
187
+ else:
188
+ assert x_context is not None
189
+ assert x_query is not None
190
+ x = torch.cat([x_context.numerical, x_query.numerical], dim=-2)
191
+
192
+ classes: Tensor | None = None
193
+ if y_context is not None and y_context.categorical.size(-1) > 0:
194
+ y = y_context.categorical.code.squeeze(-1)
195
+ classes = y_context.categorical.categories[0]
196
+ elif y_context is not None and y_context.numerical.size(-1) > 0:
197
+ y = y_context.numerical.squeeze(-1)
198
+ else:
199
+ assert cache is not None
200
+ classes = cast(Tensor | None, cache["classes"])
201
+ y = x.new_empty(
202
+ (*x.size()[:-2], 0),
203
+ dtype=torch.int64 if classes is not None else x.dtype,
204
+ )
205
+
206
+ categorical_mask = categorical_mask.expand(*x.size()[:-2], -1)
207
+
208
+ if classes is None:
209
+ out = self.models[Task.regression](
210
+ x=x,
211
+ y=y,
212
+ categorical_mask=categorical_mask,
213
+ cache=cache,
214
+ )
215
+ return TableTensor(
216
+ columns={
217
+ Stype.numerical: [f"q{i:03d}" for i in range(1, 1000)]
218
+ },
219
+ numerical=out,
220
+ )
221
+
222
+ out = self.ecoc(
223
+ model=self.models[Task.classification],
224
+ x=x,
225
+ y=y,
226
+ num_classes=len(classes),
227
+ num_members=x.size(0) if x.dim() > 2 else 1,
228
+ cache=cache,
229
+ generator=generator,
230
+ categorical_mask=categorical_mask,
231
+ )
232
+ return TableTensor(
233
+ columns={Stype.numerical: [str(i) for i in classes.tolist()]},
234
+ numerical=out,
235
+ )
236
+
237
+
238
+ class _KumoTabular(torch.nn.Module):
239
+ def __init__(
240
+ self,
241
+ num_classes: int,
242
+ num_quantiles: int,
243
+ cell_channels: int = 128,
244
+ num_embedding_layers: int = 4,
245
+ num_embedding_heads: int = 4,
246
+ num_inducing_points: int = 128,
247
+ group_size: int = 3,
248
+ num_frequencies: int = 32,
249
+ num_readout_tokens: int = 4,
250
+ icl_channels: int = 512,
251
+ num_icl_layers: int = 12,
252
+ num_icl_heads: int = 8,
253
+ num_icl_key_value_heads_for_query: int | None = None,
254
+ device: torch.device | str | None = None,
255
+ dtype: torch.dtype | None = None,
256
+ ) -> None:
257
+ super().__init__()
258
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
259
+
260
+ self.row_embedding = RowEmbedding(
261
+ num_classes=num_classes,
262
+ channels=cell_channels,
263
+ num_layers=num_embedding_layers,
264
+ num_heads=num_embedding_heads,
265
+ group_size=group_size,
266
+ num_frequencies=num_frequencies,
267
+ num_inducing_points=num_inducing_points,
268
+ num_readout_tokens=num_readout_tokens,
269
+ **factory_kwargs,
270
+ )
271
+ if cell_channels * num_readout_tokens != icl_channels:
272
+ self.row_project = Linear(
273
+ cell_channels * num_readout_tokens,
274
+ icl_channels,
275
+ **factory_kwargs,
276
+ )
277
+ else:
278
+ self.row_project = Identity()
279
+ self.icl_block = ICLBlock(
280
+ num_classes=num_classes,
281
+ out_channels=num_classes or num_quantiles,
282
+ channels=icl_channels,
283
+ num_layers=num_icl_layers,
284
+ num_heads=num_icl_heads,
285
+ num_key_value_heads_for_query=num_icl_key_value_heads_for_query,
286
+ **factory_kwargs,
287
+ )
288
+
289
+ def forward(
290
+ self,
291
+ x: Tensor, # [..., R, C]
292
+ y: Tensor, # [..., R_train]
293
+ categorical_mask: Tensor, # [..., C]
294
+ *,
295
+ cache: Cache | None = None,
296
+ ) -> Tensor: # [..., R_test, num_classes or num_quantiles]
297
+ x = self.row_embedding(x, y, categorical_mask, cache=cache)
298
+ x = self.row_project(x)
299
+ return self.icl_block(x=x, y=y, cache=cache)
@@ -0,0 +1,71 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import torch
5
+
6
+ import sdm.processing as sp
7
+
8
+
9
+ def default_recipe() -> sp.Recipe: # noqa: D103
10
+ def numerical_processor() -> sp.Sequential:
11
+ return sp.Sequential(
12
+ # In single precision, a few huge outliers collapse the
13
+ # standardized values of all other rows onto one value.
14
+ sp.Cast(torch.float64),
15
+ sp.DropConstantColumns(),
16
+ sp.Standardize(eps=1e-6),
17
+ sp.Clip(-100.0, 100.0),
18
+ sp.Choice(
19
+ sp.Identity(),
20
+ sp.PowerTransform(),
21
+ [
22
+ sp.RobustScale(),
23
+ sp.ClipSoft(3.0),
24
+ ],
25
+ sp.RankGaussian(max_knots=8192),
26
+ method="round_robin",
27
+ ),
28
+ sp.ClipSigma(threshold=4.0),
29
+ )
30
+
31
+ return sp.Recipe(
32
+ features=[
33
+ sp.StypeDispatch(
34
+ numerical=[
35
+ numerical_processor(),
36
+ sp.FlipSign(),
37
+ ],
38
+ categorical=[
39
+ sp.AlignCategories(sort_by="value"),
40
+ sp.AddCategoryCounts(min_cardinality=50),
41
+ sp.ToNumerical(),
42
+ numerical_processor(),
43
+ ],
44
+ ),
45
+ sp.ShuffleColumns(method="latin"),
46
+ sp.SelectColumns(500, method="first"),
47
+ sp.Cast(torch.float32),
48
+ ],
49
+ target=[
50
+ sp.StypeDispatch(
51
+ categorical=[
52
+ sp.AlignCategories(sort_by="value"),
53
+ sp.ShuffleCategories(method="shift"),
54
+ ],
55
+ numerical=[
56
+ sp.Standardize(),
57
+ sp.FlipSign(),
58
+ ],
59
+ ),
60
+ ],
61
+ output=sp.TaskDispatch(
62
+ classification=[
63
+ sp.AverageEstimators(),
64
+ sp.Softmax(),
65
+ ],
66
+ regression=[
67
+ sp.SortQuantiles(),
68
+ sp.AverageEstimators(trim_fraction=0.2),
69
+ ],
70
+ ),
71
+ )
@@ -0,0 +1,214 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D101, D102
5
+
6
+ from typing import Any, cast
7
+
8
+ import torch
9
+ from torch import Tensor
10
+ from torch.nn import Embedding, Linear, ModuleList, Parameter
11
+
12
+ from sdm.cache import Cache, KVCacheEntry
13
+ from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
14
+ from sdm.models.kumo.tabular.cell_embedding import CellEmbedding
15
+ from sdm.nn import (
16
+ GatedLogScale,
17
+ InducedTransformerBlock,
18
+ LogScale,
19
+ RMSNorm,
20
+ RotaryEmbedding,
21
+ )
22
+
23
+
24
+ class RowEmbedding(torch.nn.Module):
25
+ def __init__(
26
+ self,
27
+ num_classes: int,
28
+ channels: int,
29
+ num_layers: int,
30
+ num_heads: int,
31
+ group_size: int,
32
+ num_frequencies: int,
33
+ num_inducing_points: int,
34
+ num_readout_tokens: int,
35
+ device: torch.device | str | None = None,
36
+ dtype: torch.dtype | None = None,
37
+ ) -> None:
38
+ super().__init__()
39
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
40
+ self.channels = channels
41
+
42
+ self.cell_embedding = CellEmbedding(
43
+ channels=channels,
44
+ group_size=group_size,
45
+ num_frequencies=num_frequencies,
46
+ **factory_kwargs,
47
+ )
48
+
49
+ self.y_emb: torch.nn.Module | None = None
50
+ self.y_lin: torch.nn.Module | None = None
51
+ if num_classes > 0:
52
+ self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
53
+ else:
54
+ self.y_lin = Linear(1, channels, bias=False, **factory_kwargs)
55
+
56
+ self.readout_token = Parameter(
57
+ torch.empty(num_readout_tokens, channels, **factory_kwargs)
58
+ )
59
+ torch.nn.init.trunc_normal_(self.readout_token, std=0.02)
60
+
61
+ rope = RotaryEmbedding(
62
+ channels=channels // num_heads,
63
+ layout="split_half",
64
+ theta=100_000,
65
+ requires_grad=False,
66
+ **factory_kwargs,
67
+ )
68
+
69
+ self.col_blocks = ModuleList(
70
+ InducedTransformerBlock(
71
+ channels=channels,
72
+ num_inducing_points=num_inducing_points,
73
+ inducing_block=KumoTabularTransformerBlock(
74
+ channels=channels,
75
+ num_heads=num_heads,
76
+ query_scaling=LogScale(
77
+ num_heads=num_heads,
78
+ **factory_kwargs,
79
+ ),
80
+ **factory_kwargs,
81
+ ),
82
+ output_block=KumoTabularTransformerBlock(
83
+ channels=channels,
84
+ num_heads=num_heads,
85
+ query_scaling=None,
86
+ **factory_kwargs,
87
+ ),
88
+ **factory_kwargs,
89
+ )
90
+ for _ in range(num_layers)
91
+ )
92
+ self.row_blocks = ModuleList(
93
+ KumoTabularTransformerBlock(
94
+ channels=channels,
95
+ num_heads=num_heads,
96
+ query_scaling=GatedLogScale(
97
+ channels=channels // num_heads,
98
+ num_heads=num_heads,
99
+ hidden_channels=64,
100
+ **factory_kwargs,
101
+ ),
102
+ rope=rope,
103
+ **factory_kwargs,
104
+ )
105
+ for _ in range(num_layers)
106
+ )
107
+ self.norm = RMSNorm(channels, **factory_kwargs)
108
+
109
+ def forward(
110
+ self,
111
+ x: Tensor, # [..., R, C]
112
+ y: Tensor, # [..., R_train]
113
+ categorical_mask: Tensor, # [..., C]
114
+ *,
115
+ cache: Cache | None = None,
116
+ ) -> Tensor: # [..., R, K * D]
117
+
118
+ *B, R, C = x.size()
119
+ R_train = y.size(-1)
120
+ K = self.readout_token.size(-2)
121
+ D = self.channels
122
+
123
+ buffer: Tensor | None = None
124
+ if torch.is_grad_enabled():
125
+ x = self.cell_embedding(
126
+ x=x,
127
+ categorical_mask=categorical_mask,
128
+ train_size=R_train,
129
+ cache=cache,
130
+ ) # [..., R, C, D]
131
+ else:
132
+ buffer = torch.empty(
133
+ (*B, R, K + C, D),
134
+ device=x.device,
135
+ dtype=torch.get_autocast_dtype(x.device.type)
136
+ if torch.is_autocast_enabled(x.device.type)
137
+ else x.dtype,
138
+ )
139
+ buffer[..., :K, :] = self.readout_token.to(buffer.dtype)
140
+ x = self.cell_embedding(
141
+ x=x,
142
+ categorical_mask=categorical_mask,
143
+ train_size=R_train,
144
+ cache=cache,
145
+ batch_size_limit="auto",
146
+ out=buffer[..., K:, :],
147
+ )
148
+
149
+ if y.numel() > 0:
150
+ if self.y_emb is not None:
151
+ y_emb = self.y_emb(y).unsqueeze(-2) # [..., R_train, 1, D]
152
+ else:
153
+ assert self.y_lin is not None
154
+ y_emb = self.y_lin(y.unsqueeze(-1)).unsqueeze(-2)
155
+ x[..., :R_train, :, :] += y_emb.to(x.dtype)
156
+
157
+ if buffer is not None:
158
+ x = x.transpose(-2, -3) # [..., C, R, D]
159
+
160
+ for i, (col_block, row_block) in enumerate(
161
+ zip(self.col_blocks, self.row_blocks)
162
+ ):
163
+ if buffer is None:
164
+ if i > 0:
165
+ readout_token, x = x.split([K, x.size(-2) - K], dim=-2)
166
+ readout_token = readout_token.clone()
167
+ else:
168
+ readout_token = self.readout_token
169
+ readout_token = readout_token.view(*(1,) * len(B), 1, K, D)
170
+ readout_token = readout_token.expand(*B, R, K, D)
171
+ x = x.transpose(-2, -3) # [..., C, R, D]
172
+
173
+ key = f"row_embedding.col_block{i}"
174
+ result = col_block(
175
+ query=x,
176
+ key_value=cast(KVCacheEntry, cache[key])
177
+ if cache is not None and cache.is_replaying
178
+ else x[..., :R_train, :],
179
+ return_key_value=cache is not None and cache.is_recording,
180
+ batch_size_limit="auto",
181
+ out=None if buffer is None else x,
182
+ )
183
+
184
+ if cache is not None and cache.is_recording:
185
+ x, cache[key] = result
186
+ else:
187
+ x = result
188
+ del result
189
+
190
+ if buffer is None:
191
+ x = torch.cat(
192
+ [readout_token.to(x.dtype), x.transpose(-2, -3)],
193
+ dim=-2,
194
+ )
195
+ x = row_block(
196
+ query=x[..., :K, :]
197
+ if i == len(self.row_blocks) - 1
198
+ else x,
199
+ key_value=x,
200
+ batch_size_limit="auto",
201
+ )
202
+ else:
203
+ buffer = row_block(
204
+ query=buffer[..., :K, :]
205
+ if i == len(self.row_blocks) - 1
206
+ else buffer,
207
+ key_value=buffer,
208
+ batch_size_limit="auto",
209
+ out=buffer[..., :K, :]
210
+ if i == len(self.row_blocks) - 1
211
+ else buffer,
212
+ )
213
+
214
+ return self.norm(buffer if buffer is not None else x).flatten(-2)