structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,113 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from dataclasses import dataclass
5
+ from typing import Self
6
+
7
+ import torch
8
+ from torch import Tensor
9
+
10
+ from sdm import RelatedTables, TableTensor
11
+ from sdm.models.kumo.relational.graph import HomogeneousGraph
12
+ from sdm.relational.join import join_index
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class TaskGraph: # noqa: D101
17
+ x: TableTensor
18
+ related_tables: RelatedTables[TableTensor]
19
+ graph: HomogeneousGraph
20
+ readout_table: str
21
+ readout_index: Tensor # Entity-table rows ordered by task row.
22
+ task_row_by_table: dict[str, Tensor] # Per table task-row assignment.
23
+ num_hops: int
24
+
25
+ @classmethod
26
+ def from_input( # noqa: D102
27
+ cls,
28
+ x: TableTensor,
29
+ related_tables: RelatedTables[TableTensor],
30
+ num_hops: int | None = None,
31
+ ) -> Self:
32
+
33
+ if x.dim() != 2:
34
+ raise ValueError("Tables need to be two-dimensional")
35
+
36
+ for table in related_tables.tables.values():
37
+ if table.dim() != 2:
38
+ raise ValueError("Tables need to be two-dimensional")
39
+
40
+ if len(related_tables.task_links) != 1:
41
+ raise ValueError(
42
+ f"'KumoRelational' expects exactly one task link to an "
43
+ f"entity table "
44
+ f"(got {len(related_tables.task_links)})"
45
+ )
46
+
47
+ graph = HomogeneousGraph.from_tables(
48
+ tables=related_tables.tables,
49
+ relationships=related_tables.relationships,
50
+ )
51
+ readout_table = related_tables.task_links[0].table
52
+ readout_offset = graph.start_node_offsets[readout_table]
53
+
54
+ # Map each task row to exactly one entity-table row:
55
+ task_index, readout_index = join_index(
56
+ left_table=x,
57
+ right_table=related_tables.tables[readout_table],
58
+ left_keys=related_tables.task_links[0].task_columns,
59
+ right_keys=related_tables.task_links[0].table_columns,
60
+ how="inner",
61
+ )
62
+ task_index, perm = task_index.sort()
63
+ readout_index = readout_index[perm]
64
+ global_readout_index = readout_index + readout_offset
65
+
66
+ arange = torch.arange(
67
+ x.size(-2),
68
+ dtype=task_index.dtype,
69
+ device=task_index.device,
70
+ )
71
+ if (
72
+ not task_index.equal(arange)
73
+ or readout_index.unique().numel() != readout_index.numel()
74
+ ):
75
+ raise ValueError(
76
+ f"Expected each task row to match exactly one distinct row in "
77
+ f"{readout_table!r}"
78
+ )
79
+
80
+ task_row = readout_index.new_full((graph.num_nodes,), fill_value=-1)
81
+ task_row[global_readout_index] = task_index
82
+ frontier = torch.zeros_like(task_row, dtype=torch.bool)
83
+ frontier[global_readout_index] = True
84
+
85
+ # Propagate task assignment along graph edges.
86
+ # NOTE This assumes neighborhoods do not overlap.
87
+ propagated_hops = 0
88
+ while True:
89
+ if num_hops is not None and propagated_hops >= num_hops:
90
+ break
91
+ mask = frontier[graph.row] & (task_row[graph.col] < 0)
92
+ row = graph.row[mask]
93
+ if row.numel() == 0:
94
+ break
95
+ col = graph.col[mask]
96
+
97
+ task_row[col] = task_row[row]
98
+ frontier.fill_(False)
99
+ frontier[col] = True
100
+ propagated_hops += 1
101
+
102
+ return cls(
103
+ x=x,
104
+ related_tables=related_tables,
105
+ graph=graph,
106
+ readout_table=readout_table,
107
+ readout_index=readout_index,
108
+ task_row_by_table={
109
+ table_name: task_row[graph.node_slice(table_name)]
110
+ for table_name in related_tables.tables
111
+ },
112
+ num_hops=propagated_hops if num_hops is None else num_hops,
113
+ )
@@ -0,0 +1,17 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """KumoTabular."""
5
+
6
+ from sdm.models.kumo.tabular.model import KumoTabular
7
+ from sdm.models.kumo.tabular.batching import (
8
+ estimate_fit_batch_size,
9
+ estimate_predict_batch_size,
10
+ )
11
+
12
+
13
+ __all__ = [
14
+ "KumoTabular",
15
+ "estimate_fit_batch_size",
16
+ "estimate_predict_batch_size",
17
+ ]
@@ -0,0 +1,139 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import math
5
+ from typing import cast
6
+
7
+ import torch
8
+
9
+ from sdm import TableTensor, Task
10
+ from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
11
+ from sdm.models.kumo.tabular.model import KumoTabular, _KumoTabular
12
+ from sdm.nn import InducedTransformerBlock
13
+
14
+
15
+ def estimate_fit_batch_size(
16
+ model: KumoTabular,
17
+ x: TableTensor,
18
+ y: TableTensor,
19
+ *,
20
+ memory_budget: int,
21
+ num_estimators: int = 1,
22
+ dtype: torch.dtype = torch.float16,
23
+ ) -> int:
24
+ """Estimate how many ensemble members to fit together.
25
+
26
+ For default-recipe inference. Uses half the budget; the estimate does
27
+ not guarantee that it fits in memory.
28
+
29
+ Args:
30
+ model: Kumo Tabular model with the target task initialized.
31
+ x: Unprocessed context features of shape ``[R, C]``.
32
+ y: Context targets of shape ``[R, 1]``. Categorical metadata defines
33
+ all classes; numerical targets select regression.
34
+ memory_budget: Available bytes excluding weights and inputs.
35
+ num_estimators: Total number of ensemble members.
36
+ dtype: Neural execution or autocast dtype.
37
+
38
+ Returns:
39
+ At least one member, capped at ``num_estimators``. Pass this as
40
+ ``estimator_batch_size`` to :meth:`~sdm.models.KumoTabular.fit`.
41
+ """
42
+ num_rows, num_columns = x.shape[-2:]
43
+ num_classes = (
44
+ len(y.categorical.categories[0]) if y.categorical.size(-1) else 0
45
+ )
46
+ workspace, cache, member = _row_bytes(
47
+ model=model,
48
+ num_columns=num_columns,
49
+ num_classes=num_classes,
50
+ dtype=dtype,
51
+ )
52
+ capacity = (memory_budget // 2) // max(
53
+ num_rows * (workspace + cache) + member, 1
54
+ )
55
+ return max(1, min(num_estimators, capacity))
56
+
57
+
58
+ def estimate_predict_batch_size(
59
+ model: KumoTabular,
60
+ x: TableTensor,
61
+ y: TableTensor,
62
+ *,
63
+ memory_budget: int,
64
+ num_estimators: int = 1,
65
+ estimator_batch_size: int = 1,
66
+ dtype: torch.dtype = torch.float16,
67
+ ) -> int:
68
+ """Estimate how many query rows to predict together.
69
+
70
+ For default-recipe inference. Uses half the budget; the estimate does
71
+ not guarantee that it fits in memory.
72
+
73
+ Args:
74
+ model: Kumo Tabular model with the target task initialized.
75
+ x: Unprocessed context features of shape ``[R, C]``.
76
+ y: Context targets of shape ``[R, 1]``. Categorical metadata defines
77
+ all classes; numerical targets select regression.
78
+ memory_budget: Available bytes excluding weights, inputs, and context
79
+ caches, including overlapping transfers.
80
+ num_estimators: Total number of ensemble members used for fitting.
81
+ estimator_batch_size: Ensemble members run together, as set during fit.
82
+ dtype: Neural execution or autocast dtype.
83
+
84
+ Returns:
85
+ At least one query row per prediction call.
86
+ """
87
+ num_columns = x.size(-1)
88
+ num_classes = (
89
+ len(y.categorical.categories[0]) if y.categorical.size(-1) else 0
90
+ )
91
+ workspace, _, _ = _row_bytes(
92
+ model=model,
93
+ num_columns=num_columns,
94
+ num_classes=num_classes,
95
+ dtype=dtype,
96
+ )
97
+ # Preprocessing and output reduction retain all ensemble members.
98
+ row_bytes = workspace * estimator_batch_size
99
+ row_bytes += (
100
+ num_estimators * 8 * (8 * num_columns + 4 * (num_classes or 999))
101
+ )
102
+ return max(1, (memory_budget // 2) // max(row_bytes, 1))
103
+
104
+
105
+ def _row_bytes(
106
+ model: KumoTabular,
107
+ num_columns: int,
108
+ num_classes: int,
109
+ dtype: torch.dtype,
110
+ ) -> tuple[int, int, int]:
111
+ task = Task.classification if num_classes else Task.regression
112
+ network = cast(_KumoTabular, model.models[task])
113
+ row = network.row_embedding
114
+ icl = network.icl_block
115
+ layer = cast(KumoTabularTransformerBlock, icl.layers[0])
116
+ tasks = 1
117
+ if num_classes and num_classes > model.ecoc.max_classes:
118
+ tasks = max(
119
+ math.ceil(num_classes / (model.ecoc.max_classes - 1)),
120
+ 4 * math.ceil(math.log(num_classes, model.ecoc.max_classes)),
121
+ )
122
+ # The default recipe adds count columns and keeps at most 500 features.
123
+ columns = min(2 * num_columns, 500)
124
+ element_size = dtype.itemsize
125
+ workspace = tasks * (
126
+ element_size * 4 * (columns + row.readout_token.size(0)) * row.channels
127
+ + layer.peak_bytes_per_example(
128
+ element_size=element_size, query_length=1
129
+ )
130
+ )
131
+ # Fit projects all heads before retaining the smaller query KV heads.
132
+ cache = tasks * element_size * 2 * layer.attn.q_dim * len(icl.layers)
133
+ # Fit records column-block caches per estimator, not per row.
134
+ inducing = sum(
135
+ cast(InducedTransformerBlock, block).inducing_points.size(0)
136
+ for block in row.col_blocks
137
+ )
138
+ member = tasks * element_size * 2 * columns * inducing * row.channels
139
+ return workspace, cache, member
@@ -0,0 +1,100 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D101
5
+
6
+ from typing import Any, cast
7
+
8
+ import torch
9
+ from torch import Tensor
10
+ from torch.nn import GELU, Linear, Sequential
11
+
12
+ from sdm.nn import QueryScaling, RMSNorm, RotaryEmbedding, TransformerBlock
13
+
14
+
15
+ class _RoPERMSNorm(Sequential):
16
+ def forward(self, input: Tensor) -> Tensor: # noqa: A002
17
+ rope, norm = self
18
+ return norm(input, rope=rope)
19
+
20
+
21
+ class KumoTabularTransformerBlock(TransformerBlock):
22
+ def __init__(
23
+ self,
24
+ channels: int,
25
+ num_heads: int,
26
+ query_scaling: QueryScaling | None,
27
+ rope: RotaryEmbedding | None = None,
28
+ device: torch.device | str | None = None,
29
+ dtype: torch.dtype | None = None,
30
+ ) -> None:
31
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
32
+
33
+ query_transforms: list[torch.nn.Module] = []
34
+ key_transforms: list[torch.nn.Module] = []
35
+ if rope is not None:
36
+ query_transforms.append(rope)
37
+ key_transforms.append(rope)
38
+ query_transforms.append(
39
+ RMSNorm(
40
+ channels // num_heads,
41
+ eps=1e-6,
42
+ elementwise_affine=False,
43
+ **factory_kwargs,
44
+ )
45
+ )
46
+ key_transforms.append(
47
+ RMSNorm(
48
+ channels // num_heads,
49
+ eps=1e-6,
50
+ elementwise_affine=False,
51
+ **factory_kwargs,
52
+ )
53
+ )
54
+
55
+ mlp = Sequential(
56
+ RMSNorm(channels, **factory_kwargs),
57
+ Linear(channels, 2 * channels, **factory_kwargs),
58
+ GELU(),
59
+ Linear(2 * channels, channels, **factory_kwargs),
60
+ )
61
+ torch.nn.init.zeros_(cast(Linear, mlp[-1]).weight)
62
+ torch.nn.init.zeros_(cast(Linear, mlp[-1]).bias)
63
+
64
+ transform = Sequential if rope is None else _RoPERMSNorm
65
+ super().__init__(
66
+ channels=channels,
67
+ num_query_heads=num_heads,
68
+ mlp=mlp,
69
+ query_norm=RMSNorm(channels, **factory_kwargs),
70
+ key_value_norm=RMSNorm(channels, **factory_kwargs),
71
+ query_transform=transform(*query_transforms),
72
+ key_transform=transform(*key_transforms),
73
+ query_scaling=query_scaling,
74
+ **factory_kwargs,
75
+ )
76
+
77
+ def peak_bytes_per_example(
78
+ self,
79
+ element_size: int,
80
+ query_length: int,
81
+ key_value_length: int | None = None,
82
+ ) -> int:
83
+ r""":meta private:""" # noqa: D415
84
+ length = max(query_length, key_value_length or 0)
85
+ factor = 15 if element_size <= 2 else 8
86
+ return factor * length * element_size * self.attn.q_dim
87
+
88
+
89
+ if __name__ == "__main__":
90
+ from sdm.nn import LogScale
91
+ from sdm.testing.memory import benchmark_transformer_block_memory_peak
92
+
93
+ benchmark_transformer_block_memory_peak(
94
+ block=lambda channels, num_heads: KumoTabularTransformerBlock(
95
+ channels=channels,
96
+ num_heads=num_heads,
97
+ query_scaling=LogScale(num_heads=num_heads),
98
+ ),
99
+ channels_and_heads=[(256, 4), (512, 4)],
100
+ )
@@ -0,0 +1,85 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D101, D102
5
+
6
+ from typing import Literal, cast
7
+
8
+ import torch
9
+ from torch import Tensor
10
+ from torch.nn import Linear
11
+
12
+ from sdm.cache import Cache
13
+ from sdm.models.tabfm.cell_embedding import CellEmbedding as TabFMCellEmbedding
14
+
15
+
16
+ class CellEmbedding(TabFMCellEmbedding):
17
+ def __init__(
18
+ self,
19
+ channels: int,
20
+ group_size: int,
21
+ num_frequencies: int,
22
+ device: torch.device | str | None = None,
23
+ dtype: torch.dtype | None = None,
24
+ ) -> None:
25
+ super().__init__(
26
+ channels=channels,
27
+ group_size=group_size,
28
+ num_frequencies=num_frequencies,
29
+ device=device,
30
+ dtype=dtype,
31
+ )
32
+ self.nan_lin = Linear(
33
+ group_size,
34
+ channels,
35
+ bias=False,
36
+ device=device,
37
+ dtype=dtype,
38
+ )
39
+
40
+ def forward( # type: ignore
41
+ self,
42
+ x: Tensor, # [..., R, C]
43
+ categorical_mask: Tensor, # [..., C]
44
+ *,
45
+ train_size: int | None = None,
46
+ cache: Cache | None = None,
47
+ batch_size_limit: int | Literal["auto"] | None = None,
48
+ out: Tensor | None = None,
49
+ ) -> Tensor: # [..., R, C, D]
50
+ missing = x.isnan()
51
+
52
+ if cache is not None and cache.is_replaying:
53
+ impute_mean = cast(Tensor, cache["cell_impute_mean"])
54
+ else:
55
+ assert train_size is not None
56
+ impute_mean = x[..., :train_size, :].nanmean(dim=-2, keepdim=True)
57
+ impute_mean = impute_mean.masked_fill(impute_mean.isnan(), 0.0)
58
+
59
+ if cache is not None and cache.is_recording:
60
+ cache["cell_impute_mean"] = impute_mean
61
+
62
+ x = torch.where(missing, impute_mean, x)
63
+
64
+ return super().forward(
65
+ x,
66
+ categorical_mask,
67
+ batch_size_limit=batch_size_limit,
68
+ out=out,
69
+ missing=missing,
70
+ )
71
+
72
+ def _forward( # type: ignore
73
+ self,
74
+ x: Tensor, # [..., R, C, G]
75
+ freq: Tensor, # [..., 1, C, G, F]
76
+ weight: Tensor, # [..., 1, C, G, D, 2F]
77
+ bias: Tensor, # [..., 1, C, D]
78
+ *,
79
+ missing: Tensor, # [..., R, C, G]
80
+ out: Tensor | None = None,
81
+ ) -> Tensor:
82
+
83
+ out = super()._forward(x, freq, weight, bias, out=out)
84
+ out += self.nan_lin(missing.to(out.dtype))
85
+ return out
@@ -0,0 +1,136 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D101, D102
5
+
6
+ from typing import Any, cast
7
+
8
+ import torch
9
+ from torch import Tensor
10
+ from torch.nn import GELU, Embedding, Linear, ModuleList, Sequential
11
+
12
+ from sdm.cache import Cache, KVCacheEntry
13
+ from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
14
+ from sdm.nn import LogScale, RMSNorm
15
+
16
+
17
+ class ICLBlock(torch.nn.Module):
18
+ def __init__(
19
+ self,
20
+ num_classes: int,
21
+ out_channels: int,
22
+ channels: int,
23
+ num_layers: int,
24
+ num_heads: int,
25
+ num_key_value_heads_for_query: int | None = None,
26
+ device: torch.device | str | None = None,
27
+ dtype: torch.dtype | None = None,
28
+ ) -> None:
29
+ super().__init__()
30
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
31
+ self.kv_heads = num_key_value_heads_for_query
32
+
33
+ self.y_emb: torch.nn.Module | None = None
34
+ self.y_lin: torch.nn.Module | None = None
35
+ if num_classes > 0:
36
+ self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
37
+ else:
38
+ self.y_lin = Linear(1, channels, bias=False, **factory_kwargs)
39
+
40
+ self.layers = ModuleList(
41
+ KumoTabularTransformerBlock(
42
+ channels=channels,
43
+ num_heads=num_heads,
44
+ query_scaling=LogScale(
45
+ num_heads=num_heads,
46
+ **factory_kwargs,
47
+ ),
48
+ **factory_kwargs,
49
+ )
50
+ for _ in range(num_layers)
51
+ )
52
+
53
+ self.norm = RMSNorm(channels, **factory_kwargs)
54
+ self.head = Sequential(
55
+ Linear(channels, 2 * channels, **factory_kwargs),
56
+ GELU(),
57
+ Linear(2 * channels, out_channels, **factory_kwargs),
58
+ )
59
+
60
+ def forward(
61
+ self,
62
+ x: Tensor, # [..., R, D]
63
+ y: Tensor, # [..., R_train]
64
+ *,
65
+ cache: Cache | None = None,
66
+ ) -> Tensor: # [..., R_test, out_channels]
67
+ R_train = y.size(-1)
68
+
69
+ if y.numel() > 0:
70
+ if self.y_emb is not None:
71
+ y_emb = self.y_emb(y) # [..., R_train, D]
72
+ else:
73
+ assert self.y_lin is not None
74
+ y_emb = self.y_lin(y.unsqueeze(-1)) # [..., R_train, D]
75
+
76
+ x[..., :R_train, :] += y_emb.to(x.dtype)
77
+ del y_emb
78
+
79
+ for i, layer in enumerate(self.layers):
80
+ cache_key = f"icl_block.layer{i}"
81
+ last_layer = i == len(self.layers) - 1
82
+
83
+ if self.kv_heads is None or cache is not None:
84
+ result = layer(
85
+ query=x[..., R_train:, :] if last_layer else x,
86
+ key_value=(
87
+ cast(KVCacheEntry, cache[cache_key])
88
+ if cache is not None and cache.is_replaying
89
+ else x[..., :R_train, :]
90
+ ),
91
+ return_key_value=cache is not None and cache.is_recording,
92
+ out=None
93
+ if torch.is_grad_enabled()
94
+ else x[..., R_train:, :]
95
+ if last_layer
96
+ else x,
97
+ )
98
+
99
+ if cache is not None and cache.is_recording:
100
+ x, (key, value) = result
101
+ if self.kv_heads is not None:
102
+ key = key[..., : self.kv_heads, :].contiguous()
103
+ value = value[..., : self.kv_heads, :].contiguous()
104
+ cache[cache_key] = KVCacheEntry(key, value)
105
+ del key, value
106
+ else:
107
+ x = result
108
+ del result
109
+ continue
110
+
111
+ x_context, (key, value) = layer(
112
+ query=x[..., :0, :] if last_layer else x[..., :R_train, :],
113
+ key_value=x[..., :R_train, :],
114
+ return_key_value=True,
115
+ out=None
116
+ if torch.is_grad_enabled()
117
+ else x[..., :0, :]
118
+ if last_layer
119
+ else x[..., :R_train, :],
120
+ )
121
+ key_value = KVCacheEntry(
122
+ key=key[..., : self.kv_heads, :].contiguous(),
123
+ value=value[..., : self.kv_heads, :].contiguous(),
124
+ )
125
+ del key, value
126
+ x_query = layer(
127
+ query=x[..., R_train:, :],
128
+ key_value=key_value,
129
+ out=None if torch.is_grad_enabled() else x[..., R_train:, :],
130
+ )
131
+ if last_layer:
132
+ x = x_query
133
+ elif torch.is_grad_enabled():
134
+ x = torch.cat([x_context, x_query], -2)
135
+
136
+ return self.head(self.norm(x))