structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import sdm.processing as sp
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def default_recipe() -> sp.Recipe: # noqa: D103
|
|
8
|
+
return sp.Recipe(
|
|
9
|
+
features=[
|
|
10
|
+
sp.StypeDispatch(
|
|
11
|
+
categorical=[
|
|
12
|
+
sp.AlignCategories(sort_by="value"),
|
|
13
|
+
sp.ToNumerical(),
|
|
14
|
+
],
|
|
15
|
+
),
|
|
16
|
+
sp.StypeDispatch(
|
|
17
|
+
numerical=[
|
|
18
|
+
sp.ImputeMean(),
|
|
19
|
+
sp.DropConstantColumns(),
|
|
20
|
+
sp.Standardize(eps=1e-6),
|
|
21
|
+
sp.Clip(-100.0, 100.0),
|
|
22
|
+
sp.Choice(
|
|
23
|
+
sp.Identity(),
|
|
24
|
+
sp.PowerTransform(),
|
|
25
|
+
method="round_robin",
|
|
26
|
+
),
|
|
27
|
+
sp.ClipSigma(threshold=4.0),
|
|
28
|
+
sp.ShuffleColumns(method="latin"),
|
|
29
|
+
],
|
|
30
|
+
),
|
|
31
|
+
],
|
|
32
|
+
target=[
|
|
33
|
+
sp.StypeDispatch(
|
|
34
|
+
categorical=[
|
|
35
|
+
sp.AlignCategories(sort_by="value"),
|
|
36
|
+
sp.ShuffleCategories(method="shift"),
|
|
37
|
+
],
|
|
38
|
+
numerical=sp.Standardize(),
|
|
39
|
+
),
|
|
40
|
+
],
|
|
41
|
+
output=[
|
|
42
|
+
sp.TaskDispatch(regression=sp.SortQuantiles()),
|
|
43
|
+
sp.AverageEstimators(),
|
|
44
|
+
sp.TaskDispatch(classification=sp.Softmax(temperature=0.9)),
|
|
45
|
+
],
|
|
46
|
+
)
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# Copyright (c) 2025, Soda team @ Inria
|
|
2
|
+
# Licensed under the BSD 3-Clause License; see LICENSE.
|
|
3
|
+
|
|
4
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
6
|
+
|
|
7
|
+
# ruff: noqa: D101, D102
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from typing import Any, cast
|
|
11
|
+
|
|
12
|
+
import torch
|
|
13
|
+
from torch import Tensor
|
|
14
|
+
from torch.nn import Embedding, LayerNorm, ModuleList, Parameter
|
|
15
|
+
|
|
16
|
+
from sdm.cache import Cache, KVCacheEntry
|
|
17
|
+
from sdm.models.tabiclv2.block import TabICLv2TransformerBlock
|
|
18
|
+
from sdm.nn import InducedTransformerBlock, RotaryEmbedding
|
|
19
|
+
from sdm.nn.linear import Linear
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RowEmbedding(torch.nn.Module):
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
num_classes: int,
|
|
26
|
+
channels: int,
|
|
27
|
+
num_layers: int,
|
|
28
|
+
num_heads: int,
|
|
29
|
+
group_size: int,
|
|
30
|
+
num_inducing_points: int,
|
|
31
|
+
num_readout_tokens: int,
|
|
32
|
+
norm_bias: bool,
|
|
33
|
+
device: torch.device | str | None = None,
|
|
34
|
+
dtype: torch.dtype | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
super().__init__()
|
|
37
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
38
|
+
|
|
39
|
+
self.lin = Linear(group_size, channels, **factory_kwargs)
|
|
40
|
+
|
|
41
|
+
self.num_classes = num_classes
|
|
42
|
+
self.y_emb: torch.nn.Module | None = None
|
|
43
|
+
self.y_lin: torch.nn.Module | None = None
|
|
44
|
+
if num_classes > 0:
|
|
45
|
+
self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
|
|
46
|
+
else:
|
|
47
|
+
self.y_lin = torch.nn.Linear(1, channels, **factory_kwargs)
|
|
48
|
+
|
|
49
|
+
self.col_layers = ModuleList(
|
|
50
|
+
InducedTransformerBlock(
|
|
51
|
+
channels=channels,
|
|
52
|
+
num_inducing_points=num_inducing_points,
|
|
53
|
+
inducing_block=TabICLv2TransformerBlock(
|
|
54
|
+
channels=channels,
|
|
55
|
+
num_heads=num_heads,
|
|
56
|
+
norm_bias=norm_bias,
|
|
57
|
+
qassmax=True,
|
|
58
|
+
**factory_kwargs,
|
|
59
|
+
),
|
|
60
|
+
output_block=TabICLv2TransformerBlock(
|
|
61
|
+
channels=channels,
|
|
62
|
+
num_heads=num_heads,
|
|
63
|
+
norm_bias=norm_bias,
|
|
64
|
+
qassmax=False,
|
|
65
|
+
**factory_kwargs,
|
|
66
|
+
),
|
|
67
|
+
**factory_kwargs,
|
|
68
|
+
)
|
|
69
|
+
for _ in range(num_layers)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
self.readout_token = Parameter(
|
|
73
|
+
torch.empty((num_readout_tokens, channels), **factory_kwargs)
|
|
74
|
+
)
|
|
75
|
+
torch.nn.init.trunc_normal_(self.readout_token, std=0.02)
|
|
76
|
+
|
|
77
|
+
rope = RotaryEmbedding(
|
|
78
|
+
channels=channels // num_heads,
|
|
79
|
+
layout="split_half",
|
|
80
|
+
theta=100_000,
|
|
81
|
+
**factory_kwargs,
|
|
82
|
+
)
|
|
83
|
+
self.row_layers = ModuleList(
|
|
84
|
+
TabICLv2TransformerBlock(
|
|
85
|
+
channels=channels,
|
|
86
|
+
num_heads=num_heads,
|
|
87
|
+
norm_bias=norm_bias,
|
|
88
|
+
qassmax=False,
|
|
89
|
+
rope=rope,
|
|
90
|
+
**factory_kwargs,
|
|
91
|
+
)
|
|
92
|
+
for _ in range(num_layers)
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
self.norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs)
|
|
96
|
+
|
|
97
|
+
def forward(
|
|
98
|
+
self,
|
|
99
|
+
x: Tensor, # [..., R, C]
|
|
100
|
+
y: Tensor, # [..., R_train]
|
|
101
|
+
*,
|
|
102
|
+
train_mask: Tensor | None = None, # [R],
|
|
103
|
+
max_keys: int | None = None,
|
|
104
|
+
num_classes: int | None = None,
|
|
105
|
+
cache: Cache | None = None,
|
|
106
|
+
generator: torch.Generator | None = None,
|
|
107
|
+
) -> Tensor: # [..., R, K * D]
|
|
108
|
+
*B, R, C = x.size()
|
|
109
|
+
R_train = y.size(-1)
|
|
110
|
+
G, D = self.lin.in_features, self.lin.out_features
|
|
111
|
+
K = self.readout_token.size(-2)
|
|
112
|
+
train_mask: Any = slice(R_train) if train_mask is None else train_mask
|
|
113
|
+
|
|
114
|
+
# Feature grouping: gather G columns into each token.
|
|
115
|
+
shift = 2 ** torch.arange(G, device=x.device)
|
|
116
|
+
index = torch.arange(C, device=x.device)
|
|
117
|
+
index = (index.view(C, 1) + shift.view(1, G)) % C # [C, G]
|
|
118
|
+
x = x[..., index] # [..., R, C, G]
|
|
119
|
+
|
|
120
|
+
num_digits = 1
|
|
121
|
+
if (
|
|
122
|
+
self.y_emb is not None
|
|
123
|
+
and num_classes is not None
|
|
124
|
+
and num_classes > self.num_classes
|
|
125
|
+
):
|
|
126
|
+
bases = _mixed_radix_bases(num_classes, self.num_classes)
|
|
127
|
+
num_digits = len(bases)
|
|
128
|
+
if y.numel() > 0:
|
|
129
|
+
y = _mixed_radix_digits(y, bases) # [F, ..., R_train]
|
|
130
|
+
|
|
131
|
+
buffer: Tensor | None = None
|
|
132
|
+
if torch.is_grad_enabled():
|
|
133
|
+
x = self.lin(x) # [..., R, C, D]
|
|
134
|
+
if num_digits > 1:
|
|
135
|
+
if y.numel() > 0:
|
|
136
|
+
x = x.repeat(num_digits, *(1,) * x.dim())
|
|
137
|
+
else:
|
|
138
|
+
x = x.expand(num_digits, *x.size())
|
|
139
|
+
else:
|
|
140
|
+
buffer = torch.empty(
|
|
141
|
+
((num_digits,) if num_digits > 1 else ())
|
|
142
|
+
+ (*x.size()[:-3], R, K + C, D),
|
|
143
|
+
device=x.device,
|
|
144
|
+
dtype=torch.get_autocast_dtype(x.device.type)
|
|
145
|
+
if torch.is_autocast_enabled(x.device.type)
|
|
146
|
+
else x.dtype,
|
|
147
|
+
)
|
|
148
|
+
if num_digits > 1:
|
|
149
|
+
self.lin(x, out=buffer[0, ..., K:, :])
|
|
150
|
+
buffer[1:, ..., K:, :] = buffer[:1, ..., K:, :]
|
|
151
|
+
x = buffer[..., K:, :]
|
|
152
|
+
else:
|
|
153
|
+
x = self.lin(x, out=buffer[..., K:, :]) # [..., R, C, D]
|
|
154
|
+
|
|
155
|
+
if y.numel() > 0:
|
|
156
|
+
if self.y_emb is not None:
|
|
157
|
+
y_emb = self.y_emb(y).unsqueeze(-2) # [..., R_train, 1, D]
|
|
158
|
+
else:
|
|
159
|
+
assert self.y_lin is not None
|
|
160
|
+
y_emb = self.y_lin(y.unsqueeze(-1)).unsqueeze(-2)
|
|
161
|
+
|
|
162
|
+
x[..., train_mask, :, :] += y_emb.to(x.dtype)
|
|
163
|
+
|
|
164
|
+
# Column-wise induced set attention (B * C as the batch axis).
|
|
165
|
+
x = x.transpose(-2, -3) # [..., C, R, D]
|
|
166
|
+
for i, col_layer in enumerate(self.col_layers):
|
|
167
|
+
key = f"row_embedding.col_layer{i}"
|
|
168
|
+
if cache is not None and cache.is_replaying:
|
|
169
|
+
key_value = cast(KVCacheEntry, cache[key])
|
|
170
|
+
else:
|
|
171
|
+
key_value = x[..., train_mask, :]
|
|
172
|
+
if max_keys is not None and key_value.size(-2) > max_keys:
|
|
173
|
+
index = torch.randperm(
|
|
174
|
+
key_value.size(-2),
|
|
175
|
+
device=key_value.device,
|
|
176
|
+
generator=generator,
|
|
177
|
+
)[:max_keys]
|
|
178
|
+
key_value = key_value[..., index, :]
|
|
179
|
+
|
|
180
|
+
result = col_layer(
|
|
181
|
+
query=x, # [..., C, R, D]
|
|
182
|
+
key_value=key_value, # [..., C, R_train, D]
|
|
183
|
+
return_key_value=cache is not None and cache.is_recording,
|
|
184
|
+
batch_size_limit="auto",
|
|
185
|
+
out=None if torch.is_grad_enabled() else x,
|
|
186
|
+
) # [..., C, R, D]
|
|
187
|
+
del key_value
|
|
188
|
+
|
|
189
|
+
if cache is not None and cache.is_recording:
|
|
190
|
+
x, cache[key] = result
|
|
191
|
+
else:
|
|
192
|
+
x = result
|
|
193
|
+
del result
|
|
194
|
+
|
|
195
|
+
x = x.transpose(-2, -3) # [..., R, C, D]
|
|
196
|
+
|
|
197
|
+
if num_digits > 1: # Average over mixed-radix digits.
|
|
198
|
+
if buffer is not None:
|
|
199
|
+
buffer = buffer.mean(dim=0) # [..., R, K + C, D]
|
|
200
|
+
else:
|
|
201
|
+
x = x.mean(dim=0) # [..., R, C, D]
|
|
202
|
+
|
|
203
|
+
# Prepend readout tokens before row-wise attention.
|
|
204
|
+
readout_token = self.readout_token.to(x.dtype)
|
|
205
|
+
readout_token = readout_token.view(*(1,) * len(B), 1, K, D)
|
|
206
|
+
readout_token = readout_token.expand(*B, R, K, D)
|
|
207
|
+
|
|
208
|
+
if buffer is not None:
|
|
209
|
+
buffer[..., :, :K, :] = readout_token
|
|
210
|
+
x = buffer
|
|
211
|
+
else:
|
|
212
|
+
x = torch.cat([readout_token, x], dim=-2) # [..., R, K + C, D]
|
|
213
|
+
|
|
214
|
+
# Row-wise attention (B * R as the batch axis).
|
|
215
|
+
for i, row_layer in enumerate(self.row_layers):
|
|
216
|
+
x = row_layer(
|
|
217
|
+
query=x[..., :K, :] if i == len(self.row_layers) - 1 else x,
|
|
218
|
+
key_value=x, # [..., R, K + C, D]
|
|
219
|
+
batch_size_limit="auto",
|
|
220
|
+
out=None
|
|
221
|
+
if torch.is_grad_enabled()
|
|
222
|
+
else x[..., :K, :]
|
|
223
|
+
if i == len(self.row_layers) - 1
|
|
224
|
+
else x,
|
|
225
|
+
) # [..., R, K + C, D] or [..., R, K, D]
|
|
226
|
+
|
|
227
|
+
return self.norm(x).view(*B, R, K * D) # [..., R, K * D]
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _mixed_radix_bases(num_classes: int, max_classes: int) -> list[int]:
|
|
231
|
+
num_digits = math.ceil(math.log(num_classes) / math.log(max_classes))
|
|
232
|
+
base = min(math.ceil(num_classes ** (1.0 / num_digits)), max_classes)
|
|
233
|
+
bases = [base] * num_digits
|
|
234
|
+
product = base**num_digits
|
|
235
|
+
for i in range(num_digits):
|
|
236
|
+
if product >= num_classes:
|
|
237
|
+
break
|
|
238
|
+
if bases[i] < max_classes:
|
|
239
|
+
product = product // bases[i] * (bases[i] + 1)
|
|
240
|
+
bases[i] += 1
|
|
241
|
+
|
|
242
|
+
return bases
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _mixed_radix_digits(y: Tensor, bases: list[int]) -> Tensor:
|
|
246
|
+
F = len(bases)
|
|
247
|
+
divisors = [1] * F
|
|
248
|
+
for i in range(F - 2, -1, -1):
|
|
249
|
+
divisors[i] = divisors[i + 1] * bases[i + 1]
|
|
250
|
+
|
|
251
|
+
size = (-1,) + (1,) * y.dim()
|
|
252
|
+
divisor = torch.tensor(divisors, device=y.device).view(size)
|
|
253
|
+
base = torch.tensor(bases, device=y.device).view(size)
|
|
254
|
+
return (y.unsqueeze(0) // divisor) % base # [F, ...]
|
sdm/nn/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Neural network modules for structured data models."""
|
|
5
|
+
|
|
6
|
+
from sdm.nn.rope import RotaryEmbedding
|
|
7
|
+
from sdm.nn.glu import SwiGLU
|
|
8
|
+
from sdm.nn.softplus import SoftplusScale
|
|
9
|
+
from sdm.nn.rms_norm import RMSNorm
|
|
10
|
+
from sdm.nn.scaling import QueryScaling, QASSMax, LogScale, GatedLogScale
|
|
11
|
+
from sdm.nn.attention import SDPA, Attention, TransformerBlock
|
|
12
|
+
from sdm.nn.set_transformer import InducedTransformerBlock
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"RotaryEmbedding",
|
|
17
|
+
"SwiGLU",
|
|
18
|
+
"SoftplusScale",
|
|
19
|
+
"RMSNorm",
|
|
20
|
+
"QueryScaling",
|
|
21
|
+
"QASSMax",
|
|
22
|
+
"LogScale",
|
|
23
|
+
"GatedLogScale",
|
|
24
|
+
"SDPA",
|
|
25
|
+
"Attention",
|
|
26
|
+
"TransformerBlock",
|
|
27
|
+
"InducedTransformerBlock",
|
|
28
|
+
]
|
sdm/nn/_buffer.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Iterable, Iterator
|
|
7
|
+
from typing import Any, Generic, TypeVar, cast
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
from torch import Tensor
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T", bound="Tensor | BufferList[Any]")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class BufferList(torch.nn.Module, Generic[T]):
|
|
16
|
+
"""Ordered list of tensors registered as PyTorch module state.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
buffers: Tensors and buffer lists to register in order.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
buffers: Iterable[T] = (),
|
|
25
|
+
) -> None:
|
|
26
|
+
super().__init__()
|
|
27
|
+
for index, item in enumerate(buffers):
|
|
28
|
+
if isinstance(item, BufferList):
|
|
29
|
+
self.add_module(str(index), item)
|
|
30
|
+
else:
|
|
31
|
+
self.register_buffer(str(index), item)
|
|
32
|
+
|
|
33
|
+
def _load_from_state_dict(
|
|
34
|
+
self,
|
|
35
|
+
state_dict: dict[str, Any],
|
|
36
|
+
prefix: str,
|
|
37
|
+
local_metadata: dict[str, Any],
|
|
38
|
+
strict: bool,
|
|
39
|
+
missing_keys: list[str],
|
|
40
|
+
unexpected_keys: list[str],
|
|
41
|
+
error_msgs: list[str],
|
|
42
|
+
) -> None:
|
|
43
|
+
layout = state_dict[f"{prefix}_extra_state"]
|
|
44
|
+
materialized_keys = []
|
|
45
|
+
for index, is_nested in enumerate(layout):
|
|
46
|
+
if is_nested:
|
|
47
|
+
continue
|
|
48
|
+
name = str(index)
|
|
49
|
+
key = f"{prefix}{index}"
|
|
50
|
+
if name not in self._buffers:
|
|
51
|
+
# Some tensor subclasses cannot use empty_like or copy_.
|
|
52
|
+
self.register_buffer(name, state_dict.pop(key).clone())
|
|
53
|
+
materialized_keys.append(key)
|
|
54
|
+
|
|
55
|
+
super()._load_from_state_dict(
|
|
56
|
+
state_dict,
|
|
57
|
+
prefix,
|
|
58
|
+
local_metadata,
|
|
59
|
+
strict,
|
|
60
|
+
missing_keys,
|
|
61
|
+
unexpected_keys,
|
|
62
|
+
error_msgs,
|
|
63
|
+
)
|
|
64
|
+
for key in materialized_keys:
|
|
65
|
+
if key in missing_keys:
|
|
66
|
+
missing_keys.remove(key)
|
|
67
|
+
|
|
68
|
+
def __getitem__(self, index: int) -> T:
|
|
69
|
+
name = str(index)
|
|
70
|
+
if name in self._modules:
|
|
71
|
+
return cast(T, self.get_submodule(name))
|
|
72
|
+
return cast(T, self.get_buffer(name))
|
|
73
|
+
|
|
74
|
+
def __iter__(self) -> Iterator[T]:
|
|
75
|
+
return (self[index] for index in range(len(self)))
|
|
76
|
+
|
|
77
|
+
def __len__(self) -> int:
|
|
78
|
+
return len(self._buffers) + len(self._modules)
|
|
79
|
+
|
|
80
|
+
def get_extra_state(self) -> tuple[bool, ...]:
|
|
81
|
+
r""":meta private:""" # noqa: D415
|
|
82
|
+
# Layout of the buffer list in a tuple of booleans.
|
|
83
|
+
# True at positions that hold a nested BufferList rather than a tensor.
|
|
84
|
+
return tuple(str(index) in self._modules for index in range(len(self)))
|
|
85
|
+
|
|
86
|
+
def set_extra_state(self, state: object) -> None:
|
|
87
|
+
r""":meta private:""" # noqa: D415
|
|
88
|
+
for index, is_nested in enumerate(cast(tuple[bool, ...], state)):
|
|
89
|
+
name = str(index)
|
|
90
|
+
if is_nested and name not in self._modules:
|
|
91
|
+
self.add_module(name, BufferList())
|