structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/glu.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Portions derived from TabFM:
|
|
2
|
+
# Copyright 2026 Google LLC
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
17
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
18
|
+
|
|
19
|
+
# ruff: noqa: D205
|
|
20
|
+
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import torch
|
|
24
|
+
import torch.nn.functional as F
|
|
25
|
+
from torch import Tensor
|
|
26
|
+
from torch.nn import Linear
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SwiGLU(torch.nn.Module):
|
|
30
|
+
r""":math:`\mathrm{FFN}_{\mathrm{SwiGLU}}` block from the `"GLU Variants
|
|
31
|
+
Improve Transformer" <https://arxiv.org/abs/2002.05202>`_ paper.
|
|
32
|
+
|
|
33
|
+
.. math::
|
|
34
|
+
|
|
35
|
+
W_{\mathrm{down}}(
|
|
36
|
+
\operatorname{Swish}_1(W_{\mathrm{gate}} x + b_{\mathrm{gate}})
|
|
37
|
+
\otimes
|
|
38
|
+
(W_{\mathrm{up}} x + b_{\mathrm{up}})
|
|
39
|
+
) + b_{\mathrm{down}}
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
channels: The number of input and output channels.
|
|
43
|
+
hidden_channels: The hidden channels of the gate and up projections.
|
|
44
|
+
bias: If set to ``False``, the module will not learn an additive bias.
|
|
45
|
+
device: The device.
|
|
46
|
+
dtype: The dtype.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
channels: int,
|
|
52
|
+
hidden_channels: int,
|
|
53
|
+
bias: bool = True,
|
|
54
|
+
device: torch.device | str | None = None,
|
|
55
|
+
dtype: torch.dtype | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
super().__init__()
|
|
58
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
59
|
+
|
|
60
|
+
self.up_lin = Linear(
|
|
61
|
+
channels, hidden_channels, bias=bias, **factory_kwargs
|
|
62
|
+
)
|
|
63
|
+
self.gate_lin = Linear(
|
|
64
|
+
channels, hidden_channels, bias=bias, **factory_kwargs
|
|
65
|
+
)
|
|
66
|
+
self.down_lin = Linear(
|
|
67
|
+
hidden_channels, channels, bias=bias, **factory_kwargs
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def forward(self, tensor: Tensor) -> Tensor:
|
|
71
|
+
r"""The forward pass.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
tensor: The tensor.
|
|
75
|
+
"""
|
|
76
|
+
value = self.up_lin(tensor)
|
|
77
|
+
gate = F.silu(self.gate_lin(tensor))
|
|
78
|
+
return self.down_lin(gate * value)
|
sdm/nn/linear.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D101, D102, A001, A002
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Linear(torch.nn.Linear):
|
|
13
|
+
def forward(
|
|
14
|
+
self,
|
|
15
|
+
input: Tensor,
|
|
16
|
+
*,
|
|
17
|
+
out: Tensor | None = None,
|
|
18
|
+
) -> Tensor:
|
|
19
|
+
if out is None:
|
|
20
|
+
return super().forward(input)
|
|
21
|
+
|
|
22
|
+
if torch.is_grad_enabled():
|
|
23
|
+
raise RuntimeError(
|
|
24
|
+
"'out' is only supported when gradients are disabled"
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
input = input.to(out.dtype)
|
|
28
|
+
weight = self.weight.to(out.dtype).t()
|
|
29
|
+
|
|
30
|
+
if torch.compiler.is_compiling() and not out.is_contiguous():
|
|
31
|
+
out.copy_(torch.matmul(input, weight))
|
|
32
|
+
elif input.dim() == 2:
|
|
33
|
+
torch.matmul(input, weight, out=out)
|
|
34
|
+
else:
|
|
35
|
+
batch_size = math.prod(input.shape[:-2])
|
|
36
|
+
input = input.view(batch_size, input.size(-2), input.size(-1))
|
|
37
|
+
torch.bmm(
|
|
38
|
+
input,
|
|
39
|
+
weight.expand(input.size(0), -1, -1),
|
|
40
|
+
out=out.view(batch_size, out.size(-2), out.size(-1)),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
if self.bias is not None:
|
|
44
|
+
out += self.bias.to(out.dtype)
|
|
45
|
+
|
|
46
|
+
return out
|
sdm/nn/rms_norm.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch import Tensor
|
|
6
|
+
|
|
7
|
+
from sdm._kernels import rmsnorm_cast
|
|
8
|
+
from sdm.nn.rope import RotaryEmbedding
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RMSNorm(torch.nn.RMSNorm):
|
|
12
|
+
"""A :class:`torch.nn.RMSNorm` that dispatches to an efficient kernel."""
|
|
13
|
+
|
|
14
|
+
def forward(
|
|
15
|
+
self,
|
|
16
|
+
x: Tensor,
|
|
17
|
+
rope: RotaryEmbedding | None = None,
|
|
18
|
+
) -> Tensor:
|
|
19
|
+
"""Apply optional rotary embeddings before normalization.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
x: Input with shape ``[..., C]``, or ``[..., S, H, C]`` with
|
|
23
|
+
rotary embeddings, where ``S`` is the sequence length,
|
|
24
|
+
``H`` is the number of heads, and ``C`` is channels per head.
|
|
25
|
+
rope: Optional rotary embedding applied to the input.
|
|
26
|
+
"""
|
|
27
|
+
if (
|
|
28
|
+
x.is_cuda
|
|
29
|
+
and torch.is_autocast_enabled("cuda")
|
|
30
|
+
and not torch.is_grad_enabled()
|
|
31
|
+
and x.dtype in {torch.float16, torch.bfloat16, torch.float32}
|
|
32
|
+
and x.shape[-1:] == self.normalized_shape
|
|
33
|
+
and (
|
|
34
|
+
self.weight is None
|
|
35
|
+
or self.weight.dtype
|
|
36
|
+
in {torch.float16, torch.bfloat16, torch.float32}
|
|
37
|
+
)
|
|
38
|
+
):
|
|
39
|
+
cos_sin: tuple[Tensor, Tensor] | None = None
|
|
40
|
+
if rope is not None:
|
|
41
|
+
if (
|
|
42
|
+
not torch.compiler.is_compiling()
|
|
43
|
+
and rope.layout == "split_half"
|
|
44
|
+
and rope.rotary_channels == rope.channels == x.size(-1)
|
|
45
|
+
):
|
|
46
|
+
seq = torch.arange(
|
|
47
|
+
x.size(-3), device=x.device, dtype=torch.float32
|
|
48
|
+
)
|
|
49
|
+
freq = seq.view(-1, 1) * rope.inv_freq.view(1, -1)
|
|
50
|
+
cos_sin = (freq.cos().to(x.dtype), freq.sin().to(x.dtype))
|
|
51
|
+
else:
|
|
52
|
+
x = rope(x)
|
|
53
|
+
eps = (
|
|
54
|
+
self.eps
|
|
55
|
+
if self.eps is not None
|
|
56
|
+
else torch.finfo(torch.float32).eps
|
|
57
|
+
)
|
|
58
|
+
return rmsnorm_cast(x, self.weight, eps, rope=cos_sin)
|
|
59
|
+
return super().forward(x if rope is None else rope(x))
|
sdm/nn/rope.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D205
|
|
5
|
+
|
|
6
|
+
from typing import Any, Literal
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RotaryEmbedding(torch.nn.Module):
|
|
13
|
+
r"""Rotary Positional Embedding (RoPE) from the `"RoFormer: Enhanced
|
|
14
|
+
Transformer with Rotary Position Embedding"
|
|
15
|
+
<https://arxiv.org/abs/2104.09864>`_ paper.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
channels: The number of channels per attention head.
|
|
19
|
+
layout: The channel pairing layout. ``"split_half"`` pairs the first
|
|
20
|
+
half of the channels with the second half. ``"interleaved"`` pairs
|
|
21
|
+
adjacent even and odd channels.
|
|
22
|
+
theta: The base frequency used to initialize inverse frequencies.
|
|
23
|
+
requires_grad: Whether inverse frequencies are learnable.
|
|
24
|
+
partial_rotary_factor: The fraction of leading channels to which RoPE
|
|
25
|
+
is applied.
|
|
26
|
+
device: The device.
|
|
27
|
+
dtype: The dtype.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
channels: int,
|
|
33
|
+
layout: Literal["split_half", "interleaved"],
|
|
34
|
+
theta: float = 100_000,
|
|
35
|
+
requires_grad: bool = True,
|
|
36
|
+
partial_rotary_factor: float = 1.0,
|
|
37
|
+
device: torch.device | str | None = None,
|
|
38
|
+
dtype: torch.dtype | None = None,
|
|
39
|
+
) -> None:
|
|
40
|
+
super().__init__()
|
|
41
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
42
|
+
|
|
43
|
+
self.layout = layout
|
|
44
|
+
self.channels = channels
|
|
45
|
+
self.rotary_channels = int(channels * partial_rotary_factor)
|
|
46
|
+
|
|
47
|
+
if self.channels % 2 != 0:
|
|
48
|
+
raise ValueError(f"'channels' must be even (got {self.channels})")
|
|
49
|
+
if (
|
|
50
|
+
self.rotary_channels < 2
|
|
51
|
+
or self.rotary_channels > channels
|
|
52
|
+
or self.rotary_channels % 2 != 0
|
|
53
|
+
):
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"'partial_rotary_factor' must produce an even number between "
|
|
56
|
+
f"2 and {self.channels} (got {self.rotary_channels})"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
arange = torch.arange(0, self.rotary_channels, 2, **factory_kwargs)
|
|
60
|
+
inv_freq = 1.0 / (theta ** (arange / self.rotary_channels))
|
|
61
|
+
self.inv_freq = torch.nn.Parameter(
|
|
62
|
+
inv_freq,
|
|
63
|
+
requires_grad=requires_grad,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def forward(
|
|
67
|
+
self,
|
|
68
|
+
x: Tensor, # [..., S, H, C]
|
|
69
|
+
) -> Tensor: # [..., S, H, C]
|
|
70
|
+
"""The forward pass.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
x: Tensor with shape ``[..., S, H, C]``.
|
|
74
|
+
``S`` is the query sequence length, ``H`` is the number of
|
|
75
|
+
attention heads, and ``C`` is the channels per head.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
Tensor with shape ``[..., S, H, C]``.
|
|
79
|
+
"""
|
|
80
|
+
if x.size(-1) != self.channels:
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"Expected {self.channels} channels (got {x.size(-1)})"
|
|
83
|
+
)
|
|
84
|
+
seq = torch.arange(x.size(-3), device=x.device, dtype=torch.float32)
|
|
85
|
+
freq = seq.view(-1, 1) * self.inv_freq.view(1, -1) # [S, C // 2]
|
|
86
|
+
sin = freq.sin()[:, None, :].to(x.dtype) # [S, 1, C // 2]
|
|
87
|
+
cos = freq.cos()[:, None, :].to(x.dtype) # [S, 1, C // 2]
|
|
88
|
+
|
|
89
|
+
rotary = x[..., : self.rotary_channels]
|
|
90
|
+
if self.layout == "interleaved":
|
|
91
|
+
x1, x2 = rotary[..., 0::2], rotary[..., 1::2]
|
|
92
|
+
else:
|
|
93
|
+
assert self.layout == "split_half"
|
|
94
|
+
x1, x2 = rotary.split(sin.size(-1), dim=-1)
|
|
95
|
+
|
|
96
|
+
out1 = x1 * cos - x2 * sin
|
|
97
|
+
out2 = x2 * cos + x1 * sin
|
|
98
|
+
|
|
99
|
+
if self.layout == "interleaved":
|
|
100
|
+
rotary = torch.stack((out1, out2), dim=-1).flatten(-2)
|
|
101
|
+
else:
|
|
102
|
+
assert self.layout == "split_half"
|
|
103
|
+
rotary = torch.cat((out1, out2), dim=-1)
|
|
104
|
+
|
|
105
|
+
if self.rotary_channels == self.channels:
|
|
106
|
+
return rotary
|
|
107
|
+
return torch.cat((rotary, x[..., self.rotary_channels :]), dim=-1)
|
sdm/nn/scaling.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# Portions derived from TabICLv2:
|
|
2
|
+
# Copyright (c) 2025, Soda team @ Inria
|
|
3
|
+
# Licensed under the BSD 3-Clause License; see sdm/models/tabiclv2/LICENSE.
|
|
4
|
+
|
|
5
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
6
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
7
|
+
|
|
8
|
+
import abc
|
|
9
|
+
from typing import Any, cast
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
from torch.nn import GELU, Linear, Parameter, Sequential
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class QueryScaling(torch.nn.Module, abc.ABC):
|
|
17
|
+
r"""Base class for query scaling modules in :class:`SDPA`.
|
|
18
|
+
|
|
19
|
+
Query scaling modules transform projected query heads before scaled
|
|
20
|
+
dot-product attention. They may use the effective key length to implement
|
|
21
|
+
length-aware temperature scaling while preserving compatibility with
|
|
22
|
+
:func:`torch.nn.functional.scaled_dot_product_attention`.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self) -> None:
|
|
26
|
+
super().__init__()
|
|
27
|
+
|
|
28
|
+
@abc.abstractmethod
|
|
29
|
+
def forward(
|
|
30
|
+
self,
|
|
31
|
+
query: Tensor, # [..., S, H, C]
|
|
32
|
+
*,
|
|
33
|
+
key_len: Tensor | int, # [..., 1] or [..., S] or scalar
|
|
34
|
+
) -> Tensor: # [..., S, H, C]
|
|
35
|
+
r"""The forward pass.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
query: The query tensor to scale, with shape ``[..., S, H, C]``.
|
|
39
|
+
``S`` is the query sequence length, ``H`` is the number of
|
|
40
|
+
attention heads, and ``C`` is the channels per head.
|
|
41
|
+
key_len: The number of valid keys used to scale each query.
|
|
42
|
+
An ``int`` denotes one shared key length for every query.
|
|
43
|
+
A tensor shaped ``[..., 1]`` denotes one key length per batch
|
|
44
|
+
item, shared across all query positions in ``S``.
|
|
45
|
+
A tensor shaped ``[..., S]`` denotes one individual key length
|
|
46
|
+
per query position.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Tensor with shape ``[..., S, H, C]``.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class QASSMax(QueryScaling):
|
|
54
|
+
r"""Query-Aware Scalable SoftMax (QASSMax).
|
|
55
|
+
|
|
56
|
+
This scaling method was introduced in the `"TabICLv2: A better, faster,
|
|
57
|
+
scalable, and open tabular foundation model"
|
|
58
|
+
<https://arxiv.org/abs/2602.11139>`_ paper as a temperature-scaling method
|
|
59
|
+
for attention.
|
|
60
|
+
|
|
61
|
+
For a query tensor :math:`q` and key length :math:`n`, this module returns
|
|
62
|
+
a scaled query
|
|
63
|
+
|
|
64
|
+
.. math::
|
|
65
|
+
|
|
66
|
+
\tilde{q}_{hi} = q_{hi} \cdot
|
|
67
|
+
\mathrm{MLP}_{\mathrm{base}}(\log n)_{hi} \cdot
|
|
68
|
+
\left(1 + \tanh(\mathrm{MLP}_{\mathrm{gate}}(q_h)_i)\right),
|
|
69
|
+
|
|
70
|
+
where :math:`h` indexes attention heads, :math:`i` indexes head channels,
|
|
71
|
+
:math:`\mathrm{MLP}_{\mathrm{base}}` maps the log key length to per-head,
|
|
72
|
+
per-channel scale factors, and :math:`\mathrm{MLP}_{\mathrm{gate}}` maps
|
|
73
|
+
each per-head query vector to a bounded query-dependent gate.
|
|
74
|
+
|
|
75
|
+
Multiplying the query scales the subsequent attention logits while keeping
|
|
76
|
+
the attention computation compatible with standard softmax kernels.
|
|
77
|
+
The length-dependent factor counteracts attention fading as the number of
|
|
78
|
+
keys grows, while the query-dependent gate lets the scale vary across
|
|
79
|
+
queries.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
channels: The number of channels per attention head.
|
|
83
|
+
num_heads: The number of query attention heads.
|
|
84
|
+
hidden_channels: The hidden width of the scale and gate MLPs.
|
|
85
|
+
device: The device.
|
|
86
|
+
dtype: The dtype.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def __init__(
|
|
90
|
+
self,
|
|
91
|
+
channels: int,
|
|
92
|
+
num_heads: int,
|
|
93
|
+
hidden_channels: int = 64,
|
|
94
|
+
device: torch.device | str | None = None,
|
|
95
|
+
dtype: torch.dtype | None = None,
|
|
96
|
+
) -> None:
|
|
97
|
+
super().__init__()
|
|
98
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
99
|
+
|
|
100
|
+
self.scale = Sequential(
|
|
101
|
+
Linear(1, hidden_channels, **factory_kwargs),
|
|
102
|
+
GELU(),
|
|
103
|
+
Linear(hidden_channels, num_heads * channels, **factory_kwargs),
|
|
104
|
+
)
|
|
105
|
+
self.gate = Sequential(
|
|
106
|
+
Linear(channels, hidden_channels, **factory_kwargs),
|
|
107
|
+
GELU(),
|
|
108
|
+
Linear(hidden_channels, channels, **factory_kwargs),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
torch.nn.init.zeros_(cast(Linear, self.gate[-1]).weight)
|
|
112
|
+
torch.nn.init.zeros_(cast(Linear, self.gate[-1]).bias)
|
|
113
|
+
|
|
114
|
+
def forward(
|
|
115
|
+
self,
|
|
116
|
+
query: Tensor, # [..., S, H, C]
|
|
117
|
+
*,
|
|
118
|
+
key_len: Tensor | int, # [..., 1] or [..., S] or scalar
|
|
119
|
+
) -> Tensor: # [..., S, H, C]
|
|
120
|
+
r""":meta private:""" # noqa: D415
|
|
121
|
+
if isinstance(key_len, Tensor):
|
|
122
|
+
log_key_len = key_len.float().clamp(min=1.0).log().to(query.dtype)
|
|
123
|
+
else:
|
|
124
|
+
one = query.new_ones((), dtype=torch.float32)
|
|
125
|
+
log_key_len = (one * key_len).clamp(min=1.0).log().to(query.dtype)
|
|
126
|
+
log_key_len = log_key_len.view([1] * (query.dim() - 2))
|
|
127
|
+
scale = self.scale(log_key_len.unsqueeze(-1)) # [..., S or 1, H * C]
|
|
128
|
+
scale = scale.unflatten(-1, (query.size(-2), query.size(-1)))
|
|
129
|
+
gate = 1 + self.gate(query).tanh()
|
|
130
|
+
return query * scale * gate
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class LogScale(QueryScaling):
|
|
134
|
+
r"""Logarithmic query scaling.
|
|
135
|
+
|
|
136
|
+
For a query tensor :math:`q`, effective key length :math:`n`, and learned
|
|
137
|
+
per-head coefficient :math:`a_h`, this module returns a scaled query
|
|
138
|
+
|
|
139
|
+
.. math::
|
|
140
|
+
|
|
141
|
+
\tilde{q}_{hi} = q_{hi} \cdot a_h \cdot \log(\max(n, 1)),
|
|
142
|
+
|
|
143
|
+
where :math:`h` indexes attention heads and :math:`i` indexes head
|
|
144
|
+
channels.
|
|
145
|
+
|
|
146
|
+
Each attention head receives an unconstrained coefficient initialized
|
|
147
|
+
independently and uniformly in ``[0, 1)``.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
num_heads: The number of query attention heads.
|
|
151
|
+
device: The device.
|
|
152
|
+
dtype: The dtype.
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
def __init__(
|
|
156
|
+
self,
|
|
157
|
+
num_heads: int,
|
|
158
|
+
device: torch.device | str | None = None,
|
|
159
|
+
dtype: torch.dtype | None = None,
|
|
160
|
+
) -> None:
|
|
161
|
+
super().__init__()
|
|
162
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
163
|
+
self.head_scale = Parameter(torch.rand((num_heads,), **factory_kwargs))
|
|
164
|
+
|
|
165
|
+
def forward(
|
|
166
|
+
self,
|
|
167
|
+
query: Tensor, # [..., S, H, C]
|
|
168
|
+
*,
|
|
169
|
+
key_len: Tensor | int, # [..., 1] or [..., S] or scalar
|
|
170
|
+
) -> Tensor: # [..., S, H, C]
|
|
171
|
+
r""":meta private:""" # noqa: D415
|
|
172
|
+
# Avoid a silently wrong result from broadcasting the head dimension.
|
|
173
|
+
if query.size(-2) != self.head_scale.numel():
|
|
174
|
+
raise ValueError(
|
|
175
|
+
"`query.size(-2)` must equal the number of per-head scales "
|
|
176
|
+
f"(got {query.size(-2)} and {self.head_scale.numel()})"
|
|
177
|
+
)
|
|
178
|
+
if isinstance(key_len, Tensor):
|
|
179
|
+
key_len_tensor = key_len.float()
|
|
180
|
+
else:
|
|
181
|
+
key_len_tensor = query.new_full((), key_len, dtype=torch.float32)
|
|
182
|
+
log_key_len = key_len_tensor.clamp(min=1.0).log().to(query.dtype)
|
|
183
|
+
head_scale = self.head_scale.to(query.dtype).reshape(
|
|
184
|
+
*((1,) * (query.dim() - 2)), -1, 1
|
|
185
|
+
)
|
|
186
|
+
return query * log_key_len[..., None, None] * head_scale
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class GatedLogScale(LogScale):
|
|
190
|
+
r"""Gated logarithmic query scaling.
|
|
191
|
+
|
|
192
|
+
For a query tensor :math:`q`, effective key length :math:`n`, learned
|
|
193
|
+
per-head coefficient :math:`a_h`, and learned gate :math:`g`, this module
|
|
194
|
+
returns
|
|
195
|
+
|
|
196
|
+
.. math::
|
|
197
|
+
|
|
198
|
+
\tilde{q}_{hi} = q_{hi} \cdot a_h \cdot \log(\max(n, 1))
|
|
199
|
+
\cdot \left(1 + \tanh(g(q_h)_i)\right).
|
|
200
|
+
|
|
201
|
+
The multiplicative gate is initialized to one and bounded between zero and
|
|
202
|
+
two. Per-head coefficients are initialized independently and uniformly in
|
|
203
|
+
``[0, 1)``.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
channels: The number of channels per attention head.
|
|
207
|
+
num_heads: The number of query attention heads.
|
|
208
|
+
hidden_channels: The hidden width of the gate MLP.
|
|
209
|
+
device: The device.
|
|
210
|
+
dtype: The dtype.
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
def __init__(
|
|
214
|
+
self,
|
|
215
|
+
channels: int,
|
|
216
|
+
num_heads: int,
|
|
217
|
+
hidden_channels: int,
|
|
218
|
+
device: torch.device | str | None = None,
|
|
219
|
+
dtype: torch.dtype | None = None,
|
|
220
|
+
) -> None:
|
|
221
|
+
super().__init__(num_heads=num_heads, device=device, dtype=dtype)
|
|
222
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
223
|
+
|
|
224
|
+
self.gate = Sequential(
|
|
225
|
+
Linear(channels, hidden_channels, **factory_kwargs),
|
|
226
|
+
GELU(),
|
|
227
|
+
Linear(hidden_channels, channels, **factory_kwargs),
|
|
228
|
+
)
|
|
229
|
+
torch.nn.init.zeros_(cast(Linear, self.gate[-1]).weight)
|
|
230
|
+
torch.nn.init.zeros_(cast(Linear, self.gate[-1]).bias)
|
|
231
|
+
|
|
232
|
+
def forward(
|
|
233
|
+
self,
|
|
234
|
+
query: Tensor, # [..., S, H, C]
|
|
235
|
+
*,
|
|
236
|
+
key_len: Tensor | int, # [..., 1] or [..., S] or scalar
|
|
237
|
+
) -> Tensor: # [..., S, H, C]
|
|
238
|
+
r""":meta private:""" # noqa: D415
|
|
239
|
+
query_scaled = super().forward(query=query, key_len=key_len)
|
|
240
|
+
return query_scaled * (1 + self.gate(query).tanh())
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
r"""Set-transformer modules for structured tensor models."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import Any, Literal, overload
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
from torch.nn import Parameter
|
|
13
|
+
|
|
14
|
+
from sdm.cache import KVCacheEntry
|
|
15
|
+
from sdm.nn import TransformerBlock
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class InducedTransformerBlock(torch.nn.Module):
|
|
19
|
+
r"""Transformer block using learned inducing points.
|
|
20
|
+
|
|
21
|
+
Introduced in the `"Set Transformer: A Framework for Attention-based
|
|
22
|
+
Permutation-Invariant Neural Networks"
|
|
23
|
+
<https://arxiv.org/abs/1810.00825>`_ paper, the block routes attention
|
|
24
|
+
through a small set of :math:`M` learned inducing points, reducing the cost
|
|
25
|
+
of attending a query of size :math:`Q` to a key/value context of size
|
|
26
|
+
:math:`K` from :math:`O(Q \cdot K)` to :math:`O((Q + K) \cdot M)`:
|
|
27
|
+
|
|
28
|
+
.. math::
|
|
29
|
+
|
|
30
|
+
H = \mathrm{InducingBlock}_1(I, \mathrm{key\_value}), \quad
|
|
31
|
+
\mathrm{out} = \mathrm{OutputBlock}_2(\mathrm{query}, H),
|
|
32
|
+
|
|
33
|
+
where :math:`I` are the learned inducing points and :math:`H` are the
|
|
34
|
+
inducing points after attending to the key/value elements. Passing
|
|
35
|
+
``key_value=None`` recovers the induced self-attention block (ISAB).
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
channels: The number of input and output channels.
|
|
39
|
+
num_inducing_points: The number of learned inducing points :math:`M`.
|
|
40
|
+
inducing_block: The :class:`TransformerBlock` that updates the learned
|
|
41
|
+
inducing points from key/value context.
|
|
42
|
+
output_block: The :class:`TransformerBlock` that updates the input
|
|
43
|
+
queries from the induced context.
|
|
44
|
+
device: The device.
|
|
45
|
+
dtype: The dtype.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
channels: int,
|
|
51
|
+
num_inducing_points: int,
|
|
52
|
+
inducing_block: TransformerBlock,
|
|
53
|
+
output_block: TransformerBlock,
|
|
54
|
+
device: torch.device | str | None = None,
|
|
55
|
+
dtype: torch.dtype | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
super().__init__()
|
|
58
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
59
|
+
|
|
60
|
+
self.inducing_block = inducing_block
|
|
61
|
+
self.output_block = output_block
|
|
62
|
+
|
|
63
|
+
self.inducing_points = Parameter(
|
|
64
|
+
torch.empty(num_inducing_points, channels, **factory_kwargs)
|
|
65
|
+
)
|
|
66
|
+
torch.nn.init.trunc_normal_(self.inducing_points, std=0.02)
|
|
67
|
+
|
|
68
|
+
@overload
|
|
69
|
+
def forward(
|
|
70
|
+
self,
|
|
71
|
+
query: Tensor,
|
|
72
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
73
|
+
seqused_key_value: Tensor | None = None,
|
|
74
|
+
attn_mask: Tensor | None = None,
|
|
75
|
+
*,
|
|
76
|
+
return_key_value: Literal[False] = False,
|
|
77
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
78
|
+
out: Tensor | None = None,
|
|
79
|
+
) -> Tensor: ...
|
|
80
|
+
|
|
81
|
+
@overload
|
|
82
|
+
def forward(
|
|
83
|
+
self,
|
|
84
|
+
query: Tensor,
|
|
85
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
86
|
+
seqused_key_value: Tensor | None = None,
|
|
87
|
+
attn_mask: Tensor | None = None,
|
|
88
|
+
*,
|
|
89
|
+
return_key_value: Literal[True],
|
|
90
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
91
|
+
out: Tensor | None = None,
|
|
92
|
+
) -> tuple[Tensor, KVCacheEntry]: ...
|
|
93
|
+
|
|
94
|
+
@overload
|
|
95
|
+
def forward(
|
|
96
|
+
self,
|
|
97
|
+
query: Tensor,
|
|
98
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
99
|
+
seqused_key_value: Tensor | None = None,
|
|
100
|
+
attn_mask: Tensor | None = None,
|
|
101
|
+
*,
|
|
102
|
+
return_key_value: bool,
|
|
103
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
104
|
+
out: Tensor | None = None,
|
|
105
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
|
|
106
|
+
|
|
107
|
+
def forward(
|
|
108
|
+
self,
|
|
109
|
+
query: Tensor, # [..., Q, C]
|
|
110
|
+
key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
|
|
111
|
+
seqused_key_value: Tensor | None = None, # [...]
|
|
112
|
+
attn_mask: Tensor | None = None, # [..., KV]
|
|
113
|
+
*,
|
|
114
|
+
return_key_value: bool = False,
|
|
115
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
116
|
+
out: Tensor | None = None,
|
|
117
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
|
|
118
|
+
r"""The forward pass.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
query: The query tensor with shape ``[..., Q, C]``.
|
|
122
|
+
``Q`` is the query sequence length, ``C`` is the number of
|
|
123
|
+
channels.
|
|
124
|
+
key_value: The key/value tensor with shape ``[..., KV, C]`` or
|
|
125
|
+
precomputed key/value projections as a
|
|
126
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
127
|
+
``KV`` is the key/value sequence length.
|
|
128
|
+
If omitted, ``query`` is used for induced self-attention.
|
|
129
|
+
seqused_key_value: Valid key/value lengths with shape ``[...]`` and
|
|
130
|
+
:external+torch:ref:`torch.int32 <dtype-doc>` dtype.
|
|
131
|
+
attn_mask: Boolean attention mask with shape ``[..., KV]``.
|
|
132
|
+
Entries set to ``True`` participate in attention.
|
|
133
|
+
return_key_value: Whether to return the computed key and value
|
|
134
|
+
projections for the final attention site alongside the output.
|
|
135
|
+
batch_size_limit: Maximum number of batch elements processed at
|
|
136
|
+
once.
|
|
137
|
+
out: The output tensor.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
|
|
141
|
+
``False``. Otherwise, a tuple of the output tensor and a
|
|
142
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
143
|
+
"""
|
|
144
|
+
if not isinstance(key_value, KVCacheEntry):
|
|
145
|
+
if key_value is None:
|
|
146
|
+
key_value = query
|
|
147
|
+
if attn_mask is not None:
|
|
148
|
+
attn_mask = attn_mask.unsqueeze(-2) # [..., 1, KV]
|
|
149
|
+
key_value = self.inducing_block(
|
|
150
|
+
query=self.inducing_points, # [M, C]
|
|
151
|
+
key_value=key_value, # [..., KV, C]
|
|
152
|
+
seqused_key_value=seqused_key_value, # [...]
|
|
153
|
+
attn_mask=attn_mask, # [..., 1, KV]
|
|
154
|
+
batch_size_limit=batch_size_limit,
|
|
155
|
+
) # [..., M, C]
|
|
156
|
+
return self.output_block(
|
|
157
|
+
query=query, # [..., Q, C]
|
|
158
|
+
key_value=key_value, # [..., M, C]
|
|
159
|
+
return_key_value=return_key_value,
|
|
160
|
+
batch_size_limit=batch_size_limit,
|
|
161
|
+
out=out,
|
|
162
|
+
) # [..., Q, C]
|