structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/glu.py ADDED
@@ -0,0 +1,78 @@
1
+ # Portions derived from TabFM:
2
+ # Copyright 2026 Google LLC
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # https://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
17
+ # SPDX-License-Identifier: Apache-2.0
18
+
19
+ # ruff: noqa: D205
20
+
21
+ from typing import Any
22
+
23
+ import torch
24
+ import torch.nn.functional as F
25
+ from torch import Tensor
26
+ from torch.nn import Linear
27
+
28
+
29
+ class SwiGLU(torch.nn.Module):
30
+ r""":math:`\mathrm{FFN}_{\mathrm{SwiGLU}}` block from the `"GLU Variants
31
+ Improve Transformer" <https://arxiv.org/abs/2002.05202>`_ paper.
32
+
33
+ .. math::
34
+
35
+ W_{\mathrm{down}}(
36
+ \operatorname{Swish}_1(W_{\mathrm{gate}} x + b_{\mathrm{gate}})
37
+ \otimes
38
+ (W_{\mathrm{up}} x + b_{\mathrm{up}})
39
+ ) + b_{\mathrm{down}}
40
+
41
+ Args:
42
+ channels: The number of input and output channels.
43
+ hidden_channels: The hidden channels of the gate and up projections.
44
+ bias: If set to ``False``, the module will not learn an additive bias.
45
+ device: The device.
46
+ dtype: The dtype.
47
+ """
48
+
49
+ def __init__(
50
+ self,
51
+ channels: int,
52
+ hidden_channels: int,
53
+ bias: bool = True,
54
+ device: torch.device | str | None = None,
55
+ dtype: torch.dtype | None = None,
56
+ ) -> None:
57
+ super().__init__()
58
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
59
+
60
+ self.up_lin = Linear(
61
+ channels, hidden_channels, bias=bias, **factory_kwargs
62
+ )
63
+ self.gate_lin = Linear(
64
+ channels, hidden_channels, bias=bias, **factory_kwargs
65
+ )
66
+ self.down_lin = Linear(
67
+ hidden_channels, channels, bias=bias, **factory_kwargs
68
+ )
69
+
70
+ def forward(self, tensor: Tensor) -> Tensor:
71
+ r"""The forward pass.
72
+
73
+ Args:
74
+ tensor: The tensor.
75
+ """
76
+ value = self.up_lin(tensor)
77
+ gate = F.silu(self.gate_lin(tensor))
78
+ return self.down_lin(gate * value)
sdm/nn/linear.py ADDED
@@ -0,0 +1,46 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D101, D102, A001, A002
5
+
6
+ import math
7
+
8
+ import torch
9
+ from torch import Tensor
10
+
11
+
12
+ class Linear(torch.nn.Linear):
13
+ def forward(
14
+ self,
15
+ input: Tensor,
16
+ *,
17
+ out: Tensor | None = None,
18
+ ) -> Tensor:
19
+ if out is None:
20
+ return super().forward(input)
21
+
22
+ if torch.is_grad_enabled():
23
+ raise RuntimeError(
24
+ "'out' is only supported when gradients are disabled"
25
+ )
26
+
27
+ input = input.to(out.dtype)
28
+ weight = self.weight.to(out.dtype).t()
29
+
30
+ if torch.compiler.is_compiling() and not out.is_contiguous():
31
+ out.copy_(torch.matmul(input, weight))
32
+ elif input.dim() == 2:
33
+ torch.matmul(input, weight, out=out)
34
+ else:
35
+ batch_size = math.prod(input.shape[:-2])
36
+ input = input.view(batch_size, input.size(-2), input.size(-1))
37
+ torch.bmm(
38
+ input,
39
+ weight.expand(input.size(0), -1, -1),
40
+ out=out.view(batch_size, out.size(-2), out.size(-1)),
41
+ )
42
+
43
+ if self.bias is not None:
44
+ out += self.bias.to(out.dtype)
45
+
46
+ return out
sdm/nn/rms_norm.py ADDED
@@ -0,0 +1,59 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import torch
5
+ from torch import Tensor
6
+
7
+ from sdm._kernels import rmsnorm_cast
8
+ from sdm.nn.rope import RotaryEmbedding
9
+
10
+
11
+ class RMSNorm(torch.nn.RMSNorm):
12
+ """A :class:`torch.nn.RMSNorm` that dispatches to an efficient kernel."""
13
+
14
+ def forward(
15
+ self,
16
+ x: Tensor,
17
+ rope: RotaryEmbedding | None = None,
18
+ ) -> Tensor:
19
+ """Apply optional rotary embeddings before normalization.
20
+
21
+ Args:
22
+ x: Input with shape ``[..., C]``, or ``[..., S, H, C]`` with
23
+ rotary embeddings, where ``S`` is the sequence length,
24
+ ``H`` is the number of heads, and ``C`` is channels per head.
25
+ rope: Optional rotary embedding applied to the input.
26
+ """
27
+ if (
28
+ x.is_cuda
29
+ and torch.is_autocast_enabled("cuda")
30
+ and not torch.is_grad_enabled()
31
+ and x.dtype in {torch.float16, torch.bfloat16, torch.float32}
32
+ and x.shape[-1:] == self.normalized_shape
33
+ and (
34
+ self.weight is None
35
+ or self.weight.dtype
36
+ in {torch.float16, torch.bfloat16, torch.float32}
37
+ )
38
+ ):
39
+ cos_sin: tuple[Tensor, Tensor] | None = None
40
+ if rope is not None:
41
+ if (
42
+ not torch.compiler.is_compiling()
43
+ and rope.layout == "split_half"
44
+ and rope.rotary_channels == rope.channels == x.size(-1)
45
+ ):
46
+ seq = torch.arange(
47
+ x.size(-3), device=x.device, dtype=torch.float32
48
+ )
49
+ freq = seq.view(-1, 1) * rope.inv_freq.view(1, -1)
50
+ cos_sin = (freq.cos().to(x.dtype), freq.sin().to(x.dtype))
51
+ else:
52
+ x = rope(x)
53
+ eps = (
54
+ self.eps
55
+ if self.eps is not None
56
+ else torch.finfo(torch.float32).eps
57
+ )
58
+ return rmsnorm_cast(x, self.weight, eps, rope=cos_sin)
59
+ return super().forward(x if rope is None else rope(x))
sdm/nn/rope.py ADDED
@@ -0,0 +1,107 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D205
5
+
6
+ from typing import Any, Literal
7
+
8
+ import torch
9
+ from torch import Tensor
10
+
11
+
12
+ class RotaryEmbedding(torch.nn.Module):
13
+ r"""Rotary Positional Embedding (RoPE) from the `"RoFormer: Enhanced
14
+ Transformer with Rotary Position Embedding"
15
+ <https://arxiv.org/abs/2104.09864>`_ paper.
16
+
17
+ Args:
18
+ channels: The number of channels per attention head.
19
+ layout: The channel pairing layout. ``"split_half"`` pairs the first
20
+ half of the channels with the second half. ``"interleaved"`` pairs
21
+ adjacent even and odd channels.
22
+ theta: The base frequency used to initialize inverse frequencies.
23
+ requires_grad: Whether inverse frequencies are learnable.
24
+ partial_rotary_factor: The fraction of leading channels to which RoPE
25
+ is applied.
26
+ device: The device.
27
+ dtype: The dtype.
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ channels: int,
33
+ layout: Literal["split_half", "interleaved"],
34
+ theta: float = 100_000,
35
+ requires_grad: bool = True,
36
+ partial_rotary_factor: float = 1.0,
37
+ device: torch.device | str | None = None,
38
+ dtype: torch.dtype | None = None,
39
+ ) -> None:
40
+ super().__init__()
41
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
42
+
43
+ self.layout = layout
44
+ self.channels = channels
45
+ self.rotary_channels = int(channels * partial_rotary_factor)
46
+
47
+ if self.channels % 2 != 0:
48
+ raise ValueError(f"'channels' must be even (got {self.channels})")
49
+ if (
50
+ self.rotary_channels < 2
51
+ or self.rotary_channels > channels
52
+ or self.rotary_channels % 2 != 0
53
+ ):
54
+ raise ValueError(
55
+ f"'partial_rotary_factor' must produce an even number between "
56
+ f"2 and {self.channels} (got {self.rotary_channels})"
57
+ )
58
+
59
+ arange = torch.arange(0, self.rotary_channels, 2, **factory_kwargs)
60
+ inv_freq = 1.0 / (theta ** (arange / self.rotary_channels))
61
+ self.inv_freq = torch.nn.Parameter(
62
+ inv_freq,
63
+ requires_grad=requires_grad,
64
+ )
65
+
66
+ def forward(
67
+ self,
68
+ x: Tensor, # [..., S, H, C]
69
+ ) -> Tensor: # [..., S, H, C]
70
+ """The forward pass.
71
+
72
+ Args:
73
+ x: Tensor with shape ``[..., S, H, C]``.
74
+ ``S`` is the query sequence length, ``H`` is the number of
75
+ attention heads, and ``C`` is the channels per head.
76
+
77
+ Returns:
78
+ Tensor with shape ``[..., S, H, C]``.
79
+ """
80
+ if x.size(-1) != self.channels:
81
+ raise ValueError(
82
+ f"Expected {self.channels} channels (got {x.size(-1)})"
83
+ )
84
+ seq = torch.arange(x.size(-3), device=x.device, dtype=torch.float32)
85
+ freq = seq.view(-1, 1) * self.inv_freq.view(1, -1) # [S, C // 2]
86
+ sin = freq.sin()[:, None, :].to(x.dtype) # [S, 1, C // 2]
87
+ cos = freq.cos()[:, None, :].to(x.dtype) # [S, 1, C // 2]
88
+
89
+ rotary = x[..., : self.rotary_channels]
90
+ if self.layout == "interleaved":
91
+ x1, x2 = rotary[..., 0::2], rotary[..., 1::2]
92
+ else:
93
+ assert self.layout == "split_half"
94
+ x1, x2 = rotary.split(sin.size(-1), dim=-1)
95
+
96
+ out1 = x1 * cos - x2 * sin
97
+ out2 = x2 * cos + x1 * sin
98
+
99
+ if self.layout == "interleaved":
100
+ rotary = torch.stack((out1, out2), dim=-1).flatten(-2)
101
+ else:
102
+ assert self.layout == "split_half"
103
+ rotary = torch.cat((out1, out2), dim=-1)
104
+
105
+ if self.rotary_channels == self.channels:
106
+ return rotary
107
+ return torch.cat((rotary, x[..., self.rotary_channels :]), dim=-1)
sdm/nn/scaling.py ADDED
@@ -0,0 +1,240 @@
1
+ # Portions derived from TabICLv2:
2
+ # Copyright (c) 2025, Soda team @ Inria
3
+ # Licensed under the BSD 3-Clause License; see sdm/models/tabiclv2/LICENSE.
4
+
5
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6
+ # SPDX-License-Identifier: Apache-2.0
7
+
8
+ import abc
9
+ from typing import Any, cast
10
+
11
+ import torch
12
+ from torch import Tensor
13
+ from torch.nn import GELU, Linear, Parameter, Sequential
14
+
15
+
16
+ class QueryScaling(torch.nn.Module, abc.ABC):
17
+ r"""Base class for query scaling modules in :class:`SDPA`.
18
+
19
+ Query scaling modules transform projected query heads before scaled
20
+ dot-product attention. They may use the effective key length to implement
21
+ length-aware temperature scaling while preserving compatibility with
22
+ :func:`torch.nn.functional.scaled_dot_product_attention`.
23
+ """
24
+
25
+ def __init__(self) -> None:
26
+ super().__init__()
27
+
28
+ @abc.abstractmethod
29
+ def forward(
30
+ self,
31
+ query: Tensor, # [..., S, H, C]
32
+ *,
33
+ key_len: Tensor | int, # [..., 1] or [..., S] or scalar
34
+ ) -> Tensor: # [..., S, H, C]
35
+ r"""The forward pass.
36
+
37
+ Args:
38
+ query: The query tensor to scale, with shape ``[..., S, H, C]``.
39
+ ``S`` is the query sequence length, ``H`` is the number of
40
+ attention heads, and ``C`` is the channels per head.
41
+ key_len: The number of valid keys used to scale each query.
42
+ An ``int`` denotes one shared key length for every query.
43
+ A tensor shaped ``[..., 1]`` denotes one key length per batch
44
+ item, shared across all query positions in ``S``.
45
+ A tensor shaped ``[..., S]`` denotes one individual key length
46
+ per query position.
47
+
48
+ Returns:
49
+ Tensor with shape ``[..., S, H, C]``.
50
+ """
51
+
52
+
53
+ class QASSMax(QueryScaling):
54
+ r"""Query-Aware Scalable SoftMax (QASSMax).
55
+
56
+ This scaling method was introduced in the `"TabICLv2: A better, faster,
57
+ scalable, and open tabular foundation model"
58
+ <https://arxiv.org/abs/2602.11139>`_ paper as a temperature-scaling method
59
+ for attention.
60
+
61
+ For a query tensor :math:`q` and key length :math:`n`, this module returns
62
+ a scaled query
63
+
64
+ .. math::
65
+
66
+ \tilde{q}_{hi} = q_{hi} \cdot
67
+ \mathrm{MLP}_{\mathrm{base}}(\log n)_{hi} \cdot
68
+ \left(1 + \tanh(\mathrm{MLP}_{\mathrm{gate}}(q_h)_i)\right),
69
+
70
+ where :math:`h` indexes attention heads, :math:`i` indexes head channels,
71
+ :math:`\mathrm{MLP}_{\mathrm{base}}` maps the log key length to per-head,
72
+ per-channel scale factors, and :math:`\mathrm{MLP}_{\mathrm{gate}}` maps
73
+ each per-head query vector to a bounded query-dependent gate.
74
+
75
+ Multiplying the query scales the subsequent attention logits while keeping
76
+ the attention computation compatible with standard softmax kernels.
77
+ The length-dependent factor counteracts attention fading as the number of
78
+ keys grows, while the query-dependent gate lets the scale vary across
79
+ queries.
80
+
81
+ Args:
82
+ channels: The number of channels per attention head.
83
+ num_heads: The number of query attention heads.
84
+ hidden_channels: The hidden width of the scale and gate MLPs.
85
+ device: The device.
86
+ dtype: The dtype.
87
+ """
88
+
89
+ def __init__(
90
+ self,
91
+ channels: int,
92
+ num_heads: int,
93
+ hidden_channels: int = 64,
94
+ device: torch.device | str | None = None,
95
+ dtype: torch.dtype | None = None,
96
+ ) -> None:
97
+ super().__init__()
98
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
99
+
100
+ self.scale = Sequential(
101
+ Linear(1, hidden_channels, **factory_kwargs),
102
+ GELU(),
103
+ Linear(hidden_channels, num_heads * channels, **factory_kwargs),
104
+ )
105
+ self.gate = Sequential(
106
+ Linear(channels, hidden_channels, **factory_kwargs),
107
+ GELU(),
108
+ Linear(hidden_channels, channels, **factory_kwargs),
109
+ )
110
+
111
+ torch.nn.init.zeros_(cast(Linear, self.gate[-1]).weight)
112
+ torch.nn.init.zeros_(cast(Linear, self.gate[-1]).bias)
113
+
114
+ def forward(
115
+ self,
116
+ query: Tensor, # [..., S, H, C]
117
+ *,
118
+ key_len: Tensor | int, # [..., 1] or [..., S] or scalar
119
+ ) -> Tensor: # [..., S, H, C]
120
+ r""":meta private:""" # noqa: D415
121
+ if isinstance(key_len, Tensor):
122
+ log_key_len = key_len.float().clamp(min=1.0).log().to(query.dtype)
123
+ else:
124
+ one = query.new_ones((), dtype=torch.float32)
125
+ log_key_len = (one * key_len).clamp(min=1.0).log().to(query.dtype)
126
+ log_key_len = log_key_len.view([1] * (query.dim() - 2))
127
+ scale = self.scale(log_key_len.unsqueeze(-1)) # [..., S or 1, H * C]
128
+ scale = scale.unflatten(-1, (query.size(-2), query.size(-1)))
129
+ gate = 1 + self.gate(query).tanh()
130
+ return query * scale * gate
131
+
132
+
133
+ class LogScale(QueryScaling):
134
+ r"""Logarithmic query scaling.
135
+
136
+ For a query tensor :math:`q`, effective key length :math:`n`, and learned
137
+ per-head coefficient :math:`a_h`, this module returns a scaled query
138
+
139
+ .. math::
140
+
141
+ \tilde{q}_{hi} = q_{hi} \cdot a_h \cdot \log(\max(n, 1)),
142
+
143
+ where :math:`h` indexes attention heads and :math:`i` indexes head
144
+ channels.
145
+
146
+ Each attention head receives an unconstrained coefficient initialized
147
+ independently and uniformly in ``[0, 1)``.
148
+
149
+ Args:
150
+ num_heads: The number of query attention heads.
151
+ device: The device.
152
+ dtype: The dtype.
153
+ """
154
+
155
+ def __init__(
156
+ self,
157
+ num_heads: int,
158
+ device: torch.device | str | None = None,
159
+ dtype: torch.dtype | None = None,
160
+ ) -> None:
161
+ super().__init__()
162
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
163
+ self.head_scale = Parameter(torch.rand((num_heads,), **factory_kwargs))
164
+
165
+ def forward(
166
+ self,
167
+ query: Tensor, # [..., S, H, C]
168
+ *,
169
+ key_len: Tensor | int, # [..., 1] or [..., S] or scalar
170
+ ) -> Tensor: # [..., S, H, C]
171
+ r""":meta private:""" # noqa: D415
172
+ # Avoid a silently wrong result from broadcasting the head dimension.
173
+ if query.size(-2) != self.head_scale.numel():
174
+ raise ValueError(
175
+ "`query.size(-2)` must equal the number of per-head scales "
176
+ f"(got {query.size(-2)} and {self.head_scale.numel()})"
177
+ )
178
+ if isinstance(key_len, Tensor):
179
+ key_len_tensor = key_len.float()
180
+ else:
181
+ key_len_tensor = query.new_full((), key_len, dtype=torch.float32)
182
+ log_key_len = key_len_tensor.clamp(min=1.0).log().to(query.dtype)
183
+ head_scale = self.head_scale.to(query.dtype).reshape(
184
+ *((1,) * (query.dim() - 2)), -1, 1
185
+ )
186
+ return query * log_key_len[..., None, None] * head_scale
187
+
188
+
189
+ class GatedLogScale(LogScale):
190
+ r"""Gated logarithmic query scaling.
191
+
192
+ For a query tensor :math:`q`, effective key length :math:`n`, learned
193
+ per-head coefficient :math:`a_h`, and learned gate :math:`g`, this module
194
+ returns
195
+
196
+ .. math::
197
+
198
+ \tilde{q}_{hi} = q_{hi} \cdot a_h \cdot \log(\max(n, 1))
199
+ \cdot \left(1 + \tanh(g(q_h)_i)\right).
200
+
201
+ The multiplicative gate is initialized to one and bounded between zero and
202
+ two. Per-head coefficients are initialized independently and uniformly in
203
+ ``[0, 1)``.
204
+
205
+ Args:
206
+ channels: The number of channels per attention head.
207
+ num_heads: The number of query attention heads.
208
+ hidden_channels: The hidden width of the gate MLP.
209
+ device: The device.
210
+ dtype: The dtype.
211
+ """
212
+
213
+ def __init__(
214
+ self,
215
+ channels: int,
216
+ num_heads: int,
217
+ hidden_channels: int,
218
+ device: torch.device | str | None = None,
219
+ dtype: torch.dtype | None = None,
220
+ ) -> None:
221
+ super().__init__(num_heads=num_heads, device=device, dtype=dtype)
222
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
223
+
224
+ self.gate = Sequential(
225
+ Linear(channels, hidden_channels, **factory_kwargs),
226
+ GELU(),
227
+ Linear(hidden_channels, channels, **factory_kwargs),
228
+ )
229
+ torch.nn.init.zeros_(cast(Linear, self.gate[-1]).weight)
230
+ torch.nn.init.zeros_(cast(Linear, self.gate[-1]).bias)
231
+
232
+ def forward(
233
+ self,
234
+ query: Tensor, # [..., S, H, C]
235
+ *,
236
+ key_len: Tensor | int, # [..., 1] or [..., S] or scalar
237
+ ) -> Tensor: # [..., S, H, C]
238
+ r""":meta private:""" # noqa: D415
239
+ query_scaled = super().forward(query=query, key_len=key_len)
240
+ return query_scaled * (1 + self.gate(query).tanh())
@@ -0,0 +1,162 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ r"""Set-transformer modules for structured tensor models."""
5
+
6
+ from __future__ import annotations
7
+
8
+ from typing import Any, Literal, overload
9
+
10
+ import torch
11
+ from torch import Tensor
12
+ from torch.nn import Parameter
13
+
14
+ from sdm.cache import KVCacheEntry
15
+ from sdm.nn import TransformerBlock
16
+
17
+
18
+ class InducedTransformerBlock(torch.nn.Module):
19
+ r"""Transformer block using learned inducing points.
20
+
21
+ Introduced in the `"Set Transformer: A Framework for Attention-based
22
+ Permutation-Invariant Neural Networks"
23
+ <https://arxiv.org/abs/1810.00825>`_ paper, the block routes attention
24
+ through a small set of :math:`M` learned inducing points, reducing the cost
25
+ of attending a query of size :math:`Q` to a key/value context of size
26
+ :math:`K` from :math:`O(Q \cdot K)` to :math:`O((Q + K) \cdot M)`:
27
+
28
+ .. math::
29
+
30
+ H = \mathrm{InducingBlock}_1(I, \mathrm{key\_value}), \quad
31
+ \mathrm{out} = \mathrm{OutputBlock}_2(\mathrm{query}, H),
32
+
33
+ where :math:`I` are the learned inducing points and :math:`H` are the
34
+ inducing points after attending to the key/value elements. Passing
35
+ ``key_value=None`` recovers the induced self-attention block (ISAB).
36
+
37
+ Args:
38
+ channels: The number of input and output channels.
39
+ num_inducing_points: The number of learned inducing points :math:`M`.
40
+ inducing_block: The :class:`TransformerBlock` that updates the learned
41
+ inducing points from key/value context.
42
+ output_block: The :class:`TransformerBlock` that updates the input
43
+ queries from the induced context.
44
+ device: The device.
45
+ dtype: The dtype.
46
+ """
47
+
48
+ def __init__(
49
+ self,
50
+ channels: int,
51
+ num_inducing_points: int,
52
+ inducing_block: TransformerBlock,
53
+ output_block: TransformerBlock,
54
+ device: torch.device | str | None = None,
55
+ dtype: torch.dtype | None = None,
56
+ ) -> None:
57
+ super().__init__()
58
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
59
+
60
+ self.inducing_block = inducing_block
61
+ self.output_block = output_block
62
+
63
+ self.inducing_points = Parameter(
64
+ torch.empty(num_inducing_points, channels, **factory_kwargs)
65
+ )
66
+ torch.nn.init.trunc_normal_(self.inducing_points, std=0.02)
67
+
68
+ @overload
69
+ def forward(
70
+ self,
71
+ query: Tensor,
72
+ key_value: Tensor | KVCacheEntry | None = None,
73
+ seqused_key_value: Tensor | None = None,
74
+ attn_mask: Tensor | None = None,
75
+ *,
76
+ return_key_value: Literal[False] = False,
77
+ batch_size_limit: int | Literal["auto"] | None = None,
78
+ out: Tensor | None = None,
79
+ ) -> Tensor: ...
80
+
81
+ @overload
82
+ def forward(
83
+ self,
84
+ query: Tensor,
85
+ key_value: Tensor | KVCacheEntry | None = None,
86
+ seqused_key_value: Tensor | None = None,
87
+ attn_mask: Tensor | None = None,
88
+ *,
89
+ return_key_value: Literal[True],
90
+ batch_size_limit: int | Literal["auto"] | None = None,
91
+ out: Tensor | None = None,
92
+ ) -> tuple[Tensor, KVCacheEntry]: ...
93
+
94
+ @overload
95
+ def forward(
96
+ self,
97
+ query: Tensor,
98
+ key_value: Tensor | KVCacheEntry | None = None,
99
+ seqused_key_value: Tensor | None = None,
100
+ attn_mask: Tensor | None = None,
101
+ *,
102
+ return_key_value: bool,
103
+ batch_size_limit: int | Literal["auto"] | None = None,
104
+ out: Tensor | None = None,
105
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
106
+
107
+ def forward(
108
+ self,
109
+ query: Tensor, # [..., Q, C]
110
+ key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
111
+ seqused_key_value: Tensor | None = None, # [...]
112
+ attn_mask: Tensor | None = None, # [..., KV]
113
+ *,
114
+ return_key_value: bool = False,
115
+ batch_size_limit: int | Literal["auto"] | None = None,
116
+ out: Tensor | None = None,
117
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
118
+ r"""The forward pass.
119
+
120
+ Args:
121
+ query: The query tensor with shape ``[..., Q, C]``.
122
+ ``Q`` is the query sequence length, ``C`` is the number of
123
+ channels.
124
+ key_value: The key/value tensor with shape ``[..., KV, C]`` or
125
+ precomputed key/value projections as a
126
+ :class:`~sdm.cache.KVCacheEntry`.
127
+ ``KV`` is the key/value sequence length.
128
+ If omitted, ``query`` is used for induced self-attention.
129
+ seqused_key_value: Valid key/value lengths with shape ``[...]`` and
130
+ :external+torch:ref:`torch.int32 <dtype-doc>` dtype.
131
+ attn_mask: Boolean attention mask with shape ``[..., KV]``.
132
+ Entries set to ``True`` participate in attention.
133
+ return_key_value: Whether to return the computed key and value
134
+ projections for the final attention site alongside the output.
135
+ batch_size_limit: Maximum number of batch elements processed at
136
+ once.
137
+ out: The output tensor.
138
+
139
+ Returns:
140
+ Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
141
+ ``False``. Otherwise, a tuple of the output tensor and a
142
+ :class:`~sdm.cache.KVCacheEntry`.
143
+ """
144
+ if not isinstance(key_value, KVCacheEntry):
145
+ if key_value is None:
146
+ key_value = query
147
+ if attn_mask is not None:
148
+ attn_mask = attn_mask.unsqueeze(-2) # [..., 1, KV]
149
+ key_value = self.inducing_block(
150
+ query=self.inducing_points, # [M, C]
151
+ key_value=key_value, # [..., KV, C]
152
+ seqused_key_value=seqused_key_value, # [...]
153
+ attn_mask=attn_mask, # [..., 1, KV]
154
+ batch_size_limit=batch_size_limit,
155
+ ) # [..., M, C]
156
+ return self.output_block(
157
+ query=query, # [..., Q, C]
158
+ key_value=key_value, # [..., M, C]
159
+ return_key_value=return_key_value,
160
+ batch_size_limit=batch_size_limit,
161
+ out=out,
162
+ ) # [..., Q, C]