structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/attention.py ADDED
@@ -0,0 +1,825 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Attention modules for structured tensor models."""
5
+
6
+ import math
7
+ from typing import Any, Literal, overload
8
+
9
+ import torch
10
+ import torch.nn.functional as F
11
+ from torch import Tensor
12
+ from torch.nn import Linear
13
+
14
+ from sdm._memory import chunk_memory_limit
15
+ from sdm.cache import KVCacheEntry
16
+ from sdm.nn import QueryScaling
17
+
18
+
19
+ class SDPA(torch.nn.Module):
20
+ r"""Scaled Dot-Product Attention (SDPA).
21
+
22
+ This module wraps :func:`torch.nn.functional.scaled_dot_product_attention`
23
+ and extends it by arbitrary batch dimensions, optional inference-time
24
+ batch chunking, query-scaling, and padding support for key/value pairs.
25
+
26
+ Args:
27
+ num_query_heads: The number of query attention heads.
28
+ num_key_value_heads: The number of key/value attention heads.
29
+ Setting this below ``num_query_heads`` enables grouped-query
30
+ attention (GQA); setting it to ``1`` enables multi-query attention
31
+ (MQA). Must divide ``num_query_heads``. Defaults to
32
+ ``num_query_heads`` (standard multi-head attention).
33
+ query_scaling: Query scaling module to scale projected query heads
34
+ before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
35
+ scale: Scaling factor passed to
36
+ :func:`torch.nn.functional.scaled_dot_product_attention`.
37
+ ``None`` uses the default value of ``1 / sqrt(channels)``.
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ num_query_heads: int,
43
+ num_key_value_heads: int | None = None,
44
+ query_scaling: QueryScaling | None = None,
45
+ scale: float | None = None,
46
+ ) -> None:
47
+ super().__init__()
48
+ if num_key_value_heads is None:
49
+ num_key_value_heads = num_query_heads
50
+ if num_query_heads % num_key_value_heads != 0:
51
+ raise ValueError(
52
+ f"`num_query_heads` ({num_query_heads}) must be divisible by "
53
+ f"`num_key_value_heads` ({num_key_value_heads})"
54
+ )
55
+
56
+ self.num_query_heads = num_query_heads
57
+ self.num_key_value_heads = num_key_value_heads
58
+ self.query_scaling = query_scaling
59
+ self.scale = scale
60
+
61
+ def forward(
62
+ self,
63
+ query: Tensor, # [..., Q, Hq, C]
64
+ key: Tensor, # [..., KV, Hkv, C]
65
+ value: Tensor, # [..., KV, Hkv, C]
66
+ seqused_key_value: Tensor | None = None, # [...]
67
+ attn_mask: Tensor | None = None, # [..., Q, KV]
68
+ ) -> Tensor: # [..., Q, Hq, C]
69
+ r"""The forward pass.
70
+
71
+ Args:
72
+ query: The query tensor with shape ``[..., Q, Hq, C]``.
73
+ ``Q`` is the query sequence length, ``Hq`` is the number of
74
+ query attention heads (``num_query_heads``), and ``C`` is the
75
+ channels per head.
76
+ key: The key tensor with shape ``[..., KV, Hkv, C]``.
77
+ ``KV`` is the key/value sequence length and ``Hkv`` is the
78
+ number of key/value heads (``num_key_value_heads``).
79
+ value: The value tensor with shape ``[..., KV, Hkv, C]``.
80
+ seqused_key_value: Valid key/value lengths with shape ``[...]`` and
81
+ :external+torch:ref:`torch.int32 <dtype-doc>` dtype.
82
+ attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
83
+ Entries set to ``True`` participate in attention.
84
+
85
+ Returns:
86
+ Tensor with shape ``[..., Q, Hq, C]``.
87
+ """
88
+ if query.numel() == 0:
89
+ return query
90
+
91
+ if attn_mask is not None and seqused_key_value is not None:
92
+ raise ValueError(
93
+ "Cannot pass both `attn_mask` and `seqused_key_value`"
94
+ )
95
+
96
+ if (
97
+ seqused_key_value is not None
98
+ and seqused_key_value.dtype != torch.int32
99
+ ):
100
+ raise ValueError("`seqused_key_value` must have dtype torch.int32")
101
+ if attn_mask is not None and attn_mask.dtype != torch.bool:
102
+ raise ValueError("`attn_mask` must have dtype torch.bool")
103
+
104
+ batch_shapes = [query.size()[:-3], key.size()[:-3], value.size()[:-3]]
105
+ if seqused_key_value is not None:
106
+ batch_shapes.append(seqused_key_value.size())
107
+ if attn_mask is not None:
108
+ batch_shapes.append(attn_mask.size()[:-2])
109
+ batch_shape = torch.broadcast_shapes(*batch_shapes)
110
+
111
+ if self.query_scaling is not None:
112
+ if seqused_key_value is not None:
113
+ key_len = seqused_key_value.unsqueeze(-1)
114
+ elif attn_mask is not None and attn_mask.size(-1) > 1:
115
+ key_len = attn_mask.sum(dim=-1)
116
+ else:
117
+ key_len = key.size(-3)
118
+ query = self.query_scaling(query, key_len=key_len)
119
+
120
+ # Broadcast and flatten batch dimensions => [B, S, H, C].
121
+ query_size = query.size()[-3:]
122
+ key_size = key.size()[-3:]
123
+ value_size = value.size()[-3:]
124
+
125
+ # FlashAttention rejects empty batches before PyTorch 2.10.
126
+ if key_size[0] == 0 or 0 in batch_shape:
127
+ out = query.new_zeros(
128
+ batch_shape + query_size[:-1] + value_size[-1:]
129
+ )
130
+ if torch.is_grad_enabled():
131
+ # Summing an empty slice gives zero and keeps backward working.
132
+ for tensor in (query, key, value):
133
+ if tensor.requires_grad:
134
+ out = out + tensor[..., :0, :, :].sum()
135
+ return out
136
+
137
+ query = query.expand(batch_shape + query_size).reshape(-1, *query_size)
138
+ key = key.expand(batch_shape + key_size).reshape(-1, *key_size)
139
+ value = value.expand(batch_shape + value_size).reshape(-1, *value_size)
140
+
141
+ if attn_mask is not None:
142
+ attn_mask = attn_mask.expand(batch_shape + attn_mask.size()[-2:])
143
+ attn_mask = attn_mask.reshape(-1, *attn_mask.size()[-2:])
144
+
145
+ if seqused_key_value is not None:
146
+ seqused_key_value = seqused_key_value.expand(batch_shape)
147
+ seqused_key_value = seqused_key_value.reshape(-1).unsqueeze(-1)
148
+ key_index = torch.arange(key.size(-3), device=key.device)
149
+ attn_mask = key_index.unsqueeze(0) < seqused_key_value
150
+ attn_mask = attn_mask.unsqueeze(-2).expand(-1, query.size(-3), -1)
151
+
152
+ enable_gqa = False
153
+ if query.size(-2) != key.size(-2):
154
+ enable_gqa = True
155
+
156
+ out = F.scaled_dot_product_attention(
157
+ query=query.transpose(-3, -2), # [B, Hq, Q, C],
158
+ key=key.transpose(-3, -2), # [B, Hkv, KV, C],
159
+ value=value.transpose(-3, -2), # [B, Hkv, KV, C],
160
+ attn_mask=attn_mask.unsqueeze(-3) # [B, 1, Q, KV]
161
+ if attn_mask is not None
162
+ else None,
163
+ enable_gqa=enable_gqa,
164
+ scale=self.scale,
165
+ ).transpose(-3, -2) # [B, Q, Hq, C]
166
+
167
+ return out.view(batch_shape + out.size()[-3:]) # [..., Q, Hq, C]
168
+
169
+
170
+ class Attention(torch.nn.Module):
171
+ r"""Multi-head attention layer with grouped-query attention support.
172
+
173
+ This module owns the query, key, value, and output projections.
174
+ It performs self-attention when ``key_value`` is omitted and
175
+ cross-attention when ``key_value`` is given.
176
+
177
+ Args:
178
+ channels: The number of input and output channels.
179
+ num_query_heads: The number of query attention heads.
180
+ ``channels`` must be divisible by ``num_query_heads``.
181
+ num_key_value_heads: The number of key/value attention heads.
182
+ Setting this below ``num_query_heads`` enables grouped-query
183
+ attention (GQA); setting it to ``1`` enables multi-query attention
184
+ (MQA). Must divide ``num_query_heads``. Defaults to
185
+ ``num_query_heads`` (standard multi-head attention).
186
+ query_transform: Transformation applied to projected query heads before
187
+ scaled dot-product attention.
188
+ key_transform: Transformation applied to projected key heads before
189
+ scaled dot-product attention.
190
+ query_scaling: Query scaling module to scale projected query heads
191
+ before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
192
+ scale: Scaling factor passed to
193
+ :func:`torch.nn.functional.scaled_dot_product_attention`.
194
+ ``None`` uses ``1 / sqrt(channels_per_head)``.
195
+ bias: If set to ``False``, the module will not learn an additive bias.
196
+ device: The device.
197
+ dtype: The dtype.
198
+ """
199
+
200
+ def __init__(
201
+ self,
202
+ channels: int,
203
+ num_query_heads: int,
204
+ num_key_value_heads: int | None = None,
205
+ query_transform: torch.nn.Module | None = None,
206
+ key_transform: torch.nn.Module | None = None,
207
+ query_scaling: QueryScaling | None = None,
208
+ scale: float | None = None,
209
+ bias: bool = True,
210
+ device: torch.device | str | None = None,
211
+ dtype: torch.dtype | None = None,
212
+ ) -> None:
213
+ super().__init__()
214
+ if num_key_value_heads is None:
215
+ num_key_value_heads = num_query_heads
216
+ if channels % num_query_heads != 0:
217
+ raise ValueError(
218
+ f"`channels` ({channels}) must be divisible by "
219
+ f"`num_query_heads` ({num_query_heads})"
220
+ )
221
+
222
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
223
+
224
+ self.num_query_heads = num_query_heads
225
+ self.num_key_value_heads = num_key_value_heads
226
+ self.head_dim = channels // num_query_heads
227
+ # Query projection spans all channels; key/value span fewer heads.
228
+ self.q_dim = num_query_heads * self.head_dim # == channels
229
+ self.kv_dim = num_key_value_heads * self.head_dim
230
+
231
+ self.qkv_lin = Linear(
232
+ channels, self.q_dim + 2 * self.kv_dim, bias=bias, **factory_kwargs
233
+ )
234
+ self.query_transform = query_transform
235
+ self.key_transform = key_transform
236
+ self.sdpa = SDPA(
237
+ num_query_heads=num_query_heads,
238
+ num_key_value_heads=num_key_value_heads,
239
+ query_scaling=query_scaling,
240
+ scale=scale,
241
+ )
242
+ self.out_lin = Linear(channels, channels, bias=bias, **factory_kwargs)
243
+
244
+ torch.nn.init.zeros_(self.out_lin.weight)
245
+ if self.out_lin.bias is not None:
246
+ torch.nn.init.zeros_(self.out_lin.bias)
247
+
248
+ @overload
249
+ def forward(
250
+ self,
251
+ query: Tensor,
252
+ key_value: Tensor | KVCacheEntry | None = None,
253
+ seqused_key_value: Tensor | None = None,
254
+ attn_mask: Tensor | None = None,
255
+ *,
256
+ return_key_value: Literal[False] = False,
257
+ ) -> Tensor: ...
258
+
259
+ @overload
260
+ def forward(
261
+ self,
262
+ query: Tensor,
263
+ key_value: Tensor | KVCacheEntry | None = None,
264
+ seqused_key_value: Tensor | None = None,
265
+ attn_mask: Tensor | None = None,
266
+ *,
267
+ return_key_value: Literal[True],
268
+ ) -> tuple[Tensor, KVCacheEntry]: ...
269
+
270
+ @overload
271
+ def forward(
272
+ self,
273
+ query: Tensor,
274
+ key_value: Tensor | KVCacheEntry | None = None,
275
+ seqused_key_value: Tensor | None = None,
276
+ attn_mask: Tensor | None = None,
277
+ *,
278
+ return_key_value: bool,
279
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
280
+
281
+ def forward(
282
+ self,
283
+ query: Tensor, # [..., Q, C]
284
+ key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
285
+ seqused_key_value: Tensor | None = None, # [...]
286
+ attn_mask: Tensor | None = None, # [..., Q, KV]
287
+ *,
288
+ return_key_value: bool = False,
289
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
290
+ r"""The forward pass.
291
+
292
+ Args:
293
+ query: The query tensor with shape ``[..., Q, C]``.
294
+ ``Q`` is the query sequence length, ``C`` is the number of
295
+ channels.
296
+ key_value: The key/value tensor with shape ``[..., KV, C]`` or
297
+ precomputed key/value projections as a
298
+ :class:`~sdm.cache.KVCacheEntry`.
299
+ ``KV`` is the key/value sequence length.
300
+ If omitted, ``query`` is used for self-attention.
301
+ seqused_key_value: Valid key/value lengths with shape ``[...]`` and
302
+ :external+torch:ref:`torch.int32 <dtype-doc>` dtype.
303
+ attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
304
+ Entries set to ``True`` participate in attention.
305
+ return_key_value: Whether to return the computed key and value
306
+ projections alongside the attention output.
307
+
308
+ Returns:
309
+ Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
310
+ ``False``.
311
+ Otherwise, a tuple of the output tensor and a
312
+ :class:`~sdm.cache.KVCacheEntry`.
313
+ """
314
+ if isinstance(key_value, KVCacheEntry):
315
+ query = F.linear(
316
+ query,
317
+ weight=self.qkv_lin.weight[: self.q_dim],
318
+ bias=self.qkv_lin.bias[: self.q_dim]
319
+ if self.qkv_lin.bias is not None
320
+ else None,
321
+ )
322
+ if (
323
+ key_value.key.dtype != query.dtype
324
+ or key_value.value.dtype != query.dtype
325
+ ):
326
+ raise ValueError(
327
+ f"Key/value projections were cached under dtypes "
328
+ f"'{key_value.key.dtype}'/'{key_value.value.dtype}' but "
329
+ f"the query has dtype '{query.dtype}'"
330
+ )
331
+ key = key_value.key
332
+ value = key_value.value
333
+ elif key_value is None:
334
+ query, key, value = self.qkv_lin(query).split(
335
+ [self.q_dim, self.kv_dim, self.kv_dim], dim=-1
336
+ )
337
+ else:
338
+ sections = [self.q_dim, 2 * self.kv_dim]
339
+ q_weight, kv_weight = self.qkv_lin.weight.split(sections, dim=0)
340
+ q_bias = kv_bias = None
341
+ if self.qkv_lin.bias is not None:
342
+ q_bias, kv_bias = self.qkv_lin.bias.split(sections, dim=0)
343
+ query = F.linear(query, q_weight, q_bias)
344
+ key, value = F.linear(key_value, kv_weight, kv_bias).chunk(2, -1)
345
+
346
+ # [..., S, C] -> [..., S, H, C // H], with separate query/kv heads.
347
+ query = query.unflatten(-1, [self.num_query_heads, self.head_dim])
348
+ if not isinstance(key_value, KVCacheEntry):
349
+ key = key.unflatten(-1, [self.num_key_value_heads, self.head_dim])
350
+ value = value.unflatten(
351
+ -1, [self.num_key_value_heads, self.head_dim]
352
+ )
353
+
354
+ if self.query_transform is not None:
355
+ query = self.query_transform(query)
356
+ if (
357
+ not isinstance(key_value, KVCacheEntry)
358
+ and self.key_transform is not None
359
+ ):
360
+ key = self.key_transform(key)
361
+
362
+ out = self.sdpa(
363
+ query=query, # [..., Q, Hq, C // Hq]
364
+ key=key, # [..., KV, Hkv, C // Hq]
365
+ value=value, # [..., KV, Hkv, C // Hq]
366
+ seqused_key_value=seqused_key_value, # [...]
367
+ attn_mask=attn_mask, # [..., Q, KV]
368
+ ) # [..., Q, Hq, C // Hq]
369
+
370
+ out = out.flatten(-2, -1) # [..., Q, C]
371
+ out = self.out_lin(out) # [..., Q, C]
372
+ if return_key_value:
373
+ # CUDA autocast runs RMSNorm in fp32, so cast explicitly before
374
+ # caching: https://github.com/pytorch/pytorch/blob/v2.13.0/aten/src/ATen/autocast_mode.h#L875
375
+ return out, KVCacheEntry(
376
+ key=key.to(value.dtype),
377
+ value=value,
378
+ )
379
+ return out
380
+
381
+
382
+ class TransformerBlock(torch.nn.Module):
383
+ r"""Transformer block with normalization and feedforward residual modules.
384
+
385
+ Args:
386
+ channels: The number of input and output channels.
387
+ num_query_heads: The number of query attention heads.
388
+ mlp: Feedforward module applied after the attention residual.
389
+ num_key_value_heads: The number of key/value attention heads.
390
+ Defaults to ``num_query_heads`` (standard multi-head attention).
391
+ query_norm: Normalization applied to query inputs before attention.
392
+ key_value_norm: Normalization applied to key/value inputs before
393
+ attention.
394
+ post_attn_norm: Normalization applied to the attention output before
395
+ its residual addition.
396
+ query_transform: Transformation applied to projected query heads before
397
+ scaled dot-product attention.
398
+ key_transform: Transformation applied to projected key heads before
399
+ scaled dot-product attention.
400
+ query_scaling: Query scaling module to scale projected query heads
401
+ before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
402
+ scale: Scaling factor passed to
403
+ :func:`torch.nn.functional.scaled_dot_product_attention`.
404
+ ``None`` uses ``1 / sqrt(channels_per_head)``.
405
+ bias: If set to ``False``, the module will not learn an additive bias.
406
+ device: The device.
407
+ dtype: The dtype.
408
+ """
409
+
410
+ def __init__(
411
+ self,
412
+ channels: int,
413
+ num_query_heads: int,
414
+ mlp: torch.nn.Module,
415
+ num_key_value_heads: int | None = None,
416
+ query_norm: torch.nn.Module | None = None,
417
+ key_value_norm: torch.nn.Module | None = None,
418
+ post_attn_norm: torch.nn.Module | None = None,
419
+ query_transform: torch.nn.Module | None = None,
420
+ key_transform: torch.nn.Module | None = None,
421
+ query_scaling: QueryScaling | None = None,
422
+ scale: float | None = None,
423
+ bias: bool = True,
424
+ device: torch.device | str | None = None,
425
+ dtype: torch.dtype | None = None,
426
+ ) -> None:
427
+ super().__init__()
428
+ factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
429
+
430
+ self.mlp = mlp
431
+ self.query_norm = query_norm
432
+ self.key_value_norm = key_value_norm
433
+ self.post_attn_norm = post_attn_norm
434
+
435
+ self.attn = Attention(
436
+ channels=channels,
437
+ num_query_heads=num_query_heads,
438
+ num_key_value_heads=num_key_value_heads,
439
+ query_transform=query_transform,
440
+ key_transform=key_transform,
441
+ query_scaling=query_scaling,
442
+ scale=scale,
443
+ bias=bias,
444
+ **factory_kwargs,
445
+ )
446
+
447
+ @overload
448
+ def forward(
449
+ self,
450
+ query: Tensor,
451
+ key_value: Tensor | KVCacheEntry | None = None,
452
+ seqused_key_value: Tensor | None = None,
453
+ attn_mask: Tensor | None = None,
454
+ *,
455
+ return_key_value: Literal[False] = False,
456
+ batch_size_limit: int | Literal["auto"] | None = None,
457
+ out: Tensor | None = None,
458
+ ) -> Tensor: ...
459
+
460
+ @overload
461
+ def forward(
462
+ self,
463
+ query: Tensor,
464
+ key_value: Tensor | KVCacheEntry | None = None,
465
+ seqused_key_value: Tensor | None = None,
466
+ attn_mask: Tensor | None = None,
467
+ *,
468
+ return_key_value: Literal[True],
469
+ batch_size_limit: int | Literal["auto"] | None = None,
470
+ out: Tensor | None = None,
471
+ ) -> tuple[Tensor, KVCacheEntry]: ...
472
+
473
+ @overload
474
+ def forward(
475
+ self,
476
+ query: Tensor,
477
+ key_value: Tensor | KVCacheEntry | None = None,
478
+ seqused_key_value: Tensor | None = None,
479
+ attn_mask: Tensor | None = None,
480
+ *,
481
+ return_key_value: bool,
482
+ batch_size_limit: int | Literal["auto"] | None = None,
483
+ out: Tensor | None = None,
484
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
485
+
486
+ def forward(
487
+ self,
488
+ query: Tensor, # [..., Q, C]
489
+ key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
490
+ seqused_key_value: Tensor | None = None, # [...]
491
+ attn_mask: Tensor | None = None, # [..., Q, KV]
492
+ *,
493
+ return_key_value: bool = False,
494
+ batch_size_limit: int | Literal["auto"] | None = None,
495
+ out: Tensor | None = None,
496
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
497
+ r"""The forward pass.
498
+
499
+ Args:
500
+ query: The query tensor with shape ``[..., Q, C]``.
501
+ ``Q`` is the query sequence length, ``C`` is the number of
502
+ channels.
503
+ key_value: The key/value tensor with shape ``[..., KV, C]`` or
504
+ precomputed key/value projections as a
505
+ :class:`~sdm.cache.KVCacheEntry`.
506
+ ``KV`` is the key/value sequence length.
507
+ If omitted, ``query`` is used for self-attention.
508
+ seqused_key_value: Valid key/value lengths with shape ``[...]`` and
509
+ :external+torch:ref:`torch.int32 <dtype-doc>` dtype.
510
+ attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
511
+ Entries set to ``True`` participate in attention.
512
+ return_key_value: Whether to return the computed key and value
513
+ projections alongside the block output.
514
+ batch_size_limit: Maximum number of batch elements processed at
515
+ once.
516
+ out: The output tensor.
517
+
518
+ Returns:
519
+ Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
520
+ ``False``.
521
+ Otherwise, a tuple of the output tensor and a
522
+ :class:`~sdm.cache.KVCacheEntry`.
523
+ """
524
+ if out is not None and torch.is_grad_enabled():
525
+ raise RuntimeError(
526
+ "'out' is only supported when gradients are disabled"
527
+ )
528
+ if return_key_value and isinstance(key_value, KVCacheEntry):
529
+ raise ValueError(
530
+ "'return_key_value=True' is not supported when 'key_value' is "
531
+ "already cached"
532
+ )
533
+
534
+ if torch.is_grad_enabled() or torch.compiler.is_compiling():
535
+ return self._forward(
536
+ query=query,
537
+ key_value=key_value,
538
+ seqused_key_value=seqused_key_value,
539
+ attn_mask=attn_mask,
540
+ return_key_value=return_key_value,
541
+ out=out,
542
+ )
543
+
544
+ if batch_size_limit == "auto":
545
+ key_value_length: int | None = None
546
+ if isinstance(key_value, Tensor):
547
+ key_value_length = key_value.size(-2)
548
+ elif isinstance(key_value, KVCacheEntry):
549
+ key_value_length = key_value.key.size(-3)
550
+
551
+ batch_size_limit = self.auto_batch_size_limit(
552
+ device=query.device,
553
+ element_size=torch.get_autocast_dtype(
554
+ query.device.type
555
+ ).itemsize
556
+ if torch.is_autocast_enabled(query.device.type)
557
+ else query.element_size(),
558
+ query_length=query.size(-2),
559
+ key_value_length=key_value_length,
560
+ )
561
+
562
+ batch_size_limit = min(batch_size_limit or 65_535, 65_535)
563
+
564
+ batch_shape = _batch_shape(
565
+ query=query,
566
+ key_value=key_value,
567
+ seqused_key_value=seqused_key_value,
568
+ attn_mask=attn_mask,
569
+ )
570
+ batch_size = math.prod(batch_shape)
571
+
572
+ disable_chunking = False
573
+ if batch_size <= batch_size_limit or query.size(-2) == 0:
574
+ disable_chunking = True
575
+ if return_key_value:
576
+ assert not isinstance(key_value, KVCacheEntry)
577
+ if batch_shape != (
578
+ query.size()[:-2]
579
+ if key_value is None
580
+ else key_value.size()[:-2]
581
+ ):
582
+ disable_chunking = True
583
+
584
+ if disable_chunking:
585
+ return self._forward(
586
+ query=query,
587
+ key_value=key_value,
588
+ seqused_key_value=seqused_key_value,
589
+ attn_mask=attn_mask,
590
+ return_key_value=return_key_value,
591
+ out=out,
592
+ )
593
+
594
+ flat_out: Tensor | None = None
595
+ flat_key: Tensor | None = None
596
+ flat_value: Tensor | None = None
597
+
598
+ if out is not None and (out.dim() <= 3 or out.is_contiguous()):
599
+ flat_out = out.view(batch_size, *query.size()[-2:])
600
+
601
+ for start in range(0, batch_size, batch_size_limit):
602
+ end = min(start + batch_size_limit, batch_size)
603
+
604
+ result = self._forward(
605
+ query=_chunk(query, batch_shape, 2, start, end),
606
+ key_value=_chunk(key_value, batch_shape, 2, start, end),
607
+ seqused_key_value=_chunk(
608
+ seqused_key_value,
609
+ batch_shape,
610
+ trailing_dims=0,
611
+ start=start,
612
+ end=end,
613
+ ),
614
+ attn_mask=_chunk(
615
+ attn_mask,
616
+ batch_shape,
617
+ trailing_dims=2,
618
+ start=start,
619
+ end=end,
620
+ ),
621
+ return_key_value=return_key_value,
622
+ out=flat_out[start:end] if flat_out is not None else None,
623
+ )
624
+
625
+ if isinstance(result, tuple):
626
+ chunk, chunk_kv = result
627
+
628
+ if flat_key is None:
629
+ flat_key = chunk_kv.key.new_empty(
630
+ batch_size, *chunk_kv.key.size()[-3:]
631
+ )
632
+ if flat_value is None:
633
+ flat_value = chunk_kv.value.new_empty(
634
+ batch_size, *chunk_kv.value.size()[-3:]
635
+ )
636
+
637
+ flat_key[start:end] = chunk_kv.key
638
+ flat_value[start:end] = chunk_kv.value
639
+
640
+ del chunk_kv
641
+
642
+ else:
643
+ chunk = result
644
+
645
+ if flat_out is None and out is not None:
646
+ flat_index = torch.arange(start, end, device=out.device)
647
+ batch_indices: list[Tensor] = []
648
+ for size in reversed(batch_shape):
649
+ batch_indices.append(flat_index % size)
650
+ flat_index = flat_index // size
651
+ out[tuple(reversed(batch_indices))] = chunk.to(out.dtype)
652
+ elif flat_out is None:
653
+ flat_out = chunk.new_empty((batch_size, *query.size()[-2:]))
654
+ flat_out[start:end] = chunk
655
+
656
+ del chunk, result
657
+
658
+ if out is None:
659
+ assert flat_out is not None
660
+ out = flat_out.view(*batch_shape, *query.size()[-2:])
661
+
662
+ if not return_key_value:
663
+ return out
664
+
665
+ assert flat_key is not None
666
+ assert flat_value is not None
667
+
668
+ return out, KVCacheEntry(
669
+ key=flat_key.view(*batch_shape, *flat_key.size()[-3:]),
670
+ value=flat_value.view(*batch_shape, *flat_value.size()[-3:]),
671
+ )
672
+
673
+ def _forward(
674
+ self,
675
+ query: Tensor, # [..., Q, C]
676
+ key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
677
+ seqused_key_value: Tensor | None = None, # [...]
678
+ attn_mask: Tensor | None = None, # [..., Q, KV]
679
+ *,
680
+ return_key_value: bool = False,
681
+ out: Tensor | None = None,
682
+ ) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
683
+
684
+ if self.key_value_norm is None:
685
+ pass
686
+ elif key_value is None and self.key_value_norm is not self.query_norm:
687
+ key_value = self.key_value_norm(query)
688
+ elif isinstance(key_value, Tensor):
689
+ key_value = self.key_value_norm(key_value)
690
+
691
+ result = self.attn(
692
+ query=query if self.query_norm is None else self.query_norm(query),
693
+ key_value=key_value,
694
+ seqused_key_value=seqused_key_value,
695
+ attn_mask=attn_mask,
696
+ return_key_value=return_key_value,
697
+ )
698
+ del key_value
699
+
700
+ if return_key_value:
701
+ attn_out, kv = result
702
+ else:
703
+ attn_out = result
704
+
705
+ if self.post_attn_norm is not None:
706
+ attn_out = self.post_attn_norm(attn_out)
707
+
708
+ if (
709
+ out is not None
710
+ and torch.compiler.is_compiling()
711
+ and not out.is_contiguous()
712
+ ):
713
+ # Match eager out= rounding before the MLP.
714
+ tmp = (attn_out + query).to(out.dtype)
715
+ out.copy_(tmp + self.mlp(tmp))
716
+ else:
717
+ tmp = torch.add(attn_out, query, out=out)
718
+ out = torch.add(tmp, self.mlp(tmp), out=out)
719
+
720
+ return (out, kv) if return_key_value else out
721
+
722
+ def peak_bytes_per_example(
723
+ self,
724
+ element_size: int,
725
+ query_length: int,
726
+ key_value_length: int | None = None,
727
+ ) -> int:
728
+ r""":meta private:""" # noqa: D415
729
+ return 0
730
+
731
+ def auto_batch_size_limit(
732
+ self,
733
+ device: torch.device,
734
+ element_size: int,
735
+ query_length: int,
736
+ key_value_length: int | None = None,
737
+ ) -> int:
738
+ r""":meta private:""" # noqa: D415
739
+ if device.type != "cuda":
740
+ return 65_535
741
+ bytes_per_example = self.peak_bytes_per_example(
742
+ element_size=element_size,
743
+ query_length=query_length,
744
+ key_value_length=key_value_length,
745
+ )
746
+ limit = chunk_memory_limit(device) // max(bytes_per_example, 1)
747
+ return min(max(limit, 1), 65_535)
748
+
749
+
750
+ def _batch_shape(
751
+ query: Tensor,
752
+ key_value: Tensor | KVCacheEntry | None,
753
+ seqused_key_value: Tensor | None,
754
+ attn_mask: Tensor | None,
755
+ ) -> torch.Size:
756
+ shapes = [query.size()[:-2]]
757
+ if isinstance(key_value, Tensor):
758
+ shapes.append(key_value.size()[:-2])
759
+ elif isinstance(key_value, KVCacheEntry):
760
+ shapes += [key_value.key.size()[:-3], key_value.value.size()[:-3]]
761
+ if seqused_key_value is not None:
762
+ shapes.append(seqused_key_value.size())
763
+ if attn_mask is not None:
764
+ shapes.append(attn_mask.size()[:-2])
765
+ return torch.broadcast_shapes(*shapes)
766
+
767
+
768
+ @overload
769
+ def _chunk(
770
+ tensor: Tensor,
771
+ batch_shape: torch.Size,
772
+ trailing_dims: int,
773
+ start: int,
774
+ end: int,
775
+ ) -> Tensor: ...
776
+
777
+
778
+ @overload
779
+ def _chunk(
780
+ tensor: KVCacheEntry,
781
+ batch_shape: torch.Size,
782
+ trailing_dims: int,
783
+ start: int,
784
+ end: int,
785
+ ) -> KVCacheEntry: ...
786
+
787
+
788
+ @overload
789
+ def _chunk(
790
+ tensor: None,
791
+ batch_shape: torch.Size,
792
+ trailing_dims: int,
793
+ start: int,
794
+ end: int,
795
+ ) -> None: ...
796
+
797
+
798
+ def _chunk(
799
+ tensor: Tensor | KVCacheEntry | None,
800
+ batch_shape: torch.Size,
801
+ trailing_dims: int,
802
+ start: int,
803
+ end: int,
804
+ ) -> Tensor | KVCacheEntry | None:
805
+
806
+ if tensor is None:
807
+ return None
808
+
809
+ if isinstance(tensor, KVCacheEntry):
810
+ return KVCacheEntry(
811
+ key=_chunk(tensor.key, batch_shape, 3, start, end),
812
+ value=_chunk(tensor.value, batch_shape, 3, start, end),
813
+ )
814
+
815
+ trailing_shape = tensor.size()[-trailing_dims:] if trailing_dims else ()
816
+ tensor = tensor.expand(batch_shape + trailing_shape)
817
+ if len(batch_shape) == 1:
818
+ return tensor[start:end]
819
+
820
+ flat_index = torch.arange(start, end, device=tensor.device)
821
+ batch_indices: list[Tensor] = []
822
+ for size in reversed(batch_shape):
823
+ batch_indices.append(flat_index % size)
824
+ flat_index = flat_index // size
825
+ return tensor[tuple(reversed(batch_indices))]