structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/attention.py
ADDED
|
@@ -0,0 +1,825 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Attention modules for structured tensor models."""
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
from typing import Any, Literal, overload
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
import torch.nn.functional as F
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
from torch.nn import Linear
|
|
13
|
+
|
|
14
|
+
from sdm._memory import chunk_memory_limit
|
|
15
|
+
from sdm.cache import KVCacheEntry
|
|
16
|
+
from sdm.nn import QueryScaling
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SDPA(torch.nn.Module):
|
|
20
|
+
r"""Scaled Dot-Product Attention (SDPA).
|
|
21
|
+
|
|
22
|
+
This module wraps :func:`torch.nn.functional.scaled_dot_product_attention`
|
|
23
|
+
and extends it by arbitrary batch dimensions, optional inference-time
|
|
24
|
+
batch chunking, query-scaling, and padding support for key/value pairs.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
num_query_heads: The number of query attention heads.
|
|
28
|
+
num_key_value_heads: The number of key/value attention heads.
|
|
29
|
+
Setting this below ``num_query_heads`` enables grouped-query
|
|
30
|
+
attention (GQA); setting it to ``1`` enables multi-query attention
|
|
31
|
+
(MQA). Must divide ``num_query_heads``. Defaults to
|
|
32
|
+
``num_query_heads`` (standard multi-head attention).
|
|
33
|
+
query_scaling: Query scaling module to scale projected query heads
|
|
34
|
+
before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
|
|
35
|
+
scale: Scaling factor passed to
|
|
36
|
+
:func:`torch.nn.functional.scaled_dot_product_attention`.
|
|
37
|
+
``None`` uses the default value of ``1 / sqrt(channels)``.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
num_query_heads: int,
|
|
43
|
+
num_key_value_heads: int | None = None,
|
|
44
|
+
query_scaling: QueryScaling | None = None,
|
|
45
|
+
scale: float | None = None,
|
|
46
|
+
) -> None:
|
|
47
|
+
super().__init__()
|
|
48
|
+
if num_key_value_heads is None:
|
|
49
|
+
num_key_value_heads = num_query_heads
|
|
50
|
+
if num_query_heads % num_key_value_heads != 0:
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"`num_query_heads` ({num_query_heads}) must be divisible by "
|
|
53
|
+
f"`num_key_value_heads` ({num_key_value_heads})"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
self.num_query_heads = num_query_heads
|
|
57
|
+
self.num_key_value_heads = num_key_value_heads
|
|
58
|
+
self.query_scaling = query_scaling
|
|
59
|
+
self.scale = scale
|
|
60
|
+
|
|
61
|
+
def forward(
|
|
62
|
+
self,
|
|
63
|
+
query: Tensor, # [..., Q, Hq, C]
|
|
64
|
+
key: Tensor, # [..., KV, Hkv, C]
|
|
65
|
+
value: Tensor, # [..., KV, Hkv, C]
|
|
66
|
+
seqused_key_value: Tensor | None = None, # [...]
|
|
67
|
+
attn_mask: Tensor | None = None, # [..., Q, KV]
|
|
68
|
+
) -> Tensor: # [..., Q, Hq, C]
|
|
69
|
+
r"""The forward pass.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
query: The query tensor with shape ``[..., Q, Hq, C]``.
|
|
73
|
+
``Q`` is the query sequence length, ``Hq`` is the number of
|
|
74
|
+
query attention heads (``num_query_heads``), and ``C`` is the
|
|
75
|
+
channels per head.
|
|
76
|
+
key: The key tensor with shape ``[..., KV, Hkv, C]``.
|
|
77
|
+
``KV`` is the key/value sequence length and ``Hkv`` is the
|
|
78
|
+
number of key/value heads (``num_key_value_heads``).
|
|
79
|
+
value: The value tensor with shape ``[..., KV, Hkv, C]``.
|
|
80
|
+
seqused_key_value: Valid key/value lengths with shape ``[...]`` and
|
|
81
|
+
:external+torch:ref:`torch.int32 <dtype-doc>` dtype.
|
|
82
|
+
attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
|
|
83
|
+
Entries set to ``True`` participate in attention.
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
Tensor with shape ``[..., Q, Hq, C]``.
|
|
87
|
+
"""
|
|
88
|
+
if query.numel() == 0:
|
|
89
|
+
return query
|
|
90
|
+
|
|
91
|
+
if attn_mask is not None and seqused_key_value is not None:
|
|
92
|
+
raise ValueError(
|
|
93
|
+
"Cannot pass both `attn_mask` and `seqused_key_value`"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if (
|
|
97
|
+
seqused_key_value is not None
|
|
98
|
+
and seqused_key_value.dtype != torch.int32
|
|
99
|
+
):
|
|
100
|
+
raise ValueError("`seqused_key_value` must have dtype torch.int32")
|
|
101
|
+
if attn_mask is not None and attn_mask.dtype != torch.bool:
|
|
102
|
+
raise ValueError("`attn_mask` must have dtype torch.bool")
|
|
103
|
+
|
|
104
|
+
batch_shapes = [query.size()[:-3], key.size()[:-3], value.size()[:-3]]
|
|
105
|
+
if seqused_key_value is not None:
|
|
106
|
+
batch_shapes.append(seqused_key_value.size())
|
|
107
|
+
if attn_mask is not None:
|
|
108
|
+
batch_shapes.append(attn_mask.size()[:-2])
|
|
109
|
+
batch_shape = torch.broadcast_shapes(*batch_shapes)
|
|
110
|
+
|
|
111
|
+
if self.query_scaling is not None:
|
|
112
|
+
if seqused_key_value is not None:
|
|
113
|
+
key_len = seqused_key_value.unsqueeze(-1)
|
|
114
|
+
elif attn_mask is not None and attn_mask.size(-1) > 1:
|
|
115
|
+
key_len = attn_mask.sum(dim=-1)
|
|
116
|
+
else:
|
|
117
|
+
key_len = key.size(-3)
|
|
118
|
+
query = self.query_scaling(query, key_len=key_len)
|
|
119
|
+
|
|
120
|
+
# Broadcast and flatten batch dimensions => [B, S, H, C].
|
|
121
|
+
query_size = query.size()[-3:]
|
|
122
|
+
key_size = key.size()[-3:]
|
|
123
|
+
value_size = value.size()[-3:]
|
|
124
|
+
|
|
125
|
+
# FlashAttention rejects empty batches before PyTorch 2.10.
|
|
126
|
+
if key_size[0] == 0 or 0 in batch_shape:
|
|
127
|
+
out = query.new_zeros(
|
|
128
|
+
batch_shape + query_size[:-1] + value_size[-1:]
|
|
129
|
+
)
|
|
130
|
+
if torch.is_grad_enabled():
|
|
131
|
+
# Summing an empty slice gives zero and keeps backward working.
|
|
132
|
+
for tensor in (query, key, value):
|
|
133
|
+
if tensor.requires_grad:
|
|
134
|
+
out = out + tensor[..., :0, :, :].sum()
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
query = query.expand(batch_shape + query_size).reshape(-1, *query_size)
|
|
138
|
+
key = key.expand(batch_shape + key_size).reshape(-1, *key_size)
|
|
139
|
+
value = value.expand(batch_shape + value_size).reshape(-1, *value_size)
|
|
140
|
+
|
|
141
|
+
if attn_mask is not None:
|
|
142
|
+
attn_mask = attn_mask.expand(batch_shape + attn_mask.size()[-2:])
|
|
143
|
+
attn_mask = attn_mask.reshape(-1, *attn_mask.size()[-2:])
|
|
144
|
+
|
|
145
|
+
if seqused_key_value is not None:
|
|
146
|
+
seqused_key_value = seqused_key_value.expand(batch_shape)
|
|
147
|
+
seqused_key_value = seqused_key_value.reshape(-1).unsqueeze(-1)
|
|
148
|
+
key_index = torch.arange(key.size(-3), device=key.device)
|
|
149
|
+
attn_mask = key_index.unsqueeze(0) < seqused_key_value
|
|
150
|
+
attn_mask = attn_mask.unsqueeze(-2).expand(-1, query.size(-3), -1)
|
|
151
|
+
|
|
152
|
+
enable_gqa = False
|
|
153
|
+
if query.size(-2) != key.size(-2):
|
|
154
|
+
enable_gqa = True
|
|
155
|
+
|
|
156
|
+
out = F.scaled_dot_product_attention(
|
|
157
|
+
query=query.transpose(-3, -2), # [B, Hq, Q, C],
|
|
158
|
+
key=key.transpose(-3, -2), # [B, Hkv, KV, C],
|
|
159
|
+
value=value.transpose(-3, -2), # [B, Hkv, KV, C],
|
|
160
|
+
attn_mask=attn_mask.unsqueeze(-3) # [B, 1, Q, KV]
|
|
161
|
+
if attn_mask is not None
|
|
162
|
+
else None,
|
|
163
|
+
enable_gqa=enable_gqa,
|
|
164
|
+
scale=self.scale,
|
|
165
|
+
).transpose(-3, -2) # [B, Q, Hq, C]
|
|
166
|
+
|
|
167
|
+
return out.view(batch_shape + out.size()[-3:]) # [..., Q, Hq, C]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class Attention(torch.nn.Module):
|
|
171
|
+
r"""Multi-head attention layer with grouped-query attention support.
|
|
172
|
+
|
|
173
|
+
This module owns the query, key, value, and output projections.
|
|
174
|
+
It performs self-attention when ``key_value`` is omitted and
|
|
175
|
+
cross-attention when ``key_value`` is given.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
channels: The number of input and output channels.
|
|
179
|
+
num_query_heads: The number of query attention heads.
|
|
180
|
+
``channels`` must be divisible by ``num_query_heads``.
|
|
181
|
+
num_key_value_heads: The number of key/value attention heads.
|
|
182
|
+
Setting this below ``num_query_heads`` enables grouped-query
|
|
183
|
+
attention (GQA); setting it to ``1`` enables multi-query attention
|
|
184
|
+
(MQA). Must divide ``num_query_heads``. Defaults to
|
|
185
|
+
``num_query_heads`` (standard multi-head attention).
|
|
186
|
+
query_transform: Transformation applied to projected query heads before
|
|
187
|
+
scaled dot-product attention.
|
|
188
|
+
key_transform: Transformation applied to projected key heads before
|
|
189
|
+
scaled dot-product attention.
|
|
190
|
+
query_scaling: Query scaling module to scale projected query heads
|
|
191
|
+
before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
|
|
192
|
+
scale: Scaling factor passed to
|
|
193
|
+
:func:`torch.nn.functional.scaled_dot_product_attention`.
|
|
194
|
+
``None`` uses ``1 / sqrt(channels_per_head)``.
|
|
195
|
+
bias: If set to ``False``, the module will not learn an additive bias.
|
|
196
|
+
device: The device.
|
|
197
|
+
dtype: The dtype.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
def __init__(
|
|
201
|
+
self,
|
|
202
|
+
channels: int,
|
|
203
|
+
num_query_heads: int,
|
|
204
|
+
num_key_value_heads: int | None = None,
|
|
205
|
+
query_transform: torch.nn.Module | None = None,
|
|
206
|
+
key_transform: torch.nn.Module | None = None,
|
|
207
|
+
query_scaling: QueryScaling | None = None,
|
|
208
|
+
scale: float | None = None,
|
|
209
|
+
bias: bool = True,
|
|
210
|
+
device: torch.device | str | None = None,
|
|
211
|
+
dtype: torch.dtype | None = None,
|
|
212
|
+
) -> None:
|
|
213
|
+
super().__init__()
|
|
214
|
+
if num_key_value_heads is None:
|
|
215
|
+
num_key_value_heads = num_query_heads
|
|
216
|
+
if channels % num_query_heads != 0:
|
|
217
|
+
raise ValueError(
|
|
218
|
+
f"`channels` ({channels}) must be divisible by "
|
|
219
|
+
f"`num_query_heads` ({num_query_heads})"
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
223
|
+
|
|
224
|
+
self.num_query_heads = num_query_heads
|
|
225
|
+
self.num_key_value_heads = num_key_value_heads
|
|
226
|
+
self.head_dim = channels // num_query_heads
|
|
227
|
+
# Query projection spans all channels; key/value span fewer heads.
|
|
228
|
+
self.q_dim = num_query_heads * self.head_dim # == channels
|
|
229
|
+
self.kv_dim = num_key_value_heads * self.head_dim
|
|
230
|
+
|
|
231
|
+
self.qkv_lin = Linear(
|
|
232
|
+
channels, self.q_dim + 2 * self.kv_dim, bias=bias, **factory_kwargs
|
|
233
|
+
)
|
|
234
|
+
self.query_transform = query_transform
|
|
235
|
+
self.key_transform = key_transform
|
|
236
|
+
self.sdpa = SDPA(
|
|
237
|
+
num_query_heads=num_query_heads,
|
|
238
|
+
num_key_value_heads=num_key_value_heads,
|
|
239
|
+
query_scaling=query_scaling,
|
|
240
|
+
scale=scale,
|
|
241
|
+
)
|
|
242
|
+
self.out_lin = Linear(channels, channels, bias=bias, **factory_kwargs)
|
|
243
|
+
|
|
244
|
+
torch.nn.init.zeros_(self.out_lin.weight)
|
|
245
|
+
if self.out_lin.bias is not None:
|
|
246
|
+
torch.nn.init.zeros_(self.out_lin.bias)
|
|
247
|
+
|
|
248
|
+
@overload
|
|
249
|
+
def forward(
|
|
250
|
+
self,
|
|
251
|
+
query: Tensor,
|
|
252
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
253
|
+
seqused_key_value: Tensor | None = None,
|
|
254
|
+
attn_mask: Tensor | None = None,
|
|
255
|
+
*,
|
|
256
|
+
return_key_value: Literal[False] = False,
|
|
257
|
+
) -> Tensor: ...
|
|
258
|
+
|
|
259
|
+
@overload
|
|
260
|
+
def forward(
|
|
261
|
+
self,
|
|
262
|
+
query: Tensor,
|
|
263
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
264
|
+
seqused_key_value: Tensor | None = None,
|
|
265
|
+
attn_mask: Tensor | None = None,
|
|
266
|
+
*,
|
|
267
|
+
return_key_value: Literal[True],
|
|
268
|
+
) -> tuple[Tensor, KVCacheEntry]: ...
|
|
269
|
+
|
|
270
|
+
@overload
|
|
271
|
+
def forward(
|
|
272
|
+
self,
|
|
273
|
+
query: Tensor,
|
|
274
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
275
|
+
seqused_key_value: Tensor | None = None,
|
|
276
|
+
attn_mask: Tensor | None = None,
|
|
277
|
+
*,
|
|
278
|
+
return_key_value: bool,
|
|
279
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
|
|
280
|
+
|
|
281
|
+
def forward(
|
|
282
|
+
self,
|
|
283
|
+
query: Tensor, # [..., Q, C]
|
|
284
|
+
key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
|
|
285
|
+
seqused_key_value: Tensor | None = None, # [...]
|
|
286
|
+
attn_mask: Tensor | None = None, # [..., Q, KV]
|
|
287
|
+
*,
|
|
288
|
+
return_key_value: bool = False,
|
|
289
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
|
|
290
|
+
r"""The forward pass.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
query: The query tensor with shape ``[..., Q, C]``.
|
|
294
|
+
``Q`` is the query sequence length, ``C`` is the number of
|
|
295
|
+
channels.
|
|
296
|
+
key_value: The key/value tensor with shape ``[..., KV, C]`` or
|
|
297
|
+
precomputed key/value projections as a
|
|
298
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
299
|
+
``KV`` is the key/value sequence length.
|
|
300
|
+
If omitted, ``query`` is used for self-attention.
|
|
301
|
+
seqused_key_value: Valid key/value lengths with shape ``[...]`` and
|
|
302
|
+
:external+torch:ref:`torch.int32 <dtype-doc>` dtype.
|
|
303
|
+
attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
|
|
304
|
+
Entries set to ``True`` participate in attention.
|
|
305
|
+
return_key_value: Whether to return the computed key and value
|
|
306
|
+
projections alongside the attention output.
|
|
307
|
+
|
|
308
|
+
Returns:
|
|
309
|
+
Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
|
|
310
|
+
``False``.
|
|
311
|
+
Otherwise, a tuple of the output tensor and a
|
|
312
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
313
|
+
"""
|
|
314
|
+
if isinstance(key_value, KVCacheEntry):
|
|
315
|
+
query = F.linear(
|
|
316
|
+
query,
|
|
317
|
+
weight=self.qkv_lin.weight[: self.q_dim],
|
|
318
|
+
bias=self.qkv_lin.bias[: self.q_dim]
|
|
319
|
+
if self.qkv_lin.bias is not None
|
|
320
|
+
else None,
|
|
321
|
+
)
|
|
322
|
+
if (
|
|
323
|
+
key_value.key.dtype != query.dtype
|
|
324
|
+
or key_value.value.dtype != query.dtype
|
|
325
|
+
):
|
|
326
|
+
raise ValueError(
|
|
327
|
+
f"Key/value projections were cached under dtypes "
|
|
328
|
+
f"'{key_value.key.dtype}'/'{key_value.value.dtype}' but "
|
|
329
|
+
f"the query has dtype '{query.dtype}'"
|
|
330
|
+
)
|
|
331
|
+
key = key_value.key
|
|
332
|
+
value = key_value.value
|
|
333
|
+
elif key_value is None:
|
|
334
|
+
query, key, value = self.qkv_lin(query).split(
|
|
335
|
+
[self.q_dim, self.kv_dim, self.kv_dim], dim=-1
|
|
336
|
+
)
|
|
337
|
+
else:
|
|
338
|
+
sections = [self.q_dim, 2 * self.kv_dim]
|
|
339
|
+
q_weight, kv_weight = self.qkv_lin.weight.split(sections, dim=0)
|
|
340
|
+
q_bias = kv_bias = None
|
|
341
|
+
if self.qkv_lin.bias is not None:
|
|
342
|
+
q_bias, kv_bias = self.qkv_lin.bias.split(sections, dim=0)
|
|
343
|
+
query = F.linear(query, q_weight, q_bias)
|
|
344
|
+
key, value = F.linear(key_value, kv_weight, kv_bias).chunk(2, -1)
|
|
345
|
+
|
|
346
|
+
# [..., S, C] -> [..., S, H, C // H], with separate query/kv heads.
|
|
347
|
+
query = query.unflatten(-1, [self.num_query_heads, self.head_dim])
|
|
348
|
+
if not isinstance(key_value, KVCacheEntry):
|
|
349
|
+
key = key.unflatten(-1, [self.num_key_value_heads, self.head_dim])
|
|
350
|
+
value = value.unflatten(
|
|
351
|
+
-1, [self.num_key_value_heads, self.head_dim]
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
if self.query_transform is not None:
|
|
355
|
+
query = self.query_transform(query)
|
|
356
|
+
if (
|
|
357
|
+
not isinstance(key_value, KVCacheEntry)
|
|
358
|
+
and self.key_transform is not None
|
|
359
|
+
):
|
|
360
|
+
key = self.key_transform(key)
|
|
361
|
+
|
|
362
|
+
out = self.sdpa(
|
|
363
|
+
query=query, # [..., Q, Hq, C // Hq]
|
|
364
|
+
key=key, # [..., KV, Hkv, C // Hq]
|
|
365
|
+
value=value, # [..., KV, Hkv, C // Hq]
|
|
366
|
+
seqused_key_value=seqused_key_value, # [...]
|
|
367
|
+
attn_mask=attn_mask, # [..., Q, KV]
|
|
368
|
+
) # [..., Q, Hq, C // Hq]
|
|
369
|
+
|
|
370
|
+
out = out.flatten(-2, -1) # [..., Q, C]
|
|
371
|
+
out = self.out_lin(out) # [..., Q, C]
|
|
372
|
+
if return_key_value:
|
|
373
|
+
# CUDA autocast runs RMSNorm in fp32, so cast explicitly before
|
|
374
|
+
# caching: https://github.com/pytorch/pytorch/blob/v2.13.0/aten/src/ATen/autocast_mode.h#L875
|
|
375
|
+
return out, KVCacheEntry(
|
|
376
|
+
key=key.to(value.dtype),
|
|
377
|
+
value=value,
|
|
378
|
+
)
|
|
379
|
+
return out
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
class TransformerBlock(torch.nn.Module):
|
|
383
|
+
r"""Transformer block with normalization and feedforward residual modules.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
channels: The number of input and output channels.
|
|
387
|
+
num_query_heads: The number of query attention heads.
|
|
388
|
+
mlp: Feedforward module applied after the attention residual.
|
|
389
|
+
num_key_value_heads: The number of key/value attention heads.
|
|
390
|
+
Defaults to ``num_query_heads`` (standard multi-head attention).
|
|
391
|
+
query_norm: Normalization applied to query inputs before attention.
|
|
392
|
+
key_value_norm: Normalization applied to key/value inputs before
|
|
393
|
+
attention.
|
|
394
|
+
post_attn_norm: Normalization applied to the attention output before
|
|
395
|
+
its residual addition.
|
|
396
|
+
query_transform: Transformation applied to projected query heads before
|
|
397
|
+
scaled dot-product attention.
|
|
398
|
+
key_transform: Transformation applied to projected key heads before
|
|
399
|
+
scaled dot-product attention.
|
|
400
|
+
query_scaling: Query scaling module to scale projected query heads
|
|
401
|
+
before scaled dot-product attention, *e.g.*, :class:`QASSMax`.
|
|
402
|
+
scale: Scaling factor passed to
|
|
403
|
+
:func:`torch.nn.functional.scaled_dot_product_attention`.
|
|
404
|
+
``None`` uses ``1 / sqrt(channels_per_head)``.
|
|
405
|
+
bias: If set to ``False``, the module will not learn an additive bias.
|
|
406
|
+
device: The device.
|
|
407
|
+
dtype: The dtype.
|
|
408
|
+
"""
|
|
409
|
+
|
|
410
|
+
def __init__(
|
|
411
|
+
self,
|
|
412
|
+
channels: int,
|
|
413
|
+
num_query_heads: int,
|
|
414
|
+
mlp: torch.nn.Module,
|
|
415
|
+
num_key_value_heads: int | None = None,
|
|
416
|
+
query_norm: torch.nn.Module | None = None,
|
|
417
|
+
key_value_norm: torch.nn.Module | None = None,
|
|
418
|
+
post_attn_norm: torch.nn.Module | None = None,
|
|
419
|
+
query_transform: torch.nn.Module | None = None,
|
|
420
|
+
key_transform: torch.nn.Module | None = None,
|
|
421
|
+
query_scaling: QueryScaling | None = None,
|
|
422
|
+
scale: float | None = None,
|
|
423
|
+
bias: bool = True,
|
|
424
|
+
device: torch.device | str | None = None,
|
|
425
|
+
dtype: torch.dtype | None = None,
|
|
426
|
+
) -> None:
|
|
427
|
+
super().__init__()
|
|
428
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
429
|
+
|
|
430
|
+
self.mlp = mlp
|
|
431
|
+
self.query_norm = query_norm
|
|
432
|
+
self.key_value_norm = key_value_norm
|
|
433
|
+
self.post_attn_norm = post_attn_norm
|
|
434
|
+
|
|
435
|
+
self.attn = Attention(
|
|
436
|
+
channels=channels,
|
|
437
|
+
num_query_heads=num_query_heads,
|
|
438
|
+
num_key_value_heads=num_key_value_heads,
|
|
439
|
+
query_transform=query_transform,
|
|
440
|
+
key_transform=key_transform,
|
|
441
|
+
query_scaling=query_scaling,
|
|
442
|
+
scale=scale,
|
|
443
|
+
bias=bias,
|
|
444
|
+
**factory_kwargs,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
@overload
|
|
448
|
+
def forward(
|
|
449
|
+
self,
|
|
450
|
+
query: Tensor,
|
|
451
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
452
|
+
seqused_key_value: Tensor | None = None,
|
|
453
|
+
attn_mask: Tensor | None = None,
|
|
454
|
+
*,
|
|
455
|
+
return_key_value: Literal[False] = False,
|
|
456
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
457
|
+
out: Tensor | None = None,
|
|
458
|
+
) -> Tensor: ...
|
|
459
|
+
|
|
460
|
+
@overload
|
|
461
|
+
def forward(
|
|
462
|
+
self,
|
|
463
|
+
query: Tensor,
|
|
464
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
465
|
+
seqused_key_value: Tensor | None = None,
|
|
466
|
+
attn_mask: Tensor | None = None,
|
|
467
|
+
*,
|
|
468
|
+
return_key_value: Literal[True],
|
|
469
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
470
|
+
out: Tensor | None = None,
|
|
471
|
+
) -> tuple[Tensor, KVCacheEntry]: ...
|
|
472
|
+
|
|
473
|
+
@overload
|
|
474
|
+
def forward(
|
|
475
|
+
self,
|
|
476
|
+
query: Tensor,
|
|
477
|
+
key_value: Tensor | KVCacheEntry | None = None,
|
|
478
|
+
seqused_key_value: Tensor | None = None,
|
|
479
|
+
attn_mask: Tensor | None = None,
|
|
480
|
+
*,
|
|
481
|
+
return_key_value: bool,
|
|
482
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
483
|
+
out: Tensor | None = None,
|
|
484
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: ...
|
|
485
|
+
|
|
486
|
+
def forward(
|
|
487
|
+
self,
|
|
488
|
+
query: Tensor, # [..., Q, C]
|
|
489
|
+
key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
|
|
490
|
+
seqused_key_value: Tensor | None = None, # [...]
|
|
491
|
+
attn_mask: Tensor | None = None, # [..., Q, KV]
|
|
492
|
+
*,
|
|
493
|
+
return_key_value: bool = False,
|
|
494
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
495
|
+
out: Tensor | None = None,
|
|
496
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
|
|
497
|
+
r"""The forward pass.
|
|
498
|
+
|
|
499
|
+
Args:
|
|
500
|
+
query: The query tensor with shape ``[..., Q, C]``.
|
|
501
|
+
``Q`` is the query sequence length, ``C`` is the number of
|
|
502
|
+
channels.
|
|
503
|
+
key_value: The key/value tensor with shape ``[..., KV, C]`` or
|
|
504
|
+
precomputed key/value projections as a
|
|
505
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
506
|
+
``KV`` is the key/value sequence length.
|
|
507
|
+
If omitted, ``query`` is used for self-attention.
|
|
508
|
+
seqused_key_value: Valid key/value lengths with shape ``[...]`` and
|
|
509
|
+
:external+torch:ref:`torch.int32 <dtype-doc>` dtype.
|
|
510
|
+
attn_mask: Boolean attention mask with shape ``[..., Q, KV]``.
|
|
511
|
+
Entries set to ``True`` participate in attention.
|
|
512
|
+
return_key_value: Whether to return the computed key and value
|
|
513
|
+
projections alongside the block output.
|
|
514
|
+
batch_size_limit: Maximum number of batch elements processed at
|
|
515
|
+
once.
|
|
516
|
+
out: The output tensor.
|
|
517
|
+
|
|
518
|
+
Returns:
|
|
519
|
+
Tensor with shape ``[..., Q, C]`` when ``return_key_value`` is
|
|
520
|
+
``False``.
|
|
521
|
+
Otherwise, a tuple of the output tensor and a
|
|
522
|
+
:class:`~sdm.cache.KVCacheEntry`.
|
|
523
|
+
"""
|
|
524
|
+
if out is not None and torch.is_grad_enabled():
|
|
525
|
+
raise RuntimeError(
|
|
526
|
+
"'out' is only supported when gradients are disabled"
|
|
527
|
+
)
|
|
528
|
+
if return_key_value and isinstance(key_value, KVCacheEntry):
|
|
529
|
+
raise ValueError(
|
|
530
|
+
"'return_key_value=True' is not supported when 'key_value' is "
|
|
531
|
+
"already cached"
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
if torch.is_grad_enabled() or torch.compiler.is_compiling():
|
|
535
|
+
return self._forward(
|
|
536
|
+
query=query,
|
|
537
|
+
key_value=key_value,
|
|
538
|
+
seqused_key_value=seqused_key_value,
|
|
539
|
+
attn_mask=attn_mask,
|
|
540
|
+
return_key_value=return_key_value,
|
|
541
|
+
out=out,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
if batch_size_limit == "auto":
|
|
545
|
+
key_value_length: int | None = None
|
|
546
|
+
if isinstance(key_value, Tensor):
|
|
547
|
+
key_value_length = key_value.size(-2)
|
|
548
|
+
elif isinstance(key_value, KVCacheEntry):
|
|
549
|
+
key_value_length = key_value.key.size(-3)
|
|
550
|
+
|
|
551
|
+
batch_size_limit = self.auto_batch_size_limit(
|
|
552
|
+
device=query.device,
|
|
553
|
+
element_size=torch.get_autocast_dtype(
|
|
554
|
+
query.device.type
|
|
555
|
+
).itemsize
|
|
556
|
+
if torch.is_autocast_enabled(query.device.type)
|
|
557
|
+
else query.element_size(),
|
|
558
|
+
query_length=query.size(-2),
|
|
559
|
+
key_value_length=key_value_length,
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
batch_size_limit = min(batch_size_limit or 65_535, 65_535)
|
|
563
|
+
|
|
564
|
+
batch_shape = _batch_shape(
|
|
565
|
+
query=query,
|
|
566
|
+
key_value=key_value,
|
|
567
|
+
seqused_key_value=seqused_key_value,
|
|
568
|
+
attn_mask=attn_mask,
|
|
569
|
+
)
|
|
570
|
+
batch_size = math.prod(batch_shape)
|
|
571
|
+
|
|
572
|
+
disable_chunking = False
|
|
573
|
+
if batch_size <= batch_size_limit or query.size(-2) == 0:
|
|
574
|
+
disable_chunking = True
|
|
575
|
+
if return_key_value:
|
|
576
|
+
assert not isinstance(key_value, KVCacheEntry)
|
|
577
|
+
if batch_shape != (
|
|
578
|
+
query.size()[:-2]
|
|
579
|
+
if key_value is None
|
|
580
|
+
else key_value.size()[:-2]
|
|
581
|
+
):
|
|
582
|
+
disable_chunking = True
|
|
583
|
+
|
|
584
|
+
if disable_chunking:
|
|
585
|
+
return self._forward(
|
|
586
|
+
query=query,
|
|
587
|
+
key_value=key_value,
|
|
588
|
+
seqused_key_value=seqused_key_value,
|
|
589
|
+
attn_mask=attn_mask,
|
|
590
|
+
return_key_value=return_key_value,
|
|
591
|
+
out=out,
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
flat_out: Tensor | None = None
|
|
595
|
+
flat_key: Tensor | None = None
|
|
596
|
+
flat_value: Tensor | None = None
|
|
597
|
+
|
|
598
|
+
if out is not None and (out.dim() <= 3 or out.is_contiguous()):
|
|
599
|
+
flat_out = out.view(batch_size, *query.size()[-2:])
|
|
600
|
+
|
|
601
|
+
for start in range(0, batch_size, batch_size_limit):
|
|
602
|
+
end = min(start + batch_size_limit, batch_size)
|
|
603
|
+
|
|
604
|
+
result = self._forward(
|
|
605
|
+
query=_chunk(query, batch_shape, 2, start, end),
|
|
606
|
+
key_value=_chunk(key_value, batch_shape, 2, start, end),
|
|
607
|
+
seqused_key_value=_chunk(
|
|
608
|
+
seqused_key_value,
|
|
609
|
+
batch_shape,
|
|
610
|
+
trailing_dims=0,
|
|
611
|
+
start=start,
|
|
612
|
+
end=end,
|
|
613
|
+
),
|
|
614
|
+
attn_mask=_chunk(
|
|
615
|
+
attn_mask,
|
|
616
|
+
batch_shape,
|
|
617
|
+
trailing_dims=2,
|
|
618
|
+
start=start,
|
|
619
|
+
end=end,
|
|
620
|
+
),
|
|
621
|
+
return_key_value=return_key_value,
|
|
622
|
+
out=flat_out[start:end] if flat_out is not None else None,
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
if isinstance(result, tuple):
|
|
626
|
+
chunk, chunk_kv = result
|
|
627
|
+
|
|
628
|
+
if flat_key is None:
|
|
629
|
+
flat_key = chunk_kv.key.new_empty(
|
|
630
|
+
batch_size, *chunk_kv.key.size()[-3:]
|
|
631
|
+
)
|
|
632
|
+
if flat_value is None:
|
|
633
|
+
flat_value = chunk_kv.value.new_empty(
|
|
634
|
+
batch_size, *chunk_kv.value.size()[-3:]
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
flat_key[start:end] = chunk_kv.key
|
|
638
|
+
flat_value[start:end] = chunk_kv.value
|
|
639
|
+
|
|
640
|
+
del chunk_kv
|
|
641
|
+
|
|
642
|
+
else:
|
|
643
|
+
chunk = result
|
|
644
|
+
|
|
645
|
+
if flat_out is None and out is not None:
|
|
646
|
+
flat_index = torch.arange(start, end, device=out.device)
|
|
647
|
+
batch_indices: list[Tensor] = []
|
|
648
|
+
for size in reversed(batch_shape):
|
|
649
|
+
batch_indices.append(flat_index % size)
|
|
650
|
+
flat_index = flat_index // size
|
|
651
|
+
out[tuple(reversed(batch_indices))] = chunk.to(out.dtype)
|
|
652
|
+
elif flat_out is None:
|
|
653
|
+
flat_out = chunk.new_empty((batch_size, *query.size()[-2:]))
|
|
654
|
+
flat_out[start:end] = chunk
|
|
655
|
+
|
|
656
|
+
del chunk, result
|
|
657
|
+
|
|
658
|
+
if out is None:
|
|
659
|
+
assert flat_out is not None
|
|
660
|
+
out = flat_out.view(*batch_shape, *query.size()[-2:])
|
|
661
|
+
|
|
662
|
+
if not return_key_value:
|
|
663
|
+
return out
|
|
664
|
+
|
|
665
|
+
assert flat_key is not None
|
|
666
|
+
assert flat_value is not None
|
|
667
|
+
|
|
668
|
+
return out, KVCacheEntry(
|
|
669
|
+
key=flat_key.view(*batch_shape, *flat_key.size()[-3:]),
|
|
670
|
+
value=flat_value.view(*batch_shape, *flat_value.size()[-3:]),
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
def _forward(
|
|
674
|
+
self,
|
|
675
|
+
query: Tensor, # [..., Q, C]
|
|
676
|
+
key_value: Tensor | KVCacheEntry | None = None, # [..., KV, C]
|
|
677
|
+
seqused_key_value: Tensor | None = None, # [...]
|
|
678
|
+
attn_mask: Tensor | None = None, # [..., Q, KV]
|
|
679
|
+
*,
|
|
680
|
+
return_key_value: bool = False,
|
|
681
|
+
out: Tensor | None = None,
|
|
682
|
+
) -> Tensor | tuple[Tensor, KVCacheEntry]: # [..., Q, C]
|
|
683
|
+
|
|
684
|
+
if self.key_value_norm is None:
|
|
685
|
+
pass
|
|
686
|
+
elif key_value is None and self.key_value_norm is not self.query_norm:
|
|
687
|
+
key_value = self.key_value_norm(query)
|
|
688
|
+
elif isinstance(key_value, Tensor):
|
|
689
|
+
key_value = self.key_value_norm(key_value)
|
|
690
|
+
|
|
691
|
+
result = self.attn(
|
|
692
|
+
query=query if self.query_norm is None else self.query_norm(query),
|
|
693
|
+
key_value=key_value,
|
|
694
|
+
seqused_key_value=seqused_key_value,
|
|
695
|
+
attn_mask=attn_mask,
|
|
696
|
+
return_key_value=return_key_value,
|
|
697
|
+
)
|
|
698
|
+
del key_value
|
|
699
|
+
|
|
700
|
+
if return_key_value:
|
|
701
|
+
attn_out, kv = result
|
|
702
|
+
else:
|
|
703
|
+
attn_out = result
|
|
704
|
+
|
|
705
|
+
if self.post_attn_norm is not None:
|
|
706
|
+
attn_out = self.post_attn_norm(attn_out)
|
|
707
|
+
|
|
708
|
+
if (
|
|
709
|
+
out is not None
|
|
710
|
+
and torch.compiler.is_compiling()
|
|
711
|
+
and not out.is_contiguous()
|
|
712
|
+
):
|
|
713
|
+
# Match eager out= rounding before the MLP.
|
|
714
|
+
tmp = (attn_out + query).to(out.dtype)
|
|
715
|
+
out.copy_(tmp + self.mlp(tmp))
|
|
716
|
+
else:
|
|
717
|
+
tmp = torch.add(attn_out, query, out=out)
|
|
718
|
+
out = torch.add(tmp, self.mlp(tmp), out=out)
|
|
719
|
+
|
|
720
|
+
return (out, kv) if return_key_value else out
|
|
721
|
+
|
|
722
|
+
def peak_bytes_per_example(
|
|
723
|
+
self,
|
|
724
|
+
element_size: int,
|
|
725
|
+
query_length: int,
|
|
726
|
+
key_value_length: int | None = None,
|
|
727
|
+
) -> int:
|
|
728
|
+
r""":meta private:""" # noqa: D415
|
|
729
|
+
return 0
|
|
730
|
+
|
|
731
|
+
def auto_batch_size_limit(
|
|
732
|
+
self,
|
|
733
|
+
device: torch.device,
|
|
734
|
+
element_size: int,
|
|
735
|
+
query_length: int,
|
|
736
|
+
key_value_length: int | None = None,
|
|
737
|
+
) -> int:
|
|
738
|
+
r""":meta private:""" # noqa: D415
|
|
739
|
+
if device.type != "cuda":
|
|
740
|
+
return 65_535
|
|
741
|
+
bytes_per_example = self.peak_bytes_per_example(
|
|
742
|
+
element_size=element_size,
|
|
743
|
+
query_length=query_length,
|
|
744
|
+
key_value_length=key_value_length,
|
|
745
|
+
)
|
|
746
|
+
limit = chunk_memory_limit(device) // max(bytes_per_example, 1)
|
|
747
|
+
return min(max(limit, 1), 65_535)
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _batch_shape(
|
|
751
|
+
query: Tensor,
|
|
752
|
+
key_value: Tensor | KVCacheEntry | None,
|
|
753
|
+
seqused_key_value: Tensor | None,
|
|
754
|
+
attn_mask: Tensor | None,
|
|
755
|
+
) -> torch.Size:
|
|
756
|
+
shapes = [query.size()[:-2]]
|
|
757
|
+
if isinstance(key_value, Tensor):
|
|
758
|
+
shapes.append(key_value.size()[:-2])
|
|
759
|
+
elif isinstance(key_value, KVCacheEntry):
|
|
760
|
+
shapes += [key_value.key.size()[:-3], key_value.value.size()[:-3]]
|
|
761
|
+
if seqused_key_value is not None:
|
|
762
|
+
shapes.append(seqused_key_value.size())
|
|
763
|
+
if attn_mask is not None:
|
|
764
|
+
shapes.append(attn_mask.size()[:-2])
|
|
765
|
+
return torch.broadcast_shapes(*shapes)
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
@overload
|
|
769
|
+
def _chunk(
|
|
770
|
+
tensor: Tensor,
|
|
771
|
+
batch_shape: torch.Size,
|
|
772
|
+
trailing_dims: int,
|
|
773
|
+
start: int,
|
|
774
|
+
end: int,
|
|
775
|
+
) -> Tensor: ...
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
@overload
|
|
779
|
+
def _chunk(
|
|
780
|
+
tensor: KVCacheEntry,
|
|
781
|
+
batch_shape: torch.Size,
|
|
782
|
+
trailing_dims: int,
|
|
783
|
+
start: int,
|
|
784
|
+
end: int,
|
|
785
|
+
) -> KVCacheEntry: ...
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
@overload
|
|
789
|
+
def _chunk(
|
|
790
|
+
tensor: None,
|
|
791
|
+
batch_shape: torch.Size,
|
|
792
|
+
trailing_dims: int,
|
|
793
|
+
start: int,
|
|
794
|
+
end: int,
|
|
795
|
+
) -> None: ...
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _chunk(
|
|
799
|
+
tensor: Tensor | KVCacheEntry | None,
|
|
800
|
+
batch_shape: torch.Size,
|
|
801
|
+
trailing_dims: int,
|
|
802
|
+
start: int,
|
|
803
|
+
end: int,
|
|
804
|
+
) -> Tensor | KVCacheEntry | None:
|
|
805
|
+
|
|
806
|
+
if tensor is None:
|
|
807
|
+
return None
|
|
808
|
+
|
|
809
|
+
if isinstance(tensor, KVCacheEntry):
|
|
810
|
+
return KVCacheEntry(
|
|
811
|
+
key=_chunk(tensor.key, batch_shape, 3, start, end),
|
|
812
|
+
value=_chunk(tensor.value, batch_shape, 3, start, end),
|
|
813
|
+
)
|
|
814
|
+
|
|
815
|
+
trailing_shape = tensor.size()[-trailing_dims:] if trailing_dims else ()
|
|
816
|
+
tensor = tensor.expand(batch_shape + trailing_shape)
|
|
817
|
+
if len(batch_shape) == 1:
|
|
818
|
+
return tensor[start:end]
|
|
819
|
+
|
|
820
|
+
flat_index = torch.arange(start, end, device=tensor.device)
|
|
821
|
+
batch_indices: list[Tensor] = []
|
|
822
|
+
for size in reversed(batch_shape):
|
|
823
|
+
batch_indices.append(flat_index % size)
|
|
824
|
+
flat_index = flat_index // size
|
|
825
|
+
return tensor[tuple(reversed(batch_indices))]
|