broccoli-ml 4.0.0__py3-none-any.whl → 4.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- broccoli/transformer.py +3 -1
- {broccoli_ml-4.0.0.dist-info → broccoli_ml-4.0.1.dist-info}/METADATA +1 -1
- {broccoli_ml-4.0.0.dist-info → broccoli_ml-4.0.1.dist-info}/RECORD +5 -5
- {broccoli_ml-4.0.0.dist-info → broccoli_ml-4.0.1.dist-info}/LICENSE +0 -0
- {broccoli_ml-4.0.0.dist-info → broccoli_ml-4.0.1.dist-info}/WHEEL +0 -0
broccoli/transformer.py
CHANGED
|
@@ -233,7 +233,7 @@ class MHAttention(nn.Module):
|
|
|
233
233
|
q,
|
|
234
234
|
k,
|
|
235
235
|
v,
|
|
236
|
-
dropout_p=self.dropout if self.training else 0.0,
|
|
236
|
+
dropout_p=self.dropout.p if self.training else 0.0,
|
|
237
237
|
softmax_scale=scaling_factor,
|
|
238
238
|
causal=self.causal,
|
|
239
239
|
)
|
|
@@ -257,6 +257,8 @@ class MHAttention(nn.Module):
|
|
|
257
257
|
|
|
258
258
|
qk_scores = F.softmax(qk_scores, dim=-1)
|
|
259
259
|
|
|
260
|
+
qk_scores = self.dropout(qk_scores)
|
|
261
|
+
|
|
260
262
|
output_with_heads = qk_scores @ v
|
|
261
263
|
|
|
262
264
|
output_without_heads = rearrange(output_with_heads, "b h t d -> b t (h d)")
|
|
@@ -4,10 +4,10 @@ broccoli/cnn.py,sha256=WjoPDSpe3ttwxCBNfCVRdaCHvbeZ7G-a5_i8fUsK_d8,4889
|
|
|
4
4
|
broccoli/linear.py,sha256=Y7s-DzcwsOipRboNHc4HTScw4mJRalNoVFsNcxOB6a4,4872
|
|
5
5
|
broccoli/rope.py,sha256=GRqApBNmYCFaDak0WL1xE_BC5CTTYKQU_PBdeTcQcjc,12557
|
|
6
6
|
broccoli/tensor.py,sha256=um8mrxkYbvNDo-QvHlmJm8Aw6qcngOlUZPoAk_PMReA,4480
|
|
7
|
-
broccoli/transformer.py,sha256=
|
|
7
|
+
broccoli/transformer.py,sha256=uqSf8q30MF7Ds7LfqW8Pr206NXpSlf7o6770KZu81Ew,19883
|
|
8
8
|
broccoli/utils.py,sha256=oOWzn6dJ5nC_9r4zq0emmfmaYACJXJNFS48AOpW2jqc,358
|
|
9
9
|
broccoli/vit.py,sha256=_5uLcklmJ1Uoj7V7TkzF0UqroVnl8NCHun5B0mORmOg,18651
|
|
10
|
-
broccoli_ml-4.0.
|
|
11
|
-
broccoli_ml-4.0.
|
|
12
|
-
broccoli_ml-4.0.
|
|
13
|
-
broccoli_ml-4.0.
|
|
10
|
+
broccoli_ml-4.0.1.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
|
|
11
|
+
broccoli_ml-4.0.1.dist-info/METADATA,sha256=vH_utDdo0-e2q8ReDrRHQ1d6fOzG4nzb9EWlqgyl4XY,1368
|
|
12
|
+
broccoli_ml-4.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
13
|
+
broccoli_ml-4.0.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|