broccoli-ml 15.3.0__py3-none-any.whl → 15.4.1__py3-none-any.whl

broccoli/vit.py

@@ -520,9 +520,13 @@ class ViT(nn.Module):
                 "SwiGLU": SwiGLU,
             }[transformer_activation]
 
-        # Set alpha and beta according to Microsoft's DeepNorm
-        self.alpha = (2 * transformer_layers) ** 0.25
-        self.beta = (8 * transformer_layers) ** -0.25
+        # Set alpha according to Microsoft's DeepNorm if layers > 50
+        if transformer_layers > 50:
+            self.alpha = (2 * transformer_layers) ** 0.25
+            self.beta = 1.0  # beta is not needed as we norm the Q and K vectors in MSA!
+        else:
+            self.alpha = 1.0
+            self.beta = 1.0  # beta is not needed as we norm the Q and K vectors in MSA!
 
         self.encoder = ViTEncoder(
             input_size=input_size,

{broccoli_ml-15.3.0.dist-info → broccoli_ml-15.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: broccoli-ml
-Version: 15.3.0
+Version: 15.4.1
 Summary: Some useful Pytorch models, circa 2025
 License: MIT
 Author: Nicholas Bailey

{broccoli_ml-15.3.0.dist-info → broccoli_ml-15.4.1.dist-info}/RECORD

@@ -6,8 +6,8 @@ broccoli/rope.py,sha256=GRqApBNmYCFaDak0WL1xE_BC5CTTYKQU_PBdeTcQcjc,12557
 broccoli/tensor.py,sha256=um8mrxkYbvNDo-QvHlmJm8Aw6qcngOlUZPoAk_PMReA,4480
 broccoli/transformer.py,sha256=XCr3fkDrYvNEA3m5rdVDHU5qbf1IYJQS1Cjgmn5RX-o,28177
 broccoli/utils.py,sha256=oOWzn6dJ5nC_9r4zq0emmfmaYACJXJNFS48AOpW2jqc,358
-broccoli/vit.py,sha256=9z1MJce38dppX35OGJvmnNIOnt-euMVV4n160W_aQYU,22799
-broccoli_ml-15.3.0.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
-broccoli_ml-15.3.0.dist-info/METADATA,sha256=StOW7vZd1c4xNddKcA8XJ-zbXgUnAsrQlg9G97wT-2w,1369
-broccoli_ml-15.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-broccoli_ml-15.3.0.dist-info/RECORD,,
+broccoli/vit.py,sha256=lhWOcVg4GfTHJp3ABflKjQQzWGKMqrOU6QKqRWU11wo,23012
+broccoli_ml-15.4.1.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
+broccoli_ml-15.4.1.dist-info/METADATA,sha256=Vg2KdoHSlbV1IDYuNk8h07sVT1UTW69bt49JLiEr0os,1369
+broccoli_ml-15.4.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+broccoli_ml-15.4.1.dist-info/RECORD,,