broccoli-ml 10.0.1__py3-none-any.whl → 10.1.1__py3-none-any.whl

broccoli/transformer.py

@@ -710,6 +710,11 @@ class TransformerEncoder(nn.Module):
         self._utility_tokens = utility_tokens
         self.return_utility_tokens = return_utility_tokens
 
+        if layerscale:
+            self.layerscale = LayerScale(d_model)
+        else:
+            self.layerscale = None
+
         # Initialise utility tokens with normal init, like usual Pytorch embeddings
         if self._utility_tokens:
             self._utility_token_embedding = nn.Parameter(
@@ -795,13 +800,16 @@ class TransformerEncoder(nn.Module):
             x = x
 
         if self.absolute_position_embedding is not None:
-            x = x + self.absolute_position_embedding(
+            position_embedding = self.absolute_position_embedding(
                 torch.arange(
                     0, self.full_sequence_length, dtype=torch.long, device=x.device
                 ).unsqueeze(
                     0
                 )  # to shape (1, seq_len) to broadcast over batch
             )
+            if self.layerscale is not None:
+                position_embedding = self.layerscale(position_embedding)
+            x += position_embedding
 
         return x
 

{broccoli_ml-10.0.1.dist-info → broccoli_ml-10.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: broccoli-ml
-Version: 10.0.1
+Version: 10.1.1
 Summary: Some useful Pytorch models, circa 2025
 License: MIT
 Author: Nicholas Bailey

{broccoli_ml-10.0.1.dist-info → broccoli_ml-10.1.1.dist-info}/RECORD

@@ -4,10 +4,10 @@ broccoli/cnn.py,sha256=WjoPDSpe3ttwxCBNfCVRdaCHvbeZ7G-a5_i8fUsK_d8,4889
 broccoli/linear.py,sha256=W-3aNpBjd_0xRyzbCKkmg4H1qmslQOIQhB-WDDay2nM,13125
 broccoli/rope.py,sha256=GRqApBNmYCFaDak0WL1xE_BC5CTTYKQU_PBdeTcQcjc,12557
 broccoli/tensor.py,sha256=um8mrxkYbvNDo-QvHlmJm8Aw6qcngOlUZPoAk_PMReA,4480
-broccoli/transformer.py,sha256=lnfiv7UIYbABiClIluy6CefGxaiYMrvBcj2Ul0uU6xE,27693
+broccoli/transformer.py,sha256=a2l1uRWr6wD6Q84XdYmTsimNqBxDDmbK719b6fwKCik,27982
 broccoli/utils.py,sha256=oOWzn6dJ5nC_9r4zq0emmfmaYACJXJNFS48AOpW2jqc,358
 broccoli/vit.py,sha256=EGbQb-atuzG3JAx7kdTaJEbWvQR-4XgyYvwjKkN5C38,22612
-broccoli_ml-10.0.1.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
-broccoli_ml-10.0.1.dist-info/METADATA,sha256=65GKe2Jor5jgUZ8zxROntJ_t0XwAlaukrvpT7nxS0lQ,1369
-broccoli_ml-10.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-broccoli_ml-10.0.1.dist-info/RECORD,,
+broccoli_ml-10.1.1.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
+broccoli_ml-10.1.1.dist-info/METADATA,sha256=fi_mTSGNka5gnIfIVuvaunXqzbppH7NXMZR4ierpVbY,1369
+broccoli_ml-10.1.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+broccoli_ml-10.1.1.dist-info/RECORD,,