x-transformers 2.3.6__tar.gz → 2.3.8__tar.gz

{x_transformers-2.3.6 → x_transformers-2.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.3.6
+Version: 2.3.8
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.3.6 → x_transformers-2.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.3.6"
+version = "2.3.8"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{x_transformers-2.3.6 → x_transformers-2.3.8}/tests/test_x_transformers.py

@@ -858,7 +858,7 @@ def test_ff_deep_embed():
             depth = 6,
             heads = 8,
             rotary_pos_emb = True,
-            ff_deep_embed_hiddens = True,
+            ff_deep_embed = True,
             ff_deep_embed_num_tokens = 20000,
         )
     )

{x_transformers-2.3.6 → x_transformers-2.3.8}/x_transformers/x_transformers.py

@@ -1208,7 +1208,7 @@ class FeedForward(Module):
         sublayer_dropout = 0.,
         no_bias = False,
         zero_init_output = False,
-        deep_embed_hiddens = False,
+        deep_embed = False,
         deep_embed_num_tokens = None,
     ):
         super().__init__()
@@ -1249,9 +1249,9 @@ class FeedForward(Module):
         # improvements were clearest to me (on my toy setup) with multiplying on output of feedforward, will try with attention at future date
 
         self.deep_embed = None
-        if deep_embed_hiddens:
+        if deep_embed:
             assert exists(deep_embed_num_tokens)
-            self.deep_embed = nn.Parameter(torch.zeros(deep_embed_num_tokens, dim_out))
+            self.deep_embed = nn.Parameter(torch.ones(deep_embed_num_tokens, dim_out))
 
         # init last linear layer to 0
 
@@ -1266,7 +1266,7 @@ class FeedForward(Module):
         out = self.ff(x)
 
         if exists(deep_embed_ids) and exists(self.deep_embed):
-            deep_embed = self.deep_embed[deep_embed_ids] + 1.
+            deep_embed = self.deep_embed[deep_embed_ids]
             out = out * deep_embed
 
         return out