x-transformers 2.3.6__py3-none-any.whl → 2.3.8__py3-none-any.whl

x_transformers/x_transformers.py

@@ -1208,7 +1208,7 @@ class FeedForward(Module):
         sublayer_dropout = 0.,
         no_bias = False,
         zero_init_output = False,
-        deep_embed_hiddens = False,
+        deep_embed = False,
         deep_embed_num_tokens = None,
     ):
         super().__init__()
@@ -1249,9 +1249,9 @@ class FeedForward(Module):
         # improvements were clearest to me (on my toy setup) with multiplying on output of feedforward, will try with attention at future date
 
         self.deep_embed = None
-        if deep_embed_hiddens:
+        if deep_embed:
             assert exists(deep_embed_num_tokens)
-            self.deep_embed = nn.Parameter(torch.zeros(deep_embed_num_tokens, dim_out))
+            self.deep_embed = nn.Parameter(torch.ones(deep_embed_num_tokens, dim_out))
 
         # init last linear layer to 0
 
@@ -1266,7 +1266,7 @@ class FeedForward(Module):
         out = self.ff(x)
 
         if exists(deep_embed_ids) and exists(self.deep_embed):
-            deep_embed = self.deep_embed[deep_embed_ids] + 1.
+            deep_embed = self.deep_embed[deep_embed_ids]
             out = out * deep_embed
 
         return out

{x_transformers-2.3.6.dist-info → x_transformers-2.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.3.6
+Version: 2.3.8
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.3.6.dist-info → x_transformers-2.3.8.dist-info}/RECORD

@@ -8,10 +8,10 @@ x_transformers/entropy_based_tokenizer.py,sha256=F2lO8-v3aLIcVDVNhu7RR-UtRdlmaaY
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=kZKk80hxV0Pvmx1E745BR7c8YzB-S4u2cZHSMZvpZq8,113507
+x_transformers/x_transformers.py,sha256=-9YbiPLh7w408Dxe5sGxkaOCPx__itHqCP6af6KrsxY,113485
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.3.6.dist-info/METADATA,sha256=Z337g7NRRYaKGbBHkKe1UZbIQJeXPk-dtZ4aBiVvSH8,89021
-x_transformers-2.3.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.3.6.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.3.6.dist-info/RECORD,,
+x_transformers-2.3.8.dist-info/METADATA,sha256=KV2fYRQSAkf1dJwWvH4rKf8GFz-WMEhVkAcxUVu1fNk,89021
+x_transformers-2.3.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.3.8.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.3.8.dist-info/RECORD,,