x-transformers 2.2.11__py3-none-any.whl → 2.2.12__py3-none-any.whl

x_transformers/x_transformers.py

@@ -62,6 +62,9 @@ def default(val, d):
         return val
     return d() if callable(d) else d
 
+def identity(t, *args, **kwargs):
+    return t
+
 def first(it, default = None):
     return it[0] if len(it) > 0 else default
 
@@ -74,7 +77,10 @@ def cast_tuple(val, depth = 1):
 def divisible_by(num, den):
     return (num % den) == 0
 
-def maybe(fn):
+def maybe(fn = None):
+    if not exists(fn):
+        fn = identity
+
     @wraps(fn)
     def inner(x, *args, **kwargs):
         if not exists(x):
@@ -1199,6 +1205,7 @@ class FeedForward(Module):
         custom_activation = None,
         post_act_ln = False,
         dropout = 0.,
+        sublayer_dropout = 0.,
         no_bias = False,
         zero_init_output = False
     ):
@@ -1227,7 +1234,8 @@ class FeedForward(Module):
             project_in,
             LayerNorm(inner_dim) if post_act_ln else None,
             nn.Dropout(dropout),
-            nn.Linear(inner_dim, dim_out, bias = not no_bias)
+            nn.Linear(inner_dim, dim_out, bias = not no_bias),
+            nn.Dropout(sublayer_dropout) if sublayer_dropout > 0. else None
         )
 
         # init last linear layer to 0
@@ -1256,6 +1264,7 @@ class Attention(Module):
         sparse_topk_straight_through = False,
         num_mem_kv = 0,
         dropout = 0.,
+        sublayer_dropout = 0.,
         on_attn = False,
         gate_value_heads = False,
         swiglu_values = False,
@@ -1534,6 +1543,10 @@ class Attention(Module):
         dim_out = default(dim_out, dim)
         self.to_out = nn.Sequential(LinearNoBias(out_dim, dim_out * 2), nn.GLU()) if on_attn else LinearNoBias(out_dim, dim_out)
 
+        # sublayer dropout
+
+        self.sublayer_dropout = nn.Dropout(sublayer_dropout) if sublayer_dropout > 0. else None
+
         # the number of attention heads to rotate, for decoupled rope in multi-latent attention
 
         rotate_num_heads = default(rotate_num_heads, heads)
@@ -1871,6 +1884,10 @@ class Attention(Module):
 
         out = self.to_out(out)
 
+        # maybe sublayer dropout
+
+        out = maybe(self.sublayer_dropout)(out)
+
         if exists(mask):
             out = einx.where('b n, b n d, -> b n d', mask, out, 0.)
 

{x_transformers-2.2.11.dist-info → x_transformers-2.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.2.11
+Version: 2.2.12
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.2.11.dist-info → x_transformers-2.2.12.dist-info}/RECORD

@@ -8,10 +8,10 @@ x_transformers/entropy_based_tokenizer.py,sha256=F2lO8-v3aLIcVDVNhu7RR-UtRdlmaaY
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=m2xiiTafFZiII-QZLCpPerdWbY8O41I6BAYCaaPdXig,111953
+x_transformers/x_transformers.py,sha256=MF91aJGr2DOjIGe57uqwgyNxCExBg_tI9z7usAJMxOM,112401
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.2.11.dist-info/METADATA,sha256=riLwnpQD_lZYsGehLw8rt8Pja2qt2k8lRepqDgMKGwA,88687
-x_transformers-2.2.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.2.11.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.2.11.dist-info/RECORD,,
+x_transformers-2.2.12.dist-info/METADATA,sha256=cWj_UYsNQNf2botGDqO7GkyiUh3msLww0EilFMMhRS0,88687
+x_transformers-2.2.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.2.12.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.2.12.dist-info/RECORD,,