x-transformers 1.42.21__py3-none-any.whl → 1.42.23__py3-none-any.whl

x_transformers/x_transformers.py

@@ -1079,6 +1079,7 @@ class Attention(Module):
         neutreno_alpha = 0.4,
         learned_value_residual_mix = False,
         laser = False, # https://arxiv.org/abs/2411.03493v1
+        laser_softclamp_value = 15.,
         onnxable = False,
         attend_sdp_kwargs: dict = dict(
             enable_flash = True,
@@ -1119,9 +1120,9 @@ class Attention(Module):
         self.to_v = LinearNoBias(dim_kv, v_dim) if not shared_kv else None
 
         # enhancing gradients to attention through exponentiated values
-        # todo - compare it to `attn = attn * large_value + attn.detach() * (1. - large_value)`
 
         self.laser = laser
+        self.laser_softclamp_value = laser_softclamp_value
 
         # relations projection from tp-attention
 
@@ -1449,8 +1450,7 @@ class Attention(Module):
             attn_bias = pad_at_dim(attn_bias, (num_mem_kv, 0))
 
         if self.laser:
-            values_max = v.amax(dim = -2, keepdim = True).detach() # numerical stability
-            v = v - values_max
+            v = softclamp(v, self.laser_softclamp_value)
             v = v.exp()
 
         # attention is all we need
@@ -1465,7 +1465,7 @@ class Attention(Module):
         # laser
 
         if self.laser:
-            out = out.log() + values_max
+            out = log(out)
 
         # store the values for resformer or Neutreno
 

{x_transformers-1.42.21.dist-info → x_transformers-1.42.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.21
+Version: 1.42.23
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.42.21.dist-info → x_transformers-1.42.23.dist-info}/RECORD

@@ -6,11 +6,11 @@ x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=CCXzE-lRhcKoymSHeRO_ZvL1XMOhR3YlVo3Ovyk5BZw,96069
+x_transformers/x_transformers.py,sha256=a5zP8ORv3x-cdkZkgwDDLBZ9wm3LmHDrqgqK2tCx-mU,95992
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-1.42.21.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.42.21.dist-info/METADATA,sha256=UEU2DHiMgksiQW5Ks9Mfvsup-d58vP_sR6DjLE8PSTQ,739
-x_transformers-1.42.21.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-x_transformers-1.42.21.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.42.21.dist-info/RECORD,,
+x_transformers-1.42.23.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.42.23.dist-info/METADATA,sha256=e6FRIaLa5dolzIpUoq4L3T-h1bqQa8I65bHeWuY9Uck,739
+x_transformers-1.42.23.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+x_transformers-1.42.23.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.42.23.dist-info/RECORD,,