x-transformers 1.42.21__tar.gz → 1.42.23__tar.gz

{x_transformers-1.42.21/x_transformers.egg-info → x_transformers-1.42.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.21
+Version: 1.42.23
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.42.21 → x_transformers-1.42.23}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.42.21',
+  version = '1.42.23',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x_transformers-1.42.21 → x_transformers-1.42.23}/x_transformers/x_transformers.py

@@ -1079,6 +1079,7 @@ class Attention(Module):
         neutreno_alpha = 0.4,
         learned_value_residual_mix = False,
         laser = False, # https://arxiv.org/abs/2411.03493v1
+        laser_softclamp_value = 15.,
         onnxable = False,
         attend_sdp_kwargs: dict = dict(
             enable_flash = True,
@@ -1119,9 +1120,9 @@ class Attention(Module):
         self.to_v = LinearNoBias(dim_kv, v_dim) if not shared_kv else None
 
         # enhancing gradients to attention through exponentiated values
-        # todo - compare it to `attn = attn * large_value + attn.detach() * (1. - large_value)`
 
         self.laser = laser
+        self.laser_softclamp_value = laser_softclamp_value
 
         # relations projection from tp-attention
 
@@ -1449,8 +1450,7 @@ class Attention(Module):
             attn_bias = pad_at_dim(attn_bias, (num_mem_kv, 0))
 
         if self.laser:
-            values_max = v.amax(dim = -2, keepdim = True).detach() # numerical stability
-            v = v - values_max
+            v = softclamp(v, self.laser_softclamp_value)
             v = v.exp()
 
         # attention is all we need
@@ -1465,7 +1465,7 @@ class Attention(Module):
         # laser
 
         if self.laser:
-            out = out.log() + values_max
+            out = log(out)
 
         # store the values for resformer or Neutreno
 

{x_transformers-1.42.21 → x_transformers-1.42.23/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.21
+Version: 1.42.23
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang