x-transformers 1.42.25__tar.gz → 1.42.27__tar.gz

{x_transformers-1.42.25/x_transformers.egg-info → x_transformers-1.42.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.25
+Version: 1.42.27
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.42.25 → x_transformers-1.42.27}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.42.25',
+  version = '1.42.27',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x_transformers-1.42.25 → x_transformers-1.42.27}/tests/test_x_transformers.py

@@ -558,8 +558,10 @@ def test_laser():
 
     model(x)
 
+@pytest.mark.parametrize('self_attn_custom_pos', (True, False))
 @pytest.mark.parametrize('cross_attn_rotary', (True, False))
 def test_cross_attn_rotary(
+    self_attn_custom_pos: bool,
     cross_attn_rotary: bool
 ):
 
@@ -577,12 +579,14 @@ def test_cross_attn_rotary(
         cross_attn_dim_context = 512
     )
 
-    context_pos = torch.arange(128)
+    pos = torch.arange(64) if self_attn_custom_pos else None
+    context_pos = torch.arange(128) if cross_attn_rotary else None
 
     embed = model(
       x = x,
       mask = mask,
       context = context,
-      context_pos = context_pos if cross_attn_rotary else None,
+      pos = pos,
+      context_pos = context_pos,
       context_mask = context_mask
     )

{x_transformers-1.42.25 → x_transformers-1.42.27}/x_transformers/x_transformers.py

@@ -51,8 +51,8 @@ def default(val, d):
         return val
     return d() if callable(d) else d
 
-def first(it):
-    return it[0]
+def first(it, default = None):
+    return it[0] if len(it) > 0 else default
 
 def is_empty(x):
     return len(x) == 0
@@ -1077,7 +1077,7 @@ class Attention(Module):
         logit_softclamp_value = 50.,
         neutreno_value_residual = False, # Nguyen et al. https://arxiv.org/abs/2312.00751
         neutreno_alpha = 0.4,
-        learned_value_residual_mix = False,
+        learned_value_residual_mix = True,
         laser = False, # https://arxiv.org/abs/2411.03493v1
         laser_softclamp_value = 15.,
         onnxable = False,
@@ -1357,7 +1357,6 @@ class Attention(Module):
             k = k * self.qk_norm_k_scale
 
         if exists(rotary_pos_emb):
-
             freqs, xpos_scale = rotary_pos_emb
             q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale ** -1.) if exists(xpos_scale) else (1., 1.)
 
@@ -1989,7 +1988,7 @@ class AttentionLayers(Module):
 
         if exists(self.rotary_pos_emb):
             if not exists(rotary_pos_emb):
-                maybe_mem = mems[0] # todo - handle edge case where different layers get different memory lengths. don't think this will ever come up but who knows
+                maybe_mem = first(mems, None) # todo - handle edge case where different layers get different memory lengths. don't think this will ever come up but who knows
                 mem_len = maybe_mem.shape[1] if exists(maybe_mem) else 0
 
                 if not exists(pos):

{x_transformers-1.42.25 → x_transformers-1.42.27/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.25
+Version: 1.42.27
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang