x-transformers 1.40.0__tar.gz → 1.40.2__tar.gz

{x_transformers-1.40.0/x_transformers.egg-info → x_transformers-1.40.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.40.0
+Version: 1.40.2
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.40.0 → x_transformers-1.40.2}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.40.0',
+  version = '1.40.2',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x_transformers-1.40.0 → x_transformers-1.40.2}/tests/test_x_transformers.py

@@ -280,28 +280,6 @@ def test_mos():
 
     eval_logits = model(x)
 
-def test_sigsoftmax():
-    model = TransformerWrapper(
-        num_tokens = 20000,
-        max_seq_len = 1024,
-        mixture_of_softmax = True,
-        sigsoftmax_logits = True,
-        attn_layers = Decoder(
-            attn_sigsoftmax = True,
-            dim = 128,
-            depth = 6,
-            heads = 8
-        )
-    )
-
-    x = torch.randint(0, 20000, (2, 1024))
-
-    logits = model(x)
-
-    model.eval()
-
-    eval_logits = model(x)
-
 @pytest.mark.parametrize('attn_one_kv_head', (True, False))
 def test_l2_distance(attn_one_kv_head):
 

{x_transformers-1.40.0 → x_transformers-1.40.2}/x_transformers/attend.py

@@ -119,7 +119,12 @@ def one_hot_straight_through(logits, temperature = 1.):
 # sparse topk attention - only keep topk attn logits for softmax
 # optional straight through with masked out logits by setting `attn_sparse_topk_straight_through = True`
 
-def sparse_topk_attn(logits, sparse_topk, temperature = 1., straight_through = False):
+def sparse_topk_attn(
+    logits,
+    sparse_topk,
+    temperature = 1.,
+    straight_through = False
+):
     orig_logits = logits
 
     mask_value = -torch.finfo(logits.dtype).max
@@ -132,7 +137,7 @@ def sparse_topk_attn(logits, sparse_topk, temperature = 1., straight_through = F
         return topk_attn
 
     soft_attn = (orig_logits / temperature).softmax(dim = -1)
-    return topk_attn + soft_attn - soft_attn.detach()
+    return topk_attn.detach() + soft_attn - soft_attn.detach()
 
 # functions for creating causal mask
 # need a special one for onnx cpu (no support for .triu)

{x_transformers-1.40.0 → x_transformers-1.40.2}/x_transformers/x_transformers.py

@@ -1107,6 +1107,7 @@ class Attention(Module):
         context_mask = None,
         attn_mask = None,
         rel_pos = None,
+        attn_bias = None,
         rotary_pos_emb = None,
         prev_attn = None,
         mem = None,
@@ -1237,8 +1238,8 @@ class Attention(Module):
 
         # prepare relative positional bias, if needed
 
-        attn_bias = None
         if exists(rel_pos):
+            assert not exists(attn_bias)
             attn_bias = rel_pos(i, j)
             attn_bias = pad_at_dim(attn_bias, (num_mem_kv, 0), value = 0.) # handle memory key / values
 

{x_transformers-1.40.0 → x_transformers-1.40.2/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.40.0
+Version: 1.40.2
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang