x-transformers 2.10.0__py3-none-any.whl → 2.10.2__py3-none-any.whl

x_transformers/attend.py

@@ -520,6 +520,7 @@ class Attend(Module):
 
         if self.cog_signed:
             sim_sign = sim.sign()
+            sim = sim.abs()
 
         # masking
 
@@ -548,13 +549,15 @@ class Attend(Module):
         if self.head_learned_sink:
             # add learned attention sink
             attn_sink = repeat(self.head_attn_sink, 'h -> b h i 1', b = sim.shape[0], i = sim.shape[2])
+
+            if self.cog_signed:
+                attn_sink, attn_sink_sign = attn_sink.abs(), attn_sink.sign()
+                sim_sign = cat((attn_sink_sign, sim_sign), dim = -1)
+
             sim = cat((attn_sink, sim), dim = -1)
 
         pre_softmax_attn = sim
 
-        if self.cog_signed:
-            sim = sim.abs()
-
         attn = self.attn_fn(sim)
 
         attn = attn.type(dtype)

{x_transformers-2.10.0.dist-info → x_transformers-2.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.10.0
+Version: 2.10.2
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers
@@ -2587,13 +2587,14 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 ```
 
 ```bibtex
-@inproceedings{anonymous2025more,
-    title   = {More Expressive Attention with Negative Weights},
-    author  = {Anonymous},
-    booktitle = {Submitted to The Fourteenth International Conference on Learning Representations},
+@misc{lv2025expressiveattentionnegativeweights,
+    title   = {More Expressive Attention with Negative Weights}, 
+    author  = {Ang Lv and Ruobing Xie and Shuaipeng Li and Jiayi Liao and Xingwu Sun and Zhanhui Kang and Di Wang and Rui Yan},
     year    = {2025},
-    url     = {https://openreview.net/forum?id=ezRrwwbxd0},
-    note    = {under review}
+    eprint  = {2411.07176},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.CL},
+    url     = {https://arxiv.org/abs/2411.07176}, 
 }
 ```
 

{x_transformers-2.10.0.dist-info → x_transformers-2.10.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 x_transformers/__init__.py,sha256=aVuhUU0572TJHW88BVc4yA2tla0Zb8l3NH7W4RZ1AEs,1005
-x_transformers/attend.py,sha256=l968RkOaypWMb_Ba-n82zKms4b62Ng337wtigvPAums,19236
+x_transformers/attend.py,sha256=vrFPCfr3WwsyMZJxn1Pox_8VHZVLVSMuXThW3eZmd5Q,19388
 x_transformers/autoregressive_wrapper.py,sha256=BsGO9xfVYkvynqbU1__tu_S_cxl7gss0YwnkhIa2baY,18401
 x_transformers/belief_state_wrapper.py,sha256=YLUMk6t2MhFBEw5lHDDHJHcoCxTIkHvxTNY__GGZEKU,13374
 x_transformers/continuous.py,sha256=WwpQCjyVY4PtuEAOFY68zqgklbF9I7AL5w6874YlDe8,13249
@@ -13,7 +13,7 @@ x_transformers/up_wrapper.py,sha256=YC2LN14_7Xx9Wtiek2rtEJ_qHqdfSmKlh3d7Cgxwd80,
 x_transformers/x_transformers.py,sha256=ADr83Fz2cehj_F7N1bMwxhAg-r48fGhlaZqw3hxoxMQ,125765
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=AwwYUm8yDAtKQyKJDIhYMsiLTJ_skh3scUFMjp5sda8,8597
-x_transformers-2.10.0.dist-info/METADATA,sha256=1tiahG4NWO99cWEZ_qRgdgKHSWRIUKdf0xl2j0BfIXQ,95736
-x_transformers-2.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.10.0.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.10.0.dist-info/RECORD,,
+x_transformers-2.10.2.dist-info/METADATA,sha256=VS0evtudpPIsBv4vWQFSn34IHkOEXoyJ7g797HqJd_A,95799
+x_transformers-2.10.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.10.2.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.10.2.dist-info/RECORD,,