x-transformers 1.30.7__py3-none-any.whl → 1.30.9__py3-none-any.whl

x_transformers/x_transformers.py

@@ -308,28 +308,46 @@ class CoPE(Module):
     """
     Appendix B of https://arxiv.org/abs/2405.18719
     """
-    def __init__ (self, dim, max_pos):
+    def __init__ (
+        self,
+        dim,
+        max_pos,
+        soft_onehot = False,
+    ):
         super () . __init__ ()
         self.max_pos = max_pos
         self.pos_emb = nn.Parameter(torch.zeros(max_pos, dim))
 
-    def forward(self, query, attn_logits):
+        self.soft_onehot = soft_onehot
+
+        if soft_onehot:
+            self.register_buffer('positions', torch.arange(max_pos))
+
+    def forward(self, query, attn_logits, temp = 5e-2):
         # compute positions
 
         gates = attn_logits.sigmoid()
+
         pos = gates.flip(-1).cumsum(dim = -1).flip(-1)
         pos = pos.clamp(max = self.max_pos - 1)
 
-        # interpolate from integer positions
-
-        pos_ceil = pos.ceil().long()
-        pos_floor = pos.floor().long()
         logits_int = einsum('b h n d, p d -> b h n p', query, self.pos_emb)
-        logits_ceil = logits_int.gather(-1, pos_ceil)
-        logits_floor = logits_int.gather(-1, pos_floor)
 
-        w = pos - pos_floor
-        return logits_ceil * w + logits_floor * (1 - w)
+        if self.soft_onehot:
+            diff_pos = (pos[..., None] - self.positions).abs()
+            soft_onehot_pos = F.softmax(-diff_pos / temp, dim = -1)
+            cope_pos_emb = einsum('b h i j p, b h i p -> b h i j', soft_onehot_pos, logits_int)
+        else:
+            # interpolate from integer positions
+            pos_ceil = pos.ceil().long()
+            pos_floor = pos.floor().long()
+            logits_ceil = logits_int.gather(-1, pos_ceil)
+            logits_floor = logits_int.gather(-1, pos_floor)
+
+            w = pos - pos_floor
+            cope_pos_emb = logits_ceil * w + logits_floor * (1 - w)
+
+        return cope_pos_emb
 
 class DynamicPositionBias(Module):
     def __init__(self, dim, *, heads, depth, log_distance = False, norm = False):
@@ -751,6 +769,7 @@ class Attention(Module):
         rotary_embed_values = False,
         use_cope = False,
         cope_max_pos = 16,
+        cope_soft_onehot_pos = False,
         logit_softclamp_value = None,
         onnxable = False
     ):
@@ -834,7 +853,7 @@ class Attention(Module):
             assert causal, 'CoPE was designed for causal attention'
             assert not flash, 'CoPE is not flash attention compatible'
 
-            cope = CoPE(dim_head, cope_max_pos)
+            cope = CoPE(dim_head, cope_max_pos, soft_onehot = cope_soft_onehot_pos)
 
         # attend class - includes core attention algorithm + talking heads
 

{x_transformers-1.30.7.dist-info → x_transformers-1.30.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.30.7
+Version: 1.30.9
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.30.7.dist-info → x_transformers-1.30.9.dist-info}/RECORD

@@ -4,11 +4,11 @@ x_transformers/autoregressive_wrapper.py,sha256=uX8Mb0zLsQrZECt_9UGt35g7tC05Rk3n
 x_transformers/continuous.py,sha256=WO52n9lFAXv5-SGadi2cApGF8dkouN8QSTEOuC7erj8,6180
 x_transformers/dpo.py,sha256=LjvWgCkqTl-UuehrzQ8nkX5guLr4whYwsmm7SKSwdls,3450
 x_transformers/nonautoregressive_wrapper.py,sha256=ys_p8obc7lTeeodCqvkRKxOXQ1C9T3j5Jwr-JbVgnXk,10432
-x_transformers/x_transformers.py,sha256=r9F_LLp5bQyAlue3bBTRwoRx02noTCh4ICF8oWCw1wE,67657
+x_transformers/x_transformers.py,sha256=V_kR77GhWNu3TuUetu3xryWoFliwTJdNKRX2lVWnsRc,68274
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
 x_transformers/xval.py,sha256=QE1ltYZTR_eGgIHPP2BrMWVWVLqMW-OpDZh87BSmQEg,8563
-x_transformers-1.30.7.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.30.7.dist-info/METADATA,sha256=_TniCg2s6tlimpfzpWeMCsCMOjsoYwUObBiXFdY-JhA,661
-x_transformers-1.30.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-x_transformers-1.30.7.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.30.7.dist-info/RECORD,,
+x_transformers-1.30.9.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.30.9.dist-info/METADATA,sha256=auoB6F1DCe054gY70tE1k048x6wQRtftk8-Pk6nJD-I,661
+x_transformers-1.30.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+x_transformers-1.30.9.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.30.9.dist-info/RECORD,,