x-transformers 1.30.10__py3-none-any.whl → 1.30.12__py3-none-any.whl

x_transformers/x_transformers.py

@@ -80,6 +80,13 @@ class equals():
     def __call__(self, x, *args, **kwargs):
         return x == self.val
 
+class Identity(Module):
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+
+    def __call__(self, x, *args, **kwargs):
+        return x
+
 def Sequential(*modules):
     return nn.Sequential(*filter(exists, modules))
 
@@ -314,21 +321,29 @@ class CoPE(Module):
         heads,
         max_pos,
         soft_onehot = False,
-        talking_heads = False
+        talking_heads = False,
+        soft_onehot_temp = 5e-2
     ):
         super () . __init__ ()
         self.max_pos = max_pos
         self.pos_emb = nn.Parameter(torch.zeros(max_pos, dim))
 
-        self.maybe_talking_heads = nn.Conv2d(heads, heads, 1, bias = False) if talking_heads else nn.Identity()
+        self.talking_heads = nn.Conv2d(heads, heads, 1, bias = False) if talking_heads else None
         self.soft_onehot = soft_onehot
+        self.soft_onehot_temp = soft_onehot_temp
 
         if soft_onehot:
             self.register_buffer('positions', torch.arange(max_pos))
 
-    def forward(self, query, attn_logits, temp = 5e-2):
+    def forward(self, query, attn_logits):
 
-        attn_logits = self.maybe_talking_heads(attn_logits)
+        if exists(self.talking_heads):
+            i, j = attn_logits.shape[-2:]
+            causal_mask = attn_logits.new_ones(i, j).triu_(j - i + 1).bool()
+
+            attn_logits = self.talking_heads(attn_logits)
+
+            attn_logits = attn_logits.masked_fill(causal_mask, -torch.finfo(attn_logits.dtype).max)
 
         # compute positions
 
@@ -341,7 +356,7 @@ class CoPE(Module):
 
         if self.soft_onehot:
             diff_pos = (pos[..., None] - self.positions).abs()
-            soft_onehot_pos = F.softmax(-diff_pos / temp, dim = -1)
+            soft_onehot_pos = F.softmax(-diff_pos / self.soft_onehot_temp, dim = -1)
             cope_pos_emb = einsum('b h i j p, b h i p -> b h i j', soft_onehot_pos, logits_int)
         else:
             # interpolate from integer positions
@@ -1121,6 +1136,7 @@ class AttentionLayers(Module):
         use_scalenorm = False,
         use_rmsnorm = False,
         use_simple_rmsnorm = False,
+        no_pre_or_postnorm = False,
         alibi_pos_bias = False,
         alibi_num_heads = None,
         rel_pos_bias = False,
@@ -1350,6 +1366,15 @@ class AttentionLayers(Module):
             residual_fn = GRUGating if gate_residual else Residual
             residual = residual_fn(dim, scale_residual = scale_residual, scale_residual_constant = scale_residual_constant)
 
+            # for the peri-layernorm config from https://arxiv.org/abs/2405.16039
+            # must be paired with qk norm
+
+            layer_norm_fn = norm_fn
+            if no_pre_or_postnorm:
+                layer_norm_fn = Identity()
+
+            # all normalizations of the layer
+
             pre_branch_norm = norm_fn() if pre_norm else None
             post_branch_norm = norm_fn() if sandwich_norm else None
             post_main_norm = norm_fn() if not pre_norm else None

{x_transformers-1.30.10.dist-info → x_transformers-1.30.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.30.10
+Version: 1.30.12
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.30.10.dist-info → x_transformers-1.30.12.dist-info}/RECORD

@@ -4,11 +4,11 @@ x_transformers/autoregressive_wrapper.py,sha256=uX8Mb0zLsQrZECt_9UGt35g7tC05Rk3n
 x_transformers/continuous.py,sha256=WO52n9lFAXv5-SGadi2cApGF8dkouN8QSTEOuC7erj8,6180
 x_transformers/dpo.py,sha256=LjvWgCkqTl-UuehrzQ8nkX5guLr4whYwsmm7SKSwdls,3450
 x_transformers/nonautoregressive_wrapper.py,sha256=ys_p8obc7lTeeodCqvkRKxOXQ1C9T3j5Jwr-JbVgnXk,10432
-x_transformers/x_transformers.py,sha256=qErIXwntVnZEmdO4MwTTsvxMc_vQPBU8zLLpEh535tE,68690
+x_transformers/x_transformers.py,sha256=hp7upZuaffVtrhZdL1Ra5sqBHcMrUMUlNvtbK8ilBPI,69497
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
 x_transformers/xval.py,sha256=QE1ltYZTR_eGgIHPP2BrMWVWVLqMW-OpDZh87BSmQEg,8563
-x_transformers-1.30.10.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.30.10.dist-info/METADATA,sha256=Fo2EWkxK4rLMcwUqcyeSW9AljUJA6b_2hZ59W8axIO4,662
-x_transformers-1.30.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-x_transformers-1.30.10.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.30.10.dist-info/RECORD,,
+x_transformers-1.30.12.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.30.12.dist-info/METADATA,sha256=agGqwtqqfvOYid5YrtmwaW46jxFU5kQ-QvTke5EhuiE,662
+x_transformers-1.30.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+x_transformers-1.30.12.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.30.12.dist-info/RECORD,,