x-transformers 1.41.0__py3-none-any.whl → 1.41.1__py3-none-any.whl

x_transformers/x_transformers.py

@@ -101,12 +101,6 @@ def log(t, eps = 1e-20):
 def max_neg_value(tensor):
     return -torch.finfo(tensor.dtype).max
 
-def reverse_cumsum(t, dim = -1):
-    t = t.flip(dims = (dim,))
-    t = t.cumsum(dim = dim)
-    t = t.flip(dims = (dim,))
-    return t
-
 def l2norm(t, groups = 1):
     t = rearrange(t, '... (g d) -> ... g d', g = groups)
     t = F.normalize(t, p = 2, dim = -1)
@@ -514,7 +508,7 @@ class DataDependentAlibi(Module):
         self.to_forget_gates = nn.Sequential(
             linear,
             Rearrange('b n h -> b h n'),
-            nn.Sigmoid()
+            nn.LogSigmoid()
         )
 
         nn.init.constant_(linear.bias, 5.)
@@ -522,18 +516,9 @@ class DataDependentAlibi(Module):
     def forward(self, x):
         seq = x.shape[-2]
 
-        forget_gates = self.to_forget_gates(x).log()
-        forget_gates = repeat(forget_gates, 'b h j -> b h i j', i = seq)
-
-        # causal mask out, including diagonal (so token to itself attention is never masked out)
-
-        causal_mask = torch.ones((seq, seq), dtype = torch.bool, device = x.device).triu()
-
-        forget_gates = forget_gates.masked_fill(causal_mask, 0.)
-
-        # reverse cumulative sum in log space (equivalent to cumprod)
-
-        forget_gates = reverse_cumsum(forget_gates)
+        forget_gates = self.to_forget_gates(x)
+        forget_gates = forget_gates.cumsum(dim = -1)
+        forget_gates = einx.subtract('b h i, b h j -> b h i j', forget_gates, forget_gates)
 
         return forget_gates
 

{x_transformers-1.41.0.dist-info → x_transformers-1.41.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.41.0
+Version: 1.41.1
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.41.0.dist-info → x_transformers-1.41.1.dist-info}/RECORD

@@ -5,11 +5,11 @@ x_transformers/continuous.py,sha256=cIVEdhfei258__ziV7kQBrJMxCel54bExBTDrO9rfCI,
 x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=fz71zW2IQ3NQU_csHbzCwFzGHNwrdIF9rZTLhUjmM_Q,90260
+x_transformers/x_transformers.py,sha256=n8W19Pnhbz-JxbC7QATApWrhI_yC4oqTHGQ1NLuindY,89814
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-1.41.0.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.41.0.dist-info/METADATA,sha256=832V1ChJ77viLmQO7d9RU7R9SV_bspLgVl9vtdRdq5Q,689
-x_transformers-1.41.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-x_transformers-1.41.0.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.41.0.dist-info/RECORD,,
+x_transformers-1.41.1.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.41.1.dist-info/METADATA,sha256=UIPYbEVBLrWDGuezlnyh2tFKPlM_Mdj-pYTGxse_NMI,689
+x_transformers-1.41.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+x_transformers-1.41.1.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.41.1.dist-info/RECORD,,