x-transformers 1.42.9__tar.gz → 1.42.11__tar.gz

{x_transformers-1.42.9/x_transformers.egg-info → x_transformers-1.42.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.9
+Version: 1.42.11
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.42.9 → x_transformers-1.42.11}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.42.9',
+  version = '1.42.11',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x_transformers-1.42.9 → x_transformers-1.42.11}/x_transformers/autoregressive_wrapper.py

@@ -48,7 +48,7 @@ def align_right(t, lens, pad_id = 0):
     batch_arange = torch.arange(batch, device = device, dtype = torch.long)[..., None]
     prompt_len_arange = torch.arange(seq_len, device = device, dtype = torch.long)
 
-    t = F.pad(t, (max_pad_len, 0), value = 0)
+    t = F.pad(t, (max_pad_len, 0), value = pad_id)
     offset = max_pad_len - pad_lens
 
     aligned = t[batch_arange, prompt_len_arange + offset[..., None]]

{x_transformers-1.42.9 → x_transformers-1.42.11}/x_transformers/continuous.py

@@ -2,7 +2,8 @@ import torch
 from torch import nn
 import torch.nn.functional as F
 
-from einops import pack, repeat, unpack
+import einx
+from einops import reduce, pack, repeat, unpack
 
 from x_transformers.x_transformers import (
     AttentionLayers,
@@ -24,6 +25,15 @@ def default(val, d):
         return val
     return d() if callable(d) else d
 
+def masked_mean(t, mask):
+    t = einx.where('b n, b n d, -> b n d', mask, t, 0.)
+
+    num = reduce(t, 'b n d -> b', 'sum')
+    den = mask.sum(dim = -1)
+
+    masked_average = num / den.clamp(min = 1.)
+    return masked_average
+
 # main classes
 
 class ContinuousTransformerWrapper(nn.Module):
@@ -169,12 +179,15 @@ class ContinuousAutoregressiveWrapper(nn.Module):
         net: ContinuousTransformerWrapper,
         ignore_index = -100,
         pad_value = 0,
-        loss_fn = nn.MSELoss(reduction = 'none')
+        loss_fn = nn.MSELoss(reduction = 'none'),
+        equal_loss_weight_batch = False  # setting this to True, if the mask is passed in and sequences are variable in length, each sequence will be weighted the same (as opposed to each token)
     ):
         super().__init__()
         self.net = net
         self.max_seq_len = net.max_seq_len
+
         self.loss_fn = loss_fn
+        self.equal_loss_weight_batch = equal_loss_weight_batch
 
     @torch.no_grad()
     def generate(self, start_tokens, seq_len, **kwargs):
@@ -222,6 +235,10 @@ class ContinuousAutoregressiveWrapper(nn.Module):
 
         if exists(mask):
             assert loss.ndim > 1, 'loss should not be reduced if mask is passed in'
-            loss = loss[mask]
+
+            if self.equal_loss_weight_batch:
+                loss = masked_mean(loss, mask)
+            else:
+                loss = loss[mask]
 
         return loss.mean()

{x_transformers-1.42.9 → x_transformers-1.42.11/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.42.9
+Version: 1.42.11
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang