x-transformers 1.27.19__tar.gz → 1.27.21__tar.gz

{x-transformers-1.27.19/x_transformers.egg-info → x_transformers-1.27.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.27.19
+Version: 1.27.21
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x-transformers-1.27.19 → x_transformers-1.27.21}/README.md

@@ -1437,11 +1437,10 @@ model = XValAutoregressiveWrapper(model)
 
 ids = torch.randint(0, 4, (1, 777))
 nums = torch.randn(1, 777)
-mask = torch.ones(1, 777).bool()
 
 # train on a lot of data above
 
-loss = model(ids, nums, mask = mask)
+loss = model(ids, nums)
 loss.backward()
 
 # then generate

{x-transformers-1.27.19 → x_transformers-1.27.21}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.27.19',
+  version = '1.27.21',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

{x-transformers-1.27.19 → x_transformers-1.27.21}/x_transformers/x_transformers.py

@@ -489,16 +489,6 @@ class Scale(nn.Module):
 
         return (scale_fn(out[0]), *out[1:])
 
-class ScaleNorm(nn.Module):
-    def __init__(self, dim, eps = 1e-5):
-        super().__init__()
-        self.eps = eps
-        self.g = nn.Parameter(torch.ones(1) * (dim ** -0.5))
-
-    def forward(self, x):
-        norm = torch.norm(x, dim = -1, keepdim = True)
-        return x / norm.clamp(min = self.eps) * self.g
-
 class LayerNorm(nn.Module):
     def __init__(self, dim):
         """
@@ -514,6 +504,15 @@ class LayerNorm(nn.Module):
 if version.parse(torch.__version__) >= version.parse('2.1.0'):
     LayerNorm = partial(nn.LayerNorm, bias = False)
 
+class ScaleNorm(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.scale = dim ** 0.5
+        self.g = nn.Parameter(torch.ones(1))
+
+    def forward(self, x):
+        return F.normalize(x, dim = -1) * self.scale * self.g
+
 class RMSNorm(nn.Module):
     def __init__(self, dim):
         super().__init__()

{x-transformers-1.27.19 → x_transformers-1.27.21}/x_transformers/xval.py

@@ -271,8 +271,17 @@ class XValAutoregressiveWrapper(nn.Module):
 
         cross_entropy_loss = F.cross_entropy(logits, target, reduction = 'none', ignore_index = self.ignore_index)
 
+        # protect against nan in `x_num` input tensor
+
+        target_is_number_mask = target == self.net.numerical_token_id
+        x_num_target = x_num_target.masked_fill(~target_is_number_mask, 0.)
+
+        # ignore index
+
         target_mask = target != self.ignore_index
 
+        # numerical mse loss
+
         numerical_mse_loss = F.mse_loss(numerical_pred, x_num_target, reduction = 'none')
 
         numerical_mse_loss = numerical_mse_loss * target_mask

{x-transformers-1.27.19 → x_transformers-1.27.21/x_transformers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.27.19
+Version: 1.27.21
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang