x-transformers 1.27.19__py3-none-any.whl → 1.27.21__py3-none-any.whl

x_transformers/x_transformers.py

@@ -489,16 +489,6 @@ class Scale(nn.Module):
 
         return (scale_fn(out[0]), *out[1:])
 
-class ScaleNorm(nn.Module):
-    def __init__(self, dim, eps = 1e-5):
-        super().__init__()
-        self.eps = eps
-        self.g = nn.Parameter(torch.ones(1) * (dim ** -0.5))
-
-    def forward(self, x):
-        norm = torch.norm(x, dim = -1, keepdim = True)
-        return x / norm.clamp(min = self.eps) * self.g
-
 class LayerNorm(nn.Module):
     def __init__(self, dim):
         """
@@ -514,6 +504,15 @@ class LayerNorm(nn.Module):
 if version.parse(torch.__version__) >= version.parse('2.1.0'):
     LayerNorm = partial(nn.LayerNorm, bias = False)
 
+class ScaleNorm(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.scale = dim ** 0.5
+        self.g = nn.Parameter(torch.ones(1))
+
+    def forward(self, x):
+        return F.normalize(x, dim = -1) * self.scale * self.g
+
 class RMSNorm(nn.Module):
     def __init__(self, dim):
         super().__init__()

x_transformers/xval.py

@@ -271,8 +271,17 @@ class XValAutoregressiveWrapper(nn.Module):
 
         cross_entropy_loss = F.cross_entropy(logits, target, reduction = 'none', ignore_index = self.ignore_index)
 
+        # protect against nan in `x_num` input tensor
+
+        target_is_number_mask = target == self.net.numerical_token_id
+        x_num_target = x_num_target.masked_fill(~target_is_number_mask, 0.)
+
+        # ignore index
+
         target_mask = target != self.ignore_index
 
+        # numerical mse loss
+
         numerical_mse_loss = F.mse_loss(numerical_pred, x_num_target, reduction = 'none')
 
         numerical_mse_loss = numerical_mse_loss * target_mask

{x_transformers-1.27.19.dist-info → x_transformers-1.27.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.27.19
+Version: 1.27.21
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.27.19.dist-info → x_transformers-1.27.21.dist-info}/RECORD

@@ -4,11 +4,11 @@ x_transformers/autoregressive_wrapper.py,sha256=gYKIN5Rm8dMYSTX5yHpg9sPYyZf9rsRT
 x_transformers/continuous.py,sha256=dpHK4NSMDQAJQ_N3Uj9rip0fYGXyu0QCCO_OfEdbRGs,6192
 x_transformers/dpo.py,sha256=LjvWgCkqTl-UuehrzQ8nkX5guLr4whYwsmm7SKSwdls,3450
 x_transformers/nonautoregressive_wrapper.py,sha256=ys_p8obc7lTeeodCqvkRKxOXQ1C9T3j5Jwr-JbVgnXk,10432
-x_transformers/x_transformers.py,sha256=hRU3L-8fyK-ftFb69Yr1xrOwVraqgram_6mNEWFcKNg,63641
+x_transformers/x_transformers.py,sha256=kQhRUMGDsinzkdYcOfE1GriJ057j7D4xSjbH79FFRSE,63574
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
-x_transformers/xval.py,sha256=ulEPep6i5Hl7H-H9vGfdsmHdprUmK8ajB306jViyV2c,8147
-x_transformers-1.27.19.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.27.19.dist-info/METADATA,sha256=ADB735m9jq8Bok_MrwH74hm04N-N0MkW1xt7hLHmfwg,662
-x_transformers-1.27.19.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-x_transformers-1.27.19.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.27.19.dist-info/RECORD,,
+x_transformers/xval.py,sha256=to8AkYfgvR5QC-ZXDnq8PPfzOyg0yutHt_QW36jsy98,8403
+x_transformers-1.27.21.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.27.21.dist-info/METADATA,sha256=g0OEysaFW4RomYhoqjDKu7W1brHg70m_N40_nmTLRtA,662
+x_transformers-1.27.21.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+x_transformers-1.27.21.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.27.21.dist-info/RECORD,,

{x_transformers-1.27.19.dist-info → x_transformers-1.27.21.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any