x-transformers 1.26.6__py3-none-any.whl → 1.27.0__py3-none-any.whl

x_transformers/continuous.py

@@ -8,6 +8,7 @@ from x_transformers.x_transformers import (
     AttentionLayers,
     ScaledSinusoidalEmbedding,
     AbsolutePositionalEmbedding,
+    LayerNorm,
     always,
     pad_at_dim
 )
@@ -54,7 +55,7 @@ class ContinuousTransformerWrapper(nn.Module):
         else:
             self.pos_emb = AbsolutePositionalEmbedding(dim, max_seq_len)
 
-        self.post_emb_norm = nn.LayerNorm(dim) if post_emb_norm else nn.Identity()
+        self.post_emb_norm = LayerNorm(dim) if post_emb_norm else nn.Identity()
         self.emb_dropout = nn.Dropout(emb_dropout)
 
         # memory tokens
@@ -71,8 +72,8 @@ class ContinuousTransformerWrapper(nn.Module):
 
         # project in and out
 
-        self.project_in = nn.Linear(dim_in, dim) if exists(dim_in) else nn.Identity()
-        self.project_out = nn.Linear(dim, dim_out) if exists(dim_out) else nn.Identity()
+        self.project_in = nn.Linear(dim_in, dim, bias = False) if exists(dim_in) else nn.Identity()
+        self.project_out = nn.Linear(dim, dim_out, bias = False) if exists(dim_out) else nn.Identity()
 
     def forward(
         self,

x_transformers/x_transformers.py

@@ -304,7 +304,7 @@ class DynamicPositionBias(nn.Module):
 
         self.mlp.append(Sequential(
             nn.Linear(1, dim),
-            nn.LayerNorm(dim) if norm else None,
+            LayerNorm(dim) if norm else None,
             nn.SiLU()
         ))
 
@@ -498,6 +498,19 @@ class ScaleNorm(nn.Module):
         norm = torch.norm(x, dim = -1, keepdim = True)
         return x / norm.clamp(min = self.eps) * self.g
 
+class LayerNorm(nn.Module):
+    def __init__(self, dim):
+        """
+        bias-less layernorm has been shown to be more stable. most newer models have moved towards rmsnorm, also bias-less
+        latest pytorch actually has a way to turn this off in nn.LayerNorm
+        """
+        super().__init__()
+        self.gamma = nn.Parameter(torch.ones(dim))
+        self.register_buffer("beta", torch.zeros(dim))
+
+    def forward(self, x):
+        return F.layer_norm(x, x.shape[-1:], self.gamma, self.beta)
+
 class RMSNorm(nn.Module):
     def __init__(self, dim):
         super().__init__()
@@ -634,7 +647,7 @@ class FeedForward(nn.Module):
 
         self.ff = Sequential(
             project_in,
-            nn.LayerNorm(inner_dim) if post_act_ln else None,
+            LayerNorm(inner_dim) if post_act_ln else None,
             nn.Dropout(dropout),
             nn.Linear(inner_dim, dim_out, bias = not no_bias)
         )
@@ -1083,7 +1096,7 @@ class AttentionLayers(nn.Module):
         elif use_simple_rmsnorm:
             norm_class = SimpleRMSNorm
         else:
-            norm_class = nn.LayerNorm
+            norm_class = LayerNorm
 
         norm_fn = partial(norm_class, dim)
 
@@ -1415,12 +1428,12 @@ class ViTransformerWrapper(nn.Module):
             self.register_tokens = nn.Parameter(torch.randn(num_register_tokens, dim))
 
         self.patch_to_embedding = nn.Sequential(
-            nn.LayerNorm(patch_dim),
+            LayerNorm(patch_dim),
             nn.Linear(patch_dim, dim),
-            nn.LayerNorm(dim)
+            LayerNorm(dim)
         )
 
-        self.post_emb_norm = nn.LayerNorm(dim) if post_emb_norm else nn.Identity()
+        LayerNorm(dim) if post_emb_norm else nn.Identity()
         self.dropout = nn.Dropout(emb_dropout)
 
         self.attn_layers = attn_layers
@@ -1515,7 +1528,7 @@ class TransformerWrapper(nn.Module):
 
         self.emb_frac_gradient = emb_frac_gradient
 
-        self.post_emb_norm = nn.LayerNorm(emb_dim) if post_emb_norm else nn.Identity()
+        self.post_emb_norm = LayerNorm(emb_dim) if post_emb_norm else nn.Identity()
         self.emb_dropout = nn.Dropout(emb_dropout)
 
         self.project_emb = nn.Linear(emb_dim, dim) if emb_dim != dim else nn.Identity()
@@ -1524,7 +1537,7 @@ class TransformerWrapper(nn.Module):
         self.init_()
 
         logits_dim = default(logits_dim, num_tokens)
-        self.to_logits = nn.Linear(dim, logits_dim) if not tie_embedding else lambda t: t @ self.token_emb.emb.weight.t()
+        self.to_logits = nn.Linear(dim, logits_dim, bias = False) if not tie_embedding else lambda t: t @ self.token_emb.emb.weight.t()
 
         # memory tokens (like [cls]) from Memory Transformers paper
 

{x_transformers-1.26.6.dist-info → x_transformers-1.27.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.26.6
+Version: 1.27.0
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.26.6.dist-info → x_transformers-1.27.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 x_transformers/__init__.py,sha256=pXc_U4M3ONUQcpNgZySDIlCF1rp7u4FFmcOYjc4WuXw,629
 x_transformers/attend.py,sha256=MFl_FbgPsm9mziZPTi_s8QbxASETwbGeciMH8sUIwT8,10188
 x_transformers/autoregressive_wrapper.py,sha256=gYKIN5Rm8dMYSTX5yHpg9sPYyZf9rsRTJCNrYRdJ-Ww,9618
-x_transformers/continuous.py,sha256=ixfgi2_zpGN03SX_STXFkNYEOAkgwVIxuS53QgDCx-g,6026
+x_transformers/continuous.py,sha256=Ra5IClCl9G7SAiM6L9w6iY4cCznH0dSGljC9AC_bNyw,6066
 x_transformers/nonautoregressive_wrapper.py,sha256=AQLE4rA_Kh8VNoe9OzpwyeWson34sRkhks4dn4seNjI,10414
-x_transformers/x_transformers.py,sha256=l6bYUn6VQVInpftxO72bnaQbVJGk6ScWHENkkAjxzok,61736
+x_transformers/x_transformers.py,sha256=80qvAhandlAmt-mkiG7Ft6e5caCSDVRCFRfHppGvd5A,62216
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
 x_transformers/xval.py,sha256=lS9W_E_RskPQAqVZkPiUzbByoW1Ajsw_phsikA3JDAg,8139
-x_transformers-1.26.6.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.26.6.dist-info/METADATA,sha256=LWnSoRxEKYqpFsKKOPTv4JhxfYKP_alGg1exP4pzEB0,661
-x_transformers-1.26.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-x_transformers-1.26.6.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.26.6.dist-info/RECORD,,
+x_transformers-1.27.0.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.27.0.dist-info/METADATA,sha256=7kqhAXMJ-POUrT-6-QGBbgP4j7DCUR04bK8-ULmOYxQ,661
+x_transformers-1.27.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+x_transformers-1.27.0.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.27.0.dist-info/RECORD,,