x-transformers 1.30.17__py3-none-any.whl → 1.30.19__py3-none-any.whl

x_transformers/x_transformers.py

@@ -584,7 +584,7 @@ class AdaptiveLayerNorm(Module):
 
         nn.init.zeros_(self.to_gamma.weight)
 
-    def forward(self, x, condition):
+    def forward(self, x, *, condition):
         normed = self.ln(x)
         gamma = self.to_gamma(condition)
         return normed * (gamma + 1.)
@@ -607,6 +607,20 @@ class RMSNorm(Module):
     def forward(self, x):
         return F.normalize(x, dim = -1) * self.scale * self.g
 
+class AdaptiveRMSNorm(Module):
+    def __init__(self, dim, dim_condition = None):
+        super().__init__()
+        self.scale = dim ** 0.5
+        dim_condition = default(dim_condition, dim)
+
+        self.to_gamma = nn.Linear(dim_condition, dim, bias = False)
+        nn.init.zeros_(self.to_gamma.weight)
+
+    def forward(self, x, *, condition):
+        normed = F.normalize(x, dim = -1)
+        gamma = self.to_gamma(condition)
+        return normed * self.scale * (gamma + 1.)
+
 class SimpleRMSNorm(Module):
     def __init__(self, dim):
         super().__init__()
@@ -1145,7 +1159,10 @@ class AttentionLayers(Module):
         use_rmsnorm = False,
         use_simple_rmsnorm = False,
         use_adaptive_layernorm = False,
+        use_adaptive_rmsnorm = False,
         dim_condition = None,
+        adaptive_condition_mlp = False,
+        adaptive_condition_mlp_expansion = 4,
         alibi_pos_bias = False,
         alibi_num_heads = None,
         rel_pos_bias = False,
@@ -1250,11 +1267,22 @@ class AttentionLayers(Module):
 
         # determine norm
 
-        assert at_most_one_of(use_scalenorm, use_rmsnorm, use_simple_rmsnorm, use_adaptive_layernorm) <= 1, 'you can only use either scalenorm, rmsnorm, or simple rmsnorm'
+        assert at_most_one_of(use_scalenorm, use_rmsnorm, use_simple_rmsnorm, use_adaptive_layernorm, use_adaptive_rmsnorm), 'you can only use either scalenorm, rmsnorm, adaptive layernorm, adaptive rmsnorm, or simple rmsnorm'
 
         need_condition = False
         dim_condition = default(dim_condition, dim)
 
+        self.adaptive_mlp = nn.Identity()
+        dim_condition_mult = 1
+
+        if adaptive_condition_mlp:
+            dim_condition_mult = adaptive_condition_mlp_expansion
+
+            self.adaptive_mlp = nn.Sequential(
+                nn.Linear(dim_condition, dim_condition * dim_condition_mult, bias = False),
+                nn.SiLU()
+            )
+
         if use_scalenorm:
             norm_class = ScaleNorm
         elif use_rmsnorm:
@@ -1263,7 +1291,10 @@ class AttentionLayers(Module):
             norm_class = SimpleRMSNorm
         elif use_adaptive_layernorm:
             need_condition = True
-            norm_class = partial(AdaptiveLayerNorm, dim_condition = dim_condition)
+            norm_class = partial(AdaptiveLayerNorm, dim_condition = dim_condition * dim_condition_mult)
+        elif use_adaptive_rmsnorm:
+            need_condition = True
+            norm_class = partial(AdaptiveRMSNorm, dim_condition = dim_condition * dim_condition_mult)
         else:
             norm_class = LayerNorm
 
@@ -1428,7 +1459,16 @@ class AttentionLayers(Module):
         norm_kwargs = dict()
 
         if self.need_condition:
-            assert condition.shape[-1] == self.dim_condition
+            assert condition.ndim in {2, 3}
+
+            if condition.ndim == 2:
+                condition = rearrange(condition, 'b d -> b 1 d')
+
+            assert condition.shape[-1] == self.dim_condition, f'expected condition dimension of {self.dim_condition} but received {condition.shape[-1]}'
+
+            # maybe mlp
+
+            condition = self.adaptive_mlp(condition)
 
             norm_kwargs.update(condition = condition)
 

{x_transformers-1.30.17.dist-info → x_transformers-1.30.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: x-transformers
-Version: 1.30.17
+Version: 1.30.19
 Summary: X-Transformers - Pytorch
 Home-page: https://github.com/lucidrains/x-transformers
 Author: Phil Wang

{x_transformers-1.30.17.dist-info → x_transformers-1.30.19.dist-info}/RECORD

@@ -4,11 +4,11 @@ x_transformers/autoregressive_wrapper.py,sha256=uX8Mb0zLsQrZECt_9UGt35g7tC05Rk3n
 x_transformers/continuous.py,sha256=WO52n9lFAXv5-SGadi2cApGF8dkouN8QSTEOuC7erj8,6180
 x_transformers/dpo.py,sha256=LjvWgCkqTl-UuehrzQ8nkX5guLr4whYwsmm7SKSwdls,3450
 x_transformers/nonautoregressive_wrapper.py,sha256=ys_p8obc7lTeeodCqvkRKxOXQ1C9T3j5Jwr-JbVgnXk,10432
-x_transformers/x_transformers.py,sha256=kvGX4Ib1gSYV6pmVXF6P9bcutRsx5bif_XhkbG4DOZ8,70738
+x_transformers/x_transformers.py,sha256=NxNSAMgyHZEE-toXNcEO-unGwKfoqooN9oEQMDPPGfY,72268
 x_transformers/xl_autoregressive_wrapper.py,sha256=DCx4n0_c1tFai4nOqaWVnqx2p9eutsZsDMiMP1ckxNU,4117
 x_transformers/xval.py,sha256=QE1ltYZTR_eGgIHPP2BrMWVWVLqMW-OpDZh87BSmQEg,8563
-x_transformers-1.30.17.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-1.30.17.dist-info/METADATA,sha256=m_d5lvKUbiN8xS7Dx4gI5I8dHtzEa1ccp4MuKcG5O9w,662
-x_transformers-1.30.17.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-x_transformers-1.30.17.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
-x_transformers-1.30.17.dist-info/RECORD,,
+x_transformers-1.30.19.dist-info/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-1.30.19.dist-info/METADATA,sha256=2fQP9SmEys1grPsY2xHhB3sqkODtlaIz4UWtOR9pCM8,662
+x_transformers-1.30.19.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+x_transformers-1.30.19.dist-info/top_level.txt,sha256=hO6KGpFuGucRNEtRfme4A_rGcM53AKwGP7RVlRIxS5Q,15
+x_transformers-1.30.19.dist-info/RECORD,,