x-transformers 2.1.30__py3-none-any.whl → 2.1.32__py3-none-any.whl

x_transformers/x_transformers.py

@@ -852,6 +852,7 @@ class DynamicTanh(Module):
     """ https://arxiv.org/abs/2503.10622 """
     def __init__(
         self,
+        dim,
         init_alpha = 1.,
         gamma = 1.,
         beta = 0.,
@@ -860,8 +861,8 @@ class DynamicTanh(Module):
         super().__init__()
         self.pre_tanh_scale = nn.Parameter(tensor(init_alpha))
 
-        self.gamma = nn.Parameter(tensor(init_alpha))
-        self.beta = nn.Parameter(tensor(init_alpha))
+        self.gamma = nn.Parameter(torch.ones(dim))
+        self.beta = nn.Parameter(torch.zeros(dim))
 
         self.unit_offset = int(unit_offset)
 
@@ -2040,7 +2041,6 @@ class AttentionLayers(Module):
 
         assert at_most_one_of(use_scalenorm, use_rmsnorm, use_dynamic_tanh, use_simple_rmsnorm, use_adaptive_layernorm, use_adaptive_rmsnorm), 'you can only use either scalenorm, rmsnorm, adaptive layernorm, adaptive rmsnorm, or simple rmsnorm'
 
-        norm_fn = None
         norm_need_condition = False
         dim_condition = default(dim_condition, dim)
         dim_condition_mult = 1
@@ -2055,7 +2055,8 @@ class AttentionLayers(Module):
         elif use_simple_rmsnorm:
             norm_class = SimpleRMSNorm
         elif use_dynamic_tanh:
-            norm_fn = partial(DynamicTanh, init_alpha = dynamic_tanh_init_alpha)
+            assert pre_norm, 'only tested for pre-norm'
+            norm_class = partial(DynamicTanh, init_alpha = dynamic_tanh_init_alpha)
         elif use_adaptive_layernorm:
             norm_need_condition = True
             norm_class = partial(AdaptiveLayerNorm, dim_condition = dim_condition * dim_condition_mult)
@@ -2065,8 +2066,7 @@ class AttentionLayers(Module):
         else:
             norm_class = LayerNorm
 
-        if not exists(norm_fn):
-            norm_fn = partial(norm_class, dim)
+        norm_fn = partial(norm_class, dim)
 
         if not norm_need_condition and norm_add_unit_offset:
             # researcher Ohad Rubin shares in a blog post by adding an offset to gammas, they can be subjected to weight decay safely

{x_transformers-2.1.30.dist-info → x_transformers-2.1.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.1.30
+Version: 2.1.32
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.1.30.dist-info → x_transformers-2.1.32.dist-info}/RECORD

@@ -7,10 +7,10 @@ x_transformers/dpo.py,sha256=xt4OuOWhU8pN3OKN2LZAaC2NC8iiEnchqqcrPWVqf0o,3521
 x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg,9252
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=2NU58hYMgn-4Jzg3mie-mXb0XH_dCN7fjlzd3K1rLUY,10510
-x_transformers/x_transformers.py,sha256=a7k6tR9H1kCRX44PP0N9nMMb3V1_cIgFweTBK84VtEk,111476
+x_transformers/x_transformers.py,sha256=YQODc4PDB_ddgm7vi0uktV5GGetgEuwADzt3CaIdAXs,111484
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=7S00kCuab4tWQa-vf-z-XfzADjVj48MoFIr7VSIvttg,8575
-x_transformers-2.1.30.dist-info/METADATA,sha256=IdKgXNQf9aTZ_JiOYhc3q1J44ITmwBO-VRPRDaZtnEU,88161
-x_transformers-2.1.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.1.30.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.1.30.dist-info/RECORD,,
+x_transformers-2.1.32.dist-info/METADATA,sha256=Jmgj9CByp1_kveLWVm5-QM_n55A9s3MFzUiV1ciD034,88161
+x_transformers-2.1.32.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.1.32.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.1.32.dist-info/RECORD,,