x-transformers 2.11.22__tar.gz → 2.11.24__tar.gz

{x_transformers-2.11.22 → x_transformers-2.11.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.11.22
+Version: 2.11.24
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers
@@ -2618,4 +2618,16 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 }
 ```
 
+```bibtex
+@misc{chen2025strongernormalizationfreetransformers,
+    title   = {Stronger Normalization-Free Transformers}, 
+    author  = {Mingzhi Chen and Taiming Lu and Jiachen Zhu and Mingjie Sun and Zhuang Liu},
+    year    = {2025},
+    eprint  = {2512.10938},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG},
+    url     = {https://arxiv.org/abs/2512.10938}, 
+}
+```
+
 *solve intelligence... then use that to solve everything else.* - Demis Hassabis

{x_transformers-2.11.22 → x_transformers-2.11.24}/README.md

@@ -2569,4 +2569,16 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 }
 ```
 
+```bibtex
+@misc{chen2025strongernormalizationfreetransformers,
+    title   = {Stronger Normalization-Free Transformers}, 
+    author  = {Mingzhi Chen and Taiming Lu and Jiachen Zhu and Mingjie Sun and Zhuang Liu},
+    year    = {2025},
+    eprint  = {2512.10938},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG},
+    url     = {https://arxiv.org/abs/2512.10938}, 
+}
+```
+
 *solve intelligence... then use that to solve everything else.* - Demis Hassabis

{x_transformers-2.11.22 → x_transformers-2.11.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.11.22"
+version = "2.11.24"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{x_transformers-2.11.22 → x_transformers-2.11.24}/tests/test_x_transformers.py

@@ -1478,6 +1478,7 @@ def test_belief_attn(
             dim = 512,
             depth = 6,
             heads = 8,
+            attn_kv_heads = 4,
             rotary_pos_emb = True,
             attn_orthog_projected_values = orthog_project,
             attn_orthog_projected_values_per_head = orthog_project_per_head
@@ -1487,3 +1488,23 @@ def test_belief_attn(
     x = torch.randint(0, 256, (1, 10))
 
     logits = model(x)
+
+def test_derf():
+    from x_transformers import TransformerWrapper, Decoder
+
+    model = TransformerWrapper(
+        num_tokens = 256,
+        max_seq_len = 1024,
+        attn_layers = Decoder(
+            dim = 512,
+            depth = 6,
+            heads = 8,
+            attn_kv_heads = 4,
+            rotary_pos_emb = True,
+            use_derf = True
+        )
+    )
+
+    x = torch.randint(0, 256, (1, 10))
+
+    logits = model(x)

{x_transformers-2.11.22 → x_transformers-2.11.24}/x_transformers/x_transformers.py

@@ -941,6 +941,31 @@ class DynamicTanh(Module):
         gamma = self.gamma + self.gamma_offset
         return (x * pre_tanh_scale).tanh() * gamma + self.beta
 
+class Derf(Module):
+    """ https://arxiv.org/abs/2512.10938 """
+    def __init__(
+        self,
+        dim,
+        init_alpha = 0.5,
+        init_bias = 0.,
+        unit_offset = False
+    ):
+        super().__init__()
+        scale_offset = 1. if unit_offset else 0.
+
+        self.alpha = nn.Parameter(tensor(init_alpha) - scale_offset)
+        self.s = nn.Parameter(tensor(init_bias))
+
+        self.gamma = nn.Parameter(torch.ones(dim) - scale_offset)
+        self.beta = nn.Parameter(torch.zeros(dim))
+
+        self.scale_offset = scale_offset
+
+    def forward(self, x):
+        x = x * (self.alpha + self.scale_offset) + self.s
+        activated = torch.erf(x)
+        return activated * (self.gamma + self.scale_offset) + self.beta
+
 # residual and residual gates
 
 class Residual(Module):
@@ -1431,6 +1456,7 @@ class Attention(Module):
         assert divisible_by(heads, kv_heads)
 
         self.kv_heads = kv_heads
+        self.groups = heads // kv_heads
 
         q_dim = dim_head * heads
         k_dim = dim_head * kv_heads
@@ -2077,7 +2103,7 @@ class Attention(Module):
 
         if self.orthog_projected_values or self.orthog_projected_values_per_head:
             orthog_projected = []
-            v_for_proj = self.merge_heads(orig_values)
+            v_for_proj = repeat(orig_values, 'b h n d -> b n (g h d)', g = self.groups)
 
             if self.orthog_projected_values:
                 projected = orthog_project(out, v_for_proj)
@@ -2122,6 +2148,7 @@ class AttentionLayers(Module):
         use_scalenorm = False,
         use_rmsnorm = False,
         use_dynamic_tanh = False,
+        use_derf = False,
         dynamic_tanh_init_alpha = 1.,
         use_simple_rmsnorm = False,
         use_adaptive_layernorm = False,
@@ -2276,7 +2303,7 @@ class AttentionLayers(Module):
 
         # determine norm
 
-        assert at_most_one_of(use_scalenorm, use_rmsnorm, use_dynamic_tanh, use_simple_rmsnorm, use_adaptive_layernorm, use_adaptive_rmsnorm), 'you can only use either scalenorm, rmsnorm, adaptive layernorm, adaptive rmsnorm, or simple rmsnorm'
+        assert at_most_one_of(use_scalenorm, use_rmsnorm, use_dynamic_tanh, use_derf, use_simple_rmsnorm, use_adaptive_layernorm, use_adaptive_rmsnorm), 'you can only use either scalenorm, rmsnorm, adaptive layernorm, adaptive rmsnorm, or simple rmsnorm'
 
         norm_need_condition = False
         dim_condition = default(dim_condition, dim)
@@ -2294,6 +2321,8 @@ class AttentionLayers(Module):
         elif use_dynamic_tanh:
             assert pre_norm, 'dynamic tanh norm only tested for pre-norm'
             norm_class = partial(DynamicTanh, init_alpha = dynamic_tanh_init_alpha)
+        elif use_derf:
+            norm_class = Derf
         elif use_adaptive_layernorm:
             norm_need_condition = True
             norm_class = partial(AdaptiveLayerNorm, dim_condition = dim_condition * dim_condition_mult)