x-transformers 2.4.11__tar.gz → 2.4.14__tar.gz

{x_transformers-2.4.11 → x_transformers-2.4.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.4.11
+Version: 2.4.14
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers

{x_transformers-2.4.11 → x_transformers-2.4.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.4.11"
+version = "2.4.14"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{x_transformers-2.4.11 → x_transformers-2.4.14}/x_transformers/autoregressive_wrapper.py

@@ -188,7 +188,7 @@ class AutoregressiveWrapper(Module):
         temperature = 1.,
         stochastic = False,
         prompt_lens: Tensor | None = None,
-        filter_logits_fn: str | Callable = top_k,
+        filter_logits_fn: str | Callable = identity,
         restrict_to_max_seq_len = True,
         filter_kwargs: dict = dict(),
         cache_kv = True,

{x_transformers-2.4.11 → x_transformers-2.4.14}/x_transformers/x_transformers.py

@@ -1304,6 +1304,7 @@ class Attention(Module):
         qk_norm_groups = 1,
         qk_norm_scale = 10,
         qk_norm_dim_scale = False,
+        value_rmsnorm = False,      # used in alphagenome and bytedance's GR3 for further stability
         l2_distance = False,
         sigmoid = False,
         selective = False,
@@ -1458,6 +1459,10 @@ class Attention(Module):
         assert (not qk_norm) or divisible_by(dim_head, qk_norm_groups), 'dimension per attention head must be divisible by the qk norm groups'
         assert not (qk_norm and (dim_head // qk_norm_groups) <= 2), 'the group dimension may be too small (2 was too small in my tests, but 4 still works, surprisingly)'
 
+        # value rms norm
+
+        self.value_rmsnorm = MultiheadRMSNorm(dim_head, heads = heads) if value_rmsnorm else None
+
         # contextual positional encoding
         # https://arxiv.org/html/2405.18719v2
 
@@ -1697,6 +1702,10 @@ class Attention(Module):
             q = q * self.qk_norm_q_scale
             k = k * self.qk_norm_k_scale
 
+        # maybe value rmsnorm
+
+        v = maybe(self.value_rmsnorm)(v)
+
         # take care of caching
 
         if not is_multi_latent_attn: