x-transformers 2.11.18__tar.gz → 2.11.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of x-transformers might be problematic. Click here for more details.

Files changed (68) hide show
  1. {x_transformers-2.11.18 → x_transformers-2.11.19}/PKG-INFO +1 -1
  2. {x_transformers-2.11.18 → x_transformers-2.11.19}/pyproject.toml +1 -1
  3. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/x_transformers.py +7 -2
  4. {x_transformers-2.11.18 → x_transformers-2.11.19}/.github/FUNDING.yml +0 -0
  5. {x_transformers-2.11.18 → x_transformers-2.11.19}/.github/workflows/python-publish.yml +0 -0
  6. {x_transformers-2.11.18 → x_transformers-2.11.19}/.github/workflows/python-test.yaml +0 -0
  7. {x_transformers-2.11.18 → x_transformers-2.11.19}/.gitignore +0 -0
  8. {x_transformers-2.11.18 → x_transformers-2.11.19}/LICENSE +0 -0
  9. {x_transformers-2.11.18 → x_transformers-2.11.19}/README.md +0 -0
  10. {x_transformers-2.11.18 → x_transformers-2.11.19}/data/README.md +0 -0
  11. {x_transformers-2.11.18 → x_transformers-2.11.19}/data/enwik8.gz +0 -0
  12. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/all-attention.png +0 -0
  13. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/attention-on-attention.png +0 -0
  14. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/cosine-sim-attention.png +0 -0
  15. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/deepnorm.png +0 -0
  16. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/dynamic-pos-bias-linear.png +0 -0
  17. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/dynamic-pos-bias-log.png +0 -0
  18. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/dynamic-pos-bias-sinusoidal.png +0 -0
  19. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/dynamic-pos-bias.png +0 -0
  20. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/enhanced-recurrence.png +0 -0
  21. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/fcm.png +0 -0
  22. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/ffglu.png +0 -0
  23. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/flash-attention.png +0 -0
  24. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/gate_values.png +0 -0
  25. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/gating.png +0 -0
  26. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/length-extrapolation-scale.png +0 -0
  27. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/macaron-1.png +0 -0
  28. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/macaron-2.png +0 -0
  29. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/memory-transformer.png +0 -0
  30. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/normformer.png +0 -0
  31. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/pia.png +0 -0
  32. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/qknorm-analysis.png +0 -0
  33. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/resi_dual.png +0 -0
  34. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/residual_attn.png +0 -0
  35. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/rezero.png +0 -0
  36. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/rotary.png +0 -0
  37. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/sandwich-2.png +0 -0
  38. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/sandwich.png +0 -0
  39. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/sandwich_norm.png +0 -0
  40. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/scalenorm.png +0 -0
  41. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/talking-heads.png +0 -0
  42. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/topk-attention.png +0 -0
  43. {x_transformers-2.11.18 → x_transformers-2.11.19}/images/xval.png +0 -0
  44. {x_transformers-2.11.18 → x_transformers-2.11.19}/tests/test_x_transformers.py +0 -0
  45. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_belief_state.py +0 -0
  46. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_copy.py +0 -0
  47. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_entropy_tokenizer.py +0 -0
  48. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_enwik8.py +0 -0
  49. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_free.py +0 -0
  50. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_gpt_vae.py +0 -0
  51. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_length_extrapolate.py +0 -0
  52. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_parity.py +0 -0
  53. {x_transformers-2.11.18 → x_transformers-2.11.19}/train_with_muon.py +0 -0
  54. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/__init__.py +0 -0
  55. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/attend.py +0 -0
  56. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/autoregressive_wrapper.py +0 -0
  57. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/belief_state_wrapper.py +0 -0
  58. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/continuous.py +0 -0
  59. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/dpo.py +0 -0
  60. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/entropy_based_tokenizer.py +0 -0
  61. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/free_transformer.py +0 -0
  62. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/gpt_vae.py +0 -0
  63. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/multi_input.py +0 -0
  64. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/neo_mlp.py +0 -0
  65. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/nonautoregressive_wrapper.py +0 -0
  66. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/up_wrapper.py +0 -0
  67. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/xl_autoregressive_wrapper.py +0 -0
  68. {x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/xval.py +0 -0
{x_transformers-2.11.18 → x_transformers-2.11.19}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: x-transformers
3
- Version: 2.11.18
3
+ Version: 2.11.19
4
4
  Summary: X-Transformers
5
5
  Project-URL: Homepage, https://pypi.org/project/x-transformers/
6
6
  Project-URL: Repository, https://github.com/lucidrains/x-transformers
{x_transformers-2.11.18 → x_transformers-2.11.19}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "x-transformers"
3
- version = "2.11.18"
3
+ version = "2.11.19"
4
4
  description = "X-Transformers"
5
5
  authors = [
6
6
  { name = "Phil Wang", email = "lucidrains@gmail.com" }
{x_transformers-2.11.18 → x_transformers-2.11.19}/x_transformers/x_transformers.py RENAMED
@@ -276,8 +276,13 @@ class ReluSquared(Module):
276
276
  return F.relu(x) ** 2
277
277
 
278
278
  class SoLU(Module):
279
+ def __init__(self, dim):
280
+ super().__init__()
281
+ self.norm = LayerNorm(dim)
282
+
279
283
  def forward(self, x):
280
- return x.softmax(dim = -1) * x
284
+ activated = x.softmax(dim = -1) * x
285
+ return self.norm(activated)
281
286
 
282
287
  # embedding
283
288
 
@@ -1262,7 +1267,7 @@ class FeedForward(Module):
1262
1267
  elif relu_squared:
1263
1268
  activation = ReluSquared()
1264
1269
  elif solu:
1265
- activation = SoLU()
1270
+ activation = SoLU(inner_dim)
1266
1271
  elif swish:
1267
1272
  activation = nn.SiLU()
1268
1273
  else: