x-transformers 2.11.17__tar.gz → 2.11.18__tar.gz

{x_transformers-2.11.17 → x_transformers-2.11.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.11.17
+Version: 2.11.18
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers
@@ -2607,4 +2607,14 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 }
 ```
 
+```bibtex
+@article{elhage2022solu,
+    title   = {Softmax Linear Units},
+    author  = {Elhage, Nelson and Hume, Tristan and Olsson, Catherine and Nanda, Neel and Henighan, Tom and Johnston, Scott and ElShowk, Sheer and Joseph, Nicholas and DasSarma, Nova and Mann, Ben and Hernandez, Danny and Askell, Amanda and Ndousse, Kamal and Jones, Andy and Drain, Dawn and Chen, Anna and Bai, Yuntao and Ganguli, Deep and Lovitt, Liane and Hatfield-Dodds, Zac and Kernion, Jackson and Conerly, Tom and Kravec, Shauna and Fort, Stanislav and Kadavath, Saurav and Jacobson, Josh and Tran-Johnson, Eli and Kaplan, Jared and Clark, Jack and Brown, Tom and McCandlish, Sam and Amodei, Dario and Olah, Christopher},
+    year    = {2022},
+    journal = {Transformer Circuits Thread},
+    note    = {https://transformer-circuits.pub/2022/solu/index.html}
+}
+```
+
 *solve intelligence... then use that to solve everything else.* - Demis Hassabis

{x_transformers-2.11.17 → x_transformers-2.11.18}/README.md

@@ -2558,4 +2558,14 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 }
 ```
 
+```bibtex
+@article{elhage2022solu,
+    title   = {Softmax Linear Units},
+    author  = {Elhage, Nelson and Hume, Tristan and Olsson, Catherine and Nanda, Neel and Henighan, Tom and Johnston, Scott and ElShowk, Sheer and Joseph, Nicholas and DasSarma, Nova and Mann, Ben and Hernandez, Danny and Askell, Amanda and Ndousse, Kamal and Jones, Andy and Drain, Dawn and Chen, Anna and Bai, Yuntao and Ganguli, Deep and Lovitt, Liane and Hatfield-Dodds, Zac and Kernion, Jackson and Conerly, Tom and Kravec, Shauna and Fort, Stanislav and Kadavath, Saurav and Jacobson, Josh and Tran-Johnson, Eli and Kaplan, Jared and Clark, Jack and Brown, Tom and McCandlish, Sam and Amodei, Dario and Olah, Christopher},
+    year    = {2022},
+    journal = {Transformer Circuits Thread},
+    note    = {https://transformer-circuits.pub/2022/solu/index.html}
+}
+```
+
 *solve intelligence... then use that to solve everything else.* - Demis Hassabis

{x_transformers-2.11.17 → x_transformers-2.11.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.11.17"
+version = "2.11.18"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }

{x_transformers-2.11.17 → x_transformers-2.11.18}/tests/test_x_transformers.py

@@ -1462,3 +1462,14 @@ def test_kv_input_residual():
     out = attn(tokens, context = context, cross_attn_kv_residuals = condition)
 
     assert tokens.shape == out.shape
+
+def test_solu():
+    attn = Decoder(
+        dim = 256,
+        depth = 2,
+        heads = 4,
+        ff_solu = True
+    )
+
+    tokens = torch.randn(3, 32, 256)
+    attn(tokens)

{x_transformers-2.11.17 → x_transformers-2.11.18}/x_transformers/x_transformers.py

@@ -275,6 +275,10 @@ class ReluSquared(Module):
     def forward(self, x):
         return F.relu(x) ** 2
 
+class SoLU(Module):
+    def forward(self, x):
+        return x.softmax(dim = -1) * x
+
 # embedding
 
 class TokenEmbedding(Module):
@@ -1239,6 +1243,7 @@ class FeedForward(Module):
         glu_mult_bias = False,
         swish = False,
         relu_squared = False,
+        solu = False,
         custom_activation = None,
         post_act_ln = False,
         dropout = 0.,
@@ -1250,10 +1255,14 @@ class FeedForward(Module):
         inner_dim = int(dim * mult)
         dim_out = default(dim_out, dim)
 
+        assert at_most_one_of(relu_squared, solu)
+
         if exists(custom_activation):
             activation = deepcopy(custom_activation)
         elif relu_squared:
             activation = ReluSquared()
+        elif solu:
+            activation = SoLU()
         elif swish:
             activation = nn.SiLU()
         else: