x-transformers 2.11.17__py3-none-any.whl → 2.11.19__py3-none-any.whl

x_transformers/x_transformers.py

@@ -275,6 +275,15 @@ class ReluSquared(Module):
     def forward(self, x):
         return F.relu(x) ** 2
 
+class SoLU(Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.norm = LayerNorm(dim)
+
+    def forward(self, x):
+        activated = x.softmax(dim = -1) * x
+        return self.norm(activated)
+
 # embedding
 
 class TokenEmbedding(Module):
@@ -1239,6 +1248,7 @@ class FeedForward(Module):
         glu_mult_bias = False,
         swish = False,
         relu_squared = False,
+        solu = False,
         custom_activation = None,
         post_act_ln = False,
         dropout = 0.,
@@ -1250,10 +1260,14 @@ class FeedForward(Module):
         inner_dim = int(dim * mult)
         dim_out = default(dim_out, dim)
 
+        assert at_most_one_of(relu_squared, solu)
+
         if exists(custom_activation):
             activation = deepcopy(custom_activation)
         elif relu_squared:
             activation = ReluSquared()
+        elif solu:
+            activation = SoLU(inner_dim)
         elif swish:
             activation = nn.SiLU()
         else:

{x_transformers-2.11.17.dist-info → x_transformers-2.11.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: x-transformers
-Version: 2.11.17
+Version: 2.11.19
 Summary: X-Transformers
 Project-URL: Homepage, https://pypi.org/project/x-transformers/
 Project-URL: Repository, https://github.com/lucidrains/x-transformers
@@ -2607,4 +2607,14 @@ ids_out, num_out, is_number_mask = model.generate(start_ids, start_nums, 17)
 }
 ```
 
+```bibtex
+@article{elhage2022solu,
+    title   = {Softmax Linear Units},
+    author  = {Elhage, Nelson and Hume, Tristan and Olsson, Catherine and Nanda, Neel and Henighan, Tom and Johnston, Scott and ElShowk, Sheer and Joseph, Nicholas and DasSarma, Nova and Mann, Ben and Hernandez, Danny and Askell, Amanda and Ndousse, Kamal and Jones, Andy and Drain, Dawn and Chen, Anna and Bai, Yuntao and Ganguli, Deep and Lovitt, Liane and Hatfield-Dodds, Zac and Kernion, Jackson and Conerly, Tom and Kravec, Shauna and Fort, Stanislav and Kadavath, Saurav and Jacobson, Josh and Tran-Johnson, Eli and Kaplan, Jared and Clark, Jack and Brown, Tom and McCandlish, Sam and Amodei, Dario and Olah, Christopher},
+    year    = {2022},
+    journal = {Transformer Circuits Thread},
+    note    = {https://transformer-circuits.pub/2022/solu/index.html}
+}
+```
+
 *solve intelligence... then use that to solve everything else.* - Demis Hassabis

{x_transformers-2.11.17.dist-info → x_transformers-2.11.19.dist-info}/RECORD

@@ -11,10 +11,10 @@ x_transformers/multi_input.py,sha256=tCh-fTJDj2ib4SMGtsa-AM8MxKzJAQSwqAXOu3HU2mg
 x_transformers/neo_mlp.py,sha256=XCNnnop9WLarcxap1kGuYc1x8GHvwkZiDRnXOxSl3Po,3452
 x_transformers/nonautoregressive_wrapper.py,sha256=hMQqNimGtchNIe13cR5LZule1V7I1qM5LmY8VQfVdnA,11698
 x_transformers/up_wrapper.py,sha256=YC2LN14_7Xx9Wtiek2rtEJ_qHqdfSmKlh3d7Cgxwd80,7073
-x_transformers/x_transformers.py,sha256=5ctPu8tvlbUMrtW360e_LPnoGv6xcgQFsyWdbvLo6Tk,127002
+x_transformers/x_transformers.py,sha256=xFnA2eW1ZkA_uCdqAtmBU_og3AbarIJQUlNZdVgNVcE,127354
 x_transformers/xl_autoregressive_wrapper.py,sha256=CvZMJ6A6PA-Y_bQAhnORwjJBSl6Vjq2IdW5KTdk8NI8,4195
 x_transformers/xval.py,sha256=AwwYUm8yDAtKQyKJDIhYMsiLTJ_skh3scUFMjp5sda8,8597
-x_transformers-2.11.17.dist-info/METADATA,sha256=9gqVZAutVIzE5Xs5ulYv8fZ97-M2vsCacbrWhJmkXm0,96012
-x_transformers-2.11.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-x_transformers-2.11.17.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
-x_transformers-2.11.17.dist-info/RECORD,,
+x_transformers-2.11.19.dist-info/METADATA,sha256=SW6GAdWr098WuzOLJrV5h2sY6HutYKam-7n_mXBUZzk,96858
+x_transformers-2.11.19.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+x_transformers-2.11.19.dist-info/licenses/LICENSE,sha256=As9u198X-U-vph5noInuUfqsAG2zX_oXPHDmdjwlPPY,1066
+x_transformers-2.11.19.dist-info/RECORD,,