hyper-connections 0.0.20__py3-none-any.whl → 0.0.21__py3-none-any.whl

hyper_connections/hyper_connections.py

@@ -31,6 +31,17 @@ def default(v, d):
 def identity(t):
     return t
 
+# norms
+
+class RMSNorm(Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.scale = dim ** 0.5
+        self.gamma = nn.Parameter(torch.zeros(dim))
+
+    def forward(self, x):
+        return F.normalize(x, dim = -1) * self.scale * (self.gamma + 1)
+
 # main classes
 
 # residual base class
@@ -108,7 +119,7 @@ class HyperConnections(Module):
 
         self.act = nn.Tanh() if tanh else nn.Identity()
 
-        self.norm = nn.RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
+        self.norm = RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
 
         assert num_residual_streams > 0, '`num_residual_streams` must be greater than 0'
 

hyper_connections/hyper_connections_with_multi_branch_inputs.py

@@ -22,7 +22,7 @@ br - branch functions
 t - residual streams + num branch inputs
 """
 
-from hyper_connections.hyper_connections import Residual, StreamEmbed
+from hyper_connections.hyper_connections import Residual, StreamEmbed, RMSNorm
 
 # helper functions
 
@@ -74,7 +74,7 @@ class HyperConnections(Module):
 
         self.act = nn.Tanh() if tanh else nn.Identity()
 
-        self.norm = nn.RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
+        self.norm = RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
 
         self.num_residual_streams = num_residual_streams
         self.num_branch_inputs = num_branch_inputs

{hyper_connections-0.0.20.dist-info → hyper_connections-0.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.0.20
+Version: 0.0.21
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections
@@ -155,3 +155,10 @@ HyperConnections.get_init_and_expand_reduce_stream_functions(4, disable = True)
     url     = {https://api.semanticscholar.org/CorpusID:272987528}
 }
 ```
+
+```bibtex
+@misc{Rubin2024,
+    author  = {Ohad Rubin},
+    url     = {https://medium.com/@ohadrubin/exploring-weight-decay-in-layer-normalization-challenges-and-a-reparameterization-solution-ad4d12c24950}
+}
+```

hyper_connections-0.0.21.dist-info/RECORD

@@ -0,0 +1,7 @@
+hyper_connections/__init__.py,sha256=wJxbrEXRGmOIjPw8fWP-cUq6CE8bvx95mIlhWifNvYc,135
+hyper_connections/hyper_connections.py,sha256=l64d-qB8m188RuYfzgWOoIzmXnGMjIIQAvZZBQyUOGs,9755
+hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=-9K-IS-zbjMVaFWr_29okNdup3YATJTxeypUcD0Syoc,7559
+hyper_connections-0.0.21.dist-info/METADATA,sha256=ZBcfPopgUUOK6JeUIEGMSGgaeWGG3bvYoFOLTeskGoA,5288
+hyper_connections-0.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hyper_connections-0.0.21.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.0.21.dist-info/RECORD,,

hyper_connections-0.0.20.dist-info/RECORD

@@ -1,7 +0,0 @@
-hyper_connections/__init__.py,sha256=wJxbrEXRGmOIjPw8fWP-cUq6CE8bvx95mIlhWifNvYc,135
-hyper_connections/hyper_connections.py,sha256=ElPtieRLvVKaVg2Attx1k6esKq1SY2X4AVZbZmsQAOM,9486
-hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=HbLpt79xcMv_os6brMvDd90t2GOPceliE1YFusR2eJI,7553
-hyper_connections-0.0.20.dist-info/METADATA,sha256=erA-d7KNNdzPY76x8IWKd2trv2WuBO9-C2DtH-SoQ_Y,5076
-hyper_connections-0.0.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hyper_connections-0.0.20.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.0.20.dist-info/RECORD,,