hyper-connections 0.0.19__tar.gz → 0.0.21__tar.gz

{hyper_connections-0.0.19 → hyper_connections-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.0.19
+Version: 0.0.21
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections
@@ -155,3 +155,10 @@ HyperConnections.get_init_and_expand_reduce_stream_functions(4, disable = True)
     url     = {https://api.semanticscholar.org/CorpusID:272987528}
 }
 ```
+
+```bibtex
+@misc{Rubin2024,
+    author  = {Ohad Rubin},
+    url     = {https://medium.com/@ohadrubin/exploring-weight-decay-in-layer-normalization-challenges-and-a-reparameterization-solution-ad4d12c24950}
+}
+```

{hyper_connections-0.0.19 → hyper_connections-0.0.21}/README.md

@@ -112,3 +112,10 @@ HyperConnections.get_init_and_expand_reduce_stream_functions(4, disable = True)
     url     = {https://api.semanticscholar.org/CorpusID:272987528}
 }
 ```
+
+```bibtex
+@misc{Rubin2024,
+    author  = {Ohad Rubin},
+    url     = {https://medium.com/@ohadrubin/exploring-weight-decay-in-layer-normalization-challenges-and-a-reparameterization-solution-ad4d12c24950}
+}
+```

{hyper_connections-0.0.19 → hyper_connections-0.0.21}/hyper_connections/hyper_connections.py

@@ -12,6 +12,14 @@ from torch.utils._pytree import tree_flatten, tree_unflatten
 
 from einops import rearrange, repeat, reduce, einsum
 
+"""
+ein notation:
+b - batch
+d - feature dimension
+s - residual streams
+t - residual streams + num branch inputs
+"""
+
 # helper functions
 
 def exists(v):
@@ -23,6 +31,17 @@ def default(v, d):
 def identity(t):
     return t
 
+# norms
+
+class RMSNorm(Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.scale = dim ** 0.5
+        self.gamma = nn.Parameter(torch.zeros(dim))
+
+    def forward(self, x):
+        return F.normalize(x, dim = -1) * self.scale * (self.gamma + 1)
+
 # main classes
 
 # residual base class
@@ -100,7 +119,9 @@ class HyperConnections(Module):
 
         self.act = nn.Tanh() if tanh else nn.Identity()
 
-        self.norm = nn.RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
+        self.norm = RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
+
+        assert num_residual_streams > 0, '`num_residual_streams` must be greater than 0'
 
         self.num_residual_streams = num_residual_streams
         init_residual_index = default(layer_index, randrange(num_residual_streams)) % num_residual_streams # just choose one random residual stream if layer index not given

{hyper_connections-0.0.19 → hyper_connections-0.0.21}/hyper_connections/hyper_connections_with_multi_branch_inputs.py

@@ -12,7 +12,17 @@ from torch.utils._pytree import tree_flatten, tree_unflatten
 
 from einops import rearrange, repeat, reduce, einsum
 
-from hyper_connections.hyper_connections import Residual, StreamEmbed
+"""
+ein notation:
+b - batch
+d - feature dimension
+s - residual streams
+i - branch inputs
+br - branch functions
+t - residual streams + num branch inputs
+"""
+
+from hyper_connections.hyper_connections import Residual, StreamEmbed, RMSNorm
 
 # helper functions
 
@@ -64,7 +74,7 @@ class HyperConnections(Module):
 
         self.act = nn.Tanh() if tanh else nn.Identity()
 
-        self.norm = nn.RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
+        self.norm = RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
 
         self.num_residual_streams = num_residual_streams
         self.num_branch_inputs = num_branch_inputs

{hyper_connections-0.0.19 → hyper_connections-0.0.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hyper-connections"
-version = "0.0.19"
+version = "0.0.21"
 description = "Hyper-Connections"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }