hyper-connections 0.0.8__py3-none-any.whl → 0.0.9__py3-none-any.whl

hyper_connections/hyper_connections.py

@@ -79,8 +79,11 @@ class HyperConnections(Module):
 
         self.branch = branch
 
+        # activation, seemingly results were wishy washy depending on using tanh or not
+
         self.act = nn.Tanh() if tanh else nn.Identity()
-        self.norm = nn.RMSNorm(dim)
+
+        self.norm = nn.RMSNorm(dim) # they used layernorm in paper, but rmsnorm is fine given what we know now
 
         self.num_residual_streams = num_residual_streams
         init_residual_index = default(layer_index, randrange(num_residual_streams)) % num_residual_streams # just choose one random residual stream if layer index not given
@@ -163,6 +166,20 @@ class HyperConnections(Module):
 
         return output
 
+    def decorate_branch(self, branch: Callable):
+        assert not exists(self.branch), 'branch was already wrapped on init'
+
+        def forward_and_add_residual(residual, *args, **kwargs):
+            branch_input, add_residual = self.forward(residual)
+
+            branch_output = branch(branch_input)
+
+            residual = add_residual(branch_output)
+
+            return residual
+
+        return forward_and_add_residual
+
     def forward(self, residuals, *branch_args, **branch_kwargs):
 
         branch_input, residuals, residual_kwargs = self.width_connection(residuals)

{hyper_connections-0.0.8.dist-info → hyper_connections-0.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.0.8
+Version: 0.0.9
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections
@@ -130,6 +130,8 @@ branch_output = branch(branch_input)
 
 residual = add_residual(branch_output)
 
+# or you can do it in one line as so -> residual = hyper_conn.decorate_branch(branch)(residual)
+
 # 4. reduce 4 streams with a summation, this has to be done after your for loop trunk
 
 residual = reduce_stream(residual)

hyper_connections-0.0.9.dist-info/RECORD

@@ -0,0 +1,6 @@
+hyper_connections/__init__.py,sha256=mAy66IuHqXM4XOyOZGt5mo2B2hfHdUk8jW31YnWNQTg,104
+hyper_connections/hyper_connections.py,sha256=YfEDcPcT93-S599gFUpaATtuUZ908vJ_pmjPeF4Po28,7558
+hyper_connections-0.0.9.dist-info/METADATA,sha256=xur7rWt-ZdJU1XxXlpaO0D9aWDR1BgaVNhjfnQaedZQ,5075
+hyper_connections-0.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hyper_connections-0.0.9.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.0.9.dist-info/RECORD,,

hyper_connections-0.0.8.dist-info/RECORD

@@ -1,6 +0,0 @@
-hyper_connections/__init__.py,sha256=mAy66IuHqXM4XOyOZGt5mo2B2hfHdUk8jW31YnWNQTg,104
-hyper_connections/hyper_connections.py,sha256=fdcr0DODcIQ1eggy7pa6faX6MqNIZST_q2aDMevViig,6964
-hyper_connections-0.0.8.dist-info/METADATA,sha256=2q_q0AHjDyFHzliHiRXXTBaI9iA1uqECGAH_HMZmGis,4978
-hyper_connections-0.0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hyper_connections-0.0.8.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.0.8.dist-info/RECORD,,