hyper-connections 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl

hyper_connections/hyper_connections.py

@@ -19,6 +19,7 @@ b - batch
 d - feature dimension
 s - residual streams
 t - residual streams + num branch inputs
+v - number of views for branch input
 """
 
 # helper functions
@@ -32,6 +33,9 @@ def default(v, d):
 def identity(t):
     return t
 
+def add(x, y):
+    return x + y
+
 # main functions
 
 def get_expand_reduce_stream_functions(num_streams, add_stream_embed = False, dim = None, disable = False):
@@ -141,7 +145,9 @@ class HyperConnections(Module):
         channel_first = False,
         dropout = 0.,
         residual_transform: Module | None = None, # to support resnet blocks where dimension in not equal to dimension out - usually a residual conv
-        add_branch_out_to_residual = True # will disable depth connections (weighted residual sum with beta) if set False
+        add_branch_out_to_residual = True, # will disable depth connections (weighted residual sum with beta) if set False
+        num_input_views = 1, # allow for the branch module to receive multiple input views, dimension placed on the very left (before batch)
+        depth_residual_fn = add
     ):
         """
         Appendix J, Algorithm2 in - https://arxiv.org/abs/2409.19606
@@ -161,14 +167,19 @@ class HyperConnections(Module):
         self.num_residual_streams = num_residual_streams
         init_residual_index = default(layer_index, randrange(num_residual_streams)) % num_residual_streams # just choose one random residual stream if layer index not given
 
+        # width num residual streams
+
+        assert num_input_views >= 1
+        self.num_input_views = num_input_views
+
         # width connection
 
-        init_alpha0 = torch.zeros((num_residual_streams, 1))
-        init_alpha0[init_residual_index, 0] = 1.
+        init_alpha0 = torch.zeros((num_residual_streams, num_input_views))
+        init_alpha0[init_residual_index, :] = 1.
 
         self.static_alpha = nn.Parameter(torch.cat([init_alpha0, torch.eye(num_residual_streams)], dim = 1))
 
-        self.dynamic_alpha_fn = nn.Parameter(torch.zeros(dim, num_residual_streams + 1))
+        self.dynamic_alpha_fn = nn.Parameter(torch.zeros(dim, num_residual_streams + num_input_views))
         self.dynamic_alpha_scale = nn.Parameter(torch.ones(()) * 1e-2)
 
         # depth connection related (beta)
@@ -192,6 +203,12 @@ class HyperConnections(Module):
 
         self.residual_transform = default(residual_transform, nn.Identity())
 
+        # maybe custom depth connection residual function
+        # this is to prepare for gating the addition of the branch outputs to the residual streams
+        # needed for memory lanes a la RMT / LMM
+
+        self.depth_residual_fn = depth_residual_fn
+
     def width_connection(self, residuals):
 
         maybe_transformed_residuals = self.residual_transform(residuals)
@@ -222,7 +239,11 @@ class HyperConnections(Module):
 
         mix_h = einsum(alpha, residuals, '... s t, ... s d -> ... t d')
 
-        branch_input, residuals = mix_h[..., 0, :], mix_h[..., 1:, :]
+        if self.num_input_views == 1:
+            branch_input, residuals = mix_h[..., 0, :], mix_h[..., 1:, :]
+        else:
+            branch_input, residuals = mix_h[..., :self.num_input_views, :], mix_h[..., self.num_input_views:, :]
+            branch_input = rearrange(branch_input, 'b ... v d -> v b ... d')
 
         if self.channel_first:
             branch_input = rearrange(branch_input, 'b ... d -> b d ...')
@@ -243,7 +264,7 @@ class HyperConnections(Module):
         if self.channel_first:
             output = rearrange(output, 'b ... d -> b d ...')
 
-        residuals = residuals + output
+        residuals = self.depth_residual_fn(output, residuals)
 
         return self.dropout(residuals)
 

hyper_connections/residuals.py

@@ -0,0 +1,22 @@
+import torch
+from torch import nn
+from torch.nn import Module
+
+from einops import rearrange
+
+class GRUGatedResidual(Module):
+    def __init__(
+        self,
+        dim
+    ):
+        super().__init__()
+        self.gru = nn.GRUCell(dim, dim)
+
+    def forward(self, x, residual):
+
+        gated_output = self.gru(
+            rearrange(x, 'b n d -> (b n) d'),
+            rearrange(residual, 'b n d -> (b n) d')
+        )
+
+        return gated_output.reshape_as(x)

{hyper_connections-0.1.10.dist-info → hyper_connections-0.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.1.10
+Version: 0.1.12
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections

{hyper_connections-0.1.10.dist-info → hyper_connections-0.1.12.dist-info}/RECORD

@@ -1,9 +1,10 @@
 hyper_connections/__init__.py,sha256=d2zNTka0Gp9vINu4U-RhgTJFBhsVrs1fne_15Zl0oOs,224
-hyper_connections/hyper_connections.py,sha256=Jk7Ux8fJPz63EkgZgKa7fQqpqCasr6cLZt7Fd06dPoE,11563
+hyper_connections/hyper_connections.py,sha256=vpipBRUGgYQ2qLBtT4Ws-myJYVdkQDkN3IkpTMkxRxc,12485
 hyper_connections/hyper_connections_channel_first.py,sha256=BojfO2dcT4jX1rlcU3kr0B6B_CjrkkS2AZU4ZXeWvh8,6769
 hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=2JABz1slrF5_XP33L1CMNTmmixfoD464FtQpUADqneU,7806
 hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=UIKceEZEBLKFL5VuP5tR1KTDeZNIJEKjFuPAwXkcp0I,11282
-hyper_connections-0.1.10.dist-info/METADATA,sha256=K2EgcNxhmXRGTOEbVvsVsQl_dKLZ6iw88dzZqD6zaf4,5231
-hyper_connections-0.1.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hyper_connections-0.1.10.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.1.10.dist-info/RECORD,,
+hyper_connections/residuals.py,sha256=F8NMh9vHDN8Nyo0nzIm_owgR9gXPZUCNujAuduYbTiU,465
+hyper_connections-0.1.12.dist-info/METADATA,sha256=rRgT4Zhj2xUPSRPH0LKZmfJK4olGOxpxPR5ha9wEMQE,5231
+hyper_connections-0.1.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hyper_connections-0.1.12.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.1.12.dist-info/RECORD,,