hyper-connections 0.0.23__py3-none-any.whl → 0.1.0__py3-none-any.whl

hyper_connections/hyper_connections_with_multi_input_streams.py

@@ -145,7 +145,7 @@ class Residual(Module):
 
 # hyper connection with multiple input streams
 
-InputPathType = int | str  # the path to the second residual stream, where `int` points to *args[`int` + 1] and `str` points to **kwargs[`str`]
+InputPathType = int | str  # the path to the second residual stream, where `int` points to *args[`int`] and `str` points to **kwargs[`str`] - `int` needs to be > 0, as 0 is the default input residual stream
 
 class HyperConnections(Module):
     @beartype
@@ -185,7 +185,7 @@ class HyperConnections(Module):
         init_alpha0 = torch.zeros((num_residual_streams, 1))
         init_alpha0[init_residual_index, 0] = 1.
 
-        self.dynamic_alpha_and_branch_input = ProjActScale(dim, num_residual_streams + 1)
+        self.dynamic_alpha_and_branch_input = ProjActScale(dim, num_residual_streams + 1, activation = act)
         self.static_alpha = nn.Parameter(torch.cat([init_alpha0, torch.eye(num_residual_streams)], dim = 1))
 
         self.dynamic_beta = ProjActScale(dim, 1, activation = act, squeeze_output = True)
@@ -196,9 +196,11 @@ class HyperConnections(Module):
         additional_input_paths = default(additional_input_paths, [])
         additional_input_paths = [one_path if isinstance(one_path, tuple) else (one_path, dim) for one_path in additional_input_paths]
 
+        assert all([isinstance(path, str) or path > 0 for (path, _) in additional_input_paths])
+
         self.additional_norms = ModuleList([RMSNorm(dim) for _, dim in additional_input_paths])
         self.additional_to_dynamic_input = ModuleList([ProjActScale(dim, 1, activation = act, squeeze_output = True) for _ , dim in additional_input_paths])
-        self.additional_static_input = nn.ParameterList([nn.Parameter(init_alpha0[..., 0])])
+        self.additional_static_input = nn.ParameterList([nn.Parameter(init_alpha0[..., 0]) for _ in additional_input_paths])
 
         self.additional_input_paths = additional_input_paths
 
@@ -247,12 +249,14 @@ class HyperConnections(Module):
 
         # take care of additional inputs
 
+        branch_args = list(branch_args)
+
         for (path, *_), norm, proj, learned_static in zip(self.additional_input_paths, self.additional_norms, self.additional_to_dynamic_input, self.additional_static_input):
 
             # get the residual streams from additional arguments
 
             if isinstance(path, int):
-                additional_residuals = branch_args[path]
+                additional_residuals = branch_args[path - 1]
             elif isinstance(path, str):
                 additional_residuals = branch_kwargs[path]
 
@@ -280,7 +284,7 @@ class HyperConnections(Module):
             # set back transformed residual
 
             if isinstance(path, int):
-                branch_args[path] = additional_residuals
+                branch_args[path - 1] = additional_residuals
             elif isinstance(path, str):
                 branch_kwargs[path] = additional_residuals
 

{hyper_connections-0.0.23.dist-info → hyper_connections-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.0.23
+Version: 0.1.0
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections

hyper_connections-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+hyper_connections/__init__.py,sha256=d2zNTka0Gp9vINu4U-RhgTJFBhsVrs1fne_15Zl0oOs,224
+hyper_connections/hyper_connections.py,sha256=HyMz-jmICBC6L8QT-LA3EdY8djqG5XkOV7mi-i420mI,9993
+hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=x4it5IGllpZGYank8PBHCRzFeozgZxUY7UYo6YkSkcg,7778
+hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=rSKxtJReg0H-eSCFxdruf2HKU-0-iZ4_x5Fcla3sa3Y,11317
+hyper_connections-0.1.0.dist-info/METADATA,sha256=LNCK8n3-8qGa3twXYGXBg3scYDzw8KGCQDXiRQrV14Q,5314
+hyper_connections-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hyper_connections-0.1.0.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.1.0.dist-info/RECORD,,

hyper_connections-0.0.23.dist-info/RECORD

@@ -1,8 +0,0 @@
-hyper_connections/__init__.py,sha256=d2zNTka0Gp9vINu4U-RhgTJFBhsVrs1fne_15Zl0oOs,224
-hyper_connections/hyper_connections.py,sha256=HyMz-jmICBC6L8QT-LA3EdY8djqG5XkOV7mi-i420mI,9993
-hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=x4it5IGllpZGYank8PBHCRzFeozgZxUY7UYo6YkSkcg,7778
-hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=wma0du8wO6Q5690aYk7TwIza8cT6Csf5c-8EtiIaIVI,11058
-hyper_connections-0.0.23.dist-info/METADATA,sha256=6Uhck5q8NtgJZYnYiTcPjPQyD1nd-m1crZOXyDEZ_ZU,5315
-hyper_connections-0.0.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hyper_connections-0.0.23.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.0.23.dist-info/RECORD,,