hyper-connections 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl

hyper_connections/manifold_constrained_hyper_connections.py

@@ -348,22 +348,18 @@ class ManifoldConstrainedHyperConnections(Module):
 
         alpha = dynamic_alpha + static_alpha
 
-        alpha = self.split_fracs(alpha) # (batch, seq, fracs1, streams, fracs2, input + residual streams)
-
         # the alpha is now split and "manifold constrained" with sinkhorn and sigmoid
 
         alpha_pre, alpha_residual = alpha[..., :self.num_input_views], alpha[..., self.num_input_views:]
 
         alpha_pre = alpha_pre.sigmoid()
 
-        alpha_residual = rearrange(alpha_residual, '... (v s1 s2) -> ... v s1 s2', v = self.num_input_views, s1 = streams)
-
         alpha_residual = sinkhorn_knopps(alpha_residual, self.sinkhorn_iters)
 
-        alpha_residual = rearrange(alpha_residual, '... v s1 s2 -> ... (v s1 s2)')
-
         alpha = cat((alpha_pre, alpha_residual), dim = -1)
 
+        alpha = self.split_fracs(alpha) # (batch, seq, fracs1, streams, fracs2, input + residual streams)
+
         # beta for weights from branch output back to residual streams
 
         beta = None
@@ -371,8 +367,6 @@ class ManifoldConstrainedHyperConnections(Module):
         if self.add_branch_out_to_residual:
             dc_weight = normed @ self.dynamic_beta_fn.float()
 
-            dc_weight = dc_weight.sigmoid() * 2 # sigmoid * 2 for "H_post", corresponding to dc weight in original paper
-
             if not self.has_fracs:
                 dc_weight = rearrange(dc_weight, '... -> ... 1')
 
@@ -382,6 +376,8 @@ class ManifoldConstrainedHyperConnections(Module):
 
             beta = dynamic_beta + static_beta
 
+            beta = beta.sigmoid() * 2 # for "H_post" manifold constraint
+
         mix_h = einsum(alpha, residuals.float(), '... f1 s f2 t, ... f1 s d -> ... f2 t d')
 
         if self.num_input_views == 1:

{hyper_connections-0.3.10.dist-info → hyper_connections-0.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.3.10
+Version: 0.3.12
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections

{hyper_connections-0.3.10.dist-info → hyper_connections-0.3.12.dist-info}/RECORD

@@ -3,10 +3,10 @@ hyper_connections/hyper_connections.py,sha256=rqFJj3U0LF3uDKNKNPBpRrmf0oa2BGWVbD
 hyper_connections/hyper_connections_channel_first.py,sha256=Mh_hzhTi96ZoOPmhSKwUaF4TbHpNqhs83wNe5hNuL7o,6532
 hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=6BXKdSwyx6wdQVseebKG2EQkhVaVLrrepOlL8lLnex4,7855
 hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=ueT3CJPHrt5hRU7q1bFF0rANWJh_pXqclt6HiUu1gBY,11331
-hyper_connections/manifold_constrained_hyper_connections.py,sha256=SkGAWpBHnrOlIcixb0iIGej9StO82O7KXrFjYuSKx7I,17424
+hyper_connections/manifold_constrained_hyper_connections.py,sha256=JF_ncPloyNLr303xZMlCiCorCE1Qt_soo1oeOyGYIdc,17168
 hyper_connections/residuals.py,sha256=JVSFJj_H7xQ3_Fd-pZH5Hdv9SveAQu29jQNvMyom5ek,921
 hyper_connections/vit.py,sha256=fTC8hAYkD4qm-KURAj8SJ66C6ZWtsBdHf_kS-4rJZGQ,5049
-hyper_connections-0.3.10.dist-info/METADATA,sha256=tEYVvFTVY_13gYQbflz-mjWMQEwH4DvPvQk76X9Iq2E,6705
-hyper_connections-0.3.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hyper_connections-0.3.10.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.3.10.dist-info/RECORD,,
+hyper_connections-0.3.12.dist-info/METADATA,sha256=OyTViiZQQ1AmohzVqRhf71HOFPjLEVMGPVY5kY6wHsg,6705
+hyper_connections-0.3.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hyper_connections-0.3.12.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.3.12.dist-info/RECORD,,