hyper-connections 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

hyper_connections/hyper_connections.py

@@ -175,7 +175,6 @@ class HyperConnections(Module):
         tanh = True,
         channel_first = False,
         dropout = 0.,
-        residual_transform: Module | None = None, # to support resnet blocks where dimension in not equal to dimension out - usually a residual conv
         add_branch_out_to_residual = True,  # will disable depth connections (weighted residual sum with beta) if set False
         num_input_views = 1,                # allow for the branch module to receive multiple input views, dimension placed on the very left (before batch)
         depth_residual_fn = add,
@@ -255,10 +254,6 @@ class HyperConnections(Module):
 
         self.channel_first = channel_first
 
-        # maybe residual transform
-
-        self.residual_transform = default(residual_transform, nn.Identity())
-
         # maybe custom depth connection residual function
         # this is to prepare for gating the addition of the branch outputs to the residual streams
         # needed for memory lanes a la RMT / LMM
@@ -271,8 +266,6 @@ class HyperConnections(Module):
     ):
         streams = self.num_residual_streams
 
-        maybe_transformed_residuals = self.residual_transform(residuals)
-
         # width connection
 
         # handle channel first
@@ -334,7 +327,14 @@ class HyperConnections(Module):
 
         branch_input = self.merge_fracs(branch_input)
 
-        return branch_input, maybe_transformed_residuals, dict(beta = beta)
+        # reshape residuals back
+
+        if self.channel_first:
+            residuals = rearrange(residuals, 'b ... f s d -> (b s) (f d) ...')
+        else:
+            residuals = rearrange(residuals, 'b ... f s d -> (b s) ... (f d)')
+
+        return branch_input, residuals, dict(beta = beta)
 
     def depth_connection(
         self,

hyper_connections/hyper_connections_channel_first.py

@@ -84,7 +84,6 @@ class HyperConnections(Module):
         tanh = True,
         channel_first = True,
         dropout = 0.,
-        residual_transform: Module | None = None, # to support resnet blocks where dimension in not equal to dimension out - usually a residual conv
     ):
         """
         Appendix J, Algorithm2 in - https://arxiv.org/abs/2409.19606
@@ -124,19 +123,12 @@ class HyperConnections(Module):
         self.dynamic_alpha_scale = nn.Parameter(torch.ones(()) * 1e-2)
         self.dynamic_beta_scale = nn.Parameter(torch.ones(()) * 1e-2)
 
-
         # dropouts
 
         self.dropout = nn.Dropout(dropout)
 
-        # maybe residual transform
-
-        self.residual_transform = default(residual_transform, nn.Identity())
-
     def width_connection(self, residuals):
 
-        maybe_transformed_residuals = self.residual_transform(residuals)
-
         # width connection
 
         normed = self.norm(residuals)
@@ -161,7 +153,9 @@ class HyperConnections(Module):
 
         branch_input, residuals = mix_h[:, 0, ...], mix_h[:, 1:, ...]
 
-        return branch_input, maybe_transformed_residuals, dict(beta = beta)
+        residuals = rearrange(residuals, 'b s d ... -> (b s) d ...')
+
+        return branch_input, residuals, dict(beta = beta)
 
     def depth_connection(self, branch_output, residuals, *, beta):
         # 'depth' connection

hyper_connections/manifold_constrained_hyper_connections.py

@@ -307,7 +307,7 @@ class ManifoldConstrainedHyperConnections(Module):
     ):
         streams = self.num_residual_streams
 
-        maybe_transformed_residuals = self.residual_transform(residuals)
+        residuals = self.residual_transform(residuals)
 
         # width connection
 
@@ -397,13 +397,14 @@ class ManifoldConstrainedHyperConnections(Module):
 
         branch_input = self.merge_fracs(branch_input)
 
-        branch_input = branch_input.to(dtype)
-        residuals = residuals.to(dtype)
+        residuals = rearrange(residuals, 'b ... f s d -> (b s) ... (f d)')
+
+        branch_input, residuals = tuple(t.to(dtype) for t in (branch_input, residuals))
 
         if exists(beta):
             beta = beta.to(dtype)
 
-        return branch_input, maybe_transformed_residuals, dict(beta = beta)
+        return branch_input, residuals, dict(beta = beta)
 
     def depth_connection(
         self,
@@ -486,6 +487,8 @@ class ManifoldConstrainedHyperConnections(Module):
 
         return add_residual_fn(branch_output)
 
+mHC = ManifoldConstrainedHyperConnections
+
 ManifoldConstrainedHyperConnections.get_expand_reduce_stream_functions = staticmethod(get_expand_reduce_stream_functions)
 ManifoldConstrainedHyperConnections.get_init_and_expand_reduce_stream_functions = staticmethod(get_init_and_expand_reduce_stream_functions)
 

hyper_connections/vit.py

@@ -0,0 +1,163 @@
+import torch
+from torch import nn
+from torch.nn import Module, ModuleList
+
+from einops import rearrange, repeat
+from einops.layers.torch import Rearrange
+
+from hyper_connections.manifold_constrained_hyper_connections import mHC
+
+# helpers
+
+def pair(t):
+    return t if isinstance(t, tuple) else (t, t)
+
+# classes
+
+class FeedForward(Module):
+    def __init__(self, dim, hidden_dim, dropout = 0.):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.LayerNorm(dim),
+            nn.Linear(dim, hidden_dim),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim, dim),
+            nn.Dropout(dropout)
+        )
+
+    def forward(self, x):
+        return self.net(x)
+
+class Attention(Module):
+    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):
+        super().__init__()
+        inner_dim = dim_head *  heads
+        project_out = not (heads == 1 and dim_head == dim)
+
+        self.heads = heads
+        self.scale = dim_head ** -0.5
+
+        self.norm = nn.LayerNorm(dim)
+
+        self.attend = nn.Softmax(dim = -1)
+        self.dropout = nn.Dropout(dropout)
+
+        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
+
+        self.to_out = nn.Sequential(
+            nn.Linear(inner_dim, dim),
+            nn.Dropout(dropout)
+        ) if project_out else nn.Identity()
+
+    def forward(self, x):
+        x = self.norm(x)
+
+        qkv = self.to_qkv(x).chunk(3, dim = -1)
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)
+
+        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale
+
+        attn = self.attend(dots)
+        attn = self.dropout(attn)
+
+        out = torch.matmul(attn, v)
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        return self.to_out(out)
+
+class Transformer(Module):
+    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0., num_residual_streams = 4):
+        super().__init__()
+        self.norm = nn.LayerNorm(dim)
+        self.layers = ModuleList([])
+
+        init_hyper_conn, self.expand_streams, self.reduce_streams = mHC.get_init_and_expand_reduce_stream_functions(num_residual_streams)
+
+        for _ in range(depth):
+            self.layers.append(ModuleList([
+                init_hyper_conn(dim = dim , branch = Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),
+                init_hyper_conn(dim = dim, branch = FeedForward(dim, mlp_dim, dropout = dropout))
+            ]))
+
+    def forward(self, x):
+
+        x = self.expand_streams(x)
+
+        for attn, ff in self.layers:
+            x = attn(x)
+            x = ff(x)
+
+        x = self.reduce_streams(x)
+
+        return self.norm(x)
+
+class ViT(Module):
+    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0., num_residual_streams = 4):
+        super().__init__()
+        image_height, image_width = pair(image_size)
+        patch_height, patch_width = pair(patch_size)
+
+        assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'
+
+        num_patches = (image_height // patch_height) * (image_width // patch_width)
+        patch_dim = channels * patch_height * patch_width
+
+        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'
+        num_cls_tokens = 1 if pool == 'cls' else 0
+
+        self.to_patch_embedding = nn.Sequential(
+            Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width),
+            nn.LayerNorm(patch_dim),
+            nn.Linear(patch_dim, dim),
+            nn.LayerNorm(dim),
+        )
+
+        self.cls_token = nn.Parameter(torch.randn(num_cls_tokens, dim))
+        self.pos_embedding = nn.Parameter(torch.randn(num_patches + num_cls_tokens, dim))
+
+        self.dropout = nn.Dropout(emb_dropout)
+
+        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)
+
+        self.pool = pool
+        self.to_latent = nn.Identity()
+
+        self.mlp_head = nn.Linear(dim, num_classes)
+
+    def forward(self, img):
+        batch = img.shape[0]
+        x = self.to_patch_embedding(img)
+
+        cls_tokens = repeat(self.cls_token, '... d -> b ... d', b = batch)
+        x = torch.cat((cls_tokens, x), dim = 1)
+
+        seq = x.shape[1]
+
+        x = x + self.pos_embedding[:seq]
+        x = self.dropout(x)
+
+        x = self.transformer(x)
+
+        x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]
+
+        x = self.to_latent(x)
+        return self.mlp_head(x)
+
+if __name__ == '__main__':
+    v = ViT(
+        image_size = 256,
+        patch_size = 32,
+        num_classes = 1000,
+        dim = 1024,
+        depth = 6,
+        heads = 16,
+        mlp_dim = 2048,
+        dropout = 0.1,
+        emb_dropout = 0.1,
+        num_residual_streams = 4
+    )
+
+    img = torch.randn(1, 3, 256, 256)
+
+    preds = v(img) # (1, 1000)
+    assert preds.shape == (1, 1000)

{hyper_connections-0.3.8.dist-info → hyper_connections-0.3.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyper-connections
-Version: 0.3.8
+Version: 0.3.10
 Summary: Hyper-Connections
 Project-URL: Homepage, https://pypi.org/project/hyper-connections/
 Project-URL: Repository, https://github.com/lucidrains/hyper-connections

hyper_connections-0.3.10.dist-info/RECORD

@@ -0,0 +1,12 @@
+hyper_connections/__init__.py,sha256=BAGwi53ozXcnfPJAGur0RHA4vcolF1ORBhbZ9a8SkrE,602
+hyper_connections/hyper_connections.py,sha256=rqFJj3U0LF3uDKNKNPBpRrmf0oa2BGWVbD6S-xdZdLo,14904
+hyper_connections/hyper_connections_channel_first.py,sha256=Mh_hzhTi96ZoOPmhSKwUaF4TbHpNqhs83wNe5hNuL7o,6532
+hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=6BXKdSwyx6wdQVseebKG2EQkhVaVLrrepOlL8lLnex4,7855
+hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=ueT3CJPHrt5hRU7q1bFF0rANWJh_pXqclt6HiUu1gBY,11331
+hyper_connections/manifold_constrained_hyper_connections.py,sha256=SkGAWpBHnrOlIcixb0iIGej9StO82O7KXrFjYuSKx7I,17424
+hyper_connections/residuals.py,sha256=JVSFJj_H7xQ3_Fd-pZH5Hdv9SveAQu29jQNvMyom5ek,921
+hyper_connections/vit.py,sha256=fTC8hAYkD4qm-KURAj8SJ66C6ZWtsBdHf_kS-4rJZGQ,5049
+hyper_connections-0.3.10.dist-info/METADATA,sha256=tEYVvFTVY_13gYQbflz-mjWMQEwH4DvPvQk76X9Iq2E,6705
+hyper_connections-0.3.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hyper_connections-0.3.10.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
+hyper_connections-0.3.10.dist-info/RECORD,,

hyper_connections-0.3.8.dist-info/RECORD

@@ -1,11 +0,0 @@
-hyper_connections/__init__.py,sha256=BAGwi53ozXcnfPJAGur0RHA4vcolF1ORBhbZ9a8SkrE,602
-hyper_connections/hyper_connections.py,sha256=UHxZhyRwx89GRgmQVt53Gv6JeNhX8UCjjETlydMZjTk,15021
-hyper_connections/hyper_connections_channel_first.py,sha256=_1PM4LRcPpDqfCiHlBMc2nLV08sXM2nuyZGSKTiuqbE,6818
-hyper_connections/hyper_connections_with_multi_branch_inputs.py,sha256=6BXKdSwyx6wdQVseebKG2EQkhVaVLrrepOlL8lLnex4,7855
-hyper_connections/hyper_connections_with_multi_input_streams.py,sha256=ueT3CJPHrt5hRU7q1bFF0rANWJh_pXqclt6HiUu1gBY,11331
-hyper_connections/manifold_constrained_hyper_connections.py,sha256=rNRh7Hkz-mgZD4GMImIpk3gPHFLsM2_cvVy8I0x2W5U,17339
-hyper_connections/residuals.py,sha256=JVSFJj_H7xQ3_Fd-pZH5Hdv9SveAQu29jQNvMyom5ek,921
-hyper_connections-0.3.8.dist-info/METADATA,sha256=kqntUf_yXJ9fNqxGvQ8XFERHOInCCDAqijtazZfFqes,6704
-hyper_connections-0.3.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hyper_connections-0.3.8.dist-info/licenses/LICENSE,sha256=E7RGS7kpJIStk5za_-4DVhWEAamf65EU0CNML25mq4c,1066
-hyper_connections-0.3.8.dist-info/RECORD,,