broccoli-ml 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

broccoli/vit.py

@@ -49,7 +49,7 @@ class SequencePool(nn.Module):
         return self.norm(projection)
 
 
-class CCTEncoder(nn.Module):
+class DCTEncoder(nn.Module):
     """
     Based on the Compact Convolutional Transformer (CCT) of [Hasani et al. (2021)
         *''Escaping the Big Data Paradigm with Compact Transformers''*](
@@ -69,20 +69,20 @@ class CCTEncoder(nn.Module):
         cnn_kernel_padding="same",
         cnn_kernel_dilation=1,
         cnn_kernel_groups=1,
-        cnn_activation: nn.Module = nn.ReLU,
+        cnn_activation: nn.Module = ReLU,
         cnn_activation_kwargs: Optional[dict] = None,
         cnn_dropout=0.0,
-        pooling_type="maxpool",
+        pooling_type="concat",  # maxpool or concat
         pooling_kernel_size=3,
         pooling_kernel_stride=2,
         pooling_kernel_padding=1,
-        transformer_position_embedding="absolute",  # absolute or relative
+        transformer_position_embedding="relative",  # absolute or relative
         transformer_embedding_size=256,
         transformer_layers=7,
         transformer_heads=4,
         transformer_mlp_ratio=2,
         transformer_bos_tokens=4,
-        transformer_activation: nn.Module = nn.GELU,
+        transformer_activation: nn.Module = SquaredReLU,
         transformer_activation_kwargs: Optional[dict] = None,
         mlp_dropout=0.0,
         msa_dropout=0.1,
@@ -291,8 +291,9 @@ class CCTEncoder(nn.Module):
         return self.encoder(x)
 
 
-class CCT(nn.Module):
+class DCT(nn.Module):
     """
+    Denoising convolutional transformer
     Based on the Compact Convolutional Transformer (CCT) of [Hasani et al. (2021)
         *''Escaping the Big Data Paradigm with Compact Transformers''*](
         https://arxiv.org/abs/2104.05704). It's a convolutional neural network
@@ -309,20 +310,20 @@ class CCT(nn.Module):
         cnn_kernel_padding="same",
         cnn_kernel_dilation=1,
         cnn_kernel_groups=1,
-        cnn_activation: nn.Module = nn.ReLU,
+        cnn_activation: nn.Module = ReLU,
         cnn_activation_kwargs: Optional[dict] = None,
         cnn_dropout=0.0,
-        pooling_type="maxpool",
+        pooling_type="concat",  # maxpool or concat
         pooling_kernel_size=3,
         pooling_kernel_stride=2,
         pooling_kernel_padding=1,
-        transformer_position_embedding="absolute",  # absolute or relative
+        transformer_position_embedding="relative",  # absolute or relative
         transformer_embedding_size=256,
         transformer_layers=7,
         transformer_heads=4,
         transformer_mlp_ratio=2,
         transformer_bos_tokens=4,
-        transformer_activation: nn.Module = nn.GELU,
+        transformer_activation: nn.Module = SquaredReLU,
         transformer_activation_kwargs: Optional[dict] = None,
         mlp_dropout=0.0,
         msa_dropout=0.1,
@@ -350,7 +351,7 @@ class CCT(nn.Module):
                 "SwiGLU": SwiGLU,
             }[transformer_activation]
 
-        self.encoder = CCTEncoder(
+        self.encoder = DCTEncoder(
             input_size=input_size,
             cnn_in_channels=cnn_in_channels,
             minimum_cnn_out_channels=minimum_cnn_out_channels,

{broccoli_ml-0.3.0.dist-info → broccoli_ml-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: broccoli-ml
-Version: 0.3.0
+Version: 0.4.0
 Summary: Some useful Pytorch models, circa 2025
 License: MIT
 Author: Nicholas Bailey

{broccoli_ml-0.3.0.dist-info → broccoli_ml-0.4.0.dist-info}/RECORD

@@ -10,8 +10,8 @@ broccoli/rope.py,sha256=hw7kBPNR9GQXj4GxyIAffsGKPfcTPOFh8Bc7oEHtaZY,12108
 broccoli/tensor.py,sha256=E2JK5mQwJf75e23-JGcDoT7QxQf89DJReUo2et1LhRY,1716
 broccoli/transformer.py,sha256=23R58t3TLZMb9ulhCtQ3gXu0mPlfyPvLM8TaGOpaz58,16310
 broccoli/utils.py,sha256=htq_hOsdhUhL0nJi9WkKiEYOjEoWqFpK5X49PtgTf-0,299
-broccoli/vit.py,sha256=NuHW2xcaUEv_IHAZbrrGHUWKu9D7JMR1iKDCCX07RQs,13787
-broccoli_ml-0.3.0.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
-broccoli_ml-0.3.0.dist-info/METADATA,sha256=sAbHQ0Q2yM5kaovkF22cTKCk4SU_z6vi6QtmOMMwJlQ,1256
-broccoli_ml-0.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-broccoli_ml-0.3.0.dist-info/RECORD,,
+broccoli/vit.py,sha256=wPovWrrK-s7rPcAqvvGUWXu2v_77-GMIgmgb6G_y2x8,13869
+broccoli_ml-0.4.0.dist-info/LICENSE,sha256=0BAzJE5BqQ7Iixp_AFdB2W1uO-HCRX-Qfun8PHt6yVM,1073
+broccoli_ml-0.4.0.dist-info/METADATA,sha256=88B7KqQ9zmxkzelSdhVirPbla09qQbpQrwilC6xTCng,1256
+broccoli_ml-0.4.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+broccoli_ml-0.4.0.dist-info/RECORD,,