mct-nightly 1.10.0.20231204.post420__py3-none-any.whl → 1.10.0.20231206.post417__py3-none-any.whl

{mct_nightly-1.10.0.20231204.post420.dist-info → mct_nightly-1.10.0.20231206.post417.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mct-nightly
-Version: 1.10.0.20231204.post420
+Version: 1.10.0.20231206.post417
 Summary: A Model Compression Toolkit for neural networks
 Home-page: UNKNOWN
 License: UNKNOWN

{mct_nightly-1.10.0.20231204.post420.dist-info → mct_nightly-1.10.0.20231206.post417.dist-info}/RECORD

@@ -4,14 +4,14 @@ model_compression_toolkit/logger.py,sha256=b9DVktZ-LymFcRxv2aL_sdiE6S2sSrFGWltx6
 model_compression_toolkit/core/__init__.py,sha256=qnBA6aaojI7RpEQZU2vXWiELHfVJf-MnAP-4T0tcFDY,2008
 model_compression_toolkit/core/analyzer.py,sha256=dbsD61pakp_9JXNyAScLdtJvcXny9jr_cMbET0Bd3Sg,2975
 model_compression_toolkit/core/exporter.py,sha256=U_-ea-zYHsnIt2ydameMLZ_gzDaCMI1dRa5IjA8RUuc,4233
-model_compression_toolkit/core/graph_prep_runner.py,sha256=SHhFl0vpC9YpRu40xkApFzmw_dT-nfIz1MDjmKcon8Q,9913
+model_compression_toolkit/core/graph_prep_runner.py,sha256=3xp0WYqyeRdlBkf5R6uD2zWubg_JPttOwS7JRhKykBY,10043
 model_compression_toolkit/core/quantization_prep_runner.py,sha256=npv55-QsJFR7bnbHj4tBMf13Y18Ns7QGa-UDSI6WJRE,6554
 model_compression_toolkit/core/runner.py,sha256=Cb8_TWAOBz4SO1O48ehxqC9PpaR4KifbCs0nV724zMM,10454
 model_compression_toolkit/core/common/__init__.py,sha256=Wh127PbXcETZX_d1PQqZ71ETK3J9XO5A-HpadGUbj6o,1447
 model_compression_toolkit/core/common/base_substitutions.py,sha256=xDFSmVVs_iFSZfajytI0cuQaNRNcwHX3uqOoHgVUvxQ,1666
 model_compression_toolkit/core/common/data_loader.py,sha256=7YF5Mqz64Xb4rVwY3knrdIZ4JEHybXxiQqx0deR_c5k,4017
 model_compression_toolkit/core/common/defaultdict.py,sha256=P2WOZbWQTfVKtMfpGhGOS_1_5YWfYQWiJ5pBCn6F-3k,2182
-model_compression_toolkit/core/common/framework_implementation.py,sha256=rDXQHUX62DprSrYy8WPmdfWhh-9RhlykBqh6mvoeAJU,20708
+model_compression_toolkit/core/common/framework_implementation.py,sha256=3oFMtvGkUKPtNxAAiXISmNM8XyccR3DyFQbOioBE4b4,21094
 model_compression_toolkit/core/common/framework_info.py,sha256=hwmstv7IuBRfa6IxDbeG4y-7AxKx4bwCyI_Exi2C7mo,6424
 model_compression_toolkit/core/common/memory_computation.py,sha256=ixoSpV5ZYZGyzhre3kQcvR2sNA8KBsPZ3lgbkDnw9Cs,1205
 model_compression_toolkit/core/common/model_builder_mode.py,sha256=jll9-59OPaE3ug7Y9-lLyV99_FoNHxkGZMgcm0Vkpss,1324
@@ -33,7 +33,7 @@ model_compression_toolkit/core/common/fusion/__init__.py,sha256=Rf1RcYmelmdZmBV5
 model_compression_toolkit/core/common/fusion/layer_fusing.py,sha256=tIsWFYc771o59uvq5fxAaBmOCnd_gd-_xMbQI9SupQA,5479
 model_compression_toolkit/core/common/graph/__init__.py,sha256=Xr-Lt_qXMdrCnnOaUS_OJP_3iTTGfPCLf8_vSrQgCs0,773
 model_compression_toolkit/core/common/graph/base_graph.py,sha256=-YBWWl3ZS7FJvZldGqT4SetlKI8j60f4sS0rYyFQpbI,30059
-model_compression_toolkit/core/common/graph/base_node.py,sha256=9BBECpNwskLjpuC-Eif_2ilTqf42ASqBXnOzrbs34GQ,20878
+model_compression_toolkit/core/common/graph/base_node.py,sha256=csIgi5ex7EquQsF34w5waRIHzbg7XitvIqQgCC29azs,21118
 model_compression_toolkit/core/common/graph/edge.py,sha256=K6Wc2hBcIqig5PbbLhbjtTgYtkyZEohfgj4Wn_J5yEA,3733
 model_compression_toolkit/core/common/graph/functional_node.py,sha256=0TpYNa2ODZ0M9lQ2z_GsStqAbrg1Muwdni74LjphAh0,2922
 model_compression_toolkit/core/common/graph/graph_matchers.py,sha256=kQ14uXW6ecsj7IarjRLAXUzDBmakD_v6Ck7-u24_nxg,4732
@@ -118,12 +118,12 @@ model_compression_toolkit/core/common/statistics_correction/compute_bias_correct
 model_compression_toolkit/core/common/statistics_correction/statistics_correction.py,sha256=KFWY8jERabXwKm-qzQFc2V7v-fM1dqOlwRaOQ8UIiQA,5584
 model_compression_toolkit/core/common/substitutions/__init__.py,sha256=sw7LOPN1bM82o3SkMaklyH0jw-TLGK0-fl2Wq73rffI,697
 model_compression_toolkit/core/common/substitutions/apply_substitutions.py,sha256=k-bifmakHIYZeZS-4T1QpZ1Et6AwAijMRgAKs7hmMKc,1390
-model_compression_toolkit/core/common/substitutions/batchnorm_folding.py,sha256=DavK55_If8mb2DNKwZbAcuW9b5ttEPrGQhy6ZNSYiOo,13483
-model_compression_toolkit/core/common/substitutions/batchnorm_reconstruction.py,sha256=Jcvoon9thUGjuRaZAAbAFcs4V4BV8x78SK4G2byk1gs,5892
-model_compression_toolkit/core/common/substitutions/batchnorm_refusing.py,sha256=ZeVuxSsieTShdxspf7GngoaWGuvmFRbY4yms8ePQQ-4,9962
-model_compression_toolkit/core/common/substitutions/linear_collapsing.py,sha256=1emsEUCtpR06v_IEBpsjK4SDYv4ONfK7AV9dpyVaLE4,9093
-model_compression_toolkit/core/common/substitutions/linear_collapsing_substitution.py,sha256=4HX2nz4OnERp9zO5q0FjP8ROq4TlRiQXm0yNfw7J9iE,2250
-model_compression_toolkit/core/common/substitutions/residual_collapsing.py,sha256=8JVJOztpfr7yNSovy2DLbB798XipvkIEpJZASGs_KIM,4866
+model_compression_toolkit/core/common/substitutions/batchnorm_folding.py,sha256=wLlTT7sqUffKHwOrMG2VV5SktQkkP54l8taW1Fq0mh0,13392
+model_compression_toolkit/core/common/substitutions/batchnorm_reconstruction.py,sha256=lYUZobQKydFyE3nRS-CBsYF3r4YlFirLp3-EmYa9qHM,5859
+model_compression_toolkit/core/common/substitutions/batchnorm_refusing.py,sha256=eTDmac2OvqQgJMCg_dkCGFCmkvvO6mdYjsBui9HLymY,9929
+model_compression_toolkit/core/common/substitutions/linear_collapsing.py,sha256=iEtzbWCDXP6EDkTZCtREQ0rpMxhQ2kM9zlcP_0KLq9I,12367
+model_compression_toolkit/core/common/substitutions/linear_collapsing_substitution.py,sha256=uoauhmncQqUBNvD-qCLIXsIbl_IzrbxSKdxiMig-5W4,2406
+model_compression_toolkit/core/common/substitutions/residual_collapsing.py,sha256=doErjlMq-uSObYMSjA6IywSHb3Hz3QCc0HKU68ccrQ4,4767
 model_compression_toolkit/core/common/substitutions/scale_equalization.py,sha256=nmb5QC_YiQJRbsEIq6uF50y1IRWhmRAUKaeUE9hnoNw,10978
 model_compression_toolkit/core/common/substitutions/shift_negative_activation.py,sha256=QbT6LMt4Eit4i1bLGIizHhE6R_tLeJf2Ix2qVod2bcw,28749
 model_compression_toolkit/core/common/substitutions/softmax_shift.py,sha256=R-0ZqhYAuZLEFWHvB2UTPm52L6gWHGdRdEnwGxKSeGI,2625
@@ -134,10 +134,10 @@ model_compression_toolkit/core/common/visualization/final_config_visualizer.py,s
 model_compression_toolkit/core/common/visualization/nn_visualizer.py,sha256=6EjZj_KE1tICTQ0XSKIx5ivsRFpRktFywda7pW7YnNQ,5955
 model_compression_toolkit/core/common/visualization/tensorboard_writer.py,sha256=954742gUTrrKmcVjcuBJaKR-EfMMsrWZ7PXd07unA6E,21939
 model_compression_toolkit/core/keras/__init__.py,sha256=mjbqLD-KcG3eNeCYpu1GBS7VclGVOQ63x2p6mAAuba4,698
-model_compression_toolkit/core/keras/constants.py,sha256=oFYFagoFTOQTrs2RHVc93583EhOvcvbCYHleqsZdQ6s,3046
+model_compression_toolkit/core/keras/constants.py,sha256=OVa9yHaIlTKU4WatwTw_1dANk1-7ocQxDCluQwnwGy0,3094
 model_compression_toolkit/core/keras/custom_layer_validation.py,sha256=f-b14wuiIgitBe7d0MmofYhDCTO3IhwJgwrh-Hq_t_U,1192
 model_compression_toolkit/core/keras/default_framework_info.py,sha256=cMdt9KvJMqOmWjFtUiEejzOe77mCpnnd3GzERgNh8Zk,4970
-model_compression_toolkit/core/keras/keras_implementation.py,sha256=HhyK68qXyHGEsT5DkfmHTibinRxyTh0DquiXGrBAlK4,28407
+model_compression_toolkit/core/keras/keras_implementation.py,sha256=NYv0gHvv7wHs7grpTjh2SnBgu5OUb5r-fUBpQgb0PT4,28835
 model_compression_toolkit/core/keras/keras_model_validation.py,sha256=1wNV2clFdC9BzIELRLSO2uKf0xqjLqlkTJudwtCeaJk,1722
 model_compression_toolkit/core/keras/keras_node_prior_info.py,sha256=f6o5Fmpw0aDrO704_A-SqBrKSO1iNEOyofP9pm3g8yg,3936
 model_compression_toolkit/core/keras/kpi_data_facade.py,sha256=rArrfMtxWGR1P4nhKKxqh6fo7pauRDzkRsZIh_SXxO4,8502
@@ -157,7 +157,8 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_refusing.py,sha256=5df_xGfXkqNub4xVRnCWQvSohWqdv12axjJ6edVU2H0,2478
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/dwconv_to_conv.py,sha256=R3U7cjc2E0zheMem16GHygp5jZFGSaomkNOTxTjcAgw,5794
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/input_scaling.py,sha256=Yj59BMBrITJnXJHH-7de91LJwH_1l1WhY1udSQjdoi4,5598
-model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py,sha256=Yl72Io4-etnsOXfMKAZmC2lDzmskxZu5gey7IBcUukU,5925
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py,sha256=mNLcAjSYzht_-mKh_fdBs4H4YYcQSLBJBFr_k1owF3s,8473
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py,sha256=QzzKXC_WhojIjIjpqeHxI171DKXcZMdr0hNcf_78o-s,4523
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/multi_head_attention_decomposition.py,sha256=aAG5wrcnnydn1pPYqvH56LWsQXjSODbsoNbX_jtQGP4,26759
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/relu_bound_to_power_of_2.py,sha256=IdKOg6AWZWMcmDbOuNdxetS5_zTarXIIffdYL7JTdvk,3872
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/remove_relu_upper_bound.py,sha256=cJQTDzTDQKAJ7EQ20tfsmReGA_OoTIN793MwVe1Ok8g,2387
@@ -196,7 +197,7 @@ model_compression_toolkit/core/pytorch/__init__.py,sha256=Rf1RcYmelmdZmBV5qOKvKW
 model_compression_toolkit/core/pytorch/constants.py,sha256=Kt_GDwe3yX9oMS1DI2eXYuUT25_lpjeCkxpstsAiXCI,2472
 model_compression_toolkit/core/pytorch/default_framework_info.py,sha256=qee8TFcDro2lfyXe_fujjX2OlxELTyKSsLlZ7QkzeXU,4200
 model_compression_toolkit/core/pytorch/kpi_data_facade.py,sha256=J0IDOtFMVFSFyBXDzNGbwJfHu89iRBJFdid1_wFB-xQ,8482
-model_compression_toolkit/core/pytorch/pytorch_implementation.py,sha256=7zz4c-hpAS1n1yX0jPCyZ1lEneNYQXk-Bx42t4_0Als,26268
+model_compression_toolkit/core/pytorch/pytorch_implementation.py,sha256=oTAd6_XYtyvTX2fRXx0BzajvgqbYreXGKD7ij8iL2SY,26482
 model_compression_toolkit/core/pytorch/pytorch_node_prior_info.py,sha256=n_B4a6FMwM9D2w8kzy3oenBWZgXNZuIZgTJC6JEuTy0,3250
 model_compression_toolkit/core/pytorch/utils.py,sha256=rBQMAbWluyIMjVfeghzq6FZv3sR_khszSRpbWvwussw,2959
 model_compression_toolkit/core/pytorch/back2framework/__init__.py,sha256=H_WixgN0elVWf3exgGYsi58imPoYDj5eYPeh6x4yfug,813
@@ -448,8 +449,8 @@ model_compression_toolkit/trainable_infrastructure/keras/quantize_wrapper.py,sha
 model_compression_toolkit/trainable_infrastructure/keras/quantizer_utils.py,sha256=MVwXNymmFRB2NXIBx4e2mdJ1RfoHxRPYRgjb1MQP5kY,1797
 model_compression_toolkit/trainable_infrastructure/pytorch/__init__.py,sha256=huHoBUcKNB6BnY6YaUCcFvdyBtBI172ZoUD8ZYeNc6o,696
 model_compression_toolkit/trainable_infrastructure/pytorch/base_pytorch_quantizer.py,sha256=SbvRlIdE32PEBsINt1bhSqvrKL_zbM9V-aeSkOn-sw4,3083
-mct_nightly-1.10.0.20231204.post420.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
-mct_nightly-1.10.0.20231204.post420.dist-info/METADATA,sha256=pe_Xo3W_LAeGfbD03ZCsX0r3_iY_yj5oThmtLU2p1pM,16232
-mct_nightly-1.10.0.20231204.post420.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-mct_nightly-1.10.0.20231204.post420.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
-mct_nightly-1.10.0.20231204.post420.dist-info/RECORD,,
+mct_nightly-1.10.0.20231206.post417.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
+mct_nightly-1.10.0.20231206.post417.dist-info/METADATA,sha256=kcsyt4UEdtqaUF2t42UBYPuB1oav069aO3eg3MjBrII,16232
+mct_nightly-1.10.0.20231206.post417.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+mct_nightly-1.10.0.20231206.post417.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
+mct_nightly-1.10.0.20231206.post417.dist-info/RECORD,,

model_compression_toolkit/core/common/framework_implementation.py

@@ -235,6 +235,14 @@ class FrameworkImplementation(ABC):
         raise NotImplemented(f'{self.__class__.__name__} have to implement the '
                              f'framework\'s get_linear_collapsing_substitution method.')  # pragma: no cover
 
+    @abstractmethod
+    def get_op2d_add_const_collapsing_substitution(self) -> common.BaseSubstitution:
+        """
+        Returns: conv2d add const collapsing substitution
+        """
+        raise NotImplemented(f'{self.__class__.__name__} have to implement the '
+                             f'framework\'s get_op2d_add_const_collapsing_substitution method.')  # pragma: no cover
+
     @abstractmethod
     def get_substitutions_statistics_correction(self, quant_config: QuantizationConfig) -> \
             List[common.BaseSubstitution]:

model_compression_toolkit/core/common/graph/base_node.py

@@ -79,7 +79,8 @@ class BaseNode:
     def type(self):
         """
         A function to get the node's layer_class op for convenient comparison
-        :return: the node's layer_class
+        Returns:
+            the node's layer_class
         """
         return self.layer_class
 
@@ -130,6 +131,14 @@ class BaseNode:
         """
         return f'{self.type.__name__}:{self.name}'
 
+    def is_reused(self) -> bool:
+        """
+        Check whether the node is reused or not
+        Returns:
+            True if node is reused, else False
+        """
+        return self.reuse or self.reuse_group is not None
+
     def get_weights_by_keys(self, name: str) -> np.ndarray:
         """
         Get a node's weight by its name.

model_compression_toolkit/core/common/substitutions/batchnorm_folding.py

@@ -93,7 +93,7 @@ class BatchNormalizationFolding(common.BaseSubstitution):
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if conv_node.reuse or conv_node.reuse_group is not None:
+        if conv_node.is_reused():
             return graph
 
         bn_node = edge_nodes[1]
@@ -230,7 +230,7 @@ class BatchNormalizationForwardFolding(common.BaseSubstitution):
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if conv_node.reuse or conv_node.reuse_group is not None or bn_node.reuse or bn_node.reuse_group is not None:
+        if conv_node.is_reused() or bn_node.is_reused():
             return graph
 
         if len(graph.get_next_nodes(bn_node)) > 1 or len(graph.get_prev_nodes(conv_node)) > 1:

model_compression_toolkit/core/common/substitutions/batchnorm_reconstruction.py

@@ -79,7 +79,7 @@ class BatchNormalizationReconstruction(common.BaseSubstitution):
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if source_node.reuse or source_node.reuse_group is not None:
+        if source_node.is_reused():
             for qc in source_node.candidates_quantization_cfg:
                 qc.weights_quantization_cfg.weights_second_moment_correction = False
             return graph

model_compression_toolkit/core/common/substitutions/batchnorm_refusing.py

@@ -102,7 +102,7 @@ class BatchNormalizationRefusing(common.BaseSubstitution):
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if source_node.reuse or source_node.reuse_group is not None:
+        if source_node.is_reused():
             Logger.exception("If the linear operator is part of a reused group we should skip the the BN folding "
                              "substitution and SMC feature")  # pragma: no cover
 

model_compression_toolkit/core/common/substitutions/linear_collapsing.py

@@ -91,14 +91,11 @@ class Conv2DCollapsing(common.BaseSubstitution):
             Graph after applying the substitution.
         """
 
-        first_node = edge_nodes[0]
-        second_node = edge_nodes[1]
+        first_node, second_node, _ = edge_nodes
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if first_node.reuse or first_node.reuse_group is not None:
-            return graph
-        if second_node.reuse or second_node.reuse_group is not None:
+        if first_node.is_reused() or second_node.is_reused():
             return graph
 
         # If there is an extra connection between these two nodes skip the substitution
@@ -182,3 +179,83 @@ class Conv2DCollapsing(common.BaseSubstitution):
         assert num_edges_before_substition - len(graph.edges) == 1
 
         return graph
+
+
+class Op2DAddConstCollapsing(common.BaseSubstitution):
+    """
+    Collapse Add-const into preceding Op2D (Not non-linear activation between them)
+    """
+    def __init__(self,
+                 first_node: NodeOperationMatcher,
+                 second_node: NodeOperationMatcher,
+                 op2d_collapsing_fn: Callable,
+                 bias_str: str,
+                 use_bias_str: str,
+                 layer_name_str: str = None):
+        """
+        Collapsing Add-const node (2nd node) to Op2D node (first node).
+        Args:
+            first_node: Node matcher for Op2d type nodes.
+            second_node: Node matcher for add type nodes.
+            op2d_collapsing_fn: Function for updating the convolution kernel and bias
+            bias_str: The framework specific attribute name of the convolution layer's bias.
+            use_bias_str: The framework specific attribute name of the convolution layer's bias flag.
+            layer_name_str: The framework specific attribute name of layer's name.
+        """
+        super().__init__(matcher_instance=EdgeMatcher(first_node, second_node))
+        self.op2d_collapsing_fn = op2d_collapsing_fn
+        self.bias_str = bias_str
+        self.use_bias_str = use_bias_str
+        self.layer_name_str = layer_name_str
+
+    def substitute(self,
+                   graph: Graph,
+                   edge_nodes: Tuple[BaseNode, BaseNode]) -> Graph:
+        """
+        Collapse linear layer into preceding linear layers.
+        Convolution condition:
+        |-------------------------|      |------|
+        | Op2D | ---> | Add-const |  ->  | Op2D |
+        |-------------------------|      |------|
+        Args:
+            graph: Graph we apply the substitution on.
+            edge_nodes: Tuple of linear node and add nodes
+        Returns:
+            Graph after applying the substitution.
+        """
+
+        first_node, second_node, _ = edge_nodes
+
+        # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
+        # we should skip the substitution.
+        if first_node.is_reused() or second_node.is_reused():
+            return graph
+
+        # If there is an extra connection between these two nodes skip the substitution
+        if len(graph.get_next_nodes(first_node)) > 1 or len(graph.get_prev_nodes(second_node)) > 1:
+            return graph
+
+        # New collapsed bias
+        bias = self.op2d_collapsing_fn(first_node, second_node, self.bias_str)
+
+        # New collapsed node
+        op2d_collapsed = copy.deepcopy(first_node)
+        op2d_collapsed_name = first_node.name + '_collapsed'
+        op2d_collapsed.name = op2d_collapsed_name
+        op2d_collapsed.framework_attr[self.use_bias_str] = True
+        op2d_collapsed.set_weights_by_keys(self.bias_str, bias)
+
+        if self.layer_name_str is not None:
+            op2d_collapsed.framework_attr[self.layer_name_str] = op2d_collapsed_name
+
+        # Update graph
+        graph.add_node(op2d_collapsed)
+        graph.reconnect_out_edges(current_node=second_node, new_node=op2d_collapsed)
+        graph.reconnect_in_edges(current_node=first_node, new_node=op2d_collapsed)
+        graph.replace_output_node(current_node=second_node, new_node=op2d_collapsed)
+
+        graph.remove_edge(first_node, second_node)
+        graph.remove_node(first_node)
+        graph.remove_node(second_node)
+
+        return graph

model_compression_toolkit/core/common/substitutions/linear_collapsing_substitution.py

@@ -30,6 +30,9 @@ def linear_collapsing_substitute(graph: common.Graph,
     Returns:
         Transformed graph after applying all linear collapsing substitutions.
     """
+    # TODO: remove this if after adding Op2d-add_const collapse substitution in PyTorch
+    if linear_collapsing_substitution is None:
+        return graph
     matched_nodes = graph.filter(linear_collapsing_substitution.matcher_instance)
     matched_nodes_list = []
     match_indicator = True

model_compression_toolkit/core/common/substitutions/residual_collapsing.py

@@ -63,9 +63,7 @@ class ResidualCollapsing(common.BaseSubstitution):
 
         # If the linear operator is part of a reused group (it is the "base" node, or a reused node),
         # we should skip the substitution.
-        if first_node.reuse or first_node.reuse_group is not None:
-            return graph
-        if second_node.reuse or second_node.reuse_group is not None:
+        if first_node.is_reused() or second_node.is_reused():
             return graph
 
         # Check if convolution and residual satisfy the collapsing conditions, otherwise skip substitution

model_compression_toolkit/core/graph_prep_runner.py

@@ -129,6 +129,7 @@ def get_finalized_graph(initial_graph: Graph,
     transformed_graph = substitute(graph, fw_impl.get_substitutions_pre_statistics_collection(quant_config))
     if quant_config.linear_collapsing:
         transformed_graph = linear_collapsing_substitute(transformed_graph, fw_impl.get_linear_collapsing_substitution())
+        transformed_graph = linear_collapsing_substitute(transformed_graph, fw_impl.get_op2d_add_const_collapsing_substitution())
     if quant_config.residual_collapsing:
         transformed_graph = substitute(transformed_graph, fw_impl.get_residual_collapsing_substitution())
 

model_compression_toolkit/core/keras/constants.py

@@ -53,6 +53,8 @@ DIMS = 'dims'
 TARGET_SHAPE = 'target_shape'
 TRANSPOSE_A = 'transpose_a'
 TRANSPOSE_B = 'transpose_b'
+ADJOINT_A = 'adjoint_a'
+ADJOINT_B = 'adjoint_b'
 DEPTH_MULTIPLIER = 'depth_multiplier'
 DEPTHWISE_INITIALIZER = 'depthwise_initializer'
 DEPTHWISE_REGULARIZER = 'depthwise_regularizer'

model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py

@@ -15,10 +15,14 @@
 from typing import Tuple
 import numpy as np
 import tensorflow as tf
-from tensorflow.keras.layers import Conv2D
+if tf.__version__ < "2.6":
+    from tensorflow.keras.layers import Conv2D, DepthwiseConv2D, Conv2DTranspose, Dense
+else:
+    from keras.layers import Conv2D, DepthwiseConv2D, Conv2DTranspose, Dense
+
 from model_compression_toolkit.core.common import BaseNode
 from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher, NodeFrameworkAttrMatcher
-from model_compression_toolkit.core.common.substitutions.linear_collapsing import Conv2DCollapsing
+from model_compression_toolkit.core.common.substitutions.linear_collapsing import Conv2DCollapsing, Op2DAddConstCollapsing
 from model_compression_toolkit.core.keras.constants import KERNEL, KERNEL_SIZE, STRIDES, DILATIONS, LINEAR, \
     ACTIVATION, BIAS, USE_BIAS, LAYER_NAME, FILTERS, PADDING, GROUPS, DATA_FORMAT
 from model_compression_toolkit.logger import Logger
@@ -123,3 +127,69 @@ def keras_linear_collapsing() -> Conv2DCollapsing:
                             FILTERS,
                             data_format_str=DATA_FORMAT,
                             layer_name_str=LAYER_NAME)
+
+
+def op2d_add_const_collapsing_node_matchers() -> Tuple[NodeOperationMatcher, NodeOperationMatcher]:
+    """
+    Function generates matchers for matching:
+    (Op2D, Add(const)) -> Op2D.  (Op2D is one of [DepthwiseConv2D, Conv2D, Conv2DTranspose, Dense)
+    Returns:
+        Matcher for Op2D followed by Add const
+    """
+    first_node = NodeOperationMatcher(DepthwiseConv2D) | \
+                 NodeOperationMatcher(Conv2D) | \
+                 NodeOperationMatcher(Conv2DTranspose) | \
+                 NodeOperationMatcher(Dense)
+    second_node = NodeOperationMatcher(tf.math.add)
+    return first_node, second_node
+
+
+def op2d_add_const_collapsing_fn(op2d_node: BaseNode,
+                                 add_node: BaseNode,
+                                 bias_str: str) -> np.ndarray:
+    """
+    Collapsing Add-Const to previous node's bias
+    Args:
+        op2d_node: Op2d layer node
+        add_node: Add layer to collapse
+        bias_str: The framework specific attribute name of the convolution layer's bias.
+    Returns:
+        The modified conv layer node's bias
+    """
+    bias = op2d_node.get_weights_by_keys(bias_str)
+
+    # read constant from add node
+    if len(add_node.op_call_args) > 0:
+        const = add_node.op_call_args[0]
+    elif 'y' in add_node.op_call_kwargs:
+        const = add_node.op_call_kwargs['y']
+    else:
+        Logger.error(f'Unable to read constant from add node: {add_node.name}')  # pragma: no cover
+
+    # convert constant to numpy array
+    if isinstance(const, tf.Tensor):
+        const = const.numpy()
+    elif isinstance(const, list):
+        const = np.array(const)
+    else:
+        Logger.error(f'Unable to convert constant to numpy array: {add_node.name}')  # pragma: no cover
+
+    # return new bias
+    if bias is None:
+        return const
+    else:
+        return const + bias
+
+
+def keras_op2d_add_const_collapsing() -> Op2DAddConstCollapsing:
+    """
+    Returns:
+        An Op2DCollapsing initialized for Keras models.
+    """
+    first_node, second_node = op2d_add_const_collapsing_node_matchers()
+    return Op2DAddConstCollapsing(first_node,
+                                  second_node,
+                                  op2d_add_const_collapsing_fn,
+                                  BIAS,
+                                  USE_BIAS,
+                                  layer_name_str=LAYER_NAME)

model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py

@@ -0,0 +1,108 @@
+# Copyright 2023 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import numpy as np
+import tensorflow as tf
+from model_compression_toolkit.logger import Logger
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher
+from model_compression_toolkit.core.common.graph.base_node import BaseNode
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.keras.constants import TRANSPOSE_A, TRANSPOSE_B, \
+    ADJOINT_A, ADJOINT_B, UNITS, USE_BIAS, KERNEL
+
+
+class MatmulToDenseSubstitution(common.BaseSubstitution):
+    """
+    Replace a linear layer that has an activation function, with two nodes: same linear layer without
+    an activation function, and a new activation layer to replace the function the linear node had.
+    """
+
+    def __init__(self):
+        """
+        Matches: tf.linalg.matmul
+        """
+        super().__init__(matcher_instance=NodeOperationMatcher(tf.linalg.matmul))
+
+    def substitute(self,
+                   graph: Graph,
+                   matmul_node: FunctionalNode) -> Graph:
+        """
+        Replace tf.linalg.matmul with Tensor and const with Dense layer
+
+        Args:
+            graph: Graph we apply the substitution on.
+            matmul_node: Node to replace.
+
+        Returns:
+            Graph after applying the substitution.
+        """
+
+        if len(graph.get_prev_nodes(matmul_node)) > 1:
+            # matmul of 2 activation tensors -> can't replace with Dense layer
+            return graph
+
+        if matmul_node.framework_attr.get(ADJOINT_A, False) or matmul_node.framework_attr.get(ADJOINT_B, False):
+            # MCT doesn't support complex tensors
+            return graph
+
+        if matmul_node.framework_attr.get(TRANSPOSE_A, False):
+            # first input should be an activation tensor with batch axis, that shouldn't be transposed
+            return graph
+
+        # read const from matmul inputs
+        if len(matmul_node.op_call_args) > 0:
+            w = matmul_node.op_call_args[0]
+        elif 'b' in matmul_node.op_call_kwargs:
+            w = matmul_node.op_call_kwargs['b']
+        else:
+            Logger.error(f"Matmul substitution: can't locate weight for node {matmul_node.name}")  # pragma: no cover
+
+        # Convert weight const to numpy array
+        if isinstance(w, tf.Tensor):
+            w = w.numpy()
+        elif isinstance(w, list):
+            w = np.array(w)
+        elif not isinstance(w, np.ndarray):
+            Logger.error(f'Unable to convert constant to numpy array: {matmul_node.name}')  # pragma: no cover
+
+        if len(w.shape) != 2:
+            # weight tensor should be of shape (Cin, Cout)
+            return graph
+
+        # transpose const if "transpose_b" flag is True
+        if matmul_node.op_call_kwargs.get(TRANSPOSE_B, False) or (
+                len(matmul_node.op_call_args) >= 3 and matmul_node.op_call_args[2]):
+            w = w.transpose()
+
+        dense_node = BaseNode(matmul_node.name,
+                              {UNITS: w.shape[1], USE_BIAS: False},
+                              matmul_node.input_shape, matmul_node.output_shape,
+                              {KERNEL: w}, tf.keras.layers.Dense,
+                              reuse=matmul_node.reuse, reuse_group=matmul_node.reuse_group)
+
+        graph.add_node(dense_node)
+        graph.reconnect_in_edges(current_node=matmul_node,
+                                 new_node=dense_node)
+        graph.reconnect_out_edges(current_node=matmul_node,
+                                  new_node=dense_node)
+        graph.replace_output_node(current_node=matmul_node,
+                                  new_node=dense_node)
+        graph.remove_node(matmul_node)
+
+        return graph
+
+

model_compression_toolkit/core/keras/keras_implementation.py

@@ -68,6 +68,8 @@ from model_compression_toolkit.core.common.user_info import UserInformation
 from model_compression_toolkit.core.keras.default_framework_info import DEFAULT_KERAS_INFO
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.activation_decomposition import \
     ActivationDecomposition
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.matmul_substitution import \
+    MatmulToDenseSubstitution
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.softmax_shift import \
     keras_softmax_shift
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.batchnorm_folding import \
@@ -75,7 +77,7 @@ from model_compression_toolkit.core.keras.graph_substitutions.substitutions.batc
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.batchnorm_refusing import \
     keras_batchnorm_refusing
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.linear_collapsing import \
-    keras_linear_collapsing
+    keras_linear_collapsing, keras_op2d_add_const_collapsing
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.residual_collapsing import \
     keras_residual_collapsing
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.input_scaling import InputScaling, \
@@ -260,6 +262,7 @@ class KerasImplementation(FrameworkImplementation):
 
         """
         return [SeparableConvDecomposition(),
+                MatmulToDenseSubstitution(),
                 MultiHeadAttentionDecomposition(),
                 ActivationDecomposition(),
                 DwconvToConv()]
@@ -311,6 +314,12 @@ class KerasImplementation(FrameworkImplementation):
         """
         return keras_linear_collapsing()
 
+    def get_op2d_add_const_collapsing_substitution(self) -> common.BaseSubstitution:
+        """
+        Returns: Op2d add-const collapsing substitution
+        """
+        return keras_op2d_add_const_collapsing()
+
     def get_substitutions_post_statistics_collection(self, quant_config: QuantizationConfig) \
             -> List[common.BaseSubstitution]:
         """

model_compression_toolkit/core/pytorch/pytorch_implementation.py

@@ -289,6 +289,12 @@ class PytorchImplementation(FrameworkImplementation):
         """
         return pytorch_linear_collapsing()
 
+    def get_op2d_add_const_collapsing_substitution(self) -> common.BaseSubstitution:
+        """
+        Returns: None, as Op2d add-const substitution is not supported in torch yet
+        """
+        return None
+
     def get_substitutions_post_statistics_collection(self,
                                                      quant_config: QuantizationConfig) -> List[common.BaseSubstitution]:
         """