mct-nightly 2.1.0.20240813.141729__tar.gz → 2.1.0.20240815.452__tar.gz

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mct-nightly
-Version: 2.1.0.20240813.141729
+Version: 2.1.0.20240815.452
 Summary: A Model Compression Toolkit for neural networks
 Home-page: UNKNOWN
 License: UNKNOWN

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/mct_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mct-nightly
-Version: 2.1.0.20240813.141729
+Version: 2.1.0.20240815.452
 Summary: A Model Compression Toolkit for neural networks
 Home-page: UNKNOWN
 License: UNKNOWN

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/mct_nightly.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ model_compression_toolkit/core/common/mixed_precision/bit_width_setter.py
 model_compression_toolkit/core/common/mixed_precision/configurable_quant_id.py
 model_compression_toolkit/core/common/mixed_precision/configurable_quantizer_utils.py
 model_compression_toolkit/core/common/mixed_precision/distance_weighting.py
+model_compression_toolkit/core/common/mixed_precision/mixed_precision_candidates_filter.py
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_quantization_config.py
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_search_facade.py
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_search_manager.py
@@ -181,6 +182,7 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_reconstruction.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_refusing.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/concat_threshold_update.py
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/conv_funcs_to_layer.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/dwconv_to_conv.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/input_scaling.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py
@@ -192,6 +194,7 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/residual_
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/scale_equalization.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/separableconv_decomposition.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/shift_negative_activation.py
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/sigmoid_mul_to_swish.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/softmax_shift.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/virtual_activation_weights_composition.py
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/weights_activation_split.py

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/__init__.py

@@ -27,4 +27,4 @@ from model_compression_toolkit import data_generation
 from model_compression_toolkit import pruning
 from model_compression_toolkit.trainable_infrastructure.keras.load_model import keras_load_quantized_model
 
-__version__ = "2.1.0.20240813.141729"
+__version__ = "2.1.0.20240815.000452"

mct-nightly-2.1.0.20240815.452/model_compression_toolkit/core/common/mixed_precision/mixed_precision_candidates_filter.py

@@ -0,0 +1,80 @@
+# Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import numpy as np
+
+from model_compression_toolkit.core import ResourceUtilization, FrameworkInfo
+from model_compression_toolkit.core.common import Graph
+from model_compression_toolkit.logger import Logger
+from model_compression_toolkit.target_platform_capabilities.target_platform import TargetPlatformCapabilities
+
+
+def filter_candidates_for_mixed_precision(graph: Graph,
+                                          target_resource_utilization: ResourceUtilization,
+                                          fw_info: FrameworkInfo,
+                                          tpc: TargetPlatformCapabilities):
+    """
+    Filters out candidates in case of mixed precision search for only weights or activation compression.
+    For instance, if running only weights compression - filters out candidates of activation configurable nodes
+    such that only a single candidate would remain, with the bitwidth equal to the one defined in the matching layer's
+    base config in the TPC.
+
+    Note" This function modifies the graph inplace!
+
+    Args:
+        graph: A graph representation of the model to be quantized.
+        target_resource_utilization: The resource utilization of the target device.
+        fw_info: fw_info: Information needed for quantization about the specific framework.
+        tpc: TargetPlatformCapabilities object that describes the desired inference target platform.
+
+    """
+
+    no_total_restrictions = (target_resource_utilization.total_memory == np.inf and
+                             target_resource_utilization.bops == np.inf)
+
+    if target_resource_utilization.weights_memory < np.inf:
+        if target_resource_utilization.activation_memory == np.inf and no_total_restrictions:
+            # Running mixed precision for weights compression only -
+            # filter out candidates activation only configurable node
+            weights_conf = graph.get_weights_configurable_nodes(fw_info)
+            for n in graph.get_activation_configurable_nodes():
+                if n not in weights_conf:
+                    base_cfg_nbits = n.get_qco(tpc).base_config.activation_n_bits
+                    filtered_conf = [c for c in n.candidates_quantization_cfg if
+                                     c.activation_quantization_cfg.enable_activation_quantization and
+                                     c.activation_quantization_cfg.activation_n_bits == base_cfg_nbits]
+
+                    if len(filtered_conf) != 1:
+                        Logger.critical(f"Running weights only mixed precision failed on layer {n.name} with multiple "
+                                        f"activation quantization configurations.")  # pragma: no cover
+                    n.candidates_quantization_cfg = filtered_conf
+
+    elif target_resource_utilization.activation_memory < np.inf:
+        if target_resource_utilization.weights_memory == np.inf and no_total_restrictions:
+            # Running mixed precision for activation compression only -
+            # filter out candidates weights only configurable node
+            activation_conf = graph.get_activation_configurable_nodes()
+            for n in graph.get_weights_configurable_nodes(fw_info):
+                if n not in activation_conf:
+                    kernel_attr = graph.fw_info.get_kernel_op_attributes(n.type)[0]
+                    base_cfg_nbits = n.get_qco(tpc).base_config.attr_weights_configs_mapping[kernel_attr].weights_n_bits
+                    filtered_conf = [c for c in n.candidates_quantization_cfg if
+                                     c.weights_quantization_cfg.get_attr_config(
+                                         kernel_attr).enable_weights_quantization and
+                                     c.weights_quantization_cfg.get_attr_config(
+                                         kernel_attr).weights_n_bits == base_cfg_nbits]
+                    if len(filtered_conf) != 1:
+                        Logger.critical(f"Running activation only mixed precision failed on layer {n.name} with multiple "
+                                        f"weights quantization configurations.")  # pragma: no cover
+                    n.candidates_quantization_cfg = filtered_conf

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/keras/constants.py

@@ -31,7 +31,8 @@ KERNEL_SIZE = 'kernel_size'
 PADDING = 'padding'
 GROUPS = 'groups'
 STRIDES = 'strides'
-DILATIONS = 'dilation_rate'
+DILATION_RATE = 'dilation_rate'
+DILATIONS = 'dilations'
 DATA_FORMAT = 'data_format'
 LAYER_NAME = 'name'
 TRAINABLE = 'trainable'
@@ -62,6 +63,7 @@ DEPTHWISE_CONSTRAINT = 'depthwise_constraint'
 KERNEL_INITIALIZER = 'kernel_initializer'
 KERNEL_REGULARIZER = 'kernel_regularizer'
 KERNEL_CONSTRAINT = 'kernel_constraint'
+RATE = 'rate'
 
 # functional nodes attributes
 FUNCTION = 'function'
@@ -71,6 +73,7 @@ F_MATMUL = 'matmul'
 F_STACK = 'stack'
 F_STRIDED_SLICE_BEGIN = 'begin_mask'
 F_STRIDED_SLICE_END = 'end_mask'
+F_SWISH = 'nn.silu'
 
 # Layers variables names:
 KERNEL: str = 'kernel'

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/keras/default_framework_info.py

@@ -29,7 +29,7 @@ from model_compression_toolkit.core.common.framework_info import FrameworkInfo
 from model_compression_toolkit.target_platform_capabilities.target_platform import QuantizationMethod
 from model_compression_toolkit.constants import SOFTMAX_THRESHOLD
 from model_compression_toolkit.core.keras.constants import SOFTMAX, LINEAR, RELU, SWISH, SIGMOID, IDENTITY, TANH, SELU, \
-    KERNEL, DEPTHWISE_KERNEL
+    KERNEL, DEPTHWISE_KERNEL, GELU
 from model_compression_toolkit.core.keras.quantizer.fake_quant_builder import power_of_two_quantization, symmetric_quantization, uniform_quantization
 
 """
@@ -75,7 +75,8 @@ ACTIVATION2MINMAX = {SOFTMAX: (0, SOFTMAX_THRESHOLD),
                      TANH: (-1, 1),
                      SWISH: (-0.279, None),
                      RELU: (0, None),
-                     SELU: (None, None),
+                     SELU: (-1.76, None),
+                     GELU: (-0.17, None),
                      }
 
 """

mct-nightly-2.1.0.20240815.452/model_compression_toolkit/core/keras/graph_substitutions/substitutions/conv_funcs_to_layer.py

@@ -0,0 +1,241 @@
+# Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import numpy as np
+import tensorflow as tf
+from packaging import version
+if version.parse(tf.__version__) >= version.parse("2.13"):
+    from keras.src.layers.core import TFOpLambda
+    from keras.src.layers import Conv2D, DepthwiseConv2D
+else:
+    from keras.layers.core import TFOpLambda
+    from keras.layers import Conv2D, DepthwiseConv2D
+from model_compression_toolkit.logger import Logger
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph, BaseNode, OutTensor
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher
+from model_compression_toolkit.constants import REUSE, REUSE_GROUP
+from model_compression_toolkit.core.keras.constants import KERNEL, BIAS, USE_BIAS, FILTERS, PADDING, \
+    KERNEL_SIZE, DEPTH_MULTIPLIER, STRIDES, DILATIONS, DILATION_RATE, DEPTHWISE_KERNEL, RATE
+
+
+def extract_bias_node_data(_node: FunctionalNode, _graph: Graph) -> np.ndarray:
+    """
+    Check is can extract bias from next node.
+
+    Args:
+        _node: conv node to check for subsequent add\bias_add node to extract bias from.
+        _graph: model graph.
+
+    Returns:
+        The bias weight. None if couldn't extract bias.
+
+    """
+    b = None
+    next_nodes = _graph.get_next_nodes(_node)
+    if len(next_nodes) == 1 and len(_graph.get_prev_nodes(next_nodes[0])) == 1:
+        # Found pattern in graph: conv_node->next_node. Check if next node is add\bias_add that can be absorbed as bias.
+        if next_nodes[0].is_match_type(tf.add):
+            b = next_nodes[0].weights.get(0, next_nodes[0].weights.get(1))
+            if b is not None and len(b.shape) != 1:
+                # Constant input to Add node (bias) has irregular shape. Expecting a 1-D array.
+                b = None  # pragma: no cover
+        elif next_nodes[0].is_match_type(tf.nn.bias_add):
+            # In bias_add, weight is always 1-D array. Extract weight from weights or kwargs.
+            if 1 in next_nodes[0].weights:
+                b = next_nodes[0].weights[1]
+            elif BIAS in _node.op_call_kwargs:
+                b = np.array(_node.op_call_kwargs[BIAS], dtype=np.float32)
+
+    return b
+
+
+def replace_conv_node(graph: Graph, new_node: BaseNode, old_node: FunctionalNode, remove_add_node: bool):
+    """
+    Replace in-place a functional conv node (and possibly subsequent add node) with Conv layer.
+    Args:
+        graph: model Graph.
+        new_node: Conv layer node.
+        old_node: conv function node.
+        remove_add_node: whether to remove subsequent add node or not.
+    """
+    graph.add_node(new_node)
+
+    # Replace functional conv node (and potentially add node) with Conv node.
+    graph.reconnect_in_edges(old_node, new_node)
+    if remove_add_node:
+        next_nodes = graph.get_next_nodes(old_node)
+        graph.reconnect_out_edges(next_nodes[0], new_node)
+        graph.replace_output_node(current_node=next_nodes[0], new_node=new_node)
+        graph.remove_edge(old_node, next_nodes[0])
+        graph.remove_node(next_nodes[0])
+    else:
+        graph.reconnect_out_edges(old_node, new_node)
+        graph.replace_output_node(current_node=old_node, new_node=new_node)
+    graph.remove_node(old_node)
+
+
+class Conv2dFuncToConv2dLayer(common.BaseSubstitution):
+    """
+    Substitutes tf.nn.conv2d, tf.compat.v1.nn.conv2d, tf.nn.convolution, tf.compat.v1.nn.convolution functions with a Conv2D layer.
+    """
+
+    def __init__(self):
+        """
+        Initializes the Conv2dFuncToConv2dLayer substitution matcher instance.
+        """
+        conv2d_matcher = NodeOperationMatcher(tf.nn.conv2d) | NodeOperationMatcher(tf.compat.v1.nn.conv2d)
+        convolution_matcher = NodeOperationMatcher(tf.nn.convolution) | NodeOperationMatcher(tf.compat.v1.nn.convolution)
+        super().__init__(matcher_instance=conv2d_matcher | convolution_matcher)
+
+    def substitute(self,
+                   graph: Graph,
+                   conv_func_node: FunctionalNode) -> Graph:
+        """
+        Substitutes conv functions with a Conv2D layer.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            conv_func_node: The functional node to be replaced.
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        if 1 in conv_func_node.weights:
+            k = conv_func_node.weights[1]
+        elif FILTERS in conv_func_node.op_call_kwargs:
+            k = np.array(conv_func_node.op_call_kwargs[FILTERS], dtype=np.float32)
+        else:
+            # Conv weight isn't a constant -> skip substitution.
+            return graph  # pragma: no cover
+
+        if len(k.shape) != 4:
+            # Conv dimension doesn't match conv2d dimension (K1 x K2 x Cin x Cout) -> skip substitution.
+            return graph  # pragma: no cover
+
+        # Check if can extract bias from next node.
+        b = extract_bias_node_data(conv_func_node, graph)
+
+        weights = {KERNEL: k}
+        # Create Conv2D layer attributes.
+        conv_fw_attr = {FILTERS: k.shape[3], KERNEL_SIZE: k.shape[:2]}
+        if len(conv_func_node.op_call_args) > 0:
+            Logger.critical(f"node {conv_func_node.name} expected to have only kwargs but got args={conv_func_node.op_call_args}.")  # pragma: no cover
+        if STRIDES in conv_func_node.op_call_kwargs:
+            strides = conv_func_node.op_call_kwargs[STRIDES]
+            if len(strides) == 4:
+                if strides[0] > 1 or strides[3] > 1:
+                    # Non-standard strides -> skip substitution.
+                    return graph  # pragma: no cover
+                conv_fw_attr[STRIDES] = strides[1:3]
+            else:
+                conv_fw_attr[STRIDES] = strides
+        if PADDING in conv_func_node.op_call_kwargs:
+            padding = conv_func_node.op_call_kwargs[PADDING]
+            if not isinstance(padding, str):
+                # Non-standard padding, Layer only support either 'valid' or 'same' -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[PADDING] = padding
+        if DILATIONS in conv_func_node.op_call_kwargs and conv_func_node.op_call_kwargs[DILATIONS] is not None:
+            dilations = conv_func_node.op_call_kwargs[DILATIONS]
+            if isinstance(dilations, (list, tuple)) and len(dilations) == 4:
+                if dilations[0] > 1 or dilations[3] > 1:
+                    # Non-standard dilations -> skip substitution.
+                    return graph  # pragma: no cover
+                conv_fw_attr[DILATION_RATE] = dilations[1:3]
+            else:
+                conv_fw_attr[DILATION_RATE] = dilations
+        if b is None:
+            conv_fw_attr[USE_BIAS] = False
+        else:
+            weights[BIAS] = b
+
+        _reuse_params = {REUSE: conv_func_node.reuse, REUSE_GROUP: conv_func_node.reuse_group}
+        conv_node = BaseNode(conv_func_node.name, conv_fw_attr, conv_func_node.input_shape, conv_func_node.output_shape,
+                             weights, Conv2D, **_reuse_params)
+
+        replace_conv_node(graph, conv_node, conv_func_node, remove_add_node=b is not None)
+        return graph
+
+
+class DwConv2dFuncToDwConv2dLayer(common.BaseSubstitution):
+    """
+    Substitutes tf.nn.depthwise_conv2d & tf.compat.v1.nn.depthwise_conv2d functions with a DepthwiseConv2D layer.
+    """
+
+    def __init__(self):
+        """
+        Initializes the DwConv2dFuncToDwConv2dLayer substitution matcher.
+        """
+        matcher = NodeOperationMatcher(tf.nn.depthwise_conv2d) | NodeOperationMatcher(tf.compat.v1.nn.depthwise_conv2d)
+        super().__init__(matcher_instance=matcher)
+
+    def substitute(self,
+                   graph: Graph,
+                   dwconv_func_node: FunctionalNode) -> Graph:
+        """
+        Substitutes dw-conv2d functions with a DepthwiseConv2D layer.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            dwconv_func_node: The DepthwiseConv2D node to be replaced.
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        if 1 not in dwconv_func_node.weights:
+            # Conv weight isn't a constant -> skip substitution.
+            return graph  # pragma: no cover
+
+        k = dwconv_func_node.weights[1]
+
+        # Check is can extract bias from next node.
+        b = extract_bias_node_data(dwconv_func_node, graph)
+
+        weights = {DEPTHWISE_KERNEL: k}
+        k_shape = k.shape
+        conv_fw_attr = {DEPTH_MULTIPLIER: k_shape[3], KERNEL_SIZE: k_shape[:2]}
+        if len(dwconv_func_node.op_call_args) > 0:
+            Logger.critical(f"node {dwconv_func_node.name} expected to have only kwargs but got args={dwconv_func_node.op_call_args}.")  # pragma: no cover
+        if STRIDES in dwconv_func_node.op_call_kwargs:
+            strides = dwconv_func_node.op_call_kwargs[STRIDES]
+            if strides[0] > 1 or strides[3] > 1:
+                # Non-standard strides -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[STRIDES] = strides[1:3]
+        if PADDING in dwconv_func_node.op_call_kwargs:
+            padding = dwconv_func_node.op_call_kwargs[PADDING]
+            if not isinstance(padding, str):
+                # Non-standard padding, Layer only support either 'valid' or 'same' -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[PADDING] = padding
+        if RATE in dwconv_func_node.op_call_kwargs and dwconv_func_node.op_call_kwargs[RATE] is not None:
+            conv_fw_attr[DILATION_RATE] = dwconv_func_node.op_call_kwargs[RATE]
+        elif DILATIONS in dwconv_func_node.op_call_kwargs and dwconv_func_node.op_call_kwargs[DILATIONS] is not None:
+            conv_fw_attr[DILATION_RATE] = dwconv_func_node.op_call_kwargs[DILATIONS]
+        if b is None:
+            conv_fw_attr[USE_BIAS] = False
+        else:
+            weights[BIAS] = b
+
+        _reuse_params = {REUSE: dwconv_func_node.reuse, REUSE_GROUP: dwconv_func_node.reuse_group}
+        conv_node = BaseNode(dwconv_func_node.name, conv_fw_attr, dwconv_func_node.input_shape, dwconv_func_node.output_shape,
+                             weights, DepthwiseConv2D, **_reuse_params)
+
+        replace_conv_node(graph, conv_node, dwconv_func_node, remove_add_node=b is not None)
+        return graph

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py

@@ -23,7 +23,7 @@ else:
 from model_compression_toolkit.core.common import BaseNode
 from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher, NodeFrameworkAttrMatcher
 from model_compression_toolkit.core.common.substitutions.linear_collapsing import Conv2DCollapsing, Op2DAddConstCollapsing
-from model_compression_toolkit.core.keras.constants import KERNEL, KERNEL_SIZE, STRIDES, DILATIONS, LINEAR, \
+from model_compression_toolkit.core.keras.constants import KERNEL, KERNEL_SIZE, STRIDES, DILATION_RATE, LINEAR, \
     ACTIVATION, BIAS, USE_BIAS, LAYER_NAME, FILTERS, PADDING, GROUPS, DATA_FORMAT
 from model_compression_toolkit.logger import Logger
 
@@ -122,7 +122,7 @@ def keras_linear_collapsing() -> Conv2DCollapsing:
                             USE_BIAS,
                             STRIDES,
                             PADDING,
-                            DILATIONS,
+                            DILATION_RATE,
                             GROUPS,
                             FILTERS,
                             data_format_str=DATA_FORMAT,

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py

@@ -65,6 +65,8 @@ class MatmulToDenseSubstitution(common.BaseSubstitution):
 
         # read const from matmul inputs
         w = matmul_node.weights.get(1)
+        if w is None:
+            w = np.array(matmul_node.op_call_kwargs['b'], dtype=np.float32) if 'b' in matmul_node.op_call_kwargs else None
         if w is None:
             Logger.critical(f"Matmul substitution failed: Unable to locate weight for node {matmul_node.name}.")  # pragma: no cover
 

mct-nightly-2.1.0.20240815.452/model_compression_toolkit/core/keras/graph_substitutions/substitutions/sigmoid_mul_to_swish.py

@@ -0,0 +1,89 @@
+# Copyright 2024 Sony Semiconductors Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from typing import Tuple, Union
+import numpy as np
+import tensorflow as tf
+from packaging import version
+if version.parse(tf.__version__) >= version.parse("2.13"):
+    from keras.src.layers.core import TFOpLambda
+    from keras.src.layers import Multiply, Activation
+else:
+    from keras.layers.core import TFOpLambda
+    from keras.layers import Multiply, Activation
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph, BaseNode, OutTensor
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher, \
+    EdgeMatcher, NodeFrameworkAttrMatcher
+from model_compression_toolkit.constants import REUSE, REUSE_GROUP
+from model_compression_toolkit.core.keras.constants import FUNCTION, F_SWISH, ACTIVATION, SIGMOID
+
+
+class MulSigmoidToSwish(common.BaseSubstitution):
+    """
+    Substitutes mul(x, sigmoid(x)) with swish.
+    """
+
+    def __init__(self):
+        """
+        Initializes the MulSigmoidToSwish substitution matcher instance.
+        """
+        mul_matcher = NodeOperationMatcher(tf.math.multiply) | NodeOperationMatcher(Multiply)
+        activation_sigmoid = NodeOperationMatcher(Activation) & NodeFrameworkAttrMatcher(ACTIVATION, SIGMOID)
+        sigmoid_matcher = NodeOperationMatcher(tf.sigmoid) | activation_sigmoid
+        super().__init__(matcher_instance=EdgeMatcher(sigmoid_matcher, mul_matcher))
+
+    def substitute(self,
+                   graph: Graph,
+                   sigmoid_mul_edge: Tuple[FunctionalNode, Union[FunctionalNode, BaseNode], int]) -> Graph:
+        """
+        Substitutes mul(x, sigmoid(x)) with swish.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            sigmoid_mul_edge: edge between sigmoid and multiply nodes
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        sigmoid_node, mul_node, _ = sigmoid_mul_edge
+        if sigmoid_node in [o.node for o in graph.output_nodes]:
+            # Sigmoid node in outputs -> Skip substitution.
+            return graph
+
+        input_node = graph.get_prev_nodes(sigmoid_node)[0]
+        if len(graph.get_next_nodes(sigmoid_node)) > 1 or input_node not in graph.get_prev_nodes(mul_node):
+            # Structure isn't mul(x, sigmoid(x)) -> Skip substitution.
+            return graph
+        _reuse_params = {REUSE: mul_node.reuse, REUSE_GROUP: mul_node.reuse_group}
+        swish_node = FunctionalNode(f'swish__{sigmoid_node.name}_{mul_node.name}', {FUNCTION: F_SWISH},
+                                    sigmoid_node.input_shape, mul_node.output_shape, {}, TFOpLambda,
+                                    op_call_args=[], op_call_kwargs={}, functional_op=tf.nn.silu, **_reuse_params)
+
+        graph.add_node(swish_node)
+
+        # Replace functional conv node (and potentially add node) with Conv node.
+        graph.reconnect_in_edges(sigmoid_node, swish_node)
+        graph.reconnect_out_edges(mul_node, swish_node)
+        graph.replace_output_node(current_node=mul_node, new_node=swish_node)
+        graph.remove_edge(input_node, mul_node)
+        graph.remove_edge(sigmoid_node, mul_node)
+        graph.remove_node(sigmoid_node)
+        graph.remove_node(mul_node)
+
+        return graph
+

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/keras/keras_implementation.py

@@ -69,6 +69,9 @@ from model_compression_toolkit.core.keras.graph_substitutions.substitutions.acti
     ActivationDecomposition
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.matmul_substitution import \
     MatmulToDenseSubstitution
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.sigmoid_mul_to_swish import MulSigmoidToSwish
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.conv_funcs_to_layer import \
+    Conv2dFuncToConv2dLayer, DwConv2dFuncToDwConv2dLayer
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.softmax_shift import \
     keras_softmax_shift
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.batchnorm_folding import \
@@ -242,8 +245,11 @@ class KerasImplementation(FrameworkImplementation):
         Returns: A list of the framework substitutions used to prepare the graph.
 
         """
-        return [SeparableConvDecomposition(),
+        return [MulSigmoidToSwish(),
+                SeparableConvDecomposition(),
                 MatmulToDenseSubstitution(),
+                Conv2dFuncToConv2dLayer(),
+                DwConv2dFuncToDwConv2dLayer(),
                 MultiHeadAttentionDecomposition(),
                 ActivationDecomposition(),
                 DwconvToConv(),

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/pytorch/default_framework_info.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-from torch.nn import Hardsigmoid, ReLU, ReLU6, Softmax, Sigmoid
-from torch.nn.functional import hardsigmoid, relu, relu6, softmax
+from torch.nn import Hardsigmoid, ReLU, ReLU6, Softmax, Sigmoid, GELU, SELU
+from torch.nn.functional import hardsigmoid, relu, relu6, softmax, gelu, selu
 from torch.nn import Conv2d, ConvTranspose2d, Linear
 from torch import sigmoid
 
@@ -74,7 +74,12 @@ LAYER2MINMAX = {Softmax: (0, SOFTMAX_THRESHOLD),
                 ReLU: (0, None),
                 relu: (0, None),
                 ReLU6: (0, None),
-                relu6: (0, None)}
+                relu6: (0, None),
+                GELU: (-0.17, None),
+                gelu: (-0.17, None),
+                SELU: (-1.76, None),
+                selu: (-1.76, None),
+                }
 
 """
 Mapping from a QuantizationMethod to an activation quantizer function.

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/shift_negative_activation.py

@@ -17,9 +17,9 @@ from typing import Tuple, Any, Callable
 
 import numpy as np
 import torch.nn.functional
-from torch.nn import Conv2d, Linear, PReLU, ELU, Hardswish, Dropout, ZeroPad2d, SiLU
+from torch.nn import Conv2d, Linear, PReLU, ELU, Hardswish, Dropout, ZeroPad2d, SiLU, GELU
 from torch import reshape
-from torch.nn.functional import hardswish, silu, prelu, elu
+from torch.nn.functional import hardswish, silu, prelu, elu, gelu
 from torch.nn.functional import avg_pool2d
 
 from model_compression_toolkit.core import CoreConfig, FrameworkInfo
@@ -68,7 +68,9 @@ def shift_negative_activation_node_matchers():
                NodeOperationMatcher(Hardswish) | \
                NodeOperationMatcher(hardswish) | \
                NodeOperationMatcher(SiLU) | \
-               NodeOperationMatcher(silu)
+               NodeOperationMatcher(silu) | \
+               NodeOperationMatcher(GELU) | \
+               NodeOperationMatcher(gelu)
 
     # Match linear layers where we can add a correction.
     linear_node = NodeOperationMatcher(Conv2d) | \

{mct-nightly-2.1.0.20240813.141729 → mct-nightly-2.1.0.20240815.452}/model_compression_toolkit/core/runner.py

@@ -27,6 +27,8 @@ from model_compression_toolkit.core.common.graph.memory_graph.compute_graph_max_
     SchedulerInfo
 from model_compression_toolkit.core.common.graph.memory_graph.memory_graph import MemoryGraph
 from model_compression_toolkit.core.common.hessian.hessian_info_service import HessianInfoService
+from model_compression_toolkit.core.common.mixed_precision.mixed_precision_candidates_filter import \
+    filter_candidates_for_mixed_precision
 from model_compression_toolkit.core.common.mixed_precision.resource_utilization_tools.resource_utilization_data import \
     requires_mixed_precision
 from model_compression_toolkit.core.graph_prep_runner import graph_preparation_runner
@@ -137,6 +139,7 @@ def core_runner(in_model: Any,
     if core_config.mixed_precision_enable:
         if core_config.mixed_precision_config.configuration_overwrite is None:
 
+            filter_candidates_for_mixed_precision(graph, target_resource_utilization, fw_info, tpc)
             bit_widths_config = search_bit_width(tg,
                                                  fw_info,
                                                  fw_impl,