mct-nightly 2.1.0.20240813.141729__py3-none-any.whl → 2.1.0.20240815.452__py3-none-any.whl

{mct_nightly-2.1.0.20240813.141729.dist-info → mct_nightly-2.1.0.20240815.452.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mct-nightly
-Version: 2.1.0.20240813.141729
+Version: 2.1.0.20240815.452
 Summary: A Model Compression Toolkit for neural networks
 Home-page: UNKNOWN
 License: UNKNOWN

{mct_nightly-2.1.0.20240813.141729.dist-info → mct_nightly-2.1.0.20240815.452.dist-info}/RECORD

@@ -1,4 +1,4 @@
-model_compression_toolkit/__init__.py,sha256=4Ah8Ywj3HTi7gOBoXzHjZ7FBnpJqziSb4iuxGWzX9R8,1573
+model_compression_toolkit/__init__.py,sha256=R0Zwbt0JpEgVMFa8F2SnrHQ0xhwmPSq0tvWkS53l3eI,1573
 model_compression_toolkit/constants.py,sha256=i4wYheBkIdQmsQA-axIpcT3YiSO1USNc-jaNiNE8w6E,3920
 model_compression_toolkit/defaultdict.py,sha256=LSc-sbZYXENMCw3U9F4GiXuv67IKpdn0Qm7Fr11jy-4,2277
 model_compression_toolkit/logger.py,sha256=3DByV41XHRR3kLTJNbpaMmikL8icd9e1N-nkQAY9oDk,4567
@@ -8,7 +8,7 @@ model_compression_toolkit/core/__init__.py,sha256=tnDtL9KmT0vsOU27SsJ19TKDEbIH-t
 model_compression_toolkit/core/analyzer.py,sha256=X-2ZpkH1xdXnISnw1yJvXnvV-ssoUh-9LkLISSWNqiY,3691
 model_compression_toolkit/core/graph_prep_runner.py,sha256=7-b7Jd5jBVaXOWg5nSqbEyzBtdaGDbCxs8aqMV6GZ6I,11287
 model_compression_toolkit/core/quantization_prep_runner.py,sha256=K9eJ7VbB_rpeyxX4yEnorOmSxFW3DkvofzxS6QI8Hp8,6454
-model_compression_toolkit/core/runner.py,sha256=XQDNJirZkVJ_FXP72d7tbVc_Tr3Jw0Eqm_kxNHW8kPs,13636
+model_compression_toolkit/core/runner.py,sha256=kiNClmonlaqNI2U72bzGoJUzLxKHLh61iak9-HvsfQM,13880
 model_compression_toolkit/core/common/__init__.py,sha256=Wh127PbXcETZX_d1PQqZ71ETK3J9XO5A-HpadGUbj6o,1447
 model_compression_toolkit/core/common/base_substitutions.py,sha256=xDFSmVVs_iFSZfajytI0cuQaNRNcwHX3uqOoHgVUvxQ,1666
 model_compression_toolkit/core/common/framework_implementation.py,sha256=kSg2f7wS7e2EyvX6y0eKfNTTFvVFVrB8lvldJvcPvN8,20724
@@ -63,6 +63,7 @@ model_compression_toolkit/core/common/mixed_precision/bit_width_setter.py,sha256
 model_compression_toolkit/core/common/mixed_precision/configurable_quant_id.py,sha256=LLDguK7afsbN742ucLpmJr5TUfTyFpK1vbf2bpVr1v0,882
 model_compression_toolkit/core/common/mixed_precision/configurable_quantizer_utils.py,sha256=7dKMi5S0zQZ16m8NWn1XIuoXsKuZUg64G4-uK8-j1PQ,5177
 model_compression_toolkit/core/common/mixed_precision/distance_weighting.py,sha256=H8qYkJsk88OszUJo-Zde7vTmWiypLTg9KbbzIZ-hhvM,2812
+model_compression_toolkit/core/common/mixed_precision/mixed_precision_candidates_filter.py,sha256=klmaMQDeFc3IxRLf6YX4Dw1opFksbLyN10yFHdKAtLo,4875
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_quantization_config.py,sha256=rppRZJdSCQGiZsd93QxoUIhj51eETvQbuI5JiC2TUeA,4963
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_search_facade.py,sha256=pk8HRoShDhiUprBC4m1AFQv1SacS4hOrj0MRdbq-5gY,7556
 model_compression_toolkit/core/common/mixed_precision/mixed_precision_search_manager.py,sha256=TTTux4YiOnQqt-2h7Y38959XaDwNZc0eufLMx_yws5U,37578
@@ -150,10 +151,10 @@ model_compression_toolkit/core/common/visualization/final_config_visualizer.py,s
 model_compression_toolkit/core/common/visualization/nn_visualizer.py,sha256=HOq7AObkmEZiDSZXUMJDAEJzUY-fSXUT0AMgwiyH7dg,7388
 model_compression_toolkit/core/common/visualization/tensorboard_writer.py,sha256=1-OQu3RNKXA55qfKG1MPq4JxTzmFeVKFDWv5i3TktRw,23676
 model_compression_toolkit/core/keras/__init__.py,sha256=mjbqLD-KcG3eNeCYpu1GBS7VclGVOQ63x2p6mAAuba4,698
-model_compression_toolkit/core/keras/constants.py,sha256=Uv3c0UdW55pIVQNW_1HQlgl-dHXREkltOLyzp8G1mTQ,3163
+model_compression_toolkit/core/keras/constants.py,sha256=dh4elQWt6Q6NYRht5k5RiiOcnLAq1v0MMBCJqMJzzFk,3225
 model_compression_toolkit/core/keras/custom_layer_validation.py,sha256=f-b14wuiIgitBe7d0MmofYhDCTO3IhwJgwrh-Hq_t_U,1192
-model_compression_toolkit/core/keras/default_framework_info.py,sha256=HcHplb7IcnOTyK2p6uhp3OVG4-RV3RDo9C_4evaIzkQ,4981
-model_compression_toolkit/core/keras/keras_implementation.py,sha256=hzNC6wz1gtL2EqmRCMCQYl8AqIDJPu6rdOX6nvPgjCM,30193
+model_compression_toolkit/core/keras/default_framework_info.py,sha256=PYcER89eEXjKtR0T7-2Y4f7cckqoD5OQbpHePoRkMec,5030
+model_compression_toolkit/core/keras/keras_implementation.py,sha256=uOTGpsgH4h9MBduVBp8v7mm2S8njbkC72qvXcrZUjeI,30604
 model_compression_toolkit/core/keras/keras_model_validation.py,sha256=1wNV2clFdC9BzIELRLSO2uKf0xqjLqlkTJudwtCeaJk,1722
 model_compression_toolkit/core/keras/keras_node_prior_info.py,sha256=HUmzEXDQ8LGX7uOYSRiLZ2TNbYxLX9J9IeAa6QYlifg,3927
 model_compression_toolkit/core/keras/resource_utilization_data_facade.py,sha256=s56UIgiPipUQRNd2sd1xW6GFfYNMBmrocRCNtvpYLbY,4977
@@ -172,10 +173,11 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_reconstruction.py,sha256=GR1a3mCZpNUu4WxixJXF_aSm57phAdxaRoHecNx3hxw,3168
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_refusing.py,sha256=5df_xGfXkqNub4xVRnCWQvSohWqdv12axjJ6edVU2H0,2478
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/concat_threshold_update.py,sha256=Hl4LEQ_bw_Vpmf3ZqHujYUqVdvTNsPlEMvr9dZhwg2U,2806
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/conv_funcs_to_layer.py,sha256=K2svZ8xKK6LAnV86556AwIKnvIjcEqXjJicjp7KC-zY,11132
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/dwconv_to_conv.py,sha256=R3U7cjc2E0zheMem16GHygp5jZFGSaomkNOTxTjcAgw,5794
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/input_scaling.py,sha256=V6hp67CkS_A3WqdsjLjs0ETtdZAOo4P9mhy4aT7W5FE,5940
-model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py,sha256=i5kdo6-GJe5j4ZVoBp9irLLqqS_H24izrUvda17laf0,8177
-model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py,sha256=kjwlKtm5yhNgWVVcW6mN-hn7enwAnn_8-TUZvxZBiQs,4112
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py,sha256=AvquvVVVT8-ioeVn-gjqysK4L41L3I7TlNOEDfWjViY,8185
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py,sha256=9MZJp4GNTLesWN5uQ5eOQyAHLzLYDAHAjRi-LpNppSc,4257
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/multi_head_attention_decomposition.py,sha256=l9PUREBf4aRwWILiybdteveeUbh7js-i-hLt8Ma0e4c,26771
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/relu_bound_to_power_of_2.py,sha256=IdKOg6AWZWMcmDbOuNdxetS5_zTarXIIffdYL7JTdvk,3872
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/remove_identity.py,sha256=z2J2Xk7b_w_fEgJmK87lwwBmEoAZpGxPmsBrR24IkZs,2035
@@ -183,6 +185,7 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/residual_
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/scale_equalization.py,sha256=ryes9y1ie-vjBGso2TeO4EXxVk69Ew3iSAhshPz1Ou4,5542
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/separableconv_decomposition.py,sha256=TEaHlIbXj_ZjIdT5TmAICD3WLD3u_7g0fLWQcNzTJuM,7941
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/shift_negative_activation.py,sha256=13ejpU2z7c5O2w0Iy_uz3HaBbXVYrsQpEqt0nKErVvg,11169
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/sigmoid_mul_to_swish.py,sha256=4Yf-sIj6oqYENdXs2FRxbvLCI1siDo29XpGb17mISBw,4062
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/softmax_shift.py,sha256=Qk5seDALj_th9dHJehY7ynZjvFjVfCv_mJ1enA5hX0c,1623
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/virtual_activation_weights_composition.py,sha256=wH9ocMLL725-uUPU-zCxdd8NwT5nyd0ZShmI7iuTwF8,1462
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/weights_activation_split.py,sha256=rjIheZW7LbSPv9bzMSmC8wl6UUxaTkd4J2IHinObT-Y,1814
@@ -214,7 +217,7 @@ model_compression_toolkit/core/keras/statistics_correction/apply_second_moment_c
 model_compression_toolkit/core/keras/visualization/__init__.py,sha256=mjbqLD-KcG3eNeCYpu1GBS7VclGVOQ63x2p6mAAuba4,698
 model_compression_toolkit/core/pytorch/__init__.py,sha256=Rf1RcYmelmdZmBV5qOKvKWF575ofc06JFQSq83Jz99A,696
 model_compression_toolkit/core/pytorch/constants.py,sha256=YwD_joIF0vK8UG2vW1NVvg36pCNWA0vHOXjAgy_XWn0,2794
-model_compression_toolkit/core/pytorch/default_framework_info.py,sha256=r1XyzUFvrjGcJHQM5ETLsMZIG2yHCr9HMjqf0ti9inw,4175
+model_compression_toolkit/core/pytorch/default_framework_info.py,sha256=-Vls1P_8Ckm_18nnOsmQkZ71SmzHwtQLbQ383Z4Rb-U,4365
 model_compression_toolkit/core/pytorch/pytorch_device_config.py,sha256=S25cuw10AW3SEN_fRAGRcG_I3wdvvQx1ehSJzPnn-UI,4404
 model_compression_toolkit/core/pytorch/pytorch_implementation.py,sha256=xmcJyU-rkIDX1a_X9LILzf2Ko2z_4I4xnlHkezKH-2w,27669
 model_compression_toolkit/core/pytorch/pytorch_node_prior_info.py,sha256=2LDQ7qupglHQ7o1Am7LWdfYVacfQnl-aW2N6l9det1w,3264
@@ -246,7 +249,7 @@ model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/remove_
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/reshape_with_static_shapes.py,sha256=hAZXzrEinHa-dJHLj39Hy_9Q-13QyO95rtYVSLrhvT8,4915
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/residual_collapsing.py,sha256=DcJEIkGvBdIMOelNIwaJUZ5UsAHiGnDJPR20I464vWo,2929
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/scale_equalization.py,sha256=XFtU9yuBmoZlX0f0mS6otMPWMk-RcWs94XdvvTNhW8Y,3303
-model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/shift_negative_activation.py,sha256=lOPl5zDU3FoR9WmlxO04Pfi65MimK0gmnuHzQJodQdY,10668
+model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/shift_negative_activation.py,sha256=3WCLvPyx7tVkM0rwYhYq-gntCzW9R_DcImR1ucKlPac,10772
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/softmax_shift.py,sha256=05lV4pIL3hJkZl4JQPV4wk_EFD0eYLG5b8cdzvZk4P8,1588
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/transform_function_call_method.py,sha256=EC9Dvp-_UlpDWnipnf8ds65wh_Y-T8pXAFIwRScWpiY,2044
 model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/virtual_activation_weights_composition.py,sha256=WmEa8Xjji-_tIbthDxlLAGSr69nWk-YKcHNaVqLa7sg,1375
@@ -528,8 +531,8 @@ model_compression_toolkit/xquant/pytorch/model_analyzer.py,sha256=b93o800yVB3Z-i
 model_compression_toolkit/xquant/pytorch/pytorch_report_utils.py,sha256=bOc-hFL3gdoSM1Th_S2N_-9JJSlPGpZCTx_QLJHS6lg,3388
 model_compression_toolkit/xquant/pytorch/similarity_functions.py,sha256=CERxq5K8rqaiE-DlwhZBTUd9x69dtYJlkHOPLB54vm8,2354
 model_compression_toolkit/xquant/pytorch/tensorboard_utils.py,sha256=mkoEktLFFHtEKzzFRn_jCnxjhJolK12TZ5AQeDHzUO8,9767
-mct_nightly-2.1.0.20240813.141729.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
-mct_nightly-2.1.0.20240813.141729.dist-info/METADATA,sha256=rKrCpumEkmoyMZLUBKH25qud3SqUEXdhHnBzP9nqdrE,19721
-mct_nightly-2.1.0.20240813.141729.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mct_nightly-2.1.0.20240813.141729.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
-mct_nightly-2.1.0.20240813.141729.dist-info/RECORD,,
+mct_nightly-2.1.0.20240815.452.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
+mct_nightly-2.1.0.20240815.452.dist-info/METADATA,sha256=sRuvfW9Die83_at1NPFhuX1I9FZcyEEHNc38yC11mWg,19718
+mct_nightly-2.1.0.20240815.452.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mct_nightly-2.1.0.20240815.452.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
+mct_nightly-2.1.0.20240815.452.dist-info/RECORD,,

model_compression_toolkit/__init__.py

@@ -27,4 +27,4 @@ from model_compression_toolkit import data_generation
 from model_compression_toolkit import pruning
 from model_compression_toolkit.trainable_infrastructure.keras.load_model import keras_load_quantized_model
 
-__version__ = "2.1.0.20240813.141729"
+__version__ = "2.1.0.20240815.000452"

model_compression_toolkit/core/common/mixed_precision/mixed_precision_candidates_filter.py

@@ -0,0 +1,80 @@
+# Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import numpy as np
+
+from model_compression_toolkit.core import ResourceUtilization, FrameworkInfo
+from model_compression_toolkit.core.common import Graph
+from model_compression_toolkit.logger import Logger
+from model_compression_toolkit.target_platform_capabilities.target_platform import TargetPlatformCapabilities
+
+
+def filter_candidates_for_mixed_precision(graph: Graph,
+                                          target_resource_utilization: ResourceUtilization,
+                                          fw_info: FrameworkInfo,
+                                          tpc: TargetPlatformCapabilities):
+    """
+    Filters out candidates in case of mixed precision search for only weights or activation compression.
+    For instance, if running only weights compression - filters out candidates of activation configurable nodes
+    such that only a single candidate would remain, with the bitwidth equal to the one defined in the matching layer's
+    base config in the TPC.
+
+    Note" This function modifies the graph inplace!
+
+    Args:
+        graph: A graph representation of the model to be quantized.
+        target_resource_utilization: The resource utilization of the target device.
+        fw_info: fw_info: Information needed for quantization about the specific framework.
+        tpc: TargetPlatformCapabilities object that describes the desired inference target platform.
+
+    """
+
+    no_total_restrictions = (target_resource_utilization.total_memory == np.inf and
+                             target_resource_utilization.bops == np.inf)
+
+    if target_resource_utilization.weights_memory < np.inf:
+        if target_resource_utilization.activation_memory == np.inf and no_total_restrictions:
+            # Running mixed precision for weights compression only -
+            # filter out candidates activation only configurable node
+            weights_conf = graph.get_weights_configurable_nodes(fw_info)
+            for n in graph.get_activation_configurable_nodes():
+                if n not in weights_conf:
+                    base_cfg_nbits = n.get_qco(tpc).base_config.activation_n_bits
+                    filtered_conf = [c for c in n.candidates_quantization_cfg if
+                                     c.activation_quantization_cfg.enable_activation_quantization and
+                                     c.activation_quantization_cfg.activation_n_bits == base_cfg_nbits]
+
+                    if len(filtered_conf) != 1:
+                        Logger.critical(f"Running weights only mixed precision failed on layer {n.name} with multiple "
+                                        f"activation quantization configurations.")  # pragma: no cover
+                    n.candidates_quantization_cfg = filtered_conf
+
+    elif target_resource_utilization.activation_memory < np.inf:
+        if target_resource_utilization.weights_memory == np.inf and no_total_restrictions:
+            # Running mixed precision for activation compression only -
+            # filter out candidates weights only configurable node
+            activation_conf = graph.get_activation_configurable_nodes()
+            for n in graph.get_weights_configurable_nodes(fw_info):
+                if n not in activation_conf:
+                    kernel_attr = graph.fw_info.get_kernel_op_attributes(n.type)[0]
+                    base_cfg_nbits = n.get_qco(tpc).base_config.attr_weights_configs_mapping[kernel_attr].weights_n_bits
+                    filtered_conf = [c for c in n.candidates_quantization_cfg if
+                                     c.weights_quantization_cfg.get_attr_config(
+                                         kernel_attr).enable_weights_quantization and
+                                     c.weights_quantization_cfg.get_attr_config(
+                                         kernel_attr).weights_n_bits == base_cfg_nbits]
+                    if len(filtered_conf) != 1:
+                        Logger.critical(f"Running activation only mixed precision failed on layer {n.name} with multiple "
+                                        f"weights quantization configurations.")  # pragma: no cover
+                    n.candidates_quantization_cfg = filtered_conf

model_compression_toolkit/core/keras/constants.py

@@ -31,7 +31,8 @@ KERNEL_SIZE = 'kernel_size'
 PADDING = 'padding'
 GROUPS = 'groups'
 STRIDES = 'strides'
-DILATIONS = 'dilation_rate'
+DILATION_RATE = 'dilation_rate'
+DILATIONS = 'dilations'
 DATA_FORMAT = 'data_format'
 LAYER_NAME = 'name'
 TRAINABLE = 'trainable'
@@ -62,6 +63,7 @@ DEPTHWISE_CONSTRAINT = 'depthwise_constraint'
 KERNEL_INITIALIZER = 'kernel_initializer'
 KERNEL_REGULARIZER = 'kernel_regularizer'
 KERNEL_CONSTRAINT = 'kernel_constraint'
+RATE = 'rate'
 
 # functional nodes attributes
 FUNCTION = 'function'
@@ -71,6 +73,7 @@ F_MATMUL = 'matmul'
 F_STACK = 'stack'
 F_STRIDED_SLICE_BEGIN = 'begin_mask'
 F_STRIDED_SLICE_END = 'end_mask'
+F_SWISH = 'nn.silu'
 
 # Layers variables names:
 KERNEL: str = 'kernel'

model_compression_toolkit/core/keras/default_framework_info.py

@@ -29,7 +29,7 @@ from model_compression_toolkit.core.common.framework_info import FrameworkInfo
 from model_compression_toolkit.target_platform_capabilities.target_platform import QuantizationMethod
 from model_compression_toolkit.constants import SOFTMAX_THRESHOLD
 from model_compression_toolkit.core.keras.constants import SOFTMAX, LINEAR, RELU, SWISH, SIGMOID, IDENTITY, TANH, SELU, \
-    KERNEL, DEPTHWISE_KERNEL
+    KERNEL, DEPTHWISE_KERNEL, GELU
 from model_compression_toolkit.core.keras.quantizer.fake_quant_builder import power_of_two_quantization, symmetric_quantization, uniform_quantization
 
 """
@@ -75,7 +75,8 @@ ACTIVATION2MINMAX = {SOFTMAX: (0, SOFTMAX_THRESHOLD),
                      TANH: (-1, 1),
                      SWISH: (-0.279, None),
                      RELU: (0, None),
-                     SELU: (None, None),
+                     SELU: (-1.76, None),
+                     GELU: (-0.17, None),
                      }
 
 """

model_compression_toolkit/core/keras/graph_substitutions/substitutions/conv_funcs_to_layer.py

@@ -0,0 +1,241 @@
+# Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import numpy as np
+import tensorflow as tf
+from packaging import version
+if version.parse(tf.__version__) >= version.parse("2.13"):
+    from keras.src.layers.core import TFOpLambda
+    from keras.src.layers import Conv2D, DepthwiseConv2D
+else:
+    from keras.layers.core import TFOpLambda
+    from keras.layers import Conv2D, DepthwiseConv2D
+from model_compression_toolkit.logger import Logger
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph, BaseNode, OutTensor
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher
+from model_compression_toolkit.constants import REUSE, REUSE_GROUP
+from model_compression_toolkit.core.keras.constants import KERNEL, BIAS, USE_BIAS, FILTERS, PADDING, \
+    KERNEL_SIZE, DEPTH_MULTIPLIER, STRIDES, DILATIONS, DILATION_RATE, DEPTHWISE_KERNEL, RATE
+
+
+def extract_bias_node_data(_node: FunctionalNode, _graph: Graph) -> np.ndarray:
+    """
+    Check is can extract bias from next node.
+
+    Args:
+        _node: conv node to check for subsequent add\bias_add node to extract bias from.
+        _graph: model graph.
+
+    Returns:
+        The bias weight. None if couldn't extract bias.
+
+    """
+    b = None
+    next_nodes = _graph.get_next_nodes(_node)
+    if len(next_nodes) == 1 and len(_graph.get_prev_nodes(next_nodes[0])) == 1:
+        # Found pattern in graph: conv_node->next_node. Check if next node is add\bias_add that can be absorbed as bias.
+        if next_nodes[0].is_match_type(tf.add):
+            b = next_nodes[0].weights.get(0, next_nodes[0].weights.get(1))
+            if b is not None and len(b.shape) != 1:
+                # Constant input to Add node (bias) has irregular shape. Expecting a 1-D array.
+                b = None  # pragma: no cover
+        elif next_nodes[0].is_match_type(tf.nn.bias_add):
+            # In bias_add, weight is always 1-D array. Extract weight from weights or kwargs.
+            if 1 in next_nodes[0].weights:
+                b = next_nodes[0].weights[1]
+            elif BIAS in _node.op_call_kwargs:
+                b = np.array(_node.op_call_kwargs[BIAS], dtype=np.float32)
+
+    return b
+
+
+def replace_conv_node(graph: Graph, new_node: BaseNode, old_node: FunctionalNode, remove_add_node: bool):
+    """
+    Replace in-place a functional conv node (and possibly subsequent add node) with Conv layer.
+    Args:
+        graph: model Graph.
+        new_node: Conv layer node.
+        old_node: conv function node.
+        remove_add_node: whether to remove subsequent add node or not.
+    """
+    graph.add_node(new_node)
+
+    # Replace functional conv node (and potentially add node) with Conv node.
+    graph.reconnect_in_edges(old_node, new_node)
+    if remove_add_node:
+        next_nodes = graph.get_next_nodes(old_node)
+        graph.reconnect_out_edges(next_nodes[0], new_node)
+        graph.replace_output_node(current_node=next_nodes[0], new_node=new_node)
+        graph.remove_edge(old_node, next_nodes[0])
+        graph.remove_node(next_nodes[0])
+    else:
+        graph.reconnect_out_edges(old_node, new_node)
+        graph.replace_output_node(current_node=old_node, new_node=new_node)
+    graph.remove_node(old_node)
+
+
+class Conv2dFuncToConv2dLayer(common.BaseSubstitution):
+    """
+    Substitutes tf.nn.conv2d, tf.compat.v1.nn.conv2d, tf.nn.convolution, tf.compat.v1.nn.convolution functions with a Conv2D layer.
+    """
+
+    def __init__(self):
+        """
+        Initializes the Conv2dFuncToConv2dLayer substitution matcher instance.
+        """
+        conv2d_matcher = NodeOperationMatcher(tf.nn.conv2d) | NodeOperationMatcher(tf.compat.v1.nn.conv2d)
+        convolution_matcher = NodeOperationMatcher(tf.nn.convolution) | NodeOperationMatcher(tf.compat.v1.nn.convolution)
+        super().__init__(matcher_instance=conv2d_matcher | convolution_matcher)
+
+    def substitute(self,
+                   graph: Graph,
+                   conv_func_node: FunctionalNode) -> Graph:
+        """
+        Substitutes conv functions with a Conv2D layer.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            conv_func_node: The functional node to be replaced.
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        if 1 in conv_func_node.weights:
+            k = conv_func_node.weights[1]
+        elif FILTERS in conv_func_node.op_call_kwargs:
+            k = np.array(conv_func_node.op_call_kwargs[FILTERS], dtype=np.float32)
+        else:
+            # Conv weight isn't a constant -> skip substitution.
+            return graph  # pragma: no cover
+
+        if len(k.shape) != 4:
+            # Conv dimension doesn't match conv2d dimension (K1 x K2 x Cin x Cout) -> skip substitution.
+            return graph  # pragma: no cover
+
+        # Check if can extract bias from next node.
+        b = extract_bias_node_data(conv_func_node, graph)
+
+        weights = {KERNEL: k}
+        # Create Conv2D layer attributes.
+        conv_fw_attr = {FILTERS: k.shape[3], KERNEL_SIZE: k.shape[:2]}
+        if len(conv_func_node.op_call_args) > 0:
+            Logger.critical(f"node {conv_func_node.name} expected to have only kwargs but got args={conv_func_node.op_call_args}.")  # pragma: no cover
+        if STRIDES in conv_func_node.op_call_kwargs:
+            strides = conv_func_node.op_call_kwargs[STRIDES]
+            if len(strides) == 4:
+                if strides[0] > 1 or strides[3] > 1:
+                    # Non-standard strides -> skip substitution.
+                    return graph  # pragma: no cover
+                conv_fw_attr[STRIDES] = strides[1:3]
+            else:
+                conv_fw_attr[STRIDES] = strides
+        if PADDING in conv_func_node.op_call_kwargs:
+            padding = conv_func_node.op_call_kwargs[PADDING]
+            if not isinstance(padding, str):
+                # Non-standard padding, Layer only support either 'valid' or 'same' -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[PADDING] = padding
+        if DILATIONS in conv_func_node.op_call_kwargs and conv_func_node.op_call_kwargs[DILATIONS] is not None:
+            dilations = conv_func_node.op_call_kwargs[DILATIONS]
+            if isinstance(dilations, (list, tuple)) and len(dilations) == 4:
+                if dilations[0] > 1 or dilations[3] > 1:
+                    # Non-standard dilations -> skip substitution.
+                    return graph  # pragma: no cover
+                conv_fw_attr[DILATION_RATE] = dilations[1:3]
+            else:
+                conv_fw_attr[DILATION_RATE] = dilations
+        if b is None:
+            conv_fw_attr[USE_BIAS] = False
+        else:
+            weights[BIAS] = b
+
+        _reuse_params = {REUSE: conv_func_node.reuse, REUSE_GROUP: conv_func_node.reuse_group}
+        conv_node = BaseNode(conv_func_node.name, conv_fw_attr, conv_func_node.input_shape, conv_func_node.output_shape,
+                             weights, Conv2D, **_reuse_params)
+
+        replace_conv_node(graph, conv_node, conv_func_node, remove_add_node=b is not None)
+        return graph
+
+
+class DwConv2dFuncToDwConv2dLayer(common.BaseSubstitution):
+    """
+    Substitutes tf.nn.depthwise_conv2d & tf.compat.v1.nn.depthwise_conv2d functions with a DepthwiseConv2D layer.
+    """
+
+    def __init__(self):
+        """
+        Initializes the DwConv2dFuncToDwConv2dLayer substitution matcher.
+        """
+        matcher = NodeOperationMatcher(tf.nn.depthwise_conv2d) | NodeOperationMatcher(tf.compat.v1.nn.depthwise_conv2d)
+        super().__init__(matcher_instance=matcher)
+
+    def substitute(self,
+                   graph: Graph,
+                   dwconv_func_node: FunctionalNode) -> Graph:
+        """
+        Substitutes dw-conv2d functions with a DepthwiseConv2D layer.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            dwconv_func_node: The DepthwiseConv2D node to be replaced.
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        if 1 not in dwconv_func_node.weights:
+            # Conv weight isn't a constant -> skip substitution.
+            return graph  # pragma: no cover
+
+        k = dwconv_func_node.weights[1]
+
+        # Check is can extract bias from next node.
+        b = extract_bias_node_data(dwconv_func_node, graph)
+
+        weights = {DEPTHWISE_KERNEL: k}
+        k_shape = k.shape
+        conv_fw_attr = {DEPTH_MULTIPLIER: k_shape[3], KERNEL_SIZE: k_shape[:2]}
+        if len(dwconv_func_node.op_call_args) > 0:
+            Logger.critical(f"node {dwconv_func_node.name} expected to have only kwargs but got args={dwconv_func_node.op_call_args}.")  # pragma: no cover
+        if STRIDES in dwconv_func_node.op_call_kwargs:
+            strides = dwconv_func_node.op_call_kwargs[STRIDES]
+            if strides[0] > 1 or strides[3] > 1:
+                # Non-standard strides -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[STRIDES] = strides[1:3]
+        if PADDING in dwconv_func_node.op_call_kwargs:
+            padding = dwconv_func_node.op_call_kwargs[PADDING]
+            if not isinstance(padding, str):
+                # Non-standard padding, Layer only support either 'valid' or 'same' -> skip substitution.
+                return graph  # pragma: no cover
+            conv_fw_attr[PADDING] = padding
+        if RATE in dwconv_func_node.op_call_kwargs and dwconv_func_node.op_call_kwargs[RATE] is not None:
+            conv_fw_attr[DILATION_RATE] = dwconv_func_node.op_call_kwargs[RATE]
+        elif DILATIONS in dwconv_func_node.op_call_kwargs and dwconv_func_node.op_call_kwargs[DILATIONS] is not None:
+            conv_fw_attr[DILATION_RATE] = dwconv_func_node.op_call_kwargs[DILATIONS]
+        if b is None:
+            conv_fw_attr[USE_BIAS] = False
+        else:
+            weights[BIAS] = b
+
+        _reuse_params = {REUSE: dwconv_func_node.reuse, REUSE_GROUP: dwconv_func_node.reuse_group}
+        conv_node = BaseNode(dwconv_func_node.name, conv_fw_attr, dwconv_func_node.input_shape, dwconv_func_node.output_shape,
+                             weights, DepthwiseConv2D, **_reuse_params)
+
+        replace_conv_node(graph, conv_node, dwconv_func_node, remove_add_node=b is not None)
+        return graph

model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py

@@ -23,7 +23,7 @@ else:
 from model_compression_toolkit.core.common import BaseNode
 from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher, NodeFrameworkAttrMatcher
 from model_compression_toolkit.core.common.substitutions.linear_collapsing import Conv2DCollapsing, Op2DAddConstCollapsing
-from model_compression_toolkit.core.keras.constants import KERNEL, KERNEL_SIZE, STRIDES, DILATIONS, LINEAR, \
+from model_compression_toolkit.core.keras.constants import KERNEL, KERNEL_SIZE, STRIDES, DILATION_RATE, LINEAR, \
     ACTIVATION, BIAS, USE_BIAS, LAYER_NAME, FILTERS, PADDING, GROUPS, DATA_FORMAT
 from model_compression_toolkit.logger import Logger
 
@@ -122,7 +122,7 @@ def keras_linear_collapsing() -> Conv2DCollapsing:
                             USE_BIAS,
                             STRIDES,
                             PADDING,
-                            DILATIONS,
+                            DILATION_RATE,
                             GROUPS,
                             FILTERS,
                             data_format_str=DATA_FORMAT,

model_compression_toolkit/core/keras/graph_substitutions/substitutions/matmul_substitution.py

@@ -65,6 +65,8 @@ class MatmulToDenseSubstitution(common.BaseSubstitution):
 
         # read const from matmul inputs
         w = matmul_node.weights.get(1)
+        if w is None:
+            w = np.array(matmul_node.op_call_kwargs['b'], dtype=np.float32) if 'b' in matmul_node.op_call_kwargs else None
         if w is None:
             Logger.critical(f"Matmul substitution failed: Unable to locate weight for node {matmul_node.name}.")  # pragma: no cover
 

model_compression_toolkit/core/keras/graph_substitutions/substitutions/sigmoid_mul_to_swish.py

@@ -0,0 +1,89 @@
+# Copyright 2024 Sony Semiconductors Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from typing import Tuple, Union
+import numpy as np
+import tensorflow as tf
+from packaging import version
+if version.parse(tf.__version__) >= version.parse("2.13"):
+    from keras.src.layers.core import TFOpLambda
+    from keras.src.layers import Multiply, Activation
+else:
+    from keras.layers.core import TFOpLambda
+    from keras.layers import Multiply, Activation
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph, BaseNode, OutTensor
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher, \
+    EdgeMatcher, NodeFrameworkAttrMatcher
+from model_compression_toolkit.constants import REUSE, REUSE_GROUP
+from model_compression_toolkit.core.keras.constants import FUNCTION, F_SWISH, ACTIVATION, SIGMOID
+
+
+class MulSigmoidToSwish(common.BaseSubstitution):
+    """
+    Substitutes mul(x, sigmoid(x)) with swish.
+    """
+
+    def __init__(self):
+        """
+        Initializes the MulSigmoidToSwish substitution matcher instance.
+        """
+        mul_matcher = NodeOperationMatcher(tf.math.multiply) | NodeOperationMatcher(Multiply)
+        activation_sigmoid = NodeOperationMatcher(Activation) & NodeFrameworkAttrMatcher(ACTIVATION, SIGMOID)
+        sigmoid_matcher = NodeOperationMatcher(tf.sigmoid) | activation_sigmoid
+        super().__init__(matcher_instance=EdgeMatcher(sigmoid_matcher, mul_matcher))
+
+    def substitute(self,
+                   graph: Graph,
+                   sigmoid_mul_edge: Tuple[FunctionalNode, Union[FunctionalNode, BaseNode], int]) -> Graph:
+        """
+        Substitutes mul(x, sigmoid(x)) with swish.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            sigmoid_mul_edge: edge between sigmoid and multiply nodes
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        sigmoid_node, mul_node, _ = sigmoid_mul_edge
+        if sigmoid_node in [o.node for o in graph.output_nodes]:
+            # Sigmoid node in outputs -> Skip substitution.
+            return graph
+
+        input_node = graph.get_prev_nodes(sigmoid_node)[0]
+        if len(graph.get_next_nodes(sigmoid_node)) > 1 or input_node not in graph.get_prev_nodes(mul_node):
+            # Structure isn't mul(x, sigmoid(x)) -> Skip substitution.
+            return graph
+        _reuse_params = {REUSE: mul_node.reuse, REUSE_GROUP: mul_node.reuse_group}
+        swish_node = FunctionalNode(f'swish__{sigmoid_node.name}_{mul_node.name}', {FUNCTION: F_SWISH},
+                                    sigmoid_node.input_shape, mul_node.output_shape, {}, TFOpLambda,
+                                    op_call_args=[], op_call_kwargs={}, functional_op=tf.nn.silu, **_reuse_params)
+
+        graph.add_node(swish_node)
+
+        # Replace functional conv node (and potentially add node) with Conv node.
+        graph.reconnect_in_edges(sigmoid_node, swish_node)
+        graph.reconnect_out_edges(mul_node, swish_node)
+        graph.replace_output_node(current_node=mul_node, new_node=swish_node)
+        graph.remove_edge(input_node, mul_node)
+        graph.remove_edge(sigmoid_node, mul_node)
+        graph.remove_node(sigmoid_node)
+        graph.remove_node(mul_node)
+
+        return graph
+

model_compression_toolkit/core/keras/keras_implementation.py

@@ -69,6 +69,9 @@ from model_compression_toolkit.core.keras.graph_substitutions.substitutions.acti
     ActivationDecomposition
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.matmul_substitution import \
     MatmulToDenseSubstitution
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.sigmoid_mul_to_swish import MulSigmoidToSwish
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.conv_funcs_to_layer import \
+    Conv2dFuncToConv2dLayer, DwConv2dFuncToDwConv2dLayer
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.softmax_shift import \
     keras_softmax_shift
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.batchnorm_folding import \
@@ -242,8 +245,11 @@ class KerasImplementation(FrameworkImplementation):
         Returns: A list of the framework substitutions used to prepare the graph.
 
         """
-        return [SeparableConvDecomposition(),
+        return [MulSigmoidToSwish(),
+                SeparableConvDecomposition(),
                 MatmulToDenseSubstitution(),
+                Conv2dFuncToConv2dLayer(),
+                DwConv2dFuncToDwConv2dLayer(),
                 MultiHeadAttentionDecomposition(),
                 ActivationDecomposition(),
                 DwconvToConv(),

model_compression_toolkit/core/pytorch/default_framework_info.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-from torch.nn import Hardsigmoid, ReLU, ReLU6, Softmax, Sigmoid
-from torch.nn.functional import hardsigmoid, relu, relu6, softmax
+from torch.nn import Hardsigmoid, ReLU, ReLU6, Softmax, Sigmoid, GELU, SELU
+from torch.nn.functional import hardsigmoid, relu, relu6, softmax, gelu, selu
 from torch.nn import Conv2d, ConvTranspose2d, Linear
 from torch import sigmoid
 
@@ -74,7 +74,12 @@ LAYER2MINMAX = {Softmax: (0, SOFTMAX_THRESHOLD),
                 ReLU: (0, None),
                 relu: (0, None),
                 ReLU6: (0, None),
-                relu6: (0, None)}
+                relu6: (0, None),
+                GELU: (-0.17, None),
+                gelu: (-0.17, None),
+                SELU: (-1.76, None),
+                selu: (-1.76, None),
+                }
 
 """
 Mapping from a QuantizationMethod to an activation quantizer function.

model_compression_toolkit/core/pytorch/graph_substitutions/substitutions/shift_negative_activation.py

@@ -17,9 +17,9 @@ from typing import Tuple, Any, Callable
 
 import numpy as np
 import torch.nn.functional
-from torch.nn import Conv2d, Linear, PReLU, ELU, Hardswish, Dropout, ZeroPad2d, SiLU
+from torch.nn import Conv2d, Linear, PReLU, ELU, Hardswish, Dropout, ZeroPad2d, SiLU, GELU
 from torch import reshape
-from torch.nn.functional import hardswish, silu, prelu, elu
+from torch.nn.functional import hardswish, silu, prelu, elu, gelu
 from torch.nn.functional import avg_pool2d
 
 from model_compression_toolkit.core import CoreConfig, FrameworkInfo
@@ -68,7 +68,9 @@ def shift_negative_activation_node_matchers():
                NodeOperationMatcher(Hardswish) | \
                NodeOperationMatcher(hardswish) | \
                NodeOperationMatcher(SiLU) | \
-               NodeOperationMatcher(silu)
+               NodeOperationMatcher(silu) | \
+               NodeOperationMatcher(GELU) | \
+               NodeOperationMatcher(gelu)
 
     # Match linear layers where we can add a correction.
     linear_node = NodeOperationMatcher(Conv2d) | \

model_compression_toolkit/core/runner.py

@@ -27,6 +27,8 @@ from model_compression_toolkit.core.common.graph.memory_graph.compute_graph_max_
     SchedulerInfo
 from model_compression_toolkit.core.common.graph.memory_graph.memory_graph import MemoryGraph
 from model_compression_toolkit.core.common.hessian.hessian_info_service import HessianInfoService
+from model_compression_toolkit.core.common.mixed_precision.mixed_precision_candidates_filter import \
+    filter_candidates_for_mixed_precision
 from model_compression_toolkit.core.common.mixed_precision.resource_utilization_tools.resource_utilization_data import \
     requires_mixed_precision
 from model_compression_toolkit.core.graph_prep_runner import graph_preparation_runner
@@ -137,6 +139,7 @@ def core_runner(in_model: Any,
     if core_config.mixed_precision_enable:
         if core_config.mixed_precision_config.configuration_overwrite is None:
 
+            filter_candidates_for_mixed_precision(graph, target_resource_utilization, fw_info, tpc)
             bit_widths_config = search_bit_width(tg,
                                                  fw_info,
                                                  fw_impl,