mct-nightly 1.10.0.20231017.post414__py3-none-any.whl → 1.10.0.20231019.post424__py3-none-any.whl

{mct_nightly-1.10.0.20231017.post414.dist-info → mct_nightly-1.10.0.20231019.post424.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mct-nightly
-Version: 1.10.0.20231017.post414
+Version: 1.10.0.20231019.post424
 Summary: A Model Compression Toolkit for neural networks
 Home-page: UNKNOWN
 License: UNKNOWN

{mct_nightly-1.10.0.20231017.post414.dist-info → mct_nightly-1.10.0.20231019.post424.dist-info}/RECORD

@@ -10,7 +10,7 @@ model_compression_toolkit/core/common/__init__.py,sha256=Wh127PbXcETZX_d1PQqZ71E
 model_compression_toolkit/core/common/base_substitutions.py,sha256=xDFSmVVs_iFSZfajytI0cuQaNRNcwHX3uqOoHgVUvxQ,1666
 model_compression_toolkit/core/common/data_loader.py,sha256=7YF5Mqz64Xb4rVwY3knrdIZ4JEHybXxiQqx0deR_c5k,4017
 model_compression_toolkit/core/common/defaultdict.py,sha256=n-F3dP-VTMnGy9KfCwp7D_WBlvFxe3waX4LpnOX8FH0,2281
-model_compression_toolkit/core/common/framework_implementation.py,sha256=0BXJPOBpmc7sFl35R6W02o1ALhM8PGAh36oAMw1jh8w,21312
+model_compression_toolkit/core/common/framework_implementation.py,sha256=yFWOic8eEq1A0_IhgPAAMFPfMbm0XBlsoz8-PYYp2o4,20683
 model_compression_toolkit/core/common/framework_info.py,sha256=hwmstv7IuBRfa6IxDbeG4y-7AxKx4bwCyI_Exi2C7mo,6424
 model_compression_toolkit/core/common/memory_computation.py,sha256=ixoSpV5ZYZGyzhre3kQcvR2sNA8KBsPZ3lgbkDnw9Cs,1205
 model_compression_toolkit/core/common/model_builder_mode.py,sha256=jll9-59OPaE3ug7Y9-lLyV99_FoNHxkGZMgcm0Vkpss,1324
@@ -46,7 +46,7 @@ model_compression_toolkit/core/common/graph/memory_graph/max_cut_astar.py,sha256
 model_compression_toolkit/core/common/graph/memory_graph/memory_element.py,sha256=gRmBEFRmyJsNKezQfiwDwQu1cmbGd2wgKCRTH6iw8mw,3961
 model_compression_toolkit/core/common/graph/memory_graph/memory_graph.py,sha256=gw4av_rzn_3oEAPpD3B7PHZDqnxHMjIESevl6ppPnkk,7175
 model_compression_toolkit/core/common/hessian/__init__.py,sha256=bxPVbkIlHFJMiOgTdWMVCqcD9JKV5kb2bVdWUTeLpj8,1021
-model_compression_toolkit/core/common/hessian/hessian_info_service.py,sha256=sTexwOIu6mJrkslZx5exWXeXaSl1R2AZI5ZIiW_OjqI,7420
+model_compression_toolkit/core/common/hessian/hessian_info_service.py,sha256=FJlNhYaxI_3X0aG5wXSmdNsKYZm7paiSchi1TJpZQiA,8180
 model_compression_toolkit/core/common/hessian/hessian_info_utils.py,sha256=zHEcql5z4VGuvEx3oI7OOD5FeWgwuN4xpf2H01O8kY4,3324
 model_compression_toolkit/core/common/hessian/trace_hessian_calculator.py,sha256=08PapRDOgfHybDsNW5iBEEvNFKsWraPnBBlsUfcwdRE,3600
 model_compression_toolkit/core/common/hessian/trace_hessian_request.py,sha256=sPOQL7IUTrd7QlCFuDPgL6RhMkBPMl7Fg81-WhouvQY,2705
@@ -133,10 +133,10 @@ model_compression_toolkit/core/common/visualization/final_config_visualizer.py,s
 model_compression_toolkit/core/common/visualization/nn_visualizer.py,sha256=6EjZj_KE1tICTQ0XSKIx5ivsRFpRktFywda7pW7YnNQ,5955
 model_compression_toolkit/core/common/visualization/tensorboard_writer.py,sha256=1Vbr5gKqH1fJl91iTNFlIjyEMh6jm88T4AIWalMrJFw,20099
 model_compression_toolkit/core/keras/__init__.py,sha256=mjbqLD-KcG3eNeCYpu1GBS7VclGVOQ63x2p6mAAuba4,698
-model_compression_toolkit/core/keras/constants.py,sha256=nbty5PUcLhC9aTuAemXTEmKcPF--7a5adO1-nQsgB9A,2664
+model_compression_toolkit/core/keras/constants.py,sha256=0vQZ3-8-IJ735KCet858OcxlDRFx3GyDks97GBF9gS4,2968
 model_compression_toolkit/core/keras/custom_layer_validation.py,sha256=f-b14wuiIgitBe7d0MmofYhDCTO3IhwJgwrh-Hq_t_U,1192
 model_compression_toolkit/core/keras/default_framework_info.py,sha256=HLLO28tsbb9tHYQ05grUr3oJTRKdS520QnyGUYgzRK0,4994
-model_compression_toolkit/core/keras/keras_implementation.py,sha256=-jjDGAjP_8KDrGsUHu_Kcsushn78r1xZqBIKFlZVTJk,28999
+model_compression_toolkit/core/keras/keras_implementation.py,sha256=icZxFqA-f3afiaIfHYgTrcxS26w7BaJKzHWVx-oYwAU,28370
 model_compression_toolkit/core/keras/keras_model_validation.py,sha256=1wNV2clFdC9BzIELRLSO2uKf0xqjLqlkTJudwtCeaJk,1722
 model_compression_toolkit/core/keras/keras_node_prior_info.py,sha256=f6o5Fmpw0aDrO704_A-SqBrKSO1iNEOyofP9pm3g8yg,3936
 model_compression_toolkit/core/keras/kpi_data_facade.py,sha256=rArrfMtxWGR1P4nhKKxqh6fo7pauRDzkRsZIh_SXxO4,8502
@@ -154,6 +154,7 @@ model_compression_toolkit/core/keras/graph_substitutions/substitutions/activatio
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_folding.py,sha256=5ZMQLGs5Tc11WUIuAnxiOihVpwqT2bijezPGGtaOgjA,8145
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_reconstruction.py,sha256=GR1a3mCZpNUu4WxixJXF_aSm57phAdxaRoHecNx3hxw,3168
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/batchnorm_refusing.py,sha256=5df_xGfXkqNub4xVRnCWQvSohWqdv12axjJ6edVU2H0,2478
+model_compression_toolkit/core/keras/graph_substitutions/substitutions/dwconv_to_conv.py,sha256=R3U7cjc2E0zheMem16GHygp5jZFGSaomkNOTxTjcAgw,5794
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/input_scaling.py,sha256=Yj59BMBrITJnXJHH-7de91LJwH_1l1WhY1udSQjdoi4,5598
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/linear_collapsing.py,sha256=Yl72Io4-etnsOXfMKAZmC2lDzmskxZu5gey7IBcUukU,5925
 model_compression_toolkit/core/keras/graph_substitutions/substitutions/multi_head_attention_decomposition.py,sha256=aAG5wrcnnydn1pPYqvH56LWsQXjSODbsoNbX_jtQGP4,26759
@@ -194,7 +195,7 @@ model_compression_toolkit/core/pytorch/__init__.py,sha256=Rf1RcYmelmdZmBV5qOKvKW
 model_compression_toolkit/core/pytorch/constants.py,sha256=Kt_GDwe3yX9oMS1DI2eXYuUT25_lpjeCkxpstsAiXCI,2472
 model_compression_toolkit/core/pytorch/default_framework_info.py,sha256=n0efDuwJUR-BHldZEGQu5bpb3XIR7QYt1HI5tcmz--c,4224
 model_compression_toolkit/core/pytorch/kpi_data_facade.py,sha256=J0IDOtFMVFSFyBXDzNGbwJfHu89iRBJFdid1_wFB-xQ,8482
-model_compression_toolkit/core/pytorch/pytorch_implementation.py,sha256=N7NV-DIXaVjMZS9gywrzBS74p0eM_WoxQZ7S8f6qJ5Y,27029
+model_compression_toolkit/core/pytorch/pytorch_implementation.py,sha256=bTjqdPu4u7kVpH5u9kK5WgjexJBrUCSUBkCHAUYPv4A,26257
 model_compression_toolkit/core/pytorch/pytorch_node_prior_info.py,sha256=n_B4a6FMwM9D2w8kzy3oenBWZgXNZuIZgTJC6JEuTy0,3250
 model_compression_toolkit/core/pytorch/utils.py,sha256=rBQMAbWluyIMjVfeghzq6FZv3sR_khszSRpbWvwussw,2959
 model_compression_toolkit/core/pytorch/back2framework/__init__.py,sha256=H_WixgN0elVWf3exgGYsi58imPoYDj5eYPeh6x4yfug,813
@@ -357,22 +358,28 @@ model_compression_toolkit/ptq/pytorch/__init__.py,sha256=cco4TmeIDIh32nj9ZZXVkws
 model_compression_toolkit/ptq/pytorch/quantization_facade.py,sha256=nsfvvLbgERes9qqNSLi-X5EgCIhzpLBDqK_dCAKAbGk,8670
 model_compression_toolkit/qat/__init__.py,sha256=BYKgH1NwB9fqF1TszULQ5tDfLI-GqgZV5sao-lDN9EM,1091
 model_compression_toolkit/qat/common/__init__.py,sha256=6tLZ4R4pYP6QVztLVQC_jik2nES3l4uhML0qUxZrezk,829
-model_compression_toolkit/qat/common/qat_config.py,sha256=-XgSaxS0ClLNsUseoWlB0UOJeWJ1mV4yo4b5ty65jzs,3295
+model_compression_toolkit/qat/common/qat_config.py,sha256=kbSxFL6_u28furq5mW_75STWDmyX4clPt-seJAnX3IQ,3445
 model_compression_toolkit/qat/keras/__init__.py,sha256=cco4TmeIDIh32nj9ZZXVkws4dd9F2UDrmjKzTN8G0V0,697
 model_compression_toolkit/qat/keras/quantization_facade.py,sha256=bH9pw2RtFD_91pfA5LPOuZ_ot04uX8R1au_n3Vr_7-0,16173
-model_compression_toolkit/qat/keras/quantizer/__init__.py,sha256=fWDCj0kIWttYn0sV04Uwih-RAe0vJfz71cxYWeX9qWA,856
+model_compression_toolkit/qat/keras/quantizer/__init__.py,sha256=zmYyCa25_KLCSUCGUDRslh3RCIjcRMxc_oXa54Aui-4,996
 model_compression_toolkit/qat/keras/quantizer/base_keras_qat_quantizer.py,sha256=gPuIgQb8OafvC3SuA8jNsGoy8S8eTsDCEKuh36WDNss,2104
-model_compression_toolkit/qat/keras/quantizer/quant_utils.py,sha256=rS2z_ozyjzQ07MMczaAFNZ7K6RKwAnBOKyRac4UvF44,2123
+model_compression_toolkit/qat/keras/quantizer/quant_utils.py,sha256=cBULOgWUodcBO1lHevZggdTevuDYI6tQceV86U2x6DA,2543
 model_compression_toolkit/qat/keras/quantizer/quantization_builder.py,sha256=mZwghAnKagL7916CJycFHgJdD5aY6At1A_IBkmYqae4,5635
+model_compression_toolkit/qat/keras/quantizer/lsq/__init__.py,sha256=lNJ29DYxaLUPDstRDA1PGI5r9Fulq_hvrZMlhst1Z5g,697
+model_compression_toolkit/qat/keras/quantizer/lsq/symmetric_lsq.py,sha256=FdtVZepTCTbPHoHRiCV1P8MluaNR8RVx7fEoAzXyA0U,12051
+model_compression_toolkit/qat/keras/quantizer/lsq/uniform_lsq.py,sha256=P_IU8q5ipUR7r-bmmH_wSzoFsAcjWRVgCU68T_EzXVc,11159
 model_compression_toolkit/qat/keras/quantizer/ste_rounding/__init__.py,sha256=cco4TmeIDIh32nj9ZZXVkws4dd9F2UDrmjKzTN8G0V0,697
 model_compression_toolkit/qat/keras/quantizer/ste_rounding/symmetric_ste.py,sha256=I4KlaGv17k71IyjuSG9M0OlXlD5P0pfvKa6oCyRQ5FE,13517
 model_compression_toolkit/qat/keras/quantizer/ste_rounding/uniform_ste.py,sha256=EED6LfqhX_OhDRJ9e4GwbpgNC9vq7hoXyJS2VPvG2qc,10789
 model_compression_toolkit/qat/pytorch/__init__.py,sha256=cco4TmeIDIh32nj9ZZXVkws4dd9F2UDrmjKzTN8G0V0,697
 model_compression_toolkit/qat/pytorch/quantization_facade.py,sha256=LGIk_nbcQEOhSN8fe56u6QGGUtGOVhF7tlCwW2G-Ig4,12549
-model_compression_toolkit/qat/pytorch/quantizer/__init__.py,sha256=R4vwVcbg6QprCTNzibyF9PtbKKKBsfu9ffypKDNscJQ,859
+model_compression_toolkit/qat/pytorch/quantizer/__init__.py,sha256=xYa4C8pr9cG1f3mQQcBXO_u3IdJN-zl7leZxuXDs86w,1003
 model_compression_toolkit/qat/pytorch/quantizer/base_pytorch_qat_quantizer.py,sha256=FnhuFCuQoSf78FM1z1UZgXXd3k-mKSM7i9dYOuJUmeA,2213
 model_compression_toolkit/qat/pytorch/quantizer/quantization_builder.py,sha256=GOYRDXvQSGe_iUFVmvDy5BqC952hu_-rQO06n8QCyw0,5491
-model_compression_toolkit/qat/pytorch/quantizer/quantizer_utils.py,sha256=5XswoF-5aaFangkHssWKAQTsk6lf_zzndzfCsBWBVMs,5004
+model_compression_toolkit/qat/pytorch/quantizer/quantizer_utils.py,sha256=nO7IrDRo5b9Asf21WJacE4vf5voD3UzF_oGjBoGusD4,5335
+model_compression_toolkit/qat/pytorch/quantizer/lsq/__init__.py,sha256=huHoBUcKNB6BnY6YaUCcFvdyBtBI172ZoUD8ZYeNc6o,696
+model_compression_toolkit/qat/pytorch/quantizer/lsq/symmetric_lsq.py,sha256=HihuaMi0P0OQkNZlZAE-QeYlK_4AqcDKV6N405SdgI0,10712
+model_compression_toolkit/qat/pytorch/quantizer/lsq/uniform_lsq.py,sha256=fqAI151SUo9OYN4UvQJIcdB6p1r7HVeteqPqHGxv-tI,10355
 model_compression_toolkit/qat/pytorch/quantizer/ste_rounding/__init__.py,sha256=Rf1RcYmelmdZmBV5qOKvKWF575ofc06JFQSq83Jz99A,696
 model_compression_toolkit/qat/pytorch/quantizer/ste_rounding/symmetric_ste.py,sha256=4xmLmg7yN2A7iKnifwkWddgJTWMUiIjFilIuorJeK1A,9657
 model_compression_toolkit/qat/pytorch/quantizer/ste_rounding/uniform_ste.py,sha256=HshW016iVAMx7iMkUwlONN2P3K4XgDIu-2AnJnBVSGo,8778
@@ -441,8 +448,8 @@ model_compression_toolkit/trainable_infrastructure/keras/quantize_wrapper.py,sha
 model_compression_toolkit/trainable_infrastructure/keras/quantizer_utils.py,sha256=MVwXNymmFRB2NXIBx4e2mdJ1RfoHxRPYRgjb1MQP5kY,1797
 model_compression_toolkit/trainable_infrastructure/pytorch/__init__.py,sha256=huHoBUcKNB6BnY6YaUCcFvdyBtBI172ZoUD8ZYeNc6o,696
 model_compression_toolkit/trainable_infrastructure/pytorch/base_pytorch_quantizer.py,sha256=SbvRlIdE32PEBsINt1bhSqvrKL_zbM9V-aeSkOn-sw4,3083
-mct_nightly-1.10.0.20231017.post414.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
-mct_nightly-1.10.0.20231017.post414.dist-info/METADATA,sha256=jWHr3DZ16t65LzkFjN2axJRWWPWz0953Ys_wBi8_FBQ,16303
-mct_nightly-1.10.0.20231017.post414.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-mct_nightly-1.10.0.20231017.post414.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
-mct_nightly-1.10.0.20231017.post414.dist-info/RECORD,,
+mct_nightly-1.10.0.20231019.post424.dist-info/LICENSE.md,sha256=aYSSIb-5AFPeITTvXm1UAoe0uYBiMmSS8flvXaaFUks,10174
+mct_nightly-1.10.0.20231019.post424.dist-info/METADATA,sha256=7KfKPP_oFOaieSu_3UTKRv1F3HaR5jiwMOP1DiBAiIg,16303
+mct_nightly-1.10.0.20231019.post424.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+mct_nightly-1.10.0.20231019.post424.dist-info/top_level.txt,sha256=gsYA8juk0Z-ZmQRKULkb3JLGdOdz8jW_cMRjisn9ga4,26
+mct_nightly-1.10.0.20231019.post424.dist-info/RECORD,,

model_compression_toolkit/core/common/framework_implementation.py

@@ -67,18 +67,6 @@ class FrameworkImplementation(ABC):
         raise NotImplemented(f'{self.__class__.__name__} have to implement the '
                              f'framework\'s get_trace_hessian_calculator method.')  # pragma: no cover
 
-    @abstractmethod
-    def sample_single_representative_dataset(self, representative_dataset: Callable):
-        """
-        Get a single sample (namely, batch size of 1) from a representative dataset.
-
-        Args:
-            representative_dataset: Callable which returns the representative dataset at any batch size.
-
-        Returns: List of inputs from representative_dataset where each sample has a batch size of 1.
-        """
-        raise NotImplemented(f'{self.__class__.__name__} have to implement the '
-                             f'framework\'s sample_single_representative_dataset method.')  # pragma: no cover
 
     @abstractmethod
     def to_numpy(self, tensor: Any) -> np.ndarray:

model_compression_toolkit/core/common/hessian/hessian_info_service.py

@@ -54,7 +54,7 @@ class HessianInfoService:
         self.graph = graph
 
         # Create a representative_data_gen with batch size of 1
-        self.representative_dataset = partial(fw_impl.sample_single_representative_dataset,
+        self.representative_dataset = partial(self._sample_single_representative_dataset,
                                               representative_dataset=representative_dataset)
 
         self.fw_impl = fw_impl
@@ -62,6 +62,22 @@ class HessianInfoService:
 
         self.trace_hessian_request_to_score_list = {}
 
+    def _sample_single_representative_dataset(self, representative_dataset: Callable):
+        """
+        Get a single sample (namely, batch size of 1) from a representative dataset.
+
+        Args:
+            representative_dataset: Callable which returns the representative dataset at any batch size.
+
+        Returns: List of inputs from representative_dataset where each sample has a batch size of 1.
+        """
+        images = next(representative_dataset())
+        if not isinstance(images, list):
+            Logger.error(f'Images expected to be a list but is of type {type(images)}')
+
+        # Ensure each image is a single sample, if not, take the first sample
+        return [image[0:1, ...] if image.shape[0] != 1 else image for image in images]
+
     def _clear_saved_hessian_info(self):
         """Clears the saved info approximations."""
         self.trace_hessian_request_to_score_list={}

model_compression_toolkit/core/keras/constants.py

@@ -53,6 +53,13 @@ DIMS = 'dims'
 TARGET_SHAPE = 'target_shape'
 TRANSPOSE_A = 'transpose_a'
 TRANSPOSE_B = 'transpose_b'
+DEPTH_MULTIPLIER = 'depth_multiplier'
+DEPTHWISE_INITIALIZER = 'depthwise_initializer'
+DEPTHWISE_REGULARIZER = 'depthwise_regularizer'
+DEPTHWISE_CONSTRAINT = 'depthwise_constraint'
+KERNEL_INITIALIZER = 'kernel_initializer'
+KERNEL_REGULARIZER = 'kernel_regularizer'
+KERNEL_CONSTRAINT = 'kernel_constraint'
 
 # functional nodes attributes
 FUNCTION = 'function'

model_compression_toolkit/core/keras/graph_substitutions/substitutions/dwconv_to_conv.py

@@ -0,0 +1,127 @@
+# Copyright 2023 Sony Semiconductors Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import numpy as np
+import tensorflow as tf
+from packaging import version
+if version.parse(tf.__version__) >= version.parse("2.13"):
+    from keras.src.layers.core import TFOpLambda
+    from keras.src.layers import Dense, Conv2D, Softmax, Concatenate, Reshape, Permute, DepthwiseConv2D
+else:
+    from keras.layers.core import TFOpLambda
+    from keras.layers import Dense, Conv2D, Softmax, Concatenate, Reshape, Permute, DepthwiseConv2D
+from model_compression_toolkit.core import common
+from model_compression_toolkit.core.common.graph.base_graph import Graph, BaseNode, OutTensor
+from model_compression_toolkit.core.common.graph.functional_node import FunctionalNode
+from model_compression_toolkit.core.common.graph.graph_matchers import NodeOperationMatcher
+from model_compression_toolkit.constants import REUSE, REUSE_GROUP
+from model_compression_toolkit.core.keras.reader.node_builder import REUSED_IDENTIFIER
+from model_compression_toolkit.core.keras.constants import KERNEL, BIAS, USE_BIAS, NUM_HEADS, KEY_DIM, VALUE_DIM, \
+    QUERY_SHAPE, KEY_SHAPE, VALUE_SHAPE, OUTPUT_SHAPE, ATTENTION_AXES, ACTIVATION, GROUPS, LINEAR, FILTERS, PADDING, \
+    FUNCTION, DIMS, TARGET_SHAPE, F_STRIDED_SLICE, F_STACK, Q_KERNEL, Q_BIAS, K_KERNEL, K_BIAS, V_KERNEL, V_BIAS, \
+    OUTPUT_KERNEL, OUTPUT_BIAS, F_MATMUL, TRANSPOSE_B, KERNEL_SIZE, AXIS, F_STRIDED_SLICE_BEGIN, F_STRIDED_SLICE_END, \
+    DEPTH_MULTIPLIER, DEPTHWISE_INITIALIZER, DEPTHWISE_REGULARIZER, DEPTHWISE_CONSTRAINT, KERNEL_INITIALIZER, \
+    KERNEL_REGULARIZER, KERNEL_CONSTRAINT
+
+
+class DwconvToConv(common.BaseSubstitution):
+    """
+    A substitution class for replacing DepthwiseConv2D layers with Conv2D layers having 'groups' equal to the number of
+    input channels.
+    """
+
+    def __init__(self):
+        """
+        Initializes the DwconvToConv substitution
+        """
+        super().__init__(matcher_instance=NodeOperationMatcher(DepthwiseConv2D))
+
+    @staticmethod
+    def _get_weight_by_name(node, w_str):
+        """
+        Retrieve the weight with a given name from the node.
+
+        Args:
+            node: The node containing weights.
+            w_str: The name of the weight to retrieve.
+
+        Returns:
+            The weight with the specified name or None if not found.
+        """
+        w = [k for k in node.weights.keys() if w_str in k]
+        return node.weights[w[0]]
+
+    def substitute(self,
+                   graph: Graph,
+                   dwconv_node: BaseNode) -> Graph:
+        """
+        Replace a DepthwiseConv2D layer with a Conv2D layer, setting 'groups' parameter to the number of input channels.
+
+        Args:
+            graph: The graph on which the substitution is applied.
+            dwconv_node: The DepthwiseConv2D node to be replaced.
+
+        Returns:
+            The modified graph after applying the substitution.
+        """
+
+        # Skip in case mult depth_multiplier=1
+        if dwconv_node.framework_attr[DEPTH_MULTIPLIER] == 1:
+            return graph
+
+        # Build the new node
+        k = self._get_weight_by_name(dwconv_node, KERNEL).copy()
+        k_shape = k.shape
+        filters = k_shape[2] * k_shape[3] # k_shape[2] * k_shape[3] = number of output channels
+
+        # Transform the DepthwiseConv2D kernel to match the Conv2D kernel, where each input channel is convolved with
+        # 'depth_multiplier' filters.
+        k = np.reshape(k,[k_shape[0], k_shape[1], 1, filters])
+        _reuse_params = {REUSE: dwconv_node.reuse, REUSE_GROUP: dwconv_node.reuse_group}
+
+        conv_fw_attr = dwconv_node.framework_attr
+        conv_fw_attr.update({FILTERS: filters,
+                               GROUPS: k_shape[2],
+                               KERNEL_INITIALIZER: dwconv_node.framework_attr[DEPTHWISE_INITIALIZER],
+                               KERNEL_REGULARIZER: dwconv_node.framework_attr[DEPTHWISE_REGULARIZER],
+                               KERNEL_CONSTRAINT: dwconv_node.framework_attr[DEPTHWISE_CONSTRAINT]})
+
+        conv_fw_attr.pop(DEPTH_MULTIPLIER)
+        conv_fw_attr.pop(DEPTHWISE_INITIALIZER)
+        conv_fw_attr.pop(DEPTHWISE_REGULARIZER)
+        conv_fw_attr.pop(DEPTHWISE_CONSTRAINT)
+
+        conv_weights = {KERNEL: k}
+        if conv_fw_attr[USE_BIAS]:
+            b = self._get_weight_by_name(dwconv_node, BIAS).copy()
+            conv_weights.update({BIAS: b})
+
+        conv_node = BaseNode(dwconv_node.name, conv_fw_attr, dwconv_node.input_shape, dwconv_node.output_shape,
+                             conv_weights, Conv2D,
+                             **_reuse_params)
+
+        graph.add_node(conv_node)
+
+        # Replace DWconv node with Conv node
+        _in_edge = list(graph.in_edges(dwconv_node))[0]
+        _out_edges = graph.out_edges(dwconv_node)
+        graph.add_edge(_in_edge[0], conv_node, **graph.get_edge_data(*_in_edge, 0))
+        graph.remove_edge(_in_edge[0], dwconv_node)
+        graph.reconnect_out_edges(current_node=dwconv_node, new_node=conv_node)
+
+        # Finally, remove the DWconv node
+        graph.remove_node(dwconv_node, new_graph_outputs=[OutTensor(conv_node, 0)])
+
+        return graph

model_compression_toolkit/core/keras/keras_implementation.py

@@ -91,6 +91,7 @@ from model_compression_toolkit.core.keras.graph_substitutions.substitutions.sepa
     SeparableConvDecomposition, DEPTH_MULTIPLIER
 from model_compression_toolkit.core.keras.graph_substitutions.substitutions.shift_negative_activation import \
     keras_apply_shift_negative_correction
+from model_compression_toolkit.core.keras.graph_substitutions.substitutions.dwconv_to_conv import DwconvToConv
 from model_compression_toolkit.core.keras.keras_node_prior_info import create_node_prior_info
 from model_compression_toolkit.core.keras.reader.reader import model_reader
 from model_compression_toolkit.core.common.collectors.statistics_collector_generator import \
@@ -259,7 +260,8 @@ class KerasImplementation(FrameworkImplementation):
         """
         return [SeparableConvDecomposition(),
                 MultiHeadAttentionDecomposition(),
-                ActivationDecomposition()]
+                ActivationDecomposition(),
+                DwconvToConv()]
 
     def get_substitutions_pre_statistics_collection(self, quant_config: QuantizationConfig) -> \
             List[common.BaseSubstitution]:
@@ -591,19 +593,3 @@ class KerasImplementation(FrameworkImplementation):
         """
 
         return model(inputs)
-
-    def sample_single_representative_dataset(self, representative_dataset: Callable):
-        """
-        Get a single sample (namely, batch size of 1) from a representative dataset.
-
-        Args:
-            representative_dataset: Callable which returns the representative dataset at any batch size.
-
-        Returns: List of inputs from representative_dataset where each sample has a batch size of 1.
-        """
-        images = next(representative_dataset())
-        if not isinstance(images, list):
-            Logger.error(f'Images expected to be a list but is of type {type(images)}')
-
-        # Ensure each image is a single sample, if not, take the first sample
-        return [tf.expand_dims(image[0], 0) if image.shape[0] != 1 else image for image in images]

model_compression_toolkit/core/pytorch/pytorch_implementation.py

@@ -540,18 +540,3 @@ class PytorchImplementation(FrameworkImplementation):
                                                         fw_impl=self,
                                                         num_iterations_for_approximation=num_iterations_for_approximation)
 
-    def sample_single_representative_dataset(self, representative_dataset: Callable):
-        """
-        Get a single sample (namely, batch size of 1) from a representative dataset.
-
-        Args:
-            representative_dataset: Callable which returns the representative dataset at any batch size.
-
-        Returns: List of inputs from representative_dataset where each sample has a batch size of 1.
-        """
-        images = next(representative_dataset())
-        if not isinstance(images, list):
-            Logger.error(f'Images expected to be a list but is of type {type(images)}')
-
-        # Ensure each image is a single sample, if not, take the first sample
-        return [torch.unsqueeze(image[0], 0) if image.shape[0] != 1 else image for image in images]

model_compression_toolkit/qat/common/qat_config.py

@@ -45,9 +45,12 @@ class TrainingMethod(Enum):
 
     DQA -  DNN Quantization with Attention. Includes a smooth quantization introduces by DQA method
 
+    LSQ - Learned Step size Quantization. Includes PowerOfTwo, symmetric & uniform quantizers: https://arxiv.org/pdf/1902.08153.pdf
+
     """
     STE = "STE",
-    DQA = "DQA"
+    DQA = "DQA",
+    LSQ = "LSQ"
 
 
 class QATConfig:

model_compression_toolkit/qat/keras/quantizer/__init__.py

@@ -15,3 +15,5 @@
 
 import model_compression_toolkit.qat.keras.quantizer.ste_rounding.symmetric_ste
 import model_compression_toolkit.qat.keras.quantizer.ste_rounding.uniform_ste
+import model_compression_toolkit.qat.keras.quantizer.lsq.symmetric_lsq
+import model_compression_toolkit.qat.keras.quantizer.lsq.uniform_lsq

model_compression_toolkit/qat/keras/quantizer/lsq/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================

model_compression_toolkit/qat/keras/quantizer/lsq/symmetric_lsq.py

@@ -0,0 +1,254 @@
+# Copyright 2023 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from typing import Union
+
+import numpy as np
+import tensorflow as tf
+from tensorflow.python.framework.tensor_shape import TensorShape
+from model_compression_toolkit.constants import SIGNED
+
+from model_compression_toolkit.qat import TrainingMethod
+
+from model_compression_toolkit.target_platform_capabilities.target_platform import QuantizationMethod
+from model_compression_toolkit.trainable_infrastructure import KerasTrainableQuantizationWrapper
+from mct_quantizers import QuantizationTarget, mark_quantizer
+from model_compression_toolkit.qat.common import THRESHOLD_TENSOR
+from model_compression_toolkit import constants as C
+
+from model_compression_toolkit.qat.keras.quantizer.base_keras_qat_quantizer import BaseKerasQATTrainableQuantizer
+from model_compression_toolkit.trainable_infrastructure import TrainableQuantizerWeightsConfig, \
+    TrainableQuantizerActivationConfig
+from mct_quantizers.keras.quantizers import WeightsPOTInferableQuantizer, WeightsSymmetricInferableQuantizer, \
+    ActivationPOTInferableQuantizer, ActivationSymmetricInferableQuantizer
+from model_compression_toolkit.trainable_infrastructure.common.base_trainable_quantizer import VariableGroup
+from model_compression_toolkit.qat.keras.quantizer.quant_utils import ste_round, grad_scale
+
+
+def symmetric_lsq_quantizer(x: tf.Tensor,
+                            thresholds: tf.Tensor,
+                            num_bits: int,
+                            sign: bool,
+                            min_int: int,
+                            max_int:int,
+                            scale_factor: float) -> tf.Tensor:
+    """
+    Symmetric quantizer according to LSQ algorithm: https://arxiv.org/pdf/1902.08153.pdf
+    Args:
+        x: input to quantize
+        thresholds: thresholds of quantization levels
+        num_bits: number of bits for quantization
+        sign: whether x is signed or not
+        min_int: min clipping integer value
+        max_int: max clipping integer value
+        scale_factor: grad scale of LSQ algorithm
+    Returns:
+        A quantized tensor
+    """
+    delta = thresholds / (2 ** (num_bits - int(sign)))
+    delta_scaled = grad_scale(delta, scale_factor)
+    rounded = ste_round(x / delta_scaled)
+    clipped = tf.math.minimum(tf.math.maximum(rounded, min_int), max_int)
+    quantized = delta_scaled * clipped
+    return quantized
+
+
+@mark_quantizer(quantization_target=QuantizationTarget.Weights,
+                quantization_method=[QuantizationMethod.POWER_OF_TWO, QuantizationMethod.SYMMETRIC],
+                identifier=TrainingMethod.LSQ)
+class LSQWeightQATQuantizer(BaseKerasQATTrainableQuantizer):
+    """
+    Trainable constrained quantizer to quantize layer's weights.
+    """
+
+    def __init__(self, quantization_config: TrainableQuantizerWeightsConfig):
+        """
+        Initialize a LSQWeightQATQuantizer object with parameters to use
+        for the quantization.
+
+        Args:
+            quantization_config: trainable quantizer config class
+        """
+        super().__init__(quantization_config)
+        self.power_of_two = quantization_config.weights_quantization_method == QuantizationMethod.POWER_OF_TWO
+        self.threshold_values = np.array(quantization_config.weights_quantization_params[C.THRESHOLD])
+        self.threshold_shape = self.threshold_values.shape
+        self.per_channel = self.quantization_config.weights_per_channel_threshold
+        self.channel_axis = self.quantization_config.weights_channels_axis
+        self.threshold_values = np.reshape(np.asarray(self.threshold_values), [-1]) if self.per_channel else float(self.threshold_values)
+        self.num_bits = self.quantization_config.weights_n_bits
+        n_pos_bits = self.num_bits - int(C.WEIGHTS_SIGNED)
+        self.min_int = -int(C.WEIGHTS_SIGNED) * (2 ** n_pos_bits)
+        self.max_int = 2 **n_pos_bits - 1
+        self.scale_factor = 1.0 / np.sqrt(self.max_int * self.threshold_values.size)
+        if self.power_of_two:
+            self.threshold_values = np.power(2.0, np.ceil(np.log2(np.maximum(self.threshold_values, C.MIN_THRESHOLD))))
+
+    def initialize_quantization(self,
+                                tensor_shape: TensorShape,
+                                name: str,
+                                layer: KerasTrainableQuantizationWrapper):
+        """
+        Add quantizer parameters to the quantizer parameters dictionary
+
+        Args:
+            tensor_shape: tensor shape of the quantized tensor.
+            name: Tensor name.
+            layer: Layer to quantize.
+        """
+        ptq_threshold_tensor = layer.add_weight(
+            name + THRESHOLD_TENSOR,
+            shape=len(self.threshold_values) if self.per_channel else (),
+            initializer=tf.keras.initializers.Constant(1.0),
+            trainable=True)
+        ptq_threshold_tensor.assign(self.threshold_values)
+
+        # save the quantizer added parameters for later calculations
+        self.add_quantizer_variable(THRESHOLD_TENSOR, ptq_threshold_tensor, VariableGroup.QPARAMS)
+
+    def __call__(self,
+                 inputs: tf.Tensor,
+                 training: bool):
+        """
+        Quantize a tensor.
+        Args:
+            inputs: Input tensor to quantize.
+            training: Whether the graph is in training mode.
+            weights: Dictionary of weights the quantizer can use to quantize the tensor.
+            **kwargs: Additional variables the quantizer may receive.
+
+        Returns:
+            The quantized tensor.
+        """
+
+        thresholds = self.get_quantizer_variable(THRESHOLD_TENSOR)
+        q_tensor = symmetric_lsq_quantizer(inputs, thresholds, self.num_bits, C.WEIGHTS_SIGNED, self.min_int, self.max_int, self.scale_factor)
+        return q_tensor
+
+    def convert2inferable(self) -> Union[WeightsPOTInferableQuantizer, WeightsSymmetricInferableQuantizer]:
+        """
+        Convert quantizer to inferable quantizer.
+
+        Returns:
+            BaseKerasInferableQuantizer object.
+        """
+        if self.power_of_two:
+            thresholds = 2 ** np.ceil(np.log2(self.get_quantizer_variable(THRESHOLD_TENSOR).numpy()))
+            return WeightsPOTInferableQuantizer(num_bits=self.num_bits,
+                                                threshold=list(thresholds.flatten()),
+                                                per_channel=self.per_channel,
+                                                channel_axis=self.channel_axis,
+                                                input_rank=len(self.threshold_shape))
+        else:
+            thresholds = self.get_quantizer_variable(THRESHOLD_TENSOR).numpy()
+            return WeightsSymmetricInferableQuantizer(num_bits=self.num_bits,
+                                                      threshold=list(thresholds.flatten()),
+                                                      per_channel=self.per_channel,
+                                                      channel_axis=self.channel_axis,
+                                                      input_rank=len(self.threshold_shape))
+
+
+@mark_quantizer(quantization_target=QuantizationTarget.Activation,
+                quantization_method=[QuantizationMethod.POWER_OF_TWO, QuantizationMethod.SYMMETRIC],
+                identifier=TrainingMethod.LSQ)
+class LSQActivationQATQuantizer(BaseKerasQATTrainableQuantizer):
+    """
+    Trainable constrained quantizer to quantize layer activations.
+    """
+
+    def __init__(self, quantization_config: TrainableQuantizerActivationConfig):
+        """
+        Initialize a LSQActivationQATQuantizer object with parameters to use
+        for the quantization.
+
+        Args:
+            quantization_config: trainable quantizer config class
+        """
+        super().__init__(quantization_config)
+        self.power_of_two = quantization_config.activation_quantization_method == QuantizationMethod.POWER_OF_TWO
+        self.threshold_values = float(quantization_config.activation_quantization_params[C.THRESHOLD])
+        self.threshold_shape = np.asarray(self.threshold_values).shape
+        self.sign = quantization_config.activation_quantization_params[SIGNED]
+        self.num_bits = quantization_config.activation_n_bits
+        n_pos_bits = self.num_bits - int(self.sign)
+        self.min_int = -int(self.sign) * (2 ** n_pos_bits)
+        self.max_int = (2 ** n_pos_bits) - 1
+        if self.power_of_two:
+            self.threshold_values = np.power(2.0, np.ceil(np.log2(np.maximum(self.threshold_values, C.MIN_THRESHOLD))))
+
+
+    def initialize_quantization(self,
+                                tensor_shape: TensorShape,
+                                name: str,
+                                layer: KerasTrainableQuantizationWrapper):
+        """
+        Add quantizer parameters to the quantizer parameters dictionary
+
+        Args:
+            tensor_shape: tensor shape of the quantized tensor.
+            name: Tensor name.
+            layer: Layer to quantize.
+        """
+        ptq_threshold_tensor = layer.add_weight(
+            name + THRESHOLD_TENSOR,
+            shape=(),
+            initializer=tf.keras.initializers.Constant(1.0),
+            trainable=True)
+        ptq_threshold_tensor.assign(self.threshold_values)
+
+        # save the quantizer added parameters for later calculations
+        self.add_quantizer_variable(THRESHOLD_TENSOR, ptq_threshold_tensor, VariableGroup.QPARAMS)
+
+    def __call__(self,
+                 inputs: tf.Tensor,
+                 training: bool):
+        """
+        Quantize a tensor.
+        Args:
+            inputs: Input tensor to quantize.
+            training: Whether the graph is in training mode.
+
+        Returns:
+            The quantized tensor.
+        """
+
+        thresholds = self.get_quantizer_variable(THRESHOLD_TENSOR)
+        n_channels = inputs.shape[-1]
+        scale_factor = 1.0 / np.sqrt(self.max_int * n_channels)
+        q_tensor = symmetric_lsq_quantizer(inputs, thresholds, self.num_bits, self.sign, self.min_int, self.max_int, scale_factor)
+        return q_tensor
+
+    def convert2inferable(self) -> Union[ActivationPOTInferableQuantizer, ActivationSymmetricInferableQuantizer]:
+        """
+        Convert quantizer to inferable quantizer.
+
+        Returns:
+            BaseKerasInferableQuantizer object.
+        """
+
+        if self.power_of_two:
+            thresholds = 2 ** np.ceil(np.log2(self.get_quantizer_variable(THRESHOLD_TENSOR).numpy()))
+            return ActivationPOTInferableQuantizer(num_bits=self.num_bits,
+                                                   # In activation quantization is per-tensor only - thus we pass
+                                                   # the threshold as a list with a len of 1
+                                                   threshold=[thresholds],
+                                                   signed=self.sign)
+        else:
+            thresholds = self.get_quantizer_variable(THRESHOLD_TENSOR).numpy()
+            return ActivationSymmetricInferableQuantizer(num_bits=self.num_bits,
+                                                         # In activation quantization is per-tensor only - thus we
+                                                         # pass the threshold as a list with a len of 1
+                                                         threshold=[thresholds],
+                                                         signed=self.sign)