mct-nightly 2.2.0.20241012.448__py3-none-any.whl → 2.2.0.20241018.449__py3-none-any.whl

model_compression_toolkit/trainable_infrastructure/keras/activation_quantizers/ste/uniform_ste.py

@@ -0,0 +1,122 @@
+# Copyright 2022 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import tensorflow as tf
+from tensorflow.python.framework.tensor_shape import TensorShape
+from model_compression_toolkit.trainable_infrastructure.common.constants import FQ_MIN, FQ_MAX
+from model_compression_toolkit.trainable_infrastructure import KerasTrainableQuantizationWrapper
+from model_compression_toolkit.trainable_infrastructure import TrainingMethod
+
+from mct_quantizers import mark_quantizer, QuantizationMethod, QuantizationTarget
+from mct_quantizers.keras.quantizers import BaseKerasInferableQuantizer, ActivationUniformInferableQuantizer
+
+from model_compression_toolkit.qat.keras.quantizer.quant_utils import adjust_range_to_include_zero
+from model_compression_toolkit.core.common.quantization.quantizers.quantizers_helpers import fix_range_to_include_zero
+from model_compression_toolkit import constants as C
+
+from model_compression_toolkit.trainable_infrastructure import TrainableQuantizerActivationConfig
+from model_compression_toolkit.trainable_infrastructure.common.base_trainable_quantizer import VariableGroup
+from model_compression_toolkit.trainable_infrastructure.keras.activation_quantizers import BaseKerasActivationTrainableQuantizer
+
+
+@mark_quantizer(quantization_target=QuantizationTarget.Activation,
+                quantization_method=[QuantizationMethod.UNIFORM],
+                identifier=TrainingMethod.STE)
+class STEUniformActivationTrainableQuantizer(BaseKerasActivationTrainableQuantizer):
+    """
+    Trainable constrained quantizer to quantize a layer outputs.
+    """
+
+    def __init__(self, quantization_config: TrainableQuantizerActivationConfig, freeze_quant_params: bool = False):
+        """
+        Initialize a STEUniformActivationTrainableQuantizer object with parameters to use
+        for the quantization.
+
+        Args:
+            quantization_config: trainable quantizer config class
+            freeze_quant_params: whether to freeze learnable quantization parameters. This is unused here, since there is not any quantizaiton params that are learned.
+
+        """
+        super().__init__(quantization_config, freeze_quant_params)
+
+        self.num_bits = quantization_config.activation_n_bits
+        self.min_range = quantization_config.activation_quantization_params[C.RANGE_MIN]
+        self.max_range = quantization_config.activation_quantization_params[C.RANGE_MAX]
+
+    def initialize_quantization(self,
+                                tensor_shape: TensorShape,
+                                name: str,
+                                layer: KerasTrainableQuantizationWrapper):
+        """
+        Add quantizer parameters to the quantizer parameters dictionary
+
+        Args:
+            tensor_shape: tensor shape of the quantized tensor.
+            name: Tensor name.
+            layer: Layer to quantize.
+        """
+        fq_min = layer.add_weight(
+            name + FQ_MIN,
+            shape=(),
+            initializer=tf.keras.initializers.Constant(-1.0),
+            trainable=False)
+        fq_min.assign(self.min_range)
+
+        fq_max = layer.add_weight(
+            name + FQ_MAX,
+            shape=(),
+            initializer=tf.keras.initializers.Constant(1.0),
+            trainable=False)
+        fq_max.assign(self.max_range)
+
+        # save the quantizer added parameters for later calculations
+        self.add_quantizer_variable(FQ_MIN, fq_min, VariableGroup.QPARAMS)
+        self.add_quantizer_variable(FQ_MAX, fq_max, VariableGroup.QPARAMS)
+
+    def __call__(self,
+                 inputs: tf.Tensor,
+                 training: bool):
+        """
+        Quantize a tensor.
+        Args:
+            inputs: Input tensor to quantize.
+            training: Whether the graph is in training mode.
+
+        Returns:
+            The quantized tensor.
+        """
+
+        _min = self.get_quantizer_variable(FQ_MIN)
+        _max = self.get_quantizer_variable(FQ_MAX)
+        _min, _max = adjust_range_to_include_zero(_min, _max, self.num_bits)
+        q_tensor = tf.quantization.fake_quant_with_min_max_vars(inputs, _min, _max,
+                                                                num_bits=self.num_bits)
+
+        return q_tensor
+
+    def convert2inferable(self) -> BaseKerasInferableQuantizer:
+        """
+        Convert quantizer to inferable quantizer.
+
+        Returns:
+            BaseKerasInferableQuantizer object.
+        """
+        min_range, max_range = fix_range_to_include_zero(self.get_quantizer_variable(FQ_MIN).numpy(),
+                                                         self.get_quantizer_variable(FQ_MAX).numpy(),
+                                                         self.num_bits)
+        return ActivationUniformInferableQuantizer(num_bits=self.num_bits,
+                                                   # In activation quantization is per-tensor only - thus we pass
+                                                   # the min/max as lists with a len of 1
+                                                   min_range=[min_range],
+                                                   max_range=[max_range])

model_compression_toolkit/trainable_infrastructure/keras/base_keras_quantizer.py

@@ -28,16 +28,10 @@ if FOUND_TF:
     import tensorflow as tf
 
     class BaseKerasTrainableQuantizer(BaseTrainableQuantizer):
-        def __init__(self,
-                     quantization_config: Union[TrainableQuantizerWeightsConfig, TrainableQuantizerActivationConfig]):
-            """
-            This class is a base quantizer which validates provided quantization config and defines an abstract function which any quantizer needs to implement.
-            This class adds to the base quantizer a get_config and from_config functions to enable loading and saving the keras model.
-
-            Args:
-                quantization_config: quantizer config class contains all the information about a quantizer configuration.
-            """
-            super().__init__(quantization_config)
+        """
+        This class is a base quantizer which validates provided quantization config and defines an abstract function which any quantizer needs to implement.
+        This class adds to the base quantizer a get_config and from_config functions to enable loading and saving the keras model.
+        """
 
         def get_config(self) -> Dict[str, Any]:
             """
@@ -77,6 +71,14 @@ if FOUND_TF:
                 quantizer_parameter, parameter_group = parameter_dict[VAR], parameter_dict[GROUP]
                 if quantizer_parameter.trainable and parameter_group == group:
                     quantizer_trainable.append(quantizer_parameter)
+
+            # sanity check to catch inconsistent initialization
+            if self.freeze_quant_params and group == VariableGroup.QPARAMS and quantizer_trainable:
+                Logger.critical(
+                    'Found trainable quantization params despite self.freeze_quant_params=True. '
+                    'Quantization parameters were probably not initialized correctly in the Quantizer.'
+                )  # pragma: no cover
+
             return quantizer_trainable
 
 

model_compression_toolkit/trainable_infrastructure/keras/quantizer_utils.py

@@ -16,6 +16,9 @@
 import numpy as np
 import tensorflow as tf
 
+from model_compression_toolkit.qat.keras.quantizer.quant_utils import grad_scale, ste_round, \
+    adjust_range_to_include_zero
+
 
 def int_quantization_with_threshold(data: tf.Tensor,
                                     n_bits: int,
@@ -46,3 +49,60 @@ def int_quantization_with_threshold(data: tf.Tensor,
 
     return tf.clip_by_value((data / (threshold + eps)) * (2 ** (n_bits - int(signed))),
                             clip_value_max=clip_max, clip_value_min=clip_min)
+
+
+def symmetric_lsq_quantizer(x: tf.Tensor,
+                            thresholds: tf.Tensor,
+                            num_bits: int,
+                            sign: bool,
+                            min_int: int,
+                            max_int:int,
+                            scale_factor: float) -> tf.Tensor:
+    """
+    Symmetric quantizer according to LSQ algorithm: https://arxiv.org/pdf/1902.08153.pdf
+    Args:
+        x: input to quantize
+        thresholds: thresholds of quantization levels
+        num_bits: number of bits for quantization
+        sign: whether x is signed or not
+        min_int: min clipping integer value
+        max_int: max clipping integer value
+        scale_factor: grad scale of LSQ algorithm
+    Returns:
+        A quantized tensor
+    """
+    delta = thresholds / (2 ** (num_bits - int(sign)))
+    delta_scaled = grad_scale(delta, scale_factor)
+    rounded = ste_round(x / delta_scaled)
+    clipped = tf.math.minimum(tf.math.maximum(rounded, min_int), max_int)
+    quantized = delta_scaled * clipped
+    return quantized
+
+
+def uniform_lsq_quantizer(x: tf.Tensor,
+                          min_range: tf.Tensor,
+                          max_range: tf.Tensor,
+                          num_bits: int,
+                          min_int: int,
+                          max_int:int,
+                          scale_factor: float) -> tf.Tensor:
+    """
+    Uniform quantizer according to LSQ algorithm: https://arxiv.org/pdf/1902.08153.pdf
+    Args:
+        x: input to quantize
+        min_range: min range of quantization values
+        max_range: min range of quantization values
+        num_bits: number of bits for quantization
+        min_int: min clipping integer value
+        max_int: max clipping integer value
+        scale_factor: grad scale of LSQ algorithm
+    Returns:
+        A quantized tensor
+    """
+    min_range, max_range = adjust_range_to_include_zero(min_range, max_range, num_bits)
+    delta = (max_range - min_range) / (2 ** num_bits - 1)
+    delta_scaled = grad_scale(delta, scale_factor)
+    rounded = ste_round((x-min_range) / delta_scaled)
+    clipped = tf.math.minimum(tf.math.maximum(rounded, min_int), max_int)
+    quantized = delta_scaled * clipped + min_range
+    return quantized