mct-nightly 2.4.0.20250925.543__py3-none-any.whl → 2.4.2.20250926.532__py3-none-any.whl

model_compression_toolkit/xquant/pytorch/core_report_generator.py

@@ -0,0 +1,177 @@
+#  Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ==============================================================================
+from tqdm import tqdm
+from typing import Callable, Any, Dict
+
+from model_compression_toolkit.core.common.model_collector import ModelCollector
+from model_compression_toolkit.xquant import XQuantConfig
+from model_compression_toolkit.xquant.common.constants import OUTPUT_SIMILARITY_METRICS_REPR, OUTPUT_SIMILARITY_METRICS_VAL, INTERMEDIATE_SIMILARITY_METRICS_REPR, \
+    INTERMEDIATE_SIMILARITY_METRICS_VAL
+from model_compression_toolkit.xquant.pytorch.framework_report_utils import FrameworkReportUtils
+
+from model_compression_toolkit.xquant.pytorch.core_detect_degrade_layer import core_detect_degrade_layer
+from model_compression_toolkit.xquant.pytorch.core_judge_troubleshoot import core_judge_troubleshoot
+
+def core_report_generator(float_model: Any,
+                          quantized_model: Any,
+                          repr_dataset: Callable,
+                          validation_dataset: Callable,
+                          fw_report_utils: FrameworkReportUtils,
+                          xquant_config: XQuantConfig) -> Dict[str, Any]:
+    """
+    Generate report in tensorboard with a graph of the quantized model and similarity metrics that
+    have been measured when comparing to the float model (or any other two models).
+    The report also contains histograms that are collected on the baseline model (usually, the float
+    model).
+
+    Args:
+        float_model (Any): The original floating-point model.
+        quantized_model (Any): The model after quantization.
+        repr_dataset (Callable): Representative dataset used for similarity metrics computation.
+        validation_dataset (Callable): Validation dataset used for similarity metrics computation.
+        fw_report_utils (FrameworkReportUtils): Utilities for generating framework-specific reports.
+        xquant_config (XQuantConfig): Configuration settings for explainable quantization.
+
+    Returns:
+        Dict[str, Any]: A dictionary containing the collected similarity metrics and report data.
+    """
+    # Get metadata from the quantized model
+    quantized_model_metadata = fw_report_utils.get_metadata_fn(quantized_model)
+
+    # Collect histograms on the float model.
+    float_graph = fw_report_utils.model_folding_utils.create_float_folded_graph(float_model, repr_dataset)
+    mi = ModelCollector(float_graph, fw_report_utils.fw_impl, fw_report_utils.fw_info)
+    for _data in tqdm(repr_dataset(), desc="Collecting Histograms"):
+        mi.infer(_data)
+
+    # Collect histograms and add them to Tensorboard.
+    fw_report_utils.tb_utils.add_histograms_to_tensorboard(graph=float_graph)
+
+    # Compute similarity metrics on representative dataset and validation set.
+    repr_similarity = fw_report_utils.similarity_calculator.compute_similarity_metrics(float_model=float_model,
+                                                                                       quantized_model=quantized_model,
+                                                                                       dataset=repr_dataset,
+                                                                                       custom_similarity_metrics=xquant_config.custom_similarity_metrics)
+    val_similarity = fw_report_utils.similarity_calculator.compute_similarity_metrics(float_model=float_model,
+                                                                                      quantized_model=quantized_model,
+                                                                                      dataset=validation_dataset,
+                                                                                      custom_similarity_metrics=xquant_config.custom_similarity_metrics,
+                                                                                      is_validation=True)
+    similarity_metrics = {
+        OUTPUT_SIMILARITY_METRICS_REPR: repr_similarity[0],
+        OUTPUT_SIMILARITY_METRICS_VAL: val_similarity[0],
+        INTERMEDIATE_SIMILARITY_METRICS_REPR: repr_similarity[1],
+        INTERMEDIATE_SIMILARITY_METRICS_VAL: val_similarity[1]
+    }
+
+    # Add a graph of the quantized model with the similarity metrics to TensorBoard for visualization.
+    fw_report_utils.tb_utils.add_graph_to_tensorboard(quantized_model,
+                                                      similarity_metrics,
+                                                      repr_dataset,
+                                                      quantized_model_metadata)
+
+    # Adds text information (like max cut and output similarity metrics) to the tensorboard writer.
+    fw_report_utils.tb_utils.add_text_information(similarity_metrics,
+                                                  quantized_model_metadata)
+
+    # Save data to a json file.
+    fw_report_utils.dump_report_to_json(report_dir=xquant_config.report_dir,
+                                        collected_data=similarity_metrics)
+
+    return similarity_metrics
+
+def core_report_generator_troubleshoot(float_model: Any,
+                                       quantized_model: Any,
+                                       repr_dataset: Callable,
+                                       validation_dataset: Callable,
+                                       fw_report_utils: FrameworkReportUtils,
+                                       xquant_config: XQuantConfig) -> Dict[str, Any]:
+    """
+    Generate report in tensorboard with a graph of the quantized model and similarity metrics that
+    have been measured when comparing to the float model (or any other two models).
+    The report also contains histograms that are collected on the baseline model (usually, the float
+    model).
+    Furthermore, detect degraded layers by quantize similarities and analyze degrade causes by float_model and quantized_model.
+
+    Args:
+        float_model (Any): The original floating-point model.
+        quantized_model (Any): The model after quantization.
+        repr_dataset (Callable): Representative dataset used for similarity metrics computation.
+        validation_dataset (Callable): Validation dataset used for similarity metrics computation.
+        fw_report_utils (FrameworkReportUtils): Utilities for generating framework-specific reports.
+        xquant_config (XQuantConfig): Configuration settings for explainable quantization.
+
+    Returns:
+        Dict[str, Any]: A dictionary containing the collected similarity metrics and report data.
+        Dict[str, Any]: A dictionary containing the analyze degrade cause report for degraded layaers.
+    """
+    # Get metadata from the quantized model
+    quantized_model_metadata = fw_report_utils.get_metadata_fn(quantized_model)
+
+    # Collect histograms on the float model.
+    float_graph = fw_report_utils.model_folding_utils.create_float_folded_graph(float_model, repr_dataset)
+    mi = ModelCollector(float_graph, fw_report_utils.fw_impl, fw_report_utils.fw_info)
+    for _data in tqdm(repr_dataset(), desc="Collecting Histograms"):
+        mi.infer(_data)
+
+    # Collect histograms and add them to Tensorboard.
+    fw_report_utils.tb_utils.add_histograms_to_tensorboard(graph=float_graph)
+
+    # Compute similarity metrics on representative dataset and validation set.
+    repr_similarity = fw_report_utils.similarity_calculator.compute_similarity_metrics(float_model=float_model,
+                                                                                       quantized_model=quantized_model,
+                                                                                       dataset=repr_dataset,
+                                                                                       custom_similarity_metrics=xquant_config.custom_similarity_metrics)
+    val_similarity = fw_report_utils.similarity_calculator.compute_similarity_metrics(float_model=float_model,
+                                                                                      quantized_model=quantized_model,
+                                                                                      dataset=validation_dataset,
+                                                                                      custom_similarity_metrics=xquant_config.custom_similarity_metrics,
+                                                                                      is_validation=True)
+    similarity_metrics = {
+        OUTPUT_SIMILARITY_METRICS_REPR: repr_similarity[0],
+        OUTPUT_SIMILARITY_METRICS_VAL: val_similarity[0],
+        INTERMEDIATE_SIMILARITY_METRICS_REPR: repr_similarity[1],
+        INTERMEDIATE_SIMILARITY_METRICS_VAL: val_similarity[1]
+    }
+
+    # Add a graph of the quantized model with the similarity metrics to TensorBoard for visualization.
+    fw_report_utils.tb_utils.add_graph_to_tensorboard(quantized_model,
+                                                      similarity_metrics,
+                                                      repr_dataset,
+                                                      quantized_model_metadata)
+
+    # Adds text information (like max cut and output similarity metrics) to the tensorboard writer.
+    fw_report_utils.tb_utils.add_text_information(similarity_metrics,
+                                                  quantized_model_metadata)
+
+    # Save data to a json file.
+    fw_report_utils.dump_report_to_json(report_dir=xquant_config.report_dir,
+                                        collected_data=similarity_metrics)
+
+    # Detect degraded layers by quantize similarities
+    degrade_layers = core_detect_degrade_layer(repr_similarity, val_similarity, xquant_config)
+
+    # Analyze degrade causes by float_model and quantized_model. (If degrade_layer_ratio < threshold)
+    degrade_layer_ratio = len(degrade_layers) / len(repr_similarity[1].keys())
+    if(degrade_layer_ratio < xquant_config.threshold_degrade_layer_ratio):
+        _troubleshoot_data = core_judge_troubleshoot(float_model, quantized_model, float_graph, degrade_layers, repr_dataset, xquant_config)
+    else:
+        _troubleshoot_data = {}
+
+    # Save data to a json file.
+    fw_report_utils.dump_troubleshoot_report_to_json(report_dir=xquant_config.report_dir,
+                                                     collected_data=_troubleshoot_data)
+
+    return similarity_metrics, _troubleshoot_data

model_compression_toolkit/xquant/pytorch/detect_degrade_utils.py

@@ -0,0 +1,78 @@
+#  Copyright 2025 Sony Semiconductor Solutions. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ==============================================================================
+
+from model_compression_toolkit.xquant import XQuantConfig
+import matplotlib.pyplot as plt
+
+def make_similarity_graph(metrics_name: str,
+                          dataset_name: str,
+                          intermediate_similarity: dict,
+                          degrade_layers: list[str],
+                          xquant_config: XQuantConfig) -> None:
+    """
+    Detect degrade layers by caliculated similarities by XQuant.
+    And Draw and save similarity graphs.
+
+    Args:
+        metrics_name (str): Metrics name of caluclualting quantized error. (i.g. 'mse') 
+        dataset_name (str): Dataset name ('repr'(Represantation dataset) or 'val'(Validation dataset))
+        intermediate_similarity (dict): Quant error reports per layers.
+        degrade_layers (list[str]): A list of detected degrade layers.
+        xquant_config (XQuantConfig): Configuration settings for explainable quantization.
+
+    Returns:
+        None
+    """
+
+    plot_title = "quant_loss_{}_{}".format(metrics_name, dataset_name)
+
+    # Get x,y from intermediate_similarity
+    data_x = []
+    data_y = []
+    for key_layername in intermediate_similarity.keys():
+        value_metrics = intermediate_similarity[key_layername][metrics_name]
+        data_x.append(key_layername)
+        data_y.append(value_metrics)
+    degrade_layer_indexes = [data_x.index(layer) for layer in degrade_layers]
+
+    # Make Graph(adjust size to num layers)
+    plt.figure(figsize=(max(int(len(data_x)/3), 1), 5))
+    # Draw plot with red circle markers of degrade layers
+    plt.plot(data_x, data_y, markevery=degrade_layer_indexes, marker='o', markersize=20, markeredgecolor='r', markerfacecolor=[0.0, 0.0, 0.0, 0.0], markeredgewidth=3)
+    plt.grid()
+    # Add labels
+    plt.title(plot_title)
+    plt.xlabel("layer name")
+    plt.ylabel(metrics_name)
+    plt.xticks(rotation=90)
+    # Label colors of degraded layers are red
+    _, plt_x_labels = plt.xticks()
+    for degrade_layer_index in degrade_layer_indexes:
+        plt_x_labels[degrade_layer_index].set_color("red")
+    # Add threshold line
+    threshold = xquant_config.threshold_quantize_error[metrics_name]
+    plt.hlines(threshold, 0, len(data_x)-1, "red", linestyles='dashed')
+    plt.text(0, threshold+(max(data_y)-min(data_y))*0.02, "threshold={}".format(threshold), color="red")
+
+    plt.tight_layout()
+
+    # Save Image
+    path_outfile = "{}/{}.png".format(xquant_config.report_dir, plot_title, dataset_name)
+    plt.savefig(path_outfile)
+
+    plt.clf()
+    plt.close()
+
+    return None

model_compression_toolkit/xquant/pytorch/facade_xquant_report.py

@@ -16,12 +16,12 @@
 from typing import Callable
 
 from model_compression_toolkit.verify_packages import FOUND_TORCH
-from model_compression_toolkit.xquant.common.core_report_generator import core_report_generator
 from model_compression_toolkit.xquant import XQuantConfig
 from model_compression_toolkit.logger import Logger
 
 if FOUND_TORCH:
     from model_compression_toolkit.xquant.pytorch.pytorch_report_utils import PytorchReportUtils
+    from model_compression_toolkit.xquant.pytorch.core_report_generator import core_report_generator, core_report_generator_troubleshoot
     import torch
 
     def xquant_report_pytorch_experimental(float_model: torch.nn.Module,
@@ -58,7 +58,47 @@ if FOUND_TORCH:
 
         return _collected_data
 
+    def xquant_report_troubleshoot_pytorch_experimental(float_model: torch.nn.Module,
+                                                        quantized_model: torch.nn.Module,
+                                                        repr_dataset: Callable,
+                                                        validation_dataset: Callable,
+                                                        xquant_config: XQuantConfig):
+        """
+        Generate an explainable quantization report, detect degraded layaers and judge degrade causes for a quantized Pytorch model.
+
+        Args:
+            float_model (torch.nn.Module): The original floating-point Pytorch model.
+            quantized_model (torch.nn.Module): The quantized Pytorch model.
+            repr_dataset (Callable): The representative dataset used during quantization.
+            validation_dataset (Callable): The validation dataset used for evaluation.
+            xquant_config (XQuantConfig): Configuration settings for explainable quantization.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the collected similarity metrics and report data.
+            Dict[str, Any]: A dictionary containing the analyze degrade cause report for degraded layaers.
+        """
+        # Initialize the logger with the report directory.
+        Logger.set_log_file(log_folder=xquant_config.report_dir)
+
+        pytorch_report_utils = PytorchReportUtils(xquant_config.report_dir)
+
+        _collected_data, _troubleshoot_data = core_report_generator_troubleshoot(float_model=float_model,
+                                                quantized_model=quantized_model,
+                                                repr_dataset=repr_dataset,
+                                                validation_dataset=validation_dataset,
+                                                fw_report_utils=pytorch_report_utils,
+                                                xquant_config=xquant_config)
+
+        Logger.shutdown()
+
+        return _collected_data, _troubleshoot_data
+
+
 else:
     def xquant_report_pytorch_experimental(*args, **kwargs):
         Logger.critical("PyTorch must be installed to use 'xquant_report_pytorch_experimental'. "
                                      "The 'torch' package is missing.")  # pragma: no cover
+
+    def xquant_report_troubleshoot_pytorch_experimental(*args, **kwargs):
+        Logger.critical("PyTorch must be installed to use 'xquant_report_troubleshoot_pytorch_experimental'. "
+                                     "The 'torch' package is missing.")  # pragma: no cover

model_compression_toolkit/xquant/pytorch/framework_report_utils.py

@@ -0,0 +1,98 @@
+#  Copyright 2024 Sony Semiconductor Israel, Inc. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ==============================================================================
+
+import json
+import os
+
+from model_compression_toolkit.core.common.framework_implementation import FrameworkImplementation
+from model_compression_toolkit.core.common.framework_info import FrameworkInfo
+from typing import Any, Dict, Callable
+
+from model_compression_toolkit.xquant.common.constants import REPORT_FILENAME, TROUBLESHOOT_REPORT_FILENAME
+from model_compression_toolkit.xquant.common.dataset_utils import DatasetUtils
+from model_compression_toolkit.xquant.common.model_folding_utils import ModelFoldingUtils
+from model_compression_toolkit.xquant.pytorch.similarity_calculator import SimilarityCalculator
+from model_compression_toolkit.xquant.common.tensorboard_utils import TensorboardUtils
+from model_compression_toolkit.logger import Logger
+
+
+class FrameworkReportUtils:
+    """
+    Class with various utility components required for generating the report in a specific framework.
+    """
+
+    def __init__(self,
+                 fw_info: FrameworkInfo,
+                 fw_impl: FrameworkImplementation,
+                 similarity_calculator: SimilarityCalculator,
+                 dataset_utils: DatasetUtils,
+                 model_folding_utils: ModelFoldingUtils,
+                 tb_utils: TensorboardUtils,
+                 get_metadata_fn: Callable):
+        """
+        Initializes the FrameworkReportUtils class with various utility components required for generating the report.
+
+        Args:
+            fw_info (FrameworkInfo): Information about the framework being used.
+            fw_impl (FrameworkImplementation): The implemented functions of the framework.
+            similarity_calculator (SimilarityCalculator): A utility for calculating similarity metrics.
+            dataset_utils (DatasetUtils): Utilities for handling datasets.
+            model_folding_utils (ModelFoldingUtils): Utilities for model folding operations.
+            tb_utils (TensorboardUtils): Utilities for TensorBoard operations.
+            get_metadata_fn (Callable): Function to retrieve the metadata from the quantized model.
+        """
+        self.fw_info = fw_info
+        self.fw_impl = fw_impl
+        self.similarity_calculator = similarity_calculator
+        self.dataset_utils = dataset_utils
+        self.model_folding_utils = model_folding_utils
+        self.tb_utils = tb_utils
+        self.get_metadata_fn = get_metadata_fn
+
+    def dump_report_to_json(self,
+                            report_dir: str,
+                            collected_data: Dict[str, Any]):
+        """
+        Dump the collected data (similarity, etc.) into a JSON file.
+
+        Args:
+            report_dir (str): Directory where the report will be saved.
+            collected_data (Dict[str, Any]): Data collected during report generation.
+
+        """
+        report_file_name = os.path.join(report_dir, REPORT_FILENAME)
+        report_file_name = os.path.abspath(report_file_name)
+        Logger.info(f"Dumping report data to: {report_file_name}")
+
+        with open(report_file_name, 'w') as f:
+            json.dump(collected_data, f, indent=4)
+
+    def dump_troubleshoot_report_to_json(self,
+                                         report_dir: str,
+                                         collected_data: Dict[str, Any]):
+        """
+        Dump the collected data (similarity, etc.) into a JSON file.
+
+        Args:
+            report_dir (str): Directory where the report will be saved.
+            collected_data (Dict[str, Any]): Data collected during report generation.
+
+        """
+        report_file_name = os.path.join(report_dir, TROUBLESHOOT_REPORT_FILENAME)
+        report_file_name = os.path.abspath(report_file_name)
+        Logger.info(f"Dumping report data to: {report_file_name}")
+
+        with open(report_file_name, 'w') as f:
+            json.dump(collected_data, f, indent=4)