supervisely 6.73.253__py3-none-any.whl → 6.73.255__py3-none-any.whl

supervisely/nn/benchmark/comparison/model_comparison.py

@@ -1,16 +1,35 @@
 import random
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from supervisely.api.api import Api
 from supervisely.app.widgets import SlyTqdm
 from supervisely.imaging.color import get_predefined_colors, rgb2hex
+from supervisely.io import env
+from supervisely.io.fs import dir_empty, mkdir
+from supervisely.io.json import load_json_file
 from supervisely.nn.benchmark.comparison.detection_visualization.visualizer import (
     DetectionComparisonVisualizer,
 )
-from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
+from supervisely.nn.benchmark.comparison.semantic_segmentation.visualizer import (
+    SemanticSegmentationComparisonVisualizer,
+)
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.object_detection.evaluator import (
+    ObjectDetectionEvalResult,
+)
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvalResult,
+)
+from supervisely.nn.task_type import TaskType
+from supervisely.sly_logger import logger
 from supervisely.task.progress import tqdm_sly
 
+ComparisonVisualizer = Union[
+    DetectionComparisonVisualizer, SemanticSegmentationComparisonVisualizer
+]
+ComparisonEvalResult = Union[ObjectDetectionEvalResult, SemanticSegmentationEvalResult]
+
 
 class ModelComparison:
 
@@ -20,25 +39,36 @@ class ModelComparison:
         remote_eval_dirs: List[str],
         progress: Optional[SlyTqdm] = None,
         workdir: Optional[str] = "./benchmark/comparison",
+        cv_task: Optional[TaskType] = None,
+        team_id: Optional[int] = None,
     ):
         self.api = api
         self.progress = progress or tqdm_sly
         self.workdir = workdir
         self.remote_eval_dirs = remote_eval_dirs
-        self.evaluation_results: List[EvalResult] = []
+        self.eval_results: List[ComparisonEvalResult] = []
+        self.task_type = cv_task
+        self.team_id = team_id or env.team_id()
+
+        eval_cls = SemanticSegmentationEvalResult
+        eval_cls = ObjectDetectionEvalResult
 
         colors = get_predefined_colors(len(remote_eval_dirs) * 5)  # for better visualizations
         random.shuffle(colors)
         for i, eval_dir in enumerate(remote_eval_dirs):
-            local_path = str(Path(self.workdir, "eval_data"))
-            eval_result = EvalResult(eval_dir, local_path, self.api, self.progress)
-            self.evaluation_results.append(eval_result)
+            local_path = Path(self.workdir) / "eval_data" / Path(eval_dir).name
+            self._load_eval_data(eval_dir, str(local_path))
+
+            eval_cls = self._get_eval_cls(str(local_path))
+            eval_result = eval_cls(local_path / "evaluation")
+            eval_result.report_path = Path(eval_dir, "visualizations", "template.vue").as_posix()
             eval_result.color = rgb2hex(colors[i])
 
-        self.task_type = self.evaluation_results[0].inference_info.get("task_type")
+            self.eval_results.append(eval_result)
+
         self._validate_eval_data()
 
-        self.visualizer: DetectionComparisonVisualizer = None
+        self.visualizer: ComparisonVisualizer = None
         self.remote_dir = None
 
     def _validate_eval_data(self):
@@ -49,28 +79,41 @@ class ModelComparison:
         task_type = None
         img_names = None
         cat_names = None
-        for eval_result in self.evaluation_results:
+        for eval_result in self.eval_results:
             next_task_type = eval_result.cv_task
             if not task_type is None:
                 assert task_type == next_task_type, "Task types are different in the evaluations."
             task_type = next_task_type
-            next_img_names = set(
-                [img.get("file_name") for img in eval_result.coco_gt.imgs.values()]
-            )
+            if task_type == TaskType.SEMANTIC_SEGMENTATION:
+                next_img_names = set(eval_result.mp.per_image_metrics.index)
+            else:
+                next_img_names = set(
+                    [img.get("file_name") for img in eval_result.coco_gt.imgs.values()]
+                )
             if not img_names is None:
                 assert img_names == next_img_names, "Images are different in the evaluations."
             img_names = next_img_names
-            next_cat_names = set([cat.get("name") for cat in eval_result.coco_gt.cats.values()])
+            if task_type == TaskType.SEMANTIC_SEGMENTATION:
+                next_cat_names = set(eval_result.mp.class_names)
+            else:
+                next_cat_names = set(eval_result.mp.cat_names)
             if not cat_names is None:
                 assert cat_names == next_cat_names, "Categories are different in the evaluations."
             cat_names = next_cat_names
 
-    def get_metrics(self):
-        pass
-
     def visualize(self):
+        task_type = self.eval_results[0].cv_task
+        if task_type in [
+            TaskType.OBJECT_DETECTION,
+            TaskType.INSTANCE_SEGMENTATION,
+        ]:
+            vis_cls = DetectionComparisonVisualizer
+        elif task_type == TaskType.SEMANTIC_SEGMENTATION:
+            vis_cls = SemanticSegmentationComparisonVisualizer
+        else:
+            raise ValueError(f"Unsupported task type: {task_type}")
         if self.visualizer is None:
-            self.visualizer = DetectionComparisonVisualizer(self)
+            self.visualizer = vis_cls(self)
         self.visualizer.visualize()
 
     def upload_results(self, team_id: int, remote_dir: str, progress=None) -> str:
@@ -80,5 +123,55 @@ class ModelComparison:
     def get_report_link(self) -> str:
         if self.remote_dir is None:
             raise ValueError("Results are not uploaded yet.")
-        report_link = self.remote_dir.rstrip("/") + "/template.vue"
-        return report_link
+        return self.visualizer.renderer._get_report_link(self.api, self.team_id, self.remote_dir)
+
+    @property
+    def report(self):
+        return self.visualizer.renderer.report
+
+    @property
+    def lnk(self):
+        return self.visualizer.renderer.lnk
+
+    def _load_eval_data(self, src_path: str, dst_path: str) -> None:
+        dir_name = Path(src_path).name
+        if not dir_empty(dst_path):
+            logger.info(f"Directory {dst_path} is not empty. Skipping download.")
+            return
+        if not self.api.storage.dir_exists(self.team_id, src_path):
+            raise ValueError(f"Directory {src_path} not found in storage.")
+        mkdir(dst_path)
+        with self.progress(
+            message=f"Downloading evaluation data from {dir_name}",
+            total=self.api.storage.get_directory_size(self.team_id, src_path),
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as pbar:
+            self.api.storage.download_directory(
+                self.team_id, src_path, dst_path, progress_cb=pbar.update
+            )
+
+    def _get_cv_task(self, eval_dir: str) -> CVTask:
+        try:
+            eval_data = load_json_file(Path(eval_dir, "evaluation", "inference_info.json"))
+            task_type = eval_data.get("task_type")
+            return CVTask(task_type.replace(" ", "_").lower())
+        except Exception as e:
+            raise ValueError(
+                f"Could not get CV task from `inference_info.json`, try to set it manually. {e}"
+            )
+
+    def _get_eval_cls(self, eval_dir: str) -> ComparisonEvalResult:
+        if self.task_type is None:
+            self.task_type = self._get_cv_task(eval_dir)
+        if self.task_type in [
+            CVTask.OBJECT_DETECTION,
+            CVTask.INSTANCE_SEGMENTATION,
+        ]:
+            eval_cls = ObjectDetectionEvalResult
+        elif self.task_type == CVTask.SEMANTIC_SEGMENTATION:
+            eval_cls = SemanticSegmentationEvalResult
+        else:
+            raise ValueError(f"Unsupported task type: {self.task_type}")
+        return eval_cls

supervisely/nn/benchmark/comparison/semantic_segmentation/text_templates.py

@@ -0,0 +1,128 @@
+docs_url = (
+    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/semantic-segmentation"
+)
+
+# <i class="zmdi zmdi-check-circle" style="color: #13ce66; margin-right: 5px"></i>
+clickable_label = """
+> <span style="color: #5a6772">
+>     Click on the chart to explore corresponding images.
+> </span>
+"""
+
+markdown_header = """
+<h1>{}</h1>
+
+<div class="model-info-block">
+    <div>Created by <b>{}</b></div>
+    <div><i class="zmdi zmdi-calendar-alt"></i><span>{}</span></div>
+</div>
+"""
+
+markdown_common_overview = """
+- **Models**: {}
+- **Evaluation Dataset**: <a href="/projects/{}/datasets" target="_blank">{}</a>
+- **Task type**: {}
+"""
+
+markdown_overview_info = """
+<h3>{}</h3>
+- **Model**: {}
+- **Checkpoint**: {}
+- **Architecture**: {}
+- **Runtime**: {}
+- **Checkpoint file**: <a class="checkpoint-url" href="{}" target="_blank">{}</a>
+- **Evaluation Report**: <a href="{}" target="_blank">View Report</a>
+
+"""
+
+markdown_key_metrics = """## Key Metrics
+
+We provide a comprehensive analysis of models' performance using a set of metrics, including both basic (precision, recall, F1-score, IoU, etc.) and advanced (boundary IoU, error over union decomposition, etc.) metrics.
+
+- **Pixel accuracy**: reflects the percent of image pixels which were correctly classified.
+- **Precision**: reflects the number of correctly predicted positive segmentations divided by the total number of predicted positive segmentations.
+- **Recall**: reflects the number of correctly predicted positive segmentations divided by the number of all samples that should have been segmented as positive.
+- **F1-score**: reflects the tradeoff between precision and recall. It is equivalent to the Dice coefficient and calculated as a harmonic mean of precision and recall.
+- **Intersection over union (IoU, also known as the Jaccard index)**: measures the overlap between ground truth mask and predicted mask. It is calculated as the ratio of the intersection of the two masks areas to their combined areas.
+- **Boundary intersection over union**: a segmentation consistency measure that first computes the sets of ground truth  and predicted masks pixels that are located within the distance d from each contour and then computes intersection over union of these two sets. Pixel distance parameter d (pixel width of the boundary region) controls the sensitivity of the metric, it is usually set as 2% of the image diagonal for normal resolution images and 0.5% of the image diagonal for high resolution images.
+- **Error over union and its components (boundary, extent, segment)**: a metric opposite to intersection over union and can be interpreted as what the model lacked in order to show the perfect performance with IoU = 1. It reflects the ratio of incorrectly segmented pixels of ground truth and predicted masks to their combined areas. It is usually decomposed into boundary, extent and segment errors over union in order to get exhaustive information about the model's strengths and weaknesses.
+- **Renormalized error over union**: postprocessed variant of error over union which takes into consideration cause and effect relationships between different types of segmentation errors.
+"""
+
+markdown_explorer = """## Explore Predictions
+This section contains visual comparison of predictions made by different models and ground truth annotations. Sometimes a simple visualization can be more informative than any performance metric.
+
+> Click on the image to view the **Original Image** with **Ground Truth** and **Predictions** annotations side-by-side. 
+"""
+
+markdown_explore_difference = """## Explore Predictions
+
+In this section, you can explore predictions made by different models side-by-side. This helps you to understand the differences in predictions made by each model, and to identify which model performs better in different scenarios.
+
+
+> Click on the image to view the **Ground Truth**, and **Prediction** annotations side-by-side.
+"""
+
+
+### Difference in Predictions
+
+# markdown_explore_same_errors = """
+# ### Same Errors
+
+# This section helps you to identify samples where all models made the same errors. It is useful for understanding the limitations of the models and the common challenges they face.
+
+# > Click on the image to view the **Ground Truth**, and **Prediction** annotations side-by-side.
+# """
+
+
+# """
+markdown_iou = """## Intersection & Error Over Union
+
+Pie charts below demonstrate performance metrics of each model in terms of Intersection over Union (IoU) and Error over Union (EoU). It is done with the help of Error over Union (EoU) decomposition into boundary, extent, and segment errors over union. These charts help to draw conclusions on the model's strongest and weakest sides.
+"""
+
+markdown_renormalized_error_ou = """## Renormalized Error over Union
+
+Charts below are dedicated to the decomposition of the post-processed variant of Error over Union, which takes into consideration cause and effect relationships between different types of segmentation errors. Error over Union decomposition has its own pitfalls. It is important to understand that models which tend to produce segment errors (when entire segments are mispredicted and there is no intersection between ground truth and predicted mask) will face fewer occasions to produce boundary and extent errors - as a result, boundary and extent error over union values will be underestimated.
+
+In terms of localization, segment error is more fundamental than extent, while extent error is more fundamental than boundary. In order to overcome this problem, renormalized error over union proposes a slightly different calculation method - by removing more fundamental errors from the denominator - read more in our <a href="{}" target="_blank">technical report</a>
+""".format(
+    docs_url
+)
+
+markdown_eou_per_class = """## Classwise Segmentation Error Analysis
+
+This section contains information about classwise segmentation error decomposition. For each model, each column of the chart represents a certain class from the training dataset, demonstrating model performance in terms of segmenting this specific class on images and what model lacked in order to show the perfect performance.
+"""
+
+markdown_frequently_confused_empty = """### Frequently Confused Classes
+
+No frequently confused class pairs found
+"""
+
+markdown_frequently_confused = """## Frequently Confused Classes
+
+The bar chart below reveals pairs of classes which were most frequently confused for each model. Each column of the chart demonstrates the probability of confusion of a given pair of classes. It is necessary to remember that this probability is not symmetric: the probability of confusing class A with class B is not equal to the probability of confusing class B with class A.
+"""
+
+empty = """### {}
+
+> {}
+"""
+
+markdown_speedtest_intro = """## Inference Speed
+
+This is a speed test benchmark for compared models. Models were tested with the following configurations:
+"""
+
+markdown_speedtest_overview_ms = """### Latency (Inference Time)
+The table below shows the speed test results. For each test, the time taken to process one batch of images is shown. Results are averaged across **{}** iterations.
+"""
+
+markdown_speedtest_overview_fps = """### Frames per Second (FPS)
+The table below shows the speed test results. For each test, the number of frames processed per second is shown. Results are averaged across **{}** iterations.
+"""
+
+markdown_batch_inference = """
+This chart shows how the model's speed changes with different batch sizes . As the batch size increases, you can observe an increase in FPS (images per second).
+"""

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/__init__.py

@@ -0,0 +1,21 @@
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.classwise_error_analysis import (
+    ClasswiseErrorAnalysis,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.explore_predictions import (
+    ExplorePredictions,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.frequently_confused import (
+    FrequentlyConfused,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.iou_eou import (
+    IntersectionErrorOverUnion,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.overview import (
+    Overview,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.renormalized_error_ou import (
+    RenormalizedErrorOverUnion,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.speedtest import (
+    Speedtest,
+)

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/classwise_error_analysis.py

@@ -0,0 +1,68 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class ClasswiseErrorAnalysis(BaseVisMetrics):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "classwise_error_analysis",
+            "Classwise Segmentation Error Analysis",
+            text=self.vis_texts.markdown_eou_per_class,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("classwise_error_analysis", self.get_figure())
+
+    def get_figure(self):
+        import numpy as np
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        # Цветовая палитра для метрик
+        color_palette = ["#8ACAA1", "#FFE4B5", "#F7ADAA", "#dd3f3f"]
+        metrics = ["IoU", "E_extent_oU", "E_boundary_oU", "E_segment_oU"]
+
+        group_width = 0.7
+
+        for model_idx, eval_result in enumerate(self.eval_results):
+            bar_data, labels = eval_result.mp.classwise_segm_error_data
+            model_name = eval_result.name
+
+            for metric_idx, metric_name in enumerate(metrics):
+                # hover_customdata = [f"metric: {metric_name} for class '{l}' ({model_name})" for l in labels]
+                hover_customdata = [
+                    f"class: {l}<br>model: {model_name}<br>{metric_name}" for l in labels
+                ]
+                fig.add_trace(
+                    go.Bar(
+                        name=metric_name,
+                        x=np.arange(len(labels)) + model_idx * group_width * 0.3,
+                        y=bar_data[metric_name],
+                        customdata=hover_customdata,
+                        hovertemplate="%{customdata}: %{y:.2f}<extra></extra>",
+                        marker=dict(color=color_palette[metric_idx]),
+                        width=group_width / len(metrics),
+                        offsetgroup=model_idx,
+                        base=bar_data[metrics[:metric_idx]].sum(axis=1) if metric_idx > 0 else None,
+                    )
+                )
+
+        fig.update_layout(
+            showlegend=False,
+            barmode="stack",
+            xaxis=dict(
+                title="Classes",
+                tickvals=np.arange(len(labels)) + (len(self.eval_results) - 1 - group_width) / 4,
+                ticktext=labels,
+            ),
+            width=800 if len(labels) < 10 else 1000,
+        )
+
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/explore_predictions.py

@@ -0,0 +1,141 @@
+from typing import List, Tuple
+
+from supervisely.annotation.annotation import Annotation
+from supervisely.api.image_api import ImageInfo
+from supervisely.api.module_api import ApiField
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import GalleryWidget, MarkdownWidget
+from supervisely.project.project_meta import ProjectMeta
+
+
+class ExplorePredictions(BaseVisMetrics):
+
+    MARKDOWN = "markdown_explorer"
+    GALLERY_DIFFERENCE = "explore_difference_gallery"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.meta = None
+
+    def _merged_meta(self) -> ProjectMeta:
+        if self.meta is not None:
+            return self.meta
+        self.meta = self.eval_results[0].gt_project_meta
+        for eval_res in self.eval_results:
+            self.meta = self.meta.merge(eval_res.pred_project_meta)
+        return self.meta
+
+    @property
+    def difference_predictions_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_explorer
+        return MarkdownWidget(self.MARKDOWN, "Explore Predictions", text)
+
+    @property
+    def explore_gallery(self) -> GalleryWidget:
+        columns_number = len(self.eval_results) + 1
+        images, annotations = self._get_sample_data()
+        gallery = GalleryWidget(self.GALLERY_DIFFERENCE, columns_number=columns_number)
+        gallery.add_image_left_header("Click to explore more")
+        gallery.set_project_meta(self._merged_meta())
+        gallery.set_images(images, annotations)
+        click_data = self.get_click_data_explore_all()
+        gallery.set_click_data(self.explore_modal_table.id, click_data)
+        gallery.set_show_all_data(self.explore_modal_table.id, click_data)
+        gallery._gallery._update_filters()
+
+        return gallery
+
+    def _get_sample_data(self) -> Tuple[List[ImageInfo], List[Annotation], List[ProjectMeta]]:
+        images = []
+        annotations = []
+        api = self.eval_results[0].api
+        names = None
+        ds_name = None
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_info = eval_res.gt_dataset_infos[0]
+                infos = api.image.get_list(dataset_info.id, limit=5, force_metadata_for_links=False)
+                ds_name = dataset_info.name
+                images_ids = [image_info.id for image_info in infos]
+                names = [image_info.name for image_info in infos]
+                images.append(infos)
+                from supervisely.api.api import Api
+
+                api: Api
+                anns = api.annotation.download_batch(
+                    dataset_info.id, images_ids, force_metadata_for_links=False
+                )
+                annotations.append(anns)
+            assert ds_name is not None, "Failed to get GT dataset name for gallery"
+
+            dataset_info = api.dataset.get_info_by_name(eval_res.pred_project_id, ds_name)
+
+            assert names is not None, "Failed to get GT image names for gallery"
+            infos = api.image.get_list(
+                dataset_info.id,
+                filters=[
+                    {ApiField.FIELD: ApiField.NAME, ApiField.OPERATOR: "in", ApiField.VALUE: names}
+                ],
+                force_metadata_for_links=False,
+            )
+            images_ids = [image_info.id for image_info in infos]
+            images.append(infos)
+            anns = api.annotation.download_batch(
+                dataset_info.id, images_ids, force_metadata_for_links=False
+            )
+            annotations.append(anns)
+
+        images = list(i for x in zip(*images) for i in x)
+        annotations = list(i for x in zip(*annotations) for i in x)
+        return images, annotations
+
+    def get_click_data_explore_all(self) -> dict:
+        res = {}
+
+        res["projectMeta"] = self._merged_meta().to_json()
+        res["layoutTemplate"] = [{"columnTitle": "Ground Truth"}]
+        for idx, eval_res in enumerate(self.eval_results, 1):
+            res["layoutTemplate"].append({"columnTitle": f"[{idx}] {eval_res.short_name}"})
+
+        click_data = res.setdefault("clickData", {})
+        explore = click_data.setdefault("explore", {})
+        explore["title"] = "Explore all predictions"
+
+        image_names = set()
+        for eval_res in self.eval_results:
+            eval_res.mp.per_image_metrics["img_names"].apply(image_names.add)
+
+        filters = [{"field": "name", "operator": "in", "value": list(image_names)}]
+
+        images_ids = []
+        api = self.eval_results[0].api
+        names = None
+        ds_names = None
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_infos = eval_res.gt_dataset_infos
+                ds_names = [ds.name for ds in dataset_infos]
+                current_images_ids = []
+                current_images_names = []
+                for ds in dataset_infos:
+                    image_infos = api.image.get_list(ds.id, filters, force_metadata_for_links=False)
+                    image_infos = sorted(image_infos, key=lambda x: x.name)
+                    current_images_names.extend([image_info.name for image_info in image_infos])
+                    current_images_ids.extend([image_info.id for image_info in image_infos])
+                images_ids.append(current_images_ids)
+                names = current_images_names
+
+            dataset_infos = api.dataset.get_list(eval_res.pred_project_id)
+            dataset_infos = [ds for ds in dataset_infos if ds.name in ds_names]
+            dataset_infos = sorted(dataset_infos, key=lambda x: ds_names.index(x.name))
+            current_images_infos = []
+            for ds in dataset_infos:
+                image_infos = api.image.get_list(ds.id, filters, force_metadata_for_links=False)
+                image_infos = [image_info for image_info in image_infos if image_info.name in names]
+                current_images_infos.extend(image_infos)
+            current_images_infos = sorted(current_images_infos, key=lambda x: names.index(x.name))
+            images_ids.append([image_info.id for image_info in current_images_infos])
+
+        explore["imagesIds"] = list(i for x in zip(*images_ids) for i in x)
+
+        return res

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/frequently_confused.py

@@ -0,0 +1,71 @@
+from typing import List
+
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvalResult,
+)
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class FrequentlyConfused(BaseVisMetrics):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.eval_results: List[SemanticSegmentationEvalResult]
+        self.clickable = True
+        self._keypair_sep = "-"
+
+    @property
+    def md(self) -> MarkdownWidget:
+        if self.is_empty:
+            text = self.vis_texts.markdown_frequently_confused_empty
+        else:
+            text = self.vis_texts.markdown_frequently_confused
+        return MarkdownWidget("frequently_confused", "Frequently Confused Classes", text=text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("frequently_confused", self.get_figure())
+
+    @property
+    def is_empty(self) -> bool:
+        return all(len(e.mp.frequently_confused[0]) == 0 for e in self.eval_results)
+
+    def get_figure(self):
+        import numpy as np
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        classes = self.eval_results[0].classes_whitelist
+
+        model_cnt = len(self.eval_results)
+        all_models_cmat = np.zeros((model_cnt, len(classes), len(classes)))
+        for model_idx, eval_result in enumerate(self.eval_results):
+            cmat, _ = eval_result.mp.confusion_matrix
+            all_models_cmat[model_idx] = cmat[::-1].copy()
+
+        sum_cmat = all_models_cmat.sum(axis=0)
+        np.fill_diagonal(sum_cmat, 0)
+        sum_cmat_flat = sum_cmat.flatten()
+        sorted_indices = np.argsort(sum_cmat_flat)[::-1]
+        n_pairs = min(10, len(classes) * (len(classes) - 1))
+        sorted_indices = sorted_indices[:n_pairs]
+        rows = sorted_indices // len(classes)
+        cols = sorted_indices % len(classes)
+        labels = [f"{classes[rows[i]]}-{classes[cols[i]]}" for i in range(n_pairs)]
+        for model_idx, eval_result in enumerate(self.eval_results):
+            cmat = all_models_cmat[model_idx]
+            probs = cmat[rows, cols] 
+            probs = probs * 100
+            fig.add_trace(
+                go.Bar(
+                    name=eval_result.name,
+                    x=labels,
+                    y=probs,
+                    hovertemplate="%{x}: %{y:.2f}%<extra></extra>",
+                    marker=dict(color=eval_result.color, line=dict(width=0.7)),
+                )
+            )
+
+        return fig