supervisely 6.73.254__py3-none-any.whl → 6.73.256__py3-none-any.whl

supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py

@@ -1,7 +1,7 @@
-import datetime
-from pathlib import Path
+from typing import List
 
-import supervisely.nn.benchmark.comparison.detection_visualization.text_templates as vis_texts
+import supervisely.nn.benchmark.comparison.detection_visualization.text_templates as texts
+from supervisely.nn.benchmark.comparison.base_visualizer import BaseComparisonVisualizer
 from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics import (
     AveragePrecisionByClass,
     CalibrationScore,
@@ -13,7 +13,9 @@ from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics imp
     PrecisionRecallF1,
     Speedtest,
 )
-from supervisely.nn.benchmark.visualization.renderer import Renderer
+from supervisely.nn.benchmark.object_detection.evaluator import (
+    ObjectDetectionEvalResult,
+)
 from supervisely.nn.benchmark.visualization.widgets import (
     ContainerWidget,
     GalleryWidget,
@@ -22,22 +24,13 @@ from supervisely.nn.benchmark.visualization.widgets import (
 )
 
 
-class DetectionComparisonVisualizer:
-    def __init__(self, comparison):
-        self.comparison = comparison
-        self.api = comparison.api
-        self.vis_texts = vis_texts
-
-        self._create_widgets()
-        layout = self._create_layout()
-
-        self.renderer = Renderer(layout, str(Path(self.comparison.workdir, "visualizations")))
-
-    def visualize(self):
-        return self.renderer.visualize()
+class DetectionComparisonVisualizer(BaseComparisonVisualizer):
+    vis_texts = texts
+    ann_opacity = 0.5
 
-    def upload_results(self, team_id: int, remote_dir: str, progress=None):
-        return self.renderer.upload_results(self.api, team_id, remote_dir, progress)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.eval_results: List[ObjectDetectionEvalResult]
 
     def _create_widgets(self):
         # Modal Gellery
@@ -48,10 +41,11 @@ class DetectionComparisonVisualizer:
         self.clickable_label = self._create_clickable_label()
 
         # Speedtest init here for overview
-        speedtest = Speedtest(self.vis_texts, self.comparison.evaluation_results)
+        speedtest = Speedtest(self.vis_texts, self.comparison.eval_results)
 
         # Overview
-        overview = Overview(self.vis_texts, self.comparison.evaluation_results)
+        overview = Overview(self.vis_texts, self.comparison.eval_results)
+        overview.team_id = self.comparison.team_id
         self.header = self._create_header()
         self.overviews = self._create_overviews(overview)
         self.overview_md = overview.overview_md
@@ -61,11 +55,11 @@ class DetectionComparisonVisualizer:
         )
         self.overview_chart = overview.chart_widget
 
-        columns_number = len(self.comparison.evaluation_results) + 1  # +1 for GT
+        columns_number = len(self.comparison.eval_results) + 1  # +1 for GT
         self.explore_predictions_modal_gallery = self._create_explore_modal_table(columns_number)
         explore_predictions = ExplorePredictions(
             self.vis_texts,
-            self.comparison.evaluation_results,
+            self.comparison.eval_results,
             explore_modal_table=self.explore_predictions_modal_gallery,
         )
         self.explore_predictions_md = explore_predictions.difference_predictions_md
@@ -74,7 +68,7 @@ class DetectionComparisonVisualizer:
         # Outcome Counts
         outcome_counts = OutcomeCounts(
             self.vis_texts,
-            self.comparison.evaluation_results,
+            self.comparison.eval_results,
             explore_modal_table=self.explore_modal_table,
         )
         self.outcome_counts_md = self._create_outcome_counts_md()
@@ -83,7 +77,7 @@ class DetectionComparisonVisualizer:
         self.outcome_counts_comparison = outcome_counts.chart_widget_comparison
 
         # Precision-Recall Curve
-        pr_curve = PrCurve(self.vis_texts, self.comparison.evaluation_results)
+        pr_curve = PrCurve(self.vis_texts, self.comparison.eval_results)
         self.pr_curve_md = pr_curve.markdown_widget
         self.pr_curve_collapsed_widgets = pr_curve.collapsed_widget
         self.pr_curve_table = pr_curve.table_widget
@@ -92,7 +86,7 @@ class DetectionComparisonVisualizer:
         # Average Precision by Class
         avg_prec_by_class = AveragePrecisionByClass(
             self.vis_texts,
-            self.comparison.evaluation_results,
+            self.comparison.eval_results,
             explore_modal_table=self.explore_modal_table,
         )
         self.avg_prec_by_class_md = avg_prec_by_class.markdown_widget
@@ -101,7 +95,7 @@ class DetectionComparisonVisualizer:
         # Precision, Recall, F1
         precision_recall_f1 = PrecisionRecallF1(
             self.vis_texts,
-            self.comparison.evaluation_results,
+            self.comparison.eval_results,
             explore_modal_table=self.explore_modal_table,
         )
         self.precision_recall_f1_md = precision_recall_f1.markdown_widget
@@ -118,14 +112,14 @@ class DetectionComparisonVisualizer:
         # TODO: ???
 
         # Localization Accuracy (IoU)
-        loc_acc = LocalizationAccuracyIoU(self.vis_texts, self.comparison.evaluation_results)
+        loc_acc = LocalizationAccuracyIoU(self.vis_texts, self.comparison.eval_results)
         self.loc_acc_header_md = loc_acc.header_md
         self.loc_acc_iou_distribution_md = loc_acc.iou_distribution_md
         self.loc_acc_chart = loc_acc.chart
         self.loc_acc_table = loc_acc.table_widget
 
         # Calibration Score
-        cal_score = CalibrationScore(self.vis_texts, self.comparison.evaluation_results)
+        cal_score = CalibrationScore(self.vis_texts, self.comparison.eval_results)
         self.cal_score_md = cal_score.header_md
         self.cal_score_md_2 = cal_score.header_md_2
         self.cal_score_collapse_tip = cal_score.collapse_tip
@@ -140,6 +134,7 @@ class DetectionComparisonVisualizer:
 
         # SpeedTest
         self.speedtest_present = False
+        self.speedtest_multiple_batch_sizes = False
         if not speedtest.is_empty():
             self.speedtest_present = True
             self.speedtest_md_intro = speedtest.md_intro
@@ -148,8 +143,10 @@ class DetectionComparisonVisualizer:
             self.speed_inference_time_table = speedtest.inference_time_table
             self.speed_fps_md = speedtest.fps_md
             self.speed_fps_table = speedtest.fps_table
-            self.speed_batch_inference_md = speedtest.batch_inference_md
-            self.speed_chart = speedtest.chart
+            self.speedtest_multiple_batch_sizes = speedtest.multiple_batche_sizes()
+            if self.speedtest_multiple_batch_sizes:
+                self.speed_batch_inference_md = speedtest.batch_inference_md
+                self.speed_chart = speedtest.chart
 
     def _create_layout(self):
         is_anchors_widgets = [
@@ -216,10 +213,11 @@ class DetectionComparisonVisualizer:
                     (0, self.speed_inference_time_table),
                     (0, self.speed_fps_md),
                     (0, self.speed_fps_table),
-                    (0, self.speed_batch_inference_md),
-                    (0, self.speed_chart),
                 ]
             )
+            if self.speedtest_multiple_batch_sizes:
+                is_anchors_widgets.append((0, self.speed_batch_inference_md))
+                is_anchors_widgets.append((0, self.speed_chart))
         anchors = []
         for is_anchor, widget in is_anchors_widgets:
             if is_anchor:
@@ -232,30 +230,6 @@ class DetectionComparisonVisualizer:
         )
         return layout
 
-    def _create_header(self) -> MarkdownWidget:
-        me = self.api.user.get_my_info().login
-        current_date = datetime.datetime.now().strftime("%d %B %Y, %H:%M")
-        header_main_text = " ∣ ".join(  #  vs. or | or ∣
-            eval_res.name for eval_res in self.comparison.evaluation_results
-        )
-        header_text = self.vis_texts.markdown_header.format(header_main_text, me, current_date)
-        header = MarkdownWidget("markdown_header", "Header", text=header_text)
-        return header
-
-    def _create_overviews(self, vm: Overview) -> ContainerWidget:
-        grid_cols = 2
-        if len(vm.overview_widgets) > 2:
-            grid_cols = 3
-        if len(vm.overview_widgets) % 4 == 0:
-            grid_cols = 4
-        return ContainerWidget(
-            vm.overview_widgets,
-            name="overview_container",
-            title="Overview",
-            grid=True,
-            grid_cols=grid_cols,
-        )
-
     def _create_key_metrics(self) -> MarkdownWidget:
         key_metrics_text = self.vis_texts.markdown_key_metrics.format(
             self.vis_texts.definitions.average_precision,
@@ -277,22 +251,3 @@ class DetectionComparisonVisualizer:
         return MarkdownWidget(
             "markdown_outcome_counts_diff", "Outcome Counts Differences", text=outcome_counts_text
         )
-
-    def _create_explore_modal_table(self, columns_number=3):
-        # TODO: table for each evaluation?
-        all_predictions_modal_gallery = GalleryWidget(
-            "all_predictions_modal_gallery", is_modal=True, columns_number=columns_number
-        )
-        all_predictions_modal_gallery.set_project_meta(
-            self.comparison.evaluation_results[0].dt_project_meta
-        )
-        return all_predictions_modal_gallery
-
-    def _create_diff_modal_table(self, columns_number=3) -> GalleryWidget:
-        diff_modal_gallery = GalleryWidget(
-            "diff_predictions_modal_gallery", is_modal=True, columns_number=columns_number
-        )
-        return diff_modal_gallery
-
-    def _create_clickable_label(self):
-        return MarkdownWidget("clickable_label", "", text=self.vis_texts.clickable_label)

supervisely/nn/benchmark/comparison/model_comparison.py

@@ -1,16 +1,35 @@
 import random
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from supervisely.api.api import Api
 from supervisely.app.widgets import SlyTqdm
 from supervisely.imaging.color import get_predefined_colors, rgb2hex
+from supervisely.io import env
+from supervisely.io.fs import dir_empty, mkdir
+from supervisely.io.json import load_json_file
 from supervisely.nn.benchmark.comparison.detection_visualization.visualizer import (
     DetectionComparisonVisualizer,
 )
-from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
+from supervisely.nn.benchmark.comparison.semantic_segmentation.visualizer import (
+    SemanticSegmentationComparisonVisualizer,
+)
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.object_detection.evaluator import (
+    ObjectDetectionEvalResult,
+)
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvalResult,
+)
+from supervisely.nn.task_type import TaskType
+from supervisely.sly_logger import logger
 from supervisely.task.progress import tqdm_sly
 
+ComparisonVisualizer = Union[
+    DetectionComparisonVisualizer, SemanticSegmentationComparisonVisualizer
+]
+ComparisonEvalResult = Union[ObjectDetectionEvalResult, SemanticSegmentationEvalResult]
+
 
 class ModelComparison:
 
@@ -20,25 +39,36 @@ class ModelComparison:
         remote_eval_dirs: List[str],
         progress: Optional[SlyTqdm] = None,
         workdir: Optional[str] = "./benchmark/comparison",
+        cv_task: Optional[TaskType] = None,
+        team_id: Optional[int] = None,
     ):
         self.api = api
         self.progress = progress or tqdm_sly
         self.workdir = workdir
         self.remote_eval_dirs = remote_eval_dirs
-        self.evaluation_results: List[EvalResult] = []
+        self.eval_results: List[ComparisonEvalResult] = []
+        self.task_type = cv_task
+        self.team_id = team_id or env.team_id()
+
+        eval_cls = SemanticSegmentationEvalResult
+        eval_cls = ObjectDetectionEvalResult
 
         colors = get_predefined_colors(len(remote_eval_dirs) * 5)  # for better visualizations
         random.shuffle(colors)
         for i, eval_dir in enumerate(remote_eval_dirs):
-            local_path = str(Path(self.workdir, "eval_data"))
-            eval_result = EvalResult(eval_dir, local_path, self.api, self.progress)
-            self.evaluation_results.append(eval_result)
+            local_path = Path(self.workdir) / "eval_data" / Path(eval_dir).name
+            self._load_eval_data(eval_dir, str(local_path))
+
+            eval_cls = self._get_eval_cls(str(local_path))
+            eval_result = eval_cls(local_path / "evaluation")
+            eval_result.report_path = Path(eval_dir, "visualizations", "template.vue").as_posix()
             eval_result.color = rgb2hex(colors[i])
 
-        self.task_type = self.evaluation_results[0].inference_info.get("task_type")
+            self.eval_results.append(eval_result)
+
         self._validate_eval_data()
 
-        self.visualizer: DetectionComparisonVisualizer = None
+        self.visualizer: ComparisonVisualizer = None
         self.remote_dir = None
 
     def _validate_eval_data(self):
@@ -49,28 +79,41 @@ class ModelComparison:
         task_type = None
         img_names = None
         cat_names = None
-        for eval_result in self.evaluation_results:
+        for eval_result in self.eval_results:
             next_task_type = eval_result.cv_task
             if not task_type is None:
                 assert task_type == next_task_type, "Task types are different in the evaluations."
             task_type = next_task_type
-            next_img_names = set(
-                [img.get("file_name") for img in eval_result.coco_gt.imgs.values()]
-            )
+            if task_type == TaskType.SEMANTIC_SEGMENTATION:
+                next_img_names = set(eval_result.mp.per_image_metrics.index)
+            else:
+                next_img_names = set(
+                    [img.get("file_name") for img in eval_result.coco_gt.imgs.values()]
+                )
             if not img_names is None:
                 assert img_names == next_img_names, "Images are different in the evaluations."
             img_names = next_img_names
-            next_cat_names = set([cat.get("name") for cat in eval_result.coco_gt.cats.values()])
+            if task_type == TaskType.SEMANTIC_SEGMENTATION:
+                next_cat_names = set(eval_result.mp.class_names)
+            else:
+                next_cat_names = set(eval_result.mp.cat_names)
             if not cat_names is None:
                 assert cat_names == next_cat_names, "Categories are different in the evaluations."
             cat_names = next_cat_names
 
-    def get_metrics(self):
-        pass
-
     def visualize(self):
+        task_type = self.eval_results[0].cv_task
+        if task_type in [
+            TaskType.OBJECT_DETECTION,
+            TaskType.INSTANCE_SEGMENTATION,
+        ]:
+            vis_cls = DetectionComparisonVisualizer
+        elif task_type == TaskType.SEMANTIC_SEGMENTATION:
+            vis_cls = SemanticSegmentationComparisonVisualizer
+        else:
+            raise ValueError(f"Unsupported task type: {task_type}")
         if self.visualizer is None:
-            self.visualizer = DetectionComparisonVisualizer(self)
+            self.visualizer = vis_cls(self)
         self.visualizer.visualize()
 
     def upload_results(self, team_id: int, remote_dir: str, progress=None) -> str:
@@ -80,5 +123,55 @@ class ModelComparison:
     def get_report_link(self) -> str:
         if self.remote_dir is None:
             raise ValueError("Results are not uploaded yet.")
-        report_link = self.remote_dir.rstrip("/") + "/template.vue"
-        return report_link
+        return self.visualizer.renderer._get_report_link(self.api, self.team_id, self.remote_dir)
+
+    @property
+    def report(self):
+        return self.visualizer.renderer.report
+
+    @property
+    def lnk(self):
+        return self.visualizer.renderer.lnk
+
+    def _load_eval_data(self, src_path: str, dst_path: str) -> None:
+        dir_name = Path(src_path).name
+        if not dir_empty(dst_path):
+            logger.info(f"Directory {dst_path} is not empty. Skipping download.")
+            return
+        if not self.api.storage.dir_exists(self.team_id, src_path):
+            raise ValueError(f"Directory {src_path} not found in storage.")
+        mkdir(dst_path)
+        with self.progress(
+            message=f"Downloading evaluation data from {dir_name}",
+            total=self.api.storage.get_directory_size(self.team_id, src_path),
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as pbar:
+            self.api.storage.download_directory(
+                self.team_id, src_path, dst_path, progress_cb=pbar.update
+            )
+
+    def _get_cv_task(self, eval_dir: str) -> CVTask:
+        try:
+            eval_data = load_json_file(Path(eval_dir, "evaluation", "inference_info.json"))
+            task_type = eval_data.get("task_type")
+            return CVTask(task_type.replace(" ", "_").lower())
+        except Exception as e:
+            raise ValueError(
+                f"Could not get CV task from `inference_info.json`, try to set it manually. {e}"
+            )
+
+    def _get_eval_cls(self, eval_dir: str) -> ComparisonEvalResult:
+        if self.task_type is None:
+            self.task_type = self._get_cv_task(eval_dir)
+        if self.task_type in [
+            CVTask.OBJECT_DETECTION,
+            CVTask.INSTANCE_SEGMENTATION,
+        ]:
+            eval_cls = ObjectDetectionEvalResult
+        elif self.task_type == CVTask.SEMANTIC_SEGMENTATION:
+            eval_cls = SemanticSegmentationEvalResult
+        else:
+            raise ValueError(f"Unsupported task type: {self.task_type}")
+        return eval_cls

supervisely/nn/benchmark/comparison/semantic_segmentation/text_templates.py

@@ -0,0 +1,128 @@
+docs_url = (
+    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/semantic-segmentation"
+)
+
+# <i class="zmdi zmdi-check-circle" style="color: #13ce66; margin-right: 5px"></i>
+clickable_label = """
+> <span style="color: #5a6772">
+>     Click on the chart to explore corresponding images.
+> </span>
+"""
+
+markdown_header = """
+<h1>{}</h1>
+
+<div class="model-info-block">
+    <div>Created by <b>{}</b></div>
+    <div><i class="zmdi zmdi-calendar-alt"></i><span>{}</span></div>
+</div>
+"""
+
+markdown_common_overview = """
+- **Models**: {}
+- **Evaluation Dataset**: <a href="/projects/{}/datasets" target="_blank">{}</a>
+- **Task type**: {}
+"""
+
+markdown_overview_info = """
+<h3>{}</h3>
+- **Model**: {}
+- **Checkpoint**: {}
+- **Architecture**: {}
+- **Runtime**: {}
+- **Checkpoint file**: <a class="checkpoint-url" href="{}" target="_blank">{}</a>
+- **Evaluation Report**: <a href="{}" target="_blank">View Report</a>
+
+"""
+
+markdown_key_metrics = """## Key Metrics
+
+We provide a comprehensive analysis of models' performance using a set of metrics, including both basic (precision, recall, F1-score, IoU, etc.) and advanced (boundary IoU, error over union decomposition, etc.) metrics.
+
+- **Pixel accuracy**: reflects the percent of image pixels which were correctly classified.
+- **Precision**: reflects the number of correctly predicted positive segmentations divided by the total number of predicted positive segmentations.
+- **Recall**: reflects the number of correctly predicted positive segmentations divided by the number of all samples that should have been segmented as positive.
+- **F1-score**: reflects the tradeoff between precision and recall. It is equivalent to the Dice coefficient and calculated as a harmonic mean of precision and recall.
+- **Intersection over union (IoU, also known as the Jaccard index)**: measures the overlap between ground truth mask and predicted mask. It is calculated as the ratio of the intersection of the two masks areas to their combined areas.
+- **Boundary intersection over union**: a segmentation consistency measure that first computes the sets of ground truth  and predicted masks pixels that are located within the distance d from each contour and then computes intersection over union of these two sets. Pixel distance parameter d (pixel width of the boundary region) controls the sensitivity of the metric, it is usually set as 2% of the image diagonal for normal resolution images and 0.5% of the image diagonal for high resolution images.
+- **Error over union and its components (boundary, extent, segment)**: a metric opposite to intersection over union and can be interpreted as what the model lacked in order to show the perfect performance with IoU = 1. It reflects the ratio of incorrectly segmented pixels of ground truth and predicted masks to their combined areas. It is usually decomposed into boundary, extent and segment errors over union in order to get exhaustive information about the model's strengths and weaknesses.
+- **Renormalized error over union**: postprocessed variant of error over union which takes into consideration cause and effect relationships between different types of segmentation errors.
+"""
+
+markdown_explorer = """## Explore Predictions
+This section contains visual comparison of predictions made by different models and ground truth annotations. Sometimes a simple visualization can be more informative than any performance metric.
+
+> Click on the image to view the **Original Image** with **Ground Truth** and **Predictions** annotations side-by-side. 
+"""
+
+markdown_explore_difference = """## Explore Predictions
+
+In this section, you can explore predictions made by different models side-by-side. This helps you to understand the differences in predictions made by each model, and to identify which model performs better in different scenarios.
+
+
+> Click on the image to view the **Ground Truth**, and **Prediction** annotations side-by-side.
+"""
+
+
+### Difference in Predictions
+
+# markdown_explore_same_errors = """
+# ### Same Errors
+
+# This section helps you to identify samples where all models made the same errors. It is useful for understanding the limitations of the models and the common challenges they face.
+
+# > Click on the image to view the **Ground Truth**, and **Prediction** annotations side-by-side.
+# """
+
+
+# """
+markdown_iou = """## Intersection & Error Over Union
+
+Pie charts below demonstrate performance metrics of each model in terms of Intersection over Union (IoU) and Error over Union (EoU). It is done with the help of Error over Union (EoU) decomposition into boundary, extent, and segment errors over union. These charts help to draw conclusions on the model's strongest and weakest sides.
+"""
+
+markdown_renormalized_error_ou = """## Renormalized Error over Union
+
+Charts below are dedicated to the decomposition of the post-processed variant of Error over Union, which takes into consideration cause and effect relationships between different types of segmentation errors. Error over Union decomposition has its own pitfalls. It is important to understand that models which tend to produce segment errors (when entire segments are mispredicted and there is no intersection between ground truth and predicted mask) will face fewer occasions to produce boundary and extent errors - as a result, boundary and extent error over union values will be underestimated.
+
+In terms of localization, segment error is more fundamental than extent, while extent error is more fundamental than boundary. In order to overcome this problem, renormalized error over union proposes a slightly different calculation method - by removing more fundamental errors from the denominator - read more in our <a href="{}" target="_blank">technical report</a>
+""".format(
+    docs_url
+)
+
+markdown_eou_per_class = """## Classwise Segmentation Error Analysis
+
+This section contains information about classwise segmentation error decomposition. For each model, each column of the chart represents a certain class from the training dataset, demonstrating model performance in terms of segmenting this specific class on images and what model lacked in order to show the perfect performance.
+"""
+
+markdown_frequently_confused_empty = """### Frequently Confused Classes
+
+No frequently confused class pairs found
+"""
+
+markdown_frequently_confused = """## Frequently Confused Classes
+
+The bar chart below reveals pairs of classes which were most frequently confused for each model. Each column of the chart demonstrates the probability of confusion of a given pair of classes. It is necessary to remember that this probability is not symmetric: the probability of confusing class A with class B is not equal to the probability of confusing class B with class A.
+"""
+
+empty = """### {}
+
+> {}
+"""
+
+markdown_speedtest_intro = """## Inference Speed
+
+This is a speed test benchmark for compared models. Models were tested with the following configurations:
+"""
+
+markdown_speedtest_overview_ms = """### Latency (Inference Time)
+The table below shows the speed test results. For each test, the time taken to process one batch of images is shown. Results are averaged across **{}** iterations.
+"""
+
+markdown_speedtest_overview_fps = """### Frames per Second (FPS)
+The table below shows the speed test results. For each test, the number of frames processed per second is shown. Results are averaged across **{}** iterations.
+"""
+
+markdown_batch_inference = """
+This chart shows how the model's speed changes with different batch sizes . As the batch size increases, you can observe an increase in FPS (images per second).
+"""

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/__init__.py

@@ -0,0 +1,21 @@
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.classwise_error_analysis import (
+    ClasswiseErrorAnalysis,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.explore_predictions import (
+    ExplorePredictions,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.frequently_confused import (
+    FrequentlyConfused,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.iou_eou import (
+    IntersectionErrorOverUnion,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.overview import (
+    Overview,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.renormalized_error_ou import (
+    RenormalizedErrorOverUnion,
+)
+from supervisely.nn.benchmark.comparison.semantic_segmentation.vis_metrics.speedtest import (
+    Speedtest,
+)

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/classwise_error_analysis.py

@@ -0,0 +1,68 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class ClasswiseErrorAnalysis(BaseVisMetrics):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "classwise_error_analysis",
+            "Classwise Segmentation Error Analysis",
+            text=self.vis_texts.markdown_eou_per_class,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("classwise_error_analysis", self.get_figure())
+
+    def get_figure(self):
+        import numpy as np
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        # Цветовая палитра для метрик
+        color_palette = ["#8ACAA1", "#FFE4B5", "#F7ADAA", "#dd3f3f"]
+        metrics = ["IoU", "E_extent_oU", "E_boundary_oU", "E_segment_oU"]
+
+        group_width = 0.7
+
+        for model_idx, eval_result in enumerate(self.eval_results):
+            bar_data, labels = eval_result.mp.classwise_segm_error_data
+            model_name = eval_result.name
+
+            for metric_idx, metric_name in enumerate(metrics):
+                # hover_customdata = [f"metric: {metric_name} for class '{l}' ({model_name})" for l in labels]
+                hover_customdata = [
+                    f"class: {l}<br>model: {model_name}<br>{metric_name}" for l in labels
+                ]
+                fig.add_trace(
+                    go.Bar(
+                        name=metric_name,
+                        x=np.arange(len(labels)) + model_idx * group_width * 0.3,
+                        y=bar_data[metric_name],
+                        customdata=hover_customdata,
+                        hovertemplate="%{customdata}: %{y:.2f}<extra></extra>",
+                        marker=dict(color=color_palette[metric_idx]),
+                        width=group_width / len(metrics),
+                        offsetgroup=model_idx,
+                        base=bar_data[metrics[:metric_idx]].sum(axis=1) if metric_idx > 0 else None,
+                    )
+                )
+
+        fig.update_layout(
+            showlegend=False,
+            barmode="stack",
+            xaxis=dict(
+                title="Classes",
+                tickvals=np.arange(len(labels)) + (len(self.eval_results) - 1 - group_width) / 4,
+                ticktext=labels,
+            ),
+            width=800 if len(labels) < 10 else 1000,
+        )
+
+        return fig