supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/semantic_segmentation/__init__.py

@@ -0,0 +1,9 @@
+from supervisely.nn.benchmark.semantic_segmentation.benchmark import (
+    SemanticSegmentationBenchmark,
+)
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvaluator,
+)
+from supervisely.nn.benchmark.semantic_segmentation.visualizer import (
+    SemanticSegmentationVisualizer,
+)

supervisely/nn/benchmark/semantic_segmentation/base_vis_metric.py

@@ -0,0 +1,55 @@
+from typing import Dict, Optional
+
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetric
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvalResult,
+)
+
+
+class SemanticSegmVisMetric(BaseVisMetric):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.eval_result: SemanticSegmentationEvalResult
+
+    def get_click_data(self) -> Optional[Dict]:
+        if not self.clickable:
+            return
+
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+
+        for key, v in self.eval_result.images_by_class.items():
+            res["clickData"][key] = {}
+
+            title = f"Class {key}: {len(v)} image{'s' if len(v) > 1 else ''}"
+            res["clickData"][key]["title"] = title
+            img_ids = [
+                self.eval_result.matched_pair_data[gt_img_id].pred_image_info.id for gt_img_id in v
+            ]
+            res["clickData"][key]["imagesIds"] = img_ids
+
+        return res
+
+    def get_diff_data(self) -> Dict:
+        res = {}
+
+        res["layoutTemplate"] = [
+            {"columnTitle": "Original Image"},
+            {"columnTitle": "Ground Truth Masks"},
+            {"columnTitle": "Predicted Masks"},
+        ]
+
+        click_data = res.setdefault("clickData", {})
+
+        for pairs_data in self.eval_result.matched_pair_data.values():
+            gt = pairs_data.gt_image_info
+            pred = pairs_data.pred_image_info
+            diff = pairs_data.diff_image_info
+            assert gt.name == pred.name == diff.name
+            key = click_data.setdefault(str(pred.id), {})
+            key["imagesIds"] = [diff.id, gt.id, pred.id]
+            key["title"] = f"Image: {pred.name}"
+        return res

supervisely/nn/benchmark/semantic_segmentation/benchmark.py

@@ -0,0 +1,32 @@
+from supervisely.nn.benchmark.base_benchmark import BaseBenchmark
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvaluator,
+)
+from supervisely.nn.benchmark.semantic_segmentation.visualizer import (
+    SemanticSegmentationVisualizer,
+)
+
+
+class SemanticSegmentationBenchmark(BaseBenchmark):
+    visualizer_cls = SemanticSegmentationVisualizer
+
+    @property
+    def cv_task(self) -> str:
+        return CVTask.SEMANTIC_SEGMENTATION
+
+    def _get_evaluator_class(self) -> type:
+        return SemanticSegmentationEvaluator
+
+    def _evaluate(self, gt_project_path, pred_project_path):
+        eval_results_dir = self.get_eval_results_dir()
+        self.evaluator = self._get_evaluator_class()(
+            gt_project_path=gt_project_path,
+            pred_project_path=pred_project_path,
+            result_dir=eval_results_dir,
+            progress=self.pbar,
+            items_count=self.dt_project_info.items_count,
+            classes_whitelist=self.classes_whitelist,
+            evaluation_params=self.evaluation_params,
+        )
+        self.evaluator.evaluate()

supervisely/nn/benchmark/semantic_segmentation/evaluator.py

@@ -0,0 +1,162 @@
+import os
+import pickle
+import shutil
+from pathlib import Path
+from typing import List
+
+import cv2
+import numpy as np
+from tqdm import tqdm
+
+from supervisely.io.json import load_json_file
+from supervisely.nn.benchmark.base_evaluator import BaseEvalResult, BaseEvaluator
+from supervisely.nn.benchmark.semantic_segmentation.metric_provider import (
+    MetricProvider,
+)
+from supervisely.nn.benchmark.utils import (
+    calculate_semsegm_metrics as calculate_metrics,
+)
+from supervisely.project.project import Dataset, OpenMode, Project
+from supervisely.project.project_meta import ProjectMeta
+from supervisely.sly_logger import logger
+
+
+class SemanticSegmentationEvalResult(BaseEvalResult):
+    mp_cls = MetricProvider
+
+    def _read_eval_data(self):
+        self.eval_data = pickle.load(open(Path(self.directory, "eval_data.pkl"), "rb"))
+        self.inference_info = load_json_file(Path(self.directory, "inference_info.json"))
+        speedtest_info_path = Path(self.directory).parent / "speedtest" / "speedtest.json"
+        self.speedtest_info = None
+        if speedtest_info_path.exists():
+            self.speedtest_info = load_json_file(speedtest_info_path)
+
+        self.mp = MetricProvider(self.eval_data)
+        # self.mp.calculate()
+
+
+class SemanticSegmentationEvaluator(BaseEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"
+    eval_result_cls = SemanticSegmentationEvalResult
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bg_cls_name = None
+        self.bg_cls_color = None
+
+    def evaluate(self):
+        self.bg_cls_name = self._get_bg_class_name()
+        if self.bg_cls_name not in self.classes_whitelist:
+            self.classes_whitelist.append(self.bg_cls_name)
+
+        gt_prep_path, pred_prep_path = self.prepare_segmentation_data()
+
+        self.eval_data = calculate_metrics(
+            gt_dir=gt_prep_path,
+            pred_dir=pred_prep_path,
+            boundary_width=0.01,
+            boundary_iou_d=0.02,
+            num_workers=4,
+            class_names=self.classes_whitelist,
+            result_dir=self.result_dir,
+            progress=self.pbar,
+        )
+        self.eval_data["bg_cls_name"] = self.bg_cls_name
+        logger.info("Successfully calculated evaluation metrics")
+        self._dump_eval_results()
+        logger.info("Evaluation results are saved")
+
+    def _get_palette(self, project_path):
+        meta_path = Path(project_path) / "meta.json"
+        meta = ProjectMeta.from_json(load_json_file(meta_path))
+
+        palette = [obj.color for obj in meta.obj_classes if obj.name in self.classes_whitelist]
+
+        return palette
+
+    def _dump_eval_results(self):
+        eval_data_path = self._get_eval_data_path()
+        self._dump_pickle(self.eval_data, eval_data_path)  # TODO: maybe dump JSON?
+
+    def _get_eval_data_path(self):
+        base_dir = self.result_dir
+        eval_data_path = os.path.join(base_dir, "eval_data.pkl")
+        return eval_data_path
+
+    def prepare_segmentation_data(self):
+        src_dirs = [self.gt_project_path, self.pred_project_path]
+        output_dirs = [
+            Path(self.gt_project_path).parent / "preprocessed_gt",
+            Path(self.pred_project_path).parent / "preprocessed_pred",
+        ]
+
+        for src_dir, output_dir in zip(src_dirs, output_dirs):
+            if output_dir.exists():
+                logger.info(f"Preprocessed data already exists in {output_dir} directory")
+                continue
+
+            palette = self._get_palette(src_dir)
+            bg_color = palette[self.classes_whitelist.index(self.bg_cls_name)]
+            output_dir.mkdir(parents=True)
+            temp_seg_dir = src_dir + "_temp"
+            if not os.path.exists(temp_seg_dir):
+                Project.to_segmentation_task(
+                    src_dir,
+                    temp_seg_dir,
+                    target_classes=self.classes_whitelist,
+                    bg_name=self.bg_cls_name,
+                    bg_color=bg_color,
+                )
+
+            palette_lookup = np.zeros(256**3, dtype=np.int32)
+            for idx, color in enumerate(palette, 1):
+                key = (color[0] << 16) | (color[1] << 8) | color[2]
+                palette_lookup[key] = idx
+
+            temp_project = Project(temp_seg_dir, mode=OpenMode.READ)
+            temp_project.total_items
+            for dataset in temp_project.datasets:
+                dataset: Dataset
+                names = dataset.get_items_names()
+                for name in names:
+                    mask_path = dataset.get_seg_path(name)
+                    mask = cv2.imread(mask_path)[:, :, ::-1]
+
+                    mask_keys = (
+                        (mask[:, :, 0].astype(np.int32) << 16)
+                        | (mask[:, :, 1].astype(np.int32) << 8)
+                        | mask[:, :, 2].astype(np.int32)
+                    )
+                    result = palette_lookup[mask_keys]
+                    if name.count(".png") > 1:
+                        name = name[:-4]
+                    cv2.imwrite(os.path.join(output_dir, name), result)
+
+            shutil.rmtree(temp_seg_dir)
+
+        return output_dirs
+
+    def _get_bg_class_name(self):
+        possible_names = ["background", "bg", "unlabeled", "neutral", "__bg__"]
+        logger.info(f"Searching for background class in projects. Possible names: {possible_names}")
+
+        bg_cls_names = []
+        for project_path in [self.gt_project_path, self.pred_project_path]:
+            meta_path = Path(project_path) / "meta.json"
+            meta = ProjectMeta.from_json(load_json_file(meta_path))
+
+            for obj_cls in meta.obj_classes:
+                if obj_cls.name in possible_names:
+                    bg_cls_names.append(obj_cls.name)
+                    break
+
+        if len(bg_cls_names) == 0:
+            raise ValueError("Background class not found in GT and Pred projects")
+
+        if len(set(bg_cls_names)) > 1:
+            raise ValueError(
+                f"Founds multiple background class names in GT and Pred projects: {set(bg_cls_names)}"
+            )
+
+        return bg_cls_names[0]

supervisely/nn/benchmark/semantic_segmentation/metric_provider.py

@@ -0,0 +1,153 @@
+from typing import Any, Dict
+
+import numpy as np
+
+METRIC_NAMES = {
+    "mPixel": "mPixel accuracy",
+    "mF1": "mF1-score",
+    "mPrecision": "mPrecision",
+    "mRecall": "mRecall",
+    "mIoU": "mIoU",
+    "mBoundaryIoU": "mBoundaryIoU",
+    "calibration_score": "Calibration Score",
+}
+
+
+class MetricProvider:
+    def __init__(self, eval_data: Dict[str, Any]):
+        # self.params = params
+        self.metric_names = METRIC_NAMES
+
+        # eval_data
+        self.eval_data = eval_data["result"]
+        self.bg_cls_name = eval_data["bg_cls_name"]
+        self.per_image_metrics = eval_data["per_image_metrics"]
+        self.cmat_cell_img_names = eval_data["cell_img_names"]
+        self.class_names = self.eval_data.index.tolist()
+
+        self.num_classes = len(self.class_names)
+
+        # base metrics
+        overall_TP = self.eval_data["TP"][: self.num_classes].sum()
+        overall_FN = self.eval_data["FN"][: self.num_classes].sum()
+        self.pixel_accuracy = overall_TP / (overall_TP + overall_FN)
+        self.overall_TP = overall_TP
+        self.overall_FN = overall_FN
+        self.precision = round(self.eval_data.loc["mean", "precision"] * 100, 1)
+        self.recall = round(self.eval_data.loc["mean", "recall"] * 100, 1)
+        self.f1_score = round(self.eval_data.loc["mean", "F1_score"] * 100, 1)
+        self.iou = round(self.eval_data.loc["mean", "IoU"] * 100, 1)
+        self.boundary_iou = round(self.eval_data.loc["mean", "boundary_IoU"] * 100, 1)
+
+        # error metrics
+        # labels = ["mIoU", "mBoundaryEoU", "mExtentEoU", "mSegmentEoU"]
+        self.boundary_eou = round(self.eval_data.loc["mean", "E_boundary_oU"] * 100, 1)
+        self.extent_eou = round(self.eval_data.loc["mean", "E_extent_oU"] * 100, 1)
+        self.segment_eou = round(self.eval_data.loc["mean", "E_segment_oU"] * 100, 1)
+
+        # renormalized error metrics
+        # labels = ["boundary", "extent", "segment"]
+        self.boundary_renormed_eou = round(
+            self.eval_data.loc["mean", "E_boundary_oU_renormed"] * 100, 1
+        )
+        self.extent_renormed_eou = round(
+            self.eval_data.loc["mean", "E_extent_oU_renormed"] * 100, 1
+        )
+        self.segment_renormed_eou = round(
+            self.eval_data.loc["mean", "E_segment_oU_renormed"] * 100, 1
+        )
+
+        # classwise error data
+        self.classwise_segm_error_data = self.get_classwise_error_data()
+
+        # confusion matrix
+        self.confusion_matrix = self.get_confusion_matrix(eval_data["confusion_matrix"].copy())
+
+        # frequently confused classes
+        self.frequently_confused = self.get_frequently_confused(
+            eval_data["confusion_matrix"].copy()
+        )
+
+    def json_metrics(self):
+        return {
+            "mIoU": self.eval_data.loc["mean"]["IoU"] * 100,
+            "mE_boundary_oU": self.eval_data.loc["mean"]["E_boundary_oU"] * 100,
+            "mFP_boundary_oU": self.eval_data.loc["mean"]["FP_boundary_oU"] * 100,
+            "mFN_boundary_oU": self.eval_data.loc["mean"]["FN_boundary_oU"] * 100,
+            "mE_boundary_oU_renormed": self.eval_data.loc["mean"]["E_boundary_oU_renormed"] * 100,
+            "mE_extent_oU": self.eval_data.loc["mean"]["E_extent_oU"] * 100,
+            "mFP_extent_oU": self.eval_data.loc["mean"]["FP_extent_oU"] * 100,
+            "mFN_extent_oU": self.eval_data.loc["mean"]["FN_extent_oU"] * 100,
+            "mE_extent_oU_renormed": self.eval_data.loc["mean"]["E_extent_oU_renormed"] * 100,
+            "mE_segment_oU": self.eval_data.loc["mean"]["E_segment_oU"] * 100,
+            "mFP_segment_oU": self.eval_data.loc["mean"]["FP_segment_oU"] * 100,
+            "mFN_segment_oU": self.eval_data.loc["mean"]["FN_segment_oU"] * 100,
+            "mE_segment_oU_renormed": self.eval_data.loc["mean"]["E_segment_oU_renormed"] * 100,
+            "mPrecision": self.eval_data.loc["mean"]["precision"] * 100,
+            "mRecall": self.eval_data.loc["mean"]["recall"] * 100,
+            "mF1_score": self.eval_data.loc["mean"]["F1_score"] * 100,
+            "PixelAcc": self.pixel_accuracy * 100,
+            "mBoundaryIoU": self.eval_data.loc["mean"]["boundary_IoU"] * 100,
+        }
+
+    def metric_table(self):
+        names_map = {
+            "img_names": "Image name",
+            "pixel_acc": "Pixel accuracy",
+            "precision": "Precision",
+            "recall": "Recall",
+            "f1_score": "F1 score",
+            "iou": "IoU",
+            "boundary_iou": "Boundary IoU",
+            "boundary_eou": "Boundary EoU",
+            "extent_eou": "Extent EoU",
+            "segment_eou": "Segment EoU",
+            "boundary_eou_renormed": "Boundary EoU renormed",
+            "extent_eou_renormed": "Extent EoU renormed",
+            "segment_eou_renormed": "Segment EoU renormed",
+        }
+        prediction_table = self.per_image_metrics.rename(columns=names_map)
+        return prediction_table
+
+    def get_confusion_matrix(self, confusion_matrix: np.ndarray):
+        class_names = self.eval_data.index.tolist()
+        confusion_matrix = confusion_matrix[::-1]
+        return confusion_matrix, class_names
+
+    def get_frequently_confused(self, confusion_matrix: np.ndarray):
+        n_pairs = 20
+
+        non_diagonal_indexes = {}
+        for i, idx in enumerate(np.ndindex(confusion_matrix.shape)):
+            if idx[0] != idx[1]:
+                non_diagonal_indexes[i] = idx
+
+        indexes_1d = np.argsort(confusion_matrix, axis=None)
+        indexes_2d = [
+            non_diagonal_indexes[idx] for idx in indexes_1d if idx in non_diagonal_indexes
+        ][-n_pairs:]
+        indexes_2d = np.asarray(indexes_2d[::-1])
+
+        rows = indexes_2d[:, 0]
+        cols = indexes_2d[:, 1]
+        probs = confusion_matrix[rows, cols]
+        return probs, indexes_2d
+
+    def key_metrics(self):
+        return {
+            "mPixel accuracy": self.pixel_accuracy,
+            "mPrecision": self.precision,
+            "mRecall": self.recall,
+            "mF1-score": self.f1_score,
+            "mIoU": self.iou,
+            "mBoundaryIoU": self.boundary_iou,
+            "mPixel accuracy": self.pixel_accuracy,
+        }
+
+    def get_classwise_error_data(self):
+        self.eval_data.drop(["mean"], inplace=True)
+        bar_data = self.eval_data.copy()
+        bar_data = bar_data[["IoU", "E_extent_oU", "E_boundary_oU", "E_segment_oU"]]
+        bar_data.sort_values(by="IoU", ascending=False, inplace=True)
+        labels = list(bar_data.index)
+        return bar_data, labels

supervisely/nn/benchmark/semantic_segmentation/text_templates.py

@@ -0,0 +1,130 @@
+from types import SimpleNamespace
+
+docs_url = (
+    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/semantic-segmentation"
+)
+
+clickable_label = """
+> <span style="color: #5a6772">
+>     Click on the chart to explore corresponding images.
+> </span>
+"""
+
+markdown_header = """
+<h1>{}</h1>
+
+<div class="model-info-block">
+    <div>Created by <b>{}</b></div>
+    <div><i class="zmdi zmdi-calendar-alt"></i><span>{}</span></div>
+</div>
+"""
+
+
+markdown_overview = """
+- **Model**: {}
+- **Checkpoint**: {}
+- **Architecture**: {}
+- **Task type**: {}
+- **Runtime**: {}
+- **Checkpoint file**: <a class="checkpoint-url" href="{}" target="_blank">{}</a>
+- **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
+{}
+
+Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
+"""
+
+
+markdown_key_metrics = """## Key Metrics
+
+We provide a comprehensive model performance analysis using a set of metrics, including both basic (precision, recall, f1-score, IoU, etc.) and advanced (boundary IoU, error over unio decomposition, etc.) metrics.
+
+- **Pixel accuracy**: reflects the percent of image pixels which were correctly classified.
+- **Precision**: reflects the number of correctly predicted positive segmentations divided by the total number of predicted positive segmentations.
+- **Recall8**: reflects the number of correctly predicted positive segmentations divided by the number of all samples that should have been segmented as positive.
+- **F1-score**: reflects the tradeoff between precision and recall. It is equivalent to the Dice coefficient and calculated as a harmonic mean of precision and recall.
+- **Intersection over union (IoU, also known as the Jaccard index)**: measures the overlap between ground truth mask and predicted mask. It is calculated as the ratio of the intersection of the two masks areas to their combined areas.
+- **Boundary intersection over union**: a segmentation consistency measure that first computes the sets of ground truth  and predicted masks pixels that are located within the distance d from each contour and then computes intersection over union of these two sets. Pixel distance parameter d (pixel width of the boundary region) controls the sensitivity of the metric, it is usually set as 2% of the image diagonal for normal resolution images and 0.5% of the image diagonal for high resolution images.
+- **Error over union and its components (boundary, extent, segment)**: a metric opposite to intersection over union and can be interpreted as what the model lacked in order to show the perfect performance with IoU = 1. It reflects the ratio of incorrectly segmented pixels of ground truth and predicted masks to their combined areas. It is usually decomposed into boundary, extent and segment errors over union in order to get exhaustive information about the model's strengths and weaknesses.
+- **Renormalized error over union**: postprocessed variant of error over union which takes into consideration cause and effect relationships between different types of segmentation errors.
+"""
+
+markdown_explorer = """## Explore Predictions
+This section contains visual comparison of model predictions and ground truth masks. Sometimes a simple visualization can be more informative than any performance metric.
+
+> Click on the image to view the **Original Image** with **Ground Truth** and **Prediction** annotations side-by-side. 
+"""
+
+
+markdown_predictions_table = """### Prediction details for every image
+
+Table of per image metrics allows to get performance metrics for every image. It can be helpful when there is a need to find the most problematic images where the model performed worst.
+
+**Example**: you can sort by **Pixel accuracy** in ascending order to find images where the model performed worst in terms of pixel-wise accuracy.
+
+> Click on the row to view the **Original Image** with **Ground Truth** and **Prediction** annotations side-by-side. 
+"""
+
+markdown_iou = """## Intersection & Error Over Union
+
+The pie chart below demonstrates what the model lacked in order to show the perfect performance with IoU = 1. It is done with the help of Error over Union (EoU) decomposition into boundary, extent and segment errors over union. This chart helps to draw conclusions on the model's strongest and weakest sides.
+"""
+
+
+markdown_renormalized_error_ou = """## Renormalized Error Over Union
+
+The pie chart below is dedicated to decomposition of postprocessed variant of error over union which takes into consideration cause and effect relationships between different types of segmentation errors. Error over union decomposition has its own pitfalls. It is important to understand that models which tend to produce segment errors (when entire segments are mispredicted and there is no intersection between ground truth and predicted mask) will face less occasions to produce boundary and extent errors - as a result, boundary and extent error over union values will be underestimated.
+
+In terms of localization, segment error is more fundamental than extent, while extent error is more fundamental than boundary. In order to overcome this problem, renormalized error over union proposes a slightly different calculation method - by removing more fundamental errors from the denominator - read more in our <a href="{}" target="_blank">technical report</a>
+""".format(
+    docs_url
+)
+
+
+markdown_eou_per_class = """## Classwise Segmentation Error Analysis
+
+This section contains information about classwise segmentation error decomposition. Each column of the chart represents a certain class from the training dataset, demonstrating model performance in terms of segmenting this specific class on images and what model lacked in order to show the perfect performance. All classes are sorted in IoU descending order.
+"""
+
+markdown_confusion_matrix = """## Confusion Matrix
+
+The confusion matrix below reveals which classes the model commonly confuses with each other.
+
+- **Each row** of the matrix corresponds to the actual instances of a class.
+- **Each column** corresponds to the instances as predicted by the model.
+- **The diagonal elements** of the matrix represent correctly predicted instances.
+- By examining the **off-diagonal elements**, you can see if the model is confusing two classes by frequently mislabeling one as the other.
+
+"""
+
+markdown_frequently_confused = """## Frequently Confused Classes
+
+The bar chart below reveals pairs of classes which were most frequently confused by the model. Each column of the chart demonstrates the probability of confusion of a given pair of classes. It is necessary to remember that this probability is not symmetric: the probability of confusing class A with class B is not equal to the probability of confusing class B with class A.
+"""
+
+markdown_frequently_confused_empty = """### Frequently Confused Classes
+
+No frequently confused class pairs found
+"""
+
+markdown_speedtest_intro = """## Inference Speed
+
+This is a speed test benchmark for this model. The model was tested with the following configuration:
+
+- **Device**: {}
+- **Hardware**: {}
+- **Runtime**: {}
+
+The table below shows the speed test results. For each test, the time taken to process one batch of images is shown, as well as the model's throughput (i.e, the number of images processed per second, or FPS). Results are averaged across **{}** iterations.
+"""
+
+markdown_batch_inference = """
+This chart shows how the model's speed changes with different batch sizes. As the batch size increases, you can observe an increase in FPS (images per second).
+"""
+
+markdown_acknowledgement = """---
+### Acknowledgement
+
+[1] Maximilian Bernhard, Roberto Amoroso, Yannic Kindermann, Lorenzo Baraldi, Rita Cucchiara, Volker Tresp, Matthias Schubert. <a href="https://openaccess.thecvf.com/content/WACV2024/html/Bernhard_Whats_Outside_the_Intersection_Fine-Grained_Error_Analysis_for_Semantic_Segmentation_WACV_2024_paper.html" target="_blank">What's Outside the Intersection? Fine-grained Error Analysis for Semantic Segmentation Beyond IoU.</a> In Proceedings of the IEEE / CVF Conference on Computer Vision and Pattern Recognition, pages 969 - 977, 2024.
+
+[2] Bowen Cheng, Ross Girshick, Piotr Dollar, Alexander C. Berg, Alexander Kirillov. <a href="https://arxiv.org/abs/2103.16562" target="_blank">Boundary IoU: Improving object-centric image segmentation evaluation.</a> In Proceedings of the IEEE / CVF Conference on Computer Vision and Pattern Recognition, pages 15334 - 15342, 2021.
+"""

supervisely/nn/benchmark/semantic_segmentation/vis_metrics/acknowledgement.py

@@ -0,0 +1,15 @@
+from supervisely.nn.benchmark.semantic_segmentation.base_vis_metric import (
+    SemanticSegmVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import MarkdownWidget
+
+
+class Acknowledgement(SemanticSegmVisMetric):
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "acknowledgement",
+            "Acknowledgement",
+            text=self.vis_texts.markdown_acknowledgement,
+        )

supervisely/nn/benchmark/semantic_segmentation/vis_metrics/classwise_error_analysis.py

@@ -0,0 +1,57 @@
+from typing import Dict
+
+from supervisely.nn.benchmark.semantic_segmentation.base_vis_metric import (
+    SemanticSegmVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class ClasswiseErrorAnalysis(SemanticSegmVisMetric):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "classwise_error_analysis",
+            "Classwise Segmentation Error Analysis",
+            text=self.vis_texts.markdown_eou_per_class,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget("classwise_error_analysis", self.get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].label}`,",
+        )
+        return chart
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        # # Classwise Segmentation Error Analysis figure
+        bar_data, labels = self.eval_result.mp.classwise_segm_error_data
+        color_palette = ["#8ACAA1", "#FFE4B5", "#F7ADAA", "#dd3f3f"]
+
+        for i, column in enumerate(bar_data.columns):
+            fig.add_trace(
+                go.Bar(
+                    name=column,
+                    y=bar_data[column],
+                    x=labels,
+                    marker_color=color_palette[i],
+                    hovertemplate="Class: %{x}<br> %{name}: %{y:.2f}<extra></extra>",
+                )
+            )
+
+        fig.update_layout(barmode="stack", xaxis_title="Class")
+        if len(labels) < 10:
+            fig.update_layout(width=800)
+        fig.update_yaxes(range=[0, 1])
+
+        return fig