supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/base_visualizer.py

@@ -0,0 +1,265 @@
+from typing import List, Tuple
+
+from supervisely.annotation.annotation import Annotation
+from supervisely.api.api import Api
+from supervisely.api.image_api import ImageInfo
+from supervisely.api.module_api import ApiField
+from supervisely.api.project_api import ProjectInfo
+from supervisely.nn.benchmark.base_evaluator import BaseEvalResult
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.visualization.renderer import Renderer
+from supervisely.nn.benchmark.visualization.widgets import GalleryWidget
+from supervisely.project.project_meta import ProjectMeta
+from supervisely.task.progress import tqdm_sly
+
+
+class MatchedPairData:
+    def __init__(
+        self,
+        gt_image_info: ImageInfo = None,
+        pred_image_info: ImageInfo = None,
+        diff_image_info: ImageInfo = None,
+        gt_annotation: Annotation = None,
+        pred_annotation: Annotation = None,
+        diff_annotation: Annotation = None,
+    ):
+        self.gt_image_info = gt_image_info
+        self.pred_image_info = pred_image_info
+        self.diff_image_info = diff_image_info
+        self.gt_annotation = gt_annotation
+        self.pred_annotation = pred_annotation
+        self.diff_annotation = diff_annotation
+
+
+class BaseVisMetrics:
+
+    def __init__(
+        self,
+        vis_texts,
+        eval_results: List[BaseEvalResult],
+        explore_modal_table: GalleryWidget = None,
+        diff_modal_table: GalleryWidget = None,
+    ) -> None:
+        self.vis_texts = vis_texts
+        self.eval_results = eval_results
+        self.explore_modal_table = explore_modal_table
+        self.diff_modal_table = diff_modal_table
+        self.clickable = False
+
+
+class BaseVisMetric(BaseVisMetrics):
+    def __init__(
+        self,
+        vis_texts,
+        eval_result: BaseEvalResult,
+        explore_modal_table: GalleryWidget = None,
+        diff_modal_table: GalleryWidget = None,
+    ) -> None:
+        super().__init__(vis_texts, [eval_result], explore_modal_table, diff_modal_table)
+        self.eval_result = eval_result
+
+
+class BaseVisualizer:
+    cv_task = None
+
+    def __init__(
+        self,
+        api: Api,
+        eval_results: List[BaseEvalResult],
+        workdir="./visualizations",
+        progress=None,
+    ):
+        self.api = api
+        self.workdir = workdir
+        self.eval_result = eval_results[0]  # for evaluation
+        self.eval_results = eval_results  # for comparison
+
+        self.renderer = None
+        self.gt_project_info = None
+        self.gt_project_meta = None
+        self.gt_dataset_infos = None
+        self.pbar = progress or tqdm_sly
+        self.ann_opacity = 0.4
+
+        with self.pbar(message="Fetching project and dataset infos", total=len(eval_results)) as p:
+            for eval_result in self.eval_results:
+                self._get_eval_project_infos(eval_result)
+                p.update(1)
+
+    def _get_eval_project_infos(self, eval_result):
+        # get project infos
+        if self.gt_project_info is None:
+            self.gt_project_info = self.api.project.get_info_by_id(eval_result.gt_project_id)
+        eval_result.gt_project_info = self.gt_project_info
+        eval_result.pred_project_info = self.api.project.get_info_by_id(eval_result.pred_project_id)
+
+        # get project metas
+        if self.gt_project_meta is None:
+            self.gt_project_meta = ProjectMeta.from_json(
+                self.api.project.get_meta(eval_result.gt_project_id)
+            )
+        eval_result.gt_project_meta = self.gt_project_meta
+        eval_result.pred_project_meta = ProjectMeta.from_json(
+            self.api.project.get_meta(eval_result.pred_project_id)
+        )
+
+        # get dataset infos
+        filters = None
+        if eval_result.gt_dataset_ids is not None:
+            filters = [
+                {
+                    ApiField.FIELD: ApiField.ID,
+                    ApiField.OPERATOR: "in",
+                    ApiField.VALUE: eval_result.gt_dataset_ids,
+                }
+            ]
+        if self.gt_dataset_infos is None:
+            self.gt_dataset_infos = self.api.dataset.get_list(
+                eval_result.gt_project_id,
+                filters=filters,
+                recursive=True,
+            )
+        eval_result.gt_dataset_infos = self.gt_dataset_infos
+        filters = [
+            {
+                ApiField.FIELD: ApiField.NAME,
+                ApiField.OPERATOR: "in",
+                ApiField.VALUE: [ds.name for ds in self.gt_dataset_infos],
+            }
+        ]
+        eval_result.pred_dataset_infos = self.api.dataset.get_list(
+            eval_result.pred_project_id, filters=filters, recursive=True
+        )
+
+        # get train task info
+        train_info = eval_result.train_info
+        if train_info:
+            train_task_id = train_info.get("app_session_id")
+            if train_task_id:
+                eval_result.task_info = self.api.task.get_info_by_id(int(train_task_id))
+
+    def visualize(self):
+        if self.renderer is None:
+            layout = self._create_layout()
+            self.renderer = Renderer(layout, self.workdir)
+        return self.renderer.visualize()
+
+    def upload_results(self, team_id: int, remote_dir: str, progress=None):
+        if self.renderer is None:
+            raise RuntimeError("Visualize first")
+        return self.renderer.upload_results(self.api, team_id, remote_dir, progress)
+
+    def _create_layout(self):
+        raise NotImplementedError("Implement this method in a subclass")
+
+    def _get_or_create_diff_project(self) -> Tuple[ProjectInfo, List, bool]:
+        """
+        Get or create a project for diff visualizations.
+        Dataset hierarchy is copied from the prediction project.
+        """
+
+        pred_ds_id_to_diff_ds_info = {}
+        diff_ds_infos = []
+
+        def _get_or_create_diff_dataset(pred_dataset_id, pred_datasets):
+            if pred_dataset_id in pred_ds_id_to_diff_ds_info:
+                return pred_ds_id_to_diff_ds_info[pred_dataset_id]
+            pred_dataset = pred_datasets[pred_dataset_id]
+            if pred_dataset.parent_id is None:
+                diff_dataset = self.api.dataset.create(project_info.id, pred_dataset.name)
+            else:
+                parent_dataset = _get_or_create_diff_dataset(pred_dataset.parent_id, pred_datasets)
+                diff_dataset = self.api.dataset.create(
+                    project_info.id,
+                    pred_dataset.name,
+                    parent_id=parent_dataset.id,
+                )
+            pred_ds_id_to_diff_ds_info[pred_dataset_id] = diff_dataset
+            diff_ds_infos.append(diff_dataset)
+            return diff_dataset
+
+        project_name = self._generate_diff_project_name(self.eval_result.pred_project_info.name)
+        workspace_id = self.eval_result.pred_project_info.workspace_id
+        project_info = self.api.project.get_info_by_name(
+            workspace_id, project_name, raise_error=False
+        )
+        is_existed = project_info is not None
+        if not is_existed:
+            project_info = self.api.project.create(
+                workspace_id, project_name, change_name_if_conflict=True
+            )
+            pred_datasets = {ds.id: ds for ds in self.eval_result.pred_dataset_infos}
+            for dataset in pred_datasets:
+                _get_or_create_diff_dataset(dataset, pred_datasets)
+        return project_info, diff_ds_infos, is_existed
+
+    def _generate_diff_project_name(self, pred_project_name):
+        return "[diff]: " + pred_project_name
+
+    def _create_explore_modal_table(
+        self, columns_number=3, click_gallery_id=None, hover_text=None
+    ) -> GalleryWidget:
+        gallery = GalleryWidget(
+            "all_predictions_modal_gallery",
+            is_modal=True,
+            columns_number=columns_number,
+            click_gallery_id=click_gallery_id,
+            opacity=self.ann_opacity,
+        )
+        gallery.set_project_meta(self.eval_results[0].filtered_project_meta)
+        if hover_text:
+            gallery.add_image_left_header(hover_text)
+        return gallery
+
+    def _create_diff_modal_table(self, columns_number=3) -> GalleryWidget:
+        gallery = GalleryWidget(
+            "diff_predictions_modal_gallery",
+            is_modal=True,
+            columns_number=columns_number,
+            opacity=self.ann_opacity,
+        )
+        gallery.set_project_meta(self.eval_results[0].filtered_project_meta)
+        return gallery
+
+    def _get_filtered_project_meta(self, eval_result) -> ProjectMeta:
+        remove_classes = []
+        meta = eval_result.pred_project_meta.clone()
+        if eval_result.classes_whitelist:
+            for obj_class in meta.obj_classes:
+                if obj_class.name not in eval_result.classes_whitelist:
+                    remove_classes.append(obj_class.name)
+            if remove_classes:
+                meta = meta.delete_obj_classes(remove_classes)
+        return meta
+
+    def _update_match_data(
+        self,
+        gt_image_id: int,
+        gt_image_info: ImageInfo = None,
+        pred_image_info: ImageInfo = None,
+        diff_image_info: ImageInfo = None,
+        gt_annotation: Annotation = None,
+        pred_annotation: Annotation = None,
+        diff_annotation: Annotation = None,
+    ):
+        match_data = self.eval_result.matched_pair_data.get(gt_image_id, None)
+        if match_data is None:
+            self.eval_result.matched_pair_data[gt_image_id] = MatchedPairData(
+                gt_image_info=gt_image_info,
+                pred_image_info=pred_image_info,
+                diff_image_info=diff_image_info,
+                gt_annotation=gt_annotation,
+                pred_annotation=pred_annotation,
+                diff_annotation=diff_annotation,
+            )
+        else:
+            for attr, value in {
+                "gt_image_info": gt_image_info,
+                "pred_image_info": pred_image_info,
+                "diff_image_info": diff_image_info,
+                "gt_annotation": gt_annotation,
+                "pred_annotation": pred_annotation,
+                "diff_annotation": diff_annotation,
+            }.items():
+                if value is not None:
+                    setattr(match_data, attr, value)

supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py

@@ -160,7 +160,7 @@ markdown_f1_per_class_title = """### F1-score by Class"""
 
 markdown_R = """## Recall
 
-This section measures the ability of the model to detect **all relevant instances in the dataset**. In other words, it answers the question: “Of all instances in the dataset, how many of them is the model managed to find out?”
+This section measures the ability of the model to detect **all relevant instances in the dataset**. In other words, it answers the question: "Of all instances in the dataset, how many of them is the model managed to find out?"
 
 To measure this, we calculate **Recall**. Recall counts errors, when the model does not detect an object that actually is present in a dataset and should be detected. Recall is calculated as the portion of correct predictions (true positives) over all instances in the dataset (true positives + false negatives).
 """
@@ -186,9 +186,9 @@ _Bars in the chart are sorted by <abbr title="{}">F1-score</abbr> to keep a unif
 
 markdown_P = """## Precision
 
-This section measures the accuracy of all predictions made by the model. In other words, it answers the question: “Of all predictions made by the model, how many of them are actually correct?”.
+This section measures the accuracy of all predictions made by the model. In other words, it answers the question: "Of all predictions made by the model, how many of them are actually correct?".
 
-To measure this, we calculate **Precision**. Precision counts errors, when the model predicts an object (bounding box), but the image has no objects of the predicted class in this place. Precision is calculated as a portion of correct predictions (true positives) over all model’s predictions (true positives + false positives).
+To measure this, we calculate **Precision**. Precision counts errors, when the model predicts an object (bounding box), but the image has no objects of the predicted class in this place. Precision is calculated as a portion of correct predictions (true positives) over all model's predictions (true positives + false positives).
 """
 
 notification_precision = {
@@ -283,7 +283,7 @@ markdown_frequently_confused = (
 
 This chart displays the most frequently confused pairs of classes. In general, it finds out which classes visually seem very similar to the model.
 
-The chart calculates the **probability of confusion** between different pairs of classes. For instance, if the probability of confusion for the pair “{} - {}” is {}, this means that when the model predicts either “{}” or “{}”, there is a {}% chance that the model might mistakenly predict one instead of the other.
+The chart calculates the **probability of confusion** between different pairs of classes. For instance, if the probability of confusion for the pair "{} - {}" is {}, this means that when the model predicts either "{}" or "{}", there is a {}% chance that the model might mistakenly predict one instead of the other.
 
 The measure is class-symmetric, meaning that the probability of confusing a {} with a {} is equal to the probability of confusing a {} with a {}.
 
@@ -421,7 +421,7 @@ empty = """### {}
 
 markdown_speedtest_intro = """## Inference Speed
 
-This is a speed test benchmark for compared models. Models was tested with the following configurations:
+This is a speed test benchmark for compared models. Models were tested with the following configurations:
 """
 
 markdown_speedtest_overview_ms = """### Latency (Inference Time)

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py

@@ -139,7 +139,7 @@ class CalibrationScore(BaseVisMetric):
                     x=eval_result.dfsp_down["scores"],
                     y=eval_result.dfsp_down["f1"],
                     mode="lines",
-                    name=f"[{i+1}] {eval_result.name}",
+                    name=f"[{i+1}] {eval_result.model_name}",
                     line=dict(color=eval_result.color),
                     hovertemplate="Confidence Score: %{x:.2f}<br>Value: %{y:.2f}<extra></extra>",
                 )
@@ -194,7 +194,7 @@ class CalibrationScore(BaseVisMetric):
                     x=pred_probs,
                     y=true_probs,
                     mode="lines+markers",
-                    name=f"[{i+1}] {eval_result.name}",
+                    name=f"[{i+1}] {eval_result.model_name}",
                     line=dict(color=eval_result.color),
                     hovertemplate=f"{eval_result.name}<br>"
                     + "Confidence Score: %{x:.2f}<br>Fraction of True Positives: %{y:.2f}<extra></extra>",

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predicttions.py

@@ -2,6 +2,7 @@ from typing import List, Tuple
 
 from supervisely.annotation.annotation import Annotation
 from supervisely.api.image_api import ImageInfo
+from supervisely.api.module_api import ApiField
 from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
     BaseVisMetric,
 )
@@ -30,13 +31,10 @@ class ExplorePredictions(BaseVisMetric):
             self.GALLERY_DIFFERENCE, columns_number=columns_number, filters=default_filters
         )
         gallery.add_image_left_header("Click to explore more")
-        gallery.show_all_button = True
         gallery.set_project_meta(self.eval_results[0].gt_project_meta)
         gallery.set_images(*data)
-        gallery.add_on_click(
-            self.explore_modal_table.id, self.get_click_data_explore_all(), columns_number * 3
-        )
-        gallery._gallery._filters
+        gallery.set_click_data(self.explore_modal_table.id, self.get_click_data_explore_all())
+        gallery.set_show_all_data(self.explore_modal_table.id, self.get_click_data_explore_all())
         gallery._gallery._update_filters()
 
         return gallery
@@ -48,18 +46,31 @@ class ExplorePredictions(BaseVisMetric):
         skip_tags_filtering = []
         api = self.eval_results[0].api
         min_conf = float("inf")
+        names = None
+        ds_name = None
         for idx, eval_res in enumerate(self.eval_results):
             if idx == 0:
-                dataset_info = api.dataset.get_list(eval_res.gt_project_id)[0]
+                dataset_info = eval_res.gt_dataset_infos[0]
                 image_infos = api.image.get_list(dataset_info.id, limit=5)
+                ds_name = dataset_info.name
                 images_ids = [image_info.id for image_info in image_infos]
+                names = [image_info.name for image_info in image_infos]
                 images.append(image_infos)
                 anns = api.annotation.download_batch(dataset_info.id, images_ids)
                 annotations.append(anns)
                 skip_tags_filtering.append(True)
             metas.append(eval_res.dt_project_meta)
-            dataset_info = api.dataset.get_list(eval_res.dt_project_id)[0]
-            image_infos = eval_res.api.image.get_list(dataset_info.id, limit=5)
+            assert ds_name is not None, "Failed to get GT dataset name for gallery"
+
+            dataset_info = api.dataset.get_info_by_name(eval_res.dt_project_id, ds_name)
+
+            assert names is not None, "Failed to get GT image names for gallery"
+            image_infos = eval_res.api.image.get_list(
+                dataset_info.id,
+                filters=[
+                    {ApiField.FIELD: ApiField.NAME, ApiField.OPERATOR: "in", ApiField.VALUE: names}
+                ],
+            )
             images_ids = [image_info.id for image_info in image_infos]
             images.append(image_infos)
             anns = eval_res.api.annotation.download_batch(dataset_info.id, images_ids)
@@ -78,8 +89,9 @@ class ExplorePredictions(BaseVisMetric):
         res["layoutTemplate"] = [None, None, None]
 
         res["layoutTemplate"] = [{"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"}]
-        for i in range(len(self.eval_results)):
-            res["layoutTemplate"].append({"columnTitle": f"Model {i + 1}"})
+        # for i in range(len(self.eval_results)):
+        for idx, eval_res in enumerate(self.eval_results, 1):
+            res["layoutTemplate"].append({"columnTitle": f"[{idx}] {eval_res.model_name}"})
 
         click_data = res.setdefault("clickData", {})
         explore = click_data.setdefault("explore", {})
@@ -88,21 +100,32 @@ class ExplorePredictions(BaseVisMetric):
         images_ids = []
         api = self.eval_results[0].api
         min_conf = float("inf")
+        names = None
+        ds_names = None
         for idx, eval_res in enumerate(self.eval_results):
             if idx == 0:
-                dataset_infos = api.dataset.get_list(eval_res.gt_project_id)
+                dataset_infos = eval_res.gt_dataset_infos
+                ds_names = [ds.name for ds in dataset_infos]
                 current_images_ids = []
+                current_images_names = []
                 for ds in dataset_infos:
-                    image_infos = eval_res.api.image.get_list(ds.id)
+                    image_infos = eval_res.api.image.get_list(ds.id, force_metadata_for_links=False)
+                    image_infos = sorted(image_infos, key=lambda x: x.name)
+                    current_images_names.extend([image_info.name for image_info in image_infos])
                     current_images_ids.extend([image_info.id for image_info in image_infos])
                 images_ids.append(current_images_ids)
+                names = current_images_names
 
-            current_images_ids = []
             dataset_infos = api.dataset.get_list(eval_res.dt_project_id)
+            dataset_infos = [ds for ds in dataset_infos if ds.name in ds_names]
+            dataset_infos = sorted(dataset_infos, key=lambda x: ds_names.index(x.name))
+            current_images_infos = []
             for ds in dataset_infos:
-                image_infos = eval_res.api.image.get_list(ds.id)
-                current_images_ids.extend([image_info.id for image_info in image_infos])
-            images_ids.append(current_images_ids)
+                image_infos = eval_res.api.image.get_list(ds.id, force_metadata_for_links=False)
+                image_infos = [image_info for image_info in image_infos if image_info.name in names]
+                current_images_infos.extend(image_infos)
+            current_images_infos = sorted(current_images_infos, key=lambda x: names.index(x.name))
+            images_ids.append([image_info.id for image_info in current_images_infos])
 
             min_conf = min(min_conf, eval_res.f1_optimal_conf)
 

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/localization_accuracy.py

@@ -90,7 +90,7 @@ class LocalizationAccuracyIoU(BaseVisMetric):
         bin_width = min([bin_edges[1] - bin_edges[0] for _, bin_edges in hist_data])
 
         for i, (eval_result, (hist, bin_edges)) in enumerate(zip(self.eval_results, hist_data)):
-            name = f"[{i+1}] {eval_result.name}"
+            name = f"[{i+1}] {eval_result.model_name}"
             kde = gaussian_kde(eval_result.mp.ious)
             density = kde(x_range)
 

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py

@@ -97,7 +97,7 @@ class OutcomeCounts(BaseVisMetric):
         tp_counts = [eval_result.mp.TP_count for eval_result in self.eval_results][::-1]
         fn_counts = [eval_result.mp.FN_count for eval_result in self.eval_results][::-1]
         fp_counts = [eval_result.mp.FP_count for eval_result in self.eval_results][::-1]
-        model_names = [f"Model {idx}" for idx in range(1, len(self.eval_results) + 1)][::-1]
+        model_names = [f"[{i}] {e.model_name}" for i, e in enumerate(self.eval_results, 1)][::-1]
         counts = [tp_counts, fn_counts, fp_counts]
         names = ["TP", "FN", "FP"]
         colors = ["#8ACAA1", "#dd3f3f", "#F7ADAA"]
@@ -123,7 +123,7 @@ class OutcomeCounts(BaseVisMetric):
         fig = go.Figure()
 
         colors = ["#8ACAA1", "#dd3f3f", "#F7ADAA"]
-        model_names = [f"Model {idx}" for idx in range(1, len(self.eval_results) + 1)][::-1]
+        model_names = [f"[{i}] {e.model_name}" for i, e in enumerate(self.eval_results, 1)][::-1]
         model_names.append("Common")
 
         diff_tps, common_tps = self.common_and_diff_tp
@@ -263,7 +263,7 @@ class OutcomeCounts(BaseVisMetric):
         res["layoutTemplate"] = [None, None, None]
         res["clickData"] = {}
         for i, eval_result in enumerate(self.eval_results, 1):
-            model_name = f"Model {i}"
+            model_name = f"[{i}] {eval_result.model_name}"
             for outcome, matches_data in eval_result.click_data.outcome_counts.items():
                 key = f"{model_name}_{outcome}"
                 outcome_dict = res["clickData"].setdefault(key, {})
@@ -327,7 +327,7 @@ class OutcomeCounts(BaseVisMetric):
             _update_outcome_dict("Common", outcome, outcome_dict, common_ids)
 
             for i, diff_ids in enumerate(diff_ids, 1):
-                name = f"Model {i}"
+                name = f"[{i}] {self.eval_results[i - 1].model_name}"
                 key = f"{name}_{outcome}"
                 outcome_dict = res["clickData"].setdefault(key, {})
 

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -1,10 +1,10 @@
 from typing import List
 
 from supervisely._utils import abs_url
-from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
 from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
     BaseVisMetric,
 )
+from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
 from supervisely.nn.benchmark.visualization.widgets import (
     ChartWidget,
     MarkdownWidget,
@@ -162,10 +162,11 @@ class Overview(BaseVisMetric):
         train_session, images_str = "", ""
         gt_project_id = eval_result.gt_project_info.id
         gt_dataset_ids = eval_result.gt_dataset_ids
-        gt_images_ids = eval_result.gt_images_ids
+        gt_images_cnt = eval_result.val_images_cnt
         train_info = eval_result.train_info
-        if gt_images_ids is not None:
-            val_imgs_cnt = len(gt_images_ids)
+        total_imgs_cnt = eval_result.gt_project_info.items_count
+        if gt_images_cnt is not None:
+            val_imgs_cnt = gt_images_cnt
         elif gt_dataset_ids is not None:
             datasets = eval_result.gt_dataset_infos
             val_imgs_cnt = sum(ds.items_count for ds in datasets)
@@ -182,18 +183,18 @@ class Overview(BaseVisMetric):
             train_imgs_cnt = train_info.get("images_count")
             images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
 
-        if gt_images_ids is not None:
-            images_str += f". Evaluated using subset - {val_imgs_cnt} images"
+        if gt_images_cnt is not None:
+            images_str += (
+                f", total {total_imgs_cnt} images. Evaluated using subset - {val_imgs_cnt} images"
+            )
         elif gt_dataset_ids is not None:
             links = [
                 f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
                 for ds in datasets
             ]
-            images_str += (
-                f". Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
-            )
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
         else:
-            images_str += f". Evaluated on the whole project ({val_imgs_cnt} images)"
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the whole project ({val_imgs_cnt} images)"
 
         return classes_str, images_str, train_session
 
@@ -203,7 +204,7 @@ class Overview(BaseVisMetric):
         # Overall Metrics
         fig = go.Figure()
         for i, eval_result in enumerate(self.eval_results):
-            name = f"[{i + 1}] {eval_result.name}"
+            name = f"[{i + 1}] {eval_result.model_name}"
             base_metrics = eval_result.mp.base_metrics()
             r = list(base_metrics.values())
             theta = [eval_result.mp.metric_names[k] for k in base_metrics.keys()]

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py

@@ -98,7 +98,7 @@ class PrCurve(BaseVisMetric):
             pr_curve[pr_curve == -1] = np.nan
             pr_curve = np.nanmean(pr_curve, axis=-1)
 
-            name = f"[{i}] {eval_result.name}"
+            name = f"[{i}] {eval_result.model_name}"
             color = ",".join(map(str, hex2rgb(eval_result.color))) + ",0.1"
             line = go.Scatter(
                 x=eval_result.mp.recThrs,

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py

@@ -136,7 +136,7 @@ class PrecisionRecallF1(BaseVisMetric):
             precision = eval_result.mp.json_metrics()["precision"]
             recall = eval_result.mp.json_metrics()["recall"]
             f1 = eval_result.mp.json_metrics()["f1"]
-            model_name = f"[{i}] {eval_result.name}"
+            model_name = f"[{i}] {eval_result.model_name}"
             fig.add_trace(
                 go.Bar(
                     x=["Precision", "Recall", "F1-score"],
@@ -163,7 +163,7 @@ class PrecisionRecallF1(BaseVisMetric):
         fig = go.Figure()
         classes_cnt = len(self.eval_results[0].mp.cat_names)
         for i, eval_result in enumerate(self.eval_results, 1):
-            model_name = f"[{i}] {eval_result.name}"
+            model_name = f"[{i}] {eval_result.model_name}"
             sorted_by_f1 = eval_result.mp.per_class_metrics().sort_values(by="f1")
 
             fig.add_trace(
@@ -191,7 +191,7 @@ class PrecisionRecallF1(BaseVisMetric):
         res["layoutTemplate"] = [None, None, None]
         res["clickData"] = {}
         for i, eval_result in enumerate(self.eval_results):
-            model_name = f"Model [{i + 1}] {eval_result.name}"
+            model_name = f"Model [{i + 1}] {eval_result.model_name}"
             for key, v in eval_result.click_data.objects_by_class.items():
                 click_data = res["clickData"].setdefault(f"{i}_{key}", {})
                 img_ids, obj_ids = set(), set()
@@ -220,7 +220,7 @@ class PrecisionRecallF1(BaseVisMetric):
         fig = go.Figure()
         classes_cnt = len(self.eval_results[0].mp.cat_names)
         for i, eval_result in enumerate(self.eval_results, 1):
-            model_name = f"[{i}] {eval_result.name}"
+            model_name = f"[{i}] {eval_result.model_name}"
             sorted_by_f1 = eval_result.mp.per_class_metrics().sort_values(by="f1")
 
             fig.add_trace(
@@ -249,7 +249,7 @@ class PrecisionRecallF1(BaseVisMetric):
         fig = go.Figure()
         classes_cnt = len(self.eval_results[0].mp.cat_names)
         for i, eval_result in enumerate(self.eval_results, 1):
-            model_name = f"[{i}] {eval_result.name}"
+            model_name = f"[{i}] {eval_result.model_name}"
             sorted_by_f1 = eval_result.mp.per_class_metrics().sort_values(by="f1")
 
             fig.add_trace(
@@ -278,7 +278,7 @@ class PrecisionRecallF1(BaseVisMetric):
         res["clickData"] = {}
 
         for i, eval_result in enumerate(self.eval_results):
-            model_name = f"Model [{i + 1}] {eval_result.name}"
+            model_name = f"Model [{i + 1}] {eval_result.model_name}"
             click_data = res["clickData"].setdefault(i, {})
             img_ids, obj_ids = set(), set()
             objects_cnt = 0

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/speedtest.py

@@ -248,7 +248,7 @@ class Speedtest(BaseVisMetric):
 
         fig = make_subplots(cols=2)
 
-        for eval_result in self.eval_results:
+        for idx, eval_result in enumerate(self.eval_results, 1):
             if eval_result.speedtest_info is None:
                 continue
             temp_res = {}
@@ -272,7 +272,7 @@ class Speedtest(BaseVisMetric):
                 go.Scatter(
                     x=list(temp_res["ms"].keys()),
                     y=list(temp_res["ms"].values()),
-                    name="Infrence time (ms)",
+                    name=f"[{idx}] {eval_result.model_name} (ms)",
                     line=dict(color=eval_result.color),
                     customdata=list(temp_res["ms_std"].values()),
                     error_y=dict(
@@ -290,7 +290,7 @@ class Speedtest(BaseVisMetric):
                 go.Scatter(
                     x=list(temp_res["fps"].keys()),
                     y=list(temp_res["fps"].values()),
-                    name="FPS",
+                    name=f"[{idx}] {eval_result.model_name} (fps)",
                     line=dict(color=eval_result.color),
                     hovertemplate="Batch Size: %{x}<br>FPS: %{y:.2f}<extra></extra>",  # <br> Standard deviation: %{customdata:.2f}<extra></extra>",
                 ),

supervisely/nn/benchmark/{instance_segmentation_benchmark.py → instance_segmentation/benchmark.py}

@@ -1,12 +1,18 @@
-from supervisely.nn.benchmark.base_benchmark import BaseBenchmark
 from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.evaluation import InstanceSegmentationEvaluator
+from supervisely.nn.benchmark.instance_segmentation.evaluator import (
+    InstanceSegmentationEvaluator,
+)
+from supervisely.nn.benchmark.instance_segmentation.visualizer import (
+    InstanceSegmentationVisualizer,
+)
+from supervisely.nn.benchmark.object_detection.benchmark import ObjectDetectionBenchmark
 from supervisely.nn.benchmark.utils import try_set_conf_auto
 
 CONF_THRES = 0.05
 
 
-class InstanceSegmentationBenchmark(BaseBenchmark):
+class InstanceSegmentationBenchmark(ObjectDetectionBenchmark):
+    visualizer_cls = InstanceSegmentationVisualizer
 
     @property
     def cv_task(self) -> str: