supervisely 6.73.282__py3-none-any.whl → 6.73.284__py3-none-any.whl

supervisely/convert/base_converter.py

@@ -168,6 +168,7 @@ class BaseConverter:
         self._upload_as_links: bool = upload_as_links
         self._remote_files_map: Optional[Dict[str, str]] = remote_files_map
         self._supports_links = False  # if converter supports uploading by links
+        self._force_shape_for_links = False
         self._api = Api.from_env() if self._upload_as_links else None
         self._team_id = team_id() if self._upload_as_links else None
         self._converter = None

supervisely/convert/image/cityscapes/cityscapes_converter.py

@@ -34,6 +34,8 @@ class CityscapesConverter(ImageConverter):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
 
         self._classes_mapping = {}
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self):
         return AvailableImageConverters.CITYSCAPES
@@ -41,6 +43,10 @@ class CityscapesConverter(ImageConverter):
     @property
     def key_file_ext(self) -> str:
         return ".json"
+    
+    @property
+    def ann_ext(self) -> str:
+        return ".json"
 
     def ann_file_ext(self) -> str:
         return ".json"
@@ -108,6 +114,8 @@ class CityscapesConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         images_list, ann_dict = [], {}
         for root, _, files in os.walk(self._input_data):

supervisely/convert/image/coco/coco_anntotation_converter.py

@@ -37,6 +37,7 @@ class FastCOCOConverter(COCOConverter, ImageConverter):
         self._items = []
         meta = ProjectMeta()
         warnings = defaultdict(list)
+        item_names = set()
         for ann_path in ann_paths:
             try:
                 with coco_helper.HiddenCocoPrints():
@@ -74,11 +75,18 @@ class FastCOCOConverter(COCOConverter, ImageConverter):
                 coco_ann = coco_anns[image_id]
                 if len(coco_ann) == 0 or coco_ann is None or image_name is None:
                     continue
+                if image_name in item_names:
+                    # * Block to handle the case when there are mixed annotations: caption and segmentations for the same images
+                    item = next(item for item in self._items if item.name == image_name)
+                    if item.shape == (height, width):
+                        item.ann_data.extend(coco_ann)
+                        continue
                 item = self.Item(image_name) if image_url is None else self.Item(image_url)
                 item.name = image_name
                 item.ann_data = coco_ann
                 item.set_shape((height, width))
                 self._items.append(item)
+                item_names.add(image_name)
                 detected_ann_cnt += len(coco_ann)
 
         self._meta = meta

supervisely/convert/image/coco/coco_converter.py

@@ -25,6 +25,8 @@ class COCOConverter(ImageConverter):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
 
         self._coco_categories = []
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self) -> str:
         return AvailableImageConverters.COCO
@@ -56,6 +58,8 @@ class COCOConverter(ImageConverter):
     def validate_format(self) -> bool:
         from pycocotools.coco import COCO  # pylint: disable=import-error
 
+        if self.upload_as_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         images_list, ann_paths = [], []
         for root, _, files in os.walk(self._input_data):
@@ -145,7 +149,8 @@ class COCOConverter(ImageConverter):
         if item.ann_data is None:
             return Annotation.from_img_path(item.path)
         else:
-            item.set_shape()
+            if not self.upload_as_links:
+                item.set_shape()
             ann = coco_helper.create_supervisely_annotation(
                 item,
                 meta,

supervisely/convert/image/coco/coco_helper.py

@@ -182,7 +182,7 @@ def convert_rle_mask_to_polygon(coco_ann):
     return Bitmap(mask).to_contours()
 
 
-def convert_polygon_vertices(coco_ann, image_size):
+def convert_polygon_vertices(coco_ann, image_size: Tuple[int, int]):
     polygons = coco_ann["segmentation"]
     if all(type(coord) is float for coord in polygons):
         polygons = [polygons]

supervisely/convert/image/image_converter.py

@@ -136,20 +136,16 @@ class ImageConverter(BaseConverter):
             item_names = []
             item_paths = []
             item_metas = []
-            anns = []
             for item in batch:
                 item.path = self.validate_image(item.path)
                 if item.path is None:
                     continue  # image has failed validation
-                item.name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
-                if self.upload_as_links and not self.supports_links:
-                    ann = None
-                else:
-                    ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
-                name = generate_free_name(
-                    existing_names, item.name, with_ext=True, extend_used_names=True
+                name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
+
+                item.name = generate_free_name(
+                    existing_names, name, with_ext=True, extend_used_names=True
                 )
-                item_names.append(name)
+                item_names.append(item.name)
                 item_paths.append(item.path)
 
                 if isinstance(item.meta, str):  # path to file
@@ -159,9 +155,6 @@ class ImageConverter(BaseConverter):
                 else:
                     item_metas.append({})
 
-                if ann is not None:
-                    anns.append(ann)
-
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
@@ -173,7 +166,7 @@ class ImageConverter(BaseConverter):
                         metas=item_metas,
                         batch_size=batch_size,
                         conflict_resolution="rename",
-                        force_metadata_for_links=False,
+                        force_metadata_for_links=self._force_shape_for_links,
                     )
                 else:
                     img_infos = api.image.upload_paths(
@@ -183,8 +176,15 @@ class ImageConverter(BaseConverter):
                         metas=item_metas,
                         conflict_resolution="rename",
                     )
-
                 img_ids = [img_info.id for img_info in img_infos]
+
+                anns = []
+                if not (self.upload_as_links and not self.supports_links):
+                    for info, item in zip(img_infos, batch):
+                        if self._force_shape_for_links:
+                            item.set_shape((info.height, info.width))
+                        anns.append(self.to_supervisely(item, meta, renamed_classes, renamed_tags))
+
                 if len(anns) == len(img_ids):
                     api.annotation.upload_anns(
                         img_ids, anns, skip_bounds_validation=self.upload_as_links

supervisely/convert/image/multi_view/multi_view.py

@@ -1,6 +1,6 @@
 import os
 from collections import defaultdict
-from typing import Dict, Union
+from typing import Dict, Union, Optional
 
 from supervisely import ProjectMeta, generate_free_name, is_development, logger
 from supervisely.api.api import Api, ApiContext
@@ -13,6 +13,18 @@ from supervisely.project.project_settings import LabelingInterface
 
 class MultiViewImageConverter(ImageConverter):
 
+    def __init__(
+            self,
+            input_data: str,
+            labeling_interface: Optional[Union[LabelingInterface, str]],
+            upload_as_links: bool,
+            remote_files_map: Optional[Dict[str, str]] = None,
+    ):
+        super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
+
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
+
     def __str__(self):
         return AvailableImageConverters.MULTI_VIEW
 
@@ -79,13 +91,16 @@ class MultiViewImageConverter(ImageConverter):
                     logger.warn(f"Image '{name}' already exists. Renamed to '{new_name}'.")
                     os.rename(image, os.path.join(group_path, new_name))
                     image = os.path.join(group_path, new_name)
+                if self._upload_as_links:
+                    image = self.remote_files_map.get(image, image)
                 images.append(image)
 
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
+                kwarg = {"links": images} if self.upload_as_links else {"paths": images}
                 api.image.upload_multiview_images(
-                    dataset.id, group_name, images, progress_cb=progress_cb
+                    dataset.id, group_name, **kwarg, progress_cb=progress_cb
                 )
 
         if log_progress:

supervisely/convert/image/yolo/yolo_converter.py

@@ -38,6 +38,8 @@ class YOLOConverter(ImageConverter):
         self._coco_classes_dict: dict = {}
         self._num_kpts = None
         self._num_dims = None
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self) -> str:
         return AvailableImageConverters.YOLO
@@ -151,6 +153,9 @@ class YOLOConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links:
+            self._download_remote_ann_files()
+
         detected_ann_cnt = 0
         config_path = None
         images_list, ann_dict = [], {}
@@ -238,7 +243,8 @@ class YOLOConverter(ImageConverter):
 
         try:
             labels = []
-            item.set_shape()
+            if not self.upload_as_links:
+                item.set_shape()
             height, width = item.shape
             with open(item.ann_data, "r") as ann_file:
                 lines = ann_file.readlines()

supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py

@@ -61,35 +61,27 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             return False
 
         def filter_fn(path):
-            return all(
-                [
-                    (Path(path) / name).exists()
-                    for name in ["maps", "samples", "sweeps", "v1.0-mini"]
-                ]
-            )
+            return all([(Path(path) / name).exists() for name in ["maps", "samples"]])
 
-        try:
-            input_path = [d for d in fs.dirs_filter(self._input_data, filter_fn)].pop()
-        except IndexError:
+        input_path = next((d for d in fs.dirs_filter(self._input_data, filter_fn)), None)
+        if input_path is None:
             return False
 
         sample_dir = input_path + "/samples/"
         if any([not fs.dir_exists(f"{sample_dir}/{d}") for d in helpers.DIR_NAMES]):
             return False
 
-        sweeps_dir = input_path + "/sweeps/"
-        if any([not fs.dir_exists(f"{sweeps_dir}/{d}") for d in helpers.DIR_NAMES]):
-            return False
-
-        ann_dir = input_path + "/v1.0-mini/"
-        if any([not fs.file_exists(f"{ann_dir}/{d}.json") for d in helpers.TABLE_NAMES]):
+        fil_fn = lambda p: all(fs.file_exists(f"{p}/{name}.json") for name in helpers.TABLE_NAMES)
+        ann_dir = next((d for d in fs.dirs_filter(input_path, fil_fn)), None)
+        if ann_dir is None:
             return False
 
+        version = osp.basename(ann_dir)
         try:
             t = TinyTimer()
-            nuscenes = NuScenes(dataroot=input_path, verbose=False)
+            nuscenes = NuScenes(version=version, dataroot=input_path, verbose=False)
             self._nuscenes: NuScenes = nuscenes
-            logger.info(f"NuScenes initialization took {t.get_sec():.3f} sec")
+            logger.debug(f"NuScenes initialization took {t.get_sec():.3f} sec")
         except Exception as e:
             logger.debug(f"Failed to initialize NuScenes: {e}")
             return False
@@ -184,7 +176,9 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             scene_name_to_dataset[scene_names[0]] = dataset_info
 
         if log_progress:
-            progress, progress_cb = self.get_progress(total_sample_cnt, "Converting episode scenes...")
+            progress, progress_cb = self.get_progress(
+                total_sample_cnt, "Converting episode scenes..."
+            )
         else:
             progress_cb = None
 

supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py

@@ -16,11 +16,11 @@ DIR_NAMES = [
     "CAM_FRONT_LEFT",
     "CAM_FRONT_RIGHT",
     "LIDAR_TOP",
-    "RADAR_FRONT",
-    "RADAR_FRONT_LEFT",
-    "RADAR_FRONT_RIGHT",
-    "RADAR_BACK_LEFT",
-    "RADAR_BACK_RIGHT",
+    # "RADAR_FRONT",
+    # "RADAR_FRONT_LEFT",
+    # "RADAR_FRONT_RIGHT",
+    # "RADAR_BACK_LEFT",
+    # "RADAR_BACK_RIGHT",
 ]
 
 TABLE_NAMES = [

supervisely/nn/benchmark/base_benchmark.py

@@ -1,5 +1,6 @@
 import os
-from typing import Callable, List, Optional, Tuple, Union
+from pathlib import Path
+from typing import Callable, List, Optional, Tuple, Union, Type
 
 import numpy as np
 
@@ -80,7 +81,7 @@ class BaseBenchmark:
         self.report_id = None
         self._validate_evaluation_params()
 
-    def _get_evaluator_class(self) -> type:
+    def _get_evaluator_class(self) -> Type[BaseEvaluator]:
         raise NotImplementedError()
 
     @property
@@ -95,6 +96,10 @@ class BaseBenchmark:
     def key_metrics(self):
         eval_results = self.get_eval_result()
         return eval_results.key_metrics
+    
+    @property
+    def primary_metric_name(self) -> str:
+        return self._get_evaluator_class().eval_result_cls.PRIMARY_METRIC
 
     def run_evaluation(
         self,
@@ -492,6 +497,8 @@ class BaseBenchmark:
                 "It should be defined in the subclass of BaseBenchmark (e.g. ObjectDetectionBenchmark)."
             )
         eval_result = self.get_eval_result()
+        self._dump_key_metrics(eval_result)
+
         layout_dir = self.get_layout_results_dir()
         self.visualizer = self.visualizer_cls(  # pylint: disable=not-callable
             self.api, [eval_result], layout_dir, self.pbar
@@ -621,3 +628,7 @@ class BaseBenchmark:
             self.diff_project_info = eval_result.diff_project_info
             return self.diff_project_info
         return None
+
+    def _dump_key_metrics(self, eval_result: BaseEvaluator):
+        path = str(Path(self.get_eval_results_dir(), "key_metrics.json"))
+        json.dump_json_file(eval_result.key_metrics, path)

supervisely/nn/benchmark/base_evaluator.py

@@ -12,6 +12,8 @@ from supervisely.task.progress import tqdm_sly
 
 
 class BaseEvalResult:
+    PRIMARY_METRIC = None
+
     def __init__(self, directory: Optional[str] = None):
         self.directory = directory
         self.inference_info: Dict = None

supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py

@@ -87,6 +87,11 @@ In this section you can visually assess the model performance through examples.
 > Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
 """
 
+markdown_different_iou_thresholds_warning = """### IoU Thresholds Mismatch
+
+<i class="zmdi zmdi-alert-polygon" style="color: #f5a623; margin-right: 5px"></i> The models were evaluated using different IoU thresholds. Since these thresholds varied between models and classes, it may have led to unfair comparison. For fair model comparison, we suggest using the same IoU threshold across models.
+"""
+
 markdown_explore_difference = """## Explore Predictions
 
 In this section, you can explore predictions made by different models side-by-side. This helps you to understand the differences in predictions made by each model, and to identify which model performs better in different scenarios.

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from typing import List
 
 from supervisely._utils import abs_url
@@ -15,6 +16,7 @@ class Overview(BaseVisMetrics):
     MARKDOWN_OVERVIEW = "markdown_overview"
     MARKDOWN_OVERVIEW_INFO = "markdown_overview_info"
     MARKDOWN_COMMON_OVERVIEW = "markdown_common_overview"
+    MARKDOWN_DIFF_IOU = "markdown_different_iou_thresholds_warning"
     CHART = "chart_key_metrics"
 
     def __init__(self, vis_texts, eval_results: List[EvalResult]) -> None:
@@ -237,3 +239,26 @@ class Overview(BaseVisMetrics):
             ),
         )
         return fig
+
+    @property
+    def not_matched_iou_per_class_thresholds_md(self) -> MarkdownWidget:
+        if all([not r.different_iou_thresholds_per_class for r in self.eval_results]):
+            return None
+
+        iou_thrs_map = defaultdict(set)
+        matched = True
+        for eval_result in self.eval_results:
+            for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
+                iou_thrs_map[cat_id].add(iou_thr)
+                if len(iou_thrs_map[cat_id]) > 1:
+                    matched = False
+                    break
+
+        if matched:
+            return None
+
+        return MarkdownWidget(
+            name="markdown_different_iou_thresholds_warning",
+            title="IoU per class thresholds mismatch",
+            text=self.vis_texts.markdown_different_iou_thresholds_warning,
+        )

supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py

@@ -50,10 +50,9 @@ class DetectionComparisonVisualizer(BaseComparisonVisualizer):
         self.overviews = self._create_overviews(overview)
         self.overview_md = overview.overview_md
         self.key_metrics_md = self._create_key_metrics()
-        self.key_metrics_table = overview.get_table_widget(
-            latency=speedtest.latency, fps=speedtest.fps
-        )
+        self.key_metrics_table = overview.get_table_widget(speedtest.latency, speedtest.fps)
         self.overview_chart = overview.chart_widget
+        self.iou_per_class_thresholds_md = overview.not_matched_iou_per_class_thresholds_md
 
         columns_number = len(self.comparison.eval_results) + 1  # +1 for GT
         self.explore_predictions_modal_gallery = self._create_explore_modal_table(columns_number)
@@ -154,6 +153,13 @@ class DetectionComparisonVisualizer(BaseComparisonVisualizer):
             (0, self.header),
             (1, self.overview_md),
             (0, self.overviews),
+        ]
+
+        if self.iou_per_class_thresholds_md is not None:
+            is_anchors_widgets.append((0, self.iou_per_class_thresholds_md))
+
+        is_anchors_widgets += [
+            # Key Metrics
             (1, self.key_metrics_md),
             (0, self.key_metrics_table),
             (0, self.overview_chart),

supervisely/nn/benchmark/instance_segmentation/evaluator.py

@@ -14,6 +14,7 @@ from supervisely.nn.benchmark.utils import calculate_metrics, read_coco_datasets
 
 class InstanceSegmentationEvalResult(ObjectDetectionEvalResult):
     mp_cls = MetricProvider
+    PRIMARY_METRIC = "mAP"
 
     @classmethod
     def from_evaluator(

supervisely/nn/benchmark/instance_segmentation/text_templates.py

@@ -60,6 +60,13 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
+markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+
+markdown_iou_per_class = """### IoU Threshold per Class
+
+The model is evaluated using different IoU thresholds for each class.
+"""
+
 markdown_explorer = """## Explore Predictions
 In this section you can visually assess the model performance through examples. This helps users better understand model capabilities and limitations, giving an intuitive grasp of prediction quality in different scenarios.
 

supervisely/nn/benchmark/object_detection/evaluator.py

@@ -19,6 +19,7 @@ from supervisely.nn.benchmark.visualization.vis_click_data import ClickData, IdM
 
 class ObjectDetectionEvalResult(BaseEvalResult):
     mp_cls = MetricProvider
+    PRIMARY_METRIC = "mAP"
 
     def _read_files(self, path: str) -> None:
         """Read all necessary files from the directory"""
@@ -92,6 +93,10 @@ class ObjectDetectionEvalResult(BaseEvalResult):
     def key_metrics(self):
         return self.mp.key_metrics()
 
+    @property
+    def different_iou_thresholds_per_class(self) -> bool:
+        return self.mp.iou_threshold_per_class is not None
+
 
 class ObjectDetectionEvaluator(BaseEvaluator):
     EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"
@@ -120,12 +125,19 @@ class ObjectDetectionEvaluator(BaseEvaluator):
 
     @classmethod
     def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        available_thres = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
         iou_threshold = evaluation_params.get("iou_threshold")
         if iou_threshold is not None:
-            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
-                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
-                f"but got {iou_threshold}"
+            assert iou_threshold in available_thres, (
+                f"iou_threshold must be one of {available_thres}, " f"but got {iou_threshold}"
             )
+        iou_threshold_per_class = evaluation_params.get("iou_threshold_per_class")
+        if iou_threshold_per_class is not None:
+            for class_name, iou_thres in iou_threshold_per_class.items():
+                assert iou_thres in available_thres, (
+                    f"class {class_name}: iou_threshold_per_class must be one of {available_thres}, "
+                    f"but got {iou_thres}"
+                )
 
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(

supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -92,7 +92,11 @@ class MetricProvider:
 
         eval_params = params.get("evaluation_params", {})
         self.iou_threshold = eval_params.get("iou_threshold", 0.5)
-        self.iou_threshold_idx = np.searchsorted(self.iouThrs, self.iou_threshold)
+        self.iou_threshold_idx = np.where(np.isclose(self.iouThrs, self.iou_threshold))[0][0]
+
+        # IoU per class (optional)
+        self.iou_threshold_per_class = eval_params.get("iou_threshold_per_class")
+        self.iou_idx_per_class = params.get("iou_idx_per_class")  # {cat id: iou_idx}
 
     def calculate(self):
         self.m_full = _MetricProvider(
@@ -142,6 +146,8 @@ class MetricProvider:
     def json_metrics(self):
         base = self.base_metrics()
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         ap_by_class = self.AP_per_class().tolist()
         ap_by_class = dict(zip(self.cat_names, ap_by_class))
         ap_custom_by_class = self.AP_custom_per_class().tolist()
@@ -166,6 +172,8 @@ class MetricProvider:
 
     def key_metrics(self):
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         json_metrics = self.json_metrics()
         json_metrics.pop("AP_by_class")
         json_metrics.pop(f"AP{iou_name}_by_class")
@@ -174,6 +182,8 @@ class MetricProvider:
     def metric_table(self):
         table = self.json_metrics()
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         return {
             "mAP": table["mAP"],
             "AP50": table["AP50"],
@@ -196,6 +206,10 @@ class MetricProvider:
 
     def AP_custom_per_class(self):
         s = self.coco_precision[self.iou_threshold_idx, :, :, 0, 2]
+        s = s.copy()
+        if self.iou_threshold_per_class is not None:
+            for cat_id, iou_idx in self.iou_idx_per_class.items():
+                s[:, cat_id - 1] = self.coco_precision[iou_idx, :, cat_id - 1, 0, 2]
         s[s == -1] = np.nan
         ap = np.nanmean(s, axis=0)
         return ap
@@ -280,6 +294,9 @@ class _MetricProvider:
             ious.append(match["iou"])
             cats.append(cat_id_to_idx[match["category_id"]])
         ious = np.array(ious) + np.spacing(1)
+        if 0.8999999999999999 in iouThrs:
+            iouThrs = iouThrs.copy()
+            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious) - 1
         cats = np.array(cats)
         # TP
@@ -452,6 +469,9 @@ class _MetricProvider:
         )
         scores = np.array([m["score"] for m in matches_sorted])
         ious = np.array([m["iou"] if m["type"] == "TP" else 0.0 for m in matches_sorted])
+        if 0.8999999999999999 in iouThrs:
+            iouThrs = iouThrs.copy()
+            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious + np.spacing(1))
 
         # Check

supervisely/nn/benchmark/object_detection/text_templates.py

@@ -65,6 +65,13 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
+markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+
+markdown_iou_per_class = """### IoU Threshold per Class
+
+The model is evaluated using different IoU thresholds for each class.
+"""
+
 markdown_explorer = """## Explore Predictions
 In this section you can visually assess the model performance through examples. This helps users better understand model capabilities and limitations, giving an intuitive grasp of prediction quality in different scenarios.
 

supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py

@@ -29,6 +29,8 @@ class KeyMetrics(DetectionVisMetric):
         columns = ["metrics", "values"]
         content = []
         for metric, value in self.eval_result.mp.metric_table().items():
+            if metric == "AP_custom":
+                metric += "*"
             row = [metric, round(value, 2)]
             dct = {
                 "row": row,
@@ -134,3 +136,13 @@ class KeyMetrics(DetectionVisMetric):
             ]
 
         return res
+
+    @property
+    def custom_ap_description_md(self) -> MarkdownWidget:
+        if not self.eval_result.different_iou_thresholds_per_class:
+            return None
+        return MarkdownWidget(
+            "custom_ap_description",
+            "Custom AP per Class",
+            self.vis_texts.markdown_AP_custom_description,
+        )

supervisely/nn/benchmark/object_detection/vis_metrics/overview.py

@@ -2,7 +2,7 @@ import datetime
 from typing import List
 
 from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
-from supervisely.nn.benchmark.visualization.widgets import MarkdownWidget
+from supervisely.nn.benchmark.visualization.widgets import MarkdownWidget, TableWidget
 
 
 class Overview(DetectionVisMetric):
@@ -32,6 +32,10 @@ class Overview(DetectionVisMetric):
         # link to scroll to the optimal confidence section
         opt_conf_url = self.vis_texts.docs_url + "#f1-optimal-confidence-threshold"
 
+        iou_threshold = self.eval_result.mp.iou_threshold
+        if self.eval_result.different_iou_thresholds_per_class:
+            iou_threshold = "Different IoU thresholds for each class (see the table below)"
+
         formats = [
             model_name.replace("_", "\_"),
             checkpoint_name.replace("_", "\_"),
@@ -45,7 +49,7 @@ class Overview(DetectionVisMetric):
             classes_str,
             note_about_images,
             starter_app_info,
-            self.eval_result.mp.iou_threshold,
+            iou_threshold,
             round(self.eval_result.mp.f1_optimal_conf, 4),
             opt_conf_url,
             self.vis_texts.docs_url,
@@ -112,3 +116,38 @@ class Overview(DetectionVisMetric):
         starter_app_info = train_session or evaluator_session or ""
 
         return classes_str, images_str, starter_app_info
+    
+    @property
+    def iou_per_class_md(self) -> List[MarkdownWidget]:
+        if not self.eval_result.different_iou_thresholds_per_class:
+            return None
+
+        return MarkdownWidget(
+            "markdown_iou_per_class",
+            "Different IoU thresholds for each class",
+            text=self.vis_texts.markdown_iou_per_class,
+        )
+
+    @property
+    def iou_per_class_table(self) -> TableWidget:
+        if not self.eval_result.different_iou_thresholds_per_class:
+            return None
+
+        content = []
+        for name, thr in self.eval_result.mp.iou_threshold_per_class.items():
+            row = [name, round(thr, 2)]
+            dct = {"row": row, "id": name, "items": row}
+            content.append(dct)
+
+        data = {
+            "columns": ["Class name", "IoU threshold"],
+            "columnsOptions": [{"disableSort": True}, {}],
+            "content": content,
+        }
+        return TableWidget(
+            name="table_iou_per_class",
+            data=data,
+            fix_columns=1,
+            width="60%",
+            main_column="Class name",
+        )

supervisely/nn/benchmark/object_detection/visualizer.py

@@ -90,11 +90,16 @@ class ObjectDetectionVisualizer(BaseVisualizer):
         self.header = overview.get_header(me.login)
         self.overview_md = overview.md
 
+        # IOU Per Class (optional)
+        self.iou_per_class_md = overview.iou_per_class_md
+        self.iou_per_class_table = overview.iou_per_class_table
+
         # Key Metrics
         key_metrics = KeyMetrics(self.vis_texts, self.eval_result)
         self.key_metrics_md = key_metrics.md
         self.key_metrics_table = key_metrics.table
         self.overview_chart = key_metrics.chart
+        self.custom_ap_description = key_metrics.custom_ap_description_md
 
         # Explore Predictions
         explore_predictions = ExplorePredictions(
@@ -238,9 +243,24 @@ class ObjectDetectionVisualizer(BaseVisualizer):
             # Overview
             (0, self.header),
             (1, self.overview_md),
+        ]
+
+        if self.iou_per_class_table is not None:
+            is_anchors_widgets += [
+                (0, self.iou_per_class_md),
+                (0, self.iou_per_class_table),
+            ]
+
+        is_anchors_widgets += [
             # KeyMetrics
             (1, self.key_metrics_md),
             (0, self.key_metrics_table),
+        ]
+
+        if self.custom_ap_description is not None:
+            is_anchors_widgets.append((0, self.custom_ap_description))
+
+        is_anchors_widgets += [
             (0, self.overview_chart),
             # ExplorePredictions
             (1, self.explore_predictions_md),

supervisely/nn/benchmark/semantic_segmentation/evaluator.py

@@ -25,6 +25,7 @@ from supervisely.sly_logger import logger
 
 class SemanticSegmentationEvalResult(BaseEvalResult):
     mp_cls = MetricProvider
+    PRIMARY_METRIC = "mIoU"
 
     def _read_files(self, path: str) -> None:
         """Read all necessary files from the directory"""

supervisely/nn/benchmark/utils/detection/calculate_metrics.py

@@ -48,8 +48,11 @@ def calculate_metrics(
     :return: Results of the evaluation
     :rtype: dict
     """
+    from pycocotools.coco import COCO  # pylint: disable=import-error
     from pycocotools.cocoeval import COCOeval  # pylint: disable=import-error
 
+    cocoGt: COCO = cocoGt
+
     cocoEval = COCOeval(cocoGt, cocoDt, iouType=iouType)
     cocoEval.evaluate()
     progress_cb(1) if progress_cb is not None else None
@@ -66,23 +69,33 @@ def calculate_metrics(
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.summarize()
 
-    iou_t = 0
-    is_custom_iou_threshold = (
-        evaluation_params is not None
-        and evaluation_params.get("iou_threshold")
-        and evaluation_params["iou_threshold"] != 0.5
-    )
-    if is_custom_iou_threshold:
-        iou_t = np.where(cocoEval.params.iouThrs == evaluation_params["iou_threshold"])[0][0]
+    iouThrs = cocoEval.params.iouThrs
+    evaluation_params = evaluation_params or {}
+    iou_threshold = evaluation_params.get("iou_threshold", 0.5)
+    iou_threshold_per_class = evaluation_params.get("iou_threshold_per_class")
+    if iou_threshold_per_class is not None:
+        iou_idx_per_class = {
+            cocoGt.getCatIds(catNms=[class_name])[0]: np.where(np.isclose(iouThrs, iou_thres))[0][0]
+            for class_name, iou_thres in iou_threshold_per_class.items()
+        }
+    else:
+        iou_idx = np.where(np.isclose(iouThrs, iou_threshold))[0][0]
+        iou_idx_per_class = {cat_id: iou_idx for cat_id in cocoGt.getCatIds()}
 
     eval_img_dict = get_eval_img_dict(cocoEval)
     eval_img_dict_cls = get_eval_img_dict(cocoEval_cls)
-    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=iou_t)
+    matches = get_matches(
+        eval_img_dict,
+        eval_img_dict_cls,
+        cocoEval_cls,
+        iou_idx_per_class=iou_idx_per_class,
+    )
 
     params = {
         "iouThrs": cocoEval.params.iouThrs,
         "recThrs": cocoEval.params.recThrs,
-        "evaluation_params": evaluation_params or {},
+        "evaluation_params": evaluation_params,
+        "iou_idx_per_class": iou_idx_per_class,
     }
     coco_metrics = {"mAP": cocoEval.stats[0], "precision": cocoEval.eval["precision"]}
     coco_metrics["AP50"] = cocoEval.stats[1]
@@ -204,27 +217,6 @@ def get_eval_img_dict(cocoEval):
     return eval_img_dict
 
 
-def get_eval_img_dict_cls(cocoEval_cls):
-    """
-    type cocoEval_cls: COCOeval
-    """
-    # For miss-classification
-    aRng = cocoEval_cls.params.areaRng[0]
-    eval_img_dict_cls = defaultdict(list)  # img_id : dt/gt
-    for i, eval_img in enumerate(cocoEval_cls.evalImgs):
-        if eval_img is None or eval_img["aRng"] != aRng:
-            continue
-        img_id = eval_img["image_id"]
-        cat_id = eval_img["category_id"]
-        ious = cocoEval_cls.ious[(img_id, cat_id)]
-        # ! inplace operation
-        eval_img["ious"] = ious
-        eval_img_dict_cls[img_id].append(eval_img)
-    eval_img_dict_cls = dict(eval_img_dict_cls)
-    assert np.all([len(x) == 1 for x in eval_img_dict_cls.values()])
-    return eval_img_dict_cls
-
-
 def _get_missclassified_match(eval_img_cls, dt_id, gtIds_orig, dtIds_orig, iou_t):
     # Correction on miss-classification
     gt_idx = np.nonzero(eval_img_cls["gtMatches"][iou_t] == dt_id)[0]
@@ -242,7 +234,12 @@ def _get_missclassified_match(eval_img_cls, dt_id, gtIds_orig, dtIds_orig, iou_t
     return None, None
 
 
-def get_matches(eval_img_dict: dict, eval_img_dict_cls: dict, cocoEval_cls, iou_t: int = 0):
+def get_matches(
+    eval_img_dict: dict,
+    eval_img_dict_cls: dict,
+    cocoEval_cls,
+    iou_idx_per_class: dict = None,
+):
     """
     type cocoEval_cls: COCOeval
     """
@@ -255,7 +252,8 @@ def get_matches(eval_img_dict: dict, eval_img_dict_cls: dict, cocoEval_cls, iou_
         gt_ids_orig_cls = [_["id"] for i in cat_ids for _ in cocoEval_cls._gts[img_id, i]]
 
         for eval_img in eval_imgs:
-
+            cat_id = eval_img["category_id"]
+            iou_t = iou_idx_per_class[cat_id]
             dtIds = np.array(eval_img["dtIds"])
             gtIds = np.array(eval_img["gtIds"])
             dtm = eval_img["dtMatches"][iou_t]

supervisely/nn/benchmark/visualization/renderer.py

@@ -8,6 +8,7 @@ from supervisely.api.api import Api
 from supervisely.io.fs import dir_empty, get_directory_size
 from supervisely.nn.benchmark.visualization.widgets import BaseWidget
 from supervisely.task.progress import tqdm_sly
+from supervisely import logger
 
 
 class Renderer:
@@ -95,6 +96,7 @@ class Renderer:
         pth = Path(self.base_dir).joinpath(self.report_name)
         with open(pth, "w") as f:
             f.write(report_link)
+        logger.debug(f"Report link: {self._get_report_link(api, team_id, remote_dir)}")
         return str(pth)
 
     def _get_report_link(self, api: Api, team_id: int, remote_dir: str):

{supervisely-6.73.282.dist-info → supervisely-6.73.284.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.282
+Version: 6.73.284
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.282.dist-info → supervisely-6.73.284.dist-info}/RECORD

@@ -561,18 +561,18 @@ supervisely/collection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 supervisely/collection/key_indexed_collection.py,sha256=x2UVlkprspWhhae9oLUzjTWBoIouiWY9UQSS_MozfH0,37643
 supervisely/collection/str_enum.py,sha256=Zp29yFGvnxC6oJRYNNlXhO2lTSdsriU1wiGHj6ahEJE,1250
 supervisely/convert/__init__.py,sha256=pF1bOrg8SzkdFn90AWGRmVa9OQrHABY0gTlgurJ86Tw,962
-supervisely/convert/base_converter.py,sha256=NHbOYxfZ8Yfs6qJz8zuR5kzKSlkfQs9-fpuafwdAGnA,18583
+supervisely/convert/base_converter.py,sha256=m4wh1BZIW_wbzZk4eS3PN50TMIsb2ZPud2RjVcPfQxY,18627
 supervisely/convert/converter.py,sha256=tWxTDfFv7hwzQhUQrBxzfr6WP8FUGFX_ewg5T2HbUYo,8959
 supervisely/convert/image/__init__.py,sha256=JEuyaBiiyiYmEUYqdn8Mog5FVXpz0H1zFubKkOOm73I,1395
-supervisely/convert/image/image_converter.py,sha256=r-qdhuwOsk727mXIM26ucQhkoIKigu1M0BF-tw9IfGg,10321
+supervisely/convert/image/image_converter.py,sha256=8vak8ZoKTN1ye2ZmCTvCZ605-Rw1AFLIEo7bJMfnR68,10426
 supervisely/convert/image/image_helper.py,sha256=fdV0edQD6hVGQ8TXn2JGDzsnrAXPDMacHBQsApzOME8,3677
 supervisely/convert/image/cityscapes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/convert/image/cityscapes/cityscapes_converter.py,sha256=msmsR2W-Xiod06dwn-MzmkbrEmQQqlKh7zyfTrW6YQw,7854
+supervisely/convert/image/cityscapes/cityscapes_converter.py,sha256=tnelQJHvGz_IGMXWe-EKWAkBhexRzmkv_0Kln5sN12E,8100
 supervisely/convert/image/cityscapes/cityscapes_helper.py,sha256=in5nR7__q_u5dCkVtZmynfZ_ZuvsIAHrTzyTG4EvNgU,2988
 supervisely/convert/image/coco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/convert/image/coco/coco_anntotation_converter.py,sha256=79rhAy_nkudxEgJDLW0BziUz808-fSqTOnlUeN-kvn8,6603
-supervisely/convert/image/coco/coco_converter.py,sha256=7czTd4I1we_HxEc9diQiXPC2pXAtnoqSnFSVCtNOmP4,5431
-supervisely/convert/image/coco/coco_helper.py,sha256=dUk5vCsIxeZDbpjs8_oviPGQDW7CosjqbnjwaJQd0mU,32849
+supervisely/convert/image/coco/coco_anntotation_converter.py,sha256=O1PQbwrbnpQBks2pcz2nbAnhSqpKqNk13B2ARk_roFM,7078
+supervisely/convert/image/coco/coco_converter.py,sha256=7dW7vE6yTRz7O31vTVSnEA4MDCc_UXTqc2UFEqaKorI,5650
+supervisely/convert/image/coco/coco_helper.py,sha256=ykZe_M_yfDqJT9FoQXQ3zuLbQMO0l1WP75QMbvKEx5Y,32866
 supervisely/convert/image/csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/convert/image/csv/csv_converter.py,sha256=iLyc2PAVtlsAq7blnGH4iS1_D7Ai6-4UsdI_RlDVB9Q,11677
 supervisely/convert/image/csv/csv_helper.py,sha256=-nR192IfMU0vTlNRoKXu5FS6tTs9fENqySyeKKyemRs,8409
@@ -592,7 +592,7 @@ supervisely/convert/image/medical2d/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 supervisely/convert/image/medical2d/medical2d_converter.py,sha256=cYEaRfr8YFxEG_Pv-_SVMxrqZudi3kWbGQ3aArL2mds,8156
 supervisely/convert/image/medical2d/medical2d_helper.py,sha256=pfLRCSFbFa5EIhmbB7kdmdWRu01OwIEDPXeNHzAeagg,12329
 supervisely/convert/image/multi_view/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/convert/image/multi_view/multi_view.py,sha256=TF53gXEpfudMb2MPbzunNqrR44tMTCOBBMBqGGwqQAQ,3794
+supervisely/convert/image/multi_view/multi_view.py,sha256=V-6oFN6oDre7UhejfyDkGKAg4rbM3C9JCQ8pHhuUBb8,4436
 supervisely/convert/image/multispectral/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/convert/image/multispectral/multispectral_converter.py,sha256=T3etYVNI0AUUrQsQhxw_r85NthXrqhqmdZQfz8kUY0g,5194
 supervisely/convert/image/pascal_voc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -606,7 +606,7 @@ supervisely/convert/image/sly/fast_sly_image_converter.py,sha256=pZmQzhx9FrHwgVn
 supervisely/convert/image/sly/sly_image_converter.py,sha256=097ijLa_62ZBu0elRx0xX_wpi9tmwgNZonVvBccfclg,12842
 supervisely/convert/image/sly/sly_image_helper.py,sha256=5Ri8fKb5dzh5b3v8AJ5u8xVFOQfAtoWqZ7HktPsCjTI,7373
 supervisely/convert/image/yolo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/convert/image/yolo/yolo_converter.py,sha256=cg5___X5MzvR-rZbNLmaKtr0MdRnyqtEzbBq5UBnYZ0,11171
+supervisely/convert/image/yolo/yolo_converter.py,sha256=Wn5dR05y4SEPONcaxWr9ofnbvbf-SbRZN0fkksk5Dps,11391
 supervisely/convert/image/yolo/yolo_helper.py,sha256=IwyBMZE_3eblsHhw8egeZUR9h_NciwjrxvVLNuZbxY4,19194
 supervisely/convert/pointcloud/__init__.py,sha256=WPeIpPoTWDIKAa0lF6t2SMUhFNZ0l-vKujf6yD6w7SA,589
 supervisely/convert/pointcloud/pointcloud_converter.py,sha256=yCCpzm7GrvL6WT4lNesvtYWWwdO3DO32JIOBBSSQgSA,7130
@@ -637,8 +637,8 @@ supervisely/convert/pointcloud_episodes/bag/bag_converter.py,sha256=jzWKXoFUWu11
 supervisely/convert/pointcloud_episodes/lyft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/convert/pointcloud_episodes/lyft/lyft_converter.py,sha256=QXreWUJ-QhoWgLPqRxCayatYCCCuSV6Z2XCZKScrD3o,10419
 supervisely/convert/pointcloud_episodes/nuscenes_conv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py,sha256=hveKmKVe-jOvME1mMbDwynTlL5kKqd53T9BDmFitbkM,12808
-supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py,sha256=RrTlskRrqxwzrjFfT4e5aI9YeAxnun7Io9fjoicZmnY,8959
+supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py,sha256=4HWouf-H4e5M_Hwd481DpLq17mIZMGkhRVOBgY4alXM,12692
+supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py,sha256=cJTwhFn1JgblbPjrTrZu30y6FxyjGF-12sMFfvN1xzM,8969
 supervisely/convert/pointcloud_episodes/sly/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/convert/pointcloud_episodes/sly/sly_pointcloud_episodes_converter.py,sha256=fSEGxuTtFTAOLNBAZncOxw9PVALBOtB7yZ8qTCaET7w,6102
 supervisely/convert/pointcloud_episodes/sly/sly_pointcloud_episodes_helper.py,sha256=h4WvNH6cEHtjxxhCnU7Hs2vkyJMye0qwabqXNYVTywE,3570
@@ -744,23 +744,23 @@ supervisely/nn/artifacts/utils.py,sha256=C4EaMi95MAwtK5TOnhK4sQ1BWvgwYBxXyRStkhY
 supervisely/nn/artifacts/yolov5.py,sha256=slh05EpQsxqgKwB9KMClshdBxPBN3ZWZ6S4B80ECEt4,1724
 supervisely/nn/artifacts/yolov8.py,sha256=sFd9kU7Gdowq6WH1S3NdlQeoL9jjQKmRYb51fG_wbDk,1446
 supervisely/nn/benchmark/__init__.py,sha256=7jDezvavJFtO9mDeB2TqW8N4sD8TsHQBPpA9RESleIQ,610
-supervisely/nn/benchmark/base_benchmark.py,sha256=2bNKZgcU3l1CTkdP9Glfze5-G2JGtU-V3BFEOKnrdXw,25281
-supervisely/nn/benchmark/base_evaluator.py,sha256=6JbAkphcZTghfIYFyrWDWb59_exbPVx23yot_LPOfIA,5221
+supervisely/nn/benchmark/base_benchmark.py,sha256=Xnb0jL0voBPC-s_eVYSYbYv-xVfLYtQf1tHLnJ9ktq8,25713
+supervisely/nn/benchmark/base_evaluator.py,sha256=sc8gNn3myGA8sGnP6EIiTp24JPXUQ9Ou-8BmTf-Dt7w,5248
 supervisely/nn/benchmark/base_visualizer.py,sha256=7woiYmztDzYZlbhL1hTfJnIi26RFi4obF2VLA519uxQ,10092
 supervisely/nn/benchmark/cv_tasks.py,sha256=ShoAbuNzfMYj0Se-KOnl_-dJnrmvN6Aukxa0eq28bFw,239
 supervisely/nn/benchmark/comparison/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/comparison/base_visualizer.py,sha256=ldJHfGnrbudbxP6ErzBIorPv9r2L9vV8o6UTNqLBfGI,5341
 supervisely/nn/benchmark/comparison/model_comparison.py,sha256=qgd8TwXJ2aiIqB29__EnDV391fBcRfdaIzUYpTyFp9w,7055
 supervisely/nn/benchmark/comparison/detection_visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py,sha256=lBqcNsyOX04R2jL6Efu43xB2aHC1qx28Kw27DLuk66o,28336
-supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py,sha256=JSaFJnKptkITPjzBK_1T-HpYLnVuEzIvRVeismmk5O0,11159
+supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py,sha256=JQ2DqGsvUBbjVmEsq9pGc41U8WQrtaX_Gin4IBguIow,28735
+supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py,sha256=1PZU9P7VP4lPZDuv9dg_f3il03SQ_zRr5G_L6Q72gbQ,11421
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/__init__.py,sha256=2cqjAOwahJoptYhbFKEWws8gRW3K3bxlA9KnQleCrsA,1125
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/avg_precision_by_class.py,sha256=f4FMY-XCiNs0VA-e0aOFVNImjTQSnF4wTLNRSlj2oYA,4636
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py,sha256=IKg2rofo21xGxQIoQyNlZ-UqOpAVviSn_A9SDkokC0w,7592
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predictions.py,sha256=axaviTZ4dLVWIc2R-o0Kv8g8Zr1SQVfyeokoLEA9Eqw,6484
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/localization_accuracy.py,sha256=OYmnloods1UYQ8SIPAcyOK33w0iYSn637OeMKNTrgbA,5342
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py,sha256=vev6EOc7-01i-5VAyQQwm9FGOMFYcWfJ2Y6ufUWg-DQ,13143
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py,sha256=YZIadwwQQvr5XfzFGj6_NGylad00ERLV1fYnke_g_VA,9256
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py,sha256=drRm_Hn8Da4Oc_dsa4ol_mZSeMRNYT1-Zkmb0LbiEAA,10193
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py,sha256=66hs426dR1_TUps9K-UhBp5_xBiFjIouKcF_5gP-Hn8,4797
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py,sha256=602GFNpKZjeRhxUTdlcE6ZczcFEGEjp0qLoTbkM54M4,11558
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/speedtest.py,sha256=sQDkzfpVNaSYBHVcHYqydRSWN0i-yV9uhtEAggg295A,10879
@@ -778,17 +778,17 @@ supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/speedtest.
 supervisely/nn/benchmark/instance_segmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/instance_segmentation/benchmark.py,sha256=lTDzgKGpfeF5o_a2nS56wiAsUQPH1eubk37b9CaB2KI,1171
 supervisely/nn/benchmark/instance_segmentation/evaluation_params.yaml,sha256=POzpiaxnxuwAPSNQOGgjoUPfsk6Lf5hb9GLHwltWY5Y,94
-supervisely/nn/benchmark/instance_segmentation/evaluator.py,sha256=KZhCZ0UWfw_eCrNDtv4WJj3ZQI2w66qGl4Lu9Bl85_Q,2784
-supervisely/nn/benchmark/instance_segmentation/text_templates.py,sha256=sGiGnpIyuOP35f4NoDT0BWNmscsx_T1XZ_igGvCkkBg,25481
+supervisely/nn/benchmark/instance_segmentation/evaluator.py,sha256=mpCi8S6YNwlVvgcERQSHBOhC9PrSfQkQ55pPTcK6V9c,2811
+supervisely/nn/benchmark/instance_segmentation/text_templates.py,sha256=_ZIU_3-xlUGKTcbEthxB4Ngt12azdC7pxpgqHHw7M3I,25780
 supervisely/nn/benchmark/instance_segmentation/visualizer.py,sha256=8NscOKy7JK4AG-Czu3SM0qJQXLDfKD9URdG1d4nz89E,564
 supervisely/nn/benchmark/object_detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/object_detection/base_vis_metric.py,sha256=XjUnFCnCMYLrpjojIOwiRNaSsSLYpozTHWfwLkaCd5U,1612
 supervisely/nn/benchmark/object_detection/benchmark.py,sha256=Wb4xlFXilIMVfsifNNQY25uE52NeEDLzQpnq8QPYq9U,1086
 supervisely/nn/benchmark/object_detection/evaluation_params.yaml,sha256=POzpiaxnxuwAPSNQOGgjoUPfsk6Lf5hb9GLHwltWY5Y,94
-supervisely/nn/benchmark/object_detection/evaluator.py,sha256=Oajs2a-lpudSF076_99-he7uDtqnmQGp4aayoQjNRH0,6801
-supervisely/nn/benchmark/object_detection/metric_provider.py,sha256=j7-_ZGjW8qRpv_v9EMyqO1rK9pkcgjRcInTA8CL3we4,20169
-supervisely/nn/benchmark/object_detection/text_templates.py,sha256=wv6BkcrfdEKldbXtz4ljkbENIIOdeiUr79U4iRgUx6A,25727
-supervisely/nn/benchmark/object_detection/visualizer.py,sha256=Nemp2ZjipbK3S119Yx_izoZuS86jx2U-wtah8TfthS0,31734
+supervisely/nn/benchmark/object_detection/evaluator.py,sha256=EOQQbmwQqjjvbRu3tY24SRA7K8nyqshR92gUcP1lcrY,7371
+supervisely/nn/benchmark/object_detection/metric_provider.py,sha256=MLVRnSwMQ9lfrlgBt4ThIHTVKY-6zuuEWK5-yVsmaj0,21140
+supervisely/nn/benchmark/object_detection/text_templates.py,sha256=J5xUPCGY-QWxc5AEt_u9_2r5q0LBlIzsa007H0GgoeU,26026
+supervisely/nn/benchmark/object_detection/visualizer.py,sha256=NpLKVW5fo6N0kYzgLsfY66wvCv38G3k-SNm4HImXt6g,32366
 supervisely/nn/benchmark/object_detection/vis_metrics/__init__.py,sha256=AXCLHEySEdR-B-5sfDoWBmmOLBVlyW2U_xr8Ta42sQI,2096
 supervisely/nn/benchmark/object_detection/vis_metrics/confidence_distribution.py,sha256=OlwkPgzEQ-RegcLZHVUVOL0n6I_2iayPVpAIie4y2O8,3615
 supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py,sha256=r_saaZI4WB7C7ykNb1obmf8kEOkphLA4pInDoS6dXXU,4005
@@ -797,11 +797,11 @@ supervisely/nn/benchmark/object_detection/vis_metrics/explore_predictions.py,sha
 supervisely/nn/benchmark/object_detection/vis_metrics/f1_score_at_different_iou.py,sha256=6y2Kx-R_t4SdJkdWNyZQ6TGjCC-u6KhXb4cCno4GuTk,2882
 supervisely/nn/benchmark/object_detection/vis_metrics/frequently_confused.py,sha256=7rObk7WNsfwK7xBWl3aOxcn0uD48njEc04fQIPHc3_4,4678
 supervisely/nn/benchmark/object_detection/vis_metrics/iou_distribution.py,sha256=lv4Bk8W4X8ZhvQKyMXI46d240PNlMFx1hdji_aoTS50,3601
-supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py,sha256=Z8qArtjqjjRW5Z4mBuanzK3b4LLfH6NgNbO2Lt0kXyo,4316
+supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py,sha256=byucJuHYWSXIZU8U1Dc44QDpG3lTlhoNdUfD1b-uriw,4721
 supervisely/nn/benchmark/object_detection/vis_metrics/model_predictions.py,sha256=gsGDsesiwOcqeFvHr33b4PSJNw6MoA5brO-qRydRtsA,5944
 supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts.py,sha256=HuTgisYmXCSUeF5WOahy-uaCdvRLsNzg28BDrZ-5hww,7161
 supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py,sha256=GBq0KlPka5z4cxHcKCe2eVOI_h3qlWUqGCyhYs6mjrk,6825
-supervisely/nn/benchmark/object_detection/vis_metrics/overview.py,sha256=tEHxK8NuquAcMcDxdn2G6GEIHp7eauOiTIQP1yFvgSs,5058
+supervisely/nn/benchmark/object_detection/vis_metrics/overview.py,sha256=M6E--Yd1ztP4VBjR6VDUVrj2hgs5mwJF-vhWIjgVGkw,6376
 supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py,sha256=EeZmyNlTVQLQ-0wIDGdvFmRkahJBBiOKSmWiAJ8Bfks,3478
 supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve_by_class.py,sha256=Bl_buVvH8SVqwsc4DcHnojMOqpwTnRgXFt9yw_Y1BR0,1607
 supervisely/nn/benchmark/object_detection/vis_metrics/precision.py,sha256=cAgMrp13uulHfM8xnPDZyR6PqS8nck1Fo7YPpvHPCbw,2708
@@ -814,7 +814,7 @@ supervisely/nn/benchmark/semantic_segmentation/__init__.py,sha256=Hx5bFhRZq8BXvN
 supervisely/nn/benchmark/semantic_segmentation/base_vis_metric.py,sha256=mwGjRUTPrIj56WHsxNW_4fcZM0uw1xm6B5wh25FijyQ,1788
 supervisely/nn/benchmark/semantic_segmentation/benchmark.py,sha256=8rnU6I94q0GUdXWwluZu0_Sac_eU2-Az133tHF1dA3U,1202
 supervisely/nn/benchmark/semantic_segmentation/evaluation_params.yaml,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/nn/benchmark/semantic_segmentation/evaluator.py,sha256=0zNR60As5-8ww3BVj_qNq70VJUDyuUAxcUiSj7FQS_Q,7233
+supervisely/nn/benchmark/semantic_segmentation/evaluator.py,sha256=XafPMpGL6v0ZQ-m7DkEjoY7W6fGCJNKolql5BA3M8V0,7261
 supervisely/nn/benchmark/semantic_segmentation/metric_provider.py,sha256=GwdRvyG0_nFpng6jN8ISFcMLfDbBd-fwdtoWR2XPKw4,6552
 supervisely/nn/benchmark/semantic_segmentation/text_templates.py,sha256=7yRRD2FAdJHGSRqBVIjNjzCduKzaepA1OWtggi7B0Dg,8580
 supervisely/nn/benchmark/semantic_segmentation/visualizer.py,sha256=Nt2-OOWKQ8fbaXFk5QeEaMtMURKPQebgjzDytVgQk0g,13196
@@ -833,7 +833,7 @@ supervisely/nn/benchmark/semantic_segmentation/vis_metrics/speedtest.py,sha256=0
 supervisely/nn/benchmark/semantic_segmentation/vis_metrics/vis_texts.py,sha256=rRdYZxmhQX4T3RsXJVGp34NMZPz8jUHtVvBN5BpPJ5I,603
 supervisely/nn/benchmark/utils/__init__.py,sha256=JHT73gWdwgLJKTiCpDdwggm1t_EWB0JCC90-zD7reXM,741
 supervisely/nn/benchmark/utils/detection/__init__.py,sha256=L3QKGuKUlR2N_QFRTRsa6gfLDbksIaFMYO0Hukxxy1U,172
-supervisely/nn/benchmark/utils/detection/calculate_metrics.py,sha256=1Vrf4Npf8lHAE_WZRS90fa5jFh4BofDn6RWCFcd48r8,11484
+supervisely/nn/benchmark/utils/detection/calculate_metrics.py,sha256=LXZET5yLp9S7Uq2eX4HpAMnBMxTI5Q2CgKSc1mCfaRM,11388
 supervisely/nn/benchmark/utils/detection/metric_provider.py,sha256=cgF6uzF7XOvU2CpxyU7zuK1HH6hhNiIV3vQc8MAzwMU,19934
 supervisely/nn/benchmark/utils/detection/metrics.py,sha256=oyictdJ7rRDUkaVvHoxntywW5zZweS8pIJ1bN6JgXtE,2420
 supervisely/nn/benchmark/utils/detection/sly2coco.py,sha256=0O2LSCU5zIX34mD4hZIv8O3-j6LwnB0DqhiVPAiosO8,6883
@@ -845,7 +845,7 @@ supervisely/nn/benchmark/utils/semantic_segmentation/loader.py,sha256=_5ZZ7Nkd8W
 supervisely/nn/benchmark/utils/semantic_segmentation/utils.py,sha256=X5NiR02R-0To2_SuSGHZZccl_-Bupg5F9d7nziIMRMc,3874
 supervisely/nn/benchmark/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/visualization/evaluation_result.py,sha256=733HJL4rJa5XqCJydW9vSyaepvpHzym9wQsw1wFEgeI,10251
-supervisely/nn/benchmark/visualization/renderer.py,sha256=s1YexuKDHrI2gP-Qpz1fAXCh30dBQM7whnMwjbOf61M,3804
+supervisely/nn/benchmark/visualization/renderer.py,sha256=j6dFqIGa9M0DbVVR-jgJab-MzUUiw47coHiiBF_H_jQ,3923
 supervisely/nn/benchmark/visualization/report_template.html,sha256=tylBK5Bb2cqKACK1GZUKyIjPS9yHQFHAS-QeEEwhqTE,2172
 supervisely/nn/benchmark/visualization/vis_click_data.py,sha256=hBeVepHngTGVHK3MiWe8qZY87taifxnoUXq22W2xaqo,3724
 supervisely/nn/benchmark/visualization/widgets/__init__.py,sha256=UovmhwLH4Au81JFrFz0NwPasaIqPEI-zXN-JntTc2FU,949
@@ -1070,9 +1070,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
 supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
 supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
-supervisely-6.73.282.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-supervisely-6.73.282.dist-info/METADATA,sha256=qGp4_zjqrTMz3V40SpkZe9J1yy-1HCAPicbSc5vpeGo,33573
-supervisely-6.73.282.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-supervisely-6.73.282.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
-supervisely-6.73.282.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
-supervisely-6.73.282.dist-info/RECORD,,
+supervisely-6.73.284.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+supervisely-6.73.284.dist-info/METADATA,sha256=GriDRRnOtHK84_HcMIMCD6z4qfCCZSHH5NyJsTmW3e4,33573
+supervisely-6.73.284.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+supervisely-6.73.284.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
+supervisely-6.73.284.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
+supervisely-6.73.284.dist-info/RECORD,,