supervisely 6.73.282__tar.gz → 6.73.284__tar.gz

{supervisely-6.73.282/supervisely.egg-info → supervisely-6.73.284}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.282
+Version: 6.73.284
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/base_converter.py

@@ -168,6 +168,7 @@ class BaseConverter:
         self._upload_as_links: bool = upload_as_links
         self._remote_files_map: Optional[Dict[str, str]] = remote_files_map
         self._supports_links = False  # if converter supports uploading by links
+        self._force_shape_for_links = False
         self._api = Api.from_env() if self._upload_as_links else None
         self._team_id = team_id() if self._upload_as_links else None
         self._converter = None

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/cityscapes/cityscapes_converter.py

@@ -34,6 +34,8 @@ class CityscapesConverter(ImageConverter):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
 
         self._classes_mapping = {}
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self):
         return AvailableImageConverters.CITYSCAPES
@@ -41,6 +43,10 @@ class CityscapesConverter(ImageConverter):
     @property
     def key_file_ext(self) -> str:
         return ".json"
+    
+    @property
+    def ann_ext(self) -> str:
+        return ".json"
 
     def ann_file_ext(self) -> str:
         return ".json"
@@ -108,6 +114,8 @@ class CityscapesConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         images_list, ann_dict = [], {}
         for root, _, files in os.walk(self._input_data):

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/coco/coco_anntotation_converter.py

@@ -37,6 +37,7 @@ class FastCOCOConverter(COCOConverter, ImageConverter):
         self._items = []
         meta = ProjectMeta()
         warnings = defaultdict(list)
+        item_names = set()
         for ann_path in ann_paths:
             try:
                 with coco_helper.HiddenCocoPrints():
@@ -74,11 +75,18 @@ class FastCOCOConverter(COCOConverter, ImageConverter):
                 coco_ann = coco_anns[image_id]
                 if len(coco_ann) == 0 or coco_ann is None or image_name is None:
                     continue
+                if image_name in item_names:
+                    # * Block to handle the case when there are mixed annotations: caption and segmentations for the same images
+                    item = next(item for item in self._items if item.name == image_name)
+                    if item.shape == (height, width):
+                        item.ann_data.extend(coco_ann)
+                        continue
                 item = self.Item(image_name) if image_url is None else self.Item(image_url)
                 item.name = image_name
                 item.ann_data = coco_ann
                 item.set_shape((height, width))
                 self._items.append(item)
+                item_names.add(image_name)
                 detected_ann_cnt += len(coco_ann)
 
         self._meta = meta

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/coco/coco_converter.py

@@ -25,6 +25,8 @@ class COCOConverter(ImageConverter):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
 
         self._coco_categories = []
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self) -> str:
         return AvailableImageConverters.COCO
@@ -56,6 +58,8 @@ class COCOConverter(ImageConverter):
     def validate_format(self) -> bool:
         from pycocotools.coco import COCO  # pylint: disable=import-error
 
+        if self.upload_as_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         images_list, ann_paths = [], []
         for root, _, files in os.walk(self._input_data):
@@ -145,7 +149,8 @@ class COCOConverter(ImageConverter):
         if item.ann_data is None:
             return Annotation.from_img_path(item.path)
         else:
-            item.set_shape()
+            if not self.upload_as_links:
+                item.set_shape()
             ann = coco_helper.create_supervisely_annotation(
                 item,
                 meta,

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/coco/coco_helper.py

@@ -182,7 +182,7 @@ def convert_rle_mask_to_polygon(coco_ann):
     return Bitmap(mask).to_contours()
 
 
-def convert_polygon_vertices(coco_ann, image_size):
+def convert_polygon_vertices(coco_ann, image_size: Tuple[int, int]):
     polygons = coco_ann["segmentation"]
     if all(type(coord) is float for coord in polygons):
         polygons = [polygons]

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/image_converter.py

@@ -136,20 +136,16 @@ class ImageConverter(BaseConverter):
             item_names = []
             item_paths = []
             item_metas = []
-            anns = []
             for item in batch:
                 item.path = self.validate_image(item.path)
                 if item.path is None:
                     continue  # image has failed validation
-                item.name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
-                if self.upload_as_links and not self.supports_links:
-                    ann = None
-                else:
-                    ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
-                name = generate_free_name(
-                    existing_names, item.name, with_ext=True, extend_used_names=True
+                name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
+
+                item.name = generate_free_name(
+                    existing_names, name, with_ext=True, extend_used_names=True
                 )
-                item_names.append(name)
+                item_names.append(item.name)
                 item_paths.append(item.path)
 
                 if isinstance(item.meta, str):  # path to file
@@ -159,9 +155,6 @@ class ImageConverter(BaseConverter):
                 else:
                     item_metas.append({})
 
-                if ann is not None:
-                    anns.append(ann)
-
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
@@ -173,7 +166,7 @@ class ImageConverter(BaseConverter):
                         metas=item_metas,
                         batch_size=batch_size,
                         conflict_resolution="rename",
-                        force_metadata_for_links=False,
+                        force_metadata_for_links=self._force_shape_for_links,
                     )
                 else:
                     img_infos = api.image.upload_paths(
@@ -183,8 +176,15 @@ class ImageConverter(BaseConverter):
                         metas=item_metas,
                         conflict_resolution="rename",
                     )
-
                 img_ids = [img_info.id for img_info in img_infos]
+
+                anns = []
+                if not (self.upload_as_links and not self.supports_links):
+                    for info, item in zip(img_infos, batch):
+                        if self._force_shape_for_links:
+                            item.set_shape((info.height, info.width))
+                        anns.append(self.to_supervisely(item, meta, renamed_classes, renamed_tags))
+
                 if len(anns) == len(img_ids):
                     api.annotation.upload_anns(
                         img_ids, anns, skip_bounds_validation=self.upload_as_links

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/multi_view/multi_view.py

@@ -1,6 +1,6 @@
 import os
 from collections import defaultdict
-from typing import Dict, Union
+from typing import Dict, Union, Optional
 
 from supervisely import ProjectMeta, generate_free_name, is_development, logger
 from supervisely.api.api import Api, ApiContext
@@ -13,6 +13,18 @@ from supervisely.project.project_settings import LabelingInterface
 
 class MultiViewImageConverter(ImageConverter):
 
+    def __init__(
+            self,
+            input_data: str,
+            labeling_interface: Optional[Union[LabelingInterface, str]],
+            upload_as_links: bool,
+            remote_files_map: Optional[Dict[str, str]] = None,
+    ):
+        super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
+
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
+
     def __str__(self):
         return AvailableImageConverters.MULTI_VIEW
 
@@ -79,13 +91,16 @@ class MultiViewImageConverter(ImageConverter):
                     logger.warn(f"Image '{name}' already exists. Renamed to '{new_name}'.")
                     os.rename(image, os.path.join(group_path, new_name))
                     image = os.path.join(group_path, new_name)
+                if self._upload_as_links:
+                    image = self.remote_files_map.get(image, image)
                 images.append(image)
 
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
+                kwarg = {"links": images} if self.upload_as_links else {"paths": images}
                 api.image.upload_multiview_images(
-                    dataset.id, group_name, images, progress_cb=progress_cb
+                    dataset.id, group_name, **kwarg, progress_cb=progress_cb
                 )
 
         if log_progress:

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/image/yolo/yolo_converter.py

@@ -38,6 +38,8 @@ class YOLOConverter(ImageConverter):
         self._coco_classes_dict: dict = {}
         self._num_kpts = None
         self._num_dims = None
+        self._supports_links = True
+        self._force_shape_for_links = self.upload_as_links
 
     def __str__(self) -> str:
         return AvailableImageConverters.YOLO
@@ -151,6 +153,9 @@ class YOLOConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links:
+            self._download_remote_ann_files()
+
         detected_ann_cnt = 0
         config_path = None
         images_list, ann_dict = [], {}
@@ -238,7 +243,8 @@ class YOLOConverter(ImageConverter):
 
         try:
             labels = []
-            item.set_shape()
+            if not self.upload_as_links:
+                item.set_shape()
             height, width = item.shape
             with open(item.ann_data, "r") as ann_file:
                 lines = ann_file.readlines()

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py

@@ -61,35 +61,27 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             return False
 
         def filter_fn(path):
-            return all(
-                [
-                    (Path(path) / name).exists()
-                    for name in ["maps", "samples", "sweeps", "v1.0-mini"]
-                ]
-            )
+            return all([(Path(path) / name).exists() for name in ["maps", "samples"]])
 
-        try:
-            input_path = [d for d in fs.dirs_filter(self._input_data, filter_fn)].pop()
-        except IndexError:
+        input_path = next((d for d in fs.dirs_filter(self._input_data, filter_fn)), None)
+        if input_path is None:
             return False
 
         sample_dir = input_path + "/samples/"
         if any([not fs.dir_exists(f"{sample_dir}/{d}") for d in helpers.DIR_NAMES]):
             return False
 
-        sweeps_dir = input_path + "/sweeps/"
-        if any([not fs.dir_exists(f"{sweeps_dir}/{d}") for d in helpers.DIR_NAMES]):
-            return False
-
-        ann_dir = input_path + "/v1.0-mini/"
-        if any([not fs.file_exists(f"{ann_dir}/{d}.json") for d in helpers.TABLE_NAMES]):
+        fil_fn = lambda p: all(fs.file_exists(f"{p}/{name}.json") for name in helpers.TABLE_NAMES)
+        ann_dir = next((d for d in fs.dirs_filter(input_path, fil_fn)), None)
+        if ann_dir is None:
             return False
 
+        version = osp.basename(ann_dir)
         try:
             t = TinyTimer()
-            nuscenes = NuScenes(dataroot=input_path, verbose=False)
+            nuscenes = NuScenes(version=version, dataroot=input_path, verbose=False)
             self._nuscenes: NuScenes = nuscenes
-            logger.info(f"NuScenes initialization took {t.get_sec():.3f} sec")
+            logger.debug(f"NuScenes initialization took {t.get_sec():.3f} sec")
         except Exception as e:
             logger.debug(f"Failed to initialize NuScenes: {e}")
             return False
@@ -184,7 +176,9 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             scene_name_to_dataset[scene_names[0]] = dataset_info
 
         if log_progress:
-            progress, progress_cb = self.get_progress(total_sample_cnt, "Converting episode scenes...")
+            progress, progress_cb = self.get_progress(
+                total_sample_cnt, "Converting episode scenes..."
+            )
         else:
             progress_cb = None
 

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py

@@ -16,11 +16,11 @@ DIR_NAMES = [
     "CAM_FRONT_LEFT",
     "CAM_FRONT_RIGHT",
     "LIDAR_TOP",
-    "RADAR_FRONT",
-    "RADAR_FRONT_LEFT",
-    "RADAR_FRONT_RIGHT",
-    "RADAR_BACK_LEFT",
-    "RADAR_BACK_RIGHT",
+    # "RADAR_FRONT",
+    # "RADAR_FRONT_LEFT",
+    # "RADAR_FRONT_RIGHT",
+    # "RADAR_BACK_LEFT",
+    # "RADAR_BACK_RIGHT",
 ]
 
 TABLE_NAMES = [

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/base_benchmark.py

@@ -1,5 +1,6 @@
 import os
-from typing import Callable, List, Optional, Tuple, Union
+from pathlib import Path
+from typing import Callable, List, Optional, Tuple, Union, Type
 
 import numpy as np
 
@@ -80,7 +81,7 @@ class BaseBenchmark:
         self.report_id = None
         self._validate_evaluation_params()
 
-    def _get_evaluator_class(self) -> type:
+    def _get_evaluator_class(self) -> Type[BaseEvaluator]:
         raise NotImplementedError()
 
     @property
@@ -95,6 +96,10 @@ class BaseBenchmark:
     def key_metrics(self):
         eval_results = self.get_eval_result()
         return eval_results.key_metrics
+    
+    @property
+    def primary_metric_name(self) -> str:
+        return self._get_evaluator_class().eval_result_cls.PRIMARY_METRIC
 
     def run_evaluation(
         self,
@@ -492,6 +497,8 @@ class BaseBenchmark:
                 "It should be defined in the subclass of BaseBenchmark (e.g. ObjectDetectionBenchmark)."
             )
         eval_result = self.get_eval_result()
+        self._dump_key_metrics(eval_result)
+
         layout_dir = self.get_layout_results_dir()
         self.visualizer = self.visualizer_cls(  # pylint: disable=not-callable
             self.api, [eval_result], layout_dir, self.pbar
@@ -621,3 +628,7 @@ class BaseBenchmark:
             self.diff_project_info = eval_result.diff_project_info
             return self.diff_project_info
         return None
+
+    def _dump_key_metrics(self, eval_result: BaseEvaluator):
+        path = str(Path(self.get_eval_results_dir(), "key_metrics.json"))
+        json.dump_json_file(eval_result.key_metrics, path)

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/base_evaluator.py

@@ -12,6 +12,8 @@ from supervisely.task.progress import tqdm_sly
 
 
 class BaseEvalResult:
+    PRIMARY_METRIC = None
+
     def __init__(self, directory: Optional[str] = None):
         self.directory = directory
         self.inference_info: Dict = None

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py

@@ -87,6 +87,11 @@ In this section you can visually assess the model performance through examples.
 > Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
 """
 
+markdown_different_iou_thresholds_warning = """### IoU Thresholds Mismatch
+
+<i class="zmdi zmdi-alert-polygon" style="color: #f5a623; margin-right: 5px"></i> The models were evaluated using different IoU thresholds. Since these thresholds varied between models and classes, it may have led to unfair comparison. For fair model comparison, we suggest using the same IoU threshold across models.
+"""
+
 markdown_explore_difference = """## Explore Predictions
 
 In this section, you can explore predictions made by different models side-by-side. This helps you to understand the differences in predictions made by each model, and to identify which model performs better in different scenarios.

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from typing import List
 
 from supervisely._utils import abs_url
@@ -15,6 +16,7 @@ class Overview(BaseVisMetrics):
     MARKDOWN_OVERVIEW = "markdown_overview"
     MARKDOWN_OVERVIEW_INFO = "markdown_overview_info"
     MARKDOWN_COMMON_OVERVIEW = "markdown_common_overview"
+    MARKDOWN_DIFF_IOU = "markdown_different_iou_thresholds_warning"
     CHART = "chart_key_metrics"
 
     def __init__(self, vis_texts, eval_results: List[EvalResult]) -> None:
@@ -237,3 +239,26 @@ class Overview(BaseVisMetrics):
             ),
         )
         return fig
+
+    @property
+    def not_matched_iou_per_class_thresholds_md(self) -> MarkdownWidget:
+        if all([not r.different_iou_thresholds_per_class for r in self.eval_results]):
+            return None
+
+        iou_thrs_map = defaultdict(set)
+        matched = True
+        for eval_result in self.eval_results:
+            for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
+                iou_thrs_map[cat_id].add(iou_thr)
+                if len(iou_thrs_map[cat_id]) > 1:
+                    matched = False
+                    break
+
+        if matched:
+            return None
+
+        return MarkdownWidget(
+            name="markdown_different_iou_thresholds_warning",
+            title="IoU per class thresholds mismatch",
+            text=self.vis_texts.markdown_different_iou_thresholds_warning,
+        )

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py

@@ -50,10 +50,9 @@ class DetectionComparisonVisualizer(BaseComparisonVisualizer):
         self.overviews = self._create_overviews(overview)
         self.overview_md = overview.overview_md
         self.key_metrics_md = self._create_key_metrics()
-        self.key_metrics_table = overview.get_table_widget(
-            latency=speedtest.latency, fps=speedtest.fps
-        )
+        self.key_metrics_table = overview.get_table_widget(speedtest.latency, speedtest.fps)
         self.overview_chart = overview.chart_widget
+        self.iou_per_class_thresholds_md = overview.not_matched_iou_per_class_thresholds_md
 
         columns_number = len(self.comparison.eval_results) + 1  # +1 for GT
         self.explore_predictions_modal_gallery = self._create_explore_modal_table(columns_number)
@@ -154,6 +153,13 @@ class DetectionComparisonVisualizer(BaseComparisonVisualizer):
             (0, self.header),
             (1, self.overview_md),
             (0, self.overviews),
+        ]
+
+        if self.iou_per_class_thresholds_md is not None:
+            is_anchors_widgets.append((0, self.iou_per_class_thresholds_md))
+
+        is_anchors_widgets += [
+            # Key Metrics
             (1, self.key_metrics_md),
             (0, self.key_metrics_table),
             (0, self.overview_chart),

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/instance_segmentation/evaluator.py

@@ -14,6 +14,7 @@ from supervisely.nn.benchmark.utils import calculate_metrics, read_coco_datasets
 
 class InstanceSegmentationEvalResult(ObjectDetectionEvalResult):
     mp_cls = MetricProvider
+    PRIMARY_METRIC = "mAP"
 
     @classmethod
     def from_evaluator(

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/instance_segmentation/text_templates.py

@@ -60,6 +60,13 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
+markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+
+markdown_iou_per_class = """### IoU Threshold per Class
+
+The model is evaluated using different IoU thresholds for each class.
+"""
+
 markdown_explorer = """## Explore Predictions
 In this section you can visually assess the model performance through examples. This helps users better understand model capabilities and limitations, giving an intuitive grasp of prediction quality in different scenarios.
 

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/object_detection/evaluator.py

@@ -19,6 +19,7 @@ from supervisely.nn.benchmark.visualization.vis_click_data import ClickData, IdM
 
 class ObjectDetectionEvalResult(BaseEvalResult):
     mp_cls = MetricProvider
+    PRIMARY_METRIC = "mAP"
 
     def _read_files(self, path: str) -> None:
         """Read all necessary files from the directory"""
@@ -92,6 +93,10 @@ class ObjectDetectionEvalResult(BaseEvalResult):
     def key_metrics(self):
         return self.mp.key_metrics()
 
+    @property
+    def different_iou_thresholds_per_class(self) -> bool:
+        return self.mp.iou_threshold_per_class is not None
+
 
 class ObjectDetectionEvaluator(BaseEvaluator):
     EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"
@@ -120,12 +125,19 @@ class ObjectDetectionEvaluator(BaseEvaluator):
 
     @classmethod
     def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        available_thres = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
         iou_threshold = evaluation_params.get("iou_threshold")
         if iou_threshold is not None:
-            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
-                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
-                f"but got {iou_threshold}"
+            assert iou_threshold in available_thres, (
+                f"iou_threshold must be one of {available_thres}, " f"but got {iou_threshold}"
             )
+        iou_threshold_per_class = evaluation_params.get("iou_threshold_per_class")
+        if iou_threshold_per_class is not None:
+            for class_name, iou_thres in iou_threshold_per_class.items():
+                assert iou_thres in available_thres, (
+                    f"class {class_name}: iou_threshold_per_class must be one of {available_thres}, "
+                    f"but got {iou_thres}"
+                )
 
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -92,7 +92,11 @@ class MetricProvider:
 
         eval_params = params.get("evaluation_params", {})
         self.iou_threshold = eval_params.get("iou_threshold", 0.5)
-        self.iou_threshold_idx = np.searchsorted(self.iouThrs, self.iou_threshold)
+        self.iou_threshold_idx = np.where(np.isclose(self.iouThrs, self.iou_threshold))[0][0]
+
+        # IoU per class (optional)
+        self.iou_threshold_per_class = eval_params.get("iou_threshold_per_class")
+        self.iou_idx_per_class = params.get("iou_idx_per_class")  # {cat id: iou_idx}
 
     def calculate(self):
         self.m_full = _MetricProvider(
@@ -142,6 +146,8 @@ class MetricProvider:
     def json_metrics(self):
         base = self.base_metrics()
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         ap_by_class = self.AP_per_class().tolist()
         ap_by_class = dict(zip(self.cat_names, ap_by_class))
         ap_custom_by_class = self.AP_custom_per_class().tolist()
@@ -166,6 +172,8 @@ class MetricProvider:
 
     def key_metrics(self):
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         json_metrics = self.json_metrics()
         json_metrics.pop("AP_by_class")
         json_metrics.pop(f"AP{iou_name}_by_class")
@@ -174,6 +182,8 @@ class MetricProvider:
     def metric_table(self):
         table = self.json_metrics()
         iou_name = int(self.iou_threshold * 100)
+        if self.iou_threshold_per_class is not None:
+            iou_name = "_custom"
         return {
             "mAP": table["mAP"],
             "AP50": table["AP50"],
@@ -196,6 +206,10 @@ class MetricProvider:
 
     def AP_custom_per_class(self):
         s = self.coco_precision[self.iou_threshold_idx, :, :, 0, 2]
+        s = s.copy()
+        if self.iou_threshold_per_class is not None:
+            for cat_id, iou_idx in self.iou_idx_per_class.items():
+                s[:, cat_id - 1] = self.coco_precision[iou_idx, :, cat_id - 1, 0, 2]
         s[s == -1] = np.nan
         ap = np.nanmean(s, axis=0)
         return ap
@@ -280,6 +294,9 @@ class _MetricProvider:
             ious.append(match["iou"])
             cats.append(cat_id_to_idx[match["category_id"]])
         ious = np.array(ious) + np.spacing(1)
+        if 0.8999999999999999 in iouThrs:
+            iouThrs = iouThrs.copy()
+            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious) - 1
         cats = np.array(cats)
         # TP
@@ -452,6 +469,9 @@ class _MetricProvider:
         )
         scores = np.array([m["score"] for m in matches_sorted])
         ious = np.array([m["iou"] if m["type"] == "TP" else 0.0 for m in matches_sorted])
+        if 0.8999999999999999 in iouThrs:
+            iouThrs = iouThrs.copy()
+            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious + np.spacing(1))
 
         # Check

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/object_detection/text_templates.py

@@ -65,6 +65,13 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
+markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+
+markdown_iou_per_class = """### IoU Threshold per Class
+
+The model is evaluated using different IoU thresholds for each class.
+"""
+
 markdown_explorer = """## Explore Predictions
 In this section you can visually assess the model performance through examples. This helps users better understand model capabilities and limitations, giving an intuitive grasp of prediction quality in different scenarios.
 

{supervisely-6.73.282 → supervisely-6.73.284}/supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py

@@ -29,6 +29,8 @@ class KeyMetrics(DetectionVisMetric):
         columns = ["metrics", "values"]
         content = []
         for metric, value in self.eval_result.mp.metric_table().items():
+            if metric == "AP_custom":
+                metric += "*"
             row = [metric, round(value, 2)]
             dct = {
                 "row": row,
@@ -134,3 +136,13 @@ class KeyMetrics(DetectionVisMetric):
             ]
 
         return res
+
+    @property
+    def custom_ap_description_md(self) -> MarkdownWidget:
+        if not self.eval_result.different_iou_thresholds_per_class:
+            return None
+        return MarkdownWidget(
+            "custom_ap_description",
+            "Custom AP per Class",
+            self.vis_texts.markdown_AP_custom_description,
+        )