supervisely 6.73.293__tar.gz → 6.73.295__tar.gz

{supervisely-6.73.293/supervisely.egg-info → supervisely-6.73.295}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.293
+Version: 6.73.295
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/api/image_api.py

@@ -391,6 +391,7 @@ class ImageApi(RemoveableBulkModuleApi):
         project_id: Optional[int] = None,
         only_labelled: Optional[bool] = False,
         fields: Optional[List[str]] = None,
+        recursive: Optional[bool] = False,
     ) -> List[ImageInfo]:
         """
         List of Images in the given :class:`Dataset<supervisely.project.project.Dataset>`.
@@ -415,6 +416,8 @@ class ImageApi(RemoveableBulkModuleApi):
         :type only_labelled: bool, optional
         :param fields: List of fields to return. If None, returns all fields.
         :type fields: List[str], optional
+        :param recursive: If True, returns all images from dataset recursively (including images in nested datasets).
+        :type recursive: bool, optional
         :return: Objects with image information from Supervisely.
         :rtype: :class:`List[ImageInfo]<ImageInfo>`
         :Usage example:
@@ -474,6 +477,7 @@ class ImageApi(RemoveableBulkModuleApi):
             ApiField.SORT: sort,
             ApiField.SORT_ORDER: sort_order,
             ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
+            ApiField.RECURSIVE: recursive,
         }
         if only_labelled:
             data[ApiField.FILTERS] = [
@@ -4537,7 +4541,7 @@ class ImageApi(RemoveableBulkModuleApi):
                     loop = sly.utils.get_or_create_event_loop()
                     images = loop.run_until_complete(api.image.get_list_async(123456, per_page=600))
         """
-        
+
         method = "images.list"
         dataset_info = kwargs.get("dataset_info", None)
 

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/avg_precision_by_class.py

@@ -114,7 +114,7 @@ class AveragePrecisionByClass(BaseVisMetrics):
                     {
                         "type": "tag",
                         "tagId": "confidence",
-                        "value": [eval_result.mp.f1_optimal_conf, 1],
+                        "value": [eval_result.mp.conf_threshold, 1],
                     },
                     {"type": "tag", "tagId": "outcome", "value": "TP"},
                     {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py

@@ -154,6 +154,16 @@ class CalibrationScore(BaseVisMetrics):
                     line=dict(color="gray", width=2, dash="dash"),
                     name=f"F1-optimal threshold ({eval_result.name})",
                 )
+            if eval_result.mp.custom_conf_threshold is not None:
+                fig.add_shape(
+                    type="line",
+                    x0=eval_result.mp.custom_conf_threshold,
+                    x1=eval_result.mp.custom_conf_threshold,
+                    y0=0,
+                    y1=eval_result.mp.custom_f1,
+                    line=dict(color="black", width=2, dash="dash"),
+                    name=f"Confidence threshold ({eval_result.name})",
+                )
 
         # Update the layout
         fig.update_layout(

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predictions.py

@@ -76,7 +76,7 @@ class ExplorePredictions(BaseVisMetrics):
             anns = eval_res.api.annotation.download_batch(dataset_info.id, images_ids, force_metadata_for_links=False)
             annotations.append(anns)
             skip_tags_filtering.append(False)
-            min_conf = min(min_conf, eval_res.mp.f1_optimal_conf)
+            min_conf = min(min_conf, eval_res.mp.conf_threshold)
 
         images = list(i for x in zip(*images) for i in x)
         annotations = list(i for x in zip(*annotations) for i in x)
@@ -127,7 +127,7 @@ class ExplorePredictions(BaseVisMetrics):
             current_images_infos = sorted(current_images_infos, key=lambda x: names.index(x.name))
             images_ids.append([image_info.id for image_info in current_images_infos])
 
-            min_conf = min(min_conf, eval_res.mp.f1_optimal_conf)
+            min_conf = min(min_conf, eval_res.mp.conf_threshold)
 
         explore["imagesIds"] = list(i for x in zip(*images_ids) for i in x)
         explore["filters"] = [{"type": "tag", "tagId": "confidence", "value": [min_conf, 1]}]

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py

@@ -276,7 +276,7 @@ class OutcomeCounts(BaseVisMetrics):
                 title = f"{model_name}. {outcome}: {len(obj_ids)} object{'s' if len(obj_ids) > 1 else ''}"
                 outcome_dict["title"] = title
                 outcome_dict["imagesIds"] = list(img_ids)
-                thr = eval_result.mp.f1_optimal_conf
+                thr = eval_result.mp.conf_threshold
                 if outcome == "FN":
                     outcome_dict["filters"] = [
                         {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -27,6 +27,7 @@ class Overview(BaseVisMetrics):
         evaluation result metrics displayed
         """
         super().__init__(vis_texts, eval_results)
+        self.team_id = None  # will be set in the visualizer
 
     @property
     def overview_md(self) -> List[MarkdownWidget]:
@@ -120,8 +121,7 @@ class Overview(BaseVisMetrics):
             if idx == 3 and not same_iou_thr:
                 continue
             metric_name = metric_renames_map.get(metric, metric)
-            values = [m[metric] for m in all_metrics]
-            values = [v if v is not None else "―" for v in values]
+            values = [m.get(metric, "―") for m in all_metrics]
             values = [round(v, 2) if isinstance(v, float) else v for v in values]
             row = [metric_name] + values
             dct = {"row": row, "id": metric, "items": row}
@@ -247,12 +247,18 @@ class Overview(BaseVisMetrics):
 
         iou_thrs_map = defaultdict(set)
         matched = True
-        for eval_result in self.eval_results:
-            for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
-                iou_thrs_map[cat_id].add(iou_thr)
-                if len(iou_thrs_map[cat_id]) > 1:
-                    matched = False
-                    break
+
+        if not all([not r.different_iou_thresholds_per_class for r in self.eval_results]):
+            matched = False
+        else:
+            for eval_result in self.eval_results:
+                iou_thrs_per_class = eval_result.mp.iou_threshold_per_class
+                if iou_thrs_per_class is not None:
+                    for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
+                        iou_thrs_map[cat_id].add(iou_thr)
+                        if len(iou_thrs_map[cat_id]) > 1:
+                            matched = False
+                            break
 
         if matched:
             return None

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py

@@ -19,7 +19,7 @@ class PrCurve(BaseVisMetrics):
     @property
     def markdown_widget(self) -> MarkdownWidget:
         text: str = getattr(self.vis_texts, self.MARKDOWN_PR_CURVE).format(
-            self.vis_texts.definitions.f1_score
+            self.vis_texts.definitions.about_pr_tradeoffs
         )
         return MarkdownWidget(
             name=self.MARKDOWN_PR_CURVE, title="mAP & Precision-Recall Curve", text=text

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py

@@ -205,7 +205,7 @@ class PrecisionRecallF1(BaseVisMetrics):
                     {
                         "type": "tag",
                         "tagId": "confidence",
-                        "value": [eval_result.mp.f1_optimal_conf, 1],
+                        "value": [eval_result.mp.conf_threshold, 1],
                     },
                     {"type": "tag", "tagId": "outcome", "value": "TP"},
                     {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
@@ -293,7 +293,7 @@ class PrecisionRecallF1(BaseVisMetrics):
                 {
                     "type": "tag",
                     "tagId": "confidence",
-                    "value": [eval_result.mp.f1_optimal_conf, 1],
+                    "value": [eval_result.mp.conf_threshold, 1],
                 },
                 {"type": "tag", "tagId": "outcome", "value": "TP"},
                 {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely-6.73.295/supervisely/nn/benchmark/instance_segmentation/evaluation_params.yaml

@@ -0,0 +1,7 @@
+# Intersection over Union threshold that will be used for object matching.
+# It mostly affects visualizations, such as Outcome Counts, Confusion Matrix, and image previews.
+iou_threshold: 0.5
+
+# Confidence threshold.
+# Set 'auto' to calculate the optimal confidence threshold.
+confidence_threshold: auto

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/instance_segmentation/text_templates.py

@@ -43,7 +43,7 @@ markdown_overview = """
 - **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
 {}
 - **IoU threshold**: {}
-- **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+{}
 - **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
@@ -73,7 +73,7 @@ In this section you can visually assess the model performance through examples.
 
 > Click on the image to view the **Ground Truth**, **Prediction**, and **Difference** annotations side-by-side. 
 
-> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
+> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). {}
 """
 
 markdown_predictions_table = """### Prediction details for every image
@@ -100,7 +100,7 @@ To measure this, we calculate **Recall**. Recall counts errors, when the model d
 """
 
 notification_recall = {
-    "title": "Recall = {}",
+    "title": "{}",
     "description": "The model correctly found <b>{} of {}</b> total instances in the dataset.",
 }
 
@@ -123,7 +123,7 @@ To measure this, we calculate **Precision**. Precision counts errors, when the m
 """
 
 notification_precision = {
-    "title": "Precision = {}",
+    "title": "{}",
     "description": "The model correctly predicted <b>{} of {}</b> predictions made by the model in total.",
 }
 

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/base_vis_metric.py

@@ -42,7 +42,7 @@ class DetectionVisMetric(BaseVisMetric):
                 {
                     "type": "tag",
                     "tagId": "confidence",
-                    "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                    "value": [self.eval_result.mp.conf_threshold, 1],
                 },
                 {"type": "tag", "tagId": "outcome", "value": "TP"},
                 {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely-6.73.295/supervisely/nn/benchmark/object_detection/evaluation_params.yaml

@@ -0,0 +1,7 @@
+# Intersection over Union threshold that will be used for object matching.
+# It mostly affects visualizations, such as Outcome Counts, Confusion Matrix, and image previews.
+iou_threshold: 0.5
+
+# Confidence threshold.
+# Set 'auto' to calculate the optimal confidence threshold.
+confidence_threshold: auto

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/evaluator.py

@@ -55,9 +55,7 @@ class ObjectDetectionEvalResult(BaseEvalResult):
             self.coco_gt, self.coco_dt = read_coco_datasets(self.coco_gt, self.coco_dt)
 
         self.mp = MetricProvider(
-            self.eval_data["matches"],
-            self.eval_data["coco_metrics"],
-            self.eval_data["params"],
+            self.eval_data,
             self.coco_gt,
             self.coco_dt,
         )

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -56,7 +56,7 @@ def filter_by_conf(matches: list, conf: float):
 
 
 class MetricProvider:
-    def __init__(self, matches: list, coco_metrics: dict, params: dict, cocoGt, cocoDt):
+    def __init__(self, eval_data: dict, cocoGt, cocoDt):
         """
         Main class for calculating prediction metrics.
 
@@ -71,11 +71,16 @@ class MetricProvider:
         :param cocoDt: COCO object with predicted annotations
         :type cocoDt: COCO
         """
-        self.matches = matches
-        self.coco_metrics = coco_metrics
-        self.params = params
+        self.eval_data = eval_data
+        self.matches = eval_data["matches"]
+        self.coco_metrics = eval_data["coco_metrics"]
+        self.params = eval_data["params"]
         self.cocoGt = cocoGt
         self.cocoDt = cocoDt
+        self.coco_mAP = self.coco_metrics["mAP"]
+        self.coco_precision = self.coco_metrics["precision"]
+        self.iouThrs = self.params["iouThrs"]
+        self.recThrs = self.params["recThrs"]
 
         self.metric_names = METRIC_NAMES
 
@@ -83,41 +88,31 @@ class MetricProvider:
         self.cat_ids = cocoGt.getCatIds()
         self.cat_names = [cocoGt.cats[cat_id]["name"] for cat_id in self.cat_ids]
 
-        # eval_data
-        self.matches = matches
-        self.coco_mAP = coco_metrics["mAP"]
-        self.coco_precision = coco_metrics["precision"]
-        self.iouThrs = params["iouThrs"]
-        self.recThrs = params["recThrs"]
-
         # Evaluation params
-        eval_params = params.get("evaluation_params", {})
+        eval_params = self.params.get("evaluation_params", {})
         self.iou_threshold = eval_params.get("iou_threshold", 0.5)
         self.iou_threshold_idx = np.where(np.isclose(self.iouThrs, self.iou_threshold))[0][0]
         self.iou_threshold_per_class = eval_params.get("iou_threshold_per_class")
-        self.iou_idx_per_class = params.get("iou_idx_per_class")  # {cat id: iou_idx}
-        if self.iou_threshold_per_class is not None:
-            # TODO: temporary solution
-            eval_params["average_across_iou_thresholds"] = False
+        self.iou_idx_per_class = self.params.get("iou_idx_per_class")  # {cat id: iou_idx}
         self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
 
     def calculate(self):
-        self.m_full = _MetricProvider(
-            self.matches, self.coco_metrics, self.params, self.cocoGt, self.cocoDt
-        )
+        self.m_full = _MetricProvider(self.matches, self.eval_data, self.cocoGt, self.cocoDt)
         self.m_full._calculate_score_profile()
 
         # Find optimal confidence threshold
         self.f1_optimal_conf, self.best_f1 = self.m_full.get_f1_optimal_conf()
+        self.custom_conf_threshold, self.custom_f1 = self.m_full.get_custom_conf_threshold()
+
+        # Confidence threshold that will be used in visualizations
+        self.conf_threshold = self.custom_conf_threshold or self.f1_optimal_conf
 
         # Filter by optimal confidence threshold
-        if self.f1_optimal_conf is not None:
-            matches_filtered = filter_by_conf(self.matches, self.f1_optimal_conf)
+        if self.conf_threshold is not None:
+            matches_filtered = filter_by_conf(self.matches, self.conf_threshold)
         else:
             matches_filtered = self.matches
-        self.m = _MetricProvider(
-            matches_filtered, self.coco_metrics, self.params, self.cocoGt, self.cocoDt
-        )
+        self.m = _MetricProvider(matches_filtered, self.eval_data, self.cocoGt, self.cocoDt)
         self.matches_filtered = matches_filtered
         self.m._init_counts()
 
@@ -155,7 +150,7 @@ class MetricProvider:
         ap_by_class = dict(zip(self.cat_names, ap_by_class))
         ap_custom_by_class = self.AP_custom_per_class().tolist()
         ap_custom_by_class = dict(zip(self.cat_names, ap_custom_by_class))
-        return {
+        data = {
             "mAP": base["mAP"],
             "AP50": self.coco_metrics.get("AP50"),
             "AP75": self.coco_metrics.get("AP75"),
@@ -172,6 +167,9 @@ class MetricProvider:
             "AP_by_class": ap_by_class,
             f"AP{iou_name}_by_class": ap_custom_by_class,
         }
+        if self.custom_conf_threshold is not None:
+            data["custom_confidence_threshold"] = self.custom_conf_threshold
+        return data
 
     def key_metrics(self):
         iou_name = int(self.iou_threshold * 100)
@@ -187,7 +185,7 @@ class MetricProvider:
         iou_name = int(self.iou_threshold * 100)
         if self.iou_threshold_per_class is not None:
             iou_name = "_custom"
-        return {
+        data = {
             "mAP": table["mAP"],
             "AP50": table["AP50"],
             "AP75": table["AP75"],
@@ -198,8 +196,11 @@ class MetricProvider:
             "Avg. IoU": table["iou"],
             "Classification Acc.": table["classification_accuracy"],
             "Calibration Score": table["calibration_score"],
-            "optimal confidence threshold": table["f1_optimal_conf"],
+            "Optimal confidence threshold": table["f1_optimal_conf"],
         }
+        if self.custom_conf_threshold is not None:
+            data["Custom confidence threshold"] = table["custom_confidence_threshold"]
+        return data
 
     def AP_per_class(self):
         s = self.coco_precision[:, :, :, 0, 2].copy()
@@ -262,25 +263,27 @@ class MetricProvider:
 
 
 class _MetricProvider:
-    def __init__(self, matches: list, coco_metrics: dict, params: dict, cocoGt, cocoDt):
+    def __init__(self, matches: list, eval_data: dict, cocoGt, cocoDt):
         """
         type cocoGt: COCO
         type cocoDt: COCO
         """
 
+        self.matches = matches
+        self.eval_data = eval_data
+        self.coco_metrics = eval_data["coco_metrics"]
+        self.params = eval_data["params"]
         self.cocoGt = cocoGt
+        self.cocoDt = cocoDt
+        self.coco_mAP = self.coco_metrics["mAP"]
+        self.coco_precision = self.coco_metrics["precision"]
+        self.iouThrs = self.params["iouThrs"]
+        self.recThrs = self.params["recThrs"]
 
         # metainfo
         self.cat_ids = cocoGt.getCatIds()
         self.cat_names = [cocoGt.cats[cat_id]["name"] for cat_id in self.cat_ids]
 
-        # eval_data
-        self.matches = matches
-        self.coco_mAP = coco_metrics["mAP"]
-        self.coco_precision = coco_metrics["precision"]
-        self.iouThrs = params["iouThrs"]
-        self.recThrs = params["recThrs"]
-
         # Matches
         self.tp_matches = [m for m in self.matches if m["type"] == "TP"]
         self.fp_matches = [m for m in self.matches if m["type"] == "FP"]
@@ -290,13 +293,12 @@ class _MetricProvider:
         self.ious = np.array([m["iou"] for m in self.tp_matches])
 
         # Evaluation params
-        self.params = params
         self.iou_idx_per_class = np.array(
-            [params["iou_idx_per_class"][cat_id] for cat_id in self.cat_ids]
+            [self.params["iou_idx_per_class"][cat_id] for cat_id in self.cat_ids]
         )[:, None]
-        eval_params = params.get("evaluation_params", {})
+        eval_params = self.params.get("evaluation_params", {})
         self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
-        
+
     def _init_counts(self):
         cat_ids = self.cat_ids
         iouThrs = self.iouThrs
@@ -307,9 +309,6 @@ class _MetricProvider:
             ious.append(match["iou"])
             cats.append(cat_id_to_idx[match["category_id"]])
         ious = np.array(ious) + np.spacing(1)
-        if 0.8999999999999999 in iouThrs:
-            iouThrs = iouThrs.copy()
-            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious) - 1
         cats = np.array(cats)
         # TP
@@ -345,9 +344,16 @@ class _MetricProvider:
         self.FP_count = int(self._take_iou_thresholds(false_positives).sum())
         self.FN_count = int(self._take_iou_thresholds(false_negatives).sum())
 
+        # self.true_positives = self.eval_data["true_positives"]
+        # self.false_negatives = self.eval_data["false_negatives"]
+        # self.false_positives = self.eval_data["false_positives"]
+        # self.TP_count = int(self._take_iou_thresholds(self.true_positives).sum())
+        # self.FP_count = int(self._take_iou_thresholds(self.false_positives).sum())
+        # self.FN_count = int(self._take_iou_thresholds(self.false_negatives).sum())
+
     def _take_iou_thresholds(self, x):
         return np.take_along_axis(x, self.iou_idx_per_class, axis=1)
-    
+
     def base_metrics(self):
         if self.average_across_iou_thresholds:
             tp = self.true_positives
@@ -495,9 +501,6 @@ class _MetricProvider:
         )
         scores = np.array([m["score"] for m in matches_sorted])
         ious = np.array([m["iou"] if m["type"] == "TP" else 0.0 for m in matches_sorted])
-        if 0.8999999999999999 in iouThrs:
-            iouThrs = iouThrs.copy()
-            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious + np.spacing(1))
 
         # Check
@@ -565,6 +568,16 @@ class _MetricProvider:
         best_f1 = self.score_profile["f1"][argmax]
         return f1_optimal_conf, best_f1
 
+    def get_custom_conf_threshold(self):
+        if (~np.isnan(self.score_profile["f1"])).sum() == 0:
+            return None, None
+        conf_threshold = self.params.get("evaluation_params", {}).get("confidence_threshold")
+        if conf_threshold is not None and conf_threshold != "auto":
+            idx = np.argmin(np.abs(self.score_profile["scores"] - conf_threshold))
+            custom_f1 = self.score_profile["f1"][idx]
+            return conf_threshold, custom_f1
+        return None, None
+
     def calibration_curve(self):
         from sklearn.calibration import (  # pylint: disable=import-error
             calibration_curve,

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/text_templates.py

@@ -43,7 +43,7 @@ markdown_overview = """
 - **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
 {}
 - **IoU threshold**: {}
-- **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+{}
 - **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
@@ -78,7 +78,7 @@ In this section you can visually assess the model performance through examples.
 
 > Click on the image to view the **Ground Truth**, **Prediction**, and **Difference** annotations side-by-side. 
 
-> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
+> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). {}
 """
 
 markdown_predictions_gallery = """
@@ -125,7 +125,7 @@ To measure this, we calculate **Recall**. Recall counts errors, when the model d
 """
 
 notification_recall = {
-    "title": "Recall = {}",
+    "title": "{}",
     "description": "The model correctly found <b>{} of {}</b> total instances in the dataset.",
 }
 
@@ -148,7 +148,7 @@ To measure this, we calculate **Precision**. Precision counts errors, when the m
 """
 
 notification_precision = {
-    "title": "Precision = {}",
+    "title": "{}",
     "description": "The model correctly predicted <b>{} of {}</b> predictions made by the model in total.",
 }
 

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/vis_metrics/confidence_distribution.py

@@ -28,7 +28,8 @@ class ConfidenceDistribution(DetectionVisMetric):
     def _get_figure(self):  # -> go.Figure:
         import plotly.graph_objects as go  # pylint: disable=import-error
 
-        f1_optimal_conf, best_f1 = self.eval_result.mp.m_full.get_f1_optimal_conf()
+        f1_optimal_conf = self.eval_result.mp.f1_optimal_conf
+        custom_conf_threshold = self.eval_result.mp.custom_conf_threshold
 
         # Histogram of confidence scores (TP vs FP)
         scores_tp, scores_fp = self.eval_result.mp.m_full.scores_tp_and_fp()
@@ -88,7 +89,7 @@ class ConfidenceDistribution(DetectionVisMetric):
                 x1=f1_optimal_conf,
                 y0=0,
                 y1=tp_y.max() * 1.3,
-                line=dict(color="orange", width=1, dash="dash"),
+                line=dict(color="orange", width=2, dash="dash"),
             )
             fig.add_annotation(
                 x=f1_optimal_conf,
@@ -104,4 +105,21 @@ class ConfidenceDistribution(DetectionVisMetric):
             )
             fig.update_xaxes(title_text="Confidence Score", range=[0, 1])
             fig.update_yaxes(title_text="Count", range=[0, tp_y.max() * 1.3])
+
+        if custom_conf_threshold is not None:
+            # Custom threshold
+            fig.add_shape(
+                type="line",
+                x0=custom_conf_threshold,
+                x1=custom_conf_threshold,
+                y0=0,
+                y1=tp_y.max() * 1.3,
+                line=dict(color="orange", width=2, dash="dash"),
+            )
+            fig.add_annotation(
+                x=custom_conf_threshold,
+                y=tp_y.max() * 1.3,
+                text=f"Confidence threshold: {custom_conf_threshold:.2f}",
+                showarrow=False,
+            )
         return fig

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py

@@ -101,6 +101,22 @@ class ConfidenceScore(DetectionVisMetric):
                 text=f"F1-optimal threshold: {self.eval_result.mp.f1_optimal_conf:.2f}",
                 showarrow=False,
             )
+        if self.eval_result.mp.custom_conf_threshold is not None:
+            # Add vertical line for the custom threshold
+            fig.add_shape(
+                type="line",
+                x0=self.eval_result.mp.custom_conf_threshold,
+                x1=self.eval_result.mp.custom_conf_threshold,
+                y0=0,
+                y1=self.eval_result.mp.custom_f1,
+                line=dict(color="black", width=2, dash="dash"),
+            )
+            fig.add_annotation(
+                x=self.eval_result.mp.custom_conf_threshold,
+                y=self.eval_result.mp.custom_f1 + 0.04,
+                text=f"Confidence threshold: {self.eval_result.mp.custom_conf_threshold:.2f}",
+                showarrow=False,
+            )
         fig.update_layout(
             dragmode=False,
             modebar=dict(

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/vis_metrics/explore_predictions.py

@@ -14,12 +14,17 @@ class ExplorePredictions(DetectionVisMetric):
 
     @property
     def md(self) -> MarkdownWidget:
-        text = self.vis_texts.markdown_explorer
+        conf_threshold_info = "Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model."
+        if self.eval_result.mp.custom_conf_threshold is not None:
+            conf_threshold_info = (
+                "Differences are calculated for the custom confidence threshold (set manually)."
+            )
+        text = self.vis_texts.markdown_explorer.format(conf_threshold_info)
+
         return MarkdownWidget(self.MARKDOWN, "Explore Predictions", text)
 
     def gallery(self, opacity) -> GalleryWidget:
-        optimal_conf = self.eval_result.mp.f1_optimal_conf
-        default_filters = [{"confidence": [optimal_conf, 1]}]
+        default_filters = [{"confidence": [self.eval_result.mp.conf_threshold, 1]}]
         gallery = GalleryWidget(
             self.GALLERY, columns_number=3, filters=default_filters, opacity=opacity
         )
@@ -62,7 +67,7 @@ class ExplorePredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             }
         ]
         explore["title"] = "Explore all predictions"
@@ -89,7 +94,7 @@ class ExplorePredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             },
         ]
         for pairs_data in self.eval_result.matched_pair_data.values():

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py

@@ -56,6 +56,7 @@ class KeyMetrics(DetectionVisMetric):
             width="60%",
             show_header_controls=False,
             main_column=columns[0],
+            page_size=15,
         )
         return table
 

{supervisely-6.73.293 → supervisely-6.73.295}/supervisely/nn/benchmark/object_detection/vis_metrics/model_predictions.py

@@ -99,7 +99,7 @@ class ModelPredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             },
             # {"type": "tag", "tagId": "outcome", "value": "FP"},
         ]