supervisely 6.73.294__py3-none-any.whl → 6.73.296__py3-none-any.whl

supervisely/cli/release/run.py

@@ -1,33 +1,41 @@
-from pathlib import Path
-import sys
-import os
 import json
-import traceback
-import click
+import os
 import re
 import subprocess
+import sys
+import traceback
+from pathlib import Path
+
+import click
 import git
 from dotenv import load_dotenv
 from rich.console import Console
 
-
 MIN_SUPPORTED_INSTANCE_VERSION = "6.8.3"
 
+from supervisely._utils import setup_certificates
 from supervisely.cli.release.release import (
+    delete_tag,
     find_tag_in_repo,
-    push_tag,
     get_app_from_instance,
-    get_module_root,
-    get_module_path,
     get_appKey,
-    release,
-    slug_is_valid,
-    delete_tag,
+    get_created_at,
     get_instance_version,
+    get_module_path,
+    get_module_root,
     get_user,
-    get_created_at,
+    push_tag,
+    release,
+    slug_is_valid,
 )
 
+load_dotenv(os.path.expanduser("~/supervisely.env"))
+try:
+    setup_certificates()
+except Exception as e:
+    print(f"Error setting up certificates: {e}")
+    traceback.print_exc()
+
 
 def _check_git(repo: git.Repo):
     console = Console()
@@ -39,11 +47,7 @@ def _check_git(repo: git.Repo):
         )
         console.print("  Untracked files:")
         console.print(
-            "\n".join(
-                [f"  {i+1}) " + file for i, file in enumerate(repo.untracked_files)][
-                    :20
-                ]
-            )
+            "\n".join([f"  {i+1}) " + file for i, file in enumerate(repo.untracked_files)][:20])
         )
         if len(repo.untracked_files) > 20:
             console.print(f"  ... and {len(repo.untracked_files) - 20} more.")
@@ -98,17 +102,13 @@ def _ask_release_version(repo: git.Repo):
     console = Console()
 
     def extract_version(tag_name):
-        return tag_name[len("sly-release-"):] if tag_name.startswith("sly-release-") else tag_name
+        return tag_name[len("sly-release-") :] if tag_name.startswith("sly-release-") else tag_name
 
     try:
         sly_release_tags = [
-            tag.name
-            for tag in repo.tags
-            if _check_release_version(extract_version(tag.name))
+            tag.name for tag in repo.tags if _check_release_version(extract_version(tag.name))
         ]
-        sly_release_tags.sort(
-            key=lambda tag: [int(n) for n in extract_version(tag)[1:].split(".")]
-        )
+        sly_release_tags.sort(key=lambda tag: [int(n) for n in extract_version(tag)[1:].split(".")])
         current_release_version = extract_version(sly_release_tags[-1])[1:]
         suggested_release_version = ".".join(
             [
@@ -232,9 +232,7 @@ def run(
     # get module path and check if it is a git repo
     module_root = get_module_root(app_directory)
     if sub_app_directory is not None:
-        sub_app_directory = (
-            Path(sub_app_directory).absolute().relative_to(module_root).as_posix()
-        )
+        sub_app_directory = Path(sub_app_directory).absolute().relative_to(module_root).as_posix()
     module_path = get_module_path(module_root, sub_app_directory)
     try:
         repo = git.Repo(module_root)
@@ -244,8 +242,7 @@ def run(
         )
         return False
 
-    # get server address
-    load_dotenv(os.path.expanduser("~/supervisely.env"))
+    # load server address
     server_address = os.getenv("SERVER_ADDRESS", None)
     if server_address is None:
         console.print(
@@ -314,9 +311,7 @@ def run(
         with open(module_path.joinpath("config.json"), "r") as f:
             config = json.load(f)
     except FileNotFoundError:
-        console.print(
-            f'[red][Error][/] Cannot find "config.json" file at "{module_path}"'
-        )
+        console.print(f'[red][Error][/] Cannot find "config.json" file at "{module_path}"')
         return False
     except json.JSONDecodeError as e:
         console.print(
@@ -379,9 +374,7 @@ def run(
             f'[red][Error][/] Could not access "{server_address}". Check that instance is running and accessible'
         )
         return False
-    module_exists_label = (
-        "[yellow bold]updated[/]" if app_exist else "[green bold]created[/]"
-    )
+    module_exists_label = "[yellow bold]updated[/]" if app_exist else "[green bold]created[/]"
 
     # print details
     console.print(f"Application directory:\t[green]{module_path}[/]")
@@ -390,9 +383,7 @@ def run(
     console.print(f"User:\t\t\t[green]{user_login} (id: {user_id})[/]")
     if release_token:
         console.print(f"Release token:\t\t[green]{hided(release_token)}[/]")
-        console.print(
-            f"Release User: \t\t[green]{release_user_login} (id: {release_user_id})[/]"
-        )
+        console.print(f"Release User: \t\t[green]{release_user_login} (id: {release_user_id})[/]")
     console.print(f"Git branch:\t\t[green]{repo.active_branch}[/]")
     console.print(f"App Name:\t\t[green]{app_name}[/]")
     console.print(f"App Key:\t\t[green]{hided(appKey)}[/]\n")
@@ -411,14 +402,10 @@ def run(
         if release_version is None:
             release_version = _ask_release_version(repo)
         if not _check_release_version(release_version):
-            console.print(
-                '[red][Error][/] Incorrect release version. Should be of format "vX.X.X"'
-            )
+            console.print('[red][Error][/] Incorrect release version. Should be of format "vX.X.X"')
             return False
     else:
-        console.print(
-            f'Release version will be "{repo.active_branch.name}" as the branch name'
-        )
+        console.print(f'Release version will be "{repo.active_branch.name}" as the branch name')
         release_version = repo.active_branch.name
 
     # get release name
@@ -534,9 +521,7 @@ def run(
     return True
 
 
-@click.command(
-    help="This app allows you to release your aplication to Supervisely platform"
-)
+@click.command(help="This app allows you to release your aplication to Supervisely platform")
 @click.option(
     "-p",
     "--path",
@@ -554,9 +539,7 @@ def run(
     required=False,
     help='[Optional] Release version in format "vX.X.X"',
 )
-@click.option(
-    "--release-description", required=False, help="[Optional] Release description"
-)
+@click.option("--release-description", required=False, help="[Optional] Release description")
 @click.option(
     "--share",
     is_flag=True,

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/avg_precision_by_class.py

@@ -114,7 +114,7 @@ class AveragePrecisionByClass(BaseVisMetrics):
                     {
                         "type": "tag",
                         "tagId": "confidence",
-                        "value": [eval_result.mp.f1_optimal_conf, 1],
+                        "value": [eval_result.mp.conf_threshold, 1],
                     },
                     {"type": "tag", "tagId": "outcome", "value": "TP"},
                     {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py

@@ -154,6 +154,16 @@ class CalibrationScore(BaseVisMetrics):
                     line=dict(color="gray", width=2, dash="dash"),
                     name=f"F1-optimal threshold ({eval_result.name})",
                 )
+            if eval_result.mp.custom_conf_threshold is not None:
+                fig.add_shape(
+                    type="line",
+                    x0=eval_result.mp.custom_conf_threshold,
+                    x1=eval_result.mp.custom_conf_threshold,
+                    y0=0,
+                    y1=eval_result.mp.custom_f1,
+                    line=dict(color="black", width=2, dash="dash"),
+                    name=f"Confidence threshold ({eval_result.name})",
+                )
 
         # Update the layout
         fig.update_layout(

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predictions.py

@@ -76,7 +76,7 @@ class ExplorePredictions(BaseVisMetrics):
             anns = eval_res.api.annotation.download_batch(dataset_info.id, images_ids, force_metadata_for_links=False)
             annotations.append(anns)
             skip_tags_filtering.append(False)
-            min_conf = min(min_conf, eval_res.mp.f1_optimal_conf)
+            min_conf = min(min_conf, eval_res.mp.conf_threshold)
 
         images = list(i for x in zip(*images) for i in x)
         annotations = list(i for x in zip(*annotations) for i in x)
@@ -127,7 +127,7 @@ class ExplorePredictions(BaseVisMetrics):
             current_images_infos = sorted(current_images_infos, key=lambda x: names.index(x.name))
             images_ids.append([image_info.id for image_info in current_images_infos])
 
-            min_conf = min(min_conf, eval_res.mp.f1_optimal_conf)
+            min_conf = min(min_conf, eval_res.mp.conf_threshold)
 
         explore["imagesIds"] = list(i for x in zip(*images_ids) for i in x)
         explore["filters"] = [{"type": "tag", "tagId": "confidence", "value": [min_conf, 1]}]

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py

@@ -276,7 +276,7 @@ class OutcomeCounts(BaseVisMetrics):
                 title = f"{model_name}. {outcome}: {len(obj_ids)} object{'s' if len(obj_ids) > 1 else ''}"
                 outcome_dict["title"] = title
                 outcome_dict["imagesIds"] = list(img_ids)
-                thr = eval_result.mp.f1_optimal_conf
+                thr = eval_result.mp.conf_threshold
                 if outcome == "FN":
                     outcome_dict["filters"] = [
                         {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -27,6 +27,7 @@ class Overview(BaseVisMetrics):
         evaluation result metrics displayed
         """
         super().__init__(vis_texts, eval_results)
+        self.team_id = None  # will be set in the visualizer
 
     @property
     def overview_md(self) -> List[MarkdownWidget]:
@@ -120,8 +121,7 @@ class Overview(BaseVisMetrics):
             if idx == 3 and not same_iou_thr:
                 continue
             metric_name = metric_renames_map.get(metric, metric)
-            values = [m[metric] for m in all_metrics]
-            values = [v if v is not None else "―" for v in values]
+            values = [m.get(metric, "―") for m in all_metrics]
             values = [round(v, 2) if isinstance(v, float) else v for v in values]
             row = [metric_name] + values
             dct = {"row": row, "id": metric, "items": row}
@@ -247,12 +247,18 @@ class Overview(BaseVisMetrics):
 
         iou_thrs_map = defaultdict(set)
         matched = True
-        for eval_result in self.eval_results:
-            for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
-                iou_thrs_map[cat_id].add(iou_thr)
-                if len(iou_thrs_map[cat_id]) > 1:
-                    matched = False
-                    break
+
+        if not all([not r.different_iou_thresholds_per_class for r in self.eval_results]):
+            matched = False
+        else:
+            for eval_result in self.eval_results:
+                iou_thrs_per_class = eval_result.mp.iou_threshold_per_class
+                if iou_thrs_per_class is not None:
+                    for cat_id, iou_thr in eval_result.mp.iou_threshold_per_class.items():
+                        iou_thrs_map[cat_id].add(iou_thr)
+                        if len(iou_thrs_map[cat_id]) > 1:
+                            matched = False
+                            break
 
         if matched:
             return None

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py

@@ -19,7 +19,7 @@ class PrCurve(BaseVisMetrics):
     @property
     def markdown_widget(self) -> MarkdownWidget:
         text: str = getattr(self.vis_texts, self.MARKDOWN_PR_CURVE).format(
-            self.vis_texts.definitions.f1_score
+            self.vis_texts.definitions.about_pr_tradeoffs
         )
         return MarkdownWidget(
             name=self.MARKDOWN_PR_CURVE, title="mAP & Precision-Recall Curve", text=text

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py

@@ -205,7 +205,7 @@ class PrecisionRecallF1(BaseVisMetrics):
                     {
                         "type": "tag",
                         "tagId": "confidence",
-                        "value": [eval_result.mp.f1_optimal_conf, 1],
+                        "value": [eval_result.mp.conf_threshold, 1],
                     },
                     {"type": "tag", "tagId": "outcome", "value": "TP"},
                     {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
@@ -293,7 +293,7 @@ class PrecisionRecallF1(BaseVisMetrics):
                 {
                     "type": "tag",
                     "tagId": "confidence",
-                    "value": [eval_result.mp.f1_optimal_conf, 1],
+                    "value": [eval_result.mp.conf_threshold, 1],
                 },
                 {"type": "tag", "tagId": "outcome", "value": "TP"},
                 {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely/nn/benchmark/instance_segmentation/evaluation_params.yaml

@@ -1,2 +1,7 @@
-# Intersection over Union threshold that will be used for objects mathcing
+# Intersection over Union threshold that will be used for object matching.
+# It mostly affects visualizations, such as Outcome Counts, Confusion Matrix, and image previews.
 iou_threshold: 0.5
+
+# Confidence threshold.
+# Set 'auto' to calculate the optimal confidence threshold.
+confidence_threshold: auto

supervisely/nn/benchmark/instance_segmentation/text_templates.py

@@ -43,7 +43,7 @@ markdown_overview = """
 - **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
 {}
 - **IoU threshold**: {}
-- **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+{}
 - **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
@@ -73,7 +73,7 @@ In this section you can visually assess the model performance through examples.
 
 > Click on the image to view the **Ground Truth**, **Prediction**, and **Difference** annotations side-by-side. 
 
-> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
+> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). {}
 """
 
 markdown_predictions_table = """### Prediction details for every image
@@ -100,7 +100,7 @@ To measure this, we calculate **Recall**. Recall counts errors, when the model d
 """
 
 notification_recall = {
-    "title": "Recall = {}",
+    "title": "{}",
     "description": "The model correctly found <b>{} of {}</b> total instances in the dataset.",
 }
 
@@ -123,7 +123,7 @@ To measure this, we calculate **Precision**. Precision counts errors, when the m
 """
 
 notification_precision = {
-    "title": "Precision = {}",
+    "title": "{}",
     "description": "The model correctly predicted <b>{} of {}</b> predictions made by the model in total.",
 }
 

supervisely/nn/benchmark/object_detection/base_vis_metric.py

@@ -42,7 +42,7 @@ class DetectionVisMetric(BaseVisMetric):
                 {
                     "type": "tag",
                     "tagId": "confidence",
-                    "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                    "value": [self.eval_result.mp.conf_threshold, 1],
                 },
                 {"type": "tag", "tagId": "outcome", "value": "TP"},
                 {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},

supervisely/nn/benchmark/object_detection/evaluation_params.yaml

@@ -1,2 +1,7 @@
-# Intersection over Union threshold that will be used for objects mathcing
+# Intersection over Union threshold that will be used for object matching.
+# It mostly affects visualizations, such as Outcome Counts, Confusion Matrix, and image previews.
 iou_threshold: 0.5
+
+# Confidence threshold.
+# Set 'auto' to calculate the optimal confidence threshold.
+confidence_threshold: auto

supervisely/nn/benchmark/object_detection/evaluator.py

@@ -55,9 +55,7 @@ class ObjectDetectionEvalResult(BaseEvalResult):
             self.coco_gt, self.coco_dt = read_coco_datasets(self.coco_gt, self.coco_dt)
 
         self.mp = MetricProvider(
-            self.eval_data["matches"],
-            self.eval_data["coco_metrics"],
-            self.eval_data["params"],
+            self.eval_data,
             self.coco_gt,
             self.coco_dt,
         )

supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -56,7 +56,7 @@ def filter_by_conf(matches: list, conf: float):
 
 
 class MetricProvider:
-    def __init__(self, matches: list, coco_metrics: dict, params: dict, cocoGt, cocoDt):
+    def __init__(self, eval_data: dict, cocoGt, cocoDt):
         """
         Main class for calculating prediction metrics.
 
@@ -71,11 +71,16 @@ class MetricProvider:
         :param cocoDt: COCO object with predicted annotations
         :type cocoDt: COCO
         """
-        self.matches = matches
-        self.coco_metrics = coco_metrics
-        self.params = params
+        self.eval_data = eval_data
+        self.matches = eval_data["matches"]
+        self.coco_metrics = eval_data["coco_metrics"]
+        self.params = eval_data["params"]
         self.cocoGt = cocoGt
         self.cocoDt = cocoDt
+        self.coco_mAP = self.coco_metrics["mAP"]
+        self.coco_precision = self.coco_metrics["precision"]
+        self.iouThrs = self.params["iouThrs"]
+        self.recThrs = self.params["recThrs"]
 
         self.metric_names = METRIC_NAMES
 
@@ -83,41 +88,31 @@ class MetricProvider:
         self.cat_ids = cocoGt.getCatIds()
         self.cat_names = [cocoGt.cats[cat_id]["name"] for cat_id in self.cat_ids]
 
-        # eval_data
-        self.matches = matches
-        self.coco_mAP = coco_metrics["mAP"]
-        self.coco_precision = coco_metrics["precision"]
-        self.iouThrs = params["iouThrs"]
-        self.recThrs = params["recThrs"]
-
         # Evaluation params
-        eval_params = params.get("evaluation_params", {})
+        eval_params = self.params.get("evaluation_params", {})
         self.iou_threshold = eval_params.get("iou_threshold", 0.5)
         self.iou_threshold_idx = np.where(np.isclose(self.iouThrs, self.iou_threshold))[0][0]
         self.iou_threshold_per_class = eval_params.get("iou_threshold_per_class")
-        self.iou_idx_per_class = params.get("iou_idx_per_class")  # {cat id: iou_idx}
-        if self.iou_threshold_per_class is not None:
-            # TODO: temporary solution
-            eval_params["average_across_iou_thresholds"] = False
+        self.iou_idx_per_class = self.params.get("iou_idx_per_class")  # {cat id: iou_idx}
         self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
 
     def calculate(self):
-        self.m_full = _MetricProvider(
-            self.matches, self.coco_metrics, self.params, self.cocoGt, self.cocoDt
-        )
+        self.m_full = _MetricProvider(self.matches, self.eval_data, self.cocoGt, self.cocoDt)
         self.m_full._calculate_score_profile()
 
         # Find optimal confidence threshold
         self.f1_optimal_conf, self.best_f1 = self.m_full.get_f1_optimal_conf()
+        self.custom_conf_threshold, self.custom_f1 = self.m_full.get_custom_conf_threshold()
+
+        # Confidence threshold that will be used in visualizations
+        self.conf_threshold = self.custom_conf_threshold or self.f1_optimal_conf
 
         # Filter by optimal confidence threshold
-        if self.f1_optimal_conf is not None:
-            matches_filtered = filter_by_conf(self.matches, self.f1_optimal_conf)
+        if self.conf_threshold is not None:
+            matches_filtered = filter_by_conf(self.matches, self.conf_threshold)
         else:
             matches_filtered = self.matches
-        self.m = _MetricProvider(
-            matches_filtered, self.coco_metrics, self.params, self.cocoGt, self.cocoDt
-        )
+        self.m = _MetricProvider(matches_filtered, self.eval_data, self.cocoGt, self.cocoDt)
         self.matches_filtered = matches_filtered
         self.m._init_counts()
 
@@ -155,7 +150,7 @@ class MetricProvider:
         ap_by_class = dict(zip(self.cat_names, ap_by_class))
         ap_custom_by_class = self.AP_custom_per_class().tolist()
         ap_custom_by_class = dict(zip(self.cat_names, ap_custom_by_class))
-        return {
+        data = {
             "mAP": base["mAP"],
             "AP50": self.coco_metrics.get("AP50"),
             "AP75": self.coco_metrics.get("AP75"),
@@ -172,6 +167,9 @@ class MetricProvider:
             "AP_by_class": ap_by_class,
             f"AP{iou_name}_by_class": ap_custom_by_class,
         }
+        if self.custom_conf_threshold is not None:
+            data["custom_confidence_threshold"] = self.custom_conf_threshold
+        return data
 
     def key_metrics(self):
         iou_name = int(self.iou_threshold * 100)
@@ -187,7 +185,7 @@ class MetricProvider:
         iou_name = int(self.iou_threshold * 100)
         if self.iou_threshold_per_class is not None:
             iou_name = "_custom"
-        return {
+        data = {
             "mAP": table["mAP"],
             "AP50": table["AP50"],
             "AP75": table["AP75"],
@@ -198,8 +196,11 @@ class MetricProvider:
             "Avg. IoU": table["iou"],
             "Classification Acc.": table["classification_accuracy"],
             "Calibration Score": table["calibration_score"],
-            "optimal confidence threshold": table["f1_optimal_conf"],
+            "Optimal confidence threshold": table["f1_optimal_conf"],
         }
+        if self.custom_conf_threshold is not None:
+            data["Custom confidence threshold"] = table["custom_confidence_threshold"]
+        return data
 
     def AP_per_class(self):
         s = self.coco_precision[:, :, :, 0, 2].copy()
@@ -262,25 +263,27 @@ class MetricProvider:
 
 
 class _MetricProvider:
-    def __init__(self, matches: list, coco_metrics: dict, params: dict, cocoGt, cocoDt):
+    def __init__(self, matches: list, eval_data: dict, cocoGt, cocoDt):
         """
         type cocoGt: COCO
         type cocoDt: COCO
         """
 
+        self.matches = matches
+        self.eval_data = eval_data
+        self.coco_metrics = eval_data["coco_metrics"]
+        self.params = eval_data["params"]
         self.cocoGt = cocoGt
+        self.cocoDt = cocoDt
+        self.coco_mAP = self.coco_metrics["mAP"]
+        self.coco_precision = self.coco_metrics["precision"]
+        self.iouThrs = self.params["iouThrs"]
+        self.recThrs = self.params["recThrs"]
 
         # metainfo
         self.cat_ids = cocoGt.getCatIds()
         self.cat_names = [cocoGt.cats[cat_id]["name"] for cat_id in self.cat_ids]
 
-        # eval_data
-        self.matches = matches
-        self.coco_mAP = coco_metrics["mAP"]
-        self.coco_precision = coco_metrics["precision"]
-        self.iouThrs = params["iouThrs"]
-        self.recThrs = params["recThrs"]
-
         # Matches
         self.tp_matches = [m for m in self.matches if m["type"] == "TP"]
         self.fp_matches = [m for m in self.matches if m["type"] == "FP"]
@@ -290,13 +293,12 @@ class _MetricProvider:
         self.ious = np.array([m["iou"] for m in self.tp_matches])
 
         # Evaluation params
-        self.params = params
         self.iou_idx_per_class = np.array(
-            [params["iou_idx_per_class"][cat_id] for cat_id in self.cat_ids]
+            [self.params["iou_idx_per_class"][cat_id] for cat_id in self.cat_ids]
         )[:, None]
-        eval_params = params.get("evaluation_params", {})
+        eval_params = self.params.get("evaluation_params", {})
         self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
-        
+
     def _init_counts(self):
         cat_ids = self.cat_ids
         iouThrs = self.iouThrs
@@ -307,9 +309,6 @@ class _MetricProvider:
             ious.append(match["iou"])
             cats.append(cat_id_to_idx[match["category_id"]])
         ious = np.array(ious) + np.spacing(1)
-        if 0.8999999999999999 in iouThrs:
-            iouThrs = iouThrs.copy()
-            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious) - 1
         cats = np.array(cats)
         # TP
@@ -345,9 +344,16 @@ class _MetricProvider:
         self.FP_count = int(self._take_iou_thresholds(false_positives).sum())
         self.FN_count = int(self._take_iou_thresholds(false_negatives).sum())
 
+        # self.true_positives = self.eval_data["true_positives"]
+        # self.false_negatives = self.eval_data["false_negatives"]
+        # self.false_positives = self.eval_data["false_positives"]
+        # self.TP_count = int(self._take_iou_thresholds(self.true_positives).sum())
+        # self.FP_count = int(self._take_iou_thresholds(self.false_positives).sum())
+        # self.FN_count = int(self._take_iou_thresholds(self.false_negatives).sum())
+
     def _take_iou_thresholds(self, x):
         return np.take_along_axis(x, self.iou_idx_per_class, axis=1)
-    
+
     def base_metrics(self):
         if self.average_across_iou_thresholds:
             tp = self.true_positives
@@ -495,9 +501,6 @@ class _MetricProvider:
         )
         scores = np.array([m["score"] for m in matches_sorted])
         ious = np.array([m["iou"] if m["type"] == "TP" else 0.0 for m in matches_sorted])
-        if 0.8999999999999999 in iouThrs:
-            iouThrs = iouThrs.copy()
-            iouThrs[iouThrs == 0.8999999999999999] = 0.9
         iou_idxs = np.searchsorted(iouThrs, ious + np.spacing(1))
 
         # Check
@@ -565,6 +568,16 @@ class _MetricProvider:
         best_f1 = self.score_profile["f1"][argmax]
         return f1_optimal_conf, best_f1
 
+    def get_custom_conf_threshold(self):
+        if (~np.isnan(self.score_profile["f1"])).sum() == 0:
+            return None, None
+        conf_threshold = self.params.get("evaluation_params", {}).get("confidence_threshold")
+        if conf_threshold is not None and conf_threshold != "auto":
+            idx = np.argmin(np.abs(self.score_profile["scores"] - conf_threshold))
+            custom_f1 = self.score_profile["f1"][idx]
+            return conf_threshold, custom_f1
+        return None, None
+
     def calibration_curve(self):
         from sklearn.calibration import (  # pylint: disable=import-error
             calibration_curve,

supervisely/nn/benchmark/object_detection/text_templates.py

@@ -43,7 +43,7 @@ markdown_overview = """
 - **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
 {}
 - **IoU threshold**: {}
-- **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+{}
 - **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
@@ -78,7 +78,7 @@ In this section you can visually assess the model performance through examples.
 
 > Click on the image to view the **Ground Truth**, **Prediction**, and **Difference** annotations side-by-side. 
 
-> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
+> Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). {}
 """
 
 markdown_predictions_gallery = """
@@ -125,7 +125,7 @@ To measure this, we calculate **Recall**. Recall counts errors, when the model d
 """
 
 notification_recall = {
-    "title": "Recall = {}",
+    "title": "{}",
     "description": "The model correctly found <b>{} of {}</b> total instances in the dataset.",
 }
 
@@ -148,7 +148,7 @@ To measure this, we calculate **Precision**. Precision counts errors, when the m
 """
 
 notification_precision = {
-    "title": "Precision = {}",
+    "title": "{}",
     "description": "The model correctly predicted <b>{} of {}</b> predictions made by the model in total.",
 }