supervisely 6.73.294__py3-none-any.whl → 6.73.296__py3-none-any.whl

supervisely/nn/benchmark/object_detection/vis_metrics/confidence_distribution.py

@@ -28,7 +28,8 @@ class ConfidenceDistribution(DetectionVisMetric):
     def _get_figure(self):  # -> go.Figure:
         import plotly.graph_objects as go  # pylint: disable=import-error
 
-        f1_optimal_conf, best_f1 = self.eval_result.mp.m_full.get_f1_optimal_conf()
+        f1_optimal_conf = self.eval_result.mp.f1_optimal_conf
+        custom_conf_threshold = self.eval_result.mp.custom_conf_threshold
 
         # Histogram of confidence scores (TP vs FP)
         scores_tp, scores_fp = self.eval_result.mp.m_full.scores_tp_and_fp()
@@ -88,7 +89,7 @@ class ConfidenceDistribution(DetectionVisMetric):
                 x1=f1_optimal_conf,
                 y0=0,
                 y1=tp_y.max() * 1.3,
-                line=dict(color="orange", width=1, dash="dash"),
+                line=dict(color="orange", width=2, dash="dash"),
             )
             fig.add_annotation(
                 x=f1_optimal_conf,
@@ -104,4 +105,21 @@ class ConfidenceDistribution(DetectionVisMetric):
             )
             fig.update_xaxes(title_text="Confidence Score", range=[0, 1])
             fig.update_yaxes(title_text="Count", range=[0, tp_y.max() * 1.3])
+
+        if custom_conf_threshold is not None:
+            # Custom threshold
+            fig.add_shape(
+                type="line",
+                x0=custom_conf_threshold,
+                x1=custom_conf_threshold,
+                y0=0,
+                y1=tp_y.max() * 1.3,
+                line=dict(color="orange", width=2, dash="dash"),
+            )
+            fig.add_annotation(
+                x=custom_conf_threshold,
+                y=tp_y.max() * 1.3,
+                text=f"Confidence threshold: {custom_conf_threshold:.2f}",
+                showarrow=False,
+            )
         return fig

supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py

@@ -101,6 +101,22 @@ class ConfidenceScore(DetectionVisMetric):
                 text=f"F1-optimal threshold: {self.eval_result.mp.f1_optimal_conf:.2f}",
                 showarrow=False,
             )
+        if self.eval_result.mp.custom_conf_threshold is not None:
+            # Add vertical line for the custom threshold
+            fig.add_shape(
+                type="line",
+                x0=self.eval_result.mp.custom_conf_threshold,
+                x1=self.eval_result.mp.custom_conf_threshold,
+                y0=0,
+                y1=self.eval_result.mp.custom_f1,
+                line=dict(color="black", width=2, dash="dash"),
+            )
+            fig.add_annotation(
+                x=self.eval_result.mp.custom_conf_threshold,
+                y=self.eval_result.mp.custom_f1 + 0.04,
+                text=f"Confidence threshold: {self.eval_result.mp.custom_conf_threshold:.2f}",
+                showarrow=False,
+            )
         fig.update_layout(
             dragmode=False,
             modebar=dict(

supervisely/nn/benchmark/object_detection/vis_metrics/explore_predictions.py

@@ -14,12 +14,17 @@ class ExplorePredictions(DetectionVisMetric):
 
     @property
     def md(self) -> MarkdownWidget:
-        text = self.vis_texts.markdown_explorer
+        conf_threshold_info = "Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model."
+        if self.eval_result.mp.custom_conf_threshold is not None:
+            conf_threshold_info = (
+                "Differences are calculated for the custom confidence threshold (set manually)."
+            )
+        text = self.vis_texts.markdown_explorer.format(conf_threshold_info)
+
         return MarkdownWidget(self.MARKDOWN, "Explore Predictions", text)
 
     def gallery(self, opacity) -> GalleryWidget:
-        optimal_conf = self.eval_result.mp.f1_optimal_conf
-        default_filters = [{"confidence": [optimal_conf, 1]}]
+        default_filters = [{"confidence": [self.eval_result.mp.conf_threshold, 1]}]
         gallery = GalleryWidget(
             self.GALLERY, columns_number=3, filters=default_filters, opacity=opacity
         )
@@ -62,7 +67,7 @@ class ExplorePredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             }
         ]
         explore["title"] = "Explore all predictions"
@@ -89,7 +94,7 @@ class ExplorePredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             },
         ]
         for pairs_data in self.eval_result.matched_pair_data.values():

supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py

@@ -56,6 +56,7 @@ class KeyMetrics(DetectionVisMetric):
             width="60%",
             show_header_controls=False,
             main_column=columns[0],
+            page_size=15,
         )
         return table
 

supervisely/nn/benchmark/object_detection/vis_metrics/model_predictions.py

@@ -99,7 +99,7 @@ class ModelPredictions(DetectionVisMetric):
             {
                 "type": "tag",
                 "tagId": "confidence",
-                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                "value": [self.eval_result.mp.conf_threshold, 1],
             },
             # {"type": "tag", "tagId": "outcome", "value": "FP"},
         ]

supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts.py

@@ -120,69 +120,14 @@ class OutcomeCounts(DetectionVisMetric):
                 "title"
             ] = f"{outcome}: {len(matches_data)} object{'s' if len(matches_data) > 1 else ''}"
             res["clickData"][outcome]["imagesIds"] = list(img_ids)
+
             res["clickData"][outcome]["filters"] = [
                 {
                     "type": "tag",
                     "tagId": "confidence",
-                    "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                    "value": [self.eval_result.mp.conf_threshold, 1],
                 },
                 {"type": "tag", "tagId": "outcome", "value": outcome},
             ]
 
         return res
-
-    # def get_diff_data(self) -> Dict:
-    #     res = {}
-
-    #     res["layoutTemplate"] = [
-    #         {"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"},
-    #         {"skipObjectTagsFiltering": ["outcome"], "columnTitle": "Prediction"},
-    #         {"skipObjectTagsFiltering": ["confidence"], "columnTitle": "Difference"},
-    #     ]
-
-    #     click_data = res.setdefault("clickData", {})
-    #     for outcome, matches_data in self.eval_result.click_data.outcome_counts.items():
-    #         filters = [
-    #             {
-    #                 "type": "tag",
-    #                 "tagId": "confidence",
-    #                 "value": [self.eval_result.mp.f1_optimal_conf, 1],
-    #             },
-    #             {"type": "tag", "tagId": "outcome", "value": outcome},
-    #         ]
-    #         for match_data in matches_data:
-    #             pairs_data = self.eval_result.matched_pair_data[match_data["gt_img_id"]]
-    #             gt = pairs_data.gt_image_info
-    #             pred = pairs_data.pred_image_info
-    #             diff = pairs_data.diff_image_info
-    #             assert gt.name == pred.name == diff.name
-    #             for img_id in [pred.id, diff.id]:
-    #                 key = click_data.setdefault(str(img_id), {})
-    #                 key["imagesIds"] = [gt.id, pred.id, diff.id]
-    #                 key["filters"] = filters
-    #                 key["title"] = f"Image: {gt.name}"
-
-    #                 object_bindings = []
-    #                 for img in [pred, diff]:
-    #                     if img == pred:
-    #                         ann_json = pairs_data.pred_annotation.to_json()
-    #                     else:
-    #                         ann_json = pairs_data.diff_annotation.to_json()
-    #                     for obj in ann_json["objects"]:
-    #                         for tag in obj["tags"]:
-    #                             if tag["name"] == "matched_gt_id":
-    #                                 object_bindings.append(
-    #                                     [
-    #                                         {
-    #                                             "id": obj["id"],
-    #                                             "annotationKey": img.id,
-    #                                         },
-    #                                         {
-    #                                             "id": int(tag["value"]),
-    #                                             "annotationKey": gt.id if img == pred else pred.id,
-    #                                         },
-    #                                     ]
-    #                                 )
-    #                 key["objectsBindings"] = object_bindings
-
-    #     return res

supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py

@@ -183,7 +183,7 @@ class PerClassOutcomeCounts(DetectionVisMetric):
                     {
                         "type": "tag",
                         "tagId": "confidence",
-                        "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                        "value": [self.eval_result.mp.conf_threshold, 1],
                     },
                     {"type": "tag", "tagId": "outcome", "value": outcome},
                 ]

supervisely/nn/benchmark/object_detection/vis_metrics/overview.py

@@ -37,6 +37,15 @@ class Overview(DetectionVisMetric):
         if self.eval_result.different_iou_thresholds_per_class:
             iou_threshold = "Different IoU thresholds for each class (see the table below)"
 
+        conf_text = (
+            f"- **Optimal confidence threshold**: "
+            f"{round(self.eval_result.mp.f1_optimal_conf, 4)} (calculated automatically), "
+            f"<a href='{opt_conf_url}' target='_blank'>learn more</a>."
+        )
+        custom_conf_thrs = self.eval_result.mp.custom_conf_threshold
+        if custom_conf_thrs is not None:
+            conf_text += f"\n- **Custom confidence threshold**: {custom_conf_thrs}"
+
         formats = [
             model_name.replace("_", "\_"),
             checkpoint_name.replace("_", "\_"),
@@ -51,8 +60,7 @@ class Overview(DetectionVisMetric):
             note_about_images,
             starter_app_info,
             iou_threshold,
-            round(self.eval_result.mp.f1_optimal_conf, 4),
-            opt_conf_url,
+            conf_text,
             self.eval_result.mp.average_across_iou_thresholds,
             average_url,
             self.vis_texts.docs_url,
@@ -119,7 +127,7 @@ class Overview(DetectionVisMetric):
         starter_app_info = train_session or evaluator_session or ""
 
         return classes_str, images_str, starter_app_info
-    
+
     @property
     def iou_per_class_md(self) -> List[MarkdownWidget]:
         if not self.eval_result.different_iou_thresholds_per_class:

supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py

@@ -19,7 +19,7 @@ class PRCurve(DetectionVisMetric):
 
     @property
     def md(self) -> MarkdownWidget:
-        text = self.vis_texts.markdown_pr_curve.format(self.vis_texts.definitions.f1_score)
+        text = self.vis_texts.markdown_pr_curve.format(self.vis_texts.definitions.about_pr_tradeoffs)
         return MarkdownWidget(self.MARKDOWN, "Precision-Recall Curve", text)
 
     @property

supervisely/nn/benchmark/object_detection/vis_metrics/precision.py

@@ -26,11 +26,21 @@ class Precision(DetectionVisMetric):
     @property
     def notification(self) -> NotificationWidget:
         title, desc = self.vis_texts.notification_precision.values()
-        tp_plus_fp = self.eval_result.mp.TP_count + self.eval_result.mp.FP_count
+        mp = self.eval_result.mp
+        tp_plus_fp = mp.TP_count + mp.FP_count
+        precision = mp.base_metrics()["precision"].round(2)
+        if mp.average_across_iou_thresholds:
+            iou_text = "[0.5,0.55,...,0.95]"
+        else:
+            if mp.iou_threshold_per_class is not None:
+                iou_text = "custom"
+            else:
+                iou_text = mp.iou_threshold
+        title = f"Precision (IoU={iou_text}) = {precision}"
         return NotificationWidget(
             self.NOTIFICATION,
-            title.format(self.eval_result.mp.base_metrics()["precision"].round(2)),
-            desc.format(self.eval_result.mp.TP_count, tp_plus_fp),
+            title,
+            desc.format(mp.TP_count, tp_plus_fp),
         )
 
     @property
@@ -51,9 +61,9 @@ class Precision(DetectionVisMetric):
     def _get_figure(self):  #  -> go.Figure
         import plotly.express as px  # pylint: disable=import-error
 
-        sorted_by_precision = self.eval_result.mp.per_class_metrics().sort_values(by="precision")
+        sorted_by_f1 = self.eval_result.mp.per_class_metrics().sort_values(by="f1")
         fig = px.bar(
-            sorted_by_precision,
+            sorted_by_f1,
             x="category",
             y="precision",
             # title="Per-class Precision (Sorted by F1)",
@@ -62,14 +72,14 @@ class Precision(DetectionVisMetric):
             color_continuous_scale="Plasma",
         )
         fig.update_traces(hovertemplate="Class: %{x}<br>Precision: %{y:.2f}<extra></extra>")
-        if len(sorted_by_precision) <= 20:
+        if len(sorted_by_f1) <= 20:
             fig.update_traces(
-                text=sorted_by_precision.round(2),
+                text=sorted_by_f1["precision"].round(2),
                 textposition="outside",
             )
         fig.update_xaxes(title_text="Class")
         fig.update_yaxes(title_text="Precision", range=[0, 1])
         fig.update_layout(
-            width=700 if len(sorted_by_precision) < 10 else None,
+            width=700 if len(sorted_by_f1) < 10 else None,
         )
         return fig

supervisely/nn/benchmark/object_detection/vis_metrics/recall.py

@@ -26,11 +26,21 @@ class Recall(DetectionVisMetric):
     @property
     def notification(self) -> NotificationWidget:
         title, desc = self.vis_texts.notification_recall.values()
-        tp_plus_fn = self.eval_result.mp.TP_count + self.eval_result.mp.FN_count
+        mp = self.eval_result.mp
+        tp_plus_fn = mp.TP_count + mp.FN_count
+        recall = mp.base_metrics()["recall"].round(2)
+        if mp.average_across_iou_thresholds:
+            iou_text = "[0.5,0.55,...,0.95]"
+        else:
+            if mp.iou_threshold_per_class is not None:
+                iou_text = "custom"
+            else:
+                iou_text = mp.iou_threshold
+        title = f"Recall (IoU={iou_text}) = {recall}"
         return NotificationWidget(
             self.NOTIFICATION,
-            title.format(self.eval_result.mp.base_metrics()["recall"].round(2)),
-            desc.format(self.eval_result.mp.TP_count, tp_plus_fn),
+            title,
+            desc.format(mp.TP_count, tp_plus_fn),
         )
 
     @property

supervisely/nn/benchmark/object_detection/visualizer.py

@@ -452,7 +452,7 @@ class ObjectDetectionVisualizer(BaseVisualizer):
                                     conf = tag.value
                                     break
 
-                            if conf < self.eval_result.mp.f1_optimal_conf:
+                            if conf < self.eval_result.mp.conf_threshold:
                                 continue  # do not add labels with low confidence to diff project
                             if match_tag_id:
                                 continue  # do not add TP labels to diff project

supervisely/nn/benchmark/utils/__init__.py

@@ -1,6 +1,5 @@
 # isort: skip_file
 from supervisely.nn.benchmark.utils.detection.calculate_metrics import calculate_metrics
-from supervisely.nn.benchmark.utils.detection.metric_provider import MetricProvider
 from supervisely.nn.benchmark.utils.detection.sly2coco import sly2coco
 from supervisely.nn.benchmark.utils.detection.utlis import read_coco_datasets
 from supervisely.nn.benchmark.utils.detection.utlis import try_set_conf_auto

supervisely/nn/benchmark/utils/detection/__init__.py

@@ -1,2 +1 @@
-from supervisely.nn.benchmark.utils.detection.calculate_metrics import calculate_metrics
-from supervisely.nn.benchmark.utils.detection.metric_provider import MetricProvider
+from supervisely.nn.benchmark.utils.detection.calculate_metrics import calculate_metrics

supervisely/nn/benchmark/utils/detection/calculate_metrics.py

@@ -82,6 +82,11 @@ def calculate_metrics(
         iou_idx = np.where(np.isclose(iouThrs, iou_threshold))[0][0]
         iou_idx_per_class = {cat_id: iou_idx for cat_id in cocoGt.getCatIds()}
 
+    # TODO: Add support for average_across_iou_thresholds
+    if iou_threshold_per_class is not None or iou_threshold != 0.5:
+        average_across_iou_thresholds = False
+        evaluation_params["average_across_iou_thresholds"] = average_across_iou_thresholds
+    
     eval_img_dict = get_eval_img_dict(cocoEval)
     eval_img_dict_cls = get_eval_img_dict(cocoEval_cls)
     matches = get_matches(
@@ -90,6 +95,7 @@ def calculate_metrics(
         cocoEval_cls,
         iou_idx_per_class=iou_idx_per_class,
     )
+    # true_positives, false_positives, false_negatives = get_counts(eval_img_dict, cocoEval_cls)
 
     params = {
         "iouThrs": cocoEval.params.iouThrs,
@@ -110,43 +116,31 @@ def calculate_metrics(
     return eval_data
 
 
-def get_counts(cocoEval):
-    """
-    true_positives, false_positives, false_negatives
-
-    type cocoEval: COCOeval
-    """
-    aRng = cocoEval.params.areaRng[0]
-    cat_ids = cocoEval.params.catIds
-    eval_imgs = [ev for ev in cocoEval.evalImgs if ev is not None and ev["aRng"] == aRng]
-
-    N = len(eval_imgs)
-    T = len(cocoEval.params.iouThrs)
-    K = max(cat_ids) + 1
-
-    true_positives = np.zeros((K, N, T))
-    false_positives = np.zeros((K, N, T))
-    false_negatives = np.zeros((K, N, T))
-
-    for i, eval_img in enumerate(eval_imgs):
-        catId = eval_img["category_id"]
-        dt_matches = eval_img["dtMatches"]
-        gt_matches = eval_img["gtMatches"]
-
-        # Ignore
-        if np.any(eval_img["gtIgnore"]):
-            dt_matches = eval_img["dtMatches"].copy()
-            dt_matches[eval_img["dtIgnore"]] = -1
-
-            gt_matches = eval_img["gtMatches"].copy()
-            gt_ignore_mask = eval_img["gtIgnore"][None,].repeat(T, axis=0).astype(bool)
-            gt_matches[gt_ignore_mask] = -1
-
-        true_positives[catId, i] = np.sum(dt_matches > 0, axis=1)
-        false_positives[catId, i] = np.sum(dt_matches == 0, axis=1)
-        false_negatives[catId, i] = np.sum(gt_matches == 0, axis=1)
-
-    return true_positives[cat_ids], false_positives[cat_ids], false_negatives[cat_ids]
+def get_counts(eval_img_dict: dict, cocoEval_cls):
+    cat_ids = cocoEval_cls.cocoGt.getCatIds()
+    iouThrs = cocoEval_cls.params.iouThrs
+    catId2idx = {cat_id: i for i, cat_id in enumerate(cat_ids)}
+    true_positives = np.zeros((len(cat_ids), len(iouThrs)))
+    false_positives = np.zeros((len(cat_ids), len(iouThrs)))
+    false_negatives = np.zeros((len(cat_ids), len(iouThrs)))
+    for img_id, eval_imgs in eval_img_dict.items():
+        for eval_img in eval_imgs:
+            cat_id = eval_img["category_id"]
+            cat_idx = catId2idx[cat_id]
+            gtIgnore = eval_img["gtIgnore"]
+            # if conf_thresh is not None:
+            #     scores = np.array(eval_img["dtScores"])
+            #     dt_conf_mask = scores < conf_thresh
+            #     dt_not_ignore[:, dt_conf_mask] = False
+            #     for idx, dt_id in enumerate(eval_img['dtIds']):
+            #         if dt_conf_mask[idx]:
+            #             eval_img["gtMatches"][eval_img["gtMatches"] == dt_id] = 0.
+            gt_not_ignore_idxs = np.where(np.logical_not(gtIgnore))[0]
+            dt_not_ignore = np.logical_not(eval_img["dtIgnore"])
+            true_positives[cat_idx] += ((eval_img["dtMatches"] > 0) & dt_not_ignore).sum(1)
+            false_positives[cat_idx] += ((eval_img["dtMatches"] == 0) & dt_not_ignore).sum(1)
+            false_negatives[cat_idx] += (eval_img["gtMatches"][:, gt_not_ignore_idxs] == 0).sum(1)
+    return true_positives.astype(int), false_positives.astype(int), false_negatives.astype(int)
 
 
 def get_counts_and_scores(cocoEval, cat_id: int, t: int):

supervisely/nn/benchmark/visualization/evaluation_result.py

@@ -13,7 +13,7 @@ from supervisely.app.widgets import SlyTqdm
 from supervisely.io.env import team_id
 from supervisely.io.fs import dir_empty, mkdir
 from supervisely.io.json import load_json_file
-from supervisely.nn.benchmark.utils.detection.metric_provider import MetricProvider
+from supervisely.nn.benchmark.object_detection.metric_provider import MetricProvider
 from supervisely.nn.benchmark.visualization.vis_click_data import ClickData, IdMapper
 from supervisely.sly_logger import logger
 from supervisely.task.progress import tqdm_sly
@@ -218,9 +218,7 @@ class EvalResult:
             )
 
         self.mp = MetricProvider(
-            self.eval_data["matches"],
-            self.eval_data["coco_metrics"],
-            self.eval_data["params"],
+            self.eval_data,
             self.coco_gt,
             self.coco_dt,
         )

supervisely/nn/benchmark/visualization/vis_click_data.py

@@ -2,9 +2,7 @@ from __future__ import annotations
 
 from collections import defaultdict
 
-from supervisely.nn.benchmark.utils.detection.metric_provider import (
-    MetricProvider,
-)
+from supervisely.nn.benchmark.object_detection.metric_provider import MetricProvider
 
 
 class IdMapper:

{supervisely-6.73.294.dist-info → supervisely-6.73.296.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.294
+Version: 6.73.296
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.294.dist-info → supervisely-6.73.296.dist-info}/RECORD

@@ -550,7 +550,7 @@ supervisely/cli/project/project_get.py,sha256=RWnMxuKd_WWhUlA5QEqI_9d4N6x8VDq0ta
 supervisely/cli/project/project_upload.py,sha256=qA_0ktOpJxUUa_Hliwvny8-uIYDQBiuzHMq8nufbLd4,1416
 supervisely/cli/release/__init__.py,sha256=5aDijgIIDsKFZVayq8anvv5ynWKhC4LZqAdESTKCW2c,335
 supervisely/cli/release/release.py,sha256=OtAi9g_2kvjp5lx4aiTE0JxczzJcaBtDFpHR-rHWFpE,9308
-supervisely/cli/release/run.py,sha256=z6jDTVweCnlaIWsdUIN282B_JUqYZefJMIPLZnq2g9U,19899
+supervisely/cli/release/run.py,sha256=nuFh9mVFcrUlB4PWagMEC6iUIa7nA2pMiM_CC1IZ42M,19814
 supervisely/cli/task/__init__.py,sha256=n0ofJDqX3AMvvTz1umfBDfEUPDFzk5Htve3nnZFd7fs,67
 supervisely/cli/task/task_set.py,sha256=KIGJ-X0iB7DzX3Ig8720FJh1WpohTVkkPk8HZt2rIzM,1337
 supervisely/cli/teamfiles/__init__.py,sha256=1V9ZFo8-xVOjFFJr7siJx2VAv-F0m44cxMEm5ZEU0Ww,251
@@ -755,14 +755,14 @@ supervisely/nn/benchmark/comparison/detection_visualization/__init__.py,sha256=4
 supervisely/nn/benchmark/comparison/detection_visualization/text_templates.py,sha256=JQ2DqGsvUBbjVmEsq9pGc41U8WQrtaX_Gin4IBguIow,28735
 supervisely/nn/benchmark/comparison/detection_visualization/visualizer.py,sha256=1PZU9P7VP4lPZDuv9dg_f3il03SQ_zRr5G_L6Q72gbQ,11421
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/__init__.py,sha256=2cqjAOwahJoptYhbFKEWws8gRW3K3bxlA9KnQleCrsA,1125
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/avg_precision_by_class.py,sha256=f4FMY-XCiNs0VA-e0aOFVNImjTQSnF4wTLNRSlj2oYA,4636
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py,sha256=IKg2rofo21xGxQIoQyNlZ-UqOpAVviSn_A9SDkokC0w,7592
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predictions.py,sha256=axaviTZ4dLVWIc2R-o0Kv8g8Zr1SQVfyeokoLEA9Eqw,6484
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/avg_precision_by_class.py,sha256=cf4jxgrau1t8SHiMisnHJ7aenVRPSLUgz1Q4Gic_jTk,4635
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py,sha256=BAYWwLASuq-OBQkjFRH6GIT3nUBTZY0RJNipP2CD9To,8075
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predictions.py,sha256=JGyvYZ4ZpbgZE2zQ6yUY6flE4n2STl0OMwmwpDF4AeU,6482
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/localization_accuracy.py,sha256=OYmnloods1UYQ8SIPAcyOK33w0iYSn637OeMKNTrgbA,5342
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py,sha256=vev6EOc7-01i-5VAyQQwm9FGOMFYcWfJ2Y6ufUWg-DQ,13143
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py,sha256=drRm_Hn8Da4Oc_dsa4ol_mZSeMRNYT1-Zkmb0LbiEAA,10193
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py,sha256=66hs426dR1_TUps9K-UhBp5_xBiFjIouKcF_5gP-Hn8,4797
-supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py,sha256=602GFNpKZjeRhxUTdlcE6ZczcFEGEjp0qLoTbkM54M4,11558
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py,sha256=01XWPppmkeovJ-9AzMNOpiqomLGdZGMu2FmQ2BIfoE8,13142
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py,sha256=p4L1lFqAlz3D6R3mMUvvtSiFdZZGT_aEzIEpBs5hsv0,10501
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/pr_curve.py,sha256=qJtswtxnuLSGWPVq599AXindPNDoem8cR1-0mgI1POc,4807
+supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/precision_recal_f1.py,sha256=is75jOQhykejNmj4Pea875s2MwBL0wZCxPhI-YostY4,11556
 supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/speedtest.py,sha256=sQDkzfpVNaSYBHVcHYqydRSWN0i-yV9uhtEAggg295A,10879
 supervisely/nn/benchmark/comparison/semantic_segmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/comparison/semantic_segmentation/text_templates.py,sha256=iHybWNpH9-LTjx8-iuyNawVoBXwXzho66a1K_afY_9Q,7622
@@ -777,36 +777,36 @@ supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/renormaliz
 supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/speedtest.py,sha256=sQDkzfpVNaSYBHVcHYqydRSWN0i-yV9uhtEAggg295A,10879
 supervisely/nn/benchmark/instance_segmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/instance_segmentation/benchmark.py,sha256=lTDzgKGpfeF5o_a2nS56wiAsUQPH1eubk37b9CaB2KI,1171
-supervisely/nn/benchmark/instance_segmentation/evaluation_params.yaml,sha256=POzpiaxnxuwAPSNQOGgjoUPfsk6Lf5hb9GLHwltWY5Y,94
+supervisely/nn/benchmark/instance_segmentation/evaluation_params.yaml,sha256=NoaecTcEp-LhsDQcHNQZi1gzNXcahgycKy_C4aDcSSw,304
 supervisely/nn/benchmark/instance_segmentation/evaluator.py,sha256=mpCi8S6YNwlVvgcERQSHBOhC9PrSfQkQ55pPTcK6V9c,2811
-supervisely/nn/benchmark/instance_segmentation/text_templates.py,sha256=l2I1PbenuBzcCPu_h2J5JE5gQsJCr6lNnoIzk5BEuwc,25868
+supervisely/nn/benchmark/instance_segmentation/text_templates.py,sha256=usKqm_FaO-WXiopxzrdjpIrOqHdqFQ89lmYoayzt6KM,25597
 supervisely/nn/benchmark/instance_segmentation/visualizer.py,sha256=8NscOKy7JK4AG-Czu3SM0qJQXLDfKD9URdG1d4nz89E,564
 supervisely/nn/benchmark/object_detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/nn/benchmark/object_detection/base_vis_metric.py,sha256=XjUnFCnCMYLrpjojIOwiRNaSsSLYpozTHWfwLkaCd5U,1612
+supervisely/nn/benchmark/object_detection/base_vis_metric.py,sha256=44Em214YPxZgn2hEzFvqBcnjsyiElD9TSuLamwUnx20,1611
 supervisely/nn/benchmark/object_detection/benchmark.py,sha256=Wb4xlFXilIMVfsifNNQY25uE52NeEDLzQpnq8QPYq9U,1086
-supervisely/nn/benchmark/object_detection/evaluation_params.yaml,sha256=POzpiaxnxuwAPSNQOGgjoUPfsk6Lf5hb9GLHwltWY5Y,94
-supervisely/nn/benchmark/object_detection/evaluator.py,sha256=EOQQbmwQqjjvbRu3tY24SRA7K8nyqshR92gUcP1lcrY,7371
-supervisely/nn/benchmark/object_detection/metric_provider.py,sha256=iV79hlyB-_wj-X25-JPjoXfwQWNxGrMVweha3JZA46M,22557
-supervisely/nn/benchmark/object_detection/text_templates.py,sha256=ZjkcP91dj98_1xqxKSy5TGrU08puXaLhpjNC-c_41A0,26113
-supervisely/nn/benchmark/object_detection/visualizer.py,sha256=NpLKVW5fo6N0kYzgLsfY66wvCv38G3k-SNm4HImXt6g,32366
+supervisely/nn/benchmark/object_detection/evaluation_params.yaml,sha256=NoaecTcEp-LhsDQcHNQZi1gzNXcahgycKy_C4aDcSSw,304
+supervisely/nn/benchmark/object_detection/evaluator.py,sha256=s-hPBm5BmoCgwoozVyDacum4kVLNtYK6I6NCt_L_LSA,7278
+supervisely/nn/benchmark/object_detection/metric_provider.py,sha256=59UnOX7VuYvVQFeUJy5v6EFIpqSDNgx5wMp9qyVixgM,23686
+supervisely/nn/benchmark/object_detection/text_templates.py,sha256=4BgTIX1Co4WK9_VSUa1qWCmh5OJzo3_opVU6LOjKSjc,25842
+supervisely/nn/benchmark/object_detection/visualizer.py,sha256=0eJ-ATDVMyjMRInoEK604vL3vJPN5Ge1HjarS11CS0A,32365
 supervisely/nn/benchmark/object_detection/vis_metrics/__init__.py,sha256=AXCLHEySEdR-B-5sfDoWBmmOLBVlyW2U_xr8Ta42sQI,2096
-supervisely/nn/benchmark/object_detection/vis_metrics/confidence_distribution.py,sha256=OlwkPgzEQ-RegcLZHVUVOL0n6I_2iayPVpAIie4y2O8,3615
-supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py,sha256=kuOhQDNwAsBbtjuMU_7GajVzu6j6n3xDJTv_hNKX6o8,4007
+supervisely/nn/benchmark/object_detection/vis_metrics/confidence_distribution.py,sha256=J6l9Vc8TGsvkTyH5u7Ry4P2jgJC2GYmcgOeMALftZBw,4254
+supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py,sha256=pmxnF_UJk0WhqEdL7O_yIjIBtxPipLQZVJwCr6XB3zc,4751
 supervisely/nn/benchmark/object_detection/vis_metrics/confusion_matrix.py,sha256=2PJUt0-njRpzN7XBGjkSt9kkh5tDPuv_Sne-2v8DWHc,3731
-supervisely/nn/benchmark/object_detection/vis_metrics/explore_predictions.py,sha256=wIYfq3izM2XNJHr56h3j5XhuU8W8Y3wO_RKAwxntQs4,4855
+supervisely/nn/benchmark/object_detection/vis_metrics/explore_predictions.py,sha256=sMSFntieAdmL8siTKwppB4RxIhfZlsg1loUeOEhiKOg,5234
 supervisely/nn/benchmark/object_detection/vis_metrics/f1_score_at_different_iou.py,sha256=Aewzu2QhxZoPT_k0QJt_G11B8M2DXLCGRjE0MlVYNko,2892
 supervisely/nn/benchmark/object_detection/vis_metrics/frequently_confused.py,sha256=7rObk7WNsfwK7xBWl3aOxcn0uD48njEc04fQIPHc3_4,4678
 supervisely/nn/benchmark/object_detection/vis_metrics/iou_distribution.py,sha256=lv4Bk8W4X8ZhvQKyMXI46d240PNlMFx1hdji_aoTS50,3601
-supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py,sha256=byucJuHYWSXIZU8U1Dc44QDpG3lTlhoNdUfD1b-uriw,4721
-supervisely/nn/benchmark/object_detection/vis_metrics/model_predictions.py,sha256=gsGDsesiwOcqeFvHr33b4PSJNw6MoA5brO-qRydRtsA,5944
-supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts.py,sha256=HuTgisYmXCSUeF5WOahy-uaCdvRLsNzg28BDrZ-5hww,7161
-supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py,sha256=BKsb1XGVsg6-aOI5G6NibxvdD5lVzkMjHisI8T85Sns,6853
-supervisely/nn/benchmark/object_detection/vis_metrics/overview.py,sha256=uBxHIz8t0ujS5MI-LASg1RrPx3YqeKKc7dWz3MozHIU,6546
-supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py,sha256=EeZmyNlTVQLQ-0wIDGdvFmRkahJBBiOKSmWiAJ8Bfks,3478
+supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py,sha256=cbGwhG95rp5NmOy0qHvpSiAjWqtQQMWCfKs7FzbuKzc,4747
+supervisely/nn/benchmark/object_detection/vis_metrics/model_predictions.py,sha256=7ynfCcZMFdwxFf2seUiFc_lc5XzkExIdxI5wr_64mmQ,5943
+supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts.py,sha256=yPC-6ionooo0eUEiKSOFb-LkhqOuWUXwWvQt6SBQ4Ws,4414
+supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py,sha256=FDba0gWJWnf7mGtl79TONC2fihc39sc-tXCVLYCt84Y,6852
+supervisely/nn/benchmark/object_detection/vis_metrics/overview.py,sha256=-22iCkQkpSQP5SZclyM0H9jSs-EynEbBgnhJKayG634,6923
+supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py,sha256=64pWJD0306D2NYupxLD1irhpGgAOCgoTpZiSVBBdqDU,3488
 supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve_by_class.py,sha256=Bl_buVvH8SVqwsc4DcHnojMOqpwTnRgXFt9yw_Y1BR0,1607
-supervisely/nn/benchmark/object_detection/vis_metrics/precision.py,sha256=cAgMrp13uulHfM8xnPDZyR6PqS8nck1Fo7YPpvHPCbw,2708
+supervisely/nn/benchmark/object_detection/vis_metrics/precision.py,sha256=lrxCH82blwMVAxofLDmqPPgs7V1S9AFgmfxtHAoWEdI,2973
 supervisely/nn/benchmark/object_detection/vis_metrics/precision_avg_per_class.py,sha256=IetoRTE9xZHHMZlPwUmQyTnaD7cQpnIQ3zJmUUr6fgY,1965
-supervisely/nn/benchmark/object_detection/vis_metrics/recall.py,sha256=AAxg3eJDjWIZEarOMZNcdIcYpVDFFDlbc5bwF4X3GIo,2579
+supervisely/nn/benchmark/object_detection/vis_metrics/recall.py,sha256=sgvxqfXrT5fPtV8_t1Sr11Ln66MPYNlQlPVSmXjqnYU,2864
 supervisely/nn/benchmark/object_detection/vis_metrics/recall_vs_precision.py,sha256=u-00HH54XzlhFuzc7U-mk2-IhUSEPYzv23BIIz8STvk,1984
 supervisely/nn/benchmark/object_detection/vis_metrics/reliability_diagram.py,sha256=_8ie3dPfwOyNCg-YhqO5jFW6kubCfQC2Obn9QSZFVeQ,3615
 supervisely/nn/benchmark/object_detection/vis_metrics/speedtest.py,sha256=uAYu42WUcQCBdfm-PZxOPAGLDsPlcnDJL7PCXDRStbY,5449
@@ -831,10 +831,9 @@ supervisely/nn/benchmark/semantic_segmentation/vis_metrics/overview.py,sha256=RT
 supervisely/nn/benchmark/semantic_segmentation/vis_metrics/renormalized_error_ou.py,sha256=w4oqirf_o7uz0fwaapaFR0ByjCBSEfMv--ZgEZTkuFQ,1575
 supervisely/nn/benchmark/semantic_segmentation/vis_metrics/speedtest.py,sha256=0UP_HReIciHQyU6sOXnWhAzyVBoGznV7ZHF8m4vEVX0,5941
 supervisely/nn/benchmark/semantic_segmentation/vis_metrics/vis_texts.py,sha256=rRdYZxmhQX4T3RsXJVGp34NMZPz8jUHtVvBN5BpPJ5I,603
-supervisely/nn/benchmark/utils/__init__.py,sha256=JHT73gWdwgLJKTiCpDdwggm1t_EWB0JCC90-zD7reXM,741
-supervisely/nn/benchmark/utils/detection/__init__.py,sha256=L3QKGuKUlR2N_QFRTRsa6gfLDbksIaFMYO0Hukxxy1U,172
-supervisely/nn/benchmark/utils/detection/calculate_metrics.py,sha256=LXZET5yLp9S7Uq2eX4HpAMnBMxTI5Q2CgKSc1mCfaRM,11388
-supervisely/nn/benchmark/utils/detection/metric_provider.py,sha256=cgF6uzF7XOvU2CpxyU7zuK1HH6hhNiIV3vQc8MAzwMU,19934
+supervisely/nn/benchmark/utils/__init__.py,sha256=r0Ay4OMqfIL-9wwJykKji_Uks2Dm9vUhyA7hT8eLxII,657
+supervisely/nn/benchmark/utils/detection/__init__.py,sha256=6CsMxQqUp1GOc-2Wmnw2lamtvklHo2tcCYTxgT5NsZo,88
+supervisely/nn/benchmark/utils/detection/calculate_metrics.py,sha256=gC6by_2HT9ACuxbtW93eKeioW9sCMMDM3aPi99w1xx8,11963
 supervisely/nn/benchmark/utils/detection/metrics.py,sha256=oyictdJ7rRDUkaVvHoxntywW5zZweS8pIJ1bN6JgXtE,2420
 supervisely/nn/benchmark/utils/detection/sly2coco.py,sha256=0O2LSCU5zIX34mD4hZIv8O3-j6LwnB0DqhiVPAiosO8,6883
 supervisely/nn/benchmark/utils/detection/utlis.py,sha256=dKhsOGmQKH20-IlD90DWfZzi171j65N71hNdHRCX5Hs,954
@@ -844,10 +843,10 @@ supervisely/nn/benchmark/utils/semantic_segmentation/evaluator.py,sha256=iPHRo1L
 supervisely/nn/benchmark/utils/semantic_segmentation/loader.py,sha256=_5ZZ7Nkd8WWYJnKwc1Dx3bEPS_1R84gG_hQc0w0TXWw,1957
 supervisely/nn/benchmark/utils/semantic_segmentation/utils.py,sha256=X5NiR02R-0To2_SuSGHZZccl_-Bupg5F9d7nziIMRMc,3874
 supervisely/nn/benchmark/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-supervisely/nn/benchmark/visualization/evaluation_result.py,sha256=733HJL4rJa5XqCJydW9vSyaepvpHzym9wQsw1wFEgeI,10251
+supervisely/nn/benchmark/visualization/evaluation_result.py,sha256=QNAvxy1-W15cyHZbeAysiDWvXAgdgQZnRAlhL_Uzlxs,10159
 supervisely/nn/benchmark/visualization/renderer.py,sha256=j6dFqIGa9M0DbVVR-jgJab-MzUUiw47coHiiBF_H_jQ,3923
 supervisely/nn/benchmark/visualization/report_template.html,sha256=tylBK5Bb2cqKACK1GZUKyIjPS9yHQFHAS-QeEEwhqTE,2172
-supervisely/nn/benchmark/visualization/vis_click_data.py,sha256=hBeVepHngTGVHK3MiWe8qZY87taifxnoUXq22W2xaqo,3724
+supervisely/nn/benchmark/visualization/vis_click_data.py,sha256=4YE0nJJrzuBEbGChzkivblC-pAwEeXsKU55sXVZQyt0,3716
 supervisely/nn/benchmark/visualization/widgets/__init__.py,sha256=UovmhwLH4Au81JFrFz0NwPasaIqPEI-zXN-JntTc2FU,949
 supervisely/nn/benchmark/visualization/widgets/widget.py,sha256=1cpfyaZjug8ZQIRR5SB4BgkSxP3BMwkYFRWH36U1eF0,818
 supervisely/nn/benchmark/visualization/widgets/chart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1071,9 +1070,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
 supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
 supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
-supervisely-6.73.294.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-supervisely-6.73.294.dist-info/METADATA,sha256=tgjdB_FSwcOSyWj9_Mxkeije_FDbelZVyzWcy4vGTDo,33573
-supervisely-6.73.294.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-supervisely-6.73.294.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
-supervisely-6.73.294.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
-supervisely-6.73.294.dist-info/RECORD,,
+supervisely-6.73.296.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+supervisely-6.73.296.dist-info/METADATA,sha256=jw6IjeK21DXTFEgbISalLDxHn7BWvWAQkZ3rbhAMJ5g,33573
+supervisely-6.73.296.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+supervisely-6.73.296.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
+supervisely-6.73.296.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
+supervisely-6.73.296.dist-info/RECORD,,