supervisely 6.73.206__py3-none-any.whl → 6.73.207__py3-none-any.whl

supervisely/nn/benchmark/base_benchmark.py

@@ -1,4 +1,5 @@
 import os
+import yaml
 from typing import Callable, List, Optional, Tuple, Union
 
 import numpy as np
@@ -31,6 +32,7 @@ class BaseBenchmark:
         progress: Optional[SlyTqdm] = None,
         progress_secondary: Optional[SlyTqdm] = None,
         classes_whitelist: Optional[List[str]] = None,
+        evaluation_params: Optional[dict] = None,
     ):
         self.api = api
         self.session: SessionJSON = None
@@ -51,6 +53,8 @@ class BaseBenchmark:
         self.vis_texts = None
         self.inference_speed_text = None
         self.train_info = None
+        self.evaluation_params = evaluation_params
+        self._validate_evaluation_params()
 
     def _get_evaluator_class(self) -> type:
         raise NotImplementedError()
@@ -152,6 +156,7 @@ class BaseBenchmark:
             progress=self.pbar,
             items_count=self.dt_project_info.items_count,
             classes_whitelist=self.classes_whitelist,
+            evaluation_params=self.evaluation_params,
         )
         self.evaluator.evaluate()
 
@@ -552,3 +557,7 @@ class BaseBenchmark:
                 chagned = True
         if chagned:
             self.api.project.update_meta(pred_project_id, pred_meta.to_json())
+
+    def _validate_evaluation_params(self):
+        if self.evaluation_params:
+            self._get_evaluator_class().validate_evaluation_params(self.evaluation_params)

supervisely/nn/benchmark/evaluation/base_evaluator.py

@@ -1,12 +1,15 @@
 import os
 import pickle
-from typing import List, Optional
+from typing import List, Optional, Union
+
+import yaml
 
 from supervisely.app.widgets import SlyTqdm
 from supervisely.task.progress import tqdm_sly
 
 
 class BaseEvaluator:
+    EVALUATION_PARAMS_YAML_PATH: str = None
 
     def __init__(
         self,
@@ -16,6 +19,7 @@ class BaseEvaluator:
         progress: Optional[SlyTqdm] = None,
         items_count: Optional[int] = None,  # TODO: is it needed?
         classes_whitelist: Optional[List[str]] = None,
+        evaluation_params: Optional[dict] = None,
     ):
         self.gt_project_path = gt_project_path
         self.dt_project_path = dt_project_path
@@ -25,12 +29,36 @@ class BaseEvaluator:
         os.makedirs(result_dir, exist_ok=True)
         self.classes_whitelist = classes_whitelist
 
+        if evaluation_params is None:
+            evaluation_params = self._get_default_evaluation_params()
+        self.evaluation_params = evaluation_params
+        if self.evaluation_params:
+            self.validate_evaluation_params(self.evaluation_params)
+
     def evaluate(self):
         raise NotImplementedError()
 
     def get_result_dir(self) -> str:
         return self.result_dir
 
+    @classmethod
+    def load_yaml_evaluation_params(cls) -> Union[str, None]:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return None
+        with open(cls.EVALUATION_PARAMS_YAML_PATH, "r") as f:
+            return f.read()
+
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        pass
+
+    @classmethod
+    def _get_default_evaluation_params(cls) -> dict:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return {}
+        else:
+            return yaml.safe_load(cls.load_yaml_evaluation_params())
+
     def _dump_pickle(self, data, file_path):
         with open(file_path, "wb") as f:
             pickle.dump(data, f)

supervisely/nn/benchmark/evaluation/coco/calculate_metrics.py

@@ -1,10 +1,39 @@
 from collections import defaultdict
-from typing import Callable, Optional, Literal
+from typing import Callable, List, Literal, Optional
 
 import numpy as np
 
-
-def calculate_metrics(cocoGt, cocoDt, iouType: Literal["bbox", "segm"], progress_cb: Optional[Callable] = None):
+# pylint: disable=import-error
+from pycocotools.cocoeval import COCOeval
+
+
+def set_cocoeval_params(
+    cocoeval: COCOeval,
+    parameters: dict,
+):
+    if parameters is None:
+        return
+    param_names = (
+        "iouThrs",
+        "recThrs",
+        "maxDets",
+        "areaRng",
+        "areaRngLbl",
+        # "kpt_oks_sigmas" # For keypoints
+    )
+    for param_name in param_names:
+        cocoeval.params.__setattr__(
+            param_name, parameters.get(param_name, cocoeval.params.__getattribute__(param_name))
+        )
+
+
+def calculate_metrics(
+    cocoGt,
+    cocoDt,
+    iouType: Literal["bbox", "segm"],
+    progress_cb: Optional[Callable] = None,
+    evaluation_params: Optional[dict] = None,
+):
     """
     Calculate COCO metrics.
 
@@ -19,35 +48,42 @@ def calculate_metrics(cocoGt, cocoDt, iouType: Literal["bbox", "segm"], progress
     :return: Results of the evaluation
     :rtype: dict
     """
-    from pycocotools.cocoeval import COCOeval  # pylint: disable=import-error
 
-    progress_cb(1) if progress_cb is not None else None
     cocoEval = COCOeval(cocoGt, cocoDt, iouType=iouType)
-    progress_cb(1) if progress_cb is not None else None
     cocoEval.evaluate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval.accumulate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval.summarize()
-    progress_cb(1) if progress_cb is not None else None
 
     # For classification metrics
     cocoEval_cls = COCOeval(cocoGt, cocoDt, iouType=iouType)
-    progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.params.useCats = 0
     cocoEval_cls.evaluate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.accumulate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.summarize()
-    progress_cb(1) if progress_cb is not None else None
+
+    iou_t = 0
+    is_custom_iou_threshold = (
+        evaluation_params is not None and evaluation_params.get("iou_threshold") and evaluation_params["iou_threshold"] != 0.5
+    )
+    if is_custom_iou_threshold:
+        iou_t = np.where(cocoEval.params.iouThrs == evaluation_params["iou_threshold"])[0][0]
 
     eval_img_dict = get_eval_img_dict(cocoEval)
     eval_img_dict_cls = get_eval_img_dict(cocoEval_cls)
-    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=0)
+    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=iou_t)
 
-    params = {"iouThrs": cocoEval.params.iouThrs, "recThrs": cocoEval.params.recThrs}
+    params = {
+        "iouThrs": cocoEval.params.iouThrs,
+        "recThrs": cocoEval.params.recThrs,
+        "evaluation_params": evaluation_params or {},
+    }
     coco_metrics = {"mAP": cocoEval.stats[0], "precision": cocoEval.eval["precision"]}
+    coco_metrics["AP50"] = cocoEval.stats[1]
+    coco_metrics["AP75"] = cocoEval.stats[2]
     eval_data = {
         "matches": matches,
         "coco_metrics": coco_metrics,

supervisely/nn/benchmark/evaluation/coco/evaluation_params.yaml

@@ -0,0 +1,2 @@
+# Intersection over Union threshold that will be used for objects mathcing
+iou_threshold: 0.5

supervisely/nn/benchmark/evaluation/coco/metric_provider.py

@@ -89,6 +89,8 @@ class MetricProvider:
         self.coco_precision = coco_metrics["precision"]
         self.iouThrs = params["iouThrs"]
         self.recThrs = params["recThrs"]
+        self.iou_threshold = params["evaluation_params"]["iou_threshold"]
+        self.iou_threshold_idx = np.searchsorted(self.iouThrs, self.iou_threshold)
 
     def calculate(self):
         self.m_full = _MetricProvider(
@@ -134,7 +136,64 @@ class MetricProvider:
         self._scores_tp_and_fp = self.m_full.scores_tp_and_fp()
         self._maximum_calibration_error = self.m_full.maximum_calibration_error()
         self._expected_calibration_error = self.m_full.expected_calibration_error()
+    
+    def json_metrics(self):
+        base = self.base_metrics()
+        iou_name = int(self.iou_threshold * 100)
+        ap_by_class = self.AP_per_class().tolist()
+        ap_by_class = dict(zip(self.cat_names, ap_by_class))
+        ap_custom_by_class = self.AP_custom_per_class().tolist()
+        ap_custom_by_class = dict(zip(self.cat_names, ap_custom_by_class))
+        return {
+            "mAP": base["mAP"],
+            "AP50": self.coco_metrics["AP50"],
+            "AP75": self.coco_metrics["AP75"],
+            f"AP{iou_name}": self.AP_custom(),
+            "f1": base["f1"],
+            "precision": base["precision"],
+            "recall": base["recall"],
+            "iou": base["iou"],
+            "classification_accuracy": base["classification_accuracy"],
+            "calibration_score": base["calibration_score"],
+            "f1_optimal_conf": self.f1_optimal_conf,
+            "expected_calibration_error": self.expected_calibration_error(),
+            "maximum_calibration_error": self.maximum_calibration_error(),
+            "AP_by_class": ap_by_class,
+            f"AP{iou_name}_by_class": ap_custom_by_class,
+        }
+    
+    def metric_table(self):
+        table = self.json_metrics()
+        iou_name = int(self.iou_threshold * 100)
+        return {
+            "mAP": table["mAP"],
+            "AP50": table["AP50"],
+            "AP75": table["AP75"],
+            f"AP{iou_name}": table[f"AP{iou_name}"],
+            "f1": table["f1"],
+            "precision": table["precision"],
+            "recall": table["recall"],
+            "Avg. IoU": table["iou"],
+            "Classification Acc.": table["classification_accuracy"],
+            "Calibration Score": table["calibration_score"],
+            "optimal confidence threshold": table["f1_optimal_conf"],
+        }
 
+    def AP_per_class(self):
+        s = self.coco_precision[:, :, :, 0, 2]
+        s[s == -1] = np.nan
+        ap = np.nanmean(s, axis=(0, 1))
+        return ap
+
+    def AP_custom_per_class(self):
+        s = self.coco_precision[self.iou_threshold_idx, :, :, 0, 2]
+        s[s == -1] = np.nan
+        ap = np.nanmean(s, axis=0)
+        return ap
+    
+    def AP_custom(self):
+        return np.nanmean(self.AP_custom_per_class())
+    
     def base_metrics(self):
         base = self._base_metrics
         calibration_score = 1 - self._expected_calibration_error

supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py

@@ -4,9 +4,12 @@ from supervisely.io.json import dump_json_file
 from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
 from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
+from pathlib import Path
 
 
 class InstanceSegmentationEvaluator(BaseEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/coco/evaluation_params.yaml"
+
     def evaluate(self):
         try:
             self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
@@ -19,12 +22,25 @@ class InstanceSegmentationEvaluator(BaseEvaluator):
 
         self._dump_datasets()
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
-        with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
+        with self.pbar(message="Evaluation: Calculating metrics", total=5) as p:
             self.eval_data = calculate_metrics(
-                self.cocoGt, self.cocoDt, iouType="segm", progress_cb=p.update
+                self.cocoGt,
+                self.cocoDt,
+                iouType="segm",
+                progress_cb=p.update,
+                evaluation_params=self.evaluation_params,
             )
         self._dump_eval_results()
 
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        iou_threshold = evaluation_params.get("iou_threshold")
+        if iou_threshold is not None:
+            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
+                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
+                f"but got {iou_threshold}"
+            )
+
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(
             self.gt_project_path,

supervisely/nn/benchmark/evaluation/object_detection_evaluator.py

@@ -4,9 +4,12 @@ from supervisely.io.json import dump_json_file
 from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
 from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
+from pathlib import Path
 
 
 class ObjectDetectionEvaluator(BaseEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/coco/evaluation_params.yaml"
+
     def evaluate(self):
         try:
             self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
@@ -17,12 +20,25 @@ class ObjectDetectionEvaluator(BaseEvaluator):
                 "try to use newer version of NN app."
             )
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
-        with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
+        with self.pbar(message="Evaluation: Calculating metrics", total=5) as p:
             self.eval_data = calculate_metrics(
-                self.cocoGt, self.cocoDt, iouType="bbox", progress_cb=p.update
+                self.cocoGt,
+                self.cocoDt,
+                iouType="bbox",
+                progress_cb=p.update,
+                evaluation_params=self.evaluation_params,
             )
         self._dump_eval_results()
 
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        iou_threshold = evaluation_params.get("iou_threshold")
+        if iou_threshold is not None:
+            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
+                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
+                f"but got {iou_threshold}"
+            )
+
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(
             self.gt_project_path,

supervisely/nn/benchmark/visualization/inference_speed/speedtest_overview.py

@@ -62,7 +62,7 @@ class SpeedtestOverview(MetricVis):
         res["columns"] = columns
         res["columnsOptions"] = columns_options
 
-        widget.main_column = columns[0]
+        widget.main_column = "Batch size"
         widget.fixed_columns = 1
         widget.show_header_controls = False
         return res

supervisely/nn/benchmark/visualization/vis_metric_base.py

@@ -216,6 +216,7 @@ class MetricVis:
                         "mainColumn": widget.main_column,
                         "fixColumns": widget.fixed_columns,
                         "showHeaderControls": widget.show_header_controls,
+                        "width": f"width:{widget.width}" if widget.width else "",
                     }
                 )
 

supervisely/nn/benchmark/visualization/vis_metrics/overview.py

@@ -15,12 +15,13 @@ class Overview(MetricVis):
     def __init__(self, loader: Visualizer) -> None:
         super().__init__(loader)
         self._is_overview = True
-        info = loader.inference_info
+        info = loader.inference_info or {}
         url = info.get("checkpoint_url")
         link_text = info.get("custom_checkpoint_path")
         if link_text is None:
             link_text = url
-        link_text = link_text.replace("_", "\_")
+        if link_text is not None:
+            link_text = link_text.replace("_", "\_")
 
         # Note about validation dataset
         classes_str, note_about_val_dataset, train_session = self.get_overview_info()
@@ -65,6 +66,7 @@ class Overview(MetricVis):
                     self._loader.vis_texts.definitions.confidence_score,
                 ],
             ),
+            table_key_metrics=Widget.Table(),
             chart=Widget.Chart(),
         )
 
@@ -117,7 +119,7 @@ class Overview(MetricVis):
         return fig
 
     def get_overview_info(self):
-        classes_cnt = len(self._loader._benchmark.classes_whitelist)
+        classes_cnt = len(self._loader._benchmark.classes_whitelist or [])
         classes_str = "classes" if classes_cnt > 1 else "class"
         classes_str = f"{classes_cnt} {classes_str}"
 
@@ -158,3 +160,30 @@ class Overview(MetricVis):
             images_str += f". Evaluated on the whole project ({val_imgs_cnt} images)"
 
         return classes_str, images_str, train_session
+
+    def get_table(self, widget: Widget.Table) -> dict:
+        res = {}
+
+        columns = ["metrics", "values"]
+        res["content"] = []
+        for metric, value in self._loader.mp.metric_table().items():
+            row = [metric, round(value, 2)]
+            dct = {
+                "row": row,
+                "id": metric,
+                "items": row,
+            }
+            res["content"].append(dct)
+
+        columns_options = [
+            {"customCell": True, "disableSort": True},
+            {"disableSort": True},
+        ]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        widget.main_column = columns[0]
+        widget.show_header_controls = False
+        widget.width = "60%"
+        return res

supervisely/nn/benchmark/visualization/vis_templates.py

@@ -177,7 +177,7 @@ template_gallery_str = """<sly-iw-gallery
 
 
 template_table_str = """
-            <div style="margin-top: 20px; margin-bottom: 30px;">
+            <div style="margin-top: 20px; margin-bottom: 30px; {{ width }}">
                 <sly-iw-table
                     iw-widget-id="{{ widget_id }}"
                     {% if clickable %}
@@ -208,11 +208,17 @@ template_table_str = """
                     slot-scope="{ row, column, cellValue }"
                     >
                     <div
-                        v-if="column === '{{ mainColumn }}'"
+                        v-if="column === ' ' && '{{ mainColumn }}' === 'Batch size'"
                         class="fflex"
                     >
                         <b>Batch size {{ '{{ cellValue }}' }}</b>
                     </div>
+                    <div
+                        v-if="column === '{{ mainColumn }}'"
+                        class="fflex"
+                    >
+                        <b>{{ '{{ cellValue }}' }}</b>
+                    </div>
                     </span>
                 </sly-iw-table>
             </div>

supervisely/nn/benchmark/visualization/vis_widgets.py

@@ -103,6 +103,7 @@ class Widget:
             self.main_column = ""
             self.fixed_columns = 0
             self.show_header_controls = True
+            self.width = None
             super().__init__()
 
     class Gallery(BaseWidget):

supervisely/nn/benchmark/visualization/visualizer.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import json
 import pickle
 from typing import TYPE_CHECKING, Dict, List, Tuple
@@ -155,6 +156,7 @@ class Visualizer:
             cocoDt,
         )
         self.mp.calculate()
+        self._dump_key_metrics()
 
         self.df_score_profile = pd.DataFrame(
             self.mp.confidence_score_profile(), columns=["scores", "precision", "recall", "f1"]
@@ -187,6 +189,8 @@ class Visualizer:
 
         initialized = [mv(self) for mv in ALL_METRICS]
         if self.speedtest is not None:
+            if len(self.speedtest["speedtest"]) < 2:
+                SPEEDTEST_METRICS.pop()
             initialized = initialized + [mv(self) for mv in SPEEDTEST_METRICS]
         initialized = [mv for mv in initialized if self.cv_task.value in mv.cv_tasks]
         with self.pbar(
@@ -351,6 +355,13 @@ class Visualizer:
             json.dump(self._generate_state(metric_visualizations), f)
         logger.info("Saved: %r", "state.json")
 
+    def _dump_key_metrics(self):
+        key_metrics = self.mp.json_metrics()
+        path = os.path.join(self._benchmark.get_base_dir(), "evaluation", "key_metrics.json")
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(key_metrics, f)
+        return path
+
     def update_diff_annotations(self):
         meta = self._update_pred_meta_with_tags(self.dt_project_info.id, self.dt_project_meta)
         self._update_diff_meta(meta)

{supervisely-6.73.206.dist-info → supervisely-6.73.207.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.206
+Version: 6.73.207
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.206.dist-info → supervisely-6.73.207.dist-info}/RECORD

@@ -721,7 +721,7 @@ supervisely/nn/artifacts/unet.py,sha256=Gn8ADfwC4F-MABVDPRY7g_ZaAIaaOAEbhhIGII-o
 supervisely/nn/artifacts/yolov5.py,sha256=6KDCyDlLO7AT9of1qHjCaG5mmxCv6C0p-zCk9KJ0PH4,1478
 supervisely/nn/artifacts/yolov8.py,sha256=c3MzbOTYD6RT5N4F9oZ0SWXxyonjJ6ZQfZLYUHPRZg4,1204
 supervisely/nn/benchmark/__init__.py,sha256=RxqbBx7cbzookq2DRvxYIaRofON9uxHeY5h8DqDbZq0,187
-supervisely/nn/benchmark/base_benchmark.py,sha256=f0TlgPdtl5-hWe38k1q4Jhld48kiDIhhWmGGJoO-FGA,22366
+supervisely/nn/benchmark/base_benchmark.py,sha256=LoDsT_F86Y9xztrTyfz74FmT619_rrZVUKnEtTzav0A,22755
 supervisely/nn/benchmark/cv_tasks.py,sha256=ShoAbuNzfMYj0Se-KOnl_-dJnrmvN6Aukxa0eq28bFw,239
 supervisely/nn/benchmark/instance_segmentation_benchmark.py,sha256=9iiWEH7KDw7ps0mQQdzIrCtCKg4umHekF3ws7jIGjmE,938
 supervisely/nn/benchmark/object_detection_benchmark.py,sha256=s1S-L952etgz-UsDPyg69AgmFfAoJXvFHhITT8zB5iw,956
@@ -730,23 +730,24 @@ supervisely/nn/benchmark/coco_utils/__init__.py,sha256=MKxuzzBWpRCwR8kOb5NXUK8vD
 supervisely/nn/benchmark/coco_utils/sly2coco.py,sha256=iudlcHNynthscH-V5qwCLk6VgIcxYrMEuAfGIjrOjZ0,6867
 supervisely/nn/benchmark/coco_utils/utils.py,sha256=J9kM_Cn4XxfsrSQ8Rx6eb1UsS65-wOybaCkI9rQDeiU,504
 supervisely/nn/benchmark/evaluation/__init__.py,sha256=1NGV_xEGe9lyPdE5gJ8AASKzm2WyZ_jKlh9WVvCQIaY,287
-supervisely/nn/benchmark/evaluation/base_evaluator.py,sha256=Ac1EsvRrMH-Fck1aVS9T2Tx1m9PfialRA3z8XJs5e8U,1039
-supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py,sha256=oskpLBSwo_u224m_fc-oVJD0GGWgsyR9HrBVEEQ-FAE,3101
-supervisely/nn/benchmark/evaluation/object_detection_evaluator.py,sha256=5XrTnNpgdZgJ-LgXdUd74OvZLTQEenltxTSO12bTwqg,2943
+supervisely/nn/benchmark/evaluation/base_evaluator.py,sha256=htei2QGHsx-1DO16j-lUjflmUFpfSIv0drhGfe60qAU,1995
+supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py,sha256=p2VC4WXB2RQlmeRt14u74QTXSOwhkyQPCznw7Kqe32k,3773
+supervisely/nn/benchmark/evaluation/object_detection_evaluator.py,sha256=0Sp0h9kNQUQYVRL3WrV9Vm7OniJPMYriEFbwcCejOBs,3615
 supervisely/nn/benchmark/evaluation/coco/__init__.py,sha256=l6dFxp9aenywosQzQkIaDEI1p-DDQ63OgJJXxSVB4Mk,172
-supervisely/nn/benchmark/evaluation/coco/calculate_metrics.py,sha256=tgUAXngl0QcGpSdGvZRVo6f_0YP_PF4Leu2fpx5a_Us,10702
-supervisely/nn/benchmark/evaluation/coco/metric_provider.py,sha256=j4YMk20t3lsX3QnsSIRjEYx8EayHw77I4KdXxKfgxeI,17513
+supervisely/nn/benchmark/evaluation/coco/calculate_metrics.py,sha256=Pe5_bXJ57343PQ0TuYEkCMNUyp-YTyIXnPXUESuXQBk,11430
+supervisely/nn/benchmark/evaluation/coco/evaluation_params.yaml,sha256=POzpiaxnxuwAPSNQOGgjoUPfsk6Lf5hb9GLHwltWY5Y,94
+supervisely/nn/benchmark/evaluation/coco/metric_provider.py,sha256=sEHY9mob6fjKG4_deMxu3lVGTNXhqbRDoFerBEwH5a0,19907
 supervisely/nn/benchmark/evaluation/coco/metrics.py,sha256=oyictdJ7rRDUkaVvHoxntywW5zZweS8pIJ1bN6JgXtE,2420
 supervisely/nn/benchmark/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/visualization/vis_click_data.py,sha256=4QdBZqJmmPYdcB7x565zOtXhDFRyXIB4tpu0V-_otoc,3724
-supervisely/nn/benchmark/visualization/vis_metric_base.py,sha256=NJBduyKE0UPihPtM2BR2eOdDwkUWZ3lMBpZGy9UFOZ0,13859
-supervisely/nn/benchmark/visualization/vis_templates.py,sha256=tDPQcuByvnDdfGdDaT-KhemnKCtieunp-MgnGAPsbrQ,9905
-supervisely/nn/benchmark/visualization/vis_widgets.py,sha256=CsT7DSfxH4g4zHsmm_7RCJf3YR6zXiADuYhUIIGdn7w,4073
-supervisely/nn/benchmark/visualization/visualizer.py,sha256=BLu31ETO202AgpT1gjiAry-m_hk3ExzyFUsQtOHaeqU,31729
+supervisely/nn/benchmark/visualization/vis_metric_base.py,sha256=hXnbGAijnZ700GuzfvaHBxU5elQR0wXkBUNbmcSWCno,13941
+supervisely/nn/benchmark/visualization/vis_templates.py,sha256=Vy019K8G8oJ9vN35tvsSjYA21xdldqIP-BALGEuy_eM,10169
+supervisely/nn/benchmark/visualization/vis_widgets.py,sha256=oavMM2Z-05Hp_Fj086NgXAqDq2KPAqXfT-nJb5qlDsg,4103
+supervisely/nn/benchmark/visualization/visualizer.py,sha256=tmx1coNpKv9dDmJb-lHhdjuiSq0oRnFC0GvSHwcXV4E,32149
 supervisely/nn/benchmark/visualization/inference_speed/__init__.py,sha256=6Nahwt9R61_Jc1eWupXa70CgyRQ7tbUeiDWR26017rY,554
 supervisely/nn/benchmark/visualization/inference_speed/speedtest_batch.py,sha256=73gbXs1uTfxxWH-UCJdR72m-48jMD5qVyMyolf5jNoc,6140
 supervisely/nn/benchmark/visualization/inference_speed/speedtest_intro.py,sha256=ivUVriKyhx9ZtwVSqrAkUqq1SJGYYxNLwLQR1UgE4aM,900
-supervisely/nn/benchmark/visualization/inference_speed/speedtest_overview.py,sha256=w-2F2lN89C2amHzPeI02WXvtjN6rdAMlRIgPLTR1f-U,4885
+supervisely/nn/benchmark/visualization/inference_speed/speedtest_overview.py,sha256=QLgiiHJzmasnNmn6OGWfLef01gLOiM84uVBK5P8c954,4887
 supervisely/nn/benchmark/visualization/inference_speed/speedtest_real_time.py,sha256=bVpNS3YBP0TGsqE_XQBuFMJI5ybDM0RZpEzFyT7cbkA,2157
 supervisely/nn/benchmark/visualization/text_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 supervisely/nn/benchmark/visualization/text_templates/inference_speed_text.py,sha256=XGeBrbP-ROyKYbqYZzA281_IG45Ygu9NKyqG2I3o5TU,1124
@@ -765,7 +766,7 @@ supervisely/nn/benchmark/visualization/vis_metrics/model_predictions.py,sha256=3
 supervisely/nn/benchmark/visualization/vis_metrics/outcome_counts.py,sha256=rsm_hdE0pYCHY-5v0pjDIid71y2tPbzYbmH2Qw-RS-4,3983
 supervisely/nn/benchmark/visualization/vis_metrics/outcome_counts_per_class.py,sha256=lSb2-jfplyERIUCi8_6P9aq6C77JGOKOJK20J824sEE,5623
 supervisely/nn/benchmark/visualization/vis_metrics/overall_error_analysis.py,sha256=YHfueea2EkUgNGP4FCyKyCaCtCwaYeYNJ3WwfF-Hzi4,3553
-supervisely/nn/benchmark/visualization/vis_metrics/overview.py,sha256=_TwrBnkEfw4zCn2Vr1BrOQf1ixsKYtN5aQY7seHBS8M,6141
+supervisely/nn/benchmark/visualization/vis_metrics/overview.py,sha256=V-uNrtNhR5idywyfFSNOA4zFesTf5d6i2g8MNtLOhIw,6997
 supervisely/nn/benchmark/visualization/vis_metrics/percision_avg_per_class.py,sha256=mm8IVM90EoIC_9GsiM-Jyhh6jPqQcHMo788VAvRAzMY,1877
 supervisely/nn/benchmark/visualization/vis_metrics/pr_curve.py,sha256=4-AwEQk1ywuW4zXO_EXo7_aFMjenwhnLlGX2PWqiu0k,3574
 supervisely/nn/benchmark/visualization/vis_metrics/pr_curve_by_class.py,sha256=9Uuibo38HVGPChPbCW8i3cMYdb6-NFlys1TBisp5zOU,1442
@@ -956,9 +957,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
 supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
 supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
-supervisely-6.73.206.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-supervisely-6.73.206.dist-info/METADATA,sha256=MQ9ZNLjLwIGBKpZiYf2S-F148rr3-bWJgg55ndrpleo,33077
-supervisely-6.73.206.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-supervisely-6.73.206.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
-supervisely-6.73.206.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
-supervisely-6.73.206.dist-info/RECORD,,
+supervisely-6.73.207.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+supervisely-6.73.207.dist-info/METADATA,sha256=jPDiqTxPbkSOwBCqz4V6xiMiBRiQ-n5fB6ZxFAaBYVw,33077
+supervisely-6.73.207.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+supervisely-6.73.207.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
+supervisely-6.73.207.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
+supervisely-6.73.207.dist-info/RECORD,,