supervisely 6.73.206__tar.gz → 6.73.208__tar.gz

{supervisely-6.73.206/supervisely.egg-info → supervisely-6.73.208}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.206
+Version: 6.73.208
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.206 → supervisely-6.73.208}/setup.py

@@ -198,6 +198,7 @@ setup(
             "imaging/colors.json.gz",
             "nn/tracker/bot_sort/configs/MOT17/*.yml",
             "nn/tracker/bot_sort/configs/MOT20/*.yml",
+            "nn/benchmark/evaluation/coco/evaluation_params.yaml",
         ],
     },
     entry_points={

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/base_benchmark.py

@@ -1,4 +1,5 @@
 import os
+import yaml
 from typing import Callable, List, Optional, Tuple, Union
 
 import numpy as np
@@ -31,6 +32,7 @@ class BaseBenchmark:
         progress: Optional[SlyTqdm] = None,
         progress_secondary: Optional[SlyTqdm] = None,
         classes_whitelist: Optional[List[str]] = None,
+        evaluation_params: Optional[dict] = None,
     ):
         self.api = api
         self.session: SessionJSON = None
@@ -51,6 +53,8 @@ class BaseBenchmark:
         self.vis_texts = None
         self.inference_speed_text = None
         self.train_info = None
+        self.evaluation_params = evaluation_params
+        self._validate_evaluation_params()
 
     def _get_evaluator_class(self) -> type:
         raise NotImplementedError()
@@ -152,6 +156,7 @@ class BaseBenchmark:
             progress=self.pbar,
             items_count=self.dt_project_info.items_count,
             classes_whitelist=self.classes_whitelist,
+            evaluation_params=self.evaluation_params,
         )
         self.evaluator.evaluate()
 
@@ -552,3 +557,7 @@ class BaseBenchmark:
                 chagned = True
         if chagned:
             self.api.project.update_meta(pred_project_id, pred_meta.to_json())
+
+    def _validate_evaluation_params(self):
+        if self.evaluation_params:
+            self._get_evaluator_class().validate_evaluation_params(self.evaluation_params)

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/evaluation/base_evaluator.py

@@ -1,12 +1,15 @@
 import os
 import pickle
-from typing import List, Optional
+from typing import List, Optional, Union
+
+import yaml
 
 from supervisely.app.widgets import SlyTqdm
 from supervisely.task.progress import tqdm_sly
 
 
 class BaseEvaluator:
+    EVALUATION_PARAMS_YAML_PATH: str = None
 
     def __init__(
         self,
@@ -16,6 +19,7 @@ class BaseEvaluator:
         progress: Optional[SlyTqdm] = None,
         items_count: Optional[int] = None,  # TODO: is it needed?
         classes_whitelist: Optional[List[str]] = None,
+        evaluation_params: Optional[dict] = None,
     ):
         self.gt_project_path = gt_project_path
         self.dt_project_path = dt_project_path
@@ -25,12 +29,36 @@ class BaseEvaluator:
         os.makedirs(result_dir, exist_ok=True)
         self.classes_whitelist = classes_whitelist
 
+        if evaluation_params is None:
+            evaluation_params = self._get_default_evaluation_params()
+        self.evaluation_params = evaluation_params
+        if self.evaluation_params:
+            self.validate_evaluation_params(self.evaluation_params)
+
     def evaluate(self):
         raise NotImplementedError()
 
     def get_result_dir(self) -> str:
         return self.result_dir
 
+    @classmethod
+    def load_yaml_evaluation_params(cls) -> Union[str, None]:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return None
+        with open(cls.EVALUATION_PARAMS_YAML_PATH, "r") as f:
+            return f.read()
+
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        pass
+
+    @classmethod
+    def _get_default_evaluation_params(cls) -> dict:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return {}
+        else:
+            return yaml.safe_load(cls.load_yaml_evaluation_params())
+
     def _dump_pickle(self, data, file_path):
         with open(file_path, "wb") as f:
             pickle.dump(data, f)

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/evaluation/coco/calculate_metrics.py

@@ -1,10 +1,39 @@
 from collections import defaultdict
-from typing import Callable, Optional, Literal
+from typing import Callable, List, Literal, Optional
 
 import numpy as np
 
-
-def calculate_metrics(cocoGt, cocoDt, iouType: Literal["bbox", "segm"], progress_cb: Optional[Callable] = None):
+# pylint: disable=import-error
+from pycocotools.cocoeval import COCOeval
+
+
+def set_cocoeval_params(
+    cocoeval: COCOeval,
+    parameters: dict,
+):
+    if parameters is None:
+        return
+    param_names = (
+        "iouThrs",
+        "recThrs",
+        "maxDets",
+        "areaRng",
+        "areaRngLbl",
+        # "kpt_oks_sigmas" # For keypoints
+    )
+    for param_name in param_names:
+        cocoeval.params.__setattr__(
+            param_name, parameters.get(param_name, cocoeval.params.__getattribute__(param_name))
+        )
+
+
+def calculate_metrics(
+    cocoGt,
+    cocoDt,
+    iouType: Literal["bbox", "segm"],
+    progress_cb: Optional[Callable] = None,
+    evaluation_params: Optional[dict] = None,
+):
     """
     Calculate COCO metrics.
 
@@ -19,35 +48,42 @@ def calculate_metrics(cocoGt, cocoDt, iouType: Literal["bbox", "segm"], progress
     :return: Results of the evaluation
     :rtype: dict
     """
-    from pycocotools.cocoeval import COCOeval  # pylint: disable=import-error
 
-    progress_cb(1) if progress_cb is not None else None
     cocoEval = COCOeval(cocoGt, cocoDt, iouType=iouType)
-    progress_cb(1) if progress_cb is not None else None
     cocoEval.evaluate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval.accumulate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval.summarize()
-    progress_cb(1) if progress_cb is not None else None
 
     # For classification metrics
     cocoEval_cls = COCOeval(cocoGt, cocoDt, iouType=iouType)
-    progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.params.useCats = 0
     cocoEval_cls.evaluate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.accumulate()
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.summarize()
-    progress_cb(1) if progress_cb is not None else None
+
+    iou_t = 0
+    is_custom_iou_threshold = (
+        evaluation_params is not None and evaluation_params.get("iou_threshold") and evaluation_params["iou_threshold"] != 0.5
+    )
+    if is_custom_iou_threshold:
+        iou_t = np.where(cocoEval.params.iouThrs == evaluation_params["iou_threshold"])[0][0]
 
     eval_img_dict = get_eval_img_dict(cocoEval)
     eval_img_dict_cls = get_eval_img_dict(cocoEval_cls)
-    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=0)
+    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=iou_t)
 
-    params = {"iouThrs": cocoEval.params.iouThrs, "recThrs": cocoEval.params.recThrs}
+    params = {
+        "iouThrs": cocoEval.params.iouThrs,
+        "recThrs": cocoEval.params.recThrs,
+        "evaluation_params": evaluation_params or {},
+    }
     coco_metrics = {"mAP": cocoEval.stats[0], "precision": cocoEval.eval["precision"]}
+    coco_metrics["AP50"] = cocoEval.stats[1]
+    coco_metrics["AP75"] = cocoEval.stats[2]
     eval_data = {
         "matches": matches,
         "coco_metrics": coco_metrics,

supervisely-6.73.208/supervisely/nn/benchmark/evaluation/coco/evaluation_params.yaml

@@ -0,0 +1,2 @@
+# Intersection over Union threshold that will be used for objects mathcing
+iou_threshold: 0.5

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/evaluation/coco/metric_provider.py

@@ -89,6 +89,8 @@ class MetricProvider:
         self.coco_precision = coco_metrics["precision"]
         self.iouThrs = params["iouThrs"]
         self.recThrs = params["recThrs"]
+        self.iou_threshold = params["evaluation_params"]["iou_threshold"]
+        self.iou_threshold_idx = np.searchsorted(self.iouThrs, self.iou_threshold)
 
     def calculate(self):
         self.m_full = _MetricProvider(
@@ -134,7 +136,64 @@ class MetricProvider:
         self._scores_tp_and_fp = self.m_full.scores_tp_and_fp()
         self._maximum_calibration_error = self.m_full.maximum_calibration_error()
         self._expected_calibration_error = self.m_full.expected_calibration_error()
+    
+    def json_metrics(self):
+        base = self.base_metrics()
+        iou_name = int(self.iou_threshold * 100)
+        ap_by_class = self.AP_per_class().tolist()
+        ap_by_class = dict(zip(self.cat_names, ap_by_class))
+        ap_custom_by_class = self.AP_custom_per_class().tolist()
+        ap_custom_by_class = dict(zip(self.cat_names, ap_custom_by_class))
+        return {
+            "mAP": base["mAP"],
+            "AP50": self.coco_metrics["AP50"],
+            "AP75": self.coco_metrics["AP75"],
+            f"AP{iou_name}": self.AP_custom(),
+            "f1": base["f1"],
+            "precision": base["precision"],
+            "recall": base["recall"],
+            "iou": base["iou"],
+            "classification_accuracy": base["classification_accuracy"],
+            "calibration_score": base["calibration_score"],
+            "f1_optimal_conf": self.f1_optimal_conf,
+            "expected_calibration_error": self.expected_calibration_error(),
+            "maximum_calibration_error": self.maximum_calibration_error(),
+            "AP_by_class": ap_by_class,
+            f"AP{iou_name}_by_class": ap_custom_by_class,
+        }
+    
+    def metric_table(self):
+        table = self.json_metrics()
+        iou_name = int(self.iou_threshold * 100)
+        return {
+            "mAP": table["mAP"],
+            "AP50": table["AP50"],
+            "AP75": table["AP75"],
+            f"AP{iou_name}": table[f"AP{iou_name}"],
+            "f1": table["f1"],
+            "precision": table["precision"],
+            "recall": table["recall"],
+            "Avg. IoU": table["iou"],
+            "Classification Acc.": table["classification_accuracy"],
+            "Calibration Score": table["calibration_score"],
+            "optimal confidence threshold": table["f1_optimal_conf"],
+        }
 
+    def AP_per_class(self):
+        s = self.coco_precision[:, :, :, 0, 2]
+        s[s == -1] = np.nan
+        ap = np.nanmean(s, axis=(0, 1))
+        return ap
+
+    def AP_custom_per_class(self):
+        s = self.coco_precision[self.iou_threshold_idx, :, :, 0, 2]
+        s[s == -1] = np.nan
+        ap = np.nanmean(s, axis=0)
+        return ap
+    
+    def AP_custom(self):
+        return np.nanmean(self.AP_custom_per_class())
+    
     def base_metrics(self):
         base = self._base_metrics
         calibration_score = 1 - self._expected_calibration_error

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py

@@ -4,9 +4,12 @@ from supervisely.io.json import dump_json_file
 from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
 from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
+from pathlib import Path
 
 
 class InstanceSegmentationEvaluator(BaseEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/coco/evaluation_params.yaml"
+
     def evaluate(self):
         try:
             self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
@@ -19,12 +22,25 @@ class InstanceSegmentationEvaluator(BaseEvaluator):
 
         self._dump_datasets()
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
-        with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
+        with self.pbar(message="Evaluation: Calculating metrics", total=5) as p:
             self.eval_data = calculate_metrics(
-                self.cocoGt, self.cocoDt, iouType="segm", progress_cb=p.update
+                self.cocoGt,
+                self.cocoDt,
+                iouType="segm",
+                progress_cb=p.update,
+                evaluation_params=self.evaluation_params,
             )
         self._dump_eval_results()
 
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        iou_threshold = evaluation_params.get("iou_threshold")
+        if iou_threshold is not None:
+            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
+                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
+                f"but got {iou_threshold}"
+            )
+
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(
             self.gt_project_path,

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/evaluation/object_detection_evaluator.py

@@ -4,9 +4,12 @@ from supervisely.io.json import dump_json_file
 from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
 from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
+from pathlib import Path
 
 
 class ObjectDetectionEvaluator(BaseEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/coco/evaluation_params.yaml"
+
     def evaluate(self):
         try:
             self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
@@ -17,12 +20,25 @@ class ObjectDetectionEvaluator(BaseEvaluator):
                 "try to use newer version of NN app."
             )
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
-        with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
+        with self.pbar(message="Evaluation: Calculating metrics", total=5) as p:
             self.eval_data = calculate_metrics(
-                self.cocoGt, self.cocoDt, iouType="bbox", progress_cb=p.update
+                self.cocoGt,
+                self.cocoDt,
+                iouType="bbox",
+                progress_cb=p.update,
+                evaluation_params=self.evaluation_params,
             )
         self._dump_eval_results()
 
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        iou_threshold = evaluation_params.get("iou_threshold")
+        if iou_threshold is not None:
+            assert iou_threshold in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], (
+                f"iou_threshold must be one of [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95], "
+                f"but got {iou_threshold}"
+            )
+
     def _convert_to_coco(self):
         cocoGt_json = sly2coco(
             self.gt_project_path,

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/inference_speed/speedtest_overview.py

@@ -62,7 +62,7 @@ class SpeedtestOverview(MetricVis):
         res["columns"] = columns
         res["columnsOptions"] = columns_options
 
-        widget.main_column = columns[0]
+        widget.main_column = "Batch size"
         widget.fixed_columns = 1
         widget.show_header_controls = False
         return res

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/vis_metric_base.py

@@ -216,6 +216,7 @@ class MetricVis:
                         "mainColumn": widget.main_column,
                         "fixColumns": widget.fixed_columns,
                         "showHeaderControls": widget.show_header_controls,
+                        "width": f"width:{widget.width}" if widget.width else "",
                     }
                 )
 

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/vis_metrics/model_predictions.py

@@ -71,14 +71,14 @@ class ModelPredictions(MetricVis):
 
         self._row_ids = []
 
-        for row in tbl["data"]["data"]:
-            sly_id = row["items"].pop(0)
+        for row in tbl["data"]:
+            sly_id = row.pop(0)
             info = self._loader.comparison_data[sly_id].gt_image_info
 
             dct = {
                 "row": {key_mapping[k]: v for k, v in info._asdict().items()},
                 "id": info.name,
-                "items": row["items"],
+                "items": row,
             }
 
             self._row_ids.append(dct["id"])

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/vis_metrics/overview.py

@@ -15,12 +15,13 @@ class Overview(MetricVis):
     def __init__(self, loader: Visualizer) -> None:
         super().__init__(loader)
         self._is_overview = True
-        info = loader.inference_info
+        info = loader.inference_info or {}
         url = info.get("checkpoint_url")
         link_text = info.get("custom_checkpoint_path")
         if link_text is None:
             link_text = url
-        link_text = link_text.replace("_", "\_")
+        if link_text is not None:
+            link_text = link_text.replace("_", "\_")
 
         # Note about validation dataset
         classes_str, note_about_val_dataset, train_session = self.get_overview_info()
@@ -65,6 +66,7 @@ class Overview(MetricVis):
                     self._loader.vis_texts.definitions.confidence_score,
                 ],
             ),
+            table_key_metrics=Widget.Table(),
             chart=Widget.Chart(),
         )
 
@@ -117,7 +119,7 @@ class Overview(MetricVis):
         return fig
 
     def get_overview_info(self):
-        classes_cnt = len(self._loader._benchmark.classes_whitelist)
+        classes_cnt = len(self._loader._benchmark.classes_whitelist or [])
         classes_str = "classes" if classes_cnt > 1 else "class"
         classes_str = f"{classes_cnt} {classes_str}"
 
@@ -158,3 +160,30 @@ class Overview(MetricVis):
             images_str += f". Evaluated on the whole project ({val_imgs_cnt} images)"
 
         return classes_str, images_str, train_session
+
+    def get_table(self, widget: Widget.Table) -> dict:
+        res = {}
+
+        columns = ["metrics", "values"]
+        res["content"] = []
+        for metric, value in self._loader.mp.metric_table().items():
+            row = [metric, round(value, 2)]
+            dct = {
+                "row": row,
+                "id": metric,
+                "items": row,
+            }
+            res["content"].append(dct)
+
+        columns_options = [
+            {"customCell": True, "disableSort": True},
+            {"disableSort": True},
+        ]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        widget.main_column = columns[0]
+        widget.show_header_controls = False
+        widget.width = "60%"
+        return res

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/vis_templates.py

@@ -177,7 +177,7 @@ template_gallery_str = """<sly-iw-gallery
 
 
 template_table_str = """
-            <div style="margin-top: 20px; margin-bottom: 30px;">
+            <div style="margin-top: 20px; margin-bottom: 30px; {{ width }}">
                 <sly-iw-table
                     iw-widget-id="{{ widget_id }}"
                     {% if clickable %}
@@ -208,11 +208,17 @@ template_table_str = """
                     slot-scope="{ row, column, cellValue }"
                     >
                     <div
-                        v-if="column === '{{ mainColumn }}'"
+                        v-if="column === ' ' && '{{ mainColumn }}' === 'Batch size'"
                         class="fflex"
                     >
                         <b>Batch size {{ '{{ cellValue }}' }}</b>
                     </div>
+                    <div
+                        v-if="column === '{{ mainColumn }}'"
+                        class="fflex"
+                    >
+                        <b>{{ '{{ cellValue }}' }}</b>
+                    </div>
                     </span>
                 </sly-iw-table>
             </div>

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/vis_widgets.py

@@ -103,6 +103,7 @@ class Widget:
             self.main_column = ""
             self.fixed_columns = 0
             self.show_header_controls = True
+            self.width = None
             super().__init__()
 
     class Gallery(BaseWidget):

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely/nn/benchmark/visualization/visualizer.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import json
 import pickle
 from typing import TYPE_CHECKING, Dict, List, Tuple
@@ -155,6 +156,7 @@ class Visualizer:
             cocoDt,
         )
         self.mp.calculate()
+        self._dump_key_metrics()
 
         self.df_score_profile = pd.DataFrame(
             self.mp.confidence_score_profile(), columns=["scores", "precision", "recall", "f1"]
@@ -187,6 +189,8 @@ class Visualizer:
 
         initialized = [mv(self) for mv in ALL_METRICS]
         if self.speedtest is not None:
+            if len(self.speedtest["speedtest"]) < 2:
+                SPEEDTEST_METRICS.pop()
             initialized = initialized + [mv(self) for mv in SPEEDTEST_METRICS]
         initialized = [mv for mv in initialized if self.cv_task.value in mv.cv_tasks]
         with self.pbar(
@@ -351,6 +355,13 @@ class Visualizer:
             json.dump(self._generate_state(metric_visualizations), f)
         logger.info("Saved: %r", "state.json")
 
+    def _dump_key_metrics(self):
+        key_metrics = self.mp.json_metrics()
+        path = os.path.join(self._benchmark.get_base_dir(), "evaluation", "key_metrics.json")
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(key_metrics, f)
+        return path
+
     def update_diff_annotations(self):
         meta = self._update_pred_meta_with_tags(self.dt_project_info.id, self.dt_project_meta)
         self._update_diff_meta(meta)

{supervisely-6.73.206 → supervisely-6.73.208/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.206
+Version: 6.73.208
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.206 → supervisely-6.73.208}/supervisely.egg-info/SOURCES.txt

@@ -745,6 +745,7 @@ supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py
 supervisely/nn/benchmark/evaluation/object_detection_evaluator.py
 supervisely/nn/benchmark/evaluation/coco/__init__.py
 supervisely/nn/benchmark/evaluation/coco/calculate_metrics.py
+supervisely/nn/benchmark/evaluation/coco/evaluation_params.yaml
 supervisely/nn/benchmark/evaluation/coco/metric_provider.py
 supervisely/nn/benchmark/evaluation/coco/metrics.py
 supervisely/nn/benchmark/visualization/__init__.py