supervisely 6.73.250__py3-none-any.whl → 6.73.251__py3-none-any.whl

supervisely/app/widgets/experiment_selector/experiment_selector.py

@@ -1,6 +1,7 @@
 import os
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import asdict
 from typing import Any, Callable, Dict, List, Union
 
 from supervisely import env, logger
@@ -91,7 +92,7 @@ class ExperimentSelector(Widget):
             self._session_link = self._generate_session_link()
 
             # col 6 benchmark report
-            self._benchmark_report = None  # experiment_infos.benchmark_report_path
+            self._benchmark_report_id = experiment_info.evaluation_report_id
 
             # widgets
             self._task_widget = self._create_task_widget()
@@ -191,7 +192,7 @@ class ExperimentSelector(Widget):
             task_widget = Container(
                 [
                     Text(
-                        f"<i class='zmdi zmdi-folder' style='color: #7f858e'></i> <a href='{self._task_link}'>{self._task_id}</a>",
+                        f"<i class='zmdi zmdi-folder' style='color: #7f858e'></i> <a href='{self._task_link}' target='_blank'>{self._task_id}</a>",
                         "text",
                     ),
                     Text(
@@ -237,22 +238,29 @@ class ExperimentSelector(Widget):
 
         def _create_session_widget(self) -> Text:
             session_link_widget = Text(
-                f"<a href='{self._session_link}'>Preview</a> <i class='zmdi zmdi-open-in-new'></i>",
+                f"<a href='{self._session_link}' target='_blank'>Preview</a> <i class='zmdi zmdi-open-in-new'></i>",
                 "text",
             )
             return session_link_widget
 
         def _create_benchmark_widget(self) -> Text:
-            if self._benchmark_report is None:
-                self._benchmark_report = "No benchmark report available"
+            if self._benchmark_report_id is None:
+                self._benchmark_report_id = "No evaluation report available"
                 benchmark_widget = Text(
-                    "<span class='field-description text-muted' style='color: #7f858e'>No benchmark report available</span>",
+                    "<span class='field-description text-muted' style='color: #7f858e'>No evaluation report available</span>",
                     "text",
                     font_size=13,
                 )
             else:
+                if is_development():
+                    benchmark_report_link = abs_url(
+                        f"/model-benchmark?id={self._benchmark_report_id}"
+                    )
+                else:
+                    benchmark_report_link = f"/model-benchmark?id={self._benchmark_report_id}"
+
                 benchmark_widget = Text(
-                    f"<a href='{self._benchmark_report}'>Benchmark Report</a> <i class='zmdi zmdi-chart'></i>",
+                    f"<i class='zmdi zmdi-chart' style='color: #7f858e'></i> <a href='{benchmark_report_link}' target='_blank'>evaluation report</a>",
                     "text",
                 )
             return benchmark_widget
@@ -432,7 +440,7 @@ class ExperimentSelector(Widget):
         if len(self._rows) == 0:
             return
         selected_row = self.get_selected_row()
-        selected_row_json = selected_row._experiment_info._asdict()
+        selected_row_json = asdict(selected_row._experiment_info)
         return selected_row_json
 
     def get_selected_checkpoint_path(self) -> str:

supervisely/nn/benchmark/base_benchmark.py

@@ -74,6 +74,8 @@ class BaseBenchmark:
         self.train_info = None
         self.evaluator_app_info = None
         self.evaluation_params = evaluation_params
+        self._eval_results = None
+        self.report_id = None
         self._validate_evaluation_params()
 
     def _get_evaluator_class(self) -> type:
@@ -87,6 +89,11 @@ class BaseBenchmark:
     def hardware(self) -> str:
         return self._hardware
 
+    @property
+    def key_metrics(self):
+        eval_results = self.get_eval_result()
+        return eval_results.key_metrics
+
     def run_evaluation(
         self,
         model_session: Union[int, str, SessionJSON],
@@ -482,8 +489,10 @@ class BaseBenchmark:
                 f"Visualizer class is not defined in {self.__class__.__name__}. "
                 "It should be defined in the subclass of BaseBenchmark (e.g. ObjectDetectionBenchmark)."
             )
-        eval_result = self.evaluator.get_eval_result()
-        vis = self.visualizer_cls(self.api, [eval_result], self.get_layout_results_dir(), self.pbar)  # pylint: disable=not-callable
+        eval_result = self.get_eval_result()
+        vis = self.visualizer_cls(  # pylint: disable=not-callable
+            self.api, [eval_result], self.get_layout_results_dir(), self.pbar
+        )
         with self.pbar(message="Visualizations: Rendering layout", total=1) as p:
             vis.visualize()
             p.update(1)
@@ -566,6 +575,7 @@ class BaseBenchmark:
     def upload_report_link(self, remote_dir: str):
         template_path = os.path.join(remote_dir, "template.vue")
         vue_template_info = self.api.file.get_info_by_path(self.team_id, template_path)
+        self.report_id = vue_template_info.id
 
         report_link = "/model-benchmark?id=" + str(vue_template_info.id)
         local_path = os.path.join(self.get_layout_results_dir(), "open.lnk")
@@ -608,3 +618,8 @@ class BaseBenchmark:
             return sum(ds.items_count for ds in self.gt_dataset_infos)
         else:
             return self.gt_project_info.items_count
+
+    def get_eval_result(self):
+        if self._eval_results is None:
+            self._eval_results = self.evaluator.get_eval_result()
+        return self._eval_results

supervisely/nn/benchmark/base_evaluator.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import pickle
 from typing import Dict, List, Optional, Union
@@ -9,23 +11,34 @@ from supervisely.task.progress import tqdm_sly
 
 
 class BaseEvalResult:
-    def __init__(self, directory: str):
+    def __init__(self, directory: Optional[str] = None):
         self.directory = directory
         self.inference_info: Dict = None
         self.speedtest_info: Dict = None
         self.eval_data: Dict = None
         self.mp = None
 
-        self._read_eval_data()
+        if self.directory is not None:
+            self._read_files(self.directory)
+            self._prepare_data()
+
+    @classmethod
+    def from_evaluator(cls, evaulator: BaseEvaluator) -> BaseEvalResult:
+        """Method to customize loading of the evaluation result."""
+        raise NotImplementedError()
 
     @property
     def cv_task(self):
         return self.inference_info.get("task_type")
 
     @property
-    def name(self) -> str:
-        model_name = self.inference_info.get("model_name", self.directory)
-        return self.inference_info.get("deploy_params", {}).get("checkpoint_name", model_name)
+    def name(self) -> Union[str, None]:
+        deploy_params = self.inference_info.get("deploy_params", {})
+        return (
+            deploy_params.get("checkpoint_name")
+            or deploy_params.get("model_name")
+            or self.inference_info.get("model_name")
+        )
 
     @property
     def gt_project_id(self) -> int:
@@ -59,7 +72,16 @@ class BaseEvalResult:
     def classes_whitelist(self):
         return self.inference_info.get("inference_settings", {}).get("classes", [])  # TODO: check
 
-    def _read_eval_data(self):
+    def _read_files(self, path: str) -> None:
+        """Read all necessary files from the directory"""
+        raise NotImplementedError()
+
+    def _prepare_data(self) -> None:
+        """Prepare data to allow easy access to the data"""
+        raise NotImplementedError()
+    
+    @property
+    def key_metrics(self):
         raise NotImplementedError()
 
 

supervisely/nn/benchmark/instance_segmentation/benchmark.py

@@ -29,5 +29,5 @@ class InstanceSegmentationBenchmark(ObjectDetectionBenchmark):
     ):
         assert try_set_conf_auto(
             self.session, CONF_THRES
-        ), f"Unable to set the confidence threshold to {CONF_THRES} for evalation."
+        ), f"Unable to set the confidence threshold to {CONF_THRES} for evaluation."
         return super()._run_inference(output_project_id, batch_size, cache_project_on_agent)

supervisely/nn/benchmark/instance_segmentation/evaluator.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 from pathlib import Path
 
@@ -13,6 +15,18 @@ from supervisely.nn.benchmark.utils import calculate_metrics, read_coco_datasets
 class InstanceSegmentationEvalResult(ObjectDetectionEvalResult):
     mp_cls = MetricProvider
 
+    @classmethod
+    def from_evaluator(
+        cls, evaulator: InstanceSegmentationEvaluator
+    ) -> InstanceSegmentationEvalResult:
+        """Method to customize loading of the evaluation result."""
+        eval_result = cls()
+        eval_result.eval_data = evaulator.eval_data
+        eval_result.coco_gt = evaulator.cocoGt
+        eval_result.coco_dt = evaulator.cocoDt
+        eval_result._prepare_data()
+        return eval_result
+
 
 class InstanceSegmentationEvaluator(ObjectDetectionEvaluator):
     EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"

supervisely/nn/benchmark/object_detection/benchmark.py

@@ -27,5 +27,5 @@ class ObjectDetectionBenchmark(BaseBenchmark):
     ):
         assert try_set_conf_auto(
             self.session, CONF_THRES
-        ), f"Unable to set the confidence threshold to {CONF_THRES} for evalation."
+        ), f"Unable to set the confidence threshold to {CONF_THRES} for evaluation."
         return super()._run_inference(output_project_id, batch_size, cache_project_on_agent)

supervisely/nn/benchmark/object_detection/evaluator.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import pickle
 from pathlib import Path
@@ -18,25 +20,39 @@ from supervisely.nn.benchmark.visualization.vis_click_data import ClickData, IdM
 class ObjectDetectionEvalResult(BaseEvalResult):
     mp_cls = MetricProvider
 
-    def _read_eval_data(self):
-        from pycocotools.coco import COCO  # pylint: disable=import-error
+    def _read_files(self, path: str) -> None:
+        """Read all necessary files from the directory"""
+
+        self.coco_gt = Path(path) / "cocoGt.json"
+        self.coco_dt = Path(path) / "cocoDt.json"
 
-        gt_path = str(Path(self.directory) / "cocoGt.json")
-        dt_path = str(Path(self.directory) / "cocoDt.json")
-        coco_gt, coco_dt = COCO(gt_path), COCO(dt_path)
-        self.coco_gt = coco_gt
-        self.coco_dt = coco_dt
-        self.eval_data = None
-        with open(Path(self.directory, "eval_data.pkl"), "rb") as f:
-            self.eval_data = pickle.load(f)
+        if self.coco_gt.exists() and self.coco_dt.exists():
+            self.coco_gt, self.coco_dt = read_coco_datasets(self.coco_gt, self.coco_dt)
 
-        inference_info_path = Path(self.directory) / "inference_info.json"
-        self.inference_info = load_json_file(str(inference_info_path))
+        eval_data_path = Path(path) / "eval_data.pkl"
+        if eval_data_path.exists():
+            with open(Path(path, "eval_data.pkl"), "rb") as f:
+                self.eval_data = pickle.load(f)
 
-        speedtest_info_path = Path(self.directory).parent / "speedtest" / "speedtest.json"
+        inference_info_path = Path(path) / "inference_info.json"
+        if inference_info_path.exists():
+            self.inference_info = load_json_file(str(inference_info_path))
+
+        speedtest_info_path = Path(path).parent / "speedtest" / "speedtest.json"
         if speedtest_info_path.exists():
             self.speedtest_info = load_json_file(str(speedtest_info_path))
 
+    def _prepare_data(self) -> None:
+        """Prepare data to allow easy access to the most important parts"""
+
+        from pycocotools.coco import COCO  # pylint: disable=import-error
+
+        if not hasattr(self, "coco_gt") or not hasattr(self, "coco_dt"):
+            raise ValueError("GT and DT datasets are not provided")
+
+        if not isinstance(self.coco_gt, COCO) and not isinstance(self.coco_dt, COCO):
+            self.coco_gt, self.coco_dt = read_coco_datasets(self.coco_gt, self.coco_dt)
+
         self.mp = MetricProvider(
             self.eval_data["matches"],
             self.eval_data["coco_metrics"],
@@ -62,6 +78,20 @@ class ObjectDetectionEvalResult(BaseEvalResult):
 
         self.click_data = ClickData(self.mp.m, gt_id_mapper, dt_id_mapper)
 
+    @classmethod
+    def from_evaluator(cls, evaulator: ObjectDetectionEvaluator) -> ObjectDetectionEvalResult:
+        """Method to customize loading of the evaluation result."""
+        eval_result = cls()
+        eval_result.eval_data = evaulator.eval_data
+        eval_result.coco_gt = evaulator.cocoGt
+        eval_result.coco_dt = evaulator.cocoDt
+        eval_result._prepare_data()
+        return eval_result
+
+    @property
+    def key_metrics(self):
+        return self.mp.key_metrics()
+
 
 class ObjectDetectionEvaluator(BaseEvaluator):
     EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"

supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -164,6 +164,13 @@ class MetricProvider:
             f"AP{iou_name}_by_class": ap_custom_by_class,
         }
 
+    def key_metrics(self):
+        iou_name = int(self.iou_threshold * 100)
+        json_metrics = self.json_metrics()
+        json_metrics.pop("AP_by_class")
+        json_metrics.pop(f"AP{iou_name}_by_class")
+        return json_metrics
+
     def metric_table(self):
         table = self.json_metrics()
         iou_name = int(self.iou_threshold * 100)

supervisely/nn/benchmark/semantic_segmentation/evaluator.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import pickle
 import shutil
@@ -24,16 +26,40 @@ from supervisely.sly_logger import logger
 class SemanticSegmentationEvalResult(BaseEvalResult):
     mp_cls = MetricProvider
 
-    def _read_eval_data(self):
-        self.eval_data = pickle.load(open(Path(self.directory, "eval_data.pkl"), "rb"))
-        self.inference_info = load_json_file(Path(self.directory, "inference_info.json"))
-        speedtest_info_path = Path(self.directory).parent / "speedtest" / "speedtest.json"
-        self.speedtest_info = None
+    def _read_files(self, path: str) -> None:
+        """Read all necessary files from the directory"""
+
+        eval_data_path = Path(path) / "eval_data.pkl"
+        if eval_data_path.exists():
+            with open(Path(path, "eval_data.pkl"), "rb") as f:
+                self.eval_data = pickle.load(f)
+
+        inference_info_path = Path(path) / "inference_info.json"
+        if inference_info_path.exists():
+            self.inference_info = load_json_file(str(inference_info_path))
+
+        speedtest_info_path = Path(path).parent / "speedtest" / "speedtest.json"
         if speedtest_info_path.exists():
-            self.speedtest_info = load_json_file(speedtest_info_path)
+            self.speedtest_info = load_json_file(str(speedtest_info_path))
+
+    def _prepare_data(self) -> None:
+        """Prepare data to allow easy access to the most important parts"""
 
         self.mp = MetricProvider(self.eval_data)
-        # self.mp.calculate()
+
+    @classmethod
+    def from_evaluator(
+        cls, evaulator: SemanticSegmentationEvaluator
+    ) -> SemanticSegmentationEvalResult:
+        """Method to customize loading of the evaluation result."""
+        eval_result = cls()
+        eval_result.eval_data = evaulator.eval_data
+        eval_result._prepare_data()
+        return eval_result
+
+    @property
+    def key_metrics(self):
+        return self.mp.json_metrics()
 
 
 class SemanticSegmentationEvaluator(BaseEvaluator):

supervisely/nn/benchmark/utils/detection/utlis.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from supervisely.io.json import load_json_file
 from supervisely.nn.inference import SessionJSON
 
@@ -15,10 +17,10 @@ def try_set_conf_auto(session: SessionJSON, conf: float):
 def read_coco_datasets(cocoGt_json, cocoDt_json):
     from pycocotools.coco import COCO  # pylint: disable=import-error
 
-    if isinstance(cocoGt_json, str):
-        cocoGt_json = load_json_file(cocoGt_json)
-    if isinstance(cocoDt_json, str):
-        cocoDt_json = load_json_file(cocoDt_json)
+    if isinstance(cocoGt_json, (str, Path)):
+        cocoGt_json = load_json_file(str(cocoGt_json))
+    if isinstance(cocoDt_json, (str, Path)):
+        cocoDt_json = load_json_file(str(cocoDt_json))
     cocoGt = COCO()
     cocoGt.dataset = cocoGt_json
     cocoGt.createIndex()

supervisely/nn/experiments.py

@@ -1,15 +1,18 @@
 from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, fields
 from json import JSONDecodeError
 from os.path import dirname, join
-from typing import Any, Dict, List, NamedTuple
+from typing import Any, Dict, List
 
 import requests
 
 from supervisely import logger
 from supervisely.api.api import Api, ApiField
+from supervisely.api.file_api import FileInfo
 
 
-class ExperimentInfo(NamedTuple):
+@dataclass
+class ExperimentInfo:
     experiment_name: str
     """Name of the experiment. Defined by the user in the training app"""
     framework_name: str
@@ -22,12 +25,14 @@ class ExperimentInfo(NamedTuple):
     """Project ID in Supervisely"""
     task_id: int
     """Task ID in Supervisely"""
-    model_files: Dict[str, str]
+    model_files: dict
     """Dictionary with paths to model files that needs to be downloaded for training"""
     checkpoints: List[str]
     """List of relative paths to checkpoints"""
     best_checkpoint: str
     """Name of the best checkpoint. Defined by the user in the training app"""
+    export: dict
+    """Dictionary with exported weights in different formats"""
     app_state: str
     """Path to file with settings that were used in the app"""
     model_meta: str
@@ -42,7 +47,7 @@ class ExperimentInfo(NamedTuple):
     """Date and time when the experiment was started"""
     evaluation_report_id: int
     """ID of the evaluation report"""
-    eval_metrics: Dict[str, Any]
+    evaluation_metrics: dict
     """Evaluation metrics"""
 
 
@@ -74,7 +79,7 @@ def get_experiment_infos(api: Api, team_id: int, framework_name: str) -> List[Ex
     experiment_infos = []
 
     file_infos = api.file.list(team_id, experiments_folder, recursive=True, return_type="fileinfo")
-    sorted_file_infos = []
+    sorted_experiment_paths = []
     for file_info in file_infos:
         if not file_info.path.endswith(metadata_name):
             continue
@@ -82,32 +87,34 @@ def get_experiment_infos(api: Api, team_id: int, framework_name: str) -> List[Ex
         experiment_dir = dirname(file_info.path)
         if experiment_dir.endswith(framework_name):
             experiment_path = join(experiment_dir, metadata_name)
-            sorted_file_infos.append(experiment_path)
+            sorted_experiment_paths.append(experiment_path)
 
-    def fetch_experiment_data(file_info):
+    def fetch_experiment_data(experiment_path: str):
         try:
             response = api.post(
                 "file-storage.download",
-                {ApiField.TEAM_ID: team_id, ApiField.PATH: file_info},
+                {ApiField.TEAM_ID: team_id, ApiField.PATH: experiment_path},
                 stream=True,
             )
             response.raise_for_status()
             response_json = response.json()
-            required_fields = {field for field in ExperimentInfo._fields}
-            if not required_fields.issubset(response_json.keys()):
-                logger.debug(
-                    f"Missing required fields in JSON from '{experiment_path}': {required_fields - response_json.keys()}"
-                )
+            required_fields = {field.name for field in fields(ExperimentInfo)}
+            missing_fields = required_fields - response_json.keys()
+            if missing_fields:
+                logger.debug(f"Missing fields: {missing_fields} for '{experiment_path}'")
                 return None
             return ExperimentInfo(**response_json)
         except requests.exceptions.RequestException as e:
-            logger.debug(f"Failed to fetch train metadata from '{experiment_path}': {e}")
+            logger.debug(f"Request failed for '{experiment_path}': {e}")
         except JSONDecodeError as e:
-            logger.debug(f"Failed to decode JSON from '{experiment_path}': {e}")
+            logger.debug(f"JSON decode failed for '{experiment_path}': {e}")
+        except TypeError as e:
+            logger.error(f"TypeError for '{experiment_path}': {e}")
         return None
 
+    # Error
     with ThreadPoolExecutor() as executor:
-        experiment_infos = list(executor.map(fetch_experiment_data, sorted_file_infos))
+        experiment_infos = list(executor.map(fetch_experiment_data, sorted_experiment_paths))
 
     experiment_infos = [info for info in experiment_infos if info is not None]
     return experiment_infos

supervisely/nn/inference/gui/serving_gui_template.py

@@ -30,7 +30,7 @@ class ServingGUITemplate(ServingGUI):
     def __init__(
         self,
         framework_name: str,
-        models: Optional[str] = None,
+        models: Optional[list] = None,
         app_options: Optional[str] = None,
     ):
         if not isinstance(framework_name, str):
@@ -41,7 +41,7 @@ class ServingGUITemplate(ServingGUI):
         self.team_id = sly_env.team_id()
 
         self.framework_name = framework_name
-        self.models = self._load_models(models) if models else []
+        self.models = models
         self.app_options = self._load_app_options(app_options) if app_options else {}
 
         base_widgets = self._initialize_layout()
@@ -177,39 +177,6 @@ class ServingGUITemplate(ServingGUI):
             "runtime": self.runtime,
         }
 
-    # Loaders
-    def _load_models(self, models: str) -> List[Dict[str, Any]]:
-        """
-        Loads models from the provided file or list of model configurations.
-        """
-        if isinstance(models, str):
-            if sly_fs.file_exists(models) and sly_fs.get_file_ext(models) == ".json":
-                models = sly_json.load_json_file(models)
-            else:
-                raise ValueError("File not found or invalid file format.")
-        else:
-            raise ValueError(
-                "Invalid models file. Please provide a valid '.json' file with list of model configurations."
-            )
-
-        if not isinstance(models, list):
-            raise ValueError("models parameters must be a list of dicts")
-        for item in models:
-            if not isinstance(item, dict):
-                raise ValueError(f"Each item in models must be a dict.")
-            model_meta = item.get("meta")
-            if model_meta is None:
-                raise ValueError(
-                    "Model metadata not found. Please update provided models parameter to include key 'meta'."
-                )
-            model_files = model_meta.get("model_files")
-            if model_files is None:
-                raise ValueError(
-                    "Model files not found in model metadata. "
-                    "Please update provided models oarameter to include key 'model_files' in 'meta' key."
-                )
-        return models
-
     def _load_app_options(self, app_options: str = None) -> Dict[str, Any]:
         """
         Loads the app_options parameter to ensure it is in the correct format.