supervisely 6.73.238__py3-none-any.whl → 6.73.239__py3-none-any.whl

supervisely/annotation/annotation.py

@@ -2177,7 +2177,7 @@ class Annotation:
 
             if np.any(mask):  # figure may be entirely covered by others
                 g = lbl.geometry
-                new_mask = Bitmap(data=mask)
+                new_mask = Bitmap(data=mask, extra_validation=False)
                 new_lbl = lbl.clone(geometry=new_mask, obj_class=dest_class)
                 new_labels.append(new_lbl)
         new_ann = self.clone(labels=new_labels)
@@ -2366,7 +2366,7 @@ class Annotation:
         new_labels = []
         for obj_class, white_mask in class_mask.items():
             mask = white_mask == 255
-            bitmap = Bitmap(data=mask)
+            bitmap = Bitmap(data=mask, extra_validation=False)
             new_labels.append(Label(geometry=bitmap, obj_class=obj_class))
         return self.clone(labels=new_labels)
 

supervisely/api/entity_annotation/tag_api.py

@@ -1,12 +1,13 @@
 # coding: utf-8
 
-from typing import List
+from typing import List, Optional
 
 from supervisely._utils import batched
 from supervisely.api.module_api import ApiField, ModuleApi
 from supervisely.collection.key_indexed_collection import KeyIndexedCollection
 from supervisely.task.progress import tqdm_sly
 from supervisely.video_annotation.key_id_map import KeyIdMap
+from supervisely.task.progress import tqdm_sly
 
 
 class TagApi(ModuleApi):
@@ -241,6 +242,7 @@ class TagApi(ModuleApi):
         tags_list: List[dict],
         batch_size: int = 100,
         log_progress: bool = False,
+        progress: Optional[tqdm_sly] = None,
     ) -> List[dict]:
         """
         Add Tags to existing Annotation Figures.
@@ -255,6 +257,8 @@ class TagApi(ModuleApi):
         :type batch_size: int
         :param log_progress: If True, will display a progress bar.
         :type log_progress: bool
+        :param progress: Progress bar object to display progress.
+        :type progress: Optional[tqdm_sly]
         :return: List of tags infos as dictionaries.
         :rtype: List[dict]
 
@@ -313,9 +317,12 @@ class TagApi(ModuleApi):
         if len(tags_list) == 0:
             return []
 
+        if progress is not None:
+            log_progress = False
+
         result = []
         if log_progress:
-            ds_progress = tqdm_sly(
+            progress = tqdm_sly(
                 desc="Adding tags to figures",
                 total=len(tags_list),
             )
@@ -323,6 +330,6 @@ class TagApi(ModuleApi):
             data = {ApiField.PROJECT_ID: project_id, ApiField.TAGS: batch}
             response = self._api.post("figures.tags.bulk.add", data)
             result.extend(response.json())
-            if log_progress:
-                ds_progress.update(len(batch))
+            if progress is not None:
+                progress.update(len(batch))
         return result

supervisely/nn/__init__.py

@@ -1,4 +1,5 @@
 import supervisely.nn.artifacts as artifacts
+import supervisely.nn.benchmark as benchmark
 import supervisely.nn.inference as inference
 from supervisely.nn.artifacts.artifacts import BaseTrainArtifacts, TrainInfo
 from supervisely.nn.prediction_dto import (

supervisely/nn/benchmark/__init__.py

@@ -1,2 +1,14 @@
-from supervisely.nn.benchmark.object_detection_benchmark import ObjectDetectionBenchmark
-from supervisely.nn.benchmark.instance_segmentation_benchmark import InstanceSegmentationBenchmark
+from supervisely.nn.benchmark.instance_segmentation.benchmark import (
+    InstanceSegmentationBenchmark,
+)
+from supervisely.nn.benchmark.instance_segmentation.evaluator import (
+    InstanceSegmentationEvaluator,
+)
+from supervisely.nn.benchmark.object_detection.benchmark import ObjectDetectionBenchmark
+from supervisely.nn.benchmark.object_detection.evaluator import ObjectDetectionEvaluator
+from supervisely.nn.benchmark.semantic_segmentation.benchmark import (
+    SemanticSegmentationBenchmark,
+)
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvaluator,
+)

supervisely/nn/benchmark/base_benchmark.py

@@ -1,26 +1,31 @@
 import os
-import yaml
 from typing import Callable, List, Optional, Tuple, Union
 
 import numpy as np
 
 from supervisely._utils import is_development
 from supervisely.api.api import Api
+from supervisely.api.module_api import ApiField
 from supervisely.api.project_api import ProjectInfo
 from supervisely.app.widgets import SlyTqdm
 from supervisely.io import env, fs, json
 from supervisely.io.fs import get_directory_size
-from supervisely.nn.benchmark.evaluation import BaseEvaluator
-from supervisely.nn.benchmark.utils import WORKSPACE_DESCRIPTION, WORKSPACE_NAME
-from supervisely.nn.benchmark.visualization.visualizer import Visualizer
+from supervisely.nn.benchmark.base_evaluator import BaseEvaluator
 from supervisely.nn.inference import SessionJSON
 from supervisely.project.project import download_project
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.sly_logger import logger
 from supervisely.task.progress import tqdm_sly
 
+WORKSPACE_NAME = "Model Benchmark: predictions and differences"
+WORKSPACE_DESCRIPTION = "Technical workspace for model benchmarking. Contains predictions and differences between ground truth and predictions."
+
 
 class BaseBenchmark:
+    visualizer_cls = None
+    EVALUATION_DIR_NAME = "evaluation"
+    SPEEDTEST_DIR_NAME = "speedtest"
+    VISUALIZATIONS_DIR_NAME = "visualizations"
 
     def __init__(
         self,
@@ -41,6 +46,20 @@ class BaseBenchmark:
         self.diff_project_info: ProjectInfo = None
         self.gt_dataset_ids = gt_dataset_ids
         self.gt_images_ids = gt_images_ids
+        self.gt_dataset_infos = None
+        if gt_dataset_ids is not None:
+            self.gt_dataset_infos = self.api.dataset.get_list(
+                self.gt_project_info.id,
+                filters=[
+                    {
+                        ApiField.FIELD: ApiField.ID,
+                        ApiField.OPERATOR: "in",
+                        ApiField.VALUE: gt_dataset_ids,
+                    }
+                ],
+                recursive=True,
+            )
+        self.num_items = self._get_total_items_for_progress()
         self.output_dir = output_dir
         self.team_id = env.team_id()
         self.evaluator: BaseEvaluator = None
@@ -53,6 +72,7 @@ class BaseBenchmark:
         self.vis_texts = None
         self.inference_speed_text = None
         self.train_info = None
+        self.evaluator_app_info = None
         self.evaluation_params = evaluation_params
         self._validate_evaluation_params()
 
@@ -75,9 +95,12 @@ class BaseBenchmark:
         batch_size: int = 16,
         cache_project_on_agent: bool = False,
     ):
-        self.session = self._init_model_session(model_session, inference_settings)
-        self._eval_inference_info = self._run_inference(
-            output_project_id, batch_size, cache_project_on_agent
+        self.run_inference(
+            model_session=model_session,
+            inference_settings=inference_settings,
+            output_project_id=output_project_id,
+            batch_size=batch_size,
+            cache_project_on_agent=cache_project_on_agent,
         )
         self.evaluate(self.dt_project_info.id)
         self._dump_eval_inference_info(self._eval_inference_info)
@@ -103,6 +126,11 @@ class BaseBenchmark:
     ):
         model_info = self._fetch_model_info(self.session)
         self.dt_project_info = self._get_or_create_dt_project(output_project_id, model_info)
+        logger.info(
+            f"""
+            Predictions project ID: {self.dt_project_info.id},
+                        workspace ID: {self.dt_project_info.workspace_id}"""
+        )
         if self.gt_images_ids is None:
             iterator = self.session.inference_project_id_async(
                 self.gt_project_info.id,
@@ -118,9 +146,7 @@ class BaseBenchmark:
                 batch_size=batch_size,
             )
         output_project_id = self.dt_project_info.id
-        with self.pbar(
-            message="Evaluation: Running inference", total=self.gt_project_info.items_count
-        ) as p:
+        with self.pbar(message="Evaluation: Running inference", total=self.num_items) as p:
             for _ in iterator:
                 p.update(1)
         inference_info = {
@@ -130,6 +156,14 @@ class BaseBenchmark:
             "batch_size": batch_size,
             **model_info,
         }
+        if self.train_info:
+            self.train_info.pop("train_images_ids", None)
+            inference_info["train_info"] = self.train_info
+        if self.evaluator_app_info:
+            self.evaluator_app_info.pop("settings", None)
+            inference_info["evaluator_app_info"] = self.evaluator_app_info
+        if self.gt_images_ids:
+            inference_info["val_images_cnt"] = len(self.gt_images_ids)
         self.dt_project_info = self.api.project.get_info_by_id(self.dt_project_info.id)
         logger.debug(
             "Inference is finished.",
@@ -151,10 +185,10 @@ class BaseBenchmark:
         eval_results_dir = self.get_eval_results_dir()
         self.evaluator = self._get_evaluator_class()(
             gt_project_path=gt_project_path,
-            dt_project_path=dt_project_path,
+            pred_project_path=dt_project_path,
             result_dir=eval_results_dir,
             progress=self.pbar,
-            items_count=self.dt_project_info.items_count,
+            items_count=self.num_items,
             classes_whitelist=self.classes_whitelist,
             evaluation_params=self.evaluation_params,
         )
@@ -244,19 +278,18 @@ class BaseBenchmark:
     def get_project_paths(self):
         base_dir = self.get_base_dir()
         gt_path = os.path.join(base_dir, "gt_project")
-        dt_path = os.path.join(base_dir, "dt_project")
+        dt_path = os.path.join(base_dir, "pred_project")
         return gt_path, dt_path
 
     def get_eval_results_dir(self) -> str:
-        dir = os.path.join(self.get_base_dir(), "evaluation")
-        os.makedirs(dir, exist_ok=True)
-        return dir
+        eval_dir = os.path.join(self.get_base_dir(), self.EVALUATION_DIR_NAME)
+        os.makedirs(eval_dir, exist_ok=True)
+        return eval_dir
 
     def get_speedtest_results_dir(self) -> str:
-        checkpoint_name = self._speedtest["model_info"]["checkpoint_name"]
-        dir = os.path.join(self.output_dir, "speedtest", checkpoint_name)
-        os.makedirs(dir, exist_ok=True)
-        return dir
+        speedtest_dir = os.path.join(self.get_base_dir(), self.SPEEDTEST_DIR_NAME)
+        os.makedirs(speedtest_dir, exist_ok=True)
+        return speedtest_dir
 
     def upload_eval_results(self, remote_dir: str):
         eval_dir = self.get_eval_results_dir()
@@ -279,7 +312,7 @@ class BaseBenchmark:
             )
 
     def get_layout_results_dir(self) -> str:
-        dir = os.path.join(self.get_base_dir(), "layout")
+        dir = os.path.join(self.get_base_dir(), self.VISUALIZATIONS_DIR_NAME)
         os.makedirs(dir, exist_ok=True)
         return dir
 
@@ -321,12 +354,9 @@ class BaseBenchmark:
     def _download_projects(self, save_images=False):
         gt_path, dt_path = self.get_project_paths()
         if not os.path.exists(gt_path):
-            total = (
-                self.gt_project_info.items_count
-                if self.gt_images_ids is None
-                else len(self.gt_images_ids)
-            )
-            with self.pbar(message="Evaluation: Downloading GT annotations", total=total) as p:
+            with self.pbar(
+                message="Evaluation: Downloading GT annotations", total=self.num_items
+            ) as p:
                 download_project(
                     self.api,
                     self.gt_project_info.id,
@@ -341,12 +371,9 @@ class BaseBenchmark:
         else:
             logger.info(f"Found GT annotations in {gt_path}")
         if not os.path.exists(dt_path):
-            total = (
-                self.gt_project_info.items_count
-                if self.gt_images_ids is None
-                else len(self.gt_images_ids)
-            )
-            with self.pbar(message="Evaluation: Downloading Pred annotations", total=total) as p:
+            with self.pbar(
+                message="Evaluation: Downloading Pred annotations", total=self.num_items
+            ) as p:
                 download_project(
                     self.api,
                     self.dt_project_info.id,
@@ -374,7 +401,7 @@ class BaseBenchmark:
                 "id": app_info["id"],
             }
         else:
-            logger.warn("session.task_id is not set. App info will not be fetched.")
+            logger.warning("session.task_id is not set. App info will not be fetched.")
             app_info = None
         model_info = {
             **deploy_info,
@@ -450,8 +477,16 @@ class BaseBenchmark:
         if dt_project_id is not None:
             self.dt_project_info = self.api.project.get_info_by_id(dt_project_id)
 
-        vis = Visualizer(self)
-        vis.visualize()
+        if self.visualizer_cls is None:
+            raise RuntimeError(
+                f"Visualizer class is not defined in {self.__class__.__name__}. "
+                "It should be defined in the subclass of BaseBenchmark (e.g. ObjectDetectionBenchmark)."
+            )
+        eval_result = self.evaluator.get_eval_result()
+        vis = self.visualizer_cls(self.api, [eval_result], self.get_layout_results_dir(), self.pbar)  # pylint: disable=not-callable
+        with self.pbar(message="Visualizations: Rendering layout", total=1) as p:
+            vis.visualize()
+            p.update(1)
 
     def _get_or_create_diff_project(self) -> Tuple[ProjectInfo, bool]:
 
@@ -499,7 +534,7 @@ class BaseBenchmark:
         layout_dir = self.get_layout_results_dir()
         assert not fs.dir_empty(
             layout_dir
-        ), f"The layout dir {layout_dir!r} is empty. You should run evaluation before uploading results."
+        ), f"The layout dir {layout_dir!r} is empty. You should run visualizations before uploading results."
 
         # self.api.file.remove_dir(self.team_id, dest_dir, silent=True)
 
@@ -555,9 +590,21 @@ class BaseBenchmark:
             if not pred_meta.obj_classes.has_key(obj_cls.name):
                 pred_meta = pred_meta.add_obj_class(obj_cls)
                 chagned = True
+        for tag_meta in gt_meta.tag_metas:
+            if not pred_meta.tag_metas.has_key(tag_meta.name):
+                pred_meta = pred_meta.add_tag_meta(tag_meta)
+                chagned = True
         if chagned:
             self.api.project.update_meta(pred_project_id, pred_meta.to_json())
 
     def _validate_evaluation_params(self):
         if self.evaluation_params:
             self._get_evaluator_class().validate_evaluation_params(self.evaluation_params)
+
+    def _get_total_items_for_progress(self):
+        if self.gt_images_ids is not None:
+            return len(self.gt_images_ids)
+        elif self.gt_dataset_ids is not None:
+            return sum(ds.items_count for ds in self.gt_dataset_infos)
+        else:
+            return self.gt_project_info.items_count

supervisely/nn/benchmark/base_evaluator.py

@@ -0,0 +1,120 @@
+import os
+import pickle
+from typing import Dict, List, Optional, Union
+
+import yaml
+
+from supervisely.app.widgets import SlyTqdm
+from supervisely.task.progress import tqdm_sly
+
+
+class BaseEvalResult:
+    def __init__(self, directory: str):
+        self.directory = directory
+        self.inference_info: Dict = None
+        self.speedtest_info: Dict = None
+        self.eval_data: Dict = None
+        self.mp = None
+
+        self._read_eval_data()
+
+    @property
+    def cv_task(self):
+        return self.inference_info.get("task_type")
+
+    @property
+    def name(self) -> str:
+        model_name = self.inference_info.get("model_name", self.directory)
+        return self.inference_info.get("deploy_params", {}).get("checkpoint_name", model_name)
+
+    @property
+    def gt_project_id(self) -> int:
+        return self.inference_info.get("gt_project_id")
+
+    @property
+    def gt_dataset_ids(self) -> List[int]:
+        return self.inference_info.get("gt_dataset_ids", None)
+
+    @property
+    def dt_project_id(self):
+        return self.inference_info.get("dt_project_id")
+
+    @property
+    def pred_project_id(self):
+        return self.dt_project_id
+
+    @property
+    def train_info(self):
+        return self.inference_info.get("train_info", None)  # TODO: check
+
+    @property
+    def evaluator_app_info(self):
+        return self.inference_info.get("evaluator_app_info", None)  # TODO: check
+
+    @property
+    def val_images_cnt(self):
+        return self.inference_info.get("val_images_cnt", None)  # TODO: check
+
+    @property
+    def classes_whitelist(self):
+        return self.inference_info.get("inference_settings", {}).get("classes", [])  # TODO: check
+
+    def _read_eval_data(self):
+        raise NotImplementedError()
+
+
+class BaseEvaluator:
+    EVALUATION_PARAMS_YAML_PATH: str = None
+    eval_result_cls = BaseEvalResult
+
+    def __init__(
+        self,
+        gt_project_path: str,
+        pred_project_path: str,
+        result_dir: str = "./evaluation",
+        progress: Optional[SlyTqdm] = None,
+        items_count: Optional[int] = None,  # TODO: is it needed?
+        classes_whitelist: Optional[List[str]] = None,
+        evaluation_params: Optional[dict] = None,
+    ):
+        self.gt_project_path = gt_project_path
+        self.pred_project_path = pred_project_path
+        self.result_dir = result_dir
+        self.total_items = items_count
+        self.pbar = progress or tqdm_sly
+        os.makedirs(result_dir, exist_ok=True)
+        self.classes_whitelist = classes_whitelist
+
+        if evaluation_params is None:
+            evaluation_params = self._get_default_evaluation_params()
+        self.evaluation_params = evaluation_params
+        if self.evaluation_params:
+            self.validate_evaluation_params(self.evaluation_params)
+
+    def evaluate(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def load_yaml_evaluation_params(cls) -> Union[str, None]:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return None
+        with open(cls.EVALUATION_PARAMS_YAML_PATH, "r") as f:
+            return f.read()
+
+    @classmethod
+    def validate_evaluation_params(cls, evaluation_params: dict) -> None:
+        pass
+
+    @classmethod
+    def _get_default_evaluation_params(cls) -> dict:
+        if cls.EVALUATION_PARAMS_YAML_PATH is None:
+            return {}
+        else:
+            return yaml.safe_load(cls.load_yaml_evaluation_params())
+
+    def _dump_pickle(self, data, file_path):
+        with open(file_path, "wb") as f:
+            pickle.dump(data, f)
+
+    def get_eval_result(self) -> BaseEvalResult:
+        return self.eval_result_cls(self.result_dir)