supervisely 6.73.283__py3-none-any.whl → 6.73.285__py3-none-any.whl

supervisely/nn/benchmark/utils/detection/calculate_metrics.py

@@ -48,8 +48,11 @@ def calculate_metrics(
     :return: Results of the evaluation
     :rtype: dict
     """
+    from pycocotools.coco import COCO  # pylint: disable=import-error
     from pycocotools.cocoeval import COCOeval  # pylint: disable=import-error
 
+    cocoGt: COCO = cocoGt
+
     cocoEval = COCOeval(cocoGt, cocoDt, iouType=iouType)
     cocoEval.evaluate()
     progress_cb(1) if progress_cb is not None else None
@@ -66,23 +69,33 @@ def calculate_metrics(
     progress_cb(1) if progress_cb is not None else None
     cocoEval_cls.summarize()
 
-    iou_t = 0
-    is_custom_iou_threshold = (
-        evaluation_params is not None
-        and evaluation_params.get("iou_threshold")
-        and evaluation_params["iou_threshold"] != 0.5
-    )
-    if is_custom_iou_threshold:
-        iou_t = np.where(cocoEval.params.iouThrs == evaluation_params["iou_threshold"])[0][0]
+    iouThrs = cocoEval.params.iouThrs
+    evaluation_params = evaluation_params or {}
+    iou_threshold = evaluation_params.get("iou_threshold", 0.5)
+    iou_threshold_per_class = evaluation_params.get("iou_threshold_per_class")
+    if iou_threshold_per_class is not None:
+        iou_idx_per_class = {
+            cocoGt.getCatIds(catNms=[class_name])[0]: np.where(np.isclose(iouThrs, iou_thres))[0][0]
+            for class_name, iou_thres in iou_threshold_per_class.items()
+        }
+    else:
+        iou_idx = np.where(np.isclose(iouThrs, iou_threshold))[0][0]
+        iou_idx_per_class = {cat_id: iou_idx for cat_id in cocoGt.getCatIds()}
 
     eval_img_dict = get_eval_img_dict(cocoEval)
     eval_img_dict_cls = get_eval_img_dict(cocoEval_cls)
-    matches = get_matches(eval_img_dict, eval_img_dict_cls, cocoEval_cls, iou_t=iou_t)
+    matches = get_matches(
+        eval_img_dict,
+        eval_img_dict_cls,
+        cocoEval_cls,
+        iou_idx_per_class=iou_idx_per_class,
+    )
 
     params = {
         "iouThrs": cocoEval.params.iouThrs,
         "recThrs": cocoEval.params.recThrs,
-        "evaluation_params": evaluation_params or {},
+        "evaluation_params": evaluation_params,
+        "iou_idx_per_class": iou_idx_per_class,
     }
     coco_metrics = {"mAP": cocoEval.stats[0], "precision": cocoEval.eval["precision"]}
     coco_metrics["AP50"] = cocoEval.stats[1]
@@ -204,27 +217,6 @@ def get_eval_img_dict(cocoEval):
     return eval_img_dict
 
 
-def get_eval_img_dict_cls(cocoEval_cls):
-    """
-    type cocoEval_cls: COCOeval
-    """
-    # For miss-classification
-    aRng = cocoEval_cls.params.areaRng[0]
-    eval_img_dict_cls = defaultdict(list)  # img_id : dt/gt
-    for i, eval_img in enumerate(cocoEval_cls.evalImgs):
-        if eval_img is None or eval_img["aRng"] != aRng:
-            continue
-        img_id = eval_img["image_id"]
-        cat_id = eval_img["category_id"]
-        ious = cocoEval_cls.ious[(img_id, cat_id)]
-        # ! inplace operation
-        eval_img["ious"] = ious
-        eval_img_dict_cls[img_id].append(eval_img)
-    eval_img_dict_cls = dict(eval_img_dict_cls)
-    assert np.all([len(x) == 1 for x in eval_img_dict_cls.values()])
-    return eval_img_dict_cls
-
-
 def _get_missclassified_match(eval_img_cls, dt_id, gtIds_orig, dtIds_orig, iou_t):
     # Correction on miss-classification
     gt_idx = np.nonzero(eval_img_cls["gtMatches"][iou_t] == dt_id)[0]
@@ -242,7 +234,12 @@ def _get_missclassified_match(eval_img_cls, dt_id, gtIds_orig, dtIds_orig, iou_t
     return None, None
 
 
-def get_matches(eval_img_dict: dict, eval_img_dict_cls: dict, cocoEval_cls, iou_t: int = 0):
+def get_matches(
+    eval_img_dict: dict,
+    eval_img_dict_cls: dict,
+    cocoEval_cls,
+    iou_idx_per_class: dict = None,
+):
     """
     type cocoEval_cls: COCOeval
     """
@@ -255,7 +252,8 @@ def get_matches(eval_img_dict: dict, eval_img_dict_cls: dict, cocoEval_cls, iou_
         gt_ids_orig_cls = [_["id"] for i in cat_ids for _ in cocoEval_cls._gts[img_id, i]]
 
         for eval_img in eval_imgs:
-
+            cat_id = eval_img["category_id"]
+            iou_t = iou_idx_per_class[cat_id]
             dtIds = np.array(eval_img["dtIds"])
             gtIds = np.array(eval_img["gtIds"])
             dtm = eval_img["dtMatches"][iou_t]

supervisely/nn/benchmark/visualization/renderer.py

@@ -8,6 +8,7 @@ from supervisely.api.api import Api
 from supervisely.io.fs import dir_empty, get_directory_size
 from supervisely.nn.benchmark.visualization.widgets import BaseWidget
 from supervisely.task.progress import tqdm_sly
+from supervisely import logger
 
 
 class Renderer:
@@ -95,6 +96,7 @@ class Renderer:
         pth = Path(self.base_dir).joinpath(self.report_name)
         with open(pth, "w") as f:
             f.write(report_link)
+        logger.debug(f"Report link: {self._get_report_link(api, team_id, remote_dir)}")
         return str(pth)
 
     def _get_report_link(self, api: Api, team_id: int, remote_dir: str):

supervisely/nn/inference/cache.py

@@ -303,6 +303,22 @@ class InferenceImageCache:
                 self.get_frame_from_cache(video_id, frame_index) for frame_index in frame_indexes
             ]
 
+    def frames_loader(
+        self, api: sly.Api, video_id: int, frame_indexes: List[int]
+    ) -> Generator[np.ndarray, None, None]:
+        if not isinstance(self._cache, PersistentImageTTLCache):
+            for frame_index in frame_indexes:
+                yield self.download_frame(api, video_id, frame_index)
+            return
+        self.run_cache_task_manually(api, None, video_id=video_id)
+        for i, frame_index in enumerate(frame_indexes):
+            if video_id in self._cache:
+                break
+            yield self.download_frame(api, video_id, frame_index)
+        if i < len(frame_indexes):
+            for frame in self._read_frames_from_cached_video_iter(video_id, frame_indexes[i:]):
+                yield frame
+
     def download_frame(self, api: sly.Api, video_id: int, frame_index: int) -> np.ndarray:
         name = self._frame_name(video_id, frame_index)
         self._wait_if_in_queue(name, api.logger)
@@ -401,7 +417,9 @@ class InferenceImageCache:
         """
         return_images = kwargs.get("return_images", True)
         progress_cb = kwargs.get("progress_cb", None)
-        video_info = kwargs.get("video_info", api.video.get_info_by_id(video_id))
+        video_info = kwargs.get("video_info", None)
+        if video_info is None:
+            video_info = api.video.get_info_by_id(video_id)
 
         self._wait_if_in_queue(video_id, api.logger)
         if not video_id in self._cache:

supervisely/nn/inference/inference.py

@@ -2252,6 +2252,25 @@ class Inference:
     def is_model_deployed(self):
         return self._model_served
 
+    def schedule_task(self, func, *args, **kwargs):
+        inference_request_uuid = kwargs.get("inference_request_uuid", None)
+        if inference_request_uuid is None:
+            self._executor.submit(func, *args, **kwargs)
+        else:
+            self._on_inference_start(inference_request_uuid)
+            future = self._executor.submit(
+                self._handle_error_in_async,
+                inference_request_uuid,
+                func,
+                *args,
+                **kwargs,
+            )
+            end_callback = partial(
+                self._on_inference_end, inference_request_uuid=inference_request_uuid
+            )
+            future.add_done_callback(end_callback)
+        logger.debug("Scheduled task.", extra={"inference_request_uuid": inference_request_uuid})
+
     def serve(self):
         if not self._use_gui:
             Progress("Deploying model ...", 1)
@@ -2320,6 +2339,9 @@ class Inference:
         else:
             autostart_func()
 
+        self.cache.add_cache_endpoint(server)
+        self.cache.add_cache_files_endpoint(server)
+
         @server.post(f"/get_session_info")
         @self._check_serve_before_call
         def get_session_info(response: Response):

supervisely/nn/inference/tracking/base_tracking.py

@@ -0,0 +1,362 @@
+import functools
+import inspect
+import json
+import traceback
+from threading import Lock
+from typing import Any, BinaryIO, Dict, List, Optional, Tuple, Union
+
+from fastapi import Form, Request, Response, UploadFile, status
+from pydantic import ValidationError
+
+from supervisely._utils import find_value_by_keys
+from supervisely.api.api import Api
+from supervisely.api.module_api import ApiField
+from supervisely.io import env
+from supervisely.nn.inference.inference import (
+    Inference,
+    _convert_sly_progress_to_dict,
+    _get_log_extra_for_inference_request,
+)
+from supervisely.sly_logger import logger
+
+
+def validate_key(data: Dict, key: str, type_: type):
+    if key not in data:
+        raise ValidationError(f"Key {key} not found in inference request.")
+    if not isinstance(data[key], type_):
+        raise ValidationError(f"Key {key} is not of type {type_}.")
+
+
+def handle_validation(func):
+    def _find_response(args, kwargs):
+        for arg in args:
+            if isinstance(arg, Response):
+                return arg
+        for value in kwargs.values():
+            if isinstance(value, Response):
+                return value
+        return None
+
+    def _handle_exception(e, response):
+        if response is not None:
+            logger.error(f"ValidationError: {e}", exc_info=True)
+            response.status_code = status.HTTP_400_BAD_REQUEST
+            return {"error": str(e), "success": False}
+        raise e
+
+    if inspect.iscoroutinefunction(func):
+
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            response = _find_response(args, kwargs)
+            try:
+                return await func(*args, **kwargs)
+            except ValidationError as e:
+                return _handle_exception(e, response)
+
+        return async_wrapper
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        response = _find_response(args, kwargs)
+        try:
+            return func(*args, **kwargs)
+        except ValidationError as e:
+            return _handle_exception(e, response)
+
+    return wrapper
+
+
+class BaseTracking(Inference):
+    def __init__(
+        self,
+        model_dir: Optional[str] = None,
+        custom_inference_settings: Optional[Union[Dict[str, Any], str]] = None,
+    ):
+        Inference.__init__(
+            self,
+            model_dir,
+            custom_inference_settings,
+            sliding_window_mode=None,
+            use_gui=False,
+        )
+
+        try:
+            self.load_on_device(model_dir, "cuda")
+        except RuntimeError:
+            self.load_on_device(model_dir, "cpu")
+            logger.warning("Failed to load model on CUDA device.")
+
+        logger.debug(
+            "Smart cache params",
+            extra={"ttl": env.smart_cache_ttl(), "maxsize": env.smart_cache_size()},
+        )
+
+    def get_info(self):
+        info = super().get_info()
+        info["task type"] = "tracking"
+        return info
+
+    def _on_inference_start(self, inference_request_uuid: str):
+        super()._on_inference_start(inference_request_uuid)
+        self._inference_requests[inference_request_uuid]["lock"] = Lock()
+
+    @staticmethod
+    def _notify_error_default(
+        api: Api, track_id: str, exception: Exception, with_traceback: bool = False
+    ):
+        error_name = type(exception).__name__
+        message = str(exception)
+        if with_traceback:
+            message = f"{message}\n{traceback.format_exc()}"
+        api.video.notify_tracking_error(track_id, error_name, message)
+
+    @staticmethod
+    def _notify_error_direct(
+        api: Api,
+        session_id: str,
+        video_id,
+        track_id: str,
+        exception: Exception,
+        with_traceback: bool = False,
+    ):
+        error_name = type(exception).__name__
+        message = str(exception)
+        if with_traceback:
+            message = f"{message}\n{traceback.format_exc()}"
+        api.vid_ann_tool.set_direct_tracking_error(
+            session_id=session_id,
+            video_id=video_id,
+            track_id=track_id,
+            message=f"{error_name}: {message}",
+        )
+
+    def _handle_error_in_async(self, uuid):
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    inf_request = self._inference_requests.get(uuid, None)
+                    if inf_request is not None:
+                        inf_request["exception"] = str(e)
+                    logger.error(f"Error in {func.__name__} function: {e}", exc_info=True)
+                    raise e
+
+            return wrapper
+
+        return decorator
+
+    @staticmethod
+    def send_error_data(api, context):
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as exc:
+                    try:
+                        track_id = context["trackId"]
+                        if ApiField.USE_DIRECT_PROGRESS_MESSAGES in context:
+                            session_id = find_value_by_keys(context, ["sessionId", "session_id"])
+                            video_id = find_value_by_keys(context, ["videoId", "video_id"])
+                            BaseTracking._notify_error_direct(
+                                api=api,
+                                session_id=session_id,
+                                video_id=video_id,
+                                track_id=track_id,
+                                exception=exc,
+                                with_traceback=False,
+                            )
+                        else:
+                            BaseTracking._notify_error_default(
+                                api=api, track_id=track_id, exception=exc, with_traceback=False
+                            )
+                    except Exception:
+                        logger.error("An error occurred while sending error data", exc_info=True)
+                    raise exc
+
+            return wrapper
+
+        return decorator
+
+    def schedule_task(self, func, *args, **kwargs):
+        inference_request_uuid = kwargs.get("inference_request_uuid", None)
+        if inference_request_uuid is None:
+            self._executor.submit(func, *args, **kwargs)
+        else:
+            self._on_inference_start(inference_request_uuid)
+            fn = self._handle_error_in_async(inference_request_uuid)(func)
+            future = self._executor.submit(
+                fn,
+                *args,
+                **kwargs,
+            )
+            end_callback = functools.partial(
+                self._on_inference_end, inference_request_uuid=inference_request_uuid
+            )
+            future.add_done_callback(end_callback)
+        logger.debug("Scheduled task.", extra={"inference_request_uuid": inference_request_uuid})
+
+    def _pop_tracking_results(self, inference_request_uuid: str, frame_range: Tuple = None):
+        inference_request = self._inference_requests[inference_request_uuid]
+        logger.debug(
+            "Pop tracking results",
+            extra={
+                "inference_request_uuid": inference_request_uuid,
+                "pending_results_len": len(inference_request["pending_results"]),
+                "frame_range": frame_range,
+            },
+        )
+        with inference_request["lock"]:
+            inference_request_copy = inference_request.copy()
+
+            if frame_range is not None:
+
+                def _in_range(figure):
+                    return (
+                        figure.frame_index >= frame_range[0]
+                        and figure.frame_index <= frame_range[1]
+                    )
+
+                inference_request_copy["pending_results"] = list(
+                    filter(_in_range, inference_request_copy["pending_results"])
+                )
+                inference_request["pending_results"] = list(
+                    filter(lambda x: not _in_range(x), inference_request["pending_results"])
+                )
+            else:
+                inference_request["pending_results"] = []
+        inference_request_copy.pop("lock")
+        inference_request_copy["progress"] = _convert_sly_progress_to_dict(
+            inference_request_copy["progress"]
+        )
+
+        inference_request_copy["pending_results"] = [
+            figure.to_json() for figure in inference_request_copy["pending_results"]
+        ]
+
+        return inference_request_copy
+
+    def _clear_tracking_results(self, inference_request_uuid):
+        del self._inference_requests[inference_request_uuid]
+        logger.debug("Removed an inference request:", extra={"uuid": inference_request_uuid})
+
+    def _stop_tracking(self, inference_request_uuid: str):
+        inference_request = self._inference_requests[inference_request_uuid]
+        inference_request["cancel_inference"] = True
+        logger.debug("Stopped tracking:", extra={"uuid": inference_request_uuid})
+
+    # Implement the following methods in the derived class
+    def track(self, api: Api, state: Dict, context: Dict):
+        raise NotImplementedError("Method `track` must be implemented.")
+
+    def track_api(self, api: Api, state: Dict, context: Dict):
+        raise NotImplementedError("Method `_track_api` must be implemented.")
+
+    def track_api_files(
+        self,
+        files: List[BinaryIO],
+        settings: Dict,
+    ):
+        raise NotImplementedError("Method `track_api_files` must be implemented.")
+
+    def track_async(self, api: Api, state: Dict, context: Dict):
+        raise NotImplementedError("Method `track_async` must be implemented.")
+
+    def stop_tracking(self, state: Dict, context: Dict):
+        validate_key(context, "inference_request_uuid", str)
+        inference_request_uuid = context["inference_request_uuid"]
+        self._stop_tracking(inference_request_uuid)
+        return {"message": "Inference will be stopped.", "success": True}
+
+    def pop_tracking_results(self, state: Dict, context: Dict):
+        validate_key(context, "inference_request_uuid", str)
+        inference_request_uuid = context["inference_request_uuid"]
+        frame_range = find_value_by_keys(context, ["frameRange", "frame_range", "frames"])
+        tracking_results = self._pop_tracking_results(inference_request_uuid, frame_range)
+        log_extra = _get_log_extra_for_inference_request(inference_request_uuid, tracking_results)
+        logger.debug(f"Sending inference delta results with uuid:", extra=log_extra)
+        return tracking_results
+
+    def clear_tracking_results(self, state: Dict, context: Dict):
+        self._clear_tracking_results(context)
+        return {"message": "Inference results cleared.", "success": True}
+
+    def _register_endpoints(self):
+        server = self._app.get_server()
+
+        @server.post("/track")
+        @handle_validation
+        def track_handler(request: Request):
+            api = request.state.api
+            state = request.state.state
+            context = request.state.context
+            logger.info("Received track request.", extra={"context": context, "state": state})
+            return self.track(api, state, context)
+
+        @server.post("/track-api")
+        @handle_validation
+        async def track_api_handler(request: Request):
+            api = request.state.api
+            state = request.state.state
+            context = request.state.context
+            logger.info("Received track-api request.", extra={"context": context, "state": state})
+            result = self.track_api(api, state, context)
+            logger.info("Track-api request processed.")
+            return result
+
+        @server.post("/track-api-files")
+        @handle_validation
+        def track_api_files(
+            files: List[UploadFile],
+            settings: str = Form("{}"),
+        ):
+            files = [file.file for file in files]
+            settings = json.loads(settings)
+            return self.track_api_files(files, settings)
+
+        @server.post("/track_async")
+        @handle_validation
+        def track_async_handler(request: Request):
+            api = request.state.api
+            state = request.state.state
+            context = request.state.context
+            logger.info("Received track_async request.", extra={"context": context, "state": state})
+            return self.track_async(api, state, context)
+
+        @server.post("/stop_tracking")
+        @handle_validation
+        def stop_tracking_handler(response: Response, request: Request):
+            state = request.state.state
+            context = request.state.context
+            logger.info(
+                "Received stop_tracking request.", extra={"context": context, "state": state}
+            )
+            return self.stop_tracking(state, context)
+
+        @server.post("/pop_tracking_results")
+        @handle_validation
+        def pop_tracking_results_handler(request: Request, response: Response):
+            state = request.state.state
+            context = request.state.context
+            logger.info(
+                "Received pop_tracking_results request.", extra={"context": context, "state": state}
+            )
+            return self.pop_tracking_results(state, context)
+
+        @server.post("/clear_tracking_results")
+        @handle_validation
+        def clear_tracking_results_handler(request: Request, response: Response):
+            context = request.state.context
+            state = request.state.state
+            logger.info(
+                "Received clear_tracking_results request.",
+                extra={"context": context, "state": state},
+            )
+            return self.clear_tracking_results(state, context)
+
+    def serve(self):
+        super().serve()
+        self._register_endpoints()