supervisely 6.73.231__tar.gz → 6.73.233__tar.gz

{supervisely-6.73.231/supervisely.egg-info → supervisely-6.73.233}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.231
+Version: 6.73.233
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/api/api.py

@@ -67,6 +67,7 @@ from supervisely._utils import camel_to_snake, is_community, is_development
 from supervisely.api.module_api import ApiField
 from supervisely.io.network_exceptions import (
     process_requests_exception,
+    process_requests_exception_async,
     process_unhandled_request,
 )
 from supervisely.project.project_meta import ProjectMeta
@@ -806,10 +807,13 @@ class Api:
             reason = "Can't get reason"
 
         def decode_response_content(response: httpx.Response):
-            if hasattr(response, "is_stream_consumed"):
-                return "Content is not acessible for streaming responses"
-            else:
+            try:
                 return response.content.decode("utf-8")
+            except Exception as e:
+                if hasattr(response, "is_stream_consumed"):
+                    return f"Stream is consumed. {e}"
+                else:
+                    return f"Can't decode response content: {e}"
 
         if 400 <= response.status_code < 500:
             http_error_msg = "%s Client Error: %s for url: %s (%s)" % (
@@ -1058,7 +1062,15 @@ class Api:
                     self._check_version()
                     Api._raise_for_status_httpx(response)
                 return response
-            except httpx.RequestError as exc:
+            except (httpx.RequestError, httpx.HTTPStatusError) as exc:
+                if (
+                    isinstance(exc, httpx.HTTPStatusError)
+                    and response.status_code == 400
+                    and self.token is None
+                ):
+                    self.logger.warning(
+                        "API_TOKEN env variable is undefined. See more: https://developer.supervisely.com/getting-started/basics-of-authentication"
+                    )
                 if raise_error:
                     raise exc
                 else:
@@ -1123,7 +1135,15 @@ class Api:
                 if response.status_code != httpx.codes.OK:
                     Api._raise_for_status_httpx(response)
                 return response
-            except httpx.RequestError as exc:
+            except (httpx.RequestError, httpx.HTTPStatusError) as exc:
+                if (
+                    isinstance(exc, httpx.HTTPStatusError)
+                    and response.status_code == 400
+                    and self.token is None
+                ):
+                    self.logger.warning(
+                        "API_TOKEN env variable is undefined. See more: https://developer.supervisely.com/getting-started/basics-of-authentication"
+                    )
                 process_requests_exception(
                     self.logger,
                     exc,
@@ -1260,7 +1280,15 @@ class Api:
                         )
                     logger.trace(f"Streamed size: {total_streamed}, expected size: {expected_size}")
                     return
-            except httpx.RequestError as e:
+            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+                if (
+                    isinstance(e, httpx.HTTPStatusError)
+                    and resp.status_code == 400
+                    and self.token is None
+                ):
+                    self.logger.warning(
+                        "API_TOKEN env variable is undefined. See more: https://developer.supervisely.com/getting-started/basics-of-authentication"
+                    )
                 retry_range_start = total_streamed + (range_start or 0)
                 if total_streamed != 0:
                     retry_range_start += 1
@@ -1348,11 +1376,19 @@ class Api:
                     self._check_version()
                     Api._raise_for_status_httpx(response)
                 return response
-            except httpx.RequestError as exc:
+            except (httpx.RequestError, httpx.HTTPStatusError) as exc:
+                if (
+                    isinstance(exc, httpx.HTTPStatusError)
+                    and response.status_code == 400
+                    and self.token is None
+                ):
+                    self.logger.warning(
+                        "API_TOKEN env variable is undefined. See more: https://developer.supervisely.com/getting-started/basics-of-authentication"
+                    )
                 if raise_error:
                     raise exc
                 else:
-                    process_requests_exception(
+                    await process_requests_exception_async(
                         self.logger,
                         exc,
                         method,
@@ -1489,13 +1525,21 @@ class Api:
                         )
                     logger.trace(f"Streamed size: {total_streamed}, expected size: {expected_size}")
                     return
-            except httpx.RequestError as e:
+            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+                if (
+                    isinstance(e, httpx.HTTPStatusError)
+                    and resp.status_code == 400
+                    and self.token is None
+                ):
+                    self.logger.warning(
+                        "API_TOKEN env variable is undefined. See more: https://developer.supervisely.com/getting-started/basics-of-authentication"
+                    )
                 retry_range_start = total_streamed + (range_start or 0)
                 if total_streamed != 0:
                     retry_range_start += 1
                 headers["Range"] = f"bytes={retry_range_start}-{range_end or ''}"
                 logger.debug(f"Setting Range header {headers['Range']} for retry")
-                process_requests_exception(
+                await process_requests_exception_async(
                     self.logger,
                     e,
                     method,

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/api/video/video_api.py

@@ -1372,6 +1372,19 @@ class VideoApi(RemoveableBulkModuleApi):
             },
         )
 
+    def notify_tracking_warning(self, track_id: int, video_id: int, message: str):
+        self._api.post(
+            "videos.notify-annotation-tool",
+            data={
+                "type": "videos:tracking-warning",
+                "data": {
+                    ApiField.VIDEO_ID: str(video_id),
+                    ApiField.TRACK_ID: str(track_id),
+                    ApiField.MESSAGE: message,
+                },
+            },
+        )
+
     # def upload(self):
     #     #"/videos.bulk.upload"
     #     pass
@@ -2055,7 +2068,7 @@ class VideoApi(RemoveableBulkModuleApi):
             hashes=[h],
             metas=[meta],
             skip_download=skip_download,
-            force_metadata_for_links=force_metadata_for_links
+            force_metadata_for_links=force_metadata_for_links,
         )
         if len(links) != 1:
             raise RuntimeError(

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/io/network_exceptions.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+import asyncio
 import time
 import traceback
 
@@ -27,6 +28,92 @@ RETRY_STATUS_CODES = {
 }
 
 
+async def process_requests_exception_async(
+    external_logger,
+    exc,
+    api_method_name,
+    url,
+    verbose=True,
+    swallow_exc=False,
+    sleep_sec=None,
+    response=None,
+    retry_info=None,
+):
+    recommended_sleep = None
+    try:
+        if hasattr(exc, "response") and exc.response.status_code == 429:
+            recommended_sleep = exc.response.headers.get("Retry-After")
+        elif response is not None and response.status_code == 429:
+            recommended_sleep = response.headers.get("Retry-After")
+        if recommended_sleep and int(recommended_sleep) > sleep_sec:
+            sleep_sec = int(recommended_sleep)
+    except Exception:
+        pass
+
+    is_connection_error = isinstance(
+        exc,
+        (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout,
+            requests.exceptions.TooManyRedirects,
+            requests.exceptions.ChunkedEncodingError,
+            httpx.NetworkError,
+            httpx.TimeoutException,
+            httpx.TooManyRedirects,
+            httpx.ProtocolError,
+        ),
+    )
+
+    is_server_retryable_error = (
+        isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError))
+        and hasattr(exc, "response")
+        and (exc.response.status_code in RETRY_STATUS_CODES)
+    )
+
+    is_need_ping_error = False
+    if (
+        isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError))
+        and hasattr(exc, "response")
+        and (exc.response.status_code == 400)
+    ):
+        try:
+            server_explanation = exc.response.json()
+            is_need_ping_error = server_explanation.get("error", None) == SPECIAL_RECONNECT_ERROR
+        except (AttributeError, ValueError):
+            pass
+
+    if is_connection_error or is_server_retryable_error:
+        await process_retryable_request_async(
+            external_logger,
+            exc,
+            api_method_name,
+            url,
+            CONNECTION_ERROR,
+            verbose=verbose,
+            swallow_exc=swallow_exc,
+            sleep_sec=sleep_sec,
+            retry_info=retry_info,
+        )
+    elif is_need_ping_error:
+        await process_retryable_request_async(
+            external_logger,
+            exc,
+            api_method_name,
+            url,
+            AGENT_CONNECTION_ERROR,
+            verbose=verbose,
+            swallow_exc=swallow_exc,
+            sleep_sec=sleep_sec,
+            retry_info=retry_info,
+        )
+    elif response is None:
+        process_unhandled_request(external_logger, exc)
+    elif isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError)):
+        process_invalid_request(external_logger, exc, response, verbose)
+    else:
+        process_unhandled_request(external_logger, exc)
+
+
 def process_requests_exception(
     external_logger,
     exc,
@@ -38,6 +125,17 @@ def process_requests_exception(
     response=None,
     retry_info=None,
 ):
+    recommended_sleep = None
+    try:
+        if hasattr(exc, "response") and exc.response.status_code == 429:
+            recommended_sleep = exc.response.headers.get("Retry-After")
+        elif response is not None and response.status_code == 429:
+            recommended_sleep = response.headers.get("Retry-After")
+        if recommended_sleep and int(recommended_sleep) > sleep_sec:
+            sleep_sec = int(recommended_sleep)
+    except Exception:
+        pass
+
     is_connection_error = isinstance(
         exc,
         (
@@ -45,7 +143,7 @@ def process_requests_exception(
             requests.exceptions.Timeout,
             requests.exceptions.TooManyRedirects,
             requests.exceptions.ChunkedEncodingError,
-            httpx.ConnectError,
+            httpx.NetworkError,
             httpx.TimeoutException,
             httpx.TooManyRedirects,
             httpx.ProtocolError,
@@ -102,6 +200,33 @@ def process_requests_exception(
         process_unhandled_request(external_logger, exc)
 
 
+async def process_retryable_request_async(
+    external_logger,
+    exc,
+    api_method_name,
+    url,
+    user_message,
+    verbose=True,
+    swallow_exc=False,
+    sleep_sec=None,
+    retry_info=None,
+):
+    if retry_info is not None:
+        retry_idx = retry_info["retry_idx"]
+        retry_limit = retry_info["retry_limit"]
+        user_message = "{}:  Retrying ({}/{}).".format(user_message, retry_idx, retry_limit)
+    if verbose:
+        external_logger.warn(
+            user_message, extra={"method": api_method_name, "url": url, "details": str(exc)}
+        )
+
+    if sleep_sec is not None:
+        await asyncio.sleep(sleep_sec)
+
+    if not swallow_exc:
+        raise exc
+
+
 def process_retryable_request(
     external_logger,
     exc,

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/nn/inference/cache.py

@@ -1,7 +1,6 @@
 import json
-import os
 import shutil
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from enum import Enum
 from logging import Logger
 from pathlib import Path
@@ -16,6 +15,7 @@ from cachetools import Cache, LRUCache, TTLCache
 from fastapi import BackgroundTasks, FastAPI, Form, Request, UploadFile
 
 import supervisely as sly
+from supervisely._utils import batched
 from supervisely.io.fs import silent_remove
 
 
@@ -151,6 +151,38 @@ class PersistentImageTTLCache(TTLCache):
     def get_project_meta(self, project_meta_name):
         return self[project_meta_name]
 
+    def copy_to(self, name, path):
+        shutil.copyfile(str(self[name]), path)
+
+
+class VideoFrameReader:
+    def __init__(self, video_path: str, frame_indexes: List[int]):
+        self.video_path = video_path
+        self.frame_indexes = frame_indexes
+        self.cap = None
+        self.prev_idx = -1
+
+    def __enter__(self):
+        self.cap = cv2.VideoCapture(str(self.video_path))
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.cap is not None:
+            self.cap.release()
+
+    def read_frames(self) -> Generator:
+        try:
+            for frame_index in self.frame_indexes:
+                if frame_index != self.prev_idx + 1:
+                    self.cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index)
+                ret, frame = self.cap.read()
+                if not ret:
+                    raise KeyError(f"Frame {frame_index} not found in video {self.video_path}")
+                yield cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                self.prev_idx = frame_index
+        finally:
+            self.cap.release()
+
 
 class InferenceImageCache:
     class _LoadType(Enum):
@@ -238,25 +270,18 @@ class InferenceImageCache:
             return_images,
         )
 
+    def _read_frames_from_cached_video_iter(self, video_id, frame_indexes):
+        video_path = self._cache.get_video_path(video_id)
+        with VideoFrameReader(video_path, frame_indexes) as reader:
+            for frame in reader.read_frames():
+                yield frame
+
     def _read_frames_from_cached_video(
         self, video_id: int, frame_indexes: List[int]
     ) -> List[np.ndarray]:
-        video_path = self._cache.get_video_path(video_id)
-        if video_path is None or not video_path.exists():
-            raise KeyError(f"Video {video_id} not found in cache")
-        cap = cv2.VideoCapture(str(video_path))
-        frames = []
-        prev_idx = -1
-        for frame_index in frame_indexes:
-            if frame_index != prev_idx + 1:
-                cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index)
-            ret, frame = cap.read()
-            if not ret:
-                raise KeyError(f"Frame {frame_index} not found in video {video_id}")
-            frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
-            prev_idx = frame_index
-        cap.release()
-        return frames
+        return [
+            frame for frame in self._read_frames_from_cached_video_iter(video_id, frame_indexes)
+        ]
 
     def get_frame_from_cache(self, video_id: int, frame_index: int) -> np.ndarray:
         name = self._frame_name(video_id, frame_index)
@@ -664,3 +689,39 @@ class InferenceImageCache:
             # TODO: sleep if slowdown
             sleep(0.1)
             continue
+
+    def download_frames_to_paths(self, api, video_id, frame_indexes, paths, progress_cb=None):
+        def _download_frame(frame_index):
+            self.download_frame(api, video_id, frame_index)
+            name = self._frame_name(video_id, frame_index)
+            return frame_index, name
+
+        def _download_and_save(this_frame_indexes, this_paths):
+            if video_id in self._cache:
+                for path, frame in zip(
+                    this_paths,
+                    self._read_frames_from_cached_video_iter(video_id, this_frame_indexes),
+                ):
+                    sly.image.write(path, frame)
+                    if progress_cb is not None:
+                        progress_cb()
+                return
+
+            futures = []
+            frame_index_to_path = {}
+            for frame_index, path in zip(this_frame_indexes[:5], this_paths[:5]):
+                frame_index_to_path[frame_index] = path
+                futures.append(executor.submit(_download_frame, frame_index))
+            for future in as_completed(futures):
+                frame_index, name = future.result()
+                path = frame_index_to_path[frame_index]
+                self._cache.copy_to(name, path)
+                if progress_cb is not None:
+                    progress_cb()
+            if len(this_frame_indexes) > 5:
+                _download_and_save(this_frame_indexes[5:], this_paths[5:])
+
+        # optimization for frame read from video file
+        frame_indexes, paths = zip(*sorted(zip(frame_indexes, paths), key=lambda x: x[0]))
+        executor = ThreadPoolExecutor(max_workers=5)
+        _download_and_save(frame_indexes, paths)

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/nn/inference/inference.py

@@ -580,7 +580,8 @@ class Inference:
                 continue
             if "classes_whitelist" in inspect.signature(self._create_label).parameters:
                 # pylint: disable=unexpected-keyword-arg
-                label = self._create_label(prediction, classes_whitelist)  # pylint: disable=too-many-function-args
+                # pylint: disable=too-many-function-args
+                label = self._create_label(prediction, classes_whitelist)
             else:
                 label = self._create_label(prediction)
             if label is None:

{supervisely-6.73.231 → supervisely-6.73.233}/supervisely/project/project.py

@@ -36,6 +36,7 @@ from supervisely.collection.key_indexed_collection import (
 from supervisely.geometry.bitmap import Bitmap
 from supervisely.imaging import image as sly_image
 from supervisely.io.fs import (
+    clean_dir,
     copy_file,
     copy_file_async,
     dir_empty,
@@ -3592,7 +3593,14 @@ def _download_project(
     meta = ProjectMeta.from_json(api.project.get_meta(project_id, with_settings=True))
     if os.path.exists(dest_dir) and resume_download:
         dump_json_file(meta.to_json(), os.path.join(dest_dir, "meta.json"))
-        project_fs = Project(dest_dir, OpenMode.READ)
+        try:
+            project_fs = Project(dest_dir, OpenMode.READ)
+        except RuntimeError as e:
+            if "Project is empty" in str(e):
+                clean_dir(dest_dir)
+                project_fs = None
+            else:
+                raise
     if project_fs is None:
         project_fs = Project(dest_dir, OpenMode.CREATE)
     project_fs.set_meta(meta)
@@ -4394,7 +4402,14 @@ async def _download_project_async(
     meta = ProjectMeta.from_json(api.project.get_meta(project_id, with_settings=True))
     if os.path.exists(dest_dir) and resume_download:
         dump_json_file(meta.to_json(), os.path.join(dest_dir, "meta.json"))
-        project_fs = Project(dest_dir, OpenMode.READ)
+        try:
+            project_fs = Project(dest_dir, OpenMode.READ)
+        except RuntimeError as e:
+            if "Project is empty" in str(e):
+                clean_dir(dest_dir)
+                project_fs = None
+            else:
+                raise
     if project_fs is None:
         project_fs = Project(dest_dir, OpenMode.CREATE)
     project_fs.set_meta(meta)

{supervisely-6.73.231 → supervisely-6.73.233/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.231
+Version: 6.73.233
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.231 → supervisely-6.73.233}/tests/test_async_project.py

@@ -11,7 +11,7 @@ from supervisely.project.project import _download_project, _download_project_asy
 
 LOG_LEVEL = "INFO"
 # LOG_LEVEL = "DEBUG"
-PROJECT_ID = 35259  #  41862
+PROJECT_ID = 325865  #  41862
 DATSET_ID = 98429
 home_dir = os.path.expanduser("~")
 common_path = os.path.join(home_dir, "test_project_download/")
@@ -24,7 +24,10 @@ api = sly.Api.from_env()
 
 
 def main_dpa(project_id: int, semaphore_size: int):
-    sema = asyncio.Semaphore(semaphore_size)
+    if semaphore_size is None:
+        sema = None
+    else:
+        sema = asyncio.Semaphore(semaphore_size)
     start = time.time()
     loop = asyncio.get_event_loop()
     loop.run_until_complete(