supervisely 6.73.272__py3-none-any.whl → 6.73.274__py3-none-any.whl

supervisely/project/project.py

@@ -4432,23 +4432,29 @@ async def _download_project_async(
     switch_size = kwargs.get("switch_size", 1.28 * 1024 * 1024)
     # batch size for bulk download
     batch_size = kwargs.get("batch_size", 100)
-    # number of workers
-    num_workers = kwargs.get("num_workers", 5)
 
     if semaphore is None:
         semaphore = api.get_default_semaphore()
 
-    async def worker(queue: asyncio.Queue, semaphore: asyncio.Semaphore):
-        while True:
+    # number of workers
+    num_workers = min(kwargs.get("num_workers", semaphore._value), 10)
+
+    async def worker(queue: asyncio.Queue, stop_event: asyncio.Event):
+        while not stop_event.is_set():
             task = await queue.get()
             if task is None:
                 break
-            async with semaphore:
+            try:
                 await task
-            queue.task_done()
+            except Exception as e:
+                logger.error(f"Error in _download_project_async worker: {e}")
+                stop_event.set()
+            finally:
+                queue.task_done()
 
     queue = asyncio.Queue()
-    workers = [asyncio.create_task(worker(queue, semaphore)) for _ in range(num_workers)]
+    stop_event = asyncio.Event()
+    workers = [asyncio.create_task(worker(queue, stop_event)) for _ in range(num_workers)]
 
     dataset_ids = set(dataset_ids) if (dataset_ids is not None) else None
     project_fs = None
@@ -4494,9 +4500,11 @@ async def _download_project_async(
         )
         small_images = []
         large_images = []
+        dataset_images = []
         async for image_batch in all_images:
             for image in image_batch:
                 if images_ids is None or image.id in images_ids:
+                    dataset_images.append(image)
                     if image.size < switch_size:
                         small_images.append(image)
                     else:
@@ -4523,13 +4531,12 @@ async def _download_project_async(
                     try:
                         existing = dataset_fs.get_item_info(image.name)
                     except:
-                        pass
+                        to_download.append(image)
                     else:
-                        if existing.updated_at == image.updated_at:
-                            if ds_progress is not None:
-                                ds_progress(1)
-                            continue
-                    to_download.append(image)
+                        if existing.updated_at != image.updated_at:
+                            to_download.append(image)
+                        elif ds_progress is not None:
+                            ds_progress(1)
                 return to_download
 
             small_images = await check_items(small_images)
@@ -4568,11 +4575,9 @@ async def _download_project_async(
 
         await queue.join()
 
-        all_images = small_images + large_images
-
         if save_image_meta:
             meta_dir = dataset_fs.meta_dir
-            for image_info in all_images:
+            for image_info in dataset_images:
                 if image_info.meta:
                     sly.fs.mkdir(meta_dir)
                     sly.json.dump_json_file(
@@ -4580,7 +4585,7 @@ async def _download_project_async(
                     )
 
         # delete redundant items
-        items_names_set = set([img.name for img in all_images])
+        items_names_set = set([img.name for img in dataset_images])
         for item_name in dataset_fs.get_items_names():
             if item_name not in items_names_set:
                 dataset_fs.delete_item(item_name)
@@ -4588,6 +4593,10 @@ async def _download_project_async(
     for _ in range(num_workers):
         await queue.put(None)
     await asyncio.gather(*workers)
+
+    if stop_event.is_set():
+        raise RuntimeError("Download process was stopped due to an error in one of the workers.")
+
     try:
         create_readme(dest_dir, project_id, api)
     except Exception as e:

{supervisely-6.73.272.dist-info → supervisely-6.73.274.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.272
+Version: 6.73.274
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.272.dist-info → supervisely-6.73.274.dist-info}/RECORD

@@ -1000,7 +1000,7 @@ supervisely/project/data_version.py,sha256=nknaWJSUCwoDyNG9_d1KA-GjzidhV9zd9Cn8c
 supervisely/project/download.py,sha256=zb8sb4XZ6Qi3CP7fmtLRUAYzaxs_W0WnOfe2x3ZVRMs,24639
 supervisely/project/pointcloud_episode_project.py,sha256=yiWdNBQiI6f1O9sr1pg8JHW6O-w3XUB1rikJNn3Oung,41866
 supervisely/project/pointcloud_project.py,sha256=Kx1Vaes-krwG3BiRRtHRLQxb9G5m5bTHPN9IzRqmNWo,49399
-supervisely/project/project.py,sha256=W9KraDLPE4u5EilHgxHndSmK_8xwrAmjkUQSS1TAyRg,188731
+supervisely/project/project.py,sha256=6E3DqWF7T5XJWywuna2gAeGVb9CaK8dU3QFcrOQMLRU,189109
 supervisely/project/project_meta.py,sha256=26s8IiHC5Pg8B1AQi6_CrsWteioJP2in00cRNe8QlW0,51423
 supervisely/project/project_settings.py,sha256=NLThzU_DCynOK6hkHhVdFyezwprn9UqlnrLDe_3qhkY,9347
 supervisely/project/project_type.py,sha256=_3RqW2CnDBKFOvSIrQT1RJQaiHirs34_jiQS8CkwCpo,530
@@ -1062,9 +1062,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
 supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
 supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
-supervisely-6.73.272.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-supervisely-6.73.272.dist-info/METADATA,sha256=tBPWJYLXQq7WFB62t_mg4YimISZvWTDSS6DcGVVV-LI,33573
-supervisely-6.73.272.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-supervisely-6.73.272.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
-supervisely-6.73.272.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
-supervisely-6.73.272.dist-info/RECORD,,
+supervisely-6.73.274.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+supervisely-6.73.274.dist-info/METADATA,sha256=PHTIEhW0ErGXlcajujh7MS3qSy4cq8TDdAIhh_eIY7U,33573
+supervisely-6.73.274.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+supervisely-6.73.274.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
+supervisely-6.73.274.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
+supervisely-6.73.274.dist-info/RECORD,,