supervisely 6.73.222__py3-none-any.whl → 6.73.223__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of supervisely might be problematic. Click here for more details.
- supervisely/api/file_api.py +5 -4
- supervisely/api/image_api.py +12 -5
- supervisely/api/video/video_api.py +52 -7
- supervisely/convert/base_converter.py +69 -19
- supervisely/convert/converter.py +5 -0
- supervisely/convert/image/image_converter.py +14 -6
- supervisely/convert/image/sly/sly_image_converter.py +15 -4
- supervisely/convert/video/video_converter.py +12 -9
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/METADATA +1 -1
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/RECORD +14 -14
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/LICENSE +0 -0
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/WHEEL +0 -0
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.222.dist-info → supervisely-6.73.223.dist-info}/top_level.txt +0 -0
supervisely/api/file_api.py
CHANGED
|
@@ -421,16 +421,17 @@ class FileApi(ModuleApiBase):
|
|
|
421
421
|
remote_path,
|
|
422
422
|
local_save_path,
|
|
423
423
|
progress_cb=None,
|
|
424
|
-
|
|
424
|
+
log_progress: bool = False,
|
|
425
425
|
):
|
|
426
426
|
response = self._api.post(
|
|
427
427
|
"file-storage.download",
|
|
428
428
|
{ApiField.TEAM_ID: team_id, ApiField.PATH: remote_path},
|
|
429
429
|
stream=True,
|
|
430
430
|
)
|
|
431
|
-
if
|
|
432
|
-
|
|
433
|
-
|
|
431
|
+
if progress_cb is not None:
|
|
432
|
+
log_progress = False
|
|
433
|
+
|
|
434
|
+
if log_progress and progress_cb is None:
|
|
434
435
|
total_size = int(response.headers.get("Content-Length", 0))
|
|
435
436
|
progress_cb = tqdm_sly(
|
|
436
437
|
total=total_size,
|
supervisely/api/image_api.py
CHANGED
|
@@ -605,9 +605,13 @@ class ImageApi(RemoveableBulkModuleApi):
|
|
|
605
605
|
infos_dict = {}
|
|
606
606
|
ids_set = set(ids)
|
|
607
607
|
while any(ids_set):
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
608
|
+
img_id = ids_set.pop()
|
|
609
|
+
image_info = self.get_info_by_id(img_id, force_metadata_for_links=False)
|
|
610
|
+
if image_info is None:
|
|
611
|
+
raise KeyError(
|
|
612
|
+
f"Image (id: {img_id}) is either archived, doesn't exist or you don't have enough permissions to access it"
|
|
613
|
+
)
|
|
614
|
+
dataset_id = image_info.dataset_id
|
|
611
615
|
for batch in batched(ids):
|
|
612
616
|
filters = [{"field": ApiField.ID, "operator": "in", "value": batch}]
|
|
613
617
|
temp_results = self.get_list_all_pages(
|
|
@@ -1755,12 +1759,13 @@ class ImageApi(RemoveableBulkModuleApi):
|
|
|
1755
1759
|
raise ValueError(
|
|
1756
1760
|
f"Conflict resolution should be one of the following: {SUPPORTED_CONFLICT_RESOLUTIONS}"
|
|
1757
1761
|
)
|
|
1762
|
+
if len(set(names)) != len(names):
|
|
1763
|
+
raise ValueError("Some image names are duplicated, only unique images can be uploaded.")
|
|
1764
|
+
|
|
1758
1765
|
results = []
|
|
1759
1766
|
|
|
1760
1767
|
def _add_timestamp(name: str) -> str:
|
|
1761
|
-
|
|
1762
1768
|
now = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
|
|
1763
|
-
|
|
1764
1769
|
return f"{get_file_name(name)}_{now}{get_file_ext(name)}"
|
|
1765
1770
|
|
|
1766
1771
|
def _pack_for_request(names: List[str], items: List[Any], metas: List[Dict]) -> List[Any]:
|
|
@@ -1816,6 +1821,8 @@ class ImageApi(RemoveableBulkModuleApi):
|
|
|
1816
1821
|
break
|
|
1817
1822
|
except HTTPError as e:
|
|
1818
1823
|
error_details = e.response.json().get("details", {})
|
|
1824
|
+
if isinstance(error_details, list):
|
|
1825
|
+
error_details = error_details[0]
|
|
1819
1826
|
if (
|
|
1820
1827
|
conflict_resolution is not None
|
|
1821
1828
|
and e.response.status_code == 400
|
|
@@ -413,7 +413,12 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
413
413
|
return_first_response=False,
|
|
414
414
|
)
|
|
415
415
|
|
|
416
|
-
def get_info_by_id(
|
|
416
|
+
def get_info_by_id(
|
|
417
|
+
self,
|
|
418
|
+
id: int,
|
|
419
|
+
raise_error: Optional[bool] = False,
|
|
420
|
+
force_metadata_for_links=True,
|
|
421
|
+
) -> VideoInfo:
|
|
417
422
|
"""
|
|
418
423
|
Get Video information by ID in VideoInfo<VideoInfo> format.
|
|
419
424
|
|
|
@@ -421,6 +426,8 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
421
426
|
:type id: int
|
|
422
427
|
:param raise_error: Return an error if the video info was not received.
|
|
423
428
|
:type raise_error: bool
|
|
429
|
+
:param force_metadata_for_links: Get video metadata from server (if the video is uploaded as a link)
|
|
430
|
+
:type force_metadata_for_links: bool
|
|
424
431
|
:return: Information about Video. See :class:`info_sequence<info_sequence>`
|
|
425
432
|
:rtype: :class:`VideoInfo`
|
|
426
433
|
|
|
@@ -477,7 +484,11 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
477
484
|
# )
|
|
478
485
|
"""
|
|
479
486
|
|
|
480
|
-
info = self._get_info_by_id(
|
|
487
|
+
info = self._get_info_by_id(
|
|
488
|
+
id,
|
|
489
|
+
"videos.info",
|
|
490
|
+
fields={ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links},
|
|
491
|
+
)
|
|
481
492
|
if info is None and raise_error is True:
|
|
482
493
|
raise KeyError(f"Video with id={id} not found in your account")
|
|
483
494
|
return info
|
|
@@ -546,6 +557,7 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
546
557
|
ApiField.DATASET_ID: dataset_id,
|
|
547
558
|
ApiField.FILTER: filters,
|
|
548
559
|
ApiField.FIELDS: fields,
|
|
560
|
+
ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
|
|
549
561
|
},
|
|
550
562
|
)
|
|
551
563
|
)
|
|
@@ -555,7 +567,12 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
555
567
|
ordered_results = [temp_map[id] for id in ids]
|
|
556
568
|
return ordered_results
|
|
557
569
|
|
|
558
|
-
def get_json_info_by_id(
|
|
570
|
+
def get_json_info_by_id(
|
|
571
|
+
self,
|
|
572
|
+
id: int,
|
|
573
|
+
raise_error: Optional[bool] = False,
|
|
574
|
+
force_metadata_for_links: Optional[bool] = True,
|
|
575
|
+
) -> Dict:
|
|
559
576
|
"""
|
|
560
577
|
Get Video information by ID in json format.
|
|
561
578
|
|
|
@@ -629,7 +646,12 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
629
646
|
"""
|
|
630
647
|
|
|
631
648
|
data = None
|
|
632
|
-
response = self._get_response_by_id(
|
|
649
|
+
response = self._get_response_by_id(
|
|
650
|
+
id,
|
|
651
|
+
"videos.info",
|
|
652
|
+
id_field=ApiField.ID,
|
|
653
|
+
fields={ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links},
|
|
654
|
+
)
|
|
633
655
|
if response is None:
|
|
634
656
|
if raise_error is True:
|
|
635
657
|
raise KeyError(f"Video with id={id} not found in your account")
|
|
@@ -1050,7 +1072,14 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
1050
1072
|
return new_videos
|
|
1051
1073
|
|
|
1052
1074
|
def _upload_bulk_add(
|
|
1053
|
-
self,
|
|
1075
|
+
self,
|
|
1076
|
+
func_item_to_kv,
|
|
1077
|
+
dataset_id,
|
|
1078
|
+
names,
|
|
1079
|
+
items,
|
|
1080
|
+
metas=None,
|
|
1081
|
+
progress_cb=None,
|
|
1082
|
+
force_metadata_for_links=True,
|
|
1054
1083
|
):
|
|
1055
1084
|
if metas is None:
|
|
1056
1085
|
metas = [{}] * len(items)
|
|
@@ -1077,7 +1106,11 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
1077
1106
|
)
|
|
1078
1107
|
response = self._api.post(
|
|
1079
1108
|
"videos.bulk.add",
|
|
1080
|
-
{
|
|
1109
|
+
{
|
|
1110
|
+
ApiField.DATASET_ID: dataset_id,
|
|
1111
|
+
ApiField.VIDEOS: images,
|
|
1112
|
+
ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
|
|
1113
|
+
},
|
|
1081
1114
|
)
|
|
1082
1115
|
if progress_cb is not None:
|
|
1083
1116
|
progress_cb(len(images))
|
|
@@ -1791,6 +1824,8 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
1791
1824
|
hashes: List[str] = None,
|
|
1792
1825
|
metas: Optional[List[Dict]] = None,
|
|
1793
1826
|
skip_download: Optional[bool] = False,
|
|
1827
|
+
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
1828
|
+
force_metadata_for_links: Optional[bool] = True,
|
|
1794
1829
|
) -> List[VideoInfo]:
|
|
1795
1830
|
"""
|
|
1796
1831
|
Upload Videos from given links to Dataset.
|
|
@@ -1809,6 +1844,10 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
1809
1844
|
:type metas: List[dict], optional
|
|
1810
1845
|
:param skip_download: Skip download videos to local storage.
|
|
1811
1846
|
:type skip_download: Optional[bool]
|
|
1847
|
+
:param progress_cb: Function for tracking the progress of copying.
|
|
1848
|
+
:type progress_cb: tqdm or callable, optional
|
|
1849
|
+
:param force_metadata_for_links: Specify if metadata should be forced. Default is True.
|
|
1850
|
+
:type force_metadata_for_links: Optional[bool]
|
|
1812
1851
|
:return: List with information about Videos. See :class:`info_sequence<info_sequence>`
|
|
1813
1852
|
:rtype: :class:`List[VideoInfo]`
|
|
1814
1853
|
:Usage example:
|
|
@@ -1843,7 +1882,13 @@ class VideoApi(RemoveableBulkModuleApi):
|
|
|
1843
1882
|
# if infos is not None and hashes is not None and not skip_download:
|
|
1844
1883
|
# self.upsert_infos(hashes, infos, links)
|
|
1845
1884
|
return self._upload_bulk_add(
|
|
1846
|
-
lambda item: (ApiField.LINK, item),
|
|
1885
|
+
lambda item: (ApiField.LINK, item),
|
|
1886
|
+
dataset_id,
|
|
1887
|
+
names,
|
|
1888
|
+
links,
|
|
1889
|
+
metas,
|
|
1890
|
+
progress_cb=progress_cb,
|
|
1891
|
+
force_metadata_for_links=force_metadata_for_links,
|
|
1847
1892
|
)
|
|
1848
1893
|
|
|
1849
1894
|
def update_custom_data(self, id: int, data: dict):
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
from typing import Dict, List, Optional, Tuple, Union
|
|
5
6
|
|
|
6
7
|
from tqdm import tqdm
|
|
@@ -10,7 +11,14 @@ from supervisely.annotation.annotation import Annotation
|
|
|
10
11
|
from supervisely.annotation.tag_meta import TagValueType
|
|
11
12
|
from supervisely.api.api import Api
|
|
12
13
|
from supervisely.io.env import team_id
|
|
13
|
-
from supervisely.io.fs import
|
|
14
|
+
from supervisely.io.fs import (
|
|
15
|
+
get_file_ext,
|
|
16
|
+
get_file_name_with_ext,
|
|
17
|
+
is_archive,
|
|
18
|
+
remove_dir,
|
|
19
|
+
silent_remove,
|
|
20
|
+
unpack_archive,
|
|
21
|
+
)
|
|
14
22
|
from supervisely.project.project_meta import ProjectMeta
|
|
15
23
|
from supervisely.project.project_settings import LabelingInterface
|
|
16
24
|
from supervisely.sly_logger import logger
|
|
@@ -433,30 +441,72 @@ class BaseConverter:
|
|
|
433
441
|
if not self.upload_as_links:
|
|
434
442
|
return
|
|
435
443
|
|
|
436
|
-
|
|
444
|
+
ann_archives = {l: r for l, r in self._remote_files_map.items() if is_archive(l)}
|
|
445
|
+
|
|
446
|
+
anns_to_download = {
|
|
437
447
|
l: r for l, r in self._remote_files_map.items() if get_file_ext(l) == self.ann_ext
|
|
438
448
|
}
|
|
439
|
-
if not
|
|
449
|
+
if not anns_to_download and not ann_archives:
|
|
440
450
|
return
|
|
441
451
|
|
|
442
452
|
import asyncio
|
|
443
453
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
454
|
+
for files_type, files in {
|
|
455
|
+
"annotations": anns_to_download,
|
|
456
|
+
"archives": ann_archives,
|
|
457
|
+
}.items():
|
|
458
|
+
if not files:
|
|
459
|
+
continue
|
|
449
460
|
|
|
450
|
-
|
|
451
|
-
silent_remove(local_path)
|
|
461
|
+
is_archive_type = files_type == "archives"
|
|
452
462
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
463
|
+
file_size = None
|
|
464
|
+
if is_archive_type:
|
|
465
|
+
logger.info(f"Remote archives detected.")
|
|
466
|
+
file_size = sum(
|
|
467
|
+
self._api.storage.get_info_by_path(self._team_id, remote_path).sizeb
|
|
468
|
+
for remote_path in files.values()
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
loop = asyncio.get_event_loop()
|
|
472
|
+
_, progress_cb = self.get_progress(
|
|
473
|
+
len(files) if not is_archive_type else file_size,
|
|
474
|
+
f"Downloading {files_type} from remote storage",
|
|
475
|
+
is_size=is_archive_type,
|
|
460
476
|
)
|
|
461
|
-
|
|
462
|
-
|
|
477
|
+
|
|
478
|
+
for local_path in files.keys():
|
|
479
|
+
silent_remove(local_path)
|
|
480
|
+
|
|
481
|
+
logger.info(f"Downloading {files_type} from remote storage...")
|
|
482
|
+
loop.run_until_complete(
|
|
483
|
+
self._api.storage.download_bulk_async(
|
|
484
|
+
team_id=self._team_id,
|
|
485
|
+
remote_paths=list(files.values()),
|
|
486
|
+
local_save_paths=list(files.keys()),
|
|
487
|
+
progress_cb=progress_cb,
|
|
488
|
+
progress_cb_type="number" if not is_archive_type else "size",
|
|
489
|
+
)
|
|
490
|
+
)
|
|
491
|
+
logger.info("Possible annotations downloaded successfully.")
|
|
492
|
+
|
|
493
|
+
if is_archive_type:
|
|
494
|
+
for local_path in files.keys():
|
|
495
|
+
parent_dir = Path(local_path).parent
|
|
496
|
+
if parent_dir.name == "ann":
|
|
497
|
+
target_dir = parent_dir
|
|
498
|
+
else:
|
|
499
|
+
target_dir = parent_dir / "ann"
|
|
500
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
501
|
+
|
|
502
|
+
unpack_archive(local_path, str(target_dir))
|
|
503
|
+
silent_remove(local_path)
|
|
504
|
+
|
|
505
|
+
dirs = [d for d in target_dir.iterdir() if d.is_dir()]
|
|
506
|
+
files = [f for f in target_dir.iterdir() if f.is_file()]
|
|
507
|
+
if len(dirs) == 1 and len(files) == 0:
|
|
508
|
+
for file in dirs[0].iterdir():
|
|
509
|
+
file.rename(target_dir / file.name)
|
|
510
|
+
remove_dir(str(dirs[0]))
|
|
511
|
+
|
|
512
|
+
logger.info(f"Archive {local_path} unpacked successfully to {str(target_dir)}")
|
supervisely/convert/converter.py
CHANGED
|
@@ -173,17 +173,22 @@ class ImportManager:
|
|
|
173
173
|
else:
|
|
174
174
|
files = [self._api.storage.get_info_by_path(self._team_id, remote_path)]
|
|
175
175
|
|
|
176
|
+
unique_directories = set()
|
|
176
177
|
for file in files:
|
|
177
178
|
new_path = file.path.replace(dir_path, local_path)
|
|
178
179
|
self._remote_files_map[new_path] = file.path
|
|
179
180
|
Path(new_path).parent.mkdir(parents=True, exist_ok=True)
|
|
181
|
+
unique_directories.add(str(Path(file.path).parent))
|
|
180
182
|
touch(new_path)
|
|
181
183
|
|
|
184
|
+
logger.info(f"Scanned remote directories:\n - " + "\n - ".join(unique_directories))
|
|
182
185
|
return local_path
|
|
183
186
|
|
|
184
187
|
def _unpack_archives(self, local_path):
|
|
185
188
|
"""Unpack if input data contains an archive."""
|
|
186
189
|
|
|
190
|
+
if self._upload_as_links:
|
|
191
|
+
return
|
|
187
192
|
new_paths_to_scan = [local_path]
|
|
188
193
|
while len(new_paths_to_scan) > 0:
|
|
189
194
|
archives = []
|
|
@@ -114,6 +114,7 @@ class ImageConverter(BaseConverter):
|
|
|
114
114
|
batch_size: int = 50,
|
|
115
115
|
log_progress=True,
|
|
116
116
|
entities: List[Item] = None,
|
|
117
|
+
progress_cb=None,
|
|
117
118
|
) -> None:
|
|
118
119
|
"""Upload converted data to Supervisely"""
|
|
119
120
|
dataset_info = api.dataset.get_info_by_id(dataset_id, raise_error=True)
|
|
@@ -122,10 +123,14 @@ class ImageConverter(BaseConverter):
|
|
|
122
123
|
meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
|
|
123
124
|
|
|
124
125
|
existing_names = set([img.name for img in api.image.get_list(dataset_id)])
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
126
|
+
progress = None
|
|
127
|
+
if progress_cb is not None:
|
|
128
|
+
log_progress = True
|
|
129
|
+
elif log_progress:
|
|
130
|
+
progress, progress_cb = self.get_progress(self.items_count, "Uploading")
|
|
131
|
+
|
|
132
|
+
if self.upload_as_links:
|
|
133
|
+
batch_size = 1000
|
|
129
134
|
|
|
130
135
|
for batch in batched(entities or self._items, batch_size=batch_size):
|
|
131
136
|
item_names = []
|
|
@@ -166,6 +171,7 @@ class ImageConverter(BaseConverter):
|
|
|
166
171
|
item_names,
|
|
167
172
|
item_paths,
|
|
168
173
|
metas=item_metas,
|
|
174
|
+
batch_size=batch_size,
|
|
169
175
|
conflict_resolution="rename",
|
|
170
176
|
force_metadata_for_links=False,
|
|
171
177
|
)
|
|
@@ -188,9 +194,11 @@ class ImageConverter(BaseConverter):
|
|
|
188
194
|
progress_cb(len(batch))
|
|
189
195
|
|
|
190
196
|
if log_progress:
|
|
191
|
-
if is_development():
|
|
197
|
+
if is_development() and progress is not None:
|
|
192
198
|
progress.close()
|
|
193
|
-
logger.info(
|
|
199
|
+
logger.info(
|
|
200
|
+
f"Dataset has been successfully uploaded → {dataset_info.name}, ID:{dataset_id}"
|
|
201
|
+
)
|
|
194
202
|
|
|
195
203
|
def validate_image(self, path: str) -> Tuple[str, str]:
|
|
196
204
|
if self.upload_as_links:
|
|
@@ -12,7 +12,7 @@ from supervisely import (
|
|
|
12
12
|
Rectangle,
|
|
13
13
|
logger,
|
|
14
14
|
)
|
|
15
|
-
from supervisely._utils import generate_free_name
|
|
15
|
+
from supervisely._utils import generate_free_name, is_development
|
|
16
16
|
from supervisely.api.api import Api
|
|
17
17
|
from supervisely.convert.base_converter import AvailableImageConverters
|
|
18
18
|
from supervisely.convert.image.image_converter import ImageConverter
|
|
@@ -169,6 +169,8 @@ class SLYImageConverter(ImageConverter):
|
|
|
169
169
|
project_dirs = [d for d in find_project_dirs(input_data)]
|
|
170
170
|
if len(project_dirs) > 1:
|
|
171
171
|
logger.info("Found multiple possible Supervisely projects in the input data")
|
|
172
|
+
else:
|
|
173
|
+
logger.info("Possible Supervisely project found in the input data")
|
|
172
174
|
meta = None
|
|
173
175
|
for project_dir in project_dirs:
|
|
174
176
|
project_fs = Project(project_dir, mode=OpenMode.READ)
|
|
@@ -281,6 +283,12 @@ class SLYImageConverter(ImageConverter):
|
|
|
281
283
|
existing_datasets = api.dataset.get_list(project_id, recursive=True)
|
|
282
284
|
existing_datasets = {ds.name for ds in existing_datasets}
|
|
283
285
|
|
|
286
|
+
if log_progress:
|
|
287
|
+
progress, progress_cb = self.get_progress(self.items_count, "Uploading project")
|
|
288
|
+
else:
|
|
289
|
+
progress, progress_cb = None, None
|
|
290
|
+
|
|
291
|
+
logger.info("Uploading project structure")
|
|
284
292
|
def _upload_project(
|
|
285
293
|
project_structure: Dict,
|
|
286
294
|
project_id: int,
|
|
@@ -288,7 +296,6 @@ class SLYImageConverter(ImageConverter):
|
|
|
288
296
|
parent_id: Optional[int] = None,
|
|
289
297
|
first_dataset=False,
|
|
290
298
|
):
|
|
291
|
-
|
|
292
299
|
for ds_name, value in project_structure.items():
|
|
293
300
|
ds_name = generate_free_name(existing_datasets, ds_name, extend_used_names=True)
|
|
294
301
|
if first_dataset:
|
|
@@ -298,13 +305,17 @@ class SLYImageConverter(ImageConverter):
|
|
|
298
305
|
dataset_id = api.dataset.create(project_id, ds_name, parent_id=parent_id).id
|
|
299
306
|
|
|
300
307
|
items = value.get(DATASET_ITEMS, [])
|
|
308
|
+
nested_datasets = value.get(NESTED_DATASETS, {})
|
|
309
|
+
logger.info(f"Dataset: {ds_name}, items: {len(items)}, nested datasets: {len(nested_datasets)}")
|
|
301
310
|
if items:
|
|
302
311
|
super(SLYImageConverter, self).upload_dataset(
|
|
303
|
-
api, dataset_id, batch_size,
|
|
312
|
+
api, dataset_id, batch_size, entities=items, progress_cb=progress_cb
|
|
304
313
|
)
|
|
305
314
|
|
|
306
|
-
nested_datasets = value.get(NESTED_DATASETS, {})
|
|
307
315
|
if nested_datasets:
|
|
308
316
|
_upload_project(nested_datasets, project_id, dataset_id, dataset_id)
|
|
309
317
|
|
|
310
318
|
_upload_project(self._project_structure, project_id, dataset_id, first_dataset=True)
|
|
319
|
+
|
|
320
|
+
if is_development() and progress is not None:
|
|
321
|
+
progress.close()
|
|
@@ -140,16 +140,17 @@ class VideoConverter(BaseConverter):
|
|
|
140
140
|
has_large_files = False
|
|
141
141
|
size_progress_cb = None
|
|
142
142
|
progress_cb, progress, ann_progress, ann_progress_cb = None, None, None, None
|
|
143
|
-
if log_progress
|
|
143
|
+
if log_progress:
|
|
144
144
|
progress, progress_cb = self.get_progress(self.items_count, "Uploading videos...")
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
145
|
+
if not self.upload_as_links:
|
|
146
|
+
file_sizes = [get_file_size(item.path) for item in self._items]
|
|
147
|
+
has_large_files = any(
|
|
148
|
+
[self._check_video_file_size(file_size) for file_size in file_sizes]
|
|
149
|
+
)
|
|
150
|
+
if has_large_files:
|
|
151
|
+
upload_progress = []
|
|
152
|
+
size_progress_cb = self._get_video_upload_progress(upload_progress)
|
|
153
|
+
batch_size = 1 if has_large_files and not self.upload_as_links else batch_size
|
|
153
154
|
|
|
154
155
|
for batch in batched(self._items, batch_size=batch_size):
|
|
155
156
|
item_names = []
|
|
@@ -176,6 +177,8 @@ class VideoConverter(BaseConverter):
|
|
|
176
177
|
item_paths,
|
|
177
178
|
item_names,
|
|
178
179
|
skip_download=True,
|
|
180
|
+
progress_cb=progress_cb if log_progress else None,
|
|
181
|
+
force_metadata_for_links=False,
|
|
179
182
|
)
|
|
180
183
|
else:
|
|
181
184
|
vid_infos = api.video.upload_paths(
|
|
@@ -25,10 +25,10 @@ supervisely/api/annotation_api.py,sha256=Eps-Jf10_SQFy7DjghUnyiM6DcVJBsamHDViRAX
|
|
|
25
25
|
supervisely/api/api.py,sha256=Jvc0nQqu6q4-1ey26AJiCcw96BzqP0ORBQqsoKO7gOI,60633
|
|
26
26
|
supervisely/api/app_api.py,sha256=zX3Iy16RuGwtcLZfMs3YfUFc93S9AVGb3W_eINeMjOs,66729
|
|
27
27
|
supervisely/api/dataset_api.py,sha256=7iwAyz3pmzFG2i072gLdXjczfBGbyj-V_rRl7Tx-V30,37944
|
|
28
|
-
supervisely/api/file_api.py,sha256=
|
|
28
|
+
supervisely/api/file_api.py,sha256=UQM5susk1DMgmEEZZq4tiMCKbglKYl8MqiZn1iPLqus,76821
|
|
29
29
|
supervisely/api/github_api.py,sha256=NIexNjEer9H5rf5sw2LEZd7C1WR-tK4t6IZzsgeAAwQ,623
|
|
30
30
|
supervisely/api/image_annotation_tool_api.py,sha256=YcUo78jRDBJYvIjrd-Y6FJAasLta54nnxhyaGyanovA,5237
|
|
31
|
-
supervisely/api/image_api.py,sha256=
|
|
31
|
+
supervisely/api/image_api.py,sha256=BQ_X0z7G-AFim3u6CBqSRPVRqvfa8OFo4sYywSCYCAM,158458
|
|
32
32
|
supervisely/api/import_storage_api.py,sha256=BDCgmR0Hv6OoiRHLCVPKt3iDxSVlQp1WrnKhAK_Zl84,460
|
|
33
33
|
supervisely/api/issues_api.py,sha256=BqDJXmNoTzwc3xe6_-mA7FDFC5QQ-ahGbXk_HmpkSeQ,17925
|
|
34
34
|
supervisely/api/labeling_job_api.py,sha256=odnzZjp29yM16Gq-FYkv-OA4WFMNJCLFo4qSikW2A7c,56280
|
|
@@ -64,7 +64,7 @@ supervisely/api/pointcloud/pointcloud_object_api.py,sha256=bO1USWb9HAywG_CW4CDu1
|
|
|
64
64
|
supervisely/api/pointcloud/pointcloud_tag_api.py,sha256=iShtr052nOElxsyMyZEUT2vypEm6kP00gnP13ABX24A,4691
|
|
65
65
|
supervisely/api/video/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
66
66
|
supervisely/api/video/video_annotation_api.py,sha256=DwZh11fBLa_AWwdYbpbSH1ZGh_5TFfLJiPKcXuOVc14,9319
|
|
67
|
-
supervisely/api/video/video_api.py,sha256=
|
|
67
|
+
supervisely/api/video/video_api.py,sha256=_UOBYr6p6fjzQWU6EQQcDaAebmCsRsaIyRBxlHImPb8,93036
|
|
68
68
|
supervisely/api/video/video_figure_api.py,sha256=quksohjhgrK2l2-PtbbNE99fOW6uWXX59-_4xfc-I-k,6244
|
|
69
69
|
supervisely/api/video/video_frame_api.py,sha256=4GwSI4xdCNYEUvTqzKc-Ewd44fw5zqkFoD24jrrN_aY,10214
|
|
70
70
|
supervisely/api/video/video_object_api.py,sha256=IC0NP8EoIT_d3xxDRgz2cA3ixSiuJ5ymy64eS-RfmDM,2227
|
|
@@ -554,10 +554,10 @@ supervisely/collection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
|
|
|
554
554
|
supervisely/collection/key_indexed_collection.py,sha256=x2UVlkprspWhhae9oLUzjTWBoIouiWY9UQSS_MozfH0,37643
|
|
555
555
|
supervisely/collection/str_enum.py,sha256=Zp29yFGvnxC6oJRYNNlXhO2lTSdsriU1wiGHj6ahEJE,1250
|
|
556
556
|
supervisely/convert/__init__.py,sha256=gRTV93OYJPI3FNfy78HO2SfR59qQ3FFKFSy_jw1adJ8,2571
|
|
557
|
-
supervisely/convert/base_converter.py,sha256=
|
|
558
|
-
supervisely/convert/converter.py,sha256=
|
|
557
|
+
supervisely/convert/base_converter.py,sha256=1Fm31rm2eBjwfN0hYvCBZ5fr9omU10k5sKHttSlCFpE,18127
|
|
558
|
+
supervisely/convert/converter.py,sha256=_uJdKG6w_jISKdyZXrhl6rQOvkuxAKBxqyemIidERgg,8936
|
|
559
559
|
supervisely/convert/image/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
560
|
-
supervisely/convert/image/image_converter.py,sha256=
|
|
560
|
+
supervisely/convert/image/image_converter.py,sha256=r-qdhuwOsk727mXIM26ucQhkoIKigu1M0BF-tw9IfGg,10321
|
|
561
561
|
supervisely/convert/image/image_helper.py,sha256=RkBAyxxXmDG1Z5WFuO9kBDK8MN1Kl7Z25DX9-CwZ7OI,3647
|
|
562
562
|
supervisely/convert/image/cityscapes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
563
563
|
supervisely/convert/image/cityscapes/cityscapes_converter.py,sha256=msmsR2W-Xiod06dwn-MzmkbrEmQQqlKh7zyfTrW6YQw,7854
|
|
@@ -596,7 +596,7 @@ supervisely/convert/image/pdf/pdf_converter.py,sha256=LKvVng9jPp0cSIjYEjKLOb48wt
|
|
|
596
596
|
supervisely/convert/image/pdf/pdf_helper.py,sha256=IDwLEvsVy8lu-KC1lXvSRkZZ9BCC6ylebnNEtLQU5L4,1288
|
|
597
597
|
supervisely/convert/image/sly/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
598
598
|
supervisely/convert/image/sly/fast_sly_image_converter.py,sha256=pZmQzhx9FrHwgVnJgqp-37Cn3zAnPww6MLa1grL6aWM,5429
|
|
599
|
-
supervisely/convert/image/sly/sly_image_converter.py,sha256=
|
|
599
|
+
supervisely/convert/image/sly/sly_image_converter.py,sha256=RfCOUqe4Li7FlCSpcbR4-ScqvV1yuBkUVJDEwNUxsEw,12819
|
|
600
600
|
supervisely/convert/image/sly/sly_image_helper.py,sha256=5Ri8fKb5dzh5b3v8AJ5u8xVFOQfAtoWqZ7HktPsCjTI,7373
|
|
601
601
|
supervisely/convert/image/yolo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
602
602
|
supervisely/convert/image/yolo/yolo_converter.py,sha256=cg5___X5MzvR-rZbNLmaKtr0MdRnyqtEzbBq5UBnYZ0,11171
|
|
@@ -623,7 +623,7 @@ supervisely/convert/pointcloud_episodes/sly/__init__.py,sha256=47DEQpj8HBSa-_TIm
|
|
|
623
623
|
supervisely/convert/pointcloud_episodes/sly/sly_pointcloud_episodes_converter.py,sha256=7ONcZMOUJCNpmc0tmMX6FnNG0lu8Nj9K2SSTQHaSXFM,6188
|
|
624
624
|
supervisely/convert/pointcloud_episodes/sly/sly_pointcloud_episodes_helper.py,sha256=h4WvNH6cEHtjxxhCnU7Hs2vkyJMye0qwabqXNYVTywE,3570
|
|
625
625
|
supervisely/convert/video/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
626
|
-
supervisely/convert/video/video_converter.py,sha256=
|
|
626
|
+
supervisely/convert/video/video_converter.py,sha256=078BvRPi_2bTAdwPi9abp_3AlIpOR0fZs9du7xs1bwU,11251
|
|
627
627
|
supervisely/convert/video/davis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
628
628
|
supervisely/convert/video/davis/davis_converter.py,sha256=zaPsJdN6AvyPT7fVnswuPbgrz5T-X2RFbHEdkuMhWGk,415
|
|
629
629
|
supervisely/convert/video/mot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -997,9 +997,9 @@ supervisely/worker_proto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
|
|
|
997
997
|
supervisely/worker_proto/worker_api_pb2.py,sha256=VQfi5JRBHs2pFCK1snec3JECgGnua3Xjqw_-b3aFxuM,59142
|
|
998
998
|
supervisely/worker_proto/worker_api_pb2_grpc.py,sha256=3BwQXOaP9qpdi0Dt9EKG--Lm8KGN0C5AgmUfRv77_Jk,28940
|
|
999
999
|
supervisely_lib/__init__.py,sha256=7-3QnN8Zf0wj8NCr2oJmqoQWMKKPKTECvjH9pd2S5vY,159
|
|
1000
|
-
supervisely-6.73.
|
|
1001
|
-
supervisely-6.73.
|
|
1002
|
-
supervisely-6.73.
|
|
1003
|
-
supervisely-6.73.
|
|
1004
|
-
supervisely-6.73.
|
|
1005
|
-
supervisely-6.73.
|
|
1000
|
+
supervisely-6.73.223.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
1001
|
+
supervisely-6.73.223.dist-info/METADATA,sha256=AEHBAjRpRoqsjSChavPDTgbucFvcihpotKkjDomq-qo,33150
|
|
1002
|
+
supervisely-6.73.223.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
1003
|
+
supervisely-6.73.223.dist-info/entry_points.txt,sha256=U96-5Hxrp2ApRjnCoUiUhWMqijqh8zLR03sEhWtAcms,102
|
|
1004
|
+
supervisely-6.73.223.dist-info/top_level.txt,sha256=kcFVwb7SXtfqZifrZaSE3owHExX4gcNYe7Q2uoby084,28
|
|
1005
|
+
supervisely-6.73.223.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|