supervisely 6.73.452__py3-none-any.whl → 6.73.513__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- supervisely/__init__.py +25 -1
- supervisely/annotation/annotation.py +8 -2
- supervisely/annotation/json_geometries_map.py +13 -12
- supervisely/api/annotation_api.py +6 -3
- supervisely/api/api.py +2 -0
- supervisely/api/app_api.py +10 -1
- supervisely/api/dataset_api.py +74 -12
- supervisely/api/entities_collection_api.py +10 -0
- supervisely/api/entity_annotation/figure_api.py +28 -0
- supervisely/api/entity_annotation/object_api.py +3 -3
- supervisely/api/entity_annotation/tag_api.py +63 -12
- supervisely/api/guides_api.py +210 -0
- supervisely/api/image_api.py +4 -0
- supervisely/api/labeling_job_api.py +83 -1
- supervisely/api/labeling_queue_api.py +33 -7
- supervisely/api/module_api.py +5 -0
- supervisely/api/project_api.py +71 -26
- supervisely/api/storage_api.py +3 -1
- supervisely/api/task_api.py +13 -2
- supervisely/api/team_api.py +4 -3
- supervisely/api/video/video_annotation_api.py +119 -3
- supervisely/api/video/video_api.py +65 -14
- supervisely/app/__init__.py +1 -1
- supervisely/app/content.py +23 -7
- supervisely/app/development/development.py +18 -2
- supervisely/app/fastapi/__init__.py +1 -0
- supervisely/app/fastapi/custom_static_files.py +1 -1
- supervisely/app/fastapi/multi_user.py +105 -0
- supervisely/app/fastapi/subapp.py +88 -42
- supervisely/app/fastapi/websocket.py +77 -9
- supervisely/app/singleton.py +21 -0
- supervisely/app/v1/app_service.py +18 -2
- supervisely/app/v1/constants.py +7 -1
- supervisely/app/widgets/__init__.py +6 -0
- supervisely/app/widgets/activity_feed/__init__.py +0 -0
- supervisely/app/widgets/activity_feed/activity_feed.py +239 -0
- supervisely/app/widgets/activity_feed/style.css +78 -0
- supervisely/app/widgets/activity_feed/template.html +22 -0
- supervisely/app/widgets/card/card.py +20 -0
- supervisely/app/widgets/classes_list_selector/classes_list_selector.py +121 -9
- supervisely/app/widgets/classes_list_selector/template.html +60 -93
- supervisely/app/widgets/classes_mapping/classes_mapping.py +13 -12
- supervisely/app/widgets/classes_table/classes_table.py +1 -0
- supervisely/app/widgets/deploy_model/deploy_model.py +56 -35
- supervisely/app/widgets/ecosystem_model_selector/ecosystem_model_selector.py +1 -1
- supervisely/app/widgets/experiment_selector/experiment_selector.py +8 -0
- supervisely/app/widgets/fast_table/fast_table.py +184 -60
- supervisely/app/widgets/fast_table/template.html +1 -1
- supervisely/app/widgets/heatmap/__init__.py +0 -0
- supervisely/app/widgets/heatmap/heatmap.py +564 -0
- supervisely/app/widgets/heatmap/script.js +533 -0
- supervisely/app/widgets/heatmap/style.css +233 -0
- supervisely/app/widgets/heatmap/template.html +21 -0
- supervisely/app/widgets/modal/__init__.py +0 -0
- supervisely/app/widgets/modal/modal.py +198 -0
- supervisely/app/widgets/modal/template.html +10 -0
- supervisely/app/widgets/object_class_view/object_class_view.py +3 -0
- supervisely/app/widgets/radio_tabs/radio_tabs.py +18 -2
- supervisely/app/widgets/radio_tabs/template.html +1 -0
- supervisely/app/widgets/select/select.py +6 -3
- supervisely/app/widgets/select_class/__init__.py +0 -0
- supervisely/app/widgets/select_class/select_class.py +363 -0
- supervisely/app/widgets/select_class/template.html +50 -0
- supervisely/app/widgets/select_cuda/select_cuda.py +22 -0
- supervisely/app/widgets/select_dataset_tree/select_dataset_tree.py +65 -7
- supervisely/app/widgets/select_tag/__init__.py +0 -0
- supervisely/app/widgets/select_tag/select_tag.py +352 -0
- supervisely/app/widgets/select_tag/template.html +64 -0
- supervisely/app/widgets/select_team/select_team.py +37 -4
- supervisely/app/widgets/select_team/template.html +4 -5
- supervisely/app/widgets/select_user/__init__.py +0 -0
- supervisely/app/widgets/select_user/select_user.py +270 -0
- supervisely/app/widgets/select_user/template.html +13 -0
- supervisely/app/widgets/select_workspace/select_workspace.py +59 -10
- supervisely/app/widgets/select_workspace/template.html +9 -12
- supervisely/app/widgets/table/table.py +68 -13
- supervisely/app/widgets/tree_select/tree_select.py +2 -0
- supervisely/aug/aug.py +6 -2
- supervisely/convert/base_converter.py +1 -0
- supervisely/convert/converter.py +2 -2
- supervisely/convert/image/image_converter.py +3 -1
- supervisely/convert/image/image_helper.py +48 -4
- supervisely/convert/image/label_studio/label_studio_converter.py +2 -0
- supervisely/convert/image/medical2d/medical2d_helper.py +2 -24
- supervisely/convert/image/multispectral/multispectral_converter.py +6 -0
- supervisely/convert/image/pascal_voc/pascal_voc_converter.py +8 -5
- supervisely/convert/image/pascal_voc/pascal_voc_helper.py +7 -0
- supervisely/convert/pointcloud/kitti_3d/kitti_3d_converter.py +33 -3
- supervisely/convert/pointcloud/kitti_3d/kitti_3d_helper.py +12 -5
- supervisely/convert/pointcloud/las/las_converter.py +13 -1
- supervisely/convert/pointcloud/las/las_helper.py +110 -11
- supervisely/convert/pointcloud/nuscenes_conv/nuscenes_converter.py +27 -16
- supervisely/convert/pointcloud/pointcloud_converter.py +91 -3
- supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py +58 -22
- supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py +21 -47
- supervisely/convert/video/__init__.py +1 -0
- supervisely/convert/video/multi_view/__init__.py +0 -0
- supervisely/convert/video/multi_view/multi_view.py +543 -0
- supervisely/convert/video/sly/sly_video_converter.py +359 -3
- supervisely/convert/video/video_converter.py +22 -2
- supervisely/convert/volume/dicom/dicom_converter.py +13 -5
- supervisely/convert/volume/dicom/dicom_helper.py +30 -18
- supervisely/geometry/constants.py +1 -0
- supervisely/geometry/geometry.py +4 -0
- supervisely/geometry/helpers.py +5 -1
- supervisely/geometry/oriented_bbox.py +676 -0
- supervisely/geometry/rectangle.py +2 -1
- supervisely/io/env.py +76 -1
- supervisely/io/fs.py +21 -0
- supervisely/nn/benchmark/base_evaluator.py +104 -11
- supervisely/nn/benchmark/instance_segmentation/evaluator.py +1 -8
- supervisely/nn/benchmark/object_detection/evaluator.py +20 -4
- supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py +10 -5
- supervisely/nn/benchmark/semantic_segmentation/evaluator.py +34 -16
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/confusion_matrix.py +1 -1
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/frequently_confused.py +1 -1
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/overview.py +1 -1
- supervisely/nn/benchmark/visualization/evaluation_result.py +66 -4
- supervisely/nn/inference/cache.py +43 -18
- supervisely/nn/inference/gui/serving_gui_template.py +5 -2
- supervisely/nn/inference/inference.py +795 -199
- supervisely/nn/inference/inference_request.py +42 -9
- supervisely/nn/inference/predict_app/gui/classes_selector.py +83 -12
- supervisely/nn/inference/predict_app/gui/gui.py +676 -488
- supervisely/nn/inference/predict_app/gui/input_selector.py +205 -26
- supervisely/nn/inference/predict_app/gui/model_selector.py +2 -4
- supervisely/nn/inference/predict_app/gui/output_selector.py +46 -6
- supervisely/nn/inference/predict_app/gui/settings_selector.py +756 -59
- supervisely/nn/inference/predict_app/gui/tags_selector.py +1 -1
- supervisely/nn/inference/predict_app/gui/utils.py +236 -119
- supervisely/nn/inference/predict_app/predict_app.py +2 -2
- supervisely/nn/inference/session.py +43 -35
- supervisely/nn/inference/tracking/bbox_tracking.py +113 -34
- supervisely/nn/inference/tracking/tracker_interface.py +7 -2
- supervisely/nn/inference/uploader.py +139 -12
- supervisely/nn/live_training/__init__.py +7 -0
- supervisely/nn/live_training/api_server.py +111 -0
- supervisely/nn/live_training/artifacts_utils.py +243 -0
- supervisely/nn/live_training/checkpoint_utils.py +229 -0
- supervisely/nn/live_training/dynamic_sampler.py +44 -0
- supervisely/nn/live_training/helpers.py +14 -0
- supervisely/nn/live_training/incremental_dataset.py +146 -0
- supervisely/nn/live_training/live_training.py +497 -0
- supervisely/nn/live_training/loss_plateau_detector.py +111 -0
- supervisely/nn/live_training/request_queue.py +52 -0
- supervisely/nn/model/model_api.py +9 -0
- supervisely/nn/prediction_dto.py +12 -1
- supervisely/nn/tracker/base_tracker.py +11 -1
- supervisely/nn/tracker/botsort/botsort_config.yaml +0 -1
- supervisely/nn/tracker/botsort/tracker/mc_bot_sort.py +7 -4
- supervisely/nn/tracker/botsort_tracker.py +94 -65
- supervisely/nn/tracker/visualize.py +87 -90
- supervisely/nn/training/gui/classes_selector.py +16 -1
- supervisely/nn/training/train_app.py +28 -29
- supervisely/project/data_version.py +115 -51
- supervisely/project/download.py +1 -1
- supervisely/project/pointcloud_episode_project.py +37 -8
- supervisely/project/pointcloud_project.py +30 -2
- supervisely/project/project.py +14 -2
- supervisely/project/project_meta.py +27 -1
- supervisely/project/project_settings.py +32 -18
- supervisely/project/versioning/__init__.py +1 -0
- supervisely/project/versioning/common.py +20 -0
- supervisely/project/versioning/schema_fields.py +35 -0
- supervisely/project/versioning/video_schema.py +221 -0
- supervisely/project/versioning/volume_schema.py +87 -0
- supervisely/project/video_project.py +717 -15
- supervisely/project/volume_project.py +623 -5
- supervisely/template/experiment/experiment.html.jinja +4 -4
- supervisely/template/experiment/experiment_generator.py +14 -21
- supervisely/template/live_training/__init__.py +0 -0
- supervisely/template/live_training/header.html.jinja +96 -0
- supervisely/template/live_training/live_training.html.jinja +51 -0
- supervisely/template/live_training/live_training_generator.py +464 -0
- supervisely/template/live_training/sly-style.css +402 -0
- supervisely/template/live_training/template.html.jinja +18 -0
- supervisely/versions.json +28 -26
- supervisely/video/sampling.py +39 -20
- supervisely/video/video.py +40 -11
- supervisely/video_annotation/video_object.py +29 -4
- supervisely/volume/stl_converter.py +2 -0
- supervisely/worker_api/agent_rpc.py +24 -1
- supervisely/worker_api/rpc_servicer.py +31 -7
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info}/METADATA +56 -39
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info}/RECORD +189 -142
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info}/WHEEL +1 -1
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info/licenses}/LICENSE +0 -0
- {supervisely-6.73.452.dist-info → supervisely-6.73.513.dist-info}/top_level.txt +0 -0
|
@@ -14,7 +14,12 @@ from supervisely.api.module_api import ApiField, ModuleApiBase
|
|
|
14
14
|
from supervisely.api.project_api import ProjectInfo
|
|
15
15
|
from supervisely.io import json
|
|
16
16
|
from supervisely.io.fs import remove_dir, silent_remove
|
|
17
|
-
|
|
17
|
+
from supervisely.project.versioning.schema_fields import VersionSchemaField
|
|
18
|
+
from supervisely.project.versioning.common import (
|
|
19
|
+
DEFAULT_IMAGE_SCHEMA_VERSION,
|
|
20
|
+
DEFAULT_VIDEO_SCHEMA_VERSION,
|
|
21
|
+
DEFAULT_VOLUME_SCHEMA_VERSION,
|
|
22
|
+
)
|
|
18
23
|
|
|
19
24
|
class VersionInfo(NamedTuple):
|
|
20
25
|
"""
|
|
@@ -49,7 +54,7 @@ class DataVersion(ModuleApiBase):
|
|
|
49
54
|
|
|
50
55
|
self._api: Api = api
|
|
51
56
|
self.__storage_dir: str = "/system/versions/"
|
|
52
|
-
self.__version_format: str =
|
|
57
|
+
self.__version_format: str = DEFAULT_IMAGE_SCHEMA_VERSION
|
|
53
58
|
self.project_info = None
|
|
54
59
|
self.project_dir = None
|
|
55
60
|
self.versions_path = None
|
|
@@ -84,6 +89,31 @@ class DataVersion(ModuleApiBase):
|
|
|
84
89
|
"""
|
|
85
90
|
return "VersionInfo"
|
|
86
91
|
|
|
92
|
+
@property
|
|
93
|
+
def project_cls(self):
|
|
94
|
+
from supervisely.project import (
|
|
95
|
+
Project,
|
|
96
|
+
ProjectType,
|
|
97
|
+
VideoProject,
|
|
98
|
+
VolumeProject,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
if self.project_info is None:
|
|
102
|
+
raise ValueError("Project info is not initialized. Call 'initialize' method first.")
|
|
103
|
+
|
|
104
|
+
project_type = self.project_info.type
|
|
105
|
+
if project_type == ProjectType.IMAGES.value:
|
|
106
|
+
self.__version_format = DEFAULT_IMAGE_SCHEMA_VERSION
|
|
107
|
+
return Project
|
|
108
|
+
elif project_type == ProjectType.VIDEOS.value:
|
|
109
|
+
self.__version_format = DEFAULT_VIDEO_SCHEMA_VERSION
|
|
110
|
+
return VideoProject
|
|
111
|
+
elif project_type == ProjectType.VOLUMES.value:
|
|
112
|
+
self.__version_format = DEFAULT_VOLUME_SCHEMA_VERSION
|
|
113
|
+
return VolumeProject
|
|
114
|
+
else:
|
|
115
|
+
raise ValueError(f"Unsupported project type: {project_type}")
|
|
116
|
+
|
|
87
117
|
def initialize(self, project_info: Union[ProjectInfo, int]):
|
|
88
118
|
"""
|
|
89
119
|
Initialize project versions.
|
|
@@ -158,10 +188,10 @@ class DataVersion(ModuleApiBase):
|
|
|
158
188
|
versions = self._api.file.get_json_file_content(
|
|
159
189
|
self.project_info.team_id, self.versions_path
|
|
160
190
|
)
|
|
161
|
-
|
|
191
|
+
return versions or {}
|
|
162
192
|
except FileNotFoundError:
|
|
163
|
-
versions = {"format": self.__version_format}
|
|
164
|
-
|
|
193
|
+
# versions = {"format": self.__version_format}
|
|
194
|
+
return {}
|
|
165
195
|
|
|
166
196
|
def set_map(self, project_info: Union[ProjectInfo, int], initialize: bool = True):
|
|
167
197
|
"""
|
|
@@ -176,6 +206,8 @@ class DataVersion(ModuleApiBase):
|
|
|
176
206
|
|
|
177
207
|
if initialize:
|
|
178
208
|
self.initialize(project_info)
|
|
209
|
+
if "format" not in self.versions:
|
|
210
|
+
self.versions["format"] = self.__version_format
|
|
179
211
|
temp_dir = tempfile.mkdtemp()
|
|
180
212
|
local_versions = os.path.join(temp_dir, "versions.json")
|
|
181
213
|
json.dump_json_file(self.versions, local_versions)
|
|
@@ -329,21 +361,32 @@ class DataVersion(ModuleApiBase):
|
|
|
329
361
|
return reserve_info.get(ApiField.ID), reserve_info.get(ApiField.COMMIT_TOKEN)
|
|
330
362
|
|
|
331
363
|
except requests.exceptions.HTTPError as e:
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
364
|
+
details = {}
|
|
365
|
+
if e.response is not None:
|
|
366
|
+
try:
|
|
367
|
+
details = (e.response.json() or {}).get("details", {}) # type: ignore[union-attr]
|
|
368
|
+
except Exception:
|
|
369
|
+
details = {}
|
|
370
|
+
|
|
371
|
+
if details.get("useExistingVersion"):
|
|
372
|
+
version_id = details.get("version", {}).get("id")
|
|
373
|
+
version = details.get("version", {}).get("version")
|
|
335
374
|
logger.info(
|
|
336
375
|
f"No changes to the project since the last version '{version}' with ID '{version_id}'"
|
|
337
376
|
)
|
|
338
377
|
return (None, None)
|
|
339
|
-
|
|
378
|
+
|
|
379
|
+
message = (details.get("message") or "").lower()
|
|
380
|
+
if "is already committing" in message:
|
|
340
381
|
if retry_delay >= max_delay:
|
|
341
382
|
raise RuntimeError(
|
|
342
383
|
"Failed to reserve version. Another process is already committing a version. Maximum number of attempts reached."
|
|
343
384
|
)
|
|
344
|
-
version = e.response.json().get("details", {}).get("version").get("version")
|
|
345
385
|
time.sleep(retry_delay)
|
|
346
386
|
retry_delay *= 2
|
|
387
|
+
continue
|
|
388
|
+
|
|
389
|
+
raise
|
|
347
390
|
|
|
348
391
|
def cancel_reservation(self, version_id: int, commit_token: str):
|
|
349
392
|
"""
|
|
@@ -384,8 +427,6 @@ class DataVersion(ModuleApiBase):
|
|
|
384
427
|
:return: ProjectInfo object of the restored project
|
|
385
428
|
:rtype: ProjectInfo or None
|
|
386
429
|
"""
|
|
387
|
-
from supervisely.project.project import Project
|
|
388
|
-
|
|
389
430
|
if version_id is None and version_num is None:
|
|
390
431
|
raise ValueError("Either version_id or version_num must be provided")
|
|
391
432
|
|
|
@@ -421,7 +462,7 @@ class DataVersion(ModuleApiBase):
|
|
|
421
462
|
return
|
|
422
463
|
|
|
423
464
|
bin_io = self._download_and_extract(backup_files)
|
|
424
|
-
new_project_info =
|
|
465
|
+
new_project_info = self.project_cls.upload_bin(
|
|
425
466
|
self._api,
|
|
426
467
|
bin_io,
|
|
427
468
|
self.project_info.workspace_id,
|
|
@@ -456,15 +497,28 @@ class DataVersion(ModuleApiBase):
|
|
|
456
497
|
local_path = os.path.join(temp_dir, "download.tar.zst")
|
|
457
498
|
try:
|
|
458
499
|
self._api.file.download(self.project_info.team_id, path, local_path)
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
500
|
+
# Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
|
|
501
|
+
try:
|
|
502
|
+
dctx = zstd.ZstdDecompressor()
|
|
503
|
+
with open(local_path, "rb") as zst_f:
|
|
504
|
+
with dctx.stream_reader(zst_f) as reader:
|
|
505
|
+
with tarfile.open(fileobj=reader, mode="r|") as tar:
|
|
506
|
+
for member in tar:
|
|
507
|
+
if member.name == "version.bin":
|
|
508
|
+
file = tar.extractfile(member)
|
|
509
|
+
if not file:
|
|
510
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
511
|
+
return io.BytesIO(file.read())
|
|
512
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
513
|
+
except Exception:
|
|
514
|
+
# Fallback: one-shot decompress
|
|
515
|
+
with open(local_path, "rb") as zst_f:
|
|
516
|
+
decompressed_data = zstd.decompress(zst_f.read())
|
|
517
|
+
with tarfile.open(fileobj=io.BytesIO(decompressed_data), mode="r") as tar:
|
|
518
|
+
file = tar.extractfile("version.bin")
|
|
519
|
+
if not file:
|
|
520
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
521
|
+
return io.BytesIO(file.read())
|
|
468
522
|
except Exception as e:
|
|
469
523
|
raise RuntimeError(f"Failed to extract version: {e}")
|
|
470
524
|
finally:
|
|
@@ -501,34 +555,44 @@ class DataVersion(ModuleApiBase):
|
|
|
501
555
|
:return: File info
|
|
502
556
|
:rtype: dict
|
|
503
557
|
"""
|
|
504
|
-
from supervisely.project.project import Project
|
|
505
|
-
|
|
506
558
|
temp_dir = tempfile.mkdtemp()
|
|
559
|
+
data = None
|
|
560
|
+
try:
|
|
561
|
+
data = self.project_cls.download_bin(
|
|
562
|
+
self._api, self.project_info.id, batch_size=200, return_bytesio=True
|
|
563
|
+
)
|
|
564
|
+
info = tarfile.TarInfo(name="version.bin")
|
|
565
|
+
data.seek(0, io.SEEK_END)
|
|
566
|
+
info.size = data.tell()
|
|
567
|
+
data.seek(0)
|
|
568
|
+
zst_archive_path = os.path.join(temp_dir, "download.tar.zst")
|
|
507
569
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
570
|
+
# Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
|
|
571
|
+
try:
|
|
572
|
+
cctx = zstd.ZstdCompressor()
|
|
573
|
+
with open(zst_archive_path, "wb") as zst_f:
|
|
574
|
+
try:
|
|
575
|
+
stream = cctx.stream_writer(zst_f, closefd=False)
|
|
576
|
+
except TypeError:
|
|
577
|
+
stream = cctx.stream_writer(zst_f)
|
|
578
|
+
with stream as compressor:
|
|
579
|
+
with tarfile.open(fileobj=compressor, mode="w|") as tar:
|
|
580
|
+
tar.addfile(tarinfo=info, fileobj=data)
|
|
581
|
+
except Exception:
|
|
582
|
+
# Fallback: build tar in memory + one-shot compress
|
|
583
|
+
tar_data = io.BytesIO()
|
|
584
|
+
with tarfile.open(fileobj=tar_data, mode="w") as tar:
|
|
585
|
+
tar.addfile(tarinfo=info, fileobj=data)
|
|
586
|
+
tar_data.seek(0)
|
|
587
|
+
with open(zst_archive_path, "wb") as zst_f:
|
|
588
|
+
zst_f.write(zstd.compress(tar_data.read()))
|
|
589
|
+
|
|
590
|
+
file_info = self._api.file.upload(self.project_info.team_id, zst_archive_path, path)
|
|
591
|
+
return file_info
|
|
592
|
+
finally:
|
|
593
|
+
if data is not None:
|
|
594
|
+
try:
|
|
595
|
+
data.close()
|
|
596
|
+
except Exception:
|
|
597
|
+
pass
|
|
598
|
+
remove_dir(temp_dir)
|
supervisely/project/download.py
CHANGED
|
@@ -216,7 +216,7 @@ def download_async_or_sync(
|
|
|
216
216
|
dataset_ids: Optional[List[int]] = None,
|
|
217
217
|
log_progress: bool = True,
|
|
218
218
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
219
|
-
semaphore: Optional[asyncio.Semaphore] = None,
|
|
219
|
+
semaphore: Optional[Union[asyncio.Semaphore, int]] = None,
|
|
220
220
|
**kwargs,
|
|
221
221
|
):
|
|
222
222
|
"""
|
|
@@ -655,6 +655,25 @@ class PointcloudEpisodeProject(PointcloudProject):
|
|
|
655
655
|
f"Static method 'download_async()' is not supported for PointcloudEpisodeProject class now."
|
|
656
656
|
)
|
|
657
657
|
|
|
658
|
+
# ----------------------------------- #
|
|
659
|
+
# Pointcloud Episodes Data Versioning #
|
|
660
|
+
# ----------------------------------- #
|
|
661
|
+
@staticmethod
|
|
662
|
+
def download_bin(*args, **kwargs):
|
|
663
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
664
|
+
|
|
665
|
+
@staticmethod
|
|
666
|
+
def upload_bin(*args, **kwargs):
|
|
667
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
668
|
+
|
|
669
|
+
@staticmethod
|
|
670
|
+
def build_snapshot(*args, **kwargs):
|
|
671
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
672
|
+
|
|
673
|
+
@staticmethod
|
|
674
|
+
def restore_snapshot(*args, **kwargs):
|
|
675
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
676
|
+
|
|
658
677
|
|
|
659
678
|
def download_pointcloud_episode_project(
|
|
660
679
|
api: Api,
|
|
@@ -946,7 +965,7 @@ def upload_pointcloud_episode_project(
|
|
|
946
965
|
log_progress: bool = True,
|
|
947
966
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
948
967
|
) -> Tuple[int, str]:
|
|
949
|
-
# STEP 0
|
|
968
|
+
# STEP 0 - create project remotely
|
|
950
969
|
project_fs = PointcloudEpisodeProject.read_single(directory)
|
|
951
970
|
project_name = project_fs.name if project_name is None else project_name
|
|
952
971
|
|
|
@@ -959,8 +978,9 @@ def upload_pointcloud_episode_project(
|
|
|
959
978
|
if progress_cb is not None:
|
|
960
979
|
log_progress = False
|
|
961
980
|
|
|
981
|
+
name_to_dsinfo = {}
|
|
962
982
|
key_id_map = KeyIdMap()
|
|
963
|
-
for dataset_fs in project_fs.datasets:
|
|
983
|
+
for dataset_fs in sorted(project_fs.datasets, key=lambda ds: len(ds.parents)):
|
|
964
984
|
dataset_fs: PointcloudEpisodeDataset
|
|
965
985
|
ann_json_path = dataset_fs.get_ann_path()
|
|
966
986
|
|
|
@@ -970,14 +990,19 @@ def upload_pointcloud_episode_project(
|
|
|
970
990
|
else:
|
|
971
991
|
episode_annotation = PointcloudEpisodeAnnotation()
|
|
972
992
|
|
|
993
|
+
parent_path = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
|
|
994
|
+
parent_info = name_to_dsinfo.get(parent_path)
|
|
995
|
+
parent_id = parent_info.id if parent_info else None
|
|
973
996
|
dataset = api.dataset.create(
|
|
974
997
|
project.id,
|
|
975
|
-
dataset_fs.
|
|
998
|
+
dataset_fs.short_name,
|
|
976
999
|
description=episode_annotation.description,
|
|
977
1000
|
change_name_if_conflict=True,
|
|
1001
|
+
parent_id=parent_id,
|
|
978
1002
|
)
|
|
1003
|
+
name_to_dsinfo[dataset_fs.name] = dataset
|
|
979
1004
|
|
|
980
|
-
# STEP 1
|
|
1005
|
+
# STEP 1 - upload episodes
|
|
981
1006
|
items_infos = {"names": [], "paths": [], "metas": []}
|
|
982
1007
|
|
|
983
1008
|
for item_name in dataset_fs:
|
|
@@ -989,6 +1014,10 @@ def upload_pointcloud_episode_project(
|
|
|
989
1014
|
items_infos["paths"].append(item_path)
|
|
990
1015
|
items_infos["metas"].append(item_meta)
|
|
991
1016
|
|
|
1017
|
+
if not items_infos["names"]:
|
|
1018
|
+
logger.info(f"Dataset {dataset.name} has no items, skipping upload")
|
|
1019
|
+
continue
|
|
1020
|
+
|
|
992
1021
|
ds_progress = progress_cb
|
|
993
1022
|
if log_progress:
|
|
994
1023
|
ds_progress = tqdm_sly(
|
|
@@ -1015,7 +1044,7 @@ def upload_pointcloud_episode_project(
|
|
|
1015
1044
|
},
|
|
1016
1045
|
)
|
|
1017
1046
|
raise e
|
|
1018
|
-
# STEP 2
|
|
1047
|
+
# STEP 2 - upload annotations
|
|
1019
1048
|
frame_to_pcl_ids = {pcl_info.frame: pcl_info.id for pcl_info in pcl_infos}
|
|
1020
1049
|
try:
|
|
1021
1050
|
api.pointcloud_episode.annotation.append(
|
|
@@ -1033,9 +1062,9 @@ def upload_pointcloud_episode_project(
|
|
|
1033
1062
|
)
|
|
1034
1063
|
raise e
|
|
1035
1064
|
|
|
1036
|
-
# STEP 3
|
|
1065
|
+
# STEP 3 - upload photo context
|
|
1037
1066
|
img_infos = {"img_paths": [], "img_metas": []}
|
|
1038
|
-
# STEP 3.1
|
|
1067
|
+
# STEP 3.1 - upload images
|
|
1039
1068
|
pcl_to_rimg_figures: Dict[int, Dict[str, List[Dict]]] = {}
|
|
1040
1069
|
pcl_to_hash_to_id: Dict[int, Dict[str, int]] = {}
|
|
1041
1070
|
for pcl_info in pcl_infos:
|
|
@@ -1067,7 +1096,7 @@ def upload_pointcloud_episode_project(
|
|
|
1067
1096
|
)
|
|
1068
1097
|
raise e
|
|
1069
1098
|
|
|
1070
|
-
# STEP 3.2
|
|
1099
|
+
# STEP 3.2 - upload images metas
|
|
1071
1100
|
images_hashes_iterator = images_hashes.__iter__()
|
|
1072
1101
|
for pcl_info in pcl_infos:
|
|
1073
1102
|
related_items = dataset_fs.get_related_images(pcl_info.name)
|
|
@@ -908,6 +908,25 @@ class PointcloudProject(VideoProject):
|
|
|
908
908
|
f"Static method 'download_async()' is not supported for PointcloudProject class now."
|
|
909
909
|
)
|
|
910
910
|
|
|
911
|
+
# -------------------------- #
|
|
912
|
+
# Pointcloud Data Versioning #
|
|
913
|
+
# -------------------------- #
|
|
914
|
+
@staticmethod
|
|
915
|
+
def download_bin(*args, **kwargs):
|
|
916
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
917
|
+
|
|
918
|
+
@staticmethod
|
|
919
|
+
def upload_bin(*args, **kwargs):
|
|
920
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
921
|
+
|
|
922
|
+
@staticmethod
|
|
923
|
+
def build_snapshot(*args, **kwargs):
|
|
924
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
925
|
+
|
|
926
|
+
@staticmethod
|
|
927
|
+
def restore_snapshot(*args, **kwargs):
|
|
928
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
929
|
+
|
|
911
930
|
|
|
912
931
|
def download_pointcloud_project(
|
|
913
932
|
api: Api,
|
|
@@ -1216,8 +1235,17 @@ def upload_pointcloud_project(
|
|
|
1216
1235
|
log_progress = False
|
|
1217
1236
|
|
|
1218
1237
|
key_id_map = KeyIdMap()
|
|
1219
|
-
|
|
1220
|
-
|
|
1238
|
+
name_to_dsinfo = {}
|
|
1239
|
+
for dataset_fs in sorted(project_fs, key=lambda ds: len(ds.parents)):
|
|
1240
|
+
parent_name = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
|
|
1241
|
+
parent_info = name_to_dsinfo.get(parent_name)
|
|
1242
|
+
parent_id = None
|
|
1243
|
+
if parent_info is not None:
|
|
1244
|
+
parent_id = parent_info.id
|
|
1245
|
+
dataset = api.dataset.create(
|
|
1246
|
+
project.id, dataset_fs.short_name, change_name_if_conflict=True, parent_id=parent_id
|
|
1247
|
+
)
|
|
1248
|
+
name_to_dsinfo[dataset_fs.name] = dataset
|
|
1221
1249
|
|
|
1222
1250
|
ds_progress = progress_cb
|
|
1223
1251
|
if log_progress:
|
supervisely/project/project.py
CHANGED
|
@@ -3626,7 +3626,11 @@ class Project:
|
|
|
3626
3626
|
if project_name is None:
|
|
3627
3627
|
project_name = project_info.name
|
|
3628
3628
|
new_project_info = api.project.create(
|
|
3629
|
-
workspace_id,
|
|
3629
|
+
workspace_id,
|
|
3630
|
+
project_name,
|
|
3631
|
+
description=project_info.description,
|
|
3632
|
+
change_name_if_conflict=True,
|
|
3633
|
+
readme=project_info.readme,
|
|
3630
3634
|
)
|
|
3631
3635
|
custom_data = new_project_info.custom_data
|
|
3632
3636
|
version_num = project_info.version.get("version", None) if project_info.version else 0
|
|
@@ -4584,6 +4588,7 @@ def upload_project(
|
|
|
4584
4588
|
blob_file_infos = []
|
|
4585
4589
|
|
|
4586
4590
|
for ds_fs in project_fs.datasets:
|
|
4591
|
+
logger.debug(f"Processing dataset: {ds_fs.name}")
|
|
4587
4592
|
if len(ds_fs.parents) > 0:
|
|
4588
4593
|
parent = f"{os.path.sep}".join(ds_fs.parents)
|
|
4589
4594
|
parent_id = dataset_map.get(parent)
|
|
@@ -4624,8 +4629,15 @@ def upload_project(
|
|
|
4624
4629
|
if os.path.isfile(path):
|
|
4625
4630
|
valid_indices.append(i)
|
|
4626
4631
|
valid_paths.append(path)
|
|
4627
|
-
|
|
4632
|
+
elif len(project_fs.blob_files) > 0:
|
|
4628
4633
|
offset_indices.append(i)
|
|
4634
|
+
else:
|
|
4635
|
+
if img_infos[i] is not None:
|
|
4636
|
+
logger.debug(f"Image will be uploaded by image_info: {names[i]}")
|
|
4637
|
+
else:
|
|
4638
|
+
logger.warning(
|
|
4639
|
+
f"Image and image info file not found, image will be skipped: {names[i]}"
|
|
4640
|
+
)
|
|
4629
4641
|
img_paths = valid_paths
|
|
4630
4642
|
ann_paths = list(filter(lambda x: os.path.isfile(x), ann_paths))
|
|
4631
4643
|
# Create a mapping from name to index position for quick lookups
|
|
@@ -14,7 +14,7 @@ from supervisely.geometry.bitmap import Bitmap
|
|
|
14
14
|
from supervisely.geometry.polygon import Polygon
|
|
15
15
|
from supervisely.geometry.rectangle import Rectangle
|
|
16
16
|
from supervisely.io.json import JsonSerializable
|
|
17
|
-
from supervisely.project.project_settings import ProjectSettings
|
|
17
|
+
from supervisely.project.project_settings import LabelingInterface, ProjectSettings
|
|
18
18
|
from supervisely.project.project_type import ProjectType
|
|
19
19
|
|
|
20
20
|
|
|
@@ -288,6 +288,32 @@ class ProjectMeta(JsonSerializable):
|
|
|
288
288
|
# Output: <class 'supervisely.project.project_settings.ProjectSettings'>
|
|
289
289
|
"""
|
|
290
290
|
return self._project_settings
|
|
291
|
+
|
|
292
|
+
@property
|
|
293
|
+
def labeling_interface(self) -> Optional[LabelingInterface]:
|
|
294
|
+
"""
|
|
295
|
+
Get labeling interface settings of the project.
|
|
296
|
+
|
|
297
|
+
:return: Labeling interface settings
|
|
298
|
+
:rtype: :class: `LabelingInterface` or None
|
|
299
|
+
:Usage example:
|
|
300
|
+
|
|
301
|
+
.. code-block:: python
|
|
302
|
+
|
|
303
|
+
import supervisely as sly
|
|
304
|
+
|
|
305
|
+
s = sly.ProjectSettings(
|
|
306
|
+
multiview_enabled=True,
|
|
307
|
+
multiview_tag_name='multi_tag',
|
|
308
|
+
multiview_is_synced=False,
|
|
309
|
+
)
|
|
310
|
+
meta = sly.ProjectMeta(project_settings=s)
|
|
311
|
+
|
|
312
|
+
labeling_interface = meta.labeling_interface
|
|
313
|
+
print(labeling_interface)
|
|
314
|
+
# Output: None
|
|
315
|
+
"""
|
|
316
|
+
return self._project_settings.labeling_interface if self._project_settings else None
|
|
291
317
|
|
|
292
318
|
def to_json(self) -> Dict:
|
|
293
319
|
"""
|
|
@@ -10,6 +10,7 @@ from supervisely._utils import take_with_default
|
|
|
10
10
|
from supervisely.annotation.tag_meta import TagValueType
|
|
11
11
|
from supervisely.collection.str_enum import StrEnum
|
|
12
12
|
from supervisely.io.json import JsonSerializable
|
|
13
|
+
from supervisely.project.project_type import ProjectType
|
|
13
14
|
from supervisely.sly_logger import logger
|
|
14
15
|
|
|
15
16
|
|
|
@@ -184,31 +185,44 @@ class ProjectSettings(JsonSerializable):
|
|
|
184
185
|
|
|
185
186
|
def validate(self, meta):
|
|
186
187
|
if meta.project_settings.multiview_enabled is True:
|
|
187
|
-
|
|
188
|
-
if
|
|
189
|
-
|
|
190
|
-
return # (tag_name, tag_id) == (None, None) is OK
|
|
191
|
-
mtag_name = meta.get_tag_name_by_id(meta.project_settings.multiview_tag_id)
|
|
188
|
+
# Images multiview
|
|
189
|
+
if meta.project_type == ProjectType.IMAGES:
|
|
190
|
+
mtag_name = meta.project_settings.multiview_tag_name
|
|
192
191
|
if mtag_name is None:
|
|
192
|
+
if meta.project_settings.multiview_tag_id is None:
|
|
193
|
+
return # (tag_name, tag_id) == (None, None) is OK
|
|
194
|
+
mtag_name = meta.get_tag_name_by_id(meta.project_settings.multiview_tag_id)
|
|
195
|
+
if mtag_name is None:
|
|
196
|
+
raise RuntimeError(
|
|
197
|
+
f"The multi-view tag with ID={meta.project_settings.multiview_tag_id} was not found in the project meta. "
|
|
198
|
+
"Please directly add the tag meta that will be used for image grouping for multi-view labeling interface."
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
multi_tag = meta.get_tag_meta(mtag_name)
|
|
202
|
+
if multi_tag is None:
|
|
193
203
|
raise RuntimeError(
|
|
194
|
-
f"The multi-view tag
|
|
195
|
-
"
|
|
204
|
+
f"The multi-view tag '{mtag_name}' was not found in the project meta. Please directly add the tag meta "
|
|
205
|
+
"that will be used for image grouping for multi-view labeling interface."
|
|
206
|
+
)
|
|
207
|
+
elif multi_tag.value_type != TagValueType.ANY_STRING:
|
|
208
|
+
raise RuntimeError(
|
|
209
|
+
f"The multi-view tag value type should be '{TagValueType.ANY_STRING}'. The provided type: '{multi_tag.value_type}'."
|
|
196
210
|
)
|
|
197
211
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
212
|
+
# Video multiview
|
|
213
|
+
elif meta.project_type == ProjectType.VIDEOS:
|
|
214
|
+
if (
|
|
215
|
+
meta.project_settings.multiview_tag_name is not None
|
|
216
|
+
or meta.project_settings.multiview_tag_id is not None
|
|
217
|
+
):
|
|
218
|
+
raise RuntimeError(
|
|
219
|
+
"For video projects, multiview_tag_name and multiview_tag_id should be None. "
|
|
220
|
+
"Videos are grouped by datasets, not by tags."
|
|
221
|
+
)
|
|
208
222
|
|
|
209
223
|
if meta.project_settings.labeling_interface is not None:
|
|
210
224
|
if meta.project_settings.labeling_interface not in LabelingInterface.values():
|
|
211
225
|
raise RuntimeError(
|
|
212
226
|
f"Invalid labeling interface value: {meta.project_settings.labeling_interface}. "
|
|
213
227
|
f"The available values: {LabelingInterface.values()}"
|
|
214
|
-
)
|
|
228
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from supervisely.project.versioning.schema_fields import VersionSchemaField
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from supervisely.project.versioning.video_schema import VideoSnapshotSchema, _VIDEO_SCHEMAS
|
|
2
|
+
from supervisely.project.versioning.volume_schema import VolumeSnapshotSchema, _VOLUME_SCHEMAS
|
|
3
|
+
|
|
4
|
+
DEFAULT_IMAGE_SCHEMA_VERSION = "v1.0.0"
|
|
5
|
+
DEFAULT_VOLUME_SCHEMA_VERSION = "v2.0.0"
|
|
6
|
+
DEFAULT_VIDEO_SCHEMA_VERSION = "v2.0.0"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_video_snapshot_schema(schema_version: str) -> VideoSnapshotSchema:
|
|
10
|
+
schema = _VIDEO_SCHEMAS.get(schema_version)
|
|
11
|
+
if schema is None:
|
|
12
|
+
raise RuntimeError(f"Unsupported video snapshot schema_version: {schema_version!r}")
|
|
13
|
+
return schema
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_volume_snapshot_schema(schema_version: str) -> VolumeSnapshotSchema:
|
|
17
|
+
schema = _VOLUME_SCHEMAS.get(schema_version)
|
|
18
|
+
if schema is None:
|
|
19
|
+
raise RuntimeError(f"Unsupported volume snapshot schema_version: {schema_version!r}")
|
|
20
|
+
return schema
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
class VersionSchemaField:
|
|
2
|
+
"""String constants used as column names across snapshot table schemas."""
|
|
3
|
+
|
|
4
|
+
SCHEMA_VERSION = "schema_version"
|
|
5
|
+
TABLES = "tables"
|
|
6
|
+
NAME = "name"
|
|
7
|
+
PATH = "path"
|
|
8
|
+
ROW_COUNT = "row_count"
|
|
9
|
+
SRC_DATASET_ID = "src_dataset_id"
|
|
10
|
+
PARENT_SRC_DATASET_ID = "parent_src_dataset_id"
|
|
11
|
+
FULL_PATH = "full_path"
|
|
12
|
+
DESCRIPTION = "description"
|
|
13
|
+
SRC_VIDEO_ID = "src_video_id"
|
|
14
|
+
SRC_VOLUME_ID = "src_volume_id"
|
|
15
|
+
HASH = "hash"
|
|
16
|
+
LINK = "link"
|
|
17
|
+
FRAMES_COUNT = "frames_count"
|
|
18
|
+
FRAME_WIDTH = "frame_width"
|
|
19
|
+
FRAME_HEIGHT = "frame_height"
|
|
20
|
+
FRAMES_TO_TIMECODES = "frames_to_timecodes"
|
|
21
|
+
META = "meta"
|
|
22
|
+
CUSTOM_DATA = "custom_data"
|
|
23
|
+
CREATED_AT = "created_at"
|
|
24
|
+
UPDATED_AT = "updated_at"
|
|
25
|
+
ANN_JSON = "ann_json"
|
|
26
|
+
JSON = "json"
|
|
27
|
+
ANNOTATION = "annotation"
|
|
28
|
+
SRC_OBJECT_ID = "src_object_id"
|
|
29
|
+
CLASS_NAME = "class_name"
|
|
30
|
+
KEY = "key"
|
|
31
|
+
TAGS_JSON = "tags_json"
|
|
32
|
+
SRC_FIGURE_ID = "src_figure_id"
|
|
33
|
+
FRAME_INDEX = "frame_index"
|
|
34
|
+
GEOMETRY_TYPE = "geometry_type"
|
|
35
|
+
GEOMETRY_JSON = "geometry_json"
|