supervisely 6.73.438__py3-none-any.whl → 6.73.513__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- supervisely/__init__.py +137 -1
- supervisely/_utils.py +81 -0
- supervisely/annotation/annotation.py +8 -2
- supervisely/annotation/json_geometries_map.py +14 -11
- supervisely/annotation/label.py +80 -3
- supervisely/api/annotation_api.py +14 -11
- supervisely/api/api.py +59 -38
- supervisely/api/app_api.py +11 -2
- supervisely/api/dataset_api.py +74 -12
- supervisely/api/entities_collection_api.py +10 -0
- supervisely/api/entity_annotation/figure_api.py +52 -4
- supervisely/api/entity_annotation/object_api.py +3 -3
- supervisely/api/entity_annotation/tag_api.py +63 -12
- supervisely/api/guides_api.py +210 -0
- supervisely/api/image_api.py +72 -1
- supervisely/api/labeling_job_api.py +83 -1
- supervisely/api/labeling_queue_api.py +33 -7
- supervisely/api/module_api.py +9 -0
- supervisely/api/project_api.py +71 -26
- supervisely/api/storage_api.py +3 -1
- supervisely/api/task_api.py +13 -2
- supervisely/api/team_api.py +4 -3
- supervisely/api/video/video_annotation_api.py +119 -3
- supervisely/api/video/video_api.py +65 -14
- supervisely/api/video/video_figure_api.py +24 -11
- supervisely/app/__init__.py +1 -1
- supervisely/app/content.py +23 -7
- supervisely/app/development/development.py +18 -2
- supervisely/app/fastapi/__init__.py +1 -0
- supervisely/app/fastapi/custom_static_files.py +1 -1
- supervisely/app/fastapi/multi_user.py +105 -0
- supervisely/app/fastapi/subapp.py +88 -42
- supervisely/app/fastapi/websocket.py +77 -9
- supervisely/app/singleton.py +21 -0
- supervisely/app/v1/app_service.py +18 -2
- supervisely/app/v1/constants.py +7 -1
- supervisely/app/widgets/__init__.py +6 -0
- supervisely/app/widgets/activity_feed/__init__.py +0 -0
- supervisely/app/widgets/activity_feed/activity_feed.py +239 -0
- supervisely/app/widgets/activity_feed/style.css +78 -0
- supervisely/app/widgets/activity_feed/template.html +22 -0
- supervisely/app/widgets/card/card.py +20 -0
- supervisely/app/widgets/classes_list_selector/classes_list_selector.py +121 -9
- supervisely/app/widgets/classes_list_selector/template.html +60 -93
- supervisely/app/widgets/classes_mapping/classes_mapping.py +13 -12
- supervisely/app/widgets/classes_table/classes_table.py +1 -0
- supervisely/app/widgets/deploy_model/deploy_model.py +56 -35
- supervisely/app/widgets/dialog/dialog.py +12 -0
- supervisely/app/widgets/dialog/template.html +2 -1
- supervisely/app/widgets/ecosystem_model_selector/ecosystem_model_selector.py +1 -1
- supervisely/app/widgets/experiment_selector/experiment_selector.py +8 -0
- supervisely/app/widgets/fast_table/fast_table.py +184 -60
- supervisely/app/widgets/fast_table/template.html +1 -1
- supervisely/app/widgets/heatmap/__init__.py +0 -0
- supervisely/app/widgets/heatmap/heatmap.py +564 -0
- supervisely/app/widgets/heatmap/script.js +533 -0
- supervisely/app/widgets/heatmap/style.css +233 -0
- supervisely/app/widgets/heatmap/template.html +21 -0
- supervisely/app/widgets/modal/__init__.py +0 -0
- supervisely/app/widgets/modal/modal.py +198 -0
- supervisely/app/widgets/modal/template.html +10 -0
- supervisely/app/widgets/object_class_view/object_class_view.py +3 -0
- supervisely/app/widgets/radio_tabs/radio_tabs.py +18 -2
- supervisely/app/widgets/radio_tabs/template.html +1 -0
- supervisely/app/widgets/select/select.py +6 -3
- supervisely/app/widgets/select_class/__init__.py +0 -0
- supervisely/app/widgets/select_class/select_class.py +363 -0
- supervisely/app/widgets/select_class/template.html +50 -0
- supervisely/app/widgets/select_cuda/select_cuda.py +22 -0
- supervisely/app/widgets/select_dataset_tree/select_dataset_tree.py +65 -7
- supervisely/app/widgets/select_tag/__init__.py +0 -0
- supervisely/app/widgets/select_tag/select_tag.py +352 -0
- supervisely/app/widgets/select_tag/template.html +64 -0
- supervisely/app/widgets/select_team/select_team.py +37 -4
- supervisely/app/widgets/select_team/template.html +4 -5
- supervisely/app/widgets/select_user/__init__.py +0 -0
- supervisely/app/widgets/select_user/select_user.py +270 -0
- supervisely/app/widgets/select_user/template.html +13 -0
- supervisely/app/widgets/select_workspace/select_workspace.py +59 -10
- supervisely/app/widgets/select_workspace/template.html +9 -12
- supervisely/app/widgets/table/table.py +68 -13
- supervisely/app/widgets/tree_select/tree_select.py +2 -0
- supervisely/aug/aug.py +6 -2
- supervisely/convert/base_converter.py +1 -0
- supervisely/convert/converter.py +2 -2
- supervisely/convert/image/csv/csv_converter.py +24 -15
- supervisely/convert/image/image_converter.py +3 -1
- supervisely/convert/image/image_helper.py +48 -4
- supervisely/convert/image/label_studio/label_studio_converter.py +2 -0
- supervisely/convert/image/medical2d/medical2d_helper.py +2 -24
- supervisely/convert/image/multispectral/multispectral_converter.py +6 -0
- supervisely/convert/image/pascal_voc/pascal_voc_converter.py +8 -5
- supervisely/convert/image/pascal_voc/pascal_voc_helper.py +7 -0
- supervisely/convert/pointcloud/kitti_3d/kitti_3d_converter.py +33 -3
- supervisely/convert/pointcloud/kitti_3d/kitti_3d_helper.py +12 -5
- supervisely/convert/pointcloud/las/las_converter.py +13 -1
- supervisely/convert/pointcloud/las/las_helper.py +110 -11
- supervisely/convert/pointcloud/nuscenes_conv/nuscenes_converter.py +27 -16
- supervisely/convert/pointcloud/pointcloud_converter.py +91 -3
- supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py +58 -22
- supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py +21 -47
- supervisely/convert/video/__init__.py +1 -0
- supervisely/convert/video/multi_view/__init__.py +0 -0
- supervisely/convert/video/multi_view/multi_view.py +543 -0
- supervisely/convert/video/sly/sly_video_converter.py +359 -3
- supervisely/convert/video/video_converter.py +24 -4
- supervisely/convert/volume/dicom/dicom_converter.py +13 -5
- supervisely/convert/volume/dicom/dicom_helper.py +30 -18
- supervisely/geometry/constants.py +1 -0
- supervisely/geometry/geometry.py +4 -0
- supervisely/geometry/helpers.py +5 -1
- supervisely/geometry/oriented_bbox.py +676 -0
- supervisely/geometry/polyline_3d.py +110 -0
- supervisely/geometry/rectangle.py +2 -1
- supervisely/io/env.py +76 -1
- supervisely/io/fs.py +21 -0
- supervisely/nn/benchmark/base_evaluator.py +104 -11
- supervisely/nn/benchmark/instance_segmentation/evaluator.py +1 -8
- supervisely/nn/benchmark/object_detection/evaluator.py +20 -4
- supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py +10 -5
- supervisely/nn/benchmark/semantic_segmentation/evaluator.py +34 -16
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/confusion_matrix.py +1 -1
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/frequently_confused.py +1 -1
- supervisely/nn/benchmark/semantic_segmentation/vis_metrics/overview.py +1 -1
- supervisely/nn/benchmark/visualization/evaluation_result.py +66 -4
- supervisely/nn/inference/cache.py +43 -18
- supervisely/nn/inference/gui/serving_gui_template.py +5 -2
- supervisely/nn/inference/inference.py +916 -222
- supervisely/nn/inference/inference_request.py +55 -10
- supervisely/nn/inference/predict_app/gui/classes_selector.py +83 -12
- supervisely/nn/inference/predict_app/gui/gui.py +676 -488
- supervisely/nn/inference/predict_app/gui/input_selector.py +205 -26
- supervisely/nn/inference/predict_app/gui/model_selector.py +2 -4
- supervisely/nn/inference/predict_app/gui/output_selector.py +46 -6
- supervisely/nn/inference/predict_app/gui/settings_selector.py +756 -59
- supervisely/nn/inference/predict_app/gui/tags_selector.py +1 -1
- supervisely/nn/inference/predict_app/gui/utils.py +236 -119
- supervisely/nn/inference/predict_app/predict_app.py +2 -2
- supervisely/nn/inference/session.py +43 -35
- supervisely/nn/inference/tracking/bbox_tracking.py +118 -35
- supervisely/nn/inference/tracking/point_tracking.py +5 -1
- supervisely/nn/inference/tracking/tracker_interface.py +10 -1
- supervisely/nn/inference/uploader.py +139 -12
- supervisely/nn/live_training/__init__.py +7 -0
- supervisely/nn/live_training/api_server.py +111 -0
- supervisely/nn/live_training/artifacts_utils.py +243 -0
- supervisely/nn/live_training/checkpoint_utils.py +229 -0
- supervisely/nn/live_training/dynamic_sampler.py +44 -0
- supervisely/nn/live_training/helpers.py +14 -0
- supervisely/nn/live_training/incremental_dataset.py +146 -0
- supervisely/nn/live_training/live_training.py +497 -0
- supervisely/nn/live_training/loss_plateau_detector.py +111 -0
- supervisely/nn/live_training/request_queue.py +52 -0
- supervisely/nn/model/model_api.py +9 -0
- supervisely/nn/model/prediction.py +2 -1
- supervisely/nn/model/prediction_session.py +26 -14
- supervisely/nn/prediction_dto.py +19 -1
- supervisely/nn/tracker/base_tracker.py +11 -1
- supervisely/nn/tracker/botsort/botsort_config.yaml +0 -1
- supervisely/nn/tracker/botsort/tracker/mc_bot_sort.py +7 -4
- supervisely/nn/tracker/botsort_tracker.py +94 -65
- supervisely/nn/tracker/utils.py +4 -5
- supervisely/nn/tracker/visualize.py +93 -93
- supervisely/nn/training/gui/classes_selector.py +16 -1
- supervisely/nn/training/gui/train_val_splits_selector.py +52 -31
- supervisely/nn/training/train_app.py +46 -31
- supervisely/project/data_version.py +115 -51
- supervisely/project/download.py +1 -1
- supervisely/project/pointcloud_episode_project.py +37 -8
- supervisely/project/pointcloud_project.py +30 -2
- supervisely/project/project.py +14 -2
- supervisely/project/project_meta.py +27 -1
- supervisely/project/project_settings.py +32 -18
- supervisely/project/versioning/__init__.py +1 -0
- supervisely/project/versioning/common.py +20 -0
- supervisely/project/versioning/schema_fields.py +35 -0
- supervisely/project/versioning/video_schema.py +221 -0
- supervisely/project/versioning/volume_schema.py +87 -0
- supervisely/project/video_project.py +717 -15
- supervisely/project/volume_project.py +623 -5
- supervisely/template/experiment/experiment.html.jinja +4 -4
- supervisely/template/experiment/experiment_generator.py +14 -21
- supervisely/template/live_training/__init__.py +0 -0
- supervisely/template/live_training/header.html.jinja +96 -0
- supervisely/template/live_training/live_training.html.jinja +51 -0
- supervisely/template/live_training/live_training_generator.py +464 -0
- supervisely/template/live_training/sly-style.css +402 -0
- supervisely/template/live_training/template.html.jinja +18 -0
- supervisely/versions.json +28 -26
- supervisely/video/sampling.py +39 -20
- supervisely/video/video.py +41 -12
- supervisely/video_annotation/video_figure.py +38 -4
- supervisely/video_annotation/video_object.py +29 -4
- supervisely/volume/stl_converter.py +2 -0
- supervisely/worker_api/agent_rpc.py +24 -1
- supervisely/worker_api/rpc_servicer.py +31 -7
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/METADATA +58 -40
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/RECORD +203 -155
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/WHEEL +1 -1
- supervisely_lib/__init__.py +6 -1
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/entry_points.txt +0 -0
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info/licenses}/LICENSE +0 -0
- {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/top_level.txt +0 -0
|
@@ -43,6 +43,7 @@ from supervisely import (
|
|
|
43
43
|
logger,
|
|
44
44
|
)
|
|
45
45
|
from supervisely._utils import abs_url, get_filename_from_headers
|
|
46
|
+
from supervisely.api.entities_collection_api import EntitiesCollectionInfo
|
|
46
47
|
from supervisely.api.file_api import FileInfo
|
|
47
48
|
from supervisely.app import get_synced_data_dir, show_dialog
|
|
48
49
|
from supervisely.app.widgets import Progress
|
|
@@ -72,7 +73,6 @@ from supervisely.project.download import (
|
|
|
72
73
|
is_cached,
|
|
73
74
|
)
|
|
74
75
|
from supervisely.template.experiment.experiment_generator import ExperimentGenerator
|
|
75
|
-
from supervisely.api.entities_collection_api import EntitiesCollectionInfo
|
|
76
76
|
|
|
77
77
|
|
|
78
78
|
class TrainApp:
|
|
@@ -1598,13 +1598,18 @@ class TrainApp:
|
|
|
1598
1598
|
project_id = self.project_id
|
|
1599
1599
|
|
|
1600
1600
|
dataset_infos = [dataset for _, dataset in self._api.dataset.tree(project_id)]
|
|
1601
|
+
id_to_info = {ds.id: ds for ds in dataset_infos}
|
|
1601
1602
|
ds_infos_dict = {}
|
|
1602
1603
|
for dataset in dataset_infos:
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1604
|
+
name_parts = [dataset.name]
|
|
1605
|
+
parent_id = dataset.parent_id
|
|
1606
|
+
while parent_id is not None:
|
|
1607
|
+
parent_ds = id_to_info.get(parent_id)
|
|
1608
|
+
if parent_ds is None:
|
|
1609
|
+
parent_ds = self._api.dataset.get_info_by_id(parent_id)
|
|
1610
|
+
name_parts.append(parent_ds.name)
|
|
1611
|
+
parent_id = parent_ds.parent_id
|
|
1612
|
+
dataset_name = "/".join(reversed(name_parts))
|
|
1608
1613
|
ds_infos_dict[dataset_name] = dataset
|
|
1609
1614
|
|
|
1610
1615
|
def get_image_infos_by_split(ds_infos_dict: dict, split: list):
|
|
@@ -3157,8 +3162,11 @@ class TrainApp:
|
|
|
3157
3162
|
|
|
3158
3163
|
# Case 1: Use existing collections for training. No need to create new collections
|
|
3159
3164
|
split_method = self.gui.train_val_splits_selector.get_split_method()
|
|
3165
|
+
self.gui.train_val_splits_selector._parse_collections()
|
|
3160
3166
|
all_train_collections = self.gui.train_val_splits_selector.all_train_collections
|
|
3161
3167
|
all_val_collections = self.gui.train_val_splits_selector.all_val_collections
|
|
3168
|
+
latest_train_collection = self.gui.train_val_splits_selector.latest_train_collection
|
|
3169
|
+
latest_val_collection = self.gui.train_val_splits_selector.latest_val_collection
|
|
3162
3170
|
if split_method == "Based on collections":
|
|
3163
3171
|
current_selected_train_collection_ids = self.gui.train_val_splits_selector.train_val_splits.get_train_collections_ids()
|
|
3164
3172
|
train_match = _check_match(current_selected_train_collection_ids, all_train_collections)
|
|
@@ -3173,44 +3181,51 @@ class TrainApp:
|
|
|
3173
3181
|
# ------------------------------------------------------------ #
|
|
3174
3182
|
|
|
3175
3183
|
# Case 2: Create new collections for selected train val splits. Need to create new collections
|
|
3176
|
-
item_type = self.project_info.type
|
|
3177
|
-
experiment_name = self.gui.training_process.get_experiment_name()
|
|
3178
|
-
|
|
3179
3184
|
train_collection_idx = 1
|
|
3180
|
-
val_collection_idx = 1
|
|
3185
|
+
val_collection_idx = 1
|
|
3186
|
+
|
|
3187
|
+
def _extract_index_from_col_name(name: str, expected_prefix: str) -> Optional[int]:
|
|
3188
|
+
parts = name.split("_")
|
|
3189
|
+
if len(parts) == 2 and parts[0] == expected_prefix and parts[1].isdigit():
|
|
3190
|
+
return int(parts[1])
|
|
3191
|
+
return None
|
|
3181
3192
|
|
|
3182
3193
|
# Get train collection with max idx
|
|
3183
|
-
if
|
|
3184
|
-
train_collection_idx =
|
|
3185
|
-
|
|
3194
|
+
if latest_train_collection:
|
|
3195
|
+
train_collection_idx = (
|
|
3196
|
+
_extract_index_from_col_name(latest_train_collection.name, "train") + 1
|
|
3197
|
+
)
|
|
3198
|
+
|
|
3186
3199
|
# Get val collection with max idx
|
|
3187
|
-
if
|
|
3188
|
-
val_collection_idx =
|
|
3189
|
-
val_collection_idx += 1
|
|
3200
|
+
if latest_val_collection:
|
|
3201
|
+
val_collection_idx = _extract_index_from_col_name(latest_val_collection.name, "val") + 1
|
|
3190
3202
|
# -------------------------------- #
|
|
3191
3203
|
|
|
3192
3204
|
# Create Train Collection
|
|
3193
3205
|
train_img_ids = list(self._train_split_item_ids)
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
train_collection_id = getattr(train_collection, "id", None)
|
|
3197
|
-
if train_collection_id is None:
|
|
3198
|
-
raise AttributeError("Train EntitiesCollectionInfo object does not have 'id' attribute")
|
|
3199
|
-
self._api.entities_collection.add_items(train_collection_id, train_img_ids)
|
|
3200
|
-
self._train_collection_id = train_collection_id
|
|
3206
|
+
self._train_collection_id = self._create_collection("train", train_collection_idx)
|
|
3207
|
+
self._api.entities_collection.add_items(self._train_collection_id, train_img_ids)
|
|
3201
3208
|
|
|
3202
3209
|
# Create Val Collection
|
|
3203
3210
|
val_img_ids = list(self._val_split_item_ids)
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
val_collection_id = getattr(val_collection, "id", None)
|
|
3207
|
-
if val_collection_id is None:
|
|
3208
|
-
raise AttributeError("Val EntitiesCollectionInfo object does not have 'id' attribute")
|
|
3209
|
-
self._api.entities_collection.add_items(val_collection_id, val_img_ids)
|
|
3210
|
-
self._val_collection_id = val_collection_id
|
|
3211
|
+
self._val_collection_id = self._create_collection("val", val_collection_idx)
|
|
3212
|
+
self._api.entities_collection.add_items(self._val_collection_id, val_img_ids)
|
|
3211
3213
|
|
|
3212
3214
|
# Update Project Custom Data
|
|
3213
|
-
self._update_project_custom_data(
|
|
3215
|
+
self._update_project_custom_data(self._train_collection_id, self._val_collection_id)
|
|
3216
|
+
|
|
3217
|
+
def _create_collection(self, split_type: str, suffix: int) -> int:
|
|
3218
|
+
experiment_name = self.gui.training_process.get_experiment_name()
|
|
3219
|
+
description = f"Collection with {split_type} {self.project_info.type} for experiment: {experiment_name}"
|
|
3220
|
+
collection = self._api.entities_collection.create(
|
|
3221
|
+
project_id=self.project_id,
|
|
3222
|
+
name=f"{split_type}_{suffix:03d}",
|
|
3223
|
+
description=description,
|
|
3224
|
+
change_name_if_conflict=True,
|
|
3225
|
+
)
|
|
3226
|
+
if collection is None or collection.id is None: # pylint: disable=no-member
|
|
3227
|
+
raise RuntimeError(f"Failed to create {split_type} collection")
|
|
3228
|
+
return collection.id # pylint: disable=no-member
|
|
3214
3229
|
|
|
3215
3230
|
def _update_project_custom_data(self, train_collection_id: int, val_collection_id: int):
|
|
3216
3231
|
train_info = {
|
|
@@ -14,7 +14,12 @@ from supervisely.api.module_api import ApiField, ModuleApiBase
|
|
|
14
14
|
from supervisely.api.project_api import ProjectInfo
|
|
15
15
|
from supervisely.io import json
|
|
16
16
|
from supervisely.io.fs import remove_dir, silent_remove
|
|
17
|
-
|
|
17
|
+
from supervisely.project.versioning.schema_fields import VersionSchemaField
|
|
18
|
+
from supervisely.project.versioning.common import (
|
|
19
|
+
DEFAULT_IMAGE_SCHEMA_VERSION,
|
|
20
|
+
DEFAULT_VIDEO_SCHEMA_VERSION,
|
|
21
|
+
DEFAULT_VOLUME_SCHEMA_VERSION,
|
|
22
|
+
)
|
|
18
23
|
|
|
19
24
|
class VersionInfo(NamedTuple):
|
|
20
25
|
"""
|
|
@@ -49,7 +54,7 @@ class DataVersion(ModuleApiBase):
|
|
|
49
54
|
|
|
50
55
|
self._api: Api = api
|
|
51
56
|
self.__storage_dir: str = "/system/versions/"
|
|
52
|
-
self.__version_format: str =
|
|
57
|
+
self.__version_format: str = DEFAULT_IMAGE_SCHEMA_VERSION
|
|
53
58
|
self.project_info = None
|
|
54
59
|
self.project_dir = None
|
|
55
60
|
self.versions_path = None
|
|
@@ -84,6 +89,31 @@ class DataVersion(ModuleApiBase):
|
|
|
84
89
|
"""
|
|
85
90
|
return "VersionInfo"
|
|
86
91
|
|
|
92
|
+
@property
|
|
93
|
+
def project_cls(self):
|
|
94
|
+
from supervisely.project import (
|
|
95
|
+
Project,
|
|
96
|
+
ProjectType,
|
|
97
|
+
VideoProject,
|
|
98
|
+
VolumeProject,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
if self.project_info is None:
|
|
102
|
+
raise ValueError("Project info is not initialized. Call 'initialize' method first.")
|
|
103
|
+
|
|
104
|
+
project_type = self.project_info.type
|
|
105
|
+
if project_type == ProjectType.IMAGES.value:
|
|
106
|
+
self.__version_format = DEFAULT_IMAGE_SCHEMA_VERSION
|
|
107
|
+
return Project
|
|
108
|
+
elif project_type == ProjectType.VIDEOS.value:
|
|
109
|
+
self.__version_format = DEFAULT_VIDEO_SCHEMA_VERSION
|
|
110
|
+
return VideoProject
|
|
111
|
+
elif project_type == ProjectType.VOLUMES.value:
|
|
112
|
+
self.__version_format = DEFAULT_VOLUME_SCHEMA_VERSION
|
|
113
|
+
return VolumeProject
|
|
114
|
+
else:
|
|
115
|
+
raise ValueError(f"Unsupported project type: {project_type}")
|
|
116
|
+
|
|
87
117
|
def initialize(self, project_info: Union[ProjectInfo, int]):
|
|
88
118
|
"""
|
|
89
119
|
Initialize project versions.
|
|
@@ -158,10 +188,10 @@ class DataVersion(ModuleApiBase):
|
|
|
158
188
|
versions = self._api.file.get_json_file_content(
|
|
159
189
|
self.project_info.team_id, self.versions_path
|
|
160
190
|
)
|
|
161
|
-
|
|
191
|
+
return versions or {}
|
|
162
192
|
except FileNotFoundError:
|
|
163
|
-
versions = {"format": self.__version_format}
|
|
164
|
-
|
|
193
|
+
# versions = {"format": self.__version_format}
|
|
194
|
+
return {}
|
|
165
195
|
|
|
166
196
|
def set_map(self, project_info: Union[ProjectInfo, int], initialize: bool = True):
|
|
167
197
|
"""
|
|
@@ -176,6 +206,8 @@ class DataVersion(ModuleApiBase):
|
|
|
176
206
|
|
|
177
207
|
if initialize:
|
|
178
208
|
self.initialize(project_info)
|
|
209
|
+
if "format" not in self.versions:
|
|
210
|
+
self.versions["format"] = self.__version_format
|
|
179
211
|
temp_dir = tempfile.mkdtemp()
|
|
180
212
|
local_versions = os.path.join(temp_dir, "versions.json")
|
|
181
213
|
json.dump_json_file(self.versions, local_versions)
|
|
@@ -329,21 +361,32 @@ class DataVersion(ModuleApiBase):
|
|
|
329
361
|
return reserve_info.get(ApiField.ID), reserve_info.get(ApiField.COMMIT_TOKEN)
|
|
330
362
|
|
|
331
363
|
except requests.exceptions.HTTPError as e:
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
364
|
+
details = {}
|
|
365
|
+
if e.response is not None:
|
|
366
|
+
try:
|
|
367
|
+
details = (e.response.json() or {}).get("details", {}) # type: ignore[union-attr]
|
|
368
|
+
except Exception:
|
|
369
|
+
details = {}
|
|
370
|
+
|
|
371
|
+
if details.get("useExistingVersion"):
|
|
372
|
+
version_id = details.get("version", {}).get("id")
|
|
373
|
+
version = details.get("version", {}).get("version")
|
|
335
374
|
logger.info(
|
|
336
375
|
f"No changes to the project since the last version '{version}' with ID '{version_id}'"
|
|
337
376
|
)
|
|
338
377
|
return (None, None)
|
|
339
|
-
|
|
378
|
+
|
|
379
|
+
message = (details.get("message") or "").lower()
|
|
380
|
+
if "is already committing" in message:
|
|
340
381
|
if retry_delay >= max_delay:
|
|
341
382
|
raise RuntimeError(
|
|
342
383
|
"Failed to reserve version. Another process is already committing a version. Maximum number of attempts reached."
|
|
343
384
|
)
|
|
344
|
-
version = e.response.json().get("details", {}).get("version").get("version")
|
|
345
385
|
time.sleep(retry_delay)
|
|
346
386
|
retry_delay *= 2
|
|
387
|
+
continue
|
|
388
|
+
|
|
389
|
+
raise
|
|
347
390
|
|
|
348
391
|
def cancel_reservation(self, version_id: int, commit_token: str):
|
|
349
392
|
"""
|
|
@@ -384,8 +427,6 @@ class DataVersion(ModuleApiBase):
|
|
|
384
427
|
:return: ProjectInfo object of the restored project
|
|
385
428
|
:rtype: ProjectInfo or None
|
|
386
429
|
"""
|
|
387
|
-
from supervisely.project.project import Project
|
|
388
|
-
|
|
389
430
|
if version_id is None and version_num is None:
|
|
390
431
|
raise ValueError("Either version_id or version_num must be provided")
|
|
391
432
|
|
|
@@ -421,7 +462,7 @@ class DataVersion(ModuleApiBase):
|
|
|
421
462
|
return
|
|
422
463
|
|
|
423
464
|
bin_io = self._download_and_extract(backup_files)
|
|
424
|
-
new_project_info =
|
|
465
|
+
new_project_info = self.project_cls.upload_bin(
|
|
425
466
|
self._api,
|
|
426
467
|
bin_io,
|
|
427
468
|
self.project_info.workspace_id,
|
|
@@ -456,15 +497,28 @@ class DataVersion(ModuleApiBase):
|
|
|
456
497
|
local_path = os.path.join(temp_dir, "download.tar.zst")
|
|
457
498
|
try:
|
|
458
499
|
self._api.file.download(self.project_info.team_id, path, local_path)
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
500
|
+
# Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
|
|
501
|
+
try:
|
|
502
|
+
dctx = zstd.ZstdDecompressor()
|
|
503
|
+
with open(local_path, "rb") as zst_f:
|
|
504
|
+
with dctx.stream_reader(zst_f) as reader:
|
|
505
|
+
with tarfile.open(fileobj=reader, mode="r|") as tar:
|
|
506
|
+
for member in tar:
|
|
507
|
+
if member.name == "version.bin":
|
|
508
|
+
file = tar.extractfile(member)
|
|
509
|
+
if not file:
|
|
510
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
511
|
+
return io.BytesIO(file.read())
|
|
512
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
513
|
+
except Exception:
|
|
514
|
+
# Fallback: one-shot decompress
|
|
515
|
+
with open(local_path, "rb") as zst_f:
|
|
516
|
+
decompressed_data = zstd.decompress(zst_f.read())
|
|
517
|
+
with tarfile.open(fileobj=io.BytesIO(decompressed_data), mode="r") as tar:
|
|
518
|
+
file = tar.extractfile("version.bin")
|
|
519
|
+
if not file:
|
|
520
|
+
raise RuntimeError("version.bin not found in the archive")
|
|
521
|
+
return io.BytesIO(file.read())
|
|
468
522
|
except Exception as e:
|
|
469
523
|
raise RuntimeError(f"Failed to extract version: {e}")
|
|
470
524
|
finally:
|
|
@@ -501,34 +555,44 @@ class DataVersion(ModuleApiBase):
|
|
|
501
555
|
:return: File info
|
|
502
556
|
:rtype: dict
|
|
503
557
|
"""
|
|
504
|
-
from supervisely.project.project import Project
|
|
505
|
-
|
|
506
558
|
temp_dir = tempfile.mkdtemp()
|
|
559
|
+
data = None
|
|
560
|
+
try:
|
|
561
|
+
data = self.project_cls.download_bin(
|
|
562
|
+
self._api, self.project_info.id, batch_size=200, return_bytesio=True
|
|
563
|
+
)
|
|
564
|
+
info = tarfile.TarInfo(name="version.bin")
|
|
565
|
+
data.seek(0, io.SEEK_END)
|
|
566
|
+
info.size = data.tell()
|
|
567
|
+
data.seek(0)
|
|
568
|
+
zst_archive_path = os.path.join(temp_dir, "download.tar.zst")
|
|
507
569
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
570
|
+
# Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
|
|
571
|
+
try:
|
|
572
|
+
cctx = zstd.ZstdCompressor()
|
|
573
|
+
with open(zst_archive_path, "wb") as zst_f:
|
|
574
|
+
try:
|
|
575
|
+
stream = cctx.stream_writer(zst_f, closefd=False)
|
|
576
|
+
except TypeError:
|
|
577
|
+
stream = cctx.stream_writer(zst_f)
|
|
578
|
+
with stream as compressor:
|
|
579
|
+
with tarfile.open(fileobj=compressor, mode="w|") as tar:
|
|
580
|
+
tar.addfile(tarinfo=info, fileobj=data)
|
|
581
|
+
except Exception:
|
|
582
|
+
# Fallback: build tar in memory + one-shot compress
|
|
583
|
+
tar_data = io.BytesIO()
|
|
584
|
+
with tarfile.open(fileobj=tar_data, mode="w") as tar:
|
|
585
|
+
tar.addfile(tarinfo=info, fileobj=data)
|
|
586
|
+
tar_data.seek(0)
|
|
587
|
+
with open(zst_archive_path, "wb") as zst_f:
|
|
588
|
+
zst_f.write(zstd.compress(tar_data.read()))
|
|
589
|
+
|
|
590
|
+
file_info = self._api.file.upload(self.project_info.team_id, zst_archive_path, path)
|
|
591
|
+
return file_info
|
|
592
|
+
finally:
|
|
593
|
+
if data is not None:
|
|
594
|
+
try:
|
|
595
|
+
data.close()
|
|
596
|
+
except Exception:
|
|
597
|
+
pass
|
|
598
|
+
remove_dir(temp_dir)
|
supervisely/project/download.py
CHANGED
|
@@ -216,7 +216,7 @@ def download_async_or_sync(
|
|
|
216
216
|
dataset_ids: Optional[List[int]] = None,
|
|
217
217
|
log_progress: bool = True,
|
|
218
218
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
219
|
-
semaphore: Optional[asyncio.Semaphore] = None,
|
|
219
|
+
semaphore: Optional[Union[asyncio.Semaphore, int]] = None,
|
|
220
220
|
**kwargs,
|
|
221
221
|
):
|
|
222
222
|
"""
|
|
@@ -655,6 +655,25 @@ class PointcloudEpisodeProject(PointcloudProject):
|
|
|
655
655
|
f"Static method 'download_async()' is not supported for PointcloudEpisodeProject class now."
|
|
656
656
|
)
|
|
657
657
|
|
|
658
|
+
# ----------------------------------- #
|
|
659
|
+
# Pointcloud Episodes Data Versioning #
|
|
660
|
+
# ----------------------------------- #
|
|
661
|
+
@staticmethod
|
|
662
|
+
def download_bin(*args, **kwargs):
|
|
663
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
664
|
+
|
|
665
|
+
@staticmethod
|
|
666
|
+
def upload_bin(*args, **kwargs):
|
|
667
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
668
|
+
|
|
669
|
+
@staticmethod
|
|
670
|
+
def build_snapshot(*args, **kwargs):
|
|
671
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
672
|
+
|
|
673
|
+
@staticmethod
|
|
674
|
+
def restore_snapshot(*args, **kwargs):
|
|
675
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
|
|
676
|
+
|
|
658
677
|
|
|
659
678
|
def download_pointcloud_episode_project(
|
|
660
679
|
api: Api,
|
|
@@ -946,7 +965,7 @@ def upload_pointcloud_episode_project(
|
|
|
946
965
|
log_progress: bool = True,
|
|
947
966
|
progress_cb: Optional[Union[tqdm, Callable]] = None,
|
|
948
967
|
) -> Tuple[int, str]:
|
|
949
|
-
# STEP 0
|
|
968
|
+
# STEP 0 - create project remotely
|
|
950
969
|
project_fs = PointcloudEpisodeProject.read_single(directory)
|
|
951
970
|
project_name = project_fs.name if project_name is None else project_name
|
|
952
971
|
|
|
@@ -959,8 +978,9 @@ def upload_pointcloud_episode_project(
|
|
|
959
978
|
if progress_cb is not None:
|
|
960
979
|
log_progress = False
|
|
961
980
|
|
|
981
|
+
name_to_dsinfo = {}
|
|
962
982
|
key_id_map = KeyIdMap()
|
|
963
|
-
for dataset_fs in project_fs.datasets:
|
|
983
|
+
for dataset_fs in sorted(project_fs.datasets, key=lambda ds: len(ds.parents)):
|
|
964
984
|
dataset_fs: PointcloudEpisodeDataset
|
|
965
985
|
ann_json_path = dataset_fs.get_ann_path()
|
|
966
986
|
|
|
@@ -970,14 +990,19 @@ def upload_pointcloud_episode_project(
|
|
|
970
990
|
else:
|
|
971
991
|
episode_annotation = PointcloudEpisodeAnnotation()
|
|
972
992
|
|
|
993
|
+
parent_path = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
|
|
994
|
+
parent_info = name_to_dsinfo.get(parent_path)
|
|
995
|
+
parent_id = parent_info.id if parent_info else None
|
|
973
996
|
dataset = api.dataset.create(
|
|
974
997
|
project.id,
|
|
975
|
-
dataset_fs.
|
|
998
|
+
dataset_fs.short_name,
|
|
976
999
|
description=episode_annotation.description,
|
|
977
1000
|
change_name_if_conflict=True,
|
|
1001
|
+
parent_id=parent_id,
|
|
978
1002
|
)
|
|
1003
|
+
name_to_dsinfo[dataset_fs.name] = dataset
|
|
979
1004
|
|
|
980
|
-
# STEP 1
|
|
1005
|
+
# STEP 1 - upload episodes
|
|
981
1006
|
items_infos = {"names": [], "paths": [], "metas": []}
|
|
982
1007
|
|
|
983
1008
|
for item_name in dataset_fs:
|
|
@@ -989,6 +1014,10 @@ def upload_pointcloud_episode_project(
|
|
|
989
1014
|
items_infos["paths"].append(item_path)
|
|
990
1015
|
items_infos["metas"].append(item_meta)
|
|
991
1016
|
|
|
1017
|
+
if not items_infos["names"]:
|
|
1018
|
+
logger.info(f"Dataset {dataset.name} has no items, skipping upload")
|
|
1019
|
+
continue
|
|
1020
|
+
|
|
992
1021
|
ds_progress = progress_cb
|
|
993
1022
|
if log_progress:
|
|
994
1023
|
ds_progress = tqdm_sly(
|
|
@@ -1015,7 +1044,7 @@ def upload_pointcloud_episode_project(
|
|
|
1015
1044
|
},
|
|
1016
1045
|
)
|
|
1017
1046
|
raise e
|
|
1018
|
-
# STEP 2
|
|
1047
|
+
# STEP 2 - upload annotations
|
|
1019
1048
|
frame_to_pcl_ids = {pcl_info.frame: pcl_info.id for pcl_info in pcl_infos}
|
|
1020
1049
|
try:
|
|
1021
1050
|
api.pointcloud_episode.annotation.append(
|
|
@@ -1033,9 +1062,9 @@ def upload_pointcloud_episode_project(
|
|
|
1033
1062
|
)
|
|
1034
1063
|
raise e
|
|
1035
1064
|
|
|
1036
|
-
# STEP 3
|
|
1065
|
+
# STEP 3 - upload photo context
|
|
1037
1066
|
img_infos = {"img_paths": [], "img_metas": []}
|
|
1038
|
-
# STEP 3.1
|
|
1067
|
+
# STEP 3.1 - upload images
|
|
1039
1068
|
pcl_to_rimg_figures: Dict[int, Dict[str, List[Dict]]] = {}
|
|
1040
1069
|
pcl_to_hash_to_id: Dict[int, Dict[str, int]] = {}
|
|
1041
1070
|
for pcl_info in pcl_infos:
|
|
@@ -1067,7 +1096,7 @@ def upload_pointcloud_episode_project(
|
|
|
1067
1096
|
)
|
|
1068
1097
|
raise e
|
|
1069
1098
|
|
|
1070
|
-
# STEP 3.2
|
|
1099
|
+
# STEP 3.2 - upload images metas
|
|
1071
1100
|
images_hashes_iterator = images_hashes.__iter__()
|
|
1072
1101
|
for pcl_info in pcl_infos:
|
|
1073
1102
|
related_items = dataset_fs.get_related_images(pcl_info.name)
|
|
@@ -908,6 +908,25 @@ class PointcloudProject(VideoProject):
|
|
|
908
908
|
f"Static method 'download_async()' is not supported for PointcloudProject class now."
|
|
909
909
|
)
|
|
910
910
|
|
|
911
|
+
# -------------------------- #
|
|
912
|
+
# Pointcloud Data Versioning #
|
|
913
|
+
# -------------------------- #
|
|
914
|
+
@staticmethod
|
|
915
|
+
def download_bin(*args, **kwargs):
|
|
916
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
917
|
+
|
|
918
|
+
@staticmethod
|
|
919
|
+
def upload_bin(*args, **kwargs):
|
|
920
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
921
|
+
|
|
922
|
+
@staticmethod
|
|
923
|
+
def build_snapshot(*args, **kwargs):
|
|
924
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
925
|
+
|
|
926
|
+
@staticmethod
|
|
927
|
+
def restore_snapshot(*args, **kwargs):
|
|
928
|
+
raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
|
|
929
|
+
|
|
911
930
|
|
|
912
931
|
def download_pointcloud_project(
|
|
913
932
|
api: Api,
|
|
@@ -1216,8 +1235,17 @@ def upload_pointcloud_project(
|
|
|
1216
1235
|
log_progress = False
|
|
1217
1236
|
|
|
1218
1237
|
key_id_map = KeyIdMap()
|
|
1219
|
-
|
|
1220
|
-
|
|
1238
|
+
name_to_dsinfo = {}
|
|
1239
|
+
for dataset_fs in sorted(project_fs, key=lambda ds: len(ds.parents)):
|
|
1240
|
+
parent_name = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
|
|
1241
|
+
parent_info = name_to_dsinfo.get(parent_name)
|
|
1242
|
+
parent_id = None
|
|
1243
|
+
if parent_info is not None:
|
|
1244
|
+
parent_id = parent_info.id
|
|
1245
|
+
dataset = api.dataset.create(
|
|
1246
|
+
project.id, dataset_fs.short_name, change_name_if_conflict=True, parent_id=parent_id
|
|
1247
|
+
)
|
|
1248
|
+
name_to_dsinfo[dataset_fs.name] = dataset
|
|
1221
1249
|
|
|
1222
1250
|
ds_progress = progress_cb
|
|
1223
1251
|
if log_progress:
|
supervisely/project/project.py
CHANGED
|
@@ -3626,7 +3626,11 @@ class Project:
|
|
|
3626
3626
|
if project_name is None:
|
|
3627
3627
|
project_name = project_info.name
|
|
3628
3628
|
new_project_info = api.project.create(
|
|
3629
|
-
workspace_id,
|
|
3629
|
+
workspace_id,
|
|
3630
|
+
project_name,
|
|
3631
|
+
description=project_info.description,
|
|
3632
|
+
change_name_if_conflict=True,
|
|
3633
|
+
readme=project_info.readme,
|
|
3630
3634
|
)
|
|
3631
3635
|
custom_data = new_project_info.custom_data
|
|
3632
3636
|
version_num = project_info.version.get("version", None) if project_info.version else 0
|
|
@@ -4584,6 +4588,7 @@ def upload_project(
|
|
|
4584
4588
|
blob_file_infos = []
|
|
4585
4589
|
|
|
4586
4590
|
for ds_fs in project_fs.datasets:
|
|
4591
|
+
logger.debug(f"Processing dataset: {ds_fs.name}")
|
|
4587
4592
|
if len(ds_fs.parents) > 0:
|
|
4588
4593
|
parent = f"{os.path.sep}".join(ds_fs.parents)
|
|
4589
4594
|
parent_id = dataset_map.get(parent)
|
|
@@ -4624,8 +4629,15 @@ def upload_project(
|
|
|
4624
4629
|
if os.path.isfile(path):
|
|
4625
4630
|
valid_indices.append(i)
|
|
4626
4631
|
valid_paths.append(path)
|
|
4627
|
-
|
|
4632
|
+
elif len(project_fs.blob_files) > 0:
|
|
4628
4633
|
offset_indices.append(i)
|
|
4634
|
+
else:
|
|
4635
|
+
if img_infos[i] is not None:
|
|
4636
|
+
logger.debug(f"Image will be uploaded by image_info: {names[i]}")
|
|
4637
|
+
else:
|
|
4638
|
+
logger.warning(
|
|
4639
|
+
f"Image and image info file not found, image will be skipped: {names[i]}"
|
|
4640
|
+
)
|
|
4629
4641
|
img_paths = valid_paths
|
|
4630
4642
|
ann_paths = list(filter(lambda x: os.path.isfile(x), ann_paths))
|
|
4631
4643
|
# Create a mapping from name to index position for quick lookups
|
|
@@ -14,7 +14,7 @@ from supervisely.geometry.bitmap import Bitmap
|
|
|
14
14
|
from supervisely.geometry.polygon import Polygon
|
|
15
15
|
from supervisely.geometry.rectangle import Rectangle
|
|
16
16
|
from supervisely.io.json import JsonSerializable
|
|
17
|
-
from supervisely.project.project_settings import ProjectSettings
|
|
17
|
+
from supervisely.project.project_settings import LabelingInterface, ProjectSettings
|
|
18
18
|
from supervisely.project.project_type import ProjectType
|
|
19
19
|
|
|
20
20
|
|
|
@@ -288,6 +288,32 @@ class ProjectMeta(JsonSerializable):
|
|
|
288
288
|
# Output: <class 'supervisely.project.project_settings.ProjectSettings'>
|
|
289
289
|
"""
|
|
290
290
|
return self._project_settings
|
|
291
|
+
|
|
292
|
+
@property
|
|
293
|
+
def labeling_interface(self) -> Optional[LabelingInterface]:
|
|
294
|
+
"""
|
|
295
|
+
Get labeling interface settings of the project.
|
|
296
|
+
|
|
297
|
+
:return: Labeling interface settings
|
|
298
|
+
:rtype: :class: `LabelingInterface` or None
|
|
299
|
+
:Usage example:
|
|
300
|
+
|
|
301
|
+
.. code-block:: python
|
|
302
|
+
|
|
303
|
+
import supervisely as sly
|
|
304
|
+
|
|
305
|
+
s = sly.ProjectSettings(
|
|
306
|
+
multiview_enabled=True,
|
|
307
|
+
multiview_tag_name='multi_tag',
|
|
308
|
+
multiview_is_synced=False,
|
|
309
|
+
)
|
|
310
|
+
meta = sly.ProjectMeta(project_settings=s)
|
|
311
|
+
|
|
312
|
+
labeling_interface = meta.labeling_interface
|
|
313
|
+
print(labeling_interface)
|
|
314
|
+
# Output: None
|
|
315
|
+
"""
|
|
316
|
+
return self._project_settings.labeling_interface if self._project_settings else None
|
|
291
317
|
|
|
292
318
|
def to_json(self) -> Dict:
|
|
293
319
|
"""
|