supervisely 6.73.438__py3-none-any.whl → 6.73.513__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. supervisely/__init__.py +137 -1
  2. supervisely/_utils.py +81 -0
  3. supervisely/annotation/annotation.py +8 -2
  4. supervisely/annotation/json_geometries_map.py +14 -11
  5. supervisely/annotation/label.py +80 -3
  6. supervisely/api/annotation_api.py +14 -11
  7. supervisely/api/api.py +59 -38
  8. supervisely/api/app_api.py +11 -2
  9. supervisely/api/dataset_api.py +74 -12
  10. supervisely/api/entities_collection_api.py +10 -0
  11. supervisely/api/entity_annotation/figure_api.py +52 -4
  12. supervisely/api/entity_annotation/object_api.py +3 -3
  13. supervisely/api/entity_annotation/tag_api.py +63 -12
  14. supervisely/api/guides_api.py +210 -0
  15. supervisely/api/image_api.py +72 -1
  16. supervisely/api/labeling_job_api.py +83 -1
  17. supervisely/api/labeling_queue_api.py +33 -7
  18. supervisely/api/module_api.py +9 -0
  19. supervisely/api/project_api.py +71 -26
  20. supervisely/api/storage_api.py +3 -1
  21. supervisely/api/task_api.py +13 -2
  22. supervisely/api/team_api.py +4 -3
  23. supervisely/api/video/video_annotation_api.py +119 -3
  24. supervisely/api/video/video_api.py +65 -14
  25. supervisely/api/video/video_figure_api.py +24 -11
  26. supervisely/app/__init__.py +1 -1
  27. supervisely/app/content.py +23 -7
  28. supervisely/app/development/development.py +18 -2
  29. supervisely/app/fastapi/__init__.py +1 -0
  30. supervisely/app/fastapi/custom_static_files.py +1 -1
  31. supervisely/app/fastapi/multi_user.py +105 -0
  32. supervisely/app/fastapi/subapp.py +88 -42
  33. supervisely/app/fastapi/websocket.py +77 -9
  34. supervisely/app/singleton.py +21 -0
  35. supervisely/app/v1/app_service.py +18 -2
  36. supervisely/app/v1/constants.py +7 -1
  37. supervisely/app/widgets/__init__.py +6 -0
  38. supervisely/app/widgets/activity_feed/__init__.py +0 -0
  39. supervisely/app/widgets/activity_feed/activity_feed.py +239 -0
  40. supervisely/app/widgets/activity_feed/style.css +78 -0
  41. supervisely/app/widgets/activity_feed/template.html +22 -0
  42. supervisely/app/widgets/card/card.py +20 -0
  43. supervisely/app/widgets/classes_list_selector/classes_list_selector.py +121 -9
  44. supervisely/app/widgets/classes_list_selector/template.html +60 -93
  45. supervisely/app/widgets/classes_mapping/classes_mapping.py +13 -12
  46. supervisely/app/widgets/classes_table/classes_table.py +1 -0
  47. supervisely/app/widgets/deploy_model/deploy_model.py +56 -35
  48. supervisely/app/widgets/dialog/dialog.py +12 -0
  49. supervisely/app/widgets/dialog/template.html +2 -1
  50. supervisely/app/widgets/ecosystem_model_selector/ecosystem_model_selector.py +1 -1
  51. supervisely/app/widgets/experiment_selector/experiment_selector.py +8 -0
  52. supervisely/app/widgets/fast_table/fast_table.py +184 -60
  53. supervisely/app/widgets/fast_table/template.html +1 -1
  54. supervisely/app/widgets/heatmap/__init__.py +0 -0
  55. supervisely/app/widgets/heatmap/heatmap.py +564 -0
  56. supervisely/app/widgets/heatmap/script.js +533 -0
  57. supervisely/app/widgets/heatmap/style.css +233 -0
  58. supervisely/app/widgets/heatmap/template.html +21 -0
  59. supervisely/app/widgets/modal/__init__.py +0 -0
  60. supervisely/app/widgets/modal/modal.py +198 -0
  61. supervisely/app/widgets/modal/template.html +10 -0
  62. supervisely/app/widgets/object_class_view/object_class_view.py +3 -0
  63. supervisely/app/widgets/radio_tabs/radio_tabs.py +18 -2
  64. supervisely/app/widgets/radio_tabs/template.html +1 -0
  65. supervisely/app/widgets/select/select.py +6 -3
  66. supervisely/app/widgets/select_class/__init__.py +0 -0
  67. supervisely/app/widgets/select_class/select_class.py +363 -0
  68. supervisely/app/widgets/select_class/template.html +50 -0
  69. supervisely/app/widgets/select_cuda/select_cuda.py +22 -0
  70. supervisely/app/widgets/select_dataset_tree/select_dataset_tree.py +65 -7
  71. supervisely/app/widgets/select_tag/__init__.py +0 -0
  72. supervisely/app/widgets/select_tag/select_tag.py +352 -0
  73. supervisely/app/widgets/select_tag/template.html +64 -0
  74. supervisely/app/widgets/select_team/select_team.py +37 -4
  75. supervisely/app/widgets/select_team/template.html +4 -5
  76. supervisely/app/widgets/select_user/__init__.py +0 -0
  77. supervisely/app/widgets/select_user/select_user.py +270 -0
  78. supervisely/app/widgets/select_user/template.html +13 -0
  79. supervisely/app/widgets/select_workspace/select_workspace.py +59 -10
  80. supervisely/app/widgets/select_workspace/template.html +9 -12
  81. supervisely/app/widgets/table/table.py +68 -13
  82. supervisely/app/widgets/tree_select/tree_select.py +2 -0
  83. supervisely/aug/aug.py +6 -2
  84. supervisely/convert/base_converter.py +1 -0
  85. supervisely/convert/converter.py +2 -2
  86. supervisely/convert/image/csv/csv_converter.py +24 -15
  87. supervisely/convert/image/image_converter.py +3 -1
  88. supervisely/convert/image/image_helper.py +48 -4
  89. supervisely/convert/image/label_studio/label_studio_converter.py +2 -0
  90. supervisely/convert/image/medical2d/medical2d_helper.py +2 -24
  91. supervisely/convert/image/multispectral/multispectral_converter.py +6 -0
  92. supervisely/convert/image/pascal_voc/pascal_voc_converter.py +8 -5
  93. supervisely/convert/image/pascal_voc/pascal_voc_helper.py +7 -0
  94. supervisely/convert/pointcloud/kitti_3d/kitti_3d_converter.py +33 -3
  95. supervisely/convert/pointcloud/kitti_3d/kitti_3d_helper.py +12 -5
  96. supervisely/convert/pointcloud/las/las_converter.py +13 -1
  97. supervisely/convert/pointcloud/las/las_helper.py +110 -11
  98. supervisely/convert/pointcloud/nuscenes_conv/nuscenes_converter.py +27 -16
  99. supervisely/convert/pointcloud/pointcloud_converter.py +91 -3
  100. supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py +58 -22
  101. supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py +21 -47
  102. supervisely/convert/video/__init__.py +1 -0
  103. supervisely/convert/video/multi_view/__init__.py +0 -0
  104. supervisely/convert/video/multi_view/multi_view.py +543 -0
  105. supervisely/convert/video/sly/sly_video_converter.py +359 -3
  106. supervisely/convert/video/video_converter.py +24 -4
  107. supervisely/convert/volume/dicom/dicom_converter.py +13 -5
  108. supervisely/convert/volume/dicom/dicom_helper.py +30 -18
  109. supervisely/geometry/constants.py +1 -0
  110. supervisely/geometry/geometry.py +4 -0
  111. supervisely/geometry/helpers.py +5 -1
  112. supervisely/geometry/oriented_bbox.py +676 -0
  113. supervisely/geometry/polyline_3d.py +110 -0
  114. supervisely/geometry/rectangle.py +2 -1
  115. supervisely/io/env.py +76 -1
  116. supervisely/io/fs.py +21 -0
  117. supervisely/nn/benchmark/base_evaluator.py +104 -11
  118. supervisely/nn/benchmark/instance_segmentation/evaluator.py +1 -8
  119. supervisely/nn/benchmark/object_detection/evaluator.py +20 -4
  120. supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py +10 -5
  121. supervisely/nn/benchmark/semantic_segmentation/evaluator.py +34 -16
  122. supervisely/nn/benchmark/semantic_segmentation/vis_metrics/confusion_matrix.py +1 -1
  123. supervisely/nn/benchmark/semantic_segmentation/vis_metrics/frequently_confused.py +1 -1
  124. supervisely/nn/benchmark/semantic_segmentation/vis_metrics/overview.py +1 -1
  125. supervisely/nn/benchmark/visualization/evaluation_result.py +66 -4
  126. supervisely/nn/inference/cache.py +43 -18
  127. supervisely/nn/inference/gui/serving_gui_template.py +5 -2
  128. supervisely/nn/inference/inference.py +916 -222
  129. supervisely/nn/inference/inference_request.py +55 -10
  130. supervisely/nn/inference/predict_app/gui/classes_selector.py +83 -12
  131. supervisely/nn/inference/predict_app/gui/gui.py +676 -488
  132. supervisely/nn/inference/predict_app/gui/input_selector.py +205 -26
  133. supervisely/nn/inference/predict_app/gui/model_selector.py +2 -4
  134. supervisely/nn/inference/predict_app/gui/output_selector.py +46 -6
  135. supervisely/nn/inference/predict_app/gui/settings_selector.py +756 -59
  136. supervisely/nn/inference/predict_app/gui/tags_selector.py +1 -1
  137. supervisely/nn/inference/predict_app/gui/utils.py +236 -119
  138. supervisely/nn/inference/predict_app/predict_app.py +2 -2
  139. supervisely/nn/inference/session.py +43 -35
  140. supervisely/nn/inference/tracking/bbox_tracking.py +118 -35
  141. supervisely/nn/inference/tracking/point_tracking.py +5 -1
  142. supervisely/nn/inference/tracking/tracker_interface.py +10 -1
  143. supervisely/nn/inference/uploader.py +139 -12
  144. supervisely/nn/live_training/__init__.py +7 -0
  145. supervisely/nn/live_training/api_server.py +111 -0
  146. supervisely/nn/live_training/artifacts_utils.py +243 -0
  147. supervisely/nn/live_training/checkpoint_utils.py +229 -0
  148. supervisely/nn/live_training/dynamic_sampler.py +44 -0
  149. supervisely/nn/live_training/helpers.py +14 -0
  150. supervisely/nn/live_training/incremental_dataset.py +146 -0
  151. supervisely/nn/live_training/live_training.py +497 -0
  152. supervisely/nn/live_training/loss_plateau_detector.py +111 -0
  153. supervisely/nn/live_training/request_queue.py +52 -0
  154. supervisely/nn/model/model_api.py +9 -0
  155. supervisely/nn/model/prediction.py +2 -1
  156. supervisely/nn/model/prediction_session.py +26 -14
  157. supervisely/nn/prediction_dto.py +19 -1
  158. supervisely/nn/tracker/base_tracker.py +11 -1
  159. supervisely/nn/tracker/botsort/botsort_config.yaml +0 -1
  160. supervisely/nn/tracker/botsort/tracker/mc_bot_sort.py +7 -4
  161. supervisely/nn/tracker/botsort_tracker.py +94 -65
  162. supervisely/nn/tracker/utils.py +4 -5
  163. supervisely/nn/tracker/visualize.py +93 -93
  164. supervisely/nn/training/gui/classes_selector.py +16 -1
  165. supervisely/nn/training/gui/train_val_splits_selector.py +52 -31
  166. supervisely/nn/training/train_app.py +46 -31
  167. supervisely/project/data_version.py +115 -51
  168. supervisely/project/download.py +1 -1
  169. supervisely/project/pointcloud_episode_project.py +37 -8
  170. supervisely/project/pointcloud_project.py +30 -2
  171. supervisely/project/project.py +14 -2
  172. supervisely/project/project_meta.py +27 -1
  173. supervisely/project/project_settings.py +32 -18
  174. supervisely/project/versioning/__init__.py +1 -0
  175. supervisely/project/versioning/common.py +20 -0
  176. supervisely/project/versioning/schema_fields.py +35 -0
  177. supervisely/project/versioning/video_schema.py +221 -0
  178. supervisely/project/versioning/volume_schema.py +87 -0
  179. supervisely/project/video_project.py +717 -15
  180. supervisely/project/volume_project.py +623 -5
  181. supervisely/template/experiment/experiment.html.jinja +4 -4
  182. supervisely/template/experiment/experiment_generator.py +14 -21
  183. supervisely/template/live_training/__init__.py +0 -0
  184. supervisely/template/live_training/header.html.jinja +96 -0
  185. supervisely/template/live_training/live_training.html.jinja +51 -0
  186. supervisely/template/live_training/live_training_generator.py +464 -0
  187. supervisely/template/live_training/sly-style.css +402 -0
  188. supervisely/template/live_training/template.html.jinja +18 -0
  189. supervisely/versions.json +28 -26
  190. supervisely/video/sampling.py +39 -20
  191. supervisely/video/video.py +41 -12
  192. supervisely/video_annotation/video_figure.py +38 -4
  193. supervisely/video_annotation/video_object.py +29 -4
  194. supervisely/volume/stl_converter.py +2 -0
  195. supervisely/worker_api/agent_rpc.py +24 -1
  196. supervisely/worker_api/rpc_servicer.py +31 -7
  197. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/METADATA +58 -40
  198. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/RECORD +203 -155
  199. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/WHEEL +1 -1
  200. supervisely_lib/__init__.py +6 -1
  201. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/entry_points.txt +0 -0
  202. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info/licenses}/LICENSE +0 -0
  203. {supervisely-6.73.438.dist-info → supervisely-6.73.513.dist-info}/top_level.txt +0 -0
@@ -43,6 +43,7 @@ from supervisely import (
43
43
  logger,
44
44
  )
45
45
  from supervisely._utils import abs_url, get_filename_from_headers
46
+ from supervisely.api.entities_collection_api import EntitiesCollectionInfo
46
47
  from supervisely.api.file_api import FileInfo
47
48
  from supervisely.app import get_synced_data_dir, show_dialog
48
49
  from supervisely.app.widgets import Progress
@@ -72,7 +73,6 @@ from supervisely.project.download import (
72
73
  is_cached,
73
74
  )
74
75
  from supervisely.template.experiment.experiment_generator import ExperimentGenerator
75
- from supervisely.api.entities_collection_api import EntitiesCollectionInfo
76
76
 
77
77
 
78
78
  class TrainApp:
@@ -1598,13 +1598,18 @@ class TrainApp:
1598
1598
  project_id = self.project_id
1599
1599
 
1600
1600
  dataset_infos = [dataset for _, dataset in self._api.dataset.tree(project_id)]
1601
+ id_to_info = {ds.id: ds for ds in dataset_infos}
1601
1602
  ds_infos_dict = {}
1602
1603
  for dataset in dataset_infos:
1603
- if dataset.parent_id is not None:
1604
- parent_ds = self._api.dataset.get_info_by_id(dataset.parent_id)
1605
- dataset_name = f"{parent_ds.name}/{dataset.name}"
1606
- else:
1607
- dataset_name = dataset.name
1604
+ name_parts = [dataset.name]
1605
+ parent_id = dataset.parent_id
1606
+ while parent_id is not None:
1607
+ parent_ds = id_to_info.get(parent_id)
1608
+ if parent_ds is None:
1609
+ parent_ds = self._api.dataset.get_info_by_id(parent_id)
1610
+ name_parts.append(parent_ds.name)
1611
+ parent_id = parent_ds.parent_id
1612
+ dataset_name = "/".join(reversed(name_parts))
1608
1613
  ds_infos_dict[dataset_name] = dataset
1609
1614
 
1610
1615
  def get_image_infos_by_split(ds_infos_dict: dict, split: list):
@@ -3157,8 +3162,11 @@ class TrainApp:
3157
3162
 
3158
3163
  # Case 1: Use existing collections for training. No need to create new collections
3159
3164
  split_method = self.gui.train_val_splits_selector.get_split_method()
3165
+ self.gui.train_val_splits_selector._parse_collections()
3160
3166
  all_train_collections = self.gui.train_val_splits_selector.all_train_collections
3161
3167
  all_val_collections = self.gui.train_val_splits_selector.all_val_collections
3168
+ latest_train_collection = self.gui.train_val_splits_selector.latest_train_collection
3169
+ latest_val_collection = self.gui.train_val_splits_selector.latest_val_collection
3162
3170
  if split_method == "Based on collections":
3163
3171
  current_selected_train_collection_ids = self.gui.train_val_splits_selector.train_val_splits.get_train_collections_ids()
3164
3172
  train_match = _check_match(current_selected_train_collection_ids, all_train_collections)
@@ -3173,44 +3181,51 @@ class TrainApp:
3173
3181
  # ------------------------------------------------------------ #
3174
3182
 
3175
3183
  # Case 2: Create new collections for selected train val splits. Need to create new collections
3176
- item_type = self.project_info.type
3177
- experiment_name = self.gui.training_process.get_experiment_name()
3178
-
3179
3184
  train_collection_idx = 1
3180
- val_collection_idx = 1
3185
+ val_collection_idx = 1
3186
+
3187
+ def _extract_index_from_col_name(name: str, expected_prefix: str) -> Optional[int]:
3188
+ parts = name.split("_")
3189
+ if len(parts) == 2 and parts[0] == expected_prefix and parts[1].isdigit():
3190
+ return int(parts[1])
3191
+ return None
3181
3192
 
3182
3193
  # Get train collection with max idx
3183
- if len(all_train_collections) > 0:
3184
- train_collection_idx = max([int(collection.name.split("_")[1]) for collection in all_train_collections])
3185
- train_collection_idx += 1
3194
+ if latest_train_collection:
3195
+ train_collection_idx = (
3196
+ _extract_index_from_col_name(latest_train_collection.name, "train") + 1
3197
+ )
3198
+
3186
3199
  # Get val collection with max idx
3187
- if len(all_val_collections) > 0:
3188
- val_collection_idx = max([int(collection.name.split("_")[1]) for collection in all_val_collections])
3189
- val_collection_idx += 1
3200
+ if latest_val_collection:
3201
+ val_collection_idx = _extract_index_from_col_name(latest_val_collection.name, "val") + 1
3190
3202
  # -------------------------------- #
3191
3203
 
3192
3204
  # Create Train Collection
3193
3205
  train_img_ids = list(self._train_split_item_ids)
3194
- train_collection_description = f"Collection with train {item_type} for experiment: {experiment_name}"
3195
- train_collection = self._api.entities_collection.create(self.project_id, f"train_{train_collection_idx}", train_collection_description)
3196
- train_collection_id = getattr(train_collection, "id", None)
3197
- if train_collection_id is None:
3198
- raise AttributeError("Train EntitiesCollectionInfo object does not have 'id' attribute")
3199
- self._api.entities_collection.add_items(train_collection_id, train_img_ids)
3200
- self._train_collection_id = train_collection_id
3206
+ self._train_collection_id = self._create_collection("train", train_collection_idx)
3207
+ self._api.entities_collection.add_items(self._train_collection_id, train_img_ids)
3201
3208
 
3202
3209
  # Create Val Collection
3203
3210
  val_img_ids = list(self._val_split_item_ids)
3204
- val_collection_description = f"Collection with val {item_type} for experiment: {experiment_name}"
3205
- val_collection = self._api.entities_collection.create(self.project_id, f"val_{val_collection_idx}", val_collection_description)
3206
- val_collection_id = getattr(val_collection, "id", None)
3207
- if val_collection_id is None:
3208
- raise AttributeError("Val EntitiesCollectionInfo object does not have 'id' attribute")
3209
- self._api.entities_collection.add_items(val_collection_id, val_img_ids)
3210
- self._val_collection_id = val_collection_id
3211
+ self._val_collection_id = self._create_collection("val", val_collection_idx)
3212
+ self._api.entities_collection.add_items(self._val_collection_id, val_img_ids)
3211
3213
 
3212
3214
  # Update Project Custom Data
3213
- self._update_project_custom_data(train_collection_id, val_collection_id)
3215
+ self._update_project_custom_data(self._train_collection_id, self._val_collection_id)
3216
+
3217
+ def _create_collection(self, split_type: str, suffix: int) -> int:
3218
+ experiment_name = self.gui.training_process.get_experiment_name()
3219
+ description = f"Collection with {split_type} {self.project_info.type} for experiment: {experiment_name}"
3220
+ collection = self._api.entities_collection.create(
3221
+ project_id=self.project_id,
3222
+ name=f"{split_type}_{suffix:03d}",
3223
+ description=description,
3224
+ change_name_if_conflict=True,
3225
+ )
3226
+ if collection is None or collection.id is None: # pylint: disable=no-member
3227
+ raise RuntimeError(f"Failed to create {split_type} collection")
3228
+ return collection.id # pylint: disable=no-member
3214
3229
 
3215
3230
  def _update_project_custom_data(self, train_collection_id: int, val_collection_id: int):
3216
3231
  train_info = {
@@ -14,7 +14,12 @@ from supervisely.api.module_api import ApiField, ModuleApiBase
14
14
  from supervisely.api.project_api import ProjectInfo
15
15
  from supervisely.io import json
16
16
  from supervisely.io.fs import remove_dir, silent_remove
17
-
17
+ from supervisely.project.versioning.schema_fields import VersionSchemaField
18
+ from supervisely.project.versioning.common import (
19
+ DEFAULT_IMAGE_SCHEMA_VERSION,
20
+ DEFAULT_VIDEO_SCHEMA_VERSION,
21
+ DEFAULT_VOLUME_SCHEMA_VERSION,
22
+ )
18
23
 
19
24
  class VersionInfo(NamedTuple):
20
25
  """
@@ -49,7 +54,7 @@ class DataVersion(ModuleApiBase):
49
54
 
50
55
  self._api: Api = api
51
56
  self.__storage_dir: str = "/system/versions/"
52
- self.__version_format: str = "v1.0.0"
57
+ self.__version_format: str = DEFAULT_IMAGE_SCHEMA_VERSION
53
58
  self.project_info = None
54
59
  self.project_dir = None
55
60
  self.versions_path = None
@@ -84,6 +89,31 @@ class DataVersion(ModuleApiBase):
84
89
  """
85
90
  return "VersionInfo"
86
91
 
92
+ @property
93
+ def project_cls(self):
94
+ from supervisely.project import (
95
+ Project,
96
+ ProjectType,
97
+ VideoProject,
98
+ VolumeProject,
99
+ )
100
+
101
+ if self.project_info is None:
102
+ raise ValueError("Project info is not initialized. Call 'initialize' method first.")
103
+
104
+ project_type = self.project_info.type
105
+ if project_type == ProjectType.IMAGES.value:
106
+ self.__version_format = DEFAULT_IMAGE_SCHEMA_VERSION
107
+ return Project
108
+ elif project_type == ProjectType.VIDEOS.value:
109
+ self.__version_format = DEFAULT_VIDEO_SCHEMA_VERSION
110
+ return VideoProject
111
+ elif project_type == ProjectType.VOLUMES.value:
112
+ self.__version_format = DEFAULT_VOLUME_SCHEMA_VERSION
113
+ return VolumeProject
114
+ else:
115
+ raise ValueError(f"Unsupported project type: {project_type}")
116
+
87
117
  def initialize(self, project_info: Union[ProjectInfo, int]):
88
118
  """
89
119
  Initialize project versions.
@@ -158,10 +188,10 @@ class DataVersion(ModuleApiBase):
158
188
  versions = self._api.file.get_json_file_content(
159
189
  self.project_info.team_id, self.versions_path
160
190
  )
161
- versions = versions if versions else {}
191
+ return versions or {}
162
192
  except FileNotFoundError:
163
- versions = {"format": self.__version_format}
164
- return versions
193
+ # versions = {"format": self.__version_format}
194
+ return {}
165
195
 
166
196
  def set_map(self, project_info: Union[ProjectInfo, int], initialize: bool = True):
167
197
  """
@@ -176,6 +206,8 @@ class DataVersion(ModuleApiBase):
176
206
 
177
207
  if initialize:
178
208
  self.initialize(project_info)
209
+ if "format" not in self.versions:
210
+ self.versions["format"] = self.__version_format
179
211
  temp_dir = tempfile.mkdtemp()
180
212
  local_versions = os.path.join(temp_dir, "versions.json")
181
213
  json.dump_json_file(self.versions, local_versions)
@@ -329,21 +361,32 @@ class DataVersion(ModuleApiBase):
329
361
  return reserve_info.get(ApiField.ID), reserve_info.get(ApiField.COMMIT_TOKEN)
330
362
 
331
363
  except requests.exceptions.HTTPError as e:
332
- if e.response.json().get("details", {}).get("useExistingVersion"):
333
- version_id = e.response.json().get("details", {}).get("version").get("id")
334
- version = e.response.json().get("details", {}).get("version").get("version")
364
+ details = {}
365
+ if e.response is not None:
366
+ try:
367
+ details = (e.response.json() or {}).get("details", {}) # type: ignore[union-attr]
368
+ except Exception:
369
+ details = {}
370
+
371
+ if details.get("useExistingVersion"):
372
+ version_id = details.get("version", {}).get("id")
373
+ version = details.get("version", {}).get("version")
335
374
  logger.info(
336
375
  f"No changes to the project since the last version '{version}' with ID '{version_id}'"
337
376
  )
338
377
  return (None, None)
339
- elif "is already committing" in e.response.json().get("details", {}).get("message"):
378
+
379
+ message = (details.get("message") or "").lower()
380
+ if "is already committing" in message:
340
381
  if retry_delay >= max_delay:
341
382
  raise RuntimeError(
342
383
  "Failed to reserve version. Another process is already committing a version. Maximum number of attempts reached."
343
384
  )
344
- version = e.response.json().get("details", {}).get("version").get("version")
345
385
  time.sleep(retry_delay)
346
386
  retry_delay *= 2
387
+ continue
388
+
389
+ raise
347
390
 
348
391
  def cancel_reservation(self, version_id: int, commit_token: str):
349
392
  """
@@ -384,8 +427,6 @@ class DataVersion(ModuleApiBase):
384
427
  :return: ProjectInfo object of the restored project
385
428
  :rtype: ProjectInfo or None
386
429
  """
387
- from supervisely.project.project import Project
388
-
389
430
  if version_id is None and version_num is None:
390
431
  raise ValueError("Either version_id or version_num must be provided")
391
432
 
@@ -421,7 +462,7 @@ class DataVersion(ModuleApiBase):
421
462
  return
422
463
 
423
464
  bin_io = self._download_and_extract(backup_files)
424
- new_project_info = Project.upload_bin(
465
+ new_project_info = self.project_cls.upload_bin(
425
466
  self._api,
426
467
  bin_io,
427
468
  self.project_info.workspace_id,
@@ -456,15 +497,28 @@ class DataVersion(ModuleApiBase):
456
497
  local_path = os.path.join(temp_dir, "download.tar.zst")
457
498
  try:
458
499
  self._api.file.download(self.project_info.team_id, path, local_path)
459
- with open(local_path, "rb") as zst:
460
- decompressed_data = zstd.decompress(zst.read())
461
- with tarfile.open(fileobj=io.BytesIO(decompressed_data)) as tar:
462
- file = tar.extractfile("version.bin")
463
- if not file:
464
- raise RuntimeError("version.bin not found in the archive")
465
- data = file.read()
466
- bin_io = io.BytesIO(data)
467
- return bin_io
500
+ # Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
501
+ try:
502
+ dctx = zstd.ZstdDecompressor()
503
+ with open(local_path, "rb") as zst_f:
504
+ with dctx.stream_reader(zst_f) as reader:
505
+ with tarfile.open(fileobj=reader, mode="r|") as tar:
506
+ for member in tar:
507
+ if member.name == "version.bin":
508
+ file = tar.extractfile(member)
509
+ if not file:
510
+ raise RuntimeError("version.bin not found in the archive")
511
+ return io.BytesIO(file.read())
512
+ raise RuntimeError("version.bin not found in the archive")
513
+ except Exception:
514
+ # Fallback: one-shot decompress
515
+ with open(local_path, "rb") as zst_f:
516
+ decompressed_data = zstd.decompress(zst_f.read())
517
+ with tarfile.open(fileobj=io.BytesIO(decompressed_data), mode="r") as tar:
518
+ file = tar.extractfile("version.bin")
519
+ if not file:
520
+ raise RuntimeError("version.bin not found in the archive")
521
+ return io.BytesIO(file.read())
468
522
  except Exception as e:
469
523
  raise RuntimeError(f"Failed to extract version: {e}")
470
524
  finally:
@@ -501,34 +555,44 @@ class DataVersion(ModuleApiBase):
501
555
  :return: File info
502
556
  :rtype: dict
503
557
  """
504
- from supervisely.project.project import Project
505
-
506
558
  temp_dir = tempfile.mkdtemp()
559
+ data = None
560
+ try:
561
+ data = self.project_cls.download_bin(
562
+ self._api, self.project_info.id, batch_size=200, return_bytesio=True
563
+ )
564
+ info = tarfile.TarInfo(name="version.bin")
565
+ data.seek(0, io.SEEK_END)
566
+ info.size = data.tell()
567
+ data.seek(0)
568
+ zst_archive_path = os.path.join(temp_dir, "download.tar.zst")
507
569
 
508
- data = Project.download_bin(
509
- self._api, self.project_info.id, batch_size=200, return_bytesio=True
510
- )
511
- data.seek(0)
512
- info = tarfile.TarInfo(name="version.bin")
513
- info.size = len(data.getvalue())
514
- chunk_size = 1024 * 1024 * 50 # 50 MiB
515
- tar_data = io.BytesIO()
516
-
517
- # Create a tarfile object that writes into the BytesIO object
518
- with tarfile.open(fileobj=tar_data, mode="w") as tar:
519
- tar.addfile(tarinfo=info, fileobj=data)
520
- data.close()
521
- # Reset the BytesIO object's cursor to the beginning
522
- tar_data.seek(0)
523
- zst_archive_path = os.path.join(temp_dir, "download.tar.zst")
524
-
525
- with open(zst_archive_path, "wb") as zst:
526
- while True:
527
- chunk = tar_data.read(chunk_size)
528
- if not chunk:
529
- break
530
- zst.write(zstd.compress(chunk))
531
- file_info = self._api.file.upload(self.project_info.team_id, zst_archive_path, path)
532
- tar_data.close()
533
- remove_dir(temp_dir)
534
- return file_info
570
+ # Stream-decompress and stream-read tar to avoid loading the whole archive in memory.
571
+ try:
572
+ cctx = zstd.ZstdCompressor()
573
+ with open(zst_archive_path, "wb") as zst_f:
574
+ try:
575
+ stream = cctx.stream_writer(zst_f, closefd=False)
576
+ except TypeError:
577
+ stream = cctx.stream_writer(zst_f)
578
+ with stream as compressor:
579
+ with tarfile.open(fileobj=compressor, mode="w|") as tar:
580
+ tar.addfile(tarinfo=info, fileobj=data)
581
+ except Exception:
582
+ # Fallback: build tar in memory + one-shot compress
583
+ tar_data = io.BytesIO()
584
+ with tarfile.open(fileobj=tar_data, mode="w") as tar:
585
+ tar.addfile(tarinfo=info, fileobj=data)
586
+ tar_data.seek(0)
587
+ with open(zst_archive_path, "wb") as zst_f:
588
+ zst_f.write(zstd.compress(tar_data.read()))
589
+
590
+ file_info = self._api.file.upload(self.project_info.team_id, zst_archive_path, path)
591
+ return file_info
592
+ finally:
593
+ if data is not None:
594
+ try:
595
+ data.close()
596
+ except Exception:
597
+ pass
598
+ remove_dir(temp_dir)
@@ -216,7 +216,7 @@ def download_async_or_sync(
216
216
  dataset_ids: Optional[List[int]] = None,
217
217
  log_progress: bool = True,
218
218
  progress_cb: Optional[Union[tqdm, Callable]] = None,
219
- semaphore: Optional[asyncio.Semaphore] = None,
219
+ semaphore: Optional[Union[asyncio.Semaphore, int]] = None,
220
220
  **kwargs,
221
221
  ):
222
222
  """
@@ -655,6 +655,25 @@ class PointcloudEpisodeProject(PointcloudProject):
655
655
  f"Static method 'download_async()' is not supported for PointcloudEpisodeProject class now."
656
656
  )
657
657
 
658
+ # ----------------------------------- #
659
+ # Pointcloud Episodes Data Versioning #
660
+ # ----------------------------------- #
661
+ @staticmethod
662
+ def download_bin(*args, **kwargs):
663
+ raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
664
+
665
+ @staticmethod
666
+ def upload_bin(*args, **kwargs):
667
+ raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
668
+
669
+ @staticmethod
670
+ def build_snapshot(*args, **kwargs):
671
+ raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
672
+
673
+ @staticmethod
674
+ def restore_snapshot(*args, **kwargs):
675
+ raise NotImplementedError("Data versioning is not supported for PointcloudEpisodeProject.")
676
+
658
677
 
659
678
  def download_pointcloud_episode_project(
660
679
  api: Api,
@@ -946,7 +965,7 @@ def upload_pointcloud_episode_project(
946
965
  log_progress: bool = True,
947
966
  progress_cb: Optional[Union[tqdm, Callable]] = None,
948
967
  ) -> Tuple[int, str]:
949
- # STEP 0 create project remotely
968
+ # STEP 0 - create project remotely
950
969
  project_fs = PointcloudEpisodeProject.read_single(directory)
951
970
  project_name = project_fs.name if project_name is None else project_name
952
971
 
@@ -959,8 +978,9 @@ def upload_pointcloud_episode_project(
959
978
  if progress_cb is not None:
960
979
  log_progress = False
961
980
 
981
+ name_to_dsinfo = {}
962
982
  key_id_map = KeyIdMap()
963
- for dataset_fs in project_fs.datasets:
983
+ for dataset_fs in sorted(project_fs.datasets, key=lambda ds: len(ds.parents)):
964
984
  dataset_fs: PointcloudEpisodeDataset
965
985
  ann_json_path = dataset_fs.get_ann_path()
966
986
 
@@ -970,14 +990,19 @@ def upload_pointcloud_episode_project(
970
990
  else:
971
991
  episode_annotation = PointcloudEpisodeAnnotation()
972
992
 
993
+ parent_path = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
994
+ parent_info = name_to_dsinfo.get(parent_path)
995
+ parent_id = parent_info.id if parent_info else None
973
996
  dataset = api.dataset.create(
974
997
  project.id,
975
- dataset_fs.name,
998
+ dataset_fs.short_name,
976
999
  description=episode_annotation.description,
977
1000
  change_name_if_conflict=True,
1001
+ parent_id=parent_id,
978
1002
  )
1003
+ name_to_dsinfo[dataset_fs.name] = dataset
979
1004
 
980
- # STEP 1 upload episodes
1005
+ # STEP 1 - upload episodes
981
1006
  items_infos = {"names": [], "paths": [], "metas": []}
982
1007
 
983
1008
  for item_name in dataset_fs:
@@ -989,6 +1014,10 @@ def upload_pointcloud_episode_project(
989
1014
  items_infos["paths"].append(item_path)
990
1015
  items_infos["metas"].append(item_meta)
991
1016
 
1017
+ if not items_infos["names"]:
1018
+ logger.info(f"Dataset {dataset.name} has no items, skipping upload")
1019
+ continue
1020
+
992
1021
  ds_progress = progress_cb
993
1022
  if log_progress:
994
1023
  ds_progress = tqdm_sly(
@@ -1015,7 +1044,7 @@ def upload_pointcloud_episode_project(
1015
1044
  },
1016
1045
  )
1017
1046
  raise e
1018
- # STEP 2 upload annotations
1047
+ # STEP 2 - upload annotations
1019
1048
  frame_to_pcl_ids = {pcl_info.frame: pcl_info.id for pcl_info in pcl_infos}
1020
1049
  try:
1021
1050
  api.pointcloud_episode.annotation.append(
@@ -1033,9 +1062,9 @@ def upload_pointcloud_episode_project(
1033
1062
  )
1034
1063
  raise e
1035
1064
 
1036
- # STEP 3 upload photo context
1065
+ # STEP 3 - upload photo context
1037
1066
  img_infos = {"img_paths": [], "img_metas": []}
1038
- # STEP 3.1 upload images
1067
+ # STEP 3.1 - upload images
1039
1068
  pcl_to_rimg_figures: Dict[int, Dict[str, List[Dict]]] = {}
1040
1069
  pcl_to_hash_to_id: Dict[int, Dict[str, int]] = {}
1041
1070
  for pcl_info in pcl_infos:
@@ -1067,7 +1096,7 @@ def upload_pointcloud_episode_project(
1067
1096
  )
1068
1097
  raise e
1069
1098
 
1070
- # STEP 3.2 upload images metas
1099
+ # STEP 3.2 - upload images metas
1071
1100
  images_hashes_iterator = images_hashes.__iter__()
1072
1101
  for pcl_info in pcl_infos:
1073
1102
  related_items = dataset_fs.get_related_images(pcl_info.name)
@@ -908,6 +908,25 @@ class PointcloudProject(VideoProject):
908
908
  f"Static method 'download_async()' is not supported for PointcloudProject class now."
909
909
  )
910
910
 
911
+ # -------------------------- #
912
+ # Pointcloud Data Versioning #
913
+ # -------------------------- #
914
+ @staticmethod
915
+ def download_bin(*args, **kwargs):
916
+ raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
917
+
918
+ @staticmethod
919
+ def upload_bin(*args, **kwargs):
920
+ raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
921
+
922
+ @staticmethod
923
+ def build_snapshot(*args, **kwargs):
924
+ raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
925
+
926
+ @staticmethod
927
+ def restore_snapshot(*args, **kwargs):
928
+ raise NotImplementedError("Data versioning is not supported for PointcloudProject.")
929
+
911
930
 
912
931
  def download_pointcloud_project(
913
932
  api: Api,
@@ -1216,8 +1235,17 @@ def upload_pointcloud_project(
1216
1235
  log_progress = False
1217
1236
 
1218
1237
  key_id_map = KeyIdMap()
1219
- for dataset_fs in project_fs:
1220
- dataset = api.dataset.create(project.id, dataset_fs.name, change_name_if_conflict=True)
1238
+ name_to_dsinfo = {}
1239
+ for dataset_fs in sorted(project_fs, key=lambda ds: len(ds.parents)):
1240
+ parent_name = dataset_fs.name.removesuffix(dataset_fs.short_name).rstrip("/")
1241
+ parent_info = name_to_dsinfo.get(parent_name)
1242
+ parent_id = None
1243
+ if parent_info is not None:
1244
+ parent_id = parent_info.id
1245
+ dataset = api.dataset.create(
1246
+ project.id, dataset_fs.short_name, change_name_if_conflict=True, parent_id=parent_id
1247
+ )
1248
+ name_to_dsinfo[dataset_fs.name] = dataset
1221
1249
 
1222
1250
  ds_progress = progress_cb
1223
1251
  if log_progress:
@@ -3626,7 +3626,11 @@ class Project:
3626
3626
  if project_name is None:
3627
3627
  project_name = project_info.name
3628
3628
  new_project_info = api.project.create(
3629
- workspace_id, project_name, change_name_if_conflict=True
3629
+ workspace_id,
3630
+ project_name,
3631
+ description=project_info.description,
3632
+ change_name_if_conflict=True,
3633
+ readme=project_info.readme,
3630
3634
  )
3631
3635
  custom_data = new_project_info.custom_data
3632
3636
  version_num = project_info.version.get("version", None) if project_info.version else 0
@@ -4584,6 +4588,7 @@ def upload_project(
4584
4588
  blob_file_infos = []
4585
4589
 
4586
4590
  for ds_fs in project_fs.datasets:
4591
+ logger.debug(f"Processing dataset: {ds_fs.name}")
4587
4592
  if len(ds_fs.parents) > 0:
4588
4593
  parent = f"{os.path.sep}".join(ds_fs.parents)
4589
4594
  parent_id = dataset_map.get(parent)
@@ -4624,8 +4629,15 @@ def upload_project(
4624
4629
  if os.path.isfile(path):
4625
4630
  valid_indices.append(i)
4626
4631
  valid_paths.append(path)
4627
- else:
4632
+ elif len(project_fs.blob_files) > 0:
4628
4633
  offset_indices.append(i)
4634
+ else:
4635
+ if img_infos[i] is not None:
4636
+ logger.debug(f"Image will be uploaded by image_info: {names[i]}")
4637
+ else:
4638
+ logger.warning(
4639
+ f"Image and image info file not found, image will be skipped: {names[i]}"
4640
+ )
4629
4641
  img_paths = valid_paths
4630
4642
  ann_paths = list(filter(lambda x: os.path.isfile(x), ann_paths))
4631
4643
  # Create a mapping from name to index position for quick lookups
@@ -14,7 +14,7 @@ from supervisely.geometry.bitmap import Bitmap
14
14
  from supervisely.geometry.polygon import Polygon
15
15
  from supervisely.geometry.rectangle import Rectangle
16
16
  from supervisely.io.json import JsonSerializable
17
- from supervisely.project.project_settings import ProjectSettings
17
+ from supervisely.project.project_settings import LabelingInterface, ProjectSettings
18
18
  from supervisely.project.project_type import ProjectType
19
19
 
20
20
 
@@ -288,6 +288,32 @@ class ProjectMeta(JsonSerializable):
288
288
  # Output: <class 'supervisely.project.project_settings.ProjectSettings'>
289
289
  """
290
290
  return self._project_settings
291
+
292
+ @property
293
+ def labeling_interface(self) -> Optional[LabelingInterface]:
294
+ """
295
+ Get labeling interface settings of the project.
296
+
297
+ :return: Labeling interface settings
298
+ :rtype: :class: `LabelingInterface` or None
299
+ :Usage example:
300
+
301
+ .. code-block:: python
302
+
303
+ import supervisely as sly
304
+
305
+ s = sly.ProjectSettings(
306
+ multiview_enabled=True,
307
+ multiview_tag_name='multi_tag',
308
+ multiview_is_synced=False,
309
+ )
310
+ meta = sly.ProjectMeta(project_settings=s)
311
+
312
+ labeling_interface = meta.labeling_interface
313
+ print(labeling_interface)
314
+ # Output: None
315
+ """
316
+ return self._project_settings.labeling_interface if self._project_settings else None
291
317
 
292
318
  def to_json(self) -> Dict:
293
319
  """