dtlpy 1.115.44__py3-none-any.whl → 1.117.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +152 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +975 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +974 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1287 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1585 -1504
  167. dtlpy/repositories/downloader.py +1157 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +256 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +429 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1786 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.117.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.117.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/top_level.txt +0 -0
dtlpy/entities/item.py CHANGED
@@ -1,959 +1,975 @@
1
- from collections import namedtuple
2
- from enum import Enum
3
- import traceback
4
- import mimetypes
5
- import logging
6
- import attr
7
- import copy
8
- import os
9
- import io
10
- from .. import repositories, entities, exceptions
11
- from .annotation import ViewAnnotationOptions, ExportVersion
12
- from ..services.api_client import ApiClient
13
- from ..services.api_client import client as client_api
14
- import json
15
- from typing import List
16
- import requests
17
-
18
- logger = logging.getLogger(name='dtlpy')
19
-
20
-
21
- class ExportMetadata(Enum):
22
- FROM_JSON = 'from_json'
23
-
24
-
25
- class ItemStatus(str, Enum):
26
- COMPLETED = "completed"
27
- APPROVED = "approved"
28
- DISCARDED = "discard"
29
-
30
-
31
- @attr.s
32
- class Item(entities.BaseEntity):
33
- """
34
- Item object
35
- """
36
- # item information
37
- annotations_link = attr.ib(repr=False)
38
- dataset_url = attr.ib()
39
- thumbnail = attr.ib(repr=False)
40
- created_at = attr.ib()
41
- updated_at = attr.ib()
42
- updated_by = attr.ib()
43
- dataset_id = attr.ib()
44
- annotated = attr.ib(repr=False)
45
- metadata = attr.ib(repr=False)
46
- filename = attr.ib()
47
- stream = attr.ib(repr=False)
48
- name = attr.ib()
49
- type = attr.ib()
50
- url = attr.ib(repr=False)
51
- id = attr.ib()
52
- hidden = attr.ib(repr=False)
53
- dir = attr.ib(repr=False)
54
- spec = attr.ib()
55
- creator = attr.ib()
56
- _description = attr.ib()
57
- _src_item = attr.ib(repr=False)
58
-
59
- # name change
60
- annotations_count = attr.ib()
61
-
62
- # api
63
- _client_api = attr.ib(type=ApiClient, repr=False)
64
- _platform_dict = attr.ib(repr=False)
65
-
66
- # entities
67
- _dataset = attr.ib(repr=False)
68
- _model = attr.ib(repr=False)
69
- _project = attr.ib(repr=False)
70
- _project_id = attr.ib(repr=False)
71
-
72
- # repositories
73
- _repositories = attr.ib(repr=False)
74
-
75
- @property
76
- def createdAt(self):
77
- return self.created_at
78
-
79
- @property
80
- def datasetId(self):
81
- return self.dataset_id
82
-
83
- @staticmethod
84
- def _protected_from_json(_json, client_api, dataset=None):
85
- """
86
- Same as from_json but with try-except to catch if error
87
- :param _json: platform json
88
- :param client_api: ApiClient entity
89
- :param dataset: dataset entity
90
- :return:
91
- """
92
- try:
93
- item = Item.from_json(_json=_json,
94
- client_api=client_api,
95
- dataset=dataset)
96
- status = True
97
- except Exception:
98
- item = traceback.format_exc()
99
- status = False
100
- return status, item
101
-
102
- @classmethod
103
- def from_json(cls, _json, client_api, dataset=None, project=None, model=None, is_fetched=True):
104
- """
105
- Build an item entity object from a json
106
-
107
- :param dtlpy.entities.project.Project project: project entity
108
- :param dict _json: _json response from host
109
- :param dtlpy.entities.dataset.Dataset dataset: dataset in which the annotation's item is located
110
- :param dtlpy.entities.dataset.Model model: the model entity if item is an artifact of a model
111
- :param dlApiClient client_api: ApiClient entity
112
- :param bool is_fetched: is Entity fetched from Platform
113
- :return: Item object
114
- :rtype: dtlpy.entities.item.Item
115
- """
116
- dataset_id = None
117
- if dataset is not None:
118
- dataset_id = _json.get('datasetId', None)
119
- if dataset.id != dataset_id and dataset_id is not None:
120
- logger.warning('Item has been fetched from a dataset that is not belong to it')
121
- dataset = None
122
- else:
123
- dataset_id = dataset.id
124
-
125
- metadata = _json.get('metadata', dict())
126
- project_id = _json.get('projectId', None)
127
- if project_id is None:
128
- project_id = project.id if project else None
129
- inst = cls(
130
- # sdk
131
- platform_dict=copy.deepcopy(_json),
132
- client_api=client_api,
133
- dataset=dataset,
134
- project=project,
135
- model=model,
136
- # params
137
- annotations_link=_json.get('annotations', None),
138
- thumbnail=_json.get('thumbnail', None),
139
- dataset_id=_json.get('datasetId', dataset_id),
140
- annotated=_json.get('annotated', None),
141
- dataset_url=_json.get('dataset', None),
142
- created_at=_json.get('createdAt', None),
143
- annotations_count=_json.get('annotationsCount', None),
144
- hidden=_json.get('hidden', False),
145
- stream=_json.get('stream', None),
146
- dir=_json.get('dir', None),
147
- filename=_json.get('filename', None),
148
- metadata=metadata,
149
- name=_json.get('name', None),
150
- type=_json.get('type', None),
151
- url=_json.get('url', None),
152
- id=_json.get('id', None),
153
- spec=_json.get('spec', None),
154
- creator=_json.get('creator', None),
155
- project_id=project_id,
156
- description=_json.get('description', None),
157
- src_item=_json.get('srcItem', None),
158
- updated_at=_json.get('updatedAt', None),
159
- updated_by=_json.get('updatedBy', None)
160
- )
161
- inst.is_fetched = is_fetched
162
- return inst
163
-
164
- def __getstate__(self):
165
- # dump state to json
166
- return self.to_json()
167
-
168
- def __setstate__(self, state):
169
- # create a new item, and update the current one with the same state
170
- # this way we can have _client_api, and all the repositories and entities which are not picklable
171
- self.__dict__.update(entities.Item.from_json(_json=state,
172
- client_api=client_api).__dict__)
173
-
174
- ############
175
- # entities #
176
- ############
177
- @property
178
- def dataset(self):
179
- if self._dataset is None:
180
- self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
181
- assert isinstance(self._dataset, entities.Dataset)
182
- return self._dataset
183
-
184
- @property
185
- def model(self):
186
- return self._model
187
-
188
- def __update_item_binary(self, _json):
189
- binary = io.BytesIO()
190
- binary.write(json.dumps(_json).encode())
191
- binary.seek(0)
192
- binary.name = self.name
193
- success, resp = client_api.gen_request(req_type='post',
194
- path=f'/items/{self.id}/revisions',
195
- files={'file': (binary.name, binary)})
196
- if not success:
197
- raise exceptions.PlatformException(resp)
198
-
199
- @property
200
- def project(self):
201
- if self._project is None:
202
- if self._dataset is None:
203
- self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
204
- self._project = self._dataset.project
205
- if self._project is None:
206
- raise exceptions.PlatformException(error='2001',
207
- message='Missing entity "project".')
208
- assert isinstance(self._project, entities.Project)
209
- return self._project
210
-
211
- @property
212
- def project_id(self):
213
- if self._project_id is None:
214
- if self._dataset is None:
215
- self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
216
- self._project_id = self._dataset.project.id
217
- return self._project_id
218
-
219
- ################
220
- # repositories #
221
- ################
222
-
223
- @_repositories.default
224
- def set_repositories(self):
225
- reps = namedtuple('repositories',
226
- field_names=['annotations', 'datasets', 'items', 'codebases', 'artifacts', 'modalities',
227
- 'features', 'assignments', 'tasks', 'resource_executions', 'collections'])
228
- reps.__new__.__defaults__ = (None, None, None, None, None, None, None, None, None)
229
-
230
- if self._dataset is None:
231
- items = repositories.Items(
232
- client_api=self._client_api,
233
- dataset=self._dataset,
234
- dataset_id=self.dataset_id,
235
- datasets=repositories.Datasets(client_api=self._client_api, project=None)
236
- )
237
- datasets = items.datasets
238
-
239
- else:
240
- items = self.dataset.items
241
- datasets = self.dataset.datasets
242
-
243
- r = reps(
244
- annotations=repositories.Annotations(
245
- client_api=self._client_api,
246
- dataset_id=self.dataset_id,
247
- item=self,
248
- dataset=self._dataset
249
- ),
250
- items=items,
251
- datasets=datasets,
252
- codebases=None,
253
- artifacts=None,
254
- modalities=Modalities(item=self),
255
- features=repositories.Features(
256
- client_api=self._client_api,
257
- project=self._project,
258
- item=self
259
- ),
260
- tasks=repositories.Tasks(
261
- client_api=self._client_api,
262
- project=self._project,
263
- dataset=self._dataset
264
- ),
265
- assignments=repositories.Assignments(
266
- client_api=self._client_api,
267
- project=self._project,
268
- dataset=self._dataset
269
- ),
270
- resource_executions=repositories.ResourceExecutions(
271
- client_api=self._client_api,
272
- project=self._project,
273
- resource=self
274
- ),
275
- collections=repositories.Collections(client_api=self._client_api, item=self, dataset=self._dataset)
276
- )
277
- return r
278
-
279
- @property
280
- def modalities(self):
281
- assert isinstance(self._repositories.modalities, Modalities)
282
- return self._repositories.modalities
283
-
284
- @property
285
- def annotations(self):
286
- assert isinstance(self._repositories.annotations, repositories.Annotations)
287
- return self._repositories.annotations
288
-
289
- @property
290
- def datasets(self):
291
- assert isinstance(self._repositories.datasets, repositories.Datasets)
292
- return self._repositories.datasets
293
-
294
- @property
295
- def assignments(self):
296
- assert isinstance(self._repositories.assignments, repositories.Assignments)
297
- return self._repositories.assignments
298
-
299
- @property
300
- def tasks(self):
301
- assert isinstance(self._repositories.tasks, repositories.Tasks)
302
- return self._repositories.tasks
303
-
304
- @property
305
- def resource_executions(self):
306
- assert isinstance(self._repositories.resource_executions, repositories.ResourceExecutions)
307
- return self._repositories.resource_executions
308
-
309
- @property
310
- def items(self):
311
- assert isinstance(self._repositories.items, repositories.Items)
312
- return self._repositories.items
313
-
314
- @property
315
- def features(self):
316
- assert isinstance(self._repositories.features, repositories.Features)
317
- return self._repositories.features
318
-
319
- @property
320
- def collections(self):
321
- assert isinstance(self._repositories.collections, repositories.Collections)
322
- return self._repositories.collections
323
-
324
- ##############
325
- # Properties #
326
- ##############
327
- @property
328
- def height(self):
329
- return self.metadata.get('system', dict()).get('height', None)
330
-
331
- @height.setter
332
- def height(self, val):
333
- if 'system' not in self.metadata:
334
- self.metadata['system'] = dict()
335
- self.metadata['system']['height'] = val
336
-
337
- @property
338
- def width(self):
339
- return self.metadata.get('system', dict()).get('width', None)
340
-
341
- @width.setter
342
- def width(self, val):
343
- if 'system' not in self.metadata:
344
- self.metadata['system'] = dict()
345
- self.metadata['system']['width'] = val
346
-
347
- @property
348
- def fps(self):
349
- return self.metadata.get('fps', None)
350
-
351
- @fps.setter
352
- def fps(self, val):
353
- self.metadata['fps'] = val
354
-
355
- @property
356
- def mimetype(self):
357
- mimetype = self.metadata.get('system', dict()).get('mimetype', None)
358
- if mimetype is None:
359
- mimetype = mimetypes.guess_type(self.filename)[0]
360
- return mimetype
361
-
362
- @property
363
- def size(self):
364
- return self.metadata.get('system', dict()).get('size', None)
365
-
366
- @property
367
- def system(self):
368
- return self.metadata.get('system', dict())
369
-
370
- @property
371
- def description(self):
372
- description = None
373
- if self._description is not None:
374
- description = self._description
375
- elif 'description' in self.metadata:
376
- description = self.metadata['description'].get('text', None)
377
- return description
378
-
379
- @property
380
- def platform_url(self):
381
- return self._client_api._get_resource_url(
382
- "projects/{}/datasets/{}/items/{}".format(self.dataset.projects[-1], self.dataset.id, self.id))
383
-
384
- @description.setter
385
- def description(self, text: str):
386
- """
387
- Update Item description
388
-
389
- :param text: if None or "" description will be deleted
390
- :return
391
- """
392
- self.set_description(text=text)
393
-
394
- ###########
395
- # Functions #
396
- ###########
397
- def to_json(self):
398
- """
399
- Returns platform _json format of object
400
-
401
- :return: platform json format of object
402
- :rtype: dict
403
- """
404
- _json = attr.asdict(self,
405
- filter=attr.filters.exclude(attr.fields(Item)._repositories,
406
- attr.fields(Item)._dataset,
407
- attr.fields(Item)._model,
408
- attr.fields(Item)._project,
409
- attr.fields(Item)._client_api,
410
- attr.fields(Item)._platform_dict,
411
- attr.fields(Item).annotations_count,
412
- attr.fields(Item).dataset_url,
413
- attr.fields(Item).annotations_link,
414
- attr.fields(Item).spec,
415
- attr.fields(Item).creator,
416
- attr.fields(Item).created_at,
417
- attr.fields(Item).dataset_id,
418
- attr.fields(Item)._project_id,
419
- attr.fields(Item)._description,
420
- attr.fields(Item)._src_item,
421
- attr.fields(Item).updated_at,
422
- attr.fields(Item).updated_by
423
- ))
424
-
425
- _json.update({'annotations': self.annotations_link,
426
- 'annotationsCount': self.annotations_count,
427
- 'dataset': self.dataset_url,
428
- 'createdAt': self.created_at,
429
- 'datasetId': self.dataset_id,
430
- })
431
- if self.spec is not None:
432
- _json['spec'] = self.spec
433
- if self.creator is not None:
434
- _json['creator'] = self.creator
435
- if self._description is not None:
436
- _json['description'] = self.description
437
- if self._src_item is not None:
438
- _json['srcItem'] = self._src_item
439
-
440
- _json['updatedAt'] = self.updated_at
441
- _json['updatedBy'] = self.updated_by
442
-
443
- return _json
444
-
445
- def download(
446
- self,
447
- # download options
448
- local_path=None,
449
- file_types=None,
450
- save_locally=True,
451
- to_array=False,
452
- annotation_options: ViewAnnotationOptions = None,
453
- overwrite=False,
454
- to_items_folder=True,
455
- thickness=1,
456
- with_text=False,
457
- annotation_filters=None,
458
- alpha=1,
459
- export_version=ExportVersion.V1,
460
- dataset_lock=False,
461
- lock_timeout_sec=None,
462
- export_summary=False,
463
- raise_on_error=False
464
- ):
465
- """
466
- Download dataset by filters.
467
- Filtering the dataset for items and save them local
468
- Optional - also download annotation, mask, instance and image mask of the item
469
-
470
- :param str local_path: local folder or filename to save to.
471
- :param list file_types: a list of file type to download. e.g ['video/webm', 'video/mp4', 'image/jpeg', 'image/png']
472
- :param bool save_locally: bool. save to disk or return a buffer
473
- :param bool to_array: returns Ndarray when True and local_path = False
474
- :param list annotation_options: download annotations options: list(dl.ViewAnnotationOptions)
475
- :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
476
- :param bool overwrite: optional - default = False
477
- :param bool dataset_lock: optional - default = False
478
- :param bool export_summary: optional - default = False
479
- :param int lock_timeout_sec: optional
480
- :param bool to_items_folder: Create 'items' folder and download items to it
481
- :param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
482
- :param bool with_text: optional - add text to annotations, default = False
483
- :param float alpha: opacity value [0 1], default 1
484
- :param str export_version: exported items will have original extension in filename, `V1` - no original extension in filenames
485
- :param bool raise_on_error: raise an exception if an error occurs
486
- :return: generator of local_path per each downloaded item
487
- :rtype: generator or single item
488
-
489
- **Example**:
490
-
491
- .. code-block:: python
492
-
493
- item.download(local_path='local_path',
494
- annotation_options=dl.ViewAnnotationOptions.MASK,
495
- overwrite=False,
496
- thickness=1,
497
- with_text=False,
498
- alpha=1,
499
- save_locally=True,
500
- dataset_lock=False
501
- lock_timeout_sec=300,
502
- export_summary=False
503
- )
504
- """
505
- # if dir - concatenate local path and item name
506
- if local_path is not None:
507
- if os.path.isdir(local_path):
508
- local_path = os.path.join(local_path, self.name)
509
- else:
510
- _, ext = os.path.splitext(local_path)
511
- _, item_ext = os.path.splitext(self.name)
512
- if not ext or ext != item_ext:
513
- os.makedirs(local_path, exist_ok=True)
514
- local_path = os.path.join(local_path, self.name)
515
-
516
- # download
517
- filters = None
518
- items = self
519
- if self.type == 'dir':
520
- filters = self.datasets._bulid_folder_filter(folder_path=self.filename)
521
- items = None
522
-
523
- return self.items.download(items=items,
524
- local_path=local_path,
525
- file_types=file_types,
526
- save_locally=save_locally,
527
- to_array=to_array,
528
- annotation_options=annotation_options,
529
- overwrite=overwrite,
530
- to_items_folder=to_items_folder,
531
- annotation_filters=annotation_filters,
532
- thickness=thickness,
533
- alpha=alpha,
534
- with_text=with_text,
535
- export_version=export_version,
536
- filters=filters,
537
- dataset_lock=dataset_lock,
538
- lock_timeout_sec=lock_timeout_sec,
539
- export_summary=export_summary,
540
- raise_on_error=raise_on_error)
541
-
542
- def delete(self):
543
- """
544
- Delete item from platform
545
-
546
- :return: True
547
- :rtype: bool
548
- """
549
- return self.items.delete(item_id=self.id)
550
-
551
- def update(self, system_metadata=False):
552
- """
553
- Update items metadata
554
-
555
- :param bool system_metadata: bool - True, if you want to change metadata system
556
- :return: Item object
557
- :rtype: dtlpy.entities.item.Item
558
- """
559
- return self.items.update(item=self, system_metadata=system_metadata)
560
-
561
- def move(self, new_path):
562
- """
563
- Move item from one folder to another in Platform
564
- If the directory doesn't exist it will be created
565
-
566
- :param str new_path: new full path to move item to.
567
- :return: True if update successfully
568
- :rtype: bool
569
- """
570
- assert isinstance(new_path, str)
571
- if not new_path.startswith('/'):
572
- new_path = '/' + new_path
573
- if new_path.endswith('/'):
574
- self.filename = new_path + self.name
575
- else:
576
- try:
577
- self.items.get(filepath=new_path, is_dir=True)
578
- self.filename = new_path + '/' + self.name
579
- except exceptions.NotFound:
580
- self.filename = new_path
581
-
582
- return self.update(system_metadata=True)
583
-
584
- def clone(self, dst_dataset_id=None, remote_filepath=None, metadata=None, with_annotations=True,
585
- with_metadata=True, with_task_annotations_status=False, allow_many=False, wait=True):
586
- """
587
- Clone item
588
-
589
- :param str dst_dataset_id: destination dataset id
590
- :param str remote_filepath: complete filepath
591
- :param dict metadata: new metadata to add
592
- :param bool with_annotations: clone annotations
593
- :param bool with_metadata: clone metadata
594
- :param bool with_task_annotations_status: clone task annotations status
595
- :param bool allow_many: `bool` if True, using multiple clones in single dataset is allowed, (default=False)
596
- :param bool wait: wait for the command to finish
597
- :return: Item object
598
- :rtype: dtlpy.entities.item.Item
599
-
600
- **Example**:
601
-
602
- .. code-block:: python
603
-
604
- item.clone(item_id='item_id',
605
- dst_dataset_id='dist_dataset_id',
606
- with_metadata=True,
607
- with_task_annotations_status=False,
608
- with_annotations=False)
609
- """
610
- if remote_filepath is None:
611
- remote_filepath = self.filename
612
- if dst_dataset_id is None:
613
- dst_dataset_id = self.dataset_id
614
- return self.items.clone(item_id=self.id,
615
- dst_dataset_id=dst_dataset_id,
616
- remote_filepath=remote_filepath,
617
- metadata=metadata,
618
- with_annotations=with_annotations,
619
- with_metadata=with_metadata,
620
- with_task_annotations_status=with_task_annotations_status,
621
- allow_many=allow_many,
622
- wait=wait)
623
-
624
- def open_in_web(self):
625
- """
626
- Open the items in web platform
627
-
628
- :return:
629
- """
630
- self._client_api._open_in_web(url=self.platform_url)
631
-
632
- def _set_action(self, status: str, operation: str, assignment_id: str = None, task_id: str = None):
633
- """
634
- update item status
635
-
636
- :param status: str - string the describes the status
637
- :param operation: str - 'create' or 'delete'
638
- :param assignment_id: str - assignment id
639
- :param task_id: str - task id
640
-
641
- :return :True/False
642
- """
643
- if assignment_id:
644
- success = self.assignments.set_status(
645
- status=status,
646
- operation=operation,
647
- item_id=self.id,
648
- assignment_id=assignment_id
649
- )
650
- elif task_id:
651
- success = self.tasks.set_status(
652
- status=status,
653
- operation=operation,
654
- item_ids=[self.id],
655
- task_id=task_id
656
- )
657
-
658
- else:
659
- raise exceptions.PlatformException('400', 'Must provide task_id or assignment_id')
660
-
661
- return success
662
-
663
- def update_status(self, status: str, clear: bool = False, assignment_id: str = None, task_id: str = None):
664
- """
665
- update item status
666
-
667
- :param str status: "completed" ,"approved" ,"discard"
668
- :param bool clear: if true delete status
669
- :param str assignment_id: assignment id
670
- :param str task_id: task id
671
-
672
- :return :True/False
673
- :rtype: bool
674
-
675
- **Example**:
676
-
677
- .. code-block:: python
678
-
679
- item.update_status(status='complete',
680
- task_id='task_id')
681
-
682
- """
683
- if not assignment_id and not task_id:
684
- system_metadata = self.metadata.get('system', dict())
685
- if 'refs' in system_metadata:
686
- refs = system_metadata['refs']
687
- if len(refs) <= 2:
688
- for ref in refs:
689
- if ref.get('type', '') == 'assignment':
690
- assignment_id = ref['id']
691
- if ref.get('type', '') == 'task':
692
- task_id = ref['id']
693
-
694
- if assignment_id or task_id:
695
- if clear:
696
- self._set_action(status=status, operation='delete', assignment_id=assignment_id, task_id=task_id)
697
- else:
698
- self._set_action(status=status, operation='create', assignment_id=assignment_id, task_id=task_id)
699
- else:
700
- raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
701
-
702
- def status(self, assignment_id: str = None, task_id: str = None):
703
- """
704
- Get item status
705
-
706
- :param str assignment_id: assignment id
707
- :param str task_id: task id
708
-
709
- :return: status
710
- :rtype: str
711
-
712
- **Example**:
713
-
714
- .. code-block:: python
715
-
716
- status = item.status(task_id='task_id')
717
- """
718
- if not assignment_id and not task_id:
719
- raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
720
- status = None
721
- resource_id = assignment_id if assignment_id else task_id
722
- for ref in self.metadata.get('system', dict()).get('refs', []):
723
- if ref.get('id') == resource_id:
724
- status = ref.get('metadata', {}).get('status', None)
725
- break
726
- return status
727
-
728
- def set_description(self, text: str):
729
- """
730
- Update Item description
731
-
732
- :param str text: if None or "" description will be deleted
733
-
734
- :return
735
- """
736
- if text is None:
737
- text = ""
738
- if not isinstance(text, str):
739
- raise ValueError("Description must get string")
740
- self._description = text
741
- self._platform_dict = self.update()._platform_dict
742
- return self
743
-
744
- def assign_subset(self, subset: str):
745
- """
746
- Assign a single ML subset (train/validation/test) to this item.
747
- Sets the chosen subset to True and the others to None.
748
- Then calls item.update(system_metadata=True).
749
-
750
- :param str subset: 'train', 'validation', or 'test'
751
- """
752
- if subset not in ['train', 'validation', 'test']:
753
- raise ValueError("subset must be one of: 'train', 'validation', 'test'")
754
-
755
- if 'system' not in self.metadata:
756
- self.metadata['system'] = {}
757
- if 'tags' not in self.metadata['system']:
758
- self.metadata['system']['tags'] = {}
759
-
760
- self.metadata['system']['tags']['train'] = True if subset == 'train' else None
761
- self.metadata['system']['tags']['validation'] = True if subset == 'validation' else None
762
- self.metadata['system']['tags']['test'] = True if subset == 'test' else None
763
-
764
- return self.update(system_metadata=True)
765
-
766
-
767
- def remove_subset(self):
768
- """
769
- Remove any ML subset assignment from this item.
770
- Sets train, validation, and test to None.
771
- Then calls item.update(system_metadata=True).
772
- """
773
- if 'system' not in self.metadata:
774
- self.metadata['system'] = {}
775
- if 'tags' not in self.metadata['system']:
776
- self.metadata['system']['tags'] = {}
777
-
778
- self.metadata['system']['tags']['train'] = None
779
- self.metadata['system']['tags']['validation'] = None
780
- self.metadata['system']['tags']['test'] = None
781
-
782
- return self.update(system_metadata=True)
783
-
784
-
785
- def get_current_subset(self) -> str:
786
- """
787
- Get the current ML subset assignment of this item.
788
- Returns 'train', 'validation', 'test', or None if not assigned.
789
-
790
- :return: subset name or None
791
- :rtype: str or None
792
- """
793
- tags = self.metadata.get('system', {}).get('tags', {})
794
- for subset in ['train', 'validation', 'test']:
795
- if tags.get(subset) is True:
796
- return subset
797
- return None
798
-
799
- def assign_collection(self, collections: List[str]) -> bool:
800
- """
801
- Assign this item to one or more collections.
802
-
803
- :param collections: List of collection names to assign the item to.
804
- :return: True if the assignment was successful, otherwise False.
805
- """
806
- return self.collections.assign(dataset_id=self.dataset_id, collections=collections, item_id=self.id,)
807
-
808
- def unassign_collection(self, collections: List[str]) -> bool:
809
- """
810
- Unassign this item from one or more collections.
811
-
812
- :param collections: List of collection names to unassign the item from.
813
- :return: True if the unassignment was successful, otherwise False.
814
- """
815
- return self.collections.unassign(dataset_id=self.dataset_id, item_id=self.id, collections=collections)
816
-
817
- def list_collections(self) -> List[dict]:
818
- """
819
- List all collections associated with this item.
820
-
821
- :return: A list of dictionaries containing collection keys and their respective names.
822
- Each dictionary has the structure: {"key": <collection_key>, "name": <collection_name>}.
823
- """
824
- collections = self.metadata.get("system", {}).get("collections", {})
825
- if not isinstance(collections, dict):
826
- # Ensure collections is a dictionary
827
- return []
828
-
829
- # Retrieve collection names by their keys
830
- return [
831
- {"key": key, "name": self.collections.get_name_by_key(key)}
832
- for key in collections.keys()
833
- ]
834
-
835
- def task_scores(self, task_id: str, page_offset: int = None, page_size: int = None):
836
- """
837
- Get the scores of the item in a specific task.
838
- :param task_id: The ID of the task.
839
- :return: page of scores
840
- """
841
- return self.items.task_scores(item_id=self.id, task_id=task_id, page_offset=page_offset, page_size=page_size)
842
-
843
-
844
- class ModalityTypeEnum(str, Enum):
845
- """
846
- State enum
847
- """
848
- OVERLAY = "overlay"
849
- REPLACE = "replace"
850
- PREVIEW = "preview"
851
-
852
-
853
- class ModalityRefTypeEnum(str, Enum):
854
- """
855
- State enum
856
- """
857
- ID = "id"
858
- URL = "url"
859
-
860
-
861
- class Modality:
862
- def __init__(self, _json=None, modality_type=None, ref=None, ref_type=ModalityRefTypeEnum.ID,
863
- name=None, timestamp=None, mimetype=None):
864
- """
865
- :param _json: json represent of all modality params
866
- :param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
867
- :param ref: id or url of the item reference
868
- :param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
869
- :param name:
870
- :param timestamp: ISOString, epoch of UTC
871
- :param mimetype: str - type of the file
872
- """
873
- if _json is None:
874
- _json = dict()
875
- self.type = _json.get('type', modality_type)
876
- self.ref_type = _json.get('refType', ref_type)
877
- self.ref = _json.get('ref', ref)
878
- self.name = _json.get('name', name)
879
- self.timestamp = _json.get('timestamp', timestamp)
880
- self.mimetype = _json.get('mimetype', mimetype)
881
-
882
- def to_json(self):
883
- _json = {"type": self.type,
884
- "ref": self.ref,
885
- "refType": self.ref_type}
886
- if self.name is not None:
887
- _json['name'] = self.name
888
- if self.timestamp is not None:
889
- _json['timestamp'] = self.timestamp
890
- if self.mimetype is not None:
891
- _json['mimetype'] = self.mimetype
892
- return _json
893
-
894
-
895
- class Modalities:
896
- def __init__(self, item):
897
- assert isinstance(item, Item)
898
- self.item = item
899
- if 'system' not in self.item.metadata:
900
- self.item.metadata['system'] = dict()
901
-
902
- @property
903
- def modalities(self):
904
- mod = None
905
- if 'system' in self.item.metadata:
906
- mod = self.item.metadata['system'].get('modalities', None)
907
- return mod
908
-
909
- def create(self, name, ref,
910
- ref_type: ModalityRefTypeEnum = ModalityRefTypeEnum.ID,
911
- modality_type: ModalityTypeEnum = ModalityTypeEnum.OVERLAY,
912
- timestamp=None,
913
- mimetype=None,
914
- ):
915
- """
916
- create Modalities entity
917
-
918
- :param name: name
919
- :param ref: id or url of the item reference
920
- :param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
921
- :param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
922
- :param timestamp: ISOString, epoch of UTC
923
- :param mimetype: str - type of the file
924
- """
925
- if self.modalities is None:
926
- self.item.metadata['system']['modalities'] = list()
927
-
928
- _json = {"type": modality_type,
929
- "ref": ref,
930
- "refType": ref_type}
931
- if name is not None:
932
- _json['name'] = name
933
- if timestamp is not None:
934
- _json['timestamp'] = timestamp
935
- if mimetype is not None:
936
- _json['mimetype'] = mimetype
937
-
938
- self.item.metadata['system']['modalities'].append(_json)
939
-
940
- return Modality(_json=_json)
941
-
942
- def delete(self, name):
943
- """
944
- :param name:
945
- """
946
- if self.modalities is not None:
947
- for modality in self.item.metadata['system']['modalities']:
948
- if name == modality['name']:
949
- self.item.metadata['system']['modalities'].remove(modality)
950
- return Modality(_json=modality)
951
- return None
952
-
953
- def list(self):
954
- modalities = list()
955
- if self.modalities is not None:
956
- modalities = list()
957
- for modality in self.item.metadata['system']['modalities']:
958
- modalities.append(Modality(_json=modality))
959
- return modalities
1
+ from collections import namedtuple
2
+ from enum import Enum
3
+ import traceback
4
+ import mimetypes
5
+ import logging
6
+ import attr
7
+ import copy
8
+ import os
9
+ import io
10
+ from .. import repositories, entities, exceptions
11
+ from .annotation import ViewAnnotationOptions, ExportVersion
12
+ from ..services.api_client import ApiClient
13
+ from ..services.api_client import client as client_api
14
+ import json
15
+ from typing import List
16
+ import requests
17
+
18
+ logger = logging.getLogger(name='dtlpy')
19
+
20
+
21
+ class ExportMetadata(Enum):
22
+ FROM_JSON = 'from_json'
23
+
24
+
25
+ class ItemStatus(str, Enum):
26
+ COMPLETED = "completed"
27
+ APPROVED = "approved"
28
+ DISCARDED = "discard"
29
+
30
+
31
+ @attr.s
32
+ class Item(entities.BaseEntity):
33
+ """
34
+ Item object
35
+ """
36
+ # item information
37
+ annotations_link = attr.ib(repr=False)
38
+ dataset_url = attr.ib()
39
+ thumbnail = attr.ib(repr=False)
40
+ created_at = attr.ib()
41
+ updated_at = attr.ib()
42
+ updated_by = attr.ib()
43
+ dataset_id = attr.ib()
44
+ annotated = attr.ib(repr=False)
45
+ metadata = attr.ib(repr=False)
46
+ filename = attr.ib()
47
+ stream = attr.ib(repr=False)
48
+ name = attr.ib()
49
+ type = attr.ib()
50
+ url = attr.ib(repr=False)
51
+ id = attr.ib()
52
+ hidden = attr.ib(repr=False)
53
+ dir = attr.ib(repr=False)
54
+ spec = attr.ib()
55
+ creator = attr.ib()
56
+ _description = attr.ib()
57
+ _src_item = attr.ib(repr=False)
58
+
59
+ # name change
60
+ annotations_count = attr.ib()
61
+
62
+ # api
63
+ _client_api = attr.ib(type=ApiClient, repr=False)
64
+ _platform_dict = attr.ib(repr=False)
65
+
66
+ # entities
67
+ _dataset = attr.ib(repr=False)
68
+ _model = attr.ib(repr=False)
69
+ _project = attr.ib(repr=False)
70
+ _project_id = attr.ib(repr=False)
71
+
72
+ # repositories
73
+ _repositories = attr.ib(repr=False)
74
+
75
+ @property
76
+ def createdAt(self):
77
+ return self.created_at
78
+
79
+ @property
80
+ def datasetId(self):
81
+ return self.dataset_id
82
+
83
+ @property
84
+ def resolved_stream(self):
85
+ stream = self.metadata.get('system', dict()).get('shebang', dict()).get('linkInfo', dict()).get('ref', None)
86
+ if stream is None:
87
+ stream = self.stream
88
+ api_url = self._client_api.environment
89
+ if api_url != self._client_api.base_gate_url:
90
+ stream = stream.replace(api_url, self._client_api.base_gate_url)
91
+ else:
92
+ link_item_url_override = os.environ.get('LINK_ITEM_URL_OVERRIDE', None)
93
+ if link_item_url_override is not None:
94
+ src, target = link_item_url_override.split(',')
95
+ stream = stream.replace(src, target)
96
+
97
+ return stream
98
+
99
+ @staticmethod
100
+ def _protected_from_json(_json, client_api, dataset=None):
101
+ """
102
+ Same as from_json but with try-except to catch if error
103
+ :param _json: platform json
104
+ :param client_api: ApiClient entity
105
+ :param dataset: dataset entity
106
+ :return:
107
+ """
108
+ try:
109
+ item = Item.from_json(_json=_json,
110
+ client_api=client_api,
111
+ dataset=dataset)
112
+ status = True
113
+ except Exception:
114
+ item = traceback.format_exc()
115
+ status = False
116
+ return status, item
117
+
118
+ @classmethod
119
+ def from_json(cls, _json, client_api, dataset=None, project=None, model=None, is_fetched=True):
120
+ """
121
+ Build an item entity object from a json
122
+
123
+ :param dtlpy.entities.project.Project project: project entity
124
+ :param dict _json: _json response from host
125
+ :param dtlpy.entities.dataset.Dataset dataset: dataset in which the annotation's item is located
126
+ :param dtlpy.entities.dataset.Model model: the model entity if item is an artifact of a model
127
+ :param dlApiClient client_api: ApiClient entity
128
+ :param bool is_fetched: is Entity fetched from Platform
129
+ :return: Item object
130
+ :rtype: dtlpy.entities.item.Item
131
+ """
132
+ dataset_id = None
133
+ if dataset is not None:
134
+ dataset_id = _json.get('datasetId', None)
135
+ if dataset.id != dataset_id and dataset_id is not None:
136
+ logger.warning('Item has been fetched from a dataset that is not belong to it')
137
+ dataset = None
138
+ else:
139
+ dataset_id = dataset.id
140
+
141
+ metadata = _json.get('metadata', dict())
142
+ project_id = _json.get('projectId', None)
143
+ if project_id is None:
144
+ project_id = project.id if project else None
145
+ inst = cls(
146
+ # sdk
147
+ platform_dict=copy.deepcopy(_json),
148
+ client_api=client_api,
149
+ dataset=dataset,
150
+ project=project,
151
+ model=model,
152
+ # params
153
+ annotations_link=_json.get('annotations', None),
154
+ thumbnail=_json.get('thumbnail', None),
155
+ dataset_id=_json.get('datasetId', dataset_id),
156
+ annotated=_json.get('annotated', None),
157
+ dataset_url=_json.get('dataset', None),
158
+ created_at=_json.get('createdAt', None),
159
+ annotations_count=_json.get('annotationsCount', None),
160
+ hidden=_json.get('hidden', False),
161
+ stream=_json.get('stream', None),
162
+ dir=_json.get('dir', None),
163
+ filename=_json.get('filename', None),
164
+ metadata=metadata,
165
+ name=_json.get('name', None),
166
+ type=_json.get('type', None),
167
+ url=_json.get('url', None),
168
+ id=_json.get('id', None),
169
+ spec=_json.get('spec', None),
170
+ creator=_json.get('creator', None),
171
+ project_id=project_id,
172
+ description=_json.get('description', None),
173
+ src_item=_json.get('srcItem', None),
174
+ updated_at=_json.get('updatedAt', None),
175
+ updated_by=_json.get('updatedBy', None)
176
+ )
177
+ inst.is_fetched = is_fetched
178
+ return inst
179
+
180
+ def __getstate__(self):
181
+ # dump state to json
182
+ return self.to_json()
183
+
184
+ def __setstate__(self, state):
185
+ # create a new item, and update the current one with the same state
186
+ # this way we can have _client_api, and all the repositories and entities which are not picklable
187
+ self.__dict__.update(entities.Item.from_json(_json=state,
188
+ client_api=client_api).__dict__)
189
+
190
+ ############
191
+ # entities #
192
+ ############
193
+ @property
194
+ def dataset(self):
195
+ if self._dataset is None:
196
+ self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
197
+ assert isinstance(self._dataset, entities.Dataset)
198
+ return self._dataset
199
+
200
+ @property
201
+ def model(self):
202
+ return self._model
203
+
204
+ def __update_item_binary(self, _json):
205
+ binary = io.BytesIO()
206
+ binary.write(json.dumps(_json).encode())
207
+ binary.seek(0)
208
+ binary.name = self.name
209
+ success, resp = client_api.gen_request(req_type='post',
210
+ path=f'/items/{self.id}/revisions',
211
+ files={'file': (binary.name, binary)})
212
+ if not success:
213
+ raise exceptions.PlatformException(resp)
214
+
215
+ @property
216
+ def project(self):
217
+ if self._project is None:
218
+ if self._dataset is None:
219
+ self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
220
+ self._project = self._dataset.project
221
+ if self._project is None:
222
+ raise exceptions.PlatformException(error='2001',
223
+ message='Missing entity "project".')
224
+ assert isinstance(self._project, entities.Project)
225
+ return self._project
226
+
227
+ @property
228
+ def project_id(self):
229
+ if self._project_id is None:
230
+ if self._dataset is None:
231
+ self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
232
+ self._project_id = self._dataset.project.id
233
+ return self._project_id
234
+
235
+ ################
236
+ # repositories #
237
+ ################
238
+
239
+ @_repositories.default
240
+ def set_repositories(self):
241
+ reps = namedtuple('repositories',
242
+ field_names=['annotations', 'datasets', 'items', 'codebases', 'artifacts', 'modalities',
243
+ 'features', 'assignments', 'tasks', 'resource_executions', 'collections'])
244
+ reps.__new__.__defaults__ = (None, None, None, None, None, None, None, None, None)
245
+
246
+ if self._dataset is None:
247
+ items = repositories.Items(
248
+ client_api=self._client_api,
249
+ dataset=self._dataset,
250
+ dataset_id=self.dataset_id,
251
+ datasets=repositories.Datasets(client_api=self._client_api, project=None)
252
+ )
253
+ datasets = items.datasets
254
+
255
+ else:
256
+ items = self.dataset.items
257
+ datasets = self.dataset.datasets
258
+
259
+ r = reps(
260
+ annotations=repositories.Annotations(
261
+ client_api=self._client_api,
262
+ dataset_id=self.dataset_id,
263
+ item=self,
264
+ dataset=self._dataset
265
+ ),
266
+ items=items,
267
+ datasets=datasets,
268
+ codebases=None,
269
+ artifacts=None,
270
+ modalities=Modalities(item=self),
271
+ features=repositories.Features(
272
+ client_api=self._client_api,
273
+ project=self._project,
274
+ item=self
275
+ ),
276
+ tasks=repositories.Tasks(
277
+ client_api=self._client_api,
278
+ project=self._project,
279
+ dataset=self._dataset
280
+ ),
281
+ assignments=repositories.Assignments(
282
+ client_api=self._client_api,
283
+ project=self._project,
284
+ dataset=self._dataset
285
+ ),
286
+ resource_executions=repositories.ResourceExecutions(
287
+ client_api=self._client_api,
288
+ project=self._project,
289
+ resource=self
290
+ ),
291
+ collections=repositories.Collections(client_api=self._client_api, item=self, dataset=self._dataset)
292
+ )
293
+ return r
294
+
295
+ @property
296
+ def modalities(self):
297
+ assert isinstance(self._repositories.modalities, Modalities)
298
+ return self._repositories.modalities
299
+
300
+ @property
301
+ def annotations(self):
302
+ assert isinstance(self._repositories.annotations, repositories.Annotations)
303
+ return self._repositories.annotations
304
+
305
+ @property
306
+ def datasets(self):
307
+ assert isinstance(self._repositories.datasets, repositories.Datasets)
308
+ return self._repositories.datasets
309
+
310
+ @property
311
+ def assignments(self):
312
+ assert isinstance(self._repositories.assignments, repositories.Assignments)
313
+ return self._repositories.assignments
314
+
315
+ @property
316
+ def tasks(self):
317
+ assert isinstance(self._repositories.tasks, repositories.Tasks)
318
+ return self._repositories.tasks
319
+
320
+ @property
321
+ def resource_executions(self):
322
+ assert isinstance(self._repositories.resource_executions, repositories.ResourceExecutions)
323
+ return self._repositories.resource_executions
324
+
325
+ @property
326
+ def items(self):
327
+ assert isinstance(self._repositories.items, repositories.Items)
328
+ return self._repositories.items
329
+
330
+ @property
331
+ def features(self):
332
+ assert isinstance(self._repositories.features, repositories.Features)
333
+ return self._repositories.features
334
+
335
+ @property
336
+ def collections(self):
337
+ assert isinstance(self._repositories.collections, repositories.Collections)
338
+ return self._repositories.collections
339
+
340
+ ##############
341
+ # Properties #
342
+ ##############
343
+ @property
344
+ def height(self):
345
+ return self.metadata.get('system', dict()).get('height', None)
346
+
347
+ @height.setter
348
+ def height(self, val):
349
+ if 'system' not in self.metadata:
350
+ self.metadata['system'] = dict()
351
+ self.metadata['system']['height'] = val
352
+
353
+ @property
354
+ def width(self):
355
+ return self.metadata.get('system', dict()).get('width', None)
356
+
357
+ @width.setter
358
+ def width(self, val):
359
+ if 'system' not in self.metadata:
360
+ self.metadata['system'] = dict()
361
+ self.metadata['system']['width'] = val
362
+
363
+ @property
364
+ def fps(self):
365
+ return self.metadata.get('fps', None)
366
+
367
+ @fps.setter
368
+ def fps(self, val):
369
+ self.metadata['fps'] = val
370
+
371
+ @property
372
+ def mimetype(self):
373
+ mimetype = self.metadata.get('system', dict()).get('mimetype', None)
374
+ if mimetype is None:
375
+ mimetype = mimetypes.guess_type(self.filename)[0]
376
+ return mimetype
377
+
378
+ @property
379
+ def size(self):
380
+ return self.metadata.get('system', dict()).get('size', None)
381
+
382
+ @property
383
+ def system(self):
384
+ return self.metadata.get('system', dict())
385
+
386
+ @property
387
+ def description(self):
388
+ description = None
389
+ if self._description is not None:
390
+ description = self._description
391
+ elif 'description' in self.metadata:
392
+ description = self.metadata['description'].get('text', None)
393
+ return description
394
+
395
+ @property
396
+ def platform_url(self):
397
+ return self._client_api._get_resource_url(
398
+ "projects/{}/datasets/{}/items/{}".format(self.dataset.projects[-1], self.dataset.id, self.id))
399
+
400
+ @description.setter
401
+ def description(self, text: str):
402
+ """
403
+ Update Item description
404
+
405
+ :param text: if None or "" description will be deleted
406
+ :return
407
+ """
408
+ self.set_description(text=text)
409
+
410
+ ###########
411
+ # Functions #
412
+ ###########
413
+ def to_json(self):
414
+ """
415
+ Returns platform _json format of object
416
+
417
+ :return: platform json format of object
418
+ :rtype: dict
419
+ """
420
+ _json = attr.asdict(self,
421
+ filter=attr.filters.exclude(attr.fields(Item)._repositories,
422
+ attr.fields(Item)._dataset,
423
+ attr.fields(Item)._model,
424
+ attr.fields(Item)._project,
425
+ attr.fields(Item)._client_api,
426
+ attr.fields(Item)._platform_dict,
427
+ attr.fields(Item).annotations_count,
428
+ attr.fields(Item).dataset_url,
429
+ attr.fields(Item).annotations_link,
430
+ attr.fields(Item).spec,
431
+ attr.fields(Item).creator,
432
+ attr.fields(Item).created_at,
433
+ attr.fields(Item).dataset_id,
434
+ attr.fields(Item)._project_id,
435
+ attr.fields(Item)._description,
436
+ attr.fields(Item)._src_item,
437
+ attr.fields(Item).updated_at,
438
+ attr.fields(Item).updated_by
439
+ ))
440
+
441
+ _json.update({'annotations': self.annotations_link,
442
+ 'annotationsCount': self.annotations_count,
443
+ 'dataset': self.dataset_url,
444
+ 'createdAt': self.created_at,
445
+ 'datasetId': self.dataset_id,
446
+ })
447
+ if self.spec is not None:
448
+ _json['spec'] = self.spec
449
+ if self.creator is not None:
450
+ _json['creator'] = self.creator
451
+ if self._description is not None:
452
+ _json['description'] = self.description
453
+ if self._src_item is not None:
454
+ _json['srcItem'] = self._src_item
455
+
456
+ _json['updatedAt'] = self.updated_at
457
+ _json['updatedBy'] = self.updated_by
458
+
459
+ return _json
460
+
461
+ def download(
462
+ self,
463
+ # download options
464
+ local_path=None,
465
+ file_types=None,
466
+ save_locally=True,
467
+ to_array=False,
468
+ annotation_options: ViewAnnotationOptions = None,
469
+ overwrite=False,
470
+ to_items_folder=True,
471
+ thickness=1,
472
+ with_text=False,
473
+ annotation_filters=None,
474
+ alpha=1,
475
+ export_version=ExportVersion.V1,
476
+ dataset_lock=False,
477
+ lock_timeout_sec=None,
478
+ export_summary=False,
479
+ raise_on_error=False
480
+ ):
481
+ """
482
+ Download dataset by filters.
483
+ Filtering the dataset for items and save them local
484
+ Optional - also download annotation, mask, instance and image mask of the item
485
+
486
+ :param str local_path: local folder or filename to save to.
487
+ :param list file_types: a list of file type to download. e.g ['video/webm', 'video/mp4', 'image/jpeg', 'image/png']
488
+ :param bool save_locally: bool. save to disk or return a buffer
489
+ :param bool to_array: returns Ndarray when True and local_path = False
490
+ :param list annotation_options: download annotations options: list(dl.ViewAnnotationOptions)
491
+ :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
492
+ :param bool overwrite: optional - default = False
493
+ :param bool dataset_lock: optional - default = False
494
+ :param bool export_summary: optional - default = False
495
+ :param int lock_timeout_sec: optional
496
+ :param bool to_items_folder: Create 'items' folder and download items to it
497
+ :param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
498
+ :param bool with_text: optional - add text to annotations, default = False
499
+ :param float alpha: opacity value [0 1], default 1
500
+ :param str export_version: exported items will have original extension in filename, `V1` - no original extension in filenames
501
+ :param bool raise_on_error: raise an exception if an error occurs
502
+ :return: generator of local_path per each downloaded item
503
+ :rtype: generator or single item
504
+
505
+ **Example**:
506
+
507
+ .. code-block:: python
508
+
509
+ item.download(local_path='local_path',
510
+ annotation_options=dl.ViewAnnotationOptions.MASK,
511
+ overwrite=False,
512
+ thickness=1,
513
+ with_text=False,
514
+ alpha=1,
515
+ save_locally=True,
516
+ dataset_lock=False
517
+ lock_timeout_sec=300,
518
+ export_summary=False
519
+ )
520
+ """
521
+ # if dir - concatenate local path and item name
522
+ if local_path is not None:
523
+ if os.path.isdir(local_path):
524
+ local_path = os.path.join(local_path, self.name)
525
+ else:
526
+ _, ext = os.path.splitext(local_path)
527
+ _, item_ext = os.path.splitext(self.name)
528
+ if not ext or ext != item_ext:
529
+ os.makedirs(local_path, exist_ok=True)
530
+ local_path = os.path.join(local_path, self.name)
531
+
532
+ # download
533
+ filters = None
534
+ items = self
535
+ if self.type == 'dir':
536
+ filters = self.datasets._bulid_folder_filter(folder_path=self.filename)
537
+ items = None
538
+
539
+ return self.items.download(items=items,
540
+ local_path=local_path,
541
+ file_types=file_types,
542
+ save_locally=save_locally,
543
+ to_array=to_array,
544
+ annotation_options=annotation_options,
545
+ overwrite=overwrite,
546
+ to_items_folder=to_items_folder,
547
+ annotation_filters=annotation_filters,
548
+ thickness=thickness,
549
+ alpha=alpha,
550
+ with_text=with_text,
551
+ export_version=export_version,
552
+ filters=filters,
553
+ dataset_lock=dataset_lock,
554
+ lock_timeout_sec=lock_timeout_sec,
555
+ export_summary=export_summary,
556
+ raise_on_error=raise_on_error)
557
+
558
+ def delete(self):
559
+ """
560
+ Delete item from platform
561
+
562
+ :return: True
563
+ :rtype: bool
564
+ """
565
+ return self.items.delete(item_id=self.id)
566
+
567
+ def update(self, system_metadata=False):
568
+ """
569
+ Update items metadata
570
+
571
+ :param bool system_metadata: bool - True, if you want to change metadata system
572
+ :return: Item object
573
+ :rtype: dtlpy.entities.item.Item
574
+ """
575
+ return self.items.update(item=self, system_metadata=system_metadata)
576
+
577
+ def move(self, new_path):
578
+ """
579
+ Move item from one folder to another in Platform
580
+ If the directory doesn't exist it will be created
581
+
582
+ :param str new_path: new full path to move item to.
583
+ :return: True if update successfully
584
+ :rtype: bool
585
+ """
586
+ assert isinstance(new_path, str)
587
+ if not new_path.startswith('/'):
588
+ new_path = '/' + new_path
589
+ if new_path.endswith('/'):
590
+ self.filename = new_path + self.name
591
+ else:
592
+ try:
593
+ self.items.get(filepath=new_path, is_dir=True)
594
+ self.filename = new_path + '/' + self.name
595
+ except exceptions.NotFound:
596
+ self.filename = new_path
597
+
598
+ return self.update(system_metadata=True)
599
+
600
+ def clone(self, dst_dataset_id=None, remote_filepath=None, metadata=None, with_annotations=True,
601
+ with_metadata=True, with_task_annotations_status=False, allow_many=False, wait=True):
602
+ """
603
+ Clone item
604
+
605
+ :param str dst_dataset_id: destination dataset id
606
+ :param str remote_filepath: complete filepath
607
+ :param dict metadata: new metadata to add
608
+ :param bool with_annotations: clone annotations
609
+ :param bool with_metadata: clone metadata
610
+ :param bool with_task_annotations_status: clone task annotations status
611
+ :param bool allow_many: `bool` if True, using multiple clones in single dataset is allowed, (default=False)
612
+ :param bool wait: wait for the command to finish
613
+ :return: Item object
614
+ :rtype: dtlpy.entities.item.Item
615
+
616
+ **Example**:
617
+
618
+ .. code-block:: python
619
+
620
+ item.clone(item_id='item_id',
621
+ dst_dataset_id='dist_dataset_id',
622
+ with_metadata=True,
623
+ with_task_annotations_status=False,
624
+ with_annotations=False)
625
+ """
626
+ if remote_filepath is None:
627
+ remote_filepath = self.filename
628
+ if dst_dataset_id is None:
629
+ dst_dataset_id = self.dataset_id
630
+ return self.items.clone(item_id=self.id,
631
+ dst_dataset_id=dst_dataset_id,
632
+ remote_filepath=remote_filepath,
633
+ metadata=metadata,
634
+ with_annotations=with_annotations,
635
+ with_metadata=with_metadata,
636
+ with_task_annotations_status=with_task_annotations_status,
637
+ allow_many=allow_many,
638
+ wait=wait)
639
+
640
+ def open_in_web(self):
641
+ """
642
+ Open the items in web platform
643
+
644
+ :return:
645
+ """
646
+ self._client_api._open_in_web(url=self.platform_url)
647
+
648
+ def _set_action(self, status: str, operation: str, assignment_id: str = None, task_id: str = None):
649
+ """
650
+ update item status
651
+
652
+ :param status: str - string the describes the status
653
+ :param operation: str - 'create' or 'delete'
654
+ :param assignment_id: str - assignment id
655
+ :param task_id: str - task id
656
+
657
+ :return :True/False
658
+ """
659
+ if assignment_id:
660
+ success = self.assignments.set_status(
661
+ status=status,
662
+ operation=operation,
663
+ item_id=self.id,
664
+ assignment_id=assignment_id
665
+ )
666
+ elif task_id:
667
+ success = self.tasks.set_status(
668
+ status=status,
669
+ operation=operation,
670
+ item_ids=[self.id],
671
+ task_id=task_id
672
+ )
673
+
674
+ else:
675
+ raise exceptions.PlatformException('400', 'Must provide task_id or assignment_id')
676
+
677
+ return success
678
+
679
+ def update_status(self, status: str, clear: bool = False, assignment_id: str = None, task_id: str = None):
680
+ """
681
+ update item status
682
+
683
+ :param str status: "completed" ,"approved" ,"discard"
684
+ :param bool clear: if true delete status
685
+ :param str assignment_id: assignment id
686
+ :param str task_id: task id
687
+
688
+ :return :True/False
689
+ :rtype: bool
690
+
691
+ **Example**:
692
+
693
+ .. code-block:: python
694
+
695
+ item.update_status(status='complete',
696
+ task_id='task_id')
697
+
698
+ """
699
+ if not assignment_id and not task_id:
700
+ system_metadata = self.metadata.get('system', dict())
701
+ if 'refs' in system_metadata:
702
+ refs = system_metadata['refs']
703
+ if len(refs) <= 2:
704
+ for ref in refs:
705
+ if ref.get('type', '') == 'assignment':
706
+ assignment_id = ref['id']
707
+ if ref.get('type', '') == 'task':
708
+ task_id = ref['id']
709
+
710
+ if assignment_id or task_id:
711
+ if clear:
712
+ self._set_action(status=status, operation='delete', assignment_id=assignment_id, task_id=task_id)
713
+ else:
714
+ self._set_action(status=status, operation='create', assignment_id=assignment_id, task_id=task_id)
715
+ else:
716
+ raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
717
+
718
+ def status(self, assignment_id: str = None, task_id: str = None):
719
+ """
720
+ Get item status
721
+
722
+ :param str assignment_id: assignment id
723
+ :param str task_id: task id
724
+
725
+ :return: status
726
+ :rtype: str
727
+
728
+ **Example**:
729
+
730
+ .. code-block:: python
731
+
732
+ status = item.status(task_id='task_id')
733
+ """
734
+ if not assignment_id and not task_id:
735
+ raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
736
+ status = None
737
+ resource_id = assignment_id if assignment_id else task_id
738
+ for ref in self.metadata.get('system', dict()).get('refs', []):
739
+ if ref.get('id') == resource_id:
740
+ status = ref.get('metadata', {}).get('status', None)
741
+ break
742
+ return status
743
+
744
+ def set_description(self, text: str):
745
+ """
746
+ Update Item description
747
+
748
+ :param str text: if None or "" description will be deleted
749
+
750
+ :return
751
+ """
752
+ if text is None:
753
+ text = ""
754
+ if not isinstance(text, str):
755
+ raise ValueError("Description must get string")
756
+ self._description = text
757
+ self._platform_dict = self.update()._platform_dict
758
+ return self
759
+
760
+ def assign_subset(self, subset: str):
761
+ """
762
+ Assign a single ML subset (train/validation/test) to this item.
763
+ Sets the chosen subset to True and the others to None.
764
+ Then calls item.update(system_metadata=True).
765
+
766
+ :param str subset: 'train', 'validation', or 'test'
767
+ """
768
+ if subset not in ['train', 'validation', 'test']:
769
+ raise ValueError("subset must be one of: 'train', 'validation', 'test'")
770
+
771
+ if 'system' not in self.metadata:
772
+ self.metadata['system'] = {}
773
+ if 'tags' not in self.metadata['system']:
774
+ self.metadata['system']['tags'] = {}
775
+
776
+ self.metadata['system']['tags']['train'] = True if subset == 'train' else None
777
+ self.metadata['system']['tags']['validation'] = True if subset == 'validation' else None
778
+ self.metadata['system']['tags']['test'] = True if subset == 'test' else None
779
+
780
+ return self.update(system_metadata=True)
781
+
782
+
783
+ def remove_subset(self):
784
+ """
785
+ Remove any ML subset assignment from this item.
786
+ Sets train, validation, and test to None.
787
+ Then calls item.update(system_metadata=True).
788
+ """
789
+ if 'system' not in self.metadata:
790
+ self.metadata['system'] = {}
791
+ if 'tags' not in self.metadata['system']:
792
+ self.metadata['system']['tags'] = {}
793
+
794
+ self.metadata['system']['tags']['train'] = None
795
+ self.metadata['system']['tags']['validation'] = None
796
+ self.metadata['system']['tags']['test'] = None
797
+
798
+ return self.update(system_metadata=True)
799
+
800
+
801
+ def get_current_subset(self) -> str:
802
+ """
803
+ Get the current ML subset assignment of this item.
804
+ Returns 'train', 'validation', 'test', or None if not assigned.
805
+
806
+ :return: subset name or None
807
+ :rtype: str or None
808
+ """
809
+ tags = self.metadata.get('system', {}).get('tags', {})
810
+ for subset in ['train', 'validation', 'test']:
811
+ if tags.get(subset) is True:
812
+ return subset
813
+ return None
814
+
815
+ def assign_collection(self, collections: List[str]) -> bool:
816
+ """
817
+ Assign this item to one or more collections.
818
+
819
+ :param collections: List of collection names to assign the item to.
820
+ :return: True if the assignment was successful, otherwise False.
821
+ """
822
+ return self.collections.assign(dataset_id=self.dataset_id, collections=collections, item_id=self.id,)
823
+
824
+ def unassign_collection(self, collections: List[str]) -> bool:
825
+ """
826
+ Unassign this item from one or more collections.
827
+
828
+ :param collections: List of collection names to unassign the item from.
829
+ :return: True if the unassignment was successful, otherwise False.
830
+ """
831
+ return self.collections.unassign(dataset_id=self.dataset_id, item_id=self.id, collections=collections)
832
+
833
+ def list_collections(self) -> List[dict]:
834
+ """
835
+ List all collections associated with this item.
836
+
837
+ :return: A list of dictionaries containing collection keys and their respective names.
838
+ Each dictionary has the structure: {"key": <collection_key>, "name": <collection_name>}.
839
+ """
840
+ collections = self.metadata.get("system", {}).get("collections", {})
841
+ if not isinstance(collections, dict):
842
+ # Ensure collections is a dictionary
843
+ return []
844
+
845
+ # Retrieve collection names by their keys
846
+ return [
847
+ {"key": key, "name": self.collections.get_name_by_key(key)}
848
+ for key in collections.keys()
849
+ ]
850
+
851
+ def task_scores(self, task_id: str, page_offset: int = None, page_size: int = None):
852
+ """
853
+ Get the scores of the item in a specific task.
854
+ :param task_id: The ID of the task.
855
+ :return: page of scores
856
+ """
857
+ return self.items.task_scores(item_id=self.id, task_id=task_id, page_offset=page_offset, page_size=page_size)
858
+
859
+
860
+ class ModalityTypeEnum(str, Enum):
861
+ """
862
+ State enum
863
+ """
864
+ OVERLAY = "overlay"
865
+ REPLACE = "replace"
866
+ PREVIEW = "preview"
867
+
868
+
869
+ class ModalityRefTypeEnum(str, Enum):
870
+ """
871
+ State enum
872
+ """
873
+ ID = "id"
874
+ URL = "url"
875
+
876
+
877
+ class Modality:
878
+ def __init__(self, _json=None, modality_type=None, ref=None, ref_type=ModalityRefTypeEnum.ID,
879
+ name=None, timestamp=None, mimetype=None):
880
+ """
881
+ :param _json: json represent of all modality params
882
+ :param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
883
+ :param ref: id or url of the item reference
884
+ :param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
885
+ :param name:
886
+ :param timestamp: ISOString, epoch of UTC
887
+ :param mimetype: str - type of the file
888
+ """
889
+ if _json is None:
890
+ _json = dict()
891
+ self.type = _json.get('type', modality_type)
892
+ self.ref_type = _json.get('refType', ref_type)
893
+ self.ref = _json.get('ref', ref)
894
+ self.name = _json.get('name', name)
895
+ self.timestamp = _json.get('timestamp', timestamp)
896
+ self.mimetype = _json.get('mimetype', mimetype)
897
+
898
+ def to_json(self):
899
+ _json = {"type": self.type,
900
+ "ref": self.ref,
901
+ "refType": self.ref_type}
902
+ if self.name is not None:
903
+ _json['name'] = self.name
904
+ if self.timestamp is not None:
905
+ _json['timestamp'] = self.timestamp
906
+ if self.mimetype is not None:
907
+ _json['mimetype'] = self.mimetype
908
+ return _json
909
+
910
+
911
+ class Modalities:
912
+ def __init__(self, item):
913
+ assert isinstance(item, Item)
914
+ self.item = item
915
+ if 'system' not in self.item.metadata:
916
+ self.item.metadata['system'] = dict()
917
+
918
+ @property
919
+ def modalities(self):
920
+ mod = None
921
+ if 'system' in self.item.metadata:
922
+ mod = self.item.metadata['system'].get('modalities', None)
923
+ return mod
924
+
925
+ def create(self, name, ref,
926
+ ref_type: ModalityRefTypeEnum = ModalityRefTypeEnum.ID,
927
+ modality_type: ModalityTypeEnum = ModalityTypeEnum.OVERLAY,
928
+ timestamp=None,
929
+ mimetype=None,
930
+ ):
931
+ """
932
+ create Modalities entity
933
+
934
+ :param name: name
935
+ :param ref: id or url of the item reference
936
+ :param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
937
+ :param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
938
+ :param timestamp: ISOString, epoch of UTC
939
+ :param mimetype: str - type of the file
940
+ """
941
+ if self.modalities is None:
942
+ self.item.metadata['system']['modalities'] = list()
943
+
944
+ _json = {"type": modality_type,
945
+ "ref": ref,
946
+ "refType": ref_type}
947
+ if name is not None:
948
+ _json['name'] = name
949
+ if timestamp is not None:
950
+ _json['timestamp'] = timestamp
951
+ if mimetype is not None:
952
+ _json['mimetype'] = mimetype
953
+
954
+ self.item.metadata['system']['modalities'].append(_json)
955
+
956
+ return Modality(_json=_json)
957
+
958
+ def delete(self, name):
959
+ """
960
+ :param name:
961
+ """
962
+ if self.modalities is not None:
963
+ for modality in self.item.metadata['system']['modalities']:
964
+ if name == modality['name']:
965
+ self.item.metadata['system']['modalities'].remove(modality)
966
+ return Modality(_json=modality)
967
+ return None
968
+
969
+ def list(self):
970
+ modalities = list()
971
+ if self.modalities is not None:
972
+ modalities = list()
973
+ for modality in self.item.metadata['system']['modalities']:
974
+ modalities.append(Modality(_json=modality))
975
+ return modalities