dtlpy 1.115.44__py3-none-any.whl → 1.117.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +491 -491
- dtlpy/__version__.py +1 -1
- dtlpy/assets/__init__.py +26 -26
- dtlpy/assets/code_server/config.yaml +2 -2
- dtlpy/assets/code_server/installation.sh +24 -24
- dtlpy/assets/code_server/launch.json +13 -13
- dtlpy/assets/code_server/settings.json +2 -2
- dtlpy/assets/main.py +53 -53
- dtlpy/assets/main_partial.py +18 -18
- dtlpy/assets/mock.json +11 -11
- dtlpy/assets/model_adapter.py +83 -83
- dtlpy/assets/package.json +61 -61
- dtlpy/assets/package_catalog.json +29 -29
- dtlpy/assets/package_gitignore +307 -307
- dtlpy/assets/service_runners/__init__.py +33 -33
- dtlpy/assets/service_runners/converter.py +96 -96
- dtlpy/assets/service_runners/multi_method.py +49 -49
- dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
- dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
- dtlpy/assets/service_runners/multi_method_item.py +52 -52
- dtlpy/assets/service_runners/multi_method_json.py +52 -52
- dtlpy/assets/service_runners/single_method.py +37 -37
- dtlpy/assets/service_runners/single_method_annotation.py +43 -43
- dtlpy/assets/service_runners/single_method_dataset.py +43 -43
- dtlpy/assets/service_runners/single_method_item.py +41 -41
- dtlpy/assets/service_runners/single_method_json.py +42 -42
- dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
- dtlpy/assets/voc_annotation_template.xml +23 -23
- dtlpy/caches/base_cache.py +32 -32
- dtlpy/caches/cache.py +473 -473
- dtlpy/caches/dl_cache.py +201 -201
- dtlpy/caches/filesystem_cache.py +89 -89
- dtlpy/caches/redis_cache.py +84 -84
- dtlpy/dlp/__init__.py +20 -20
- dtlpy/dlp/cli_utilities.py +367 -367
- dtlpy/dlp/command_executor.py +764 -764
- dtlpy/dlp/dlp +1 -1
- dtlpy/dlp/dlp.bat +1 -1
- dtlpy/dlp/dlp.py +128 -128
- dtlpy/dlp/parser.py +651 -651
- dtlpy/entities/__init__.py +83 -83
- dtlpy/entities/analytic.py +347 -347
- dtlpy/entities/annotation.py +1879 -1879
- dtlpy/entities/annotation_collection.py +699 -699
- dtlpy/entities/annotation_definitions/__init__.py +20 -20
- dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
- dtlpy/entities/annotation_definitions/box.py +195 -195
- dtlpy/entities/annotation_definitions/classification.py +67 -67
- dtlpy/entities/annotation_definitions/comparison.py +72 -72
- dtlpy/entities/annotation_definitions/cube.py +204 -204
- dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
- dtlpy/entities/annotation_definitions/description.py +32 -32
- dtlpy/entities/annotation_definitions/ellipse.py +124 -124
- dtlpy/entities/annotation_definitions/free_text.py +62 -62
- dtlpy/entities/annotation_definitions/gis.py +69 -69
- dtlpy/entities/annotation_definitions/note.py +139 -139
- dtlpy/entities/annotation_definitions/point.py +117 -117
- dtlpy/entities/annotation_definitions/polygon.py +182 -182
- dtlpy/entities/annotation_definitions/polyline.py +111 -111
- dtlpy/entities/annotation_definitions/pose.py +92 -92
- dtlpy/entities/annotation_definitions/ref_image.py +86 -86
- dtlpy/entities/annotation_definitions/segmentation.py +240 -240
- dtlpy/entities/annotation_definitions/subtitle.py +34 -34
- dtlpy/entities/annotation_definitions/text.py +85 -85
- dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
- dtlpy/entities/app.py +220 -220
- dtlpy/entities/app_module.py +107 -107
- dtlpy/entities/artifact.py +174 -174
- dtlpy/entities/assignment.py +399 -399
- dtlpy/entities/base_entity.py +214 -214
- dtlpy/entities/bot.py +113 -113
- dtlpy/entities/codebase.py +292 -292
- dtlpy/entities/collection.py +38 -38
- dtlpy/entities/command.py +169 -169
- dtlpy/entities/compute.py +449 -449
- dtlpy/entities/dataset.py +1299 -1299
- dtlpy/entities/directory_tree.py +44 -44
- dtlpy/entities/dpk.py +470 -470
- dtlpy/entities/driver.py +235 -235
- dtlpy/entities/execution.py +397 -397
- dtlpy/entities/feature.py +124 -124
- dtlpy/entities/feature_set.py +152 -145
- dtlpy/entities/filters.py +798 -798
- dtlpy/entities/gis_item.py +107 -107
- dtlpy/entities/integration.py +184 -184
- dtlpy/entities/item.py +975 -959
- dtlpy/entities/label.py +123 -123
- dtlpy/entities/links.py +85 -85
- dtlpy/entities/message.py +175 -175
- dtlpy/entities/model.py +684 -684
- dtlpy/entities/node.py +1005 -1005
- dtlpy/entities/ontology.py +810 -803
- dtlpy/entities/organization.py +287 -287
- dtlpy/entities/package.py +657 -657
- dtlpy/entities/package_defaults.py +5 -5
- dtlpy/entities/package_function.py +185 -185
- dtlpy/entities/package_module.py +113 -113
- dtlpy/entities/package_slot.py +118 -118
- dtlpy/entities/paged_entities.py +299 -299
- dtlpy/entities/pipeline.py +624 -624
- dtlpy/entities/pipeline_execution.py +279 -279
- dtlpy/entities/project.py +394 -394
- dtlpy/entities/prompt_item.py +505 -505
- dtlpy/entities/recipe.py +301 -301
- dtlpy/entities/reflect_dict.py +102 -102
- dtlpy/entities/resource_execution.py +138 -138
- dtlpy/entities/service.py +974 -963
- dtlpy/entities/service_driver.py +117 -117
- dtlpy/entities/setting.py +294 -294
- dtlpy/entities/task.py +495 -495
- dtlpy/entities/time_series.py +143 -143
- dtlpy/entities/trigger.py +426 -426
- dtlpy/entities/user.py +118 -118
- dtlpy/entities/webhook.py +124 -124
- dtlpy/examples/__init__.py +19 -19
- dtlpy/examples/add_labels.py +135 -135
- dtlpy/examples/add_metadata_to_item.py +21 -21
- dtlpy/examples/annotate_items_using_model.py +65 -65
- dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
- dtlpy/examples/annotations_convert_to_voc.py +9 -9
- dtlpy/examples/annotations_convert_to_yolo.py +9 -9
- dtlpy/examples/convert_annotation_types.py +51 -51
- dtlpy/examples/converter.py +143 -143
- dtlpy/examples/copy_annotations.py +22 -22
- dtlpy/examples/copy_folder.py +31 -31
- dtlpy/examples/create_annotations.py +51 -51
- dtlpy/examples/create_video_annotations.py +83 -83
- dtlpy/examples/delete_annotations.py +26 -26
- dtlpy/examples/filters.py +113 -113
- dtlpy/examples/move_item.py +23 -23
- dtlpy/examples/play_video_annotation.py +13 -13
- dtlpy/examples/show_item_and_mask.py +53 -53
- dtlpy/examples/triggers.py +49 -49
- dtlpy/examples/upload_batch_of_items.py +20 -20
- dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
- dtlpy/examples/upload_items_with_modalities.py +43 -43
- dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
- dtlpy/examples/upload_yolo_format_annotations.py +70 -70
- dtlpy/exceptions.py +125 -125
- dtlpy/miscellaneous/__init__.py +20 -20
- dtlpy/miscellaneous/dict_differ.py +95 -95
- dtlpy/miscellaneous/git_utils.py +217 -217
- dtlpy/miscellaneous/json_utils.py +14 -14
- dtlpy/miscellaneous/list_print.py +105 -105
- dtlpy/miscellaneous/zipping.py +130 -130
- dtlpy/ml/__init__.py +20 -20
- dtlpy/ml/base_feature_extractor_adapter.py +27 -27
- dtlpy/ml/base_model_adapter.py +1287 -1230
- dtlpy/ml/metrics.py +461 -461
- dtlpy/ml/predictions_utils.py +274 -274
- dtlpy/ml/summary_writer.py +57 -57
- dtlpy/ml/train_utils.py +60 -60
- dtlpy/new_instance.py +252 -252
- dtlpy/repositories/__init__.py +56 -56
- dtlpy/repositories/analytics.py +85 -85
- dtlpy/repositories/annotations.py +916 -916
- dtlpy/repositories/apps.py +383 -383
- dtlpy/repositories/artifacts.py +452 -452
- dtlpy/repositories/assignments.py +599 -599
- dtlpy/repositories/bots.py +213 -213
- dtlpy/repositories/codebases.py +559 -559
- dtlpy/repositories/collections.py +332 -332
- dtlpy/repositories/commands.py +152 -152
- dtlpy/repositories/compositions.py +61 -61
- dtlpy/repositories/computes.py +439 -439
- dtlpy/repositories/datasets.py +1585 -1504
- dtlpy/repositories/downloader.py +1157 -923
- dtlpy/repositories/dpks.py +433 -433
- dtlpy/repositories/drivers.py +482 -482
- dtlpy/repositories/executions.py +815 -815
- dtlpy/repositories/feature_sets.py +256 -226
- dtlpy/repositories/features.py +255 -255
- dtlpy/repositories/integrations.py +484 -484
- dtlpy/repositories/items.py +912 -912
- dtlpy/repositories/messages.py +94 -94
- dtlpy/repositories/models.py +1000 -1000
- dtlpy/repositories/nodes.py +80 -80
- dtlpy/repositories/ontologies.py +511 -511
- dtlpy/repositories/organizations.py +525 -525
- dtlpy/repositories/packages.py +1941 -1941
- dtlpy/repositories/pipeline_executions.py +451 -451
- dtlpy/repositories/pipelines.py +640 -640
- dtlpy/repositories/projects.py +539 -539
- dtlpy/repositories/recipes.py +429 -399
- dtlpy/repositories/resource_executions.py +137 -137
- dtlpy/repositories/schema.py +120 -120
- dtlpy/repositories/service_drivers.py +213 -213
- dtlpy/repositories/services.py +1704 -1704
- dtlpy/repositories/settings.py +339 -339
- dtlpy/repositories/tasks.py +1477 -1477
- dtlpy/repositories/times_series.py +278 -278
- dtlpy/repositories/triggers.py +536 -536
- dtlpy/repositories/upload_element.py +257 -257
- dtlpy/repositories/uploader.py +661 -661
- dtlpy/repositories/webhooks.py +249 -249
- dtlpy/services/__init__.py +22 -22
- dtlpy/services/aihttp_retry.py +131 -131
- dtlpy/services/api_client.py +1786 -1785
- dtlpy/services/api_reference.py +40 -40
- dtlpy/services/async_utils.py +133 -133
- dtlpy/services/calls_counter.py +44 -44
- dtlpy/services/check_sdk.py +68 -68
- dtlpy/services/cookie.py +115 -115
- dtlpy/services/create_logger.py +156 -156
- dtlpy/services/events.py +84 -84
- dtlpy/services/logins.py +235 -235
- dtlpy/services/reporter.py +256 -256
- dtlpy/services/service_defaults.py +91 -91
- dtlpy/utilities/__init__.py +20 -20
- dtlpy/utilities/annotations/__init__.py +16 -16
- dtlpy/utilities/annotations/annotation_converters.py +269 -269
- dtlpy/utilities/base_package_runner.py +285 -264
- dtlpy/utilities/converter.py +1650 -1650
- dtlpy/utilities/dataset_generators/__init__.py +1 -1
- dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
- dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
- dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
- dtlpy/utilities/local_development/__init__.py +1 -1
- dtlpy/utilities/local_development/local_session.py +179 -179
- dtlpy/utilities/reports/__init__.py +2 -2
- dtlpy/utilities/reports/figures.py +343 -343
- dtlpy/utilities/reports/report.py +71 -71
- dtlpy/utilities/videos/__init__.py +17 -17
- dtlpy/utilities/videos/video_player.py +598 -598
- dtlpy/utilities/videos/videos.py +470 -470
- {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp +1 -1
- dtlpy-1.117.6.data/scripts/dlp.bat +2 -0
- {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp.py +128 -128
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/METADATA +186 -186
- dtlpy-1.117.6.dist-info/RECORD +239 -0
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/WHEEL +1 -1
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/licenses/LICENSE +200 -200
- tests/features/environment.py +551 -551
- dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
- dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
- dtlpy-1.115.44.dist-info/RECORD +0 -240
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/top_level.txt +0 -0
dtlpy/entities/item.py
CHANGED
|
@@ -1,959 +1,975 @@
|
|
|
1
|
-
from collections import namedtuple
|
|
2
|
-
from enum import Enum
|
|
3
|
-
import traceback
|
|
4
|
-
import mimetypes
|
|
5
|
-
import logging
|
|
6
|
-
import attr
|
|
7
|
-
import copy
|
|
8
|
-
import os
|
|
9
|
-
import io
|
|
10
|
-
from .. import repositories, entities, exceptions
|
|
11
|
-
from .annotation import ViewAnnotationOptions, ExportVersion
|
|
12
|
-
from ..services.api_client import ApiClient
|
|
13
|
-
from ..services.api_client import client as client_api
|
|
14
|
-
import json
|
|
15
|
-
from typing import List
|
|
16
|
-
import requests
|
|
17
|
-
|
|
18
|
-
logger = logging.getLogger(name='dtlpy')
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class ExportMetadata(Enum):
|
|
22
|
-
FROM_JSON = 'from_json'
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class ItemStatus(str, Enum):
|
|
26
|
-
COMPLETED = "completed"
|
|
27
|
-
APPROVED = "approved"
|
|
28
|
-
DISCARDED = "discard"
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@attr.s
|
|
32
|
-
class Item(entities.BaseEntity):
|
|
33
|
-
"""
|
|
34
|
-
Item object
|
|
35
|
-
"""
|
|
36
|
-
# item information
|
|
37
|
-
annotations_link = attr.ib(repr=False)
|
|
38
|
-
dataset_url = attr.ib()
|
|
39
|
-
thumbnail = attr.ib(repr=False)
|
|
40
|
-
created_at = attr.ib()
|
|
41
|
-
updated_at = attr.ib()
|
|
42
|
-
updated_by = attr.ib()
|
|
43
|
-
dataset_id = attr.ib()
|
|
44
|
-
annotated = attr.ib(repr=False)
|
|
45
|
-
metadata = attr.ib(repr=False)
|
|
46
|
-
filename = attr.ib()
|
|
47
|
-
stream = attr.ib(repr=False)
|
|
48
|
-
name = attr.ib()
|
|
49
|
-
type = attr.ib()
|
|
50
|
-
url = attr.ib(repr=False)
|
|
51
|
-
id = attr.ib()
|
|
52
|
-
hidden = attr.ib(repr=False)
|
|
53
|
-
dir = attr.ib(repr=False)
|
|
54
|
-
spec = attr.ib()
|
|
55
|
-
creator = attr.ib()
|
|
56
|
-
_description = attr.ib()
|
|
57
|
-
_src_item = attr.ib(repr=False)
|
|
58
|
-
|
|
59
|
-
# name change
|
|
60
|
-
annotations_count = attr.ib()
|
|
61
|
-
|
|
62
|
-
# api
|
|
63
|
-
_client_api = attr.ib(type=ApiClient, repr=False)
|
|
64
|
-
_platform_dict = attr.ib(repr=False)
|
|
65
|
-
|
|
66
|
-
# entities
|
|
67
|
-
_dataset = attr.ib(repr=False)
|
|
68
|
-
_model = attr.ib(repr=False)
|
|
69
|
-
_project = attr.ib(repr=False)
|
|
70
|
-
_project_id = attr.ib(repr=False)
|
|
71
|
-
|
|
72
|
-
# repositories
|
|
73
|
-
_repositories = attr.ib(repr=False)
|
|
74
|
-
|
|
75
|
-
@property
|
|
76
|
-
def createdAt(self):
|
|
77
|
-
return self.created_at
|
|
78
|
-
|
|
79
|
-
@property
|
|
80
|
-
def datasetId(self):
|
|
81
|
-
return self.dataset_id
|
|
82
|
-
|
|
83
|
-
@
|
|
84
|
-
def
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
return self.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
client_api=self._client_api,
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
self.metadata
|
|
346
|
-
|
|
347
|
-
@
|
|
348
|
-
def
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
attr.fields(Item).
|
|
423
|
-
)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
_json
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
:
|
|
487
|
-
:
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
if
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
:
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
"""
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
:
|
|
629
|
-
|
|
630
|
-
self.
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
:
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
:
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
return
|
|
743
|
-
|
|
744
|
-
def
|
|
745
|
-
"""
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
:
|
|
751
|
-
"""
|
|
752
|
-
if
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
if '
|
|
774
|
-
self.metadata['system'] = {}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
self.metadata['system']['tags']['
|
|
779
|
-
|
|
780
|
-
self.
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
return
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
List
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
"""
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
:
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
modalities
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
1
|
+
from collections import namedtuple
|
|
2
|
+
from enum import Enum
|
|
3
|
+
import traceback
|
|
4
|
+
import mimetypes
|
|
5
|
+
import logging
|
|
6
|
+
import attr
|
|
7
|
+
import copy
|
|
8
|
+
import os
|
|
9
|
+
import io
|
|
10
|
+
from .. import repositories, entities, exceptions
|
|
11
|
+
from .annotation import ViewAnnotationOptions, ExportVersion
|
|
12
|
+
from ..services.api_client import ApiClient
|
|
13
|
+
from ..services.api_client import client as client_api
|
|
14
|
+
import json
|
|
15
|
+
from typing import List
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(name='dtlpy')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ExportMetadata(Enum):
|
|
22
|
+
FROM_JSON = 'from_json'
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ItemStatus(str, Enum):
|
|
26
|
+
COMPLETED = "completed"
|
|
27
|
+
APPROVED = "approved"
|
|
28
|
+
DISCARDED = "discard"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@attr.s
|
|
32
|
+
class Item(entities.BaseEntity):
|
|
33
|
+
"""
|
|
34
|
+
Item object
|
|
35
|
+
"""
|
|
36
|
+
# item information
|
|
37
|
+
annotations_link = attr.ib(repr=False)
|
|
38
|
+
dataset_url = attr.ib()
|
|
39
|
+
thumbnail = attr.ib(repr=False)
|
|
40
|
+
created_at = attr.ib()
|
|
41
|
+
updated_at = attr.ib()
|
|
42
|
+
updated_by = attr.ib()
|
|
43
|
+
dataset_id = attr.ib()
|
|
44
|
+
annotated = attr.ib(repr=False)
|
|
45
|
+
metadata = attr.ib(repr=False)
|
|
46
|
+
filename = attr.ib()
|
|
47
|
+
stream = attr.ib(repr=False)
|
|
48
|
+
name = attr.ib()
|
|
49
|
+
type = attr.ib()
|
|
50
|
+
url = attr.ib(repr=False)
|
|
51
|
+
id = attr.ib()
|
|
52
|
+
hidden = attr.ib(repr=False)
|
|
53
|
+
dir = attr.ib(repr=False)
|
|
54
|
+
spec = attr.ib()
|
|
55
|
+
creator = attr.ib()
|
|
56
|
+
_description = attr.ib()
|
|
57
|
+
_src_item = attr.ib(repr=False)
|
|
58
|
+
|
|
59
|
+
# name change
|
|
60
|
+
annotations_count = attr.ib()
|
|
61
|
+
|
|
62
|
+
# api
|
|
63
|
+
_client_api = attr.ib(type=ApiClient, repr=False)
|
|
64
|
+
_platform_dict = attr.ib(repr=False)
|
|
65
|
+
|
|
66
|
+
# entities
|
|
67
|
+
_dataset = attr.ib(repr=False)
|
|
68
|
+
_model = attr.ib(repr=False)
|
|
69
|
+
_project = attr.ib(repr=False)
|
|
70
|
+
_project_id = attr.ib(repr=False)
|
|
71
|
+
|
|
72
|
+
# repositories
|
|
73
|
+
_repositories = attr.ib(repr=False)
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def createdAt(self):
|
|
77
|
+
return self.created_at
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def datasetId(self):
|
|
81
|
+
return self.dataset_id
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def resolved_stream(self):
|
|
85
|
+
stream = self.metadata.get('system', dict()).get('shebang', dict()).get('linkInfo', dict()).get('ref', None)
|
|
86
|
+
if stream is None:
|
|
87
|
+
stream = self.stream
|
|
88
|
+
api_url = self._client_api.environment
|
|
89
|
+
if api_url != self._client_api.base_gate_url:
|
|
90
|
+
stream = stream.replace(api_url, self._client_api.base_gate_url)
|
|
91
|
+
else:
|
|
92
|
+
link_item_url_override = os.environ.get('LINK_ITEM_URL_OVERRIDE', None)
|
|
93
|
+
if link_item_url_override is not None:
|
|
94
|
+
src, target = link_item_url_override.split(',')
|
|
95
|
+
stream = stream.replace(src, target)
|
|
96
|
+
|
|
97
|
+
return stream
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def _protected_from_json(_json, client_api, dataset=None):
|
|
101
|
+
"""
|
|
102
|
+
Same as from_json but with try-except to catch if error
|
|
103
|
+
:param _json: platform json
|
|
104
|
+
:param client_api: ApiClient entity
|
|
105
|
+
:param dataset: dataset entity
|
|
106
|
+
:return:
|
|
107
|
+
"""
|
|
108
|
+
try:
|
|
109
|
+
item = Item.from_json(_json=_json,
|
|
110
|
+
client_api=client_api,
|
|
111
|
+
dataset=dataset)
|
|
112
|
+
status = True
|
|
113
|
+
except Exception:
|
|
114
|
+
item = traceback.format_exc()
|
|
115
|
+
status = False
|
|
116
|
+
return status, item
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def from_json(cls, _json, client_api, dataset=None, project=None, model=None, is_fetched=True):
|
|
120
|
+
"""
|
|
121
|
+
Build an item entity object from a json
|
|
122
|
+
|
|
123
|
+
:param dtlpy.entities.project.Project project: project entity
|
|
124
|
+
:param dict _json: _json response from host
|
|
125
|
+
:param dtlpy.entities.dataset.Dataset dataset: dataset in which the annotation's item is located
|
|
126
|
+
:param dtlpy.entities.dataset.Model model: the model entity if item is an artifact of a model
|
|
127
|
+
:param dlApiClient client_api: ApiClient entity
|
|
128
|
+
:param bool is_fetched: is Entity fetched from Platform
|
|
129
|
+
:return: Item object
|
|
130
|
+
:rtype: dtlpy.entities.item.Item
|
|
131
|
+
"""
|
|
132
|
+
dataset_id = None
|
|
133
|
+
if dataset is not None:
|
|
134
|
+
dataset_id = _json.get('datasetId', None)
|
|
135
|
+
if dataset.id != dataset_id and dataset_id is not None:
|
|
136
|
+
logger.warning('Item has been fetched from a dataset that is not belong to it')
|
|
137
|
+
dataset = None
|
|
138
|
+
else:
|
|
139
|
+
dataset_id = dataset.id
|
|
140
|
+
|
|
141
|
+
metadata = _json.get('metadata', dict())
|
|
142
|
+
project_id = _json.get('projectId', None)
|
|
143
|
+
if project_id is None:
|
|
144
|
+
project_id = project.id if project else None
|
|
145
|
+
inst = cls(
|
|
146
|
+
# sdk
|
|
147
|
+
platform_dict=copy.deepcopy(_json),
|
|
148
|
+
client_api=client_api,
|
|
149
|
+
dataset=dataset,
|
|
150
|
+
project=project,
|
|
151
|
+
model=model,
|
|
152
|
+
# params
|
|
153
|
+
annotations_link=_json.get('annotations', None),
|
|
154
|
+
thumbnail=_json.get('thumbnail', None),
|
|
155
|
+
dataset_id=_json.get('datasetId', dataset_id),
|
|
156
|
+
annotated=_json.get('annotated', None),
|
|
157
|
+
dataset_url=_json.get('dataset', None),
|
|
158
|
+
created_at=_json.get('createdAt', None),
|
|
159
|
+
annotations_count=_json.get('annotationsCount', None),
|
|
160
|
+
hidden=_json.get('hidden', False),
|
|
161
|
+
stream=_json.get('stream', None),
|
|
162
|
+
dir=_json.get('dir', None),
|
|
163
|
+
filename=_json.get('filename', None),
|
|
164
|
+
metadata=metadata,
|
|
165
|
+
name=_json.get('name', None),
|
|
166
|
+
type=_json.get('type', None),
|
|
167
|
+
url=_json.get('url', None),
|
|
168
|
+
id=_json.get('id', None),
|
|
169
|
+
spec=_json.get('spec', None),
|
|
170
|
+
creator=_json.get('creator', None),
|
|
171
|
+
project_id=project_id,
|
|
172
|
+
description=_json.get('description', None),
|
|
173
|
+
src_item=_json.get('srcItem', None),
|
|
174
|
+
updated_at=_json.get('updatedAt', None),
|
|
175
|
+
updated_by=_json.get('updatedBy', None)
|
|
176
|
+
)
|
|
177
|
+
inst.is_fetched = is_fetched
|
|
178
|
+
return inst
|
|
179
|
+
|
|
180
|
+
def __getstate__(self):
|
|
181
|
+
# dump state to json
|
|
182
|
+
return self.to_json()
|
|
183
|
+
|
|
184
|
+
def __setstate__(self, state):
|
|
185
|
+
# create a new item, and update the current one with the same state
|
|
186
|
+
# this way we can have _client_api, and all the repositories and entities which are not picklable
|
|
187
|
+
self.__dict__.update(entities.Item.from_json(_json=state,
|
|
188
|
+
client_api=client_api).__dict__)
|
|
189
|
+
|
|
190
|
+
############
|
|
191
|
+
# entities #
|
|
192
|
+
############
|
|
193
|
+
@property
|
|
194
|
+
def dataset(self):
|
|
195
|
+
if self._dataset is None:
|
|
196
|
+
self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
|
|
197
|
+
assert isinstance(self._dataset, entities.Dataset)
|
|
198
|
+
return self._dataset
|
|
199
|
+
|
|
200
|
+
@property
|
|
201
|
+
def model(self):
|
|
202
|
+
return self._model
|
|
203
|
+
|
|
204
|
+
def __update_item_binary(self, _json):
|
|
205
|
+
binary = io.BytesIO()
|
|
206
|
+
binary.write(json.dumps(_json).encode())
|
|
207
|
+
binary.seek(0)
|
|
208
|
+
binary.name = self.name
|
|
209
|
+
success, resp = client_api.gen_request(req_type='post',
|
|
210
|
+
path=f'/items/{self.id}/revisions',
|
|
211
|
+
files={'file': (binary.name, binary)})
|
|
212
|
+
if not success:
|
|
213
|
+
raise exceptions.PlatformException(resp)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def project(self):
|
|
217
|
+
if self._project is None:
|
|
218
|
+
if self._dataset is None:
|
|
219
|
+
self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
|
|
220
|
+
self._project = self._dataset.project
|
|
221
|
+
if self._project is None:
|
|
222
|
+
raise exceptions.PlatformException(error='2001',
|
|
223
|
+
message='Missing entity "project".')
|
|
224
|
+
assert isinstance(self._project, entities.Project)
|
|
225
|
+
return self._project
|
|
226
|
+
|
|
227
|
+
@property
|
|
228
|
+
def project_id(self):
|
|
229
|
+
if self._project_id is None:
|
|
230
|
+
if self._dataset is None:
|
|
231
|
+
self._dataset = self.datasets.get(dataset_id=self.dataset_id, fetch=None)
|
|
232
|
+
self._project_id = self._dataset.project.id
|
|
233
|
+
return self._project_id
|
|
234
|
+
|
|
235
|
+
################
|
|
236
|
+
# repositories #
|
|
237
|
+
################
|
|
238
|
+
|
|
239
|
+
@_repositories.default
|
|
240
|
+
def set_repositories(self):
|
|
241
|
+
reps = namedtuple('repositories',
|
|
242
|
+
field_names=['annotations', 'datasets', 'items', 'codebases', 'artifacts', 'modalities',
|
|
243
|
+
'features', 'assignments', 'tasks', 'resource_executions', 'collections'])
|
|
244
|
+
reps.__new__.__defaults__ = (None, None, None, None, None, None, None, None, None)
|
|
245
|
+
|
|
246
|
+
if self._dataset is None:
|
|
247
|
+
items = repositories.Items(
|
|
248
|
+
client_api=self._client_api,
|
|
249
|
+
dataset=self._dataset,
|
|
250
|
+
dataset_id=self.dataset_id,
|
|
251
|
+
datasets=repositories.Datasets(client_api=self._client_api, project=None)
|
|
252
|
+
)
|
|
253
|
+
datasets = items.datasets
|
|
254
|
+
|
|
255
|
+
else:
|
|
256
|
+
items = self.dataset.items
|
|
257
|
+
datasets = self.dataset.datasets
|
|
258
|
+
|
|
259
|
+
r = reps(
|
|
260
|
+
annotations=repositories.Annotations(
|
|
261
|
+
client_api=self._client_api,
|
|
262
|
+
dataset_id=self.dataset_id,
|
|
263
|
+
item=self,
|
|
264
|
+
dataset=self._dataset
|
|
265
|
+
),
|
|
266
|
+
items=items,
|
|
267
|
+
datasets=datasets,
|
|
268
|
+
codebases=None,
|
|
269
|
+
artifacts=None,
|
|
270
|
+
modalities=Modalities(item=self),
|
|
271
|
+
features=repositories.Features(
|
|
272
|
+
client_api=self._client_api,
|
|
273
|
+
project=self._project,
|
|
274
|
+
item=self
|
|
275
|
+
),
|
|
276
|
+
tasks=repositories.Tasks(
|
|
277
|
+
client_api=self._client_api,
|
|
278
|
+
project=self._project,
|
|
279
|
+
dataset=self._dataset
|
|
280
|
+
),
|
|
281
|
+
assignments=repositories.Assignments(
|
|
282
|
+
client_api=self._client_api,
|
|
283
|
+
project=self._project,
|
|
284
|
+
dataset=self._dataset
|
|
285
|
+
),
|
|
286
|
+
resource_executions=repositories.ResourceExecutions(
|
|
287
|
+
client_api=self._client_api,
|
|
288
|
+
project=self._project,
|
|
289
|
+
resource=self
|
|
290
|
+
),
|
|
291
|
+
collections=repositories.Collections(client_api=self._client_api, item=self, dataset=self._dataset)
|
|
292
|
+
)
|
|
293
|
+
return r
|
|
294
|
+
|
|
295
|
+
@property
|
|
296
|
+
def modalities(self):
|
|
297
|
+
assert isinstance(self._repositories.modalities, Modalities)
|
|
298
|
+
return self._repositories.modalities
|
|
299
|
+
|
|
300
|
+
@property
|
|
301
|
+
def annotations(self):
|
|
302
|
+
assert isinstance(self._repositories.annotations, repositories.Annotations)
|
|
303
|
+
return self._repositories.annotations
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def datasets(self):
|
|
307
|
+
assert isinstance(self._repositories.datasets, repositories.Datasets)
|
|
308
|
+
return self._repositories.datasets
|
|
309
|
+
|
|
310
|
+
@property
|
|
311
|
+
def assignments(self):
|
|
312
|
+
assert isinstance(self._repositories.assignments, repositories.Assignments)
|
|
313
|
+
return self._repositories.assignments
|
|
314
|
+
|
|
315
|
+
@property
|
|
316
|
+
def tasks(self):
|
|
317
|
+
assert isinstance(self._repositories.tasks, repositories.Tasks)
|
|
318
|
+
return self._repositories.tasks
|
|
319
|
+
|
|
320
|
+
@property
|
|
321
|
+
def resource_executions(self):
|
|
322
|
+
assert isinstance(self._repositories.resource_executions, repositories.ResourceExecutions)
|
|
323
|
+
return self._repositories.resource_executions
|
|
324
|
+
|
|
325
|
+
@property
|
|
326
|
+
def items(self):
|
|
327
|
+
assert isinstance(self._repositories.items, repositories.Items)
|
|
328
|
+
return self._repositories.items
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def features(self):
|
|
332
|
+
assert isinstance(self._repositories.features, repositories.Features)
|
|
333
|
+
return self._repositories.features
|
|
334
|
+
|
|
335
|
+
@property
|
|
336
|
+
def collections(self):
|
|
337
|
+
assert isinstance(self._repositories.collections, repositories.Collections)
|
|
338
|
+
return self._repositories.collections
|
|
339
|
+
|
|
340
|
+
##############
|
|
341
|
+
# Properties #
|
|
342
|
+
##############
|
|
343
|
+
@property
|
|
344
|
+
def height(self):
|
|
345
|
+
return self.metadata.get('system', dict()).get('height', None)
|
|
346
|
+
|
|
347
|
+
@height.setter
|
|
348
|
+
def height(self, val):
|
|
349
|
+
if 'system' not in self.metadata:
|
|
350
|
+
self.metadata['system'] = dict()
|
|
351
|
+
self.metadata['system']['height'] = val
|
|
352
|
+
|
|
353
|
+
@property
|
|
354
|
+
def width(self):
|
|
355
|
+
return self.metadata.get('system', dict()).get('width', None)
|
|
356
|
+
|
|
357
|
+
@width.setter
|
|
358
|
+
def width(self, val):
|
|
359
|
+
if 'system' not in self.metadata:
|
|
360
|
+
self.metadata['system'] = dict()
|
|
361
|
+
self.metadata['system']['width'] = val
|
|
362
|
+
|
|
363
|
+
@property
|
|
364
|
+
def fps(self):
|
|
365
|
+
return self.metadata.get('fps', None)
|
|
366
|
+
|
|
367
|
+
@fps.setter
|
|
368
|
+
def fps(self, val):
|
|
369
|
+
self.metadata['fps'] = val
|
|
370
|
+
|
|
371
|
+
@property
|
|
372
|
+
def mimetype(self):
|
|
373
|
+
mimetype = self.metadata.get('system', dict()).get('mimetype', None)
|
|
374
|
+
if mimetype is None:
|
|
375
|
+
mimetype = mimetypes.guess_type(self.filename)[0]
|
|
376
|
+
return mimetype
|
|
377
|
+
|
|
378
|
+
@property
|
|
379
|
+
def size(self):
|
|
380
|
+
return self.metadata.get('system', dict()).get('size', None)
|
|
381
|
+
|
|
382
|
+
@property
|
|
383
|
+
def system(self):
|
|
384
|
+
return self.metadata.get('system', dict())
|
|
385
|
+
|
|
386
|
+
@property
|
|
387
|
+
def description(self):
|
|
388
|
+
description = None
|
|
389
|
+
if self._description is not None:
|
|
390
|
+
description = self._description
|
|
391
|
+
elif 'description' in self.metadata:
|
|
392
|
+
description = self.metadata['description'].get('text', None)
|
|
393
|
+
return description
|
|
394
|
+
|
|
395
|
+
@property
|
|
396
|
+
def platform_url(self):
|
|
397
|
+
return self._client_api._get_resource_url(
|
|
398
|
+
"projects/{}/datasets/{}/items/{}".format(self.dataset.projects[-1], self.dataset.id, self.id))
|
|
399
|
+
|
|
400
|
+
@description.setter
|
|
401
|
+
def description(self, text: str):
|
|
402
|
+
"""
|
|
403
|
+
Update Item description
|
|
404
|
+
|
|
405
|
+
:param text: if None or "" description will be deleted
|
|
406
|
+
:return
|
|
407
|
+
"""
|
|
408
|
+
self.set_description(text=text)
|
|
409
|
+
|
|
410
|
+
###########
|
|
411
|
+
# Functions #
|
|
412
|
+
###########
|
|
413
|
+
def to_json(self):
|
|
414
|
+
"""
|
|
415
|
+
Returns platform _json format of object
|
|
416
|
+
|
|
417
|
+
:return: platform json format of object
|
|
418
|
+
:rtype: dict
|
|
419
|
+
"""
|
|
420
|
+
_json = attr.asdict(self,
|
|
421
|
+
filter=attr.filters.exclude(attr.fields(Item)._repositories,
|
|
422
|
+
attr.fields(Item)._dataset,
|
|
423
|
+
attr.fields(Item)._model,
|
|
424
|
+
attr.fields(Item)._project,
|
|
425
|
+
attr.fields(Item)._client_api,
|
|
426
|
+
attr.fields(Item)._platform_dict,
|
|
427
|
+
attr.fields(Item).annotations_count,
|
|
428
|
+
attr.fields(Item).dataset_url,
|
|
429
|
+
attr.fields(Item).annotations_link,
|
|
430
|
+
attr.fields(Item).spec,
|
|
431
|
+
attr.fields(Item).creator,
|
|
432
|
+
attr.fields(Item).created_at,
|
|
433
|
+
attr.fields(Item).dataset_id,
|
|
434
|
+
attr.fields(Item)._project_id,
|
|
435
|
+
attr.fields(Item)._description,
|
|
436
|
+
attr.fields(Item)._src_item,
|
|
437
|
+
attr.fields(Item).updated_at,
|
|
438
|
+
attr.fields(Item).updated_by
|
|
439
|
+
))
|
|
440
|
+
|
|
441
|
+
_json.update({'annotations': self.annotations_link,
|
|
442
|
+
'annotationsCount': self.annotations_count,
|
|
443
|
+
'dataset': self.dataset_url,
|
|
444
|
+
'createdAt': self.created_at,
|
|
445
|
+
'datasetId': self.dataset_id,
|
|
446
|
+
})
|
|
447
|
+
if self.spec is not None:
|
|
448
|
+
_json['spec'] = self.spec
|
|
449
|
+
if self.creator is not None:
|
|
450
|
+
_json['creator'] = self.creator
|
|
451
|
+
if self._description is not None:
|
|
452
|
+
_json['description'] = self.description
|
|
453
|
+
if self._src_item is not None:
|
|
454
|
+
_json['srcItem'] = self._src_item
|
|
455
|
+
|
|
456
|
+
_json['updatedAt'] = self.updated_at
|
|
457
|
+
_json['updatedBy'] = self.updated_by
|
|
458
|
+
|
|
459
|
+
return _json
|
|
460
|
+
|
|
461
|
+
def download(
|
|
462
|
+
self,
|
|
463
|
+
# download options
|
|
464
|
+
local_path=None,
|
|
465
|
+
file_types=None,
|
|
466
|
+
save_locally=True,
|
|
467
|
+
to_array=False,
|
|
468
|
+
annotation_options: ViewAnnotationOptions = None,
|
|
469
|
+
overwrite=False,
|
|
470
|
+
to_items_folder=True,
|
|
471
|
+
thickness=1,
|
|
472
|
+
with_text=False,
|
|
473
|
+
annotation_filters=None,
|
|
474
|
+
alpha=1,
|
|
475
|
+
export_version=ExportVersion.V1,
|
|
476
|
+
dataset_lock=False,
|
|
477
|
+
lock_timeout_sec=None,
|
|
478
|
+
export_summary=False,
|
|
479
|
+
raise_on_error=False
|
|
480
|
+
):
|
|
481
|
+
"""
|
|
482
|
+
Download dataset by filters.
|
|
483
|
+
Filtering the dataset for items and save them local
|
|
484
|
+
Optional - also download annotation, mask, instance and image mask of the item
|
|
485
|
+
|
|
486
|
+
:param str local_path: local folder or filename to save to.
|
|
487
|
+
:param list file_types: a list of file type to download. e.g ['video/webm', 'video/mp4', 'image/jpeg', 'image/png']
|
|
488
|
+
:param bool save_locally: bool. save to disk or return a buffer
|
|
489
|
+
:param bool to_array: returns Ndarray when True and local_path = False
|
|
490
|
+
:param list annotation_options: download annotations options: list(dl.ViewAnnotationOptions)
|
|
491
|
+
:param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
|
|
492
|
+
:param bool overwrite: optional - default = False
|
|
493
|
+
:param bool dataset_lock: optional - default = False
|
|
494
|
+
:param bool export_summary: optional - default = False
|
|
495
|
+
:param int lock_timeout_sec: optional
|
|
496
|
+
:param bool to_items_folder: Create 'items' folder and download items to it
|
|
497
|
+
:param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
|
|
498
|
+
:param bool with_text: optional - add text to annotations, default = False
|
|
499
|
+
:param float alpha: opacity value [0 1], default 1
|
|
500
|
+
:param str export_version: exported items will have original extension in filename, `V1` - no original extension in filenames
|
|
501
|
+
:param bool raise_on_error: raise an exception if an error occurs
|
|
502
|
+
:return: generator of local_path per each downloaded item
|
|
503
|
+
:rtype: generator or single item
|
|
504
|
+
|
|
505
|
+
**Example**:
|
|
506
|
+
|
|
507
|
+
.. code-block:: python
|
|
508
|
+
|
|
509
|
+
item.download(local_path='local_path',
|
|
510
|
+
annotation_options=dl.ViewAnnotationOptions.MASK,
|
|
511
|
+
overwrite=False,
|
|
512
|
+
thickness=1,
|
|
513
|
+
with_text=False,
|
|
514
|
+
alpha=1,
|
|
515
|
+
save_locally=True,
|
|
516
|
+
dataset_lock=False
|
|
517
|
+
lock_timeout_sec=300,
|
|
518
|
+
export_summary=False
|
|
519
|
+
)
|
|
520
|
+
"""
|
|
521
|
+
# if dir - concatenate local path and item name
|
|
522
|
+
if local_path is not None:
|
|
523
|
+
if os.path.isdir(local_path):
|
|
524
|
+
local_path = os.path.join(local_path, self.name)
|
|
525
|
+
else:
|
|
526
|
+
_, ext = os.path.splitext(local_path)
|
|
527
|
+
_, item_ext = os.path.splitext(self.name)
|
|
528
|
+
if not ext or ext != item_ext:
|
|
529
|
+
os.makedirs(local_path, exist_ok=True)
|
|
530
|
+
local_path = os.path.join(local_path, self.name)
|
|
531
|
+
|
|
532
|
+
# download
|
|
533
|
+
filters = None
|
|
534
|
+
items = self
|
|
535
|
+
if self.type == 'dir':
|
|
536
|
+
filters = self.datasets._bulid_folder_filter(folder_path=self.filename)
|
|
537
|
+
items = None
|
|
538
|
+
|
|
539
|
+
return self.items.download(items=items,
|
|
540
|
+
local_path=local_path,
|
|
541
|
+
file_types=file_types,
|
|
542
|
+
save_locally=save_locally,
|
|
543
|
+
to_array=to_array,
|
|
544
|
+
annotation_options=annotation_options,
|
|
545
|
+
overwrite=overwrite,
|
|
546
|
+
to_items_folder=to_items_folder,
|
|
547
|
+
annotation_filters=annotation_filters,
|
|
548
|
+
thickness=thickness,
|
|
549
|
+
alpha=alpha,
|
|
550
|
+
with_text=with_text,
|
|
551
|
+
export_version=export_version,
|
|
552
|
+
filters=filters,
|
|
553
|
+
dataset_lock=dataset_lock,
|
|
554
|
+
lock_timeout_sec=lock_timeout_sec,
|
|
555
|
+
export_summary=export_summary,
|
|
556
|
+
raise_on_error=raise_on_error)
|
|
557
|
+
|
|
558
|
+
def delete(self):
|
|
559
|
+
"""
|
|
560
|
+
Delete item from platform
|
|
561
|
+
|
|
562
|
+
:return: True
|
|
563
|
+
:rtype: bool
|
|
564
|
+
"""
|
|
565
|
+
return self.items.delete(item_id=self.id)
|
|
566
|
+
|
|
567
|
+
def update(self, system_metadata=False):
|
|
568
|
+
"""
|
|
569
|
+
Update items metadata
|
|
570
|
+
|
|
571
|
+
:param bool system_metadata: bool - True, if you want to change metadata system
|
|
572
|
+
:return: Item object
|
|
573
|
+
:rtype: dtlpy.entities.item.Item
|
|
574
|
+
"""
|
|
575
|
+
return self.items.update(item=self, system_metadata=system_metadata)
|
|
576
|
+
|
|
577
|
+
def move(self, new_path):
|
|
578
|
+
"""
|
|
579
|
+
Move item from one folder to another in Platform
|
|
580
|
+
If the directory doesn't exist it will be created
|
|
581
|
+
|
|
582
|
+
:param str new_path: new full path to move item to.
|
|
583
|
+
:return: True if update successfully
|
|
584
|
+
:rtype: bool
|
|
585
|
+
"""
|
|
586
|
+
assert isinstance(new_path, str)
|
|
587
|
+
if not new_path.startswith('/'):
|
|
588
|
+
new_path = '/' + new_path
|
|
589
|
+
if new_path.endswith('/'):
|
|
590
|
+
self.filename = new_path + self.name
|
|
591
|
+
else:
|
|
592
|
+
try:
|
|
593
|
+
self.items.get(filepath=new_path, is_dir=True)
|
|
594
|
+
self.filename = new_path + '/' + self.name
|
|
595
|
+
except exceptions.NotFound:
|
|
596
|
+
self.filename = new_path
|
|
597
|
+
|
|
598
|
+
return self.update(system_metadata=True)
|
|
599
|
+
|
|
600
|
+
def clone(self, dst_dataset_id=None, remote_filepath=None, metadata=None, with_annotations=True,
|
|
601
|
+
with_metadata=True, with_task_annotations_status=False, allow_many=False, wait=True):
|
|
602
|
+
"""
|
|
603
|
+
Clone item
|
|
604
|
+
|
|
605
|
+
:param str dst_dataset_id: destination dataset id
|
|
606
|
+
:param str remote_filepath: complete filepath
|
|
607
|
+
:param dict metadata: new metadata to add
|
|
608
|
+
:param bool with_annotations: clone annotations
|
|
609
|
+
:param bool with_metadata: clone metadata
|
|
610
|
+
:param bool with_task_annotations_status: clone task annotations status
|
|
611
|
+
:param bool allow_many: `bool` if True, using multiple clones in single dataset is allowed, (default=False)
|
|
612
|
+
:param bool wait: wait for the command to finish
|
|
613
|
+
:return: Item object
|
|
614
|
+
:rtype: dtlpy.entities.item.Item
|
|
615
|
+
|
|
616
|
+
**Example**:
|
|
617
|
+
|
|
618
|
+
.. code-block:: python
|
|
619
|
+
|
|
620
|
+
item.clone(item_id='item_id',
|
|
621
|
+
dst_dataset_id='dist_dataset_id',
|
|
622
|
+
with_metadata=True,
|
|
623
|
+
with_task_annotations_status=False,
|
|
624
|
+
with_annotations=False)
|
|
625
|
+
"""
|
|
626
|
+
if remote_filepath is None:
|
|
627
|
+
remote_filepath = self.filename
|
|
628
|
+
if dst_dataset_id is None:
|
|
629
|
+
dst_dataset_id = self.dataset_id
|
|
630
|
+
return self.items.clone(item_id=self.id,
|
|
631
|
+
dst_dataset_id=dst_dataset_id,
|
|
632
|
+
remote_filepath=remote_filepath,
|
|
633
|
+
metadata=metadata,
|
|
634
|
+
with_annotations=with_annotations,
|
|
635
|
+
with_metadata=with_metadata,
|
|
636
|
+
with_task_annotations_status=with_task_annotations_status,
|
|
637
|
+
allow_many=allow_many,
|
|
638
|
+
wait=wait)
|
|
639
|
+
|
|
640
|
+
def open_in_web(self):
|
|
641
|
+
"""
|
|
642
|
+
Open the items in web platform
|
|
643
|
+
|
|
644
|
+
:return:
|
|
645
|
+
"""
|
|
646
|
+
self._client_api._open_in_web(url=self.platform_url)
|
|
647
|
+
|
|
648
|
+
def _set_action(self, status: str, operation: str, assignment_id: str = None, task_id: str = None):
|
|
649
|
+
"""
|
|
650
|
+
update item status
|
|
651
|
+
|
|
652
|
+
:param status: str - string the describes the status
|
|
653
|
+
:param operation: str - 'create' or 'delete'
|
|
654
|
+
:param assignment_id: str - assignment id
|
|
655
|
+
:param task_id: str - task id
|
|
656
|
+
|
|
657
|
+
:return :True/False
|
|
658
|
+
"""
|
|
659
|
+
if assignment_id:
|
|
660
|
+
success = self.assignments.set_status(
|
|
661
|
+
status=status,
|
|
662
|
+
operation=operation,
|
|
663
|
+
item_id=self.id,
|
|
664
|
+
assignment_id=assignment_id
|
|
665
|
+
)
|
|
666
|
+
elif task_id:
|
|
667
|
+
success = self.tasks.set_status(
|
|
668
|
+
status=status,
|
|
669
|
+
operation=operation,
|
|
670
|
+
item_ids=[self.id],
|
|
671
|
+
task_id=task_id
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
else:
|
|
675
|
+
raise exceptions.PlatformException('400', 'Must provide task_id or assignment_id')
|
|
676
|
+
|
|
677
|
+
return success
|
|
678
|
+
|
|
679
|
+
def update_status(self, status: str, clear: bool = False, assignment_id: str = None, task_id: str = None):
|
|
680
|
+
"""
|
|
681
|
+
update item status
|
|
682
|
+
|
|
683
|
+
:param str status: "completed" ,"approved" ,"discard"
|
|
684
|
+
:param bool clear: if true delete status
|
|
685
|
+
:param str assignment_id: assignment id
|
|
686
|
+
:param str task_id: task id
|
|
687
|
+
|
|
688
|
+
:return :True/False
|
|
689
|
+
:rtype: bool
|
|
690
|
+
|
|
691
|
+
**Example**:
|
|
692
|
+
|
|
693
|
+
.. code-block:: python
|
|
694
|
+
|
|
695
|
+
item.update_status(status='complete',
|
|
696
|
+
task_id='task_id')
|
|
697
|
+
|
|
698
|
+
"""
|
|
699
|
+
if not assignment_id and not task_id:
|
|
700
|
+
system_metadata = self.metadata.get('system', dict())
|
|
701
|
+
if 'refs' in system_metadata:
|
|
702
|
+
refs = system_metadata['refs']
|
|
703
|
+
if len(refs) <= 2:
|
|
704
|
+
for ref in refs:
|
|
705
|
+
if ref.get('type', '') == 'assignment':
|
|
706
|
+
assignment_id = ref['id']
|
|
707
|
+
if ref.get('type', '') == 'task':
|
|
708
|
+
task_id = ref['id']
|
|
709
|
+
|
|
710
|
+
if assignment_id or task_id:
|
|
711
|
+
if clear:
|
|
712
|
+
self._set_action(status=status, operation='delete', assignment_id=assignment_id, task_id=task_id)
|
|
713
|
+
else:
|
|
714
|
+
self._set_action(status=status, operation='create', assignment_id=assignment_id, task_id=task_id)
|
|
715
|
+
else:
|
|
716
|
+
raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
|
|
717
|
+
|
|
718
|
+
def status(self, assignment_id: str = None, task_id: str = None):
|
|
719
|
+
"""
|
|
720
|
+
Get item status
|
|
721
|
+
|
|
722
|
+
:param str assignment_id: assignment id
|
|
723
|
+
:param str task_id: task id
|
|
724
|
+
|
|
725
|
+
:return: status
|
|
726
|
+
:rtype: str
|
|
727
|
+
|
|
728
|
+
**Example**:
|
|
729
|
+
|
|
730
|
+
.. code-block:: python
|
|
731
|
+
|
|
732
|
+
status = item.status(task_id='task_id')
|
|
733
|
+
"""
|
|
734
|
+
if not assignment_id and not task_id:
|
|
735
|
+
raise exceptions.PlatformException('400', 'must provide assignment_id or task_id')
|
|
736
|
+
status = None
|
|
737
|
+
resource_id = assignment_id if assignment_id else task_id
|
|
738
|
+
for ref in self.metadata.get('system', dict()).get('refs', []):
|
|
739
|
+
if ref.get('id') == resource_id:
|
|
740
|
+
status = ref.get('metadata', {}).get('status', None)
|
|
741
|
+
break
|
|
742
|
+
return status
|
|
743
|
+
|
|
744
|
+
def set_description(self, text: str):
|
|
745
|
+
"""
|
|
746
|
+
Update Item description
|
|
747
|
+
|
|
748
|
+
:param str text: if None or "" description will be deleted
|
|
749
|
+
|
|
750
|
+
:return
|
|
751
|
+
"""
|
|
752
|
+
if text is None:
|
|
753
|
+
text = ""
|
|
754
|
+
if not isinstance(text, str):
|
|
755
|
+
raise ValueError("Description must get string")
|
|
756
|
+
self._description = text
|
|
757
|
+
self._platform_dict = self.update()._platform_dict
|
|
758
|
+
return self
|
|
759
|
+
|
|
760
|
+
def assign_subset(self, subset: str):
|
|
761
|
+
"""
|
|
762
|
+
Assign a single ML subset (train/validation/test) to this item.
|
|
763
|
+
Sets the chosen subset to True and the others to None.
|
|
764
|
+
Then calls item.update(system_metadata=True).
|
|
765
|
+
|
|
766
|
+
:param str subset: 'train', 'validation', or 'test'
|
|
767
|
+
"""
|
|
768
|
+
if subset not in ['train', 'validation', 'test']:
|
|
769
|
+
raise ValueError("subset must be one of: 'train', 'validation', 'test'")
|
|
770
|
+
|
|
771
|
+
if 'system' not in self.metadata:
|
|
772
|
+
self.metadata['system'] = {}
|
|
773
|
+
if 'tags' not in self.metadata['system']:
|
|
774
|
+
self.metadata['system']['tags'] = {}
|
|
775
|
+
|
|
776
|
+
self.metadata['system']['tags']['train'] = True if subset == 'train' else None
|
|
777
|
+
self.metadata['system']['tags']['validation'] = True if subset == 'validation' else None
|
|
778
|
+
self.metadata['system']['tags']['test'] = True if subset == 'test' else None
|
|
779
|
+
|
|
780
|
+
return self.update(system_metadata=True)
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def remove_subset(self):
|
|
784
|
+
"""
|
|
785
|
+
Remove any ML subset assignment from this item.
|
|
786
|
+
Sets train, validation, and test to None.
|
|
787
|
+
Then calls item.update(system_metadata=True).
|
|
788
|
+
"""
|
|
789
|
+
if 'system' not in self.metadata:
|
|
790
|
+
self.metadata['system'] = {}
|
|
791
|
+
if 'tags' not in self.metadata['system']:
|
|
792
|
+
self.metadata['system']['tags'] = {}
|
|
793
|
+
|
|
794
|
+
self.metadata['system']['tags']['train'] = None
|
|
795
|
+
self.metadata['system']['tags']['validation'] = None
|
|
796
|
+
self.metadata['system']['tags']['test'] = None
|
|
797
|
+
|
|
798
|
+
return self.update(system_metadata=True)
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def get_current_subset(self) -> str:
|
|
802
|
+
"""
|
|
803
|
+
Get the current ML subset assignment of this item.
|
|
804
|
+
Returns 'train', 'validation', 'test', or None if not assigned.
|
|
805
|
+
|
|
806
|
+
:return: subset name or None
|
|
807
|
+
:rtype: str or None
|
|
808
|
+
"""
|
|
809
|
+
tags = self.metadata.get('system', {}).get('tags', {})
|
|
810
|
+
for subset in ['train', 'validation', 'test']:
|
|
811
|
+
if tags.get(subset) is True:
|
|
812
|
+
return subset
|
|
813
|
+
return None
|
|
814
|
+
|
|
815
|
+
def assign_collection(self, collections: List[str]) -> bool:
|
|
816
|
+
"""
|
|
817
|
+
Assign this item to one or more collections.
|
|
818
|
+
|
|
819
|
+
:param collections: List of collection names to assign the item to.
|
|
820
|
+
:return: True if the assignment was successful, otherwise False.
|
|
821
|
+
"""
|
|
822
|
+
return self.collections.assign(dataset_id=self.dataset_id, collections=collections, item_id=self.id,)
|
|
823
|
+
|
|
824
|
+
def unassign_collection(self, collections: List[str]) -> bool:
|
|
825
|
+
"""
|
|
826
|
+
Unassign this item from one or more collections.
|
|
827
|
+
|
|
828
|
+
:param collections: List of collection names to unassign the item from.
|
|
829
|
+
:return: True if the unassignment was successful, otherwise False.
|
|
830
|
+
"""
|
|
831
|
+
return self.collections.unassign(dataset_id=self.dataset_id, item_id=self.id, collections=collections)
|
|
832
|
+
|
|
833
|
+
def list_collections(self) -> List[dict]:
|
|
834
|
+
"""
|
|
835
|
+
List all collections associated with this item.
|
|
836
|
+
|
|
837
|
+
:return: A list of dictionaries containing collection keys and their respective names.
|
|
838
|
+
Each dictionary has the structure: {"key": <collection_key>, "name": <collection_name>}.
|
|
839
|
+
"""
|
|
840
|
+
collections = self.metadata.get("system", {}).get("collections", {})
|
|
841
|
+
if not isinstance(collections, dict):
|
|
842
|
+
# Ensure collections is a dictionary
|
|
843
|
+
return []
|
|
844
|
+
|
|
845
|
+
# Retrieve collection names by their keys
|
|
846
|
+
return [
|
|
847
|
+
{"key": key, "name": self.collections.get_name_by_key(key)}
|
|
848
|
+
for key in collections.keys()
|
|
849
|
+
]
|
|
850
|
+
|
|
851
|
+
def task_scores(self, task_id: str, page_offset: int = None, page_size: int = None):
|
|
852
|
+
"""
|
|
853
|
+
Get the scores of the item in a specific task.
|
|
854
|
+
:param task_id: The ID of the task.
|
|
855
|
+
:return: page of scores
|
|
856
|
+
"""
|
|
857
|
+
return self.items.task_scores(item_id=self.id, task_id=task_id, page_offset=page_offset, page_size=page_size)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
class ModalityTypeEnum(str, Enum):
|
|
861
|
+
"""
|
|
862
|
+
State enum
|
|
863
|
+
"""
|
|
864
|
+
OVERLAY = "overlay"
|
|
865
|
+
REPLACE = "replace"
|
|
866
|
+
PREVIEW = "preview"
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
class ModalityRefTypeEnum(str, Enum):
|
|
870
|
+
"""
|
|
871
|
+
State enum
|
|
872
|
+
"""
|
|
873
|
+
ID = "id"
|
|
874
|
+
URL = "url"
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
class Modality:
|
|
878
|
+
def __init__(self, _json=None, modality_type=None, ref=None, ref_type=ModalityRefTypeEnum.ID,
|
|
879
|
+
name=None, timestamp=None, mimetype=None):
|
|
880
|
+
"""
|
|
881
|
+
:param _json: json represent of all modality params
|
|
882
|
+
:param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
|
|
883
|
+
:param ref: id or url of the item reference
|
|
884
|
+
:param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
|
|
885
|
+
:param name:
|
|
886
|
+
:param timestamp: ISOString, epoch of UTC
|
|
887
|
+
:param mimetype: str - type of the file
|
|
888
|
+
"""
|
|
889
|
+
if _json is None:
|
|
890
|
+
_json = dict()
|
|
891
|
+
self.type = _json.get('type', modality_type)
|
|
892
|
+
self.ref_type = _json.get('refType', ref_type)
|
|
893
|
+
self.ref = _json.get('ref', ref)
|
|
894
|
+
self.name = _json.get('name', name)
|
|
895
|
+
self.timestamp = _json.get('timestamp', timestamp)
|
|
896
|
+
self.mimetype = _json.get('mimetype', mimetype)
|
|
897
|
+
|
|
898
|
+
def to_json(self):
|
|
899
|
+
_json = {"type": self.type,
|
|
900
|
+
"ref": self.ref,
|
|
901
|
+
"refType": self.ref_type}
|
|
902
|
+
if self.name is not None:
|
|
903
|
+
_json['name'] = self.name
|
|
904
|
+
if self.timestamp is not None:
|
|
905
|
+
_json['timestamp'] = self.timestamp
|
|
906
|
+
if self.mimetype is not None:
|
|
907
|
+
_json['mimetype'] = self.mimetype
|
|
908
|
+
return _json
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
class Modalities:
|
|
912
|
+
def __init__(self, item):
|
|
913
|
+
assert isinstance(item, Item)
|
|
914
|
+
self.item = item
|
|
915
|
+
if 'system' not in self.item.metadata:
|
|
916
|
+
self.item.metadata['system'] = dict()
|
|
917
|
+
|
|
918
|
+
@property
|
|
919
|
+
def modalities(self):
|
|
920
|
+
mod = None
|
|
921
|
+
if 'system' in self.item.metadata:
|
|
922
|
+
mod = self.item.metadata['system'].get('modalities', None)
|
|
923
|
+
return mod
|
|
924
|
+
|
|
925
|
+
def create(self, name, ref,
|
|
926
|
+
ref_type: ModalityRefTypeEnum = ModalityRefTypeEnum.ID,
|
|
927
|
+
modality_type: ModalityTypeEnum = ModalityTypeEnum.OVERLAY,
|
|
928
|
+
timestamp=None,
|
|
929
|
+
mimetype=None,
|
|
930
|
+
):
|
|
931
|
+
"""
|
|
932
|
+
create Modalities entity
|
|
933
|
+
|
|
934
|
+
:param name: name
|
|
935
|
+
:param ref: id or url of the item reference
|
|
936
|
+
:param ref_type: ModalityRefTypeEnum.ID, ModalityRefTypeEnum.URL
|
|
937
|
+
:param modality_type: ModalityTypeEnum.OVERLAY,ModalityTypeEnum.REPLACE
|
|
938
|
+
:param timestamp: ISOString, epoch of UTC
|
|
939
|
+
:param mimetype: str - type of the file
|
|
940
|
+
"""
|
|
941
|
+
if self.modalities is None:
|
|
942
|
+
self.item.metadata['system']['modalities'] = list()
|
|
943
|
+
|
|
944
|
+
_json = {"type": modality_type,
|
|
945
|
+
"ref": ref,
|
|
946
|
+
"refType": ref_type}
|
|
947
|
+
if name is not None:
|
|
948
|
+
_json['name'] = name
|
|
949
|
+
if timestamp is not None:
|
|
950
|
+
_json['timestamp'] = timestamp
|
|
951
|
+
if mimetype is not None:
|
|
952
|
+
_json['mimetype'] = mimetype
|
|
953
|
+
|
|
954
|
+
self.item.metadata['system']['modalities'].append(_json)
|
|
955
|
+
|
|
956
|
+
return Modality(_json=_json)
|
|
957
|
+
|
|
958
|
+
def delete(self, name):
|
|
959
|
+
"""
|
|
960
|
+
:param name:
|
|
961
|
+
"""
|
|
962
|
+
if self.modalities is not None:
|
|
963
|
+
for modality in self.item.metadata['system']['modalities']:
|
|
964
|
+
if name == modality['name']:
|
|
965
|
+
self.item.metadata['system']['modalities'].remove(modality)
|
|
966
|
+
return Modality(_json=modality)
|
|
967
|
+
return None
|
|
968
|
+
|
|
969
|
+
def list(self):
|
|
970
|
+
modalities = list()
|
|
971
|
+
if self.modalities is not None:
|
|
972
|
+
modalities = list()
|
|
973
|
+
for modality in self.item.metadata['system']['modalities']:
|
|
974
|
+
modalities.append(Modality(_json=modality))
|
|
975
|
+
return modalities
|