dtlpy 1.115.44__py3-none-any.whl → 1.117.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +152 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +975 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +974 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1287 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1585 -1504
  167. dtlpy/repositories/downloader.py +1157 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +256 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +429 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1786 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.117.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.117.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/top_level.txt +0 -0
@@ -1,803 +1,810 @@
1
- from collections import namedtuple
2
- import traceback
3
- import logging
4
- import random
5
- import uuid
6
- import attr
7
- import os
8
-
9
- from .. import entities, PlatformException, repositories, exceptions
10
- from ..services.api_client import ApiClient
11
- from .label import Label
12
-
13
- logger = logging.getLogger(name='dtlpy')
14
-
15
-
16
- class AttributesTypes:
17
- CHECKBOX = "checkbox"
18
- RADIO_BUTTON = "radio_button"
19
- SLIDER = "range"
20
- YES_NO = "boolean"
21
- FREE_TEXT = "freeText"
22
-
23
-
24
- class AttributesRange:
25
- def __init__(self, min_range, max_range, step):
26
- self.min_range = min_range
27
- self.max_range = max_range
28
- self.step = step
29
-
30
- def to_json(self):
31
- return {'min': self.min_range, 'max': self.max_range, 'step': self.step}
32
-
33
-
34
- class LabelHandlerMode:
35
- ADD = "add"
36
- UPDATE = "update"
37
- UPSERT = "upsert"
38
-
39
-
40
- @attr.s
41
- class Ontology(entities.BaseEntity):
42
- """
43
- Ontology object
44
- """
45
- # api
46
- _client_api = attr.ib(type=ApiClient, repr=False)
47
-
48
- # params
49
- id = attr.ib()
50
- creator = attr.ib()
51
- url = attr.ib(repr=False)
52
- title = attr.ib()
53
- labels = attr.ib(repr=False)
54
- metadata = attr.ib(repr=False)
55
- attributes = attr.ib()
56
-
57
- # entities
58
- _recipe = attr.ib(repr=False, default=None)
59
- _dataset = attr.ib(repr=False, default=None)
60
- _project = attr.ib(repr=False, default=None)
61
-
62
- # repositories
63
- _repositories = attr.ib(repr=False)
64
-
65
- # defaults
66
- _instance_map = attr.ib(default=None, repr=False)
67
- _color_map = attr.ib(default=None, repr=False)
68
-
69
- @_repositories.default
70
- def set_repositories(self):
71
- reps = namedtuple('repositories',
72
- field_names=['ontologies', 'datasets', 'projects'])
73
-
74
- if self._recipe is None:
75
- ontologies = repositories.Ontologies(client_api=self._client_api, recipe=self._recipe)
76
- else:
77
- ontologies = self.recipe.ontologies
78
-
79
- r = reps(ontologies=ontologies, datasets=repositories.Datasets(client_api=self._client_api),
80
- projects=repositories.Projects(client_api=self._client_api))
81
- return r
82
-
83
- @property
84
- def recipe(self):
85
- if self._recipe is not None:
86
- assert isinstance(self._recipe, entities.Recipe)
87
- return self._recipe
88
-
89
- @property
90
- def dataset(self):
91
- if self._dataset is None:
92
- if self.recipe is not None:
93
- self._dataset = self.recipe.dataset
94
- if self._dataset is not None:
95
- assert isinstance(self._dataset, entities.Dataset)
96
- return self._dataset
97
-
98
- @property
99
- def project(self):
100
- if self._project is None:
101
- if 'system' in self.metadata:
102
- project_id = self.metadata['system'].get('projectIds', None)
103
- if project_id is not None:
104
- self._project = self.projects.get(project_id=project_id[0])
105
- elif self.dataset is not None:
106
- self._project = self.dataset.project
107
- if self._project is not None:
108
- assert isinstance(self._project, entities.Project)
109
- return self._project
110
-
111
- @property
112
- def ontologies(self):
113
- if self._repositories.ontologies is not None:
114
- assert isinstance(self._repositories.ontologies, repositories.Ontologies)
115
- return self._repositories.ontologies
116
-
117
- @property
118
- def projects(self):
119
- if self._repositories.projects is not None:
120
- assert isinstance(self._repositories.projects, repositories.Projects)
121
- return self._repositories.projects
122
-
123
- @property
124
- def labels_flat_dict(self):
125
- flatten_dict = dict()
126
-
127
- def add_to_dict(tag, father):
128
- flatten_dict[tag] = father
129
- for child in father.children:
130
- add_to_dict('{}.{}'.format(tag, child.tag), child)
131
-
132
- for label in self.labels:
133
- add_to_dict(label.tag, label)
134
- return flatten_dict
135
-
136
- @property
137
- def instance_map(self):
138
- """
139
- instance mapping for creating instance mask
140
-
141
- :return dictionary {label: map_id}
142
- :rtype: dict
143
- """
144
- if self._instance_map is None:
145
- labels = [label for label in self.labels_flat_dict]
146
- labels.sort()
147
- # each label gets index as instance id
148
- self._instance_map = {label: (i_label + 1) for i_label, label in enumerate(labels)}
149
- return self._instance_map
150
-
151
- @instance_map.setter
152
- def instance_map(self, value: dict):
153
- """
154
- instance mapping for creating instance mask
155
-
156
- :param value: dictionary {label: map_id}
157
- :rtype: dict
158
- """
159
- if not isinstance(value, dict):
160
- raise ValueError('input must be a dictionary of {label_name: instance_id}')
161
- self._instance_map = value
162
-
163
- @property
164
- def color_map(self):
165
- """
166
- Color mapping of labels, {label: rgb}
167
-
168
- :return: dict
169
- :rtype: dict
170
- """
171
- if self._color_map is None:
172
- self._color_map = {k: v.rgb for k, v in self.labels_flat_dict.items()}
173
- return self._color_map
174
-
175
- @color_map.setter
176
- def color_map(self, values):
177
- """
178
- Color mapping of labels, {label: rgb}
179
-
180
- :param values: dict {label: rgb}
181
- :return:
182
- """
183
- if not isinstance(values, dict):
184
- raise ValueError('input must be a dict. got: {}'.format(type(values)))
185
- self._color_map = values
186
-
187
- @staticmethod
188
- def _protected_from_json(_json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
189
- """
190
- Same as from_json but with try-except to catch if error
191
- :param _json: platform json
192
- :param client_api: ApiClient entity
193
- :return:
194
- """
195
- try:
196
- ontology = Ontology.from_json(_json=_json,
197
- client_api=client_api,
198
- project=project,
199
- dataset=dataset,
200
- recipe=recipe,
201
- is_fetched=is_fetched)
202
- status = True
203
- except Exception:
204
- ontology = traceback.format_exc()
205
- status = False
206
- return status, ontology
207
-
208
- @property
209
- def _use_attributes_2(self):
210
- if isinstance(self.metadata, dict):
211
- attributes = self.metadata.get("attributes", None)
212
- if attributes is not None:
213
- return True
214
- else:
215
- if isinstance(self.attributes, list) and len(self.attributes) > 0:
216
- return False
217
- return True
218
-
219
- @classmethod
220
- def from_json(cls, _json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
221
- """
222
- Build an Ontology entity object from a json
223
-
224
- :param bool is_fetched: is Entity fetched from Platform
225
- :param dtlpy.entities.project.Project project: project entity
226
- :param dtlpy.entities.dataset.Dataset dataset: dataset
227
- :param dict _json: _json response from host
228
- :param dtlpy.entities.recipe.Recipe recipe: ontology's recipe
229
- :param dl.ApiClient client_api: ApiClient entity
230
- :return: Ontology object
231
- :rtype: dtlpy.entities.ontology.Ontology
232
- """
233
- attributes_v2 = _json.get('metadata', {}).get("attributes", [])
234
- attributes_v1 = _json.get("attributes", [])
235
- attributes = attributes_v2 if attributes_v2 else attributes_v1
236
-
237
- labels = list()
238
- for root in _json["roots"]:
239
- labels.append(entities.Label.from_root(root=root))
240
-
241
- inst = cls(
242
- metadata=_json.get("metadata", None),
243
- creator=_json.get("creator", None),
244
- url=_json.get("url", None),
245
- id=_json["id"],
246
- title=_json.get("title", None),
247
- attributes=attributes,
248
- client_api=client_api,
249
- project=project,
250
- dataset=dataset,
251
- recipe=recipe,
252
- labels=labels,
253
- )
254
-
255
- inst.is_fetched = is_fetched
256
- return inst
257
-
258
- def to_json(self):
259
- """
260
- Returns platform _json format of object
261
-
262
- :return: platform json format of object
263
- :rtype: dict
264
- """
265
- roots = [label.to_root() for label in self.labels]
266
- _json = attr.asdict(self, filter=attr.filters.exclude(attr.fields(Ontology)._client_api,
267
- attr.fields(Ontology)._recipe,
268
- attr.fields(Ontology)._project,
269
- attr.fields(Ontology)._dataset,
270
- attr.fields(Ontology)._instance_map,
271
- attr.fields(Ontology)._color_map,
272
- attr.fields(Ontology)._repositories))
273
- _json["roots"] = roots
274
- return _json
275
-
276
- def delete(self):
277
- """
278
- Delete recipe from platform
279
-
280
- :return: True
281
- """
282
- return self.ontologies.delete(self.id)
283
-
284
- def update(self, system_metadata=False):
285
- """
286
- Update items metadata
287
-
288
- :param bool system_metadata: bool - True, if you want to change metadata system
289
- :return: Ontology object
290
- """
291
- return self.ontologies.update(self, system_metadata=system_metadata)
292
-
293
- def _add_children(self, label_name, children, labels_node, mode):
294
- for child in children:
295
- if not isinstance(child, entities.Label):
296
- if isinstance(child, dict):
297
- if "label_name" in child:
298
- child = dict(child)
299
- child["label_name"] = "{}.{}".format(label_name, child["label_name"])
300
- labels_node += self._base_labels_handler(labels=[child], update_ontology=False, mode=mode)
301
- else:
302
- raise PlatformException("400",
303
- "Invalid parameters - child list must have label name attribute")
304
- else:
305
- raise PlatformException("400", "Invalid parameters - child must be a dict type")
306
- else:
307
- child.tag = "{}.{}".format(label_name, child.tag)
308
- labels_node += self._base_labels_handler(labels=child, update_ontology=False, mode=mode)
309
-
310
- return labels_node
311
-
312
- def _labels_handler_update_mode(self, json_req, upsert=False, log_error=True):
313
- json_req['upsert'] = upsert
314
- success, response = self._client_api.gen_request(req_type="PATCH",
315
- path="/ontologies/%s/labels" % self.id,
316
- json_req=json_req,
317
- log_error=log_error)
318
- if success:
319
- logger.debug("Labels {} has been added successfully".format(json_req))
320
- else:
321
- raise exceptions.PlatformException(response)
322
- return response
323
-
324
- def _labels_handler_add_mode(self, json_req):
325
- success, response = self._client_api.gen_request(req_type="PATCH",
326
- path="/ontologies/%s/addLabels" % self.id,
327
- json_req=json_req)
328
- if success:
329
- logger.debug("Labels {} has been added successfully".format(json_req))
330
- else:
331
- raise exceptions.PlatformException(response)
332
- return response
333
-
334
- def _base_labels_handler(self, labels, update_ontology=True, mode=LabelHandlerMode.UPSERT):
335
- """
336
- Add a single label to ontology using add label endpoint , nested label is also supported
337
-
338
- :param labels = list of labels
339
- :param update_ontology - return json_req if False
340
- :param mode add, update or upsert, relevant on update_ontology=True only
341
- :return: Ontology updated entire label entity
342
- """
343
- labels_node = list()
344
- if mode not in [LabelHandlerMode.ADD,
345
- LabelHandlerMode.UPDATE,
346
- LabelHandlerMode.UPSERT]:
347
- raise ValueError('mode must be on of: "add", "update", "upsert"')
348
-
349
- if not isinstance(labels, list): # for case that add label get one label
350
- labels = [labels]
351
-
352
- for label in labels:
353
- if isinstance(label, str):
354
- # Generate label from string
355
- label = entities.Label(tag=label)
356
- elif isinstance(label, dict):
357
- # Generate label from dict
358
- label = Label.from_root(label)
359
- elif isinstance(label, entities.Label):
360
- ...
361
- else:
362
- raise ValueError(
363
- 'Unsupported type for `labels`. Expected a list of (str, dict, dl.Label). Got: {}'.format(
364
- type(label)))
365
-
366
- # label entity
367
- label_node = {"tag": label.tag}
368
- if label.color is not None:
369
- label_node["color"] = label.hex
370
- if label.attributes is not None:
371
- label_node["attributes"] = label.attributes
372
- if label.display_label is not None:
373
- label_node["displayLabel"] = label.display_label
374
- if label.display_data is not None:
375
- label_node["displayData"] = label.display_data
376
- labels_node.append(label_node)
377
- children = label.children
378
- self._add_children(label.tag, children, labels_node, mode=mode)
379
-
380
- if not update_ontology or not len(labels_node):
381
- return labels_node
382
-
383
- json_req = {
384
- "labelsNode": labels_node
385
- }
386
-
387
- if mode == LabelHandlerMode.UPDATE:
388
- response = self._labels_handler_update_mode(json_req)
389
- else:
390
- response = self._labels_handler_update_mode(json_req, upsert=True, log_error=False)
391
-
392
- added_label = list()
393
- if "roots" not in response.json():
394
- raise exceptions.PlatformException("error fetching updated labels from server")
395
-
396
- for root in response.json()["roots"]: # to get all labels
397
- added_label.append(entities.Label.from_root(root=root))
398
-
399
- self.labels = added_label
400
- return added_label
401
-
402
- def _add_image_label(self, icon_path):
403
- display_data = dict()
404
- if self.project is not None:
405
- dataset = self.project.datasets._get_binaries_dataset()
406
- elif self.dataset is not None:
407
- dataset = self.dataset.project.datasets._get_binaries_dataset()
408
- else:
409
- raise ValueError('must have project or dataset to create with icon path')
410
- platform_path = "/.dataloop/ontologies/{}/labelDisplayImages/".format(self.id)
411
- basename = os.path.basename(icon_path)
412
- item = dataset.items.upload(local_path=icon_path,
413
- remote_path=platform_path,
414
- remote_name='{}-{}'.format(uuid.uuid4().hex, basename))
415
- display_data['displayImage'] = dict()
416
- display_data['displayImage']['itemId'] = item.id
417
- display_data['displayImage']['datasetId'] = item.dataset_id
418
- return display_data
419
-
420
- def _label_handler(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
421
- add=True, icon_path=None, update_ontology=False, mode=LabelHandlerMode.UPSERT):
422
- """
423
- Add a single label to ontology
424
-
425
- :param label_name: label name
426
- :param color: optional - if not given a random color will be selected
427
- :param children: optional - children
428
- :param attributes: optional - attributes
429
- :param display_label: optional - display_label
430
- :param label: label
431
- :param add:to add or not
432
- :param icon_path: path to image to be display on label
433
- :param update_ontology: update the ontology, default = False for backward compatible
434
- :param mode add, update or upsert, relevant on update_ontology=True only
435
- :return: Label entity
436
- """
437
-
438
- if update_ontology:
439
- if isinstance(label, entities.Label) or isinstance(label, str):
440
- return self._base_labels_handler(labels=label,
441
- update_ontology=update_ontology,
442
- mode=mode)
443
- else:
444
- display_data = dict()
445
- if icon_path is not None:
446
- display_data = self._add_image_label(icon_path=icon_path)
447
- return self._base_labels_handler(labels={"tag": label_name,
448
- "displayLabel": display_label,
449
- "color": color,
450
- "attributes": attributes,
451
- "children": children,
452
- "displayData": display_data
453
- },
454
- update_ontology=update_ontology,
455
- mode=mode)
456
-
457
- if not isinstance(label, entities.Label):
458
- if "." in label_name:
459
- raise PlatformException("400",
460
- "Invalid parameters - nested label can work with update_ontology option only")
461
-
462
- if attributes is None:
463
- attributes = list()
464
- if not isinstance(attributes, list):
465
- attributes = [attributes]
466
-
467
- # get random color if none given
468
- if color is None:
469
- color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
470
-
471
- if children is None:
472
- children = list()
473
- if not isinstance(children, list):
474
- children = [children]
475
-
476
- # add children
477
- added_children = list()
478
- for child in children:
479
- if not isinstance(child, entities.Label):
480
- added_children.append(self._label_handler(**child, add=False))
481
- else:
482
- added_children.append(child)
483
-
484
- if display_label is None:
485
- display_label = ""
486
- if len(label_name.split("_")) == 1:
487
- display_label = label_name[0].upper() + label_name[1:]
488
- else:
489
- for word in label_name.split("_"):
490
- display_label += word[0].upper() + word[1:] + " "
491
- display_label = display_label[0:-1]
492
-
493
- display_data = dict()
494
- if icon_path is not None:
495
- display_data = self._add_image_label(icon_path=icon_path)
496
-
497
- root = {
498
- "value": {
499
- "tag": label_name,
500
- "displayLabel": display_label,
501
- "color": color,
502
- "attributes": attributes,
503
- "displayData": display_data
504
- },
505
- "children": list(),
506
- }
507
- added_label = entities.Label.from_root(root)
508
- added_label.children = added_children
509
- else:
510
- added_label = label
511
- if add and self._validate_label(added_label=added_label, mode=mode, color=color,
512
- children=children, attributes=attributes,
513
- display_label=display_label, display_data=icon_path):
514
- self.labels.append(added_label)
515
- self._base_labels_handler(labels=added_label, update_ontology=True, mode=mode)
516
- return added_label
517
-
518
- def _validate_label(self, added_label, mode=LabelHandlerMode.UPSERT, color=None, children=None, attributes=None,
519
- display_label=None, display_data=None):
520
- """
521
- check if the label is exist
522
- """
523
- for i in range(len(self.labels)):
524
- if self.labels[i].tag == added_label.tag:
525
- if mode == LabelHandlerMode.UPDATE:
526
- if color:
527
- self.labels[i].color = added_label.color
528
- if children:
529
- self.labels[i].children = added_label.children
530
- if attributes:
531
- self.labels[i].attributes = added_label.attributes
532
- if display_label:
533
- self.labels[i].display_label = added_label.display_label
534
- if display_data:
535
- self.labels[i].display_data = added_label.display_data
536
- return False
537
- return True
538
-
539
- def _labels_handler(self, label_list, update_ontology=False, mode=LabelHandlerMode.UPSERT):
540
- """
541
- Adds a list of labels to ontology
542
-
543
- :param list label_list: a list of labels to add to the dataset's ontology. each value should be a dict, dl.Label or a string
544
- if dictionary, should look like this: {"value": {"tag": "name of the label", "displayLabel": "display name on the platform",
545
- "color": "#hex value", "attributes": [attributes]}, "children": [children]}
546
- :param update_ontology: update the ontology, default = False for backward compatible
547
- :param mode add, update or upsert, relevant on update_ontology=True only
548
- :return: List of label entities added
549
- """
550
- if update_ontology:
551
- return self._base_labels_handler(labels=label_list, mode=mode)
552
- labels = list()
553
- for label in label_list:
554
-
555
- if isinstance(label, str):
556
- label = entities.Label(tag=label)
557
-
558
- if isinstance(label, entities.Label):
559
- # label entity
560
- labels.append(label)
561
- else:
562
- # dictionary
563
- labels.append(Label.from_root(label))
564
- added_labels = list()
565
- for label in labels:
566
- added_labels.append(self._label_handler(label.tag, label=label, update_ontology=update_ontology))
567
-
568
- return added_labels
569
-
570
- def delete_labels(self, label_names):
571
- """
572
- Delete labels from ontology
573
-
574
- :param label_names: label object/ label name / list of label objects / list of label names
575
- :return:
576
- """
577
- if not isinstance(label_names, list):
578
- label_names = [label_names]
579
-
580
- if isinstance(label_names[0], entities.Label):
581
- label_names = [label.tag for label in label_names]
582
-
583
- for label in label_names:
584
- self.__delete_label(label)
585
-
586
- self.update()
587
-
588
- def __delete_label(self, label_name):
589
- if label_name in self.instance_map.keys():
590
- labels = self.labels
591
- label_chain = label_name.split('.')
592
- while len(label_chain) > 1:
593
- label_name = label_chain.pop(0)
594
- for i_label, label in enumerate(labels):
595
- if label.tag == label_name:
596
- labels = labels[i_label].children
597
- break
598
- label_name = label_chain[0]
599
- for i_label, label in enumerate(labels):
600
- if label.tag == label_name:
601
- labels.pop(i_label)
602
-
603
- def add_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
604
- add=True, icon_path=None, update_ontology=False):
605
- """
606
- Add a single label to ontology
607
-
608
- :param str label_name: str - label name
609
- :param tuple color: color
610
- :param children: children (sub labels)
611
- :param list attributes: attributes
612
- :param str display_label: display_label
613
- :param dtlpy.entities.label.Label label: label
614
- :param bool add: to add or not
615
- :param str icon_path: path to image to be display on label
616
- :param bool update_ontology: update the ontology, default = False for backward compatible
617
- :return: Label entity
618
- :rtype: dtlpy.entities.label.Label
619
-
620
- **Example**:
621
-
622
- .. code-block:: python
623
-
624
- label = ontology.add_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
625
- """
626
- return self._label_handler(label_name=label_name, color=color, children=children, attributes=attributes,
627
- display_label=display_label, label=label, add=add, icon_path=icon_path,
628
- update_ontology=update_ontology)
629
-
630
- def add_labels(self, label_list, update_ontology=False):
631
- """
632
- Adds a list of labels to ontology
633
-
634
- :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel",
635
- "color": "#color", "attributes": [attributes]}, "children": [children]}]
636
- :param bool update_ontology: update the ontology, default = False for backward compatible
637
- :return: List of label entities added
638
-
639
- **Example**:
640
-
641
- .. code-block:: python
642
-
643
- labels = ontology.add_labels(label_list=label_list)
644
- """
645
- self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=LabelHandlerMode.UPSERT)
646
-
647
- def update_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
648
- add=True, icon_path=None, upsert=False, update_ontology=False):
649
- """
650
- Update a single label to ontology
651
-
652
- :param str label_name: str - label name
653
- :param tuple color: color
654
- :param children: children (sub labels)
655
- :param list attributes: attributes
656
- :param str display_label: display_label
657
- :param dtlpy.entities.label.Label label: label
658
- :param bool add: to add or not
659
- :param str icon_path: path to image to be display on label
660
- :param bool update_ontology: update the ontology, default = False for backward compatible
661
- :param bool upsert: if True will add in case it does not existing
662
- :return: Label entity
663
- :rtype: dtlpy.entities.label.Label
664
-
665
- **Example**:
666
-
667
- .. code-block:: python
668
-
669
- label = ontology.update_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
670
- """
671
- if upsert:
672
- mode = LabelHandlerMode.UPSERT
673
- else:
674
- mode = LabelHandlerMode.UPDATE
675
-
676
- return self._label_handler(label_name=label_name, color=color, children=children,
677
- attributes=attributes, display_label=display_label, label=label,
678
- add=add, icon_path=icon_path, update_ontology=update_ontology, mode=mode)
679
-
680
- def update_labels(self, label_list, upsert=False, update_ontology=False):
681
- """
682
- Update a list of labels to ontology
683
-
684
- :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel", "color": "#color", "attributes": [attributes]}, "children": [children]}]
685
- :param bool upsert: if True will add in case it does not existing
686
- :param bool update_ontology: update the ontology, default = False for backward compatible
687
-
688
- :return: List of label entities added
689
-
690
- **Example**:
691
-
692
- .. code-block:: python
693
-
694
- labels = ontology.update_labels(label_list=label_list)
695
- """
696
-
697
- if upsert:
698
- mode = LabelHandlerMode.UPSERT
699
- else:
700
- mode = LabelHandlerMode.UPDATE
701
- self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=mode)
702
-
703
- def update_attributes(self,
704
- title: str,
705
- key: str,
706
- attribute_type,
707
- scope: list = None,
708
- optional: bool = None,
709
- values: list = None,
710
- attribute_range=None):
711
- """
712
- ADD a new attribute or update if exist
713
-
714
- :param str title: attribute title
715
- :param str key: the key of the attribute must br unique
716
- :param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
717
- :param list scope: list of the labels or * for all labels
718
- :param bool optional: optional attribute
719
- :param list values: list of the attribute values ( for checkbox and radio button)
720
- :param dict or AttributesRange attribute_range: dl.AttributesRange object
721
- :return: true in success
722
- :rtype: bool
723
- """
724
- return self.ontologies.update_attributes(
725
- ontology_id=self.id,
726
- title=title,
727
- key=key,
728
- attribute_type=attribute_type,
729
- scope=scope,
730
- optional=optional,
731
- values=values,
732
- attribute_range=attribute_range)
733
-
734
- def delete_attributes(self, keys: list):
735
- """
736
- Delete a bulk of attributes
737
-
738
- :param list keys: Keys of attributes to delete
739
- :return: True if success
740
- :rtype: bool
741
-
742
- **Example**:
743
-
744
- .. code-block:: python
745
-
746
- success = ontology.delete_attributes(['1'])
747
- """
748
-
749
- return self.ontologies.delete_attributes(ontology_id=self.id, keys=keys)
750
-
751
- def copy_from(self, ontology_json: dict):
752
- """
753
- Import ontology to the platform.\n
754
- Notice: only the following fields will be updated: `labels`, `attributes`, `instance_map` and `color_map`.
755
-
756
- :param dict ontology_json: The source ontology json to copy from
757
- :return: Ontology object: The updated ontology entity
758
- :rtype: dtlpy.entities.ontology.Ontology
759
-
760
- **Example**:
761
-
762
- .. code-block:: python
763
-
764
- ontology = ontology.import_ontology(ontology_json=ontology_json)
765
- """
766
- # TODO: Add support for import from ontology entity in the Future
767
- if not self._use_attributes_2:
768
- raise ValueError("This method is only supported for attributes 2 mode!")
769
- new_ontology = self.from_json(_json=ontology_json, client_api=self._client_api)
770
-
771
- # Update 'labels' and 'attributes'
772
- self.labels = new_ontology.labels
773
- new_attributes = new_ontology.attributes
774
- if isinstance(new_attributes, list):
775
- for new_attribute in new_attributes:
776
- attribute_range = new_attribute.get("range", None)
777
- if attribute_range is not None:
778
- attribute_range = entities.AttributesRange(
779
- min_range=attribute_range.get("min", None),
780
- max_range=attribute_range.get("max", None),
781
- step=attribute_range.get("step", None)
782
- )
783
- script_data = new_attribute.get("scriptData", None)
784
- if script_data is None:
785
- new_attribute_key = new_attribute.get("key", None)
786
- raise Exception(f"Attribute '{new_attribute_key}' scriptData is missing in the ontology json!")
787
- self.update_attributes(
788
- title=script_data.get("title", None),
789
- key=new_attribute.get("key", None),
790
- attribute_type=new_attribute.get("type", None),
791
- scope=new_attribute.get("scope", None),
792
- optional=script_data.get("optional", None),
793
- values=new_attribute.get("values", None),
794
- attribute_range=attribute_range
795
- )
796
-
797
- # Get remote updated 'attributes'
798
- self.metadata["attributes"] = self.ontologies.get(ontology_id=self.id).attributes
799
-
800
- # Update 'instance map' and 'color map'
801
- self._instance_map = new_ontology.instance_map
802
- self._color_map = new_ontology.color_map
803
- return self.update(system_metadata=True)
1
+ from collections import namedtuple
2
+ import traceback
3
+ import logging
4
+ import random
5
+ import uuid
6
+ import attr
7
+ import os
8
+
9
+ from .. import entities, PlatformException, repositories, exceptions
10
+ from ..services.api_client import ApiClient
11
+ from .label import Label
12
+
13
+ logger = logging.getLogger(name='dtlpy')
14
+
15
+
16
+ class AttributesTypes:
17
+ CHECKBOX = "checkbox"
18
+ RADIO_BUTTON = "radio_button"
19
+ SLIDER = "range"
20
+ YES_NO = "boolean"
21
+ FREE_TEXT = "freeText"
22
+
23
+ class AttributesRange:
24
+ def __init__(self, min_range, max_range, step):
25
+ self.min_range = min_range
26
+ self.max_range = max_range
27
+ self.step = step
28
+
29
+ def to_json(self):
30
+ return {'min': self.min_range, 'max': self.max_range, 'step': self.step}
31
+
32
+
33
+ class LabelHandlerMode:
34
+ ADD = "add"
35
+ UPDATE = "update"
36
+ UPSERT = "upsert"
37
+
38
+
39
+ @attr.s
40
+ class Ontology(entities.BaseEntity):
41
+ """
42
+ Ontology object
43
+ """
44
+ # api
45
+ _client_api = attr.ib(type=ApiClient, repr=False)
46
+
47
+ # params
48
+ id = attr.ib()
49
+ creator = attr.ib()
50
+ url = attr.ib(repr=False)
51
+ title = attr.ib()
52
+ labels = attr.ib(repr=False)
53
+ metadata = attr.ib(repr=False)
54
+ attributes = attr.ib()
55
+
56
+ # entities
57
+ _recipe = attr.ib(repr=False, default=None)
58
+ _dataset = attr.ib(repr=False, default=None)
59
+ _project = attr.ib(repr=False, default=None)
60
+
61
+ # repositories
62
+ _repositories = attr.ib(repr=False)
63
+
64
+ # defaults
65
+ _instance_map = attr.ib(default=None, repr=False)
66
+ _color_map = attr.ib(default=None, repr=False)
67
+
68
+ @_repositories.default
69
+ def set_repositories(self):
70
+ reps = namedtuple('repositories',
71
+ field_names=['ontologies', 'datasets', 'projects'])
72
+
73
+ if self._recipe is None:
74
+ ontologies = repositories.Ontologies(client_api=self._client_api, recipe=self._recipe)
75
+ else:
76
+ ontologies = self.recipe.ontologies
77
+
78
+ r = reps(ontologies=ontologies, datasets=repositories.Datasets(client_api=self._client_api),
79
+ projects=repositories.Projects(client_api=self._client_api))
80
+ return r
81
+
82
+ @property
83
+ def recipe(self):
84
+ if self._recipe is None:
85
+ filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
86
+ filters.add(field="ontologies", values=self.id)
87
+ recipes = self.project.recipes.list(filters=filters)
88
+ if recipes.items_count > 0:
89
+ self._recipe = recipes.items[0]
90
+ else:
91
+ logger.warning(f"Ontology ID: {self.id} Does not belong to a recipe")
92
+ if self._recipe is not None:
93
+ assert isinstance(self._recipe, entities.Recipe)
94
+ return self._recipe
95
+
96
+ @property
97
+ def dataset(self):
98
+ if self._dataset is None:
99
+ if self.recipe is not None:
100
+ self._dataset = self.recipe.dataset
101
+ if self._dataset is not None:
102
+ assert isinstance(self._dataset, entities.Dataset)
103
+ return self._dataset
104
+
105
+ @property
106
+ def project(self):
107
+ if self._project is None:
108
+ if 'system' in self.metadata:
109
+ project_id = self.metadata['system'].get('projectIds', None)
110
+ if project_id is not None:
111
+ self._project = self.projects.get(project_id=project_id[0])
112
+ elif self.dataset is not None:
113
+ self._project = self.dataset.project
114
+ if self._project is not None:
115
+ assert isinstance(self._project, entities.Project)
116
+ return self._project
117
+
118
+ @property
119
+ def ontologies(self):
120
+ if self._repositories.ontologies is not None:
121
+ assert isinstance(self._repositories.ontologies, repositories.Ontologies)
122
+ return self._repositories.ontologies
123
+
124
+ @property
125
+ def projects(self):
126
+ if self._repositories.projects is not None:
127
+ assert isinstance(self._repositories.projects, repositories.Projects)
128
+ return self._repositories.projects
129
+
130
+ @property
131
+ def labels_flat_dict(self):
132
+ flatten_dict = dict()
133
+
134
+ def add_to_dict(tag, father):
135
+ flatten_dict[tag] = father
136
+ for child in father.children:
137
+ add_to_dict('{}.{}'.format(tag, child.tag), child)
138
+
139
+ for label in self.labels:
140
+ add_to_dict(label.tag, label)
141
+ return flatten_dict
142
+
143
+ @property
144
+ def instance_map(self):
145
+ """
146
+ instance mapping for creating instance mask
147
+
148
+ :return dictionary {label: map_id}
149
+ :rtype: dict
150
+ """
151
+ if self._instance_map is None:
152
+ labels = [label for label in self.labels_flat_dict]
153
+ labels.sort()
154
+ # each label gets index as instance id
155
+ self._instance_map = {label: (i_label + 1) for i_label, label in enumerate(labels)}
156
+ return self._instance_map
157
+
158
+ @instance_map.setter
159
+ def instance_map(self, value: dict):
160
+ """
161
+ instance mapping for creating instance mask
162
+
163
+ :param value: dictionary {label: map_id}
164
+ :rtype: dict
165
+ """
166
+ if not isinstance(value, dict):
167
+ raise ValueError('input must be a dictionary of {label_name: instance_id}')
168
+ self._instance_map = value
169
+
170
+ @property
171
+ def color_map(self):
172
+ """
173
+ Color mapping of labels, {label: rgb}
174
+
175
+ :return: dict
176
+ :rtype: dict
177
+ """
178
+ if self._color_map is None:
179
+ self._color_map = {k: v.rgb for k, v in self.labels_flat_dict.items()}
180
+ return self._color_map
181
+
182
+ @color_map.setter
183
+ def color_map(self, values):
184
+ """
185
+ Color mapping of labels, {label: rgb}
186
+
187
+ :param values: dict {label: rgb}
188
+ :return:
189
+ """
190
+ if not isinstance(values, dict):
191
+ raise ValueError('input must be a dict. got: {}'.format(type(values)))
192
+ self._color_map = values
193
+
194
+ @staticmethod
195
+ def _protected_from_json(_json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
196
+ """
197
+ Same as from_json but with try-except to catch if error
198
+ :param _json: platform json
199
+ :param client_api: ApiClient entity
200
+ :return:
201
+ """
202
+ try:
203
+ ontology = Ontology.from_json(_json=_json,
204
+ client_api=client_api,
205
+ project=project,
206
+ dataset=dataset,
207
+ recipe=recipe,
208
+ is_fetched=is_fetched)
209
+ status = True
210
+ except Exception:
211
+ ontology = traceback.format_exc()
212
+ status = False
213
+ return status, ontology
214
+
215
+ @property
216
+ def _use_attributes_2(self):
217
+ if isinstance(self.metadata, dict):
218
+ attributes = self.metadata.get("attributes", None)
219
+ if attributes is not None:
220
+ return True
221
+ else:
222
+ if isinstance(self.attributes, list) and len(self.attributes) > 0:
223
+ return False
224
+ return True
225
+
226
+ @classmethod
227
+ def from_json(cls, _json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
228
+ """
229
+ Build an Ontology entity object from a json
230
+
231
+ :param bool is_fetched: is Entity fetched from Platform
232
+ :param dtlpy.entities.project.Project project: project entity
233
+ :param dtlpy.entities.dataset.Dataset dataset: dataset
234
+ :param dict _json: _json response from host
235
+ :param dtlpy.entities.recipe.Recipe recipe: ontology's recipe
236
+ :param dl.ApiClient client_api: ApiClient entity
237
+ :return: Ontology object
238
+ :rtype: dtlpy.entities.ontology.Ontology
239
+ """
240
+ attributes_v2 = _json.get('metadata', {}).get("attributes", [])
241
+ attributes_v1 = _json.get("attributes", [])
242
+ attributes = attributes_v2 if attributes_v2 else attributes_v1
243
+
244
+ labels = list()
245
+ for root in _json["roots"]:
246
+ labels.append(entities.Label.from_root(root=root))
247
+
248
+ inst = cls(
249
+ metadata=_json.get("metadata", None),
250
+ creator=_json.get("creator", None),
251
+ url=_json.get("url", None),
252
+ id=_json["id"],
253
+ title=_json.get("title", None),
254
+ attributes=attributes,
255
+ client_api=client_api,
256
+ project=project,
257
+ dataset=dataset,
258
+ recipe=recipe,
259
+ labels=labels,
260
+ )
261
+
262
+ inst.is_fetched = is_fetched
263
+ return inst
264
+
265
+ def to_json(self):
266
+ """
267
+ Returns platform _json format of object
268
+
269
+ :return: platform json format of object
270
+ :rtype: dict
271
+ """
272
+ roots = [label.to_root() for label in self.labels]
273
+ _json = attr.asdict(self, filter=attr.filters.exclude(attr.fields(Ontology)._client_api,
274
+ attr.fields(Ontology)._recipe,
275
+ attr.fields(Ontology)._project,
276
+ attr.fields(Ontology)._dataset,
277
+ attr.fields(Ontology)._instance_map,
278
+ attr.fields(Ontology)._color_map,
279
+ attr.fields(Ontology)._repositories))
280
+ _json["roots"] = roots
281
+ return _json
282
+
283
+ def delete(self):
284
+ """
285
+ Delete recipe from platform
286
+
287
+ :return: True
288
+ """
289
+ return self.ontologies.delete(self.id)
290
+
291
+ def update(self, system_metadata=False):
292
+ """
293
+ Update Ontology attribute
294
+
295
+ :param bool system_metadata: bool - True, if you want to change metadata system
296
+ :return: Ontology object
297
+ """
298
+ return self.ontologies.update(self, system_metadata=system_metadata)
299
+
300
+ def _add_children(self, label_name, children, labels_node, mode):
301
+ for child in children:
302
+ if not isinstance(child, entities.Label):
303
+ if isinstance(child, dict):
304
+ if "label_name" in child:
305
+ child = dict(child)
306
+ child["label_name"] = "{}.{}".format(label_name, child["label_name"])
307
+ labels_node += self._base_labels_handler(labels=[child], update_ontology=False, mode=mode)
308
+ else:
309
+ raise PlatformException("400",
310
+ "Invalid parameters - child list must have label name attribute")
311
+ else:
312
+ raise PlatformException("400", "Invalid parameters - child must be a dict type")
313
+ else:
314
+ child.tag = "{}.{}".format(label_name, child.tag)
315
+ labels_node += self._base_labels_handler(labels=child, update_ontology=False, mode=mode)
316
+
317
+ return labels_node
318
+
319
+ def _labels_handler_update_mode(self, json_req, upsert=False, log_error=True):
320
+ json_req['upsert'] = upsert
321
+ success, response = self._client_api.gen_request(req_type="PATCH",
322
+ path="/ontologies/%s/labels" % self.id,
323
+ json_req=json_req,
324
+ log_error=log_error)
325
+ if success:
326
+ logger.debug("Labels {} has been added successfully".format(json_req))
327
+ else:
328
+ raise exceptions.PlatformException(response)
329
+ return response
330
+
331
+ def _labels_handler_add_mode(self, json_req):
332
+ success, response = self._client_api.gen_request(req_type="PATCH",
333
+ path="/ontologies/%s/addLabels" % self.id,
334
+ json_req=json_req)
335
+ if success:
336
+ logger.debug("Labels {} has been added successfully".format(json_req))
337
+ else:
338
+ raise exceptions.PlatformException(response)
339
+ return response
340
+
341
+ def _base_labels_handler(self, labels, update_ontology=True, mode=LabelHandlerMode.UPSERT):
342
+ """
343
+ Add a single label to ontology using add label endpoint , nested label is also supported
344
+
345
+ :param labels = list of labels
346
+ :param update_ontology - return json_req if False
347
+ :param mode add, update or upsert, relevant on update_ontology=True only
348
+ :return: Ontology updated entire label entity
349
+ """
350
+ labels_node = list()
351
+ if mode not in [LabelHandlerMode.ADD,
352
+ LabelHandlerMode.UPDATE,
353
+ LabelHandlerMode.UPSERT]:
354
+ raise ValueError('mode must be on of: "add", "update", "upsert"')
355
+
356
+ if not isinstance(labels, list): # for case that add label get one label
357
+ labels = [labels]
358
+
359
+ for label in labels:
360
+ if isinstance(label, str):
361
+ # Generate label from string
362
+ label = entities.Label(tag=label)
363
+ elif isinstance(label, dict):
364
+ # Generate label from dict
365
+ label = Label.from_root(label)
366
+ elif isinstance(label, entities.Label):
367
+ ...
368
+ else:
369
+ raise ValueError(
370
+ 'Unsupported type for `labels`. Expected a list of (str, dict, dl.Label). Got: {}'.format(
371
+ type(label)))
372
+
373
+ # label entity
374
+ label_node = {"tag": label.tag}
375
+ if label.color is not None:
376
+ label_node["color"] = label.hex
377
+ if label.attributes is not None:
378
+ label_node["attributes"] = label.attributes
379
+ if label.display_label is not None:
380
+ label_node["displayLabel"] = label.display_label
381
+ if label.display_data is not None:
382
+ label_node["displayData"] = label.display_data
383
+ labels_node.append(label_node)
384
+ children = label.children
385
+ self._add_children(label.tag, children, labels_node, mode=mode)
386
+
387
+ if not update_ontology or not len(labels_node):
388
+ return labels_node
389
+
390
+ json_req = {
391
+ "labelsNode": labels_node
392
+ }
393
+
394
+ if mode == LabelHandlerMode.UPDATE:
395
+ response = self._labels_handler_update_mode(json_req)
396
+ else:
397
+ response = self._labels_handler_update_mode(json_req, upsert=True, log_error=False)
398
+
399
+ added_label = list()
400
+ if "roots" not in response.json():
401
+ raise exceptions.PlatformException("error fetching updated labels from server")
402
+
403
+ for root in response.json()["roots"]: # to get all labels
404
+ added_label.append(entities.Label.from_root(root=root))
405
+
406
+ self.labels = added_label
407
+ return added_label
408
+
409
+ def _add_image_label(self, icon_path):
410
+ display_data = dict()
411
+ if self.project is not None:
412
+ dataset = self.project.datasets._get_binaries_dataset()
413
+ elif self.dataset is not None:
414
+ dataset = self.dataset.project.datasets._get_binaries_dataset()
415
+ else:
416
+ raise ValueError('must have project or dataset to create with icon path')
417
+ platform_path = "/.dataloop/ontologies/{}/labelDisplayImages/".format(self.id)
418
+ basename = os.path.basename(icon_path)
419
+ item = dataset.items.upload(local_path=icon_path,
420
+ remote_path=platform_path,
421
+ remote_name='{}-{}'.format(uuid.uuid4().hex, basename))
422
+ display_data['displayImage'] = dict()
423
+ display_data['displayImage']['itemId'] = item.id
424
+ display_data['displayImage']['datasetId'] = item.dataset_id
425
+ return display_data
426
+
427
+ def _label_handler(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
428
+ add=True, icon_path=None, update_ontology=False, mode=LabelHandlerMode.UPSERT):
429
+ """
430
+ Add a single label to ontology
431
+
432
+ :param label_name: label name
433
+ :param color: optional - if not given a random color will be selected
434
+ :param children: optional - children
435
+ :param attributes: optional - attributes
436
+ :param display_label: optional - display_label
437
+ :param label: label
438
+ :param add:to add or not
439
+ :param icon_path: path to image to be display on label
440
+ :param update_ontology: update the ontology, default = False for backward compatible
441
+ :param mode add, update or upsert, relevant on update_ontology=True only
442
+ :return: Label entity
443
+ """
444
+
445
+ if update_ontology:
446
+ if isinstance(label, entities.Label) or isinstance(label, str):
447
+ return self._base_labels_handler(labels=label,
448
+ update_ontology=update_ontology,
449
+ mode=mode)
450
+ else:
451
+ display_data = dict()
452
+ if icon_path is not None:
453
+ display_data = self._add_image_label(icon_path=icon_path)
454
+ return self._base_labels_handler(labels={"tag": label_name,
455
+ "displayLabel": display_label,
456
+ "color": color,
457
+ "attributes": attributes,
458
+ "children": children,
459
+ "displayData": display_data
460
+ },
461
+ update_ontology=update_ontology,
462
+ mode=mode)
463
+
464
+ if not isinstance(label, entities.Label):
465
+ if "." in label_name:
466
+ raise PlatformException("400",
467
+ "Invalid parameters - nested label can work with update_ontology option only")
468
+
469
+ if attributes is None:
470
+ attributes = list()
471
+ if not isinstance(attributes, list):
472
+ attributes = [attributes]
473
+
474
+ # get random color if none given
475
+ if color is None:
476
+ color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
477
+
478
+ if children is None:
479
+ children = list()
480
+ if not isinstance(children, list):
481
+ children = [children]
482
+
483
+ # add children
484
+ added_children = list()
485
+ for child in children:
486
+ if not isinstance(child, entities.Label):
487
+ added_children.append(self._label_handler(**child, add=False))
488
+ else:
489
+ added_children.append(child)
490
+
491
+ if display_label is None:
492
+ display_label = ""
493
+ if len(label_name.split("_")) == 1:
494
+ display_label = label_name[0].upper() + label_name[1:]
495
+ else:
496
+ for word in label_name.split("_"):
497
+ display_label += word[0].upper() + word[1:] + " "
498
+ display_label = display_label[0:-1]
499
+
500
+ display_data = dict()
501
+ if icon_path is not None:
502
+ display_data = self._add_image_label(icon_path=icon_path)
503
+
504
+ root = {
505
+ "value": {
506
+ "tag": label_name,
507
+ "displayLabel": display_label,
508
+ "color": color,
509
+ "attributes": attributes,
510
+ "displayData": display_data
511
+ },
512
+ "children": list(),
513
+ }
514
+ added_label = entities.Label.from_root(root)
515
+ added_label.children = added_children
516
+ else:
517
+ added_label = label
518
+ if add and self._validate_label(added_label=added_label, mode=mode, color=color,
519
+ children=children, attributes=attributes,
520
+ display_label=display_label, display_data=icon_path):
521
+ self.labels.append(added_label)
522
+ self._base_labels_handler(labels=added_label, update_ontology=True, mode=mode)
523
+ return added_label
524
+
525
+ def _validate_label(self, added_label, mode=LabelHandlerMode.UPSERT, color=None, children=None, attributes=None,
526
+ display_label=None, display_data=None):
527
+ """
528
+ check if the label is exist
529
+ """
530
+ for i in range(len(self.labels)):
531
+ if self.labels[i].tag == added_label.tag:
532
+ if mode == LabelHandlerMode.UPDATE:
533
+ if color:
534
+ self.labels[i].color = added_label.color
535
+ if children:
536
+ self.labels[i].children = added_label.children
537
+ if attributes:
538
+ self.labels[i].attributes = added_label.attributes
539
+ if display_label:
540
+ self.labels[i].display_label = added_label.display_label
541
+ if display_data:
542
+ self.labels[i].display_data = added_label.display_data
543
+ return False
544
+ return True
545
+
546
+ def _labels_handler(self, label_list, update_ontology=False, mode=LabelHandlerMode.UPSERT):
547
+ """
548
+ Adds a list of labels to ontology
549
+
550
+ :param list label_list: a list of labels to add to the dataset's ontology. each value should be a dict, dl.Label or a string
551
+ if dictionary, should look like this: {"value": {"tag": "name of the label", "displayLabel": "display name on the platform",
552
+ "color": "#hex value", "attributes": [attributes]}, "children": [children]}
553
+ :param update_ontology: update the ontology, default = False for backward compatible
554
+ :param mode add, update or upsert, relevant on update_ontology=True only
555
+ :return: List of label entities added
556
+ """
557
+ if update_ontology:
558
+ return self._base_labels_handler(labels=label_list, mode=mode)
559
+ labels = list()
560
+ for label in label_list:
561
+
562
+ if isinstance(label, str):
563
+ label = entities.Label(tag=label)
564
+
565
+ if isinstance(label, entities.Label):
566
+ # label entity
567
+ labels.append(label)
568
+ else:
569
+ # dictionary
570
+ labels.append(Label.from_root(label))
571
+ added_labels = list()
572
+ for label in labels:
573
+ added_labels.append(self._label_handler(label.tag, label=label, update_ontology=update_ontology))
574
+
575
+ return added_labels
576
+
577
+ def delete_labels(self, label_names):
578
+ """
579
+ Delete labels from ontology
580
+
581
+ :param label_names: label object/ label name / list of label objects / list of label names
582
+ :return:
583
+ """
584
+ if not isinstance(label_names, list):
585
+ label_names = [label_names]
586
+
587
+ if isinstance(label_names[0], entities.Label):
588
+ label_names = [label.tag for label in label_names]
589
+
590
+ for label in label_names:
591
+ self.__delete_label(label)
592
+
593
+ self.update()
594
+
595
+ def __delete_label(self, label_name):
596
+ if label_name in self.instance_map.keys():
597
+ labels = self.labels
598
+ label_chain = label_name.split('.')
599
+ while len(label_chain) > 1:
600
+ label_name = label_chain.pop(0)
601
+ for i_label, label in enumerate(labels):
602
+ if label.tag == label_name:
603
+ labels = labels[i_label].children
604
+ break
605
+ label_name = label_chain[0]
606
+ for i_label, label in enumerate(labels):
607
+ if label.tag == label_name:
608
+ labels.pop(i_label)
609
+
610
+ def add_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
611
+ add=True, icon_path=None, update_ontology=False):
612
+ """
613
+ Add a single label to ontology
614
+
615
+ :param str label_name: str - label name
616
+ :param tuple color: color
617
+ :param children: children (sub labels)
618
+ :param list attributes: attributes
619
+ :param str display_label: display_label
620
+ :param dtlpy.entities.label.Label label: label
621
+ :param bool add: to add or not
622
+ :param str icon_path: path to image to be display on label
623
+ :param bool update_ontology: update the ontology, default = False for backward compatible
624
+ :return: Label entity
625
+ :rtype: dtlpy.entities.label.Label
626
+
627
+ **Example**:
628
+
629
+ .. code-block:: python
630
+
631
+ label = ontology.add_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
632
+ """
633
+ return self._label_handler(label_name=label_name, color=color, children=children, attributes=attributes,
634
+ display_label=display_label, label=label, add=add, icon_path=icon_path,
635
+ update_ontology=update_ontology)
636
+
637
+ def add_labels(self, label_list, update_ontology=False):
638
+ """
639
+ Adds a list of labels to ontology
640
+
641
+ :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel",
642
+ "color": "#color", "attributes": [attributes]}, "children": [children]}]
643
+ :param bool update_ontology: update the ontology, default = False for backward compatible
644
+ :return: List of label entities added
645
+
646
+ **Example**:
647
+
648
+ .. code-block:: python
649
+
650
+ labels = ontology.add_labels(label_list=label_list)
651
+ """
652
+ self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=LabelHandlerMode.UPSERT)
653
+
654
+ def update_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
655
+ add=True, icon_path=None, upsert=False, update_ontology=False):
656
+ """
657
+ Update a single label to ontology
658
+
659
+ :param str label_name: str - label name
660
+ :param tuple color: color
661
+ :param children: children (sub labels)
662
+ :param list attributes: attributes
663
+ :param str display_label: display_label
664
+ :param dtlpy.entities.label.Label label: label
665
+ :param bool add: to add or not
666
+ :param str icon_path: path to image to be display on label
667
+ :param bool update_ontology: update the ontology, default = False for backward compatible
668
+ :param bool upsert: if True will add in case it does not existing
669
+ :return: Label entity
670
+ :rtype: dtlpy.entities.label.Label
671
+
672
+ **Example**:
673
+
674
+ .. code-block:: python
675
+
676
+ label = ontology.update_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
677
+ """
678
+ if upsert:
679
+ mode = LabelHandlerMode.UPSERT
680
+ else:
681
+ mode = LabelHandlerMode.UPDATE
682
+
683
+ return self._label_handler(label_name=label_name, color=color, children=children,
684
+ attributes=attributes, display_label=display_label, label=label,
685
+ add=add, icon_path=icon_path, update_ontology=update_ontology, mode=mode)
686
+
687
+ def update_labels(self, label_list, upsert=False, update_ontology=False):
688
+ """
689
+ Update a list of labels to ontology
690
+
691
+ :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel", "color": "#color", "attributes": [attributes]}, "children": [children]}]
692
+ :param bool upsert: if True will add in case it does not existing
693
+ :param bool update_ontology: update the ontology, default = False for backward compatible
694
+
695
+ :return: List of label entities added
696
+
697
+ **Example**:
698
+
699
+ .. code-block:: python
700
+
701
+ labels = ontology.update_labels(label_list=label_list)
702
+ """
703
+
704
+ if upsert:
705
+ mode = LabelHandlerMode.UPSERT
706
+ else:
707
+ mode = LabelHandlerMode.UPDATE
708
+ self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=mode)
709
+
710
+ def update_attributes(self,
711
+ title: str,
712
+ key: str,
713
+ attribute_type,
714
+ scope: list = None,
715
+ optional: bool = None,
716
+ values: list = None,
717
+ attribute_range=None):
718
+ """
719
+ ADD a new attribute or update if exist
720
+
721
+ :param str title: attribute title
722
+ :param str key: the key of the attribute must br unique
723
+ :param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
724
+ :param list scope: list of the labels or * for all labels
725
+ :param bool optional: optional attribute
726
+ :param list values: list of the attribute values ( for checkbox and radio button)
727
+ :param dict or AttributesRange attribute_range: dl.AttributesRange object
728
+ :return: true in success
729
+ :rtype: bool
730
+ """
731
+ return self.ontologies.update_attributes(
732
+ ontology_id=self.id,
733
+ title=title,
734
+ key=key,
735
+ attribute_type=attribute_type,
736
+ scope=scope,
737
+ optional=optional,
738
+ values=values,
739
+ attribute_range=attribute_range)
740
+
741
+ def delete_attributes(self, keys: list):
742
+ """
743
+ Delete a bulk of attributes
744
+
745
+ :param list keys: Keys of attributes to delete
746
+ :return: True if success
747
+ :rtype: bool
748
+
749
+ **Example**:
750
+
751
+ .. code-block:: python
752
+
753
+ success = ontology.delete_attributes(['1'])
754
+ """
755
+
756
+ return self.ontologies.delete_attributes(ontology_id=self.id, keys=keys)
757
+
758
+ def copy_from(self, ontology_json: dict):
759
+ """
760
+ Import ontology to the platform.\n
761
+ Notice: only the following fields will be updated: `labels`, `attributes`, `instance_map` and `color_map`.
762
+
763
+ :param dict ontology_json: The source ontology json to copy from
764
+ :return: Ontology object: The updated ontology entity
765
+ :rtype: dtlpy.entities.ontology.Ontology
766
+
767
+ **Example**:
768
+
769
+ .. code-block:: python
770
+
771
+ ontology = ontology.import_ontology(ontology_json=ontology_json)
772
+ """
773
+ # TODO: Add support for import from ontology entity in the Future
774
+ if not self._use_attributes_2:
775
+ raise ValueError("This method is only supported for attributes 2 mode!")
776
+ new_ontology = self.from_json(_json=ontology_json, client_api=self._client_api)
777
+
778
+ # Update 'labels' and 'attributes'
779
+ self.labels = new_ontology.labels
780
+ new_attributes = new_ontology.attributes
781
+ if isinstance(new_attributes, list):
782
+ for new_attribute in new_attributes:
783
+ attribute_range = new_attribute.get("range", None)
784
+ if attribute_range is not None:
785
+ attribute_range = entities.AttributesRange(
786
+ min_range=attribute_range.get("min", None),
787
+ max_range=attribute_range.get("max", None),
788
+ step=attribute_range.get("step", None)
789
+ )
790
+ script_data = new_attribute.get("scriptData", None)
791
+ if script_data is None:
792
+ new_attribute_key = new_attribute.get("key", None)
793
+ raise Exception(f"Attribute '{new_attribute_key}' scriptData is missing in the ontology json!")
794
+ self.update_attributes(
795
+ title=script_data.get("title", None),
796
+ key=new_attribute.get("key", None),
797
+ attribute_type=new_attribute.get("type", None),
798
+ scope=new_attribute.get("scope", None),
799
+ optional=script_data.get("optional", None),
800
+ values=new_attribute.get("values", None),
801
+ attribute_range=attribute_range
802
+ )
803
+
804
+ # Get remote updated 'attributes'
805
+ self.metadata["attributes"] = self.ontologies.get(ontology_id=self.id).attributes
806
+
807
+ # Update 'instance map' and 'color map'
808
+ self._instance_map = new_ontology.instance_map
809
+ self._color_map = new_ontology.color_map
810
+ return self.update(system_metadata=True)