dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,803 +1,803 @@
1
- from collections import namedtuple
2
- import traceback
3
- import logging
4
- import random
5
- import uuid
6
- import attr
7
- import os
8
-
9
- from .. import entities, PlatformException, repositories, exceptions
10
- from ..services.api_client import ApiClient
11
- from .label import Label
12
-
13
- logger = logging.getLogger(name='dtlpy')
14
-
15
-
16
- class AttributesTypes:
17
- CHECKBOX = "checkbox"
18
- RADIO_BUTTON = "radio_button"
19
- SLIDER = "range"
20
- YES_NO = "boolean"
21
- FREE_TEXT = "freeText"
22
-
23
-
24
- class AttributesRange:
25
- def __init__(self, min_range, max_range, step):
26
- self.min_range = min_range
27
- self.max_range = max_range
28
- self.step = step
29
-
30
- def to_json(self):
31
- return {'min': self.min_range, 'max': self.max_range, 'step': self.step}
32
-
33
-
34
- class LabelHandlerMode:
35
- ADD = "add"
36
- UPDATE = "update"
37
- UPSERT = "upsert"
38
-
39
-
40
- @attr.s
41
- class Ontology(entities.BaseEntity):
42
- """
43
- Ontology object
44
- """
45
- # api
46
- _client_api = attr.ib(type=ApiClient, repr=False)
47
-
48
- # params
49
- id = attr.ib()
50
- creator = attr.ib()
51
- url = attr.ib(repr=False)
52
- title = attr.ib()
53
- labels = attr.ib(repr=False)
54
- metadata = attr.ib(repr=False)
55
- attributes = attr.ib()
56
-
57
- # entities
58
- _recipe = attr.ib(repr=False, default=None)
59
- _dataset = attr.ib(repr=False, default=None)
60
- _project = attr.ib(repr=False, default=None)
61
-
62
- # repositories
63
- _repositories = attr.ib(repr=False)
64
-
65
- # defaults
66
- _instance_map = attr.ib(default=None, repr=False)
67
- _color_map = attr.ib(default=None, repr=False)
68
-
69
- @_repositories.default
70
- def set_repositories(self):
71
- reps = namedtuple('repositories',
72
- field_names=['ontologies', 'datasets', 'projects'])
73
-
74
- if self._recipe is None:
75
- ontologies = repositories.Ontologies(client_api=self._client_api, recipe=self._recipe)
76
- else:
77
- ontologies = self.recipe.ontologies
78
-
79
- r = reps(ontologies=ontologies, datasets=repositories.Datasets(client_api=self._client_api),
80
- projects=repositories.Projects(client_api=self._client_api))
81
- return r
82
-
83
- @property
84
- def recipe(self):
85
- if self._recipe is not None:
86
- assert isinstance(self._recipe, entities.Recipe)
87
- return self._recipe
88
-
89
- @property
90
- def dataset(self):
91
- if self._dataset is None:
92
- if self.recipe is not None:
93
- self._dataset = self.recipe.dataset
94
- if self._dataset is not None:
95
- assert isinstance(self._dataset, entities.Dataset)
96
- return self._dataset
97
-
98
- @property
99
- def project(self):
100
- if self._project is None:
101
- if 'system' in self.metadata:
102
- project_id = self.metadata['system'].get('projectIds', None)
103
- if project_id is not None:
104
- self._project = self.projects.get(project_id=project_id[0])
105
- elif self.dataset is not None:
106
- self._project = self.dataset.project
107
- if self._project is not None:
108
- assert isinstance(self._project, entities.Project)
109
- return self._project
110
-
111
- @property
112
- def ontologies(self):
113
- if self._repositories.ontologies is not None:
114
- assert isinstance(self._repositories.ontologies, repositories.Ontologies)
115
- return self._repositories.ontologies
116
-
117
- @property
118
- def projects(self):
119
- if self._repositories.projects is not None:
120
- assert isinstance(self._repositories.projects, repositories.Projects)
121
- return self._repositories.projects
122
-
123
- @property
124
- def labels_flat_dict(self):
125
- flatten_dict = dict()
126
-
127
- def add_to_dict(tag, father):
128
- flatten_dict[tag] = father
129
- for child in father.children:
130
- add_to_dict('{}.{}'.format(tag, child.tag), child)
131
-
132
- for label in self.labels:
133
- add_to_dict(label.tag, label)
134
- return flatten_dict
135
-
136
- @property
137
- def instance_map(self):
138
- """
139
- instance mapping for creating instance mask
140
-
141
- :return dictionary {label: map_id}
142
- :rtype: dict
143
- """
144
- if self._instance_map is None:
145
- labels = [label for label in self.labels_flat_dict]
146
- labels.sort()
147
- # each label gets index as instance id
148
- self._instance_map = {label: (i_label + 1) for i_label, label in enumerate(labels)}
149
- return self._instance_map
150
-
151
- @instance_map.setter
152
- def instance_map(self, value: dict):
153
- """
154
- instance mapping for creating instance mask
155
-
156
- :param value: dictionary {label: map_id}
157
- :rtype: dict
158
- """
159
- if not isinstance(value, dict):
160
- raise ValueError('input must be a dictionary of {label_name: instance_id}')
161
- self._instance_map = value
162
-
163
- @property
164
- def color_map(self):
165
- """
166
- Color mapping of labels, {label: rgb}
167
-
168
- :return: dict
169
- :rtype: dict
170
- """
171
- if self._color_map is None:
172
- self._color_map = {k: v.rgb for k, v in self.labels_flat_dict.items()}
173
- return self._color_map
174
-
175
- @color_map.setter
176
- def color_map(self, values):
177
- """
178
- Color mapping of labels, {label: rgb}
179
-
180
- :param values: dict {label: rgb}
181
- :return:
182
- """
183
- if not isinstance(values, dict):
184
- raise ValueError('input must be a dict. got: {}'.format(type(values)))
185
- self._color_map = values
186
-
187
- @staticmethod
188
- def _protected_from_json(_json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
189
- """
190
- Same as from_json but with try-except to catch if error
191
- :param _json: platform json
192
- :param client_api: ApiClient entity
193
- :return:
194
- """
195
- try:
196
- ontology = Ontology.from_json(_json=_json,
197
- client_api=client_api,
198
- project=project,
199
- dataset=dataset,
200
- recipe=recipe,
201
- is_fetched=is_fetched)
202
- status = True
203
- except Exception:
204
- ontology = traceback.format_exc()
205
- status = False
206
- return status, ontology
207
-
208
- @property
209
- def _use_attributes_2(self):
210
- if isinstance(self.metadata, dict):
211
- attributes = self.metadata.get("attributes", None)
212
- if attributes is not None:
213
- return True
214
- else:
215
- if isinstance(self.attributes, list) and len(self.attributes) > 0:
216
- return False
217
- return True
218
-
219
- @classmethod
220
- def from_json(cls, _json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
221
- """
222
- Build an Ontology entity object from a json
223
-
224
- :param bool is_fetched: is Entity fetched from Platform
225
- :param dtlpy.entities.project.Project project: project entity
226
- :param dtlpy.entities.dataset.Dataset dataset: dataset
227
- :param dict _json: _json response from host
228
- :param dtlpy.entities.recipe.Recipe recipe: ontology's recipe
229
- :param dl.ApiClient client_api: ApiClient entity
230
- :return: Ontology object
231
- :rtype: dtlpy.entities.ontology.Ontology
232
- """
233
- attributes_v2 = _json.get('metadata', {}).get("attributes", [])
234
- attributes_v1 = _json.get("attributes", [])
235
- attributes = attributes_v2 if attributes_v2 else attributes_v1
236
-
237
- labels = list()
238
- for root in _json["roots"]:
239
- labels.append(entities.Label.from_root(root=root))
240
-
241
- inst = cls(
242
- metadata=_json.get("metadata", None),
243
- creator=_json.get("creator", None),
244
- url=_json.get("url", None),
245
- id=_json["id"],
246
- title=_json.get("title", None),
247
- attributes=attributes,
248
- client_api=client_api,
249
- project=project,
250
- dataset=dataset,
251
- recipe=recipe,
252
- labels=labels,
253
- )
254
-
255
- inst.is_fetched = is_fetched
256
- return inst
257
-
258
- def to_json(self):
259
- """
260
- Returns platform _json format of object
261
-
262
- :return: platform json format of object
263
- :rtype: dict
264
- """
265
- roots = [label.to_root() for label in self.labels]
266
- _json = attr.asdict(self, filter=attr.filters.exclude(attr.fields(Ontology)._client_api,
267
- attr.fields(Ontology)._recipe,
268
- attr.fields(Ontology)._project,
269
- attr.fields(Ontology)._dataset,
270
- attr.fields(Ontology)._instance_map,
271
- attr.fields(Ontology)._color_map,
272
- attr.fields(Ontology)._repositories))
273
- _json["roots"] = roots
274
- return _json
275
-
276
- def delete(self):
277
- """
278
- Delete recipe from platform
279
-
280
- :return: True
281
- """
282
- return self.ontologies.delete(self.id)
283
-
284
- def update(self, system_metadata=False):
285
- """
286
- Update items metadata
287
-
288
- :param bool system_metadata: bool - True, if you want to change metadata system
289
- :return: Ontology object
290
- """
291
- return self.ontologies.update(self, system_metadata=system_metadata)
292
-
293
- def _add_children(self, label_name, children, labels_node, mode):
294
- for child in children:
295
- if not isinstance(child, entities.Label):
296
- if isinstance(child, dict):
297
- if "label_name" in child:
298
- child = dict(child)
299
- child["label_name"] = "{}.{}".format(label_name, child["label_name"])
300
- labels_node += self._base_labels_handler(labels=[child], update_ontology=False, mode=mode)
301
- else:
302
- raise PlatformException("400",
303
- "Invalid parameters - child list must have label name attribute")
304
- else:
305
- raise PlatformException("400", "Invalid parameters - child must be a dict type")
306
- else:
307
- child.tag = "{}.{}".format(label_name, child.tag)
308
- labels_node += self._base_labels_handler(labels=child, update_ontology=False, mode=mode)
309
-
310
- return labels_node
311
-
312
- def _labels_handler_update_mode(self, json_req, upsert=False, log_error=True):
313
- json_req['upsert'] = upsert
314
- success, response = self._client_api.gen_request(req_type="PATCH",
315
- path="/ontologies/%s/labels" % self.id,
316
- json_req=json_req,
317
- log_error=log_error)
318
- if success:
319
- logger.debug("Labels {} has been added successfully".format(json_req))
320
- else:
321
- raise exceptions.PlatformException(response)
322
- return response
323
-
324
- def _labels_handler_add_mode(self, json_req):
325
- success, response = self._client_api.gen_request(req_type="PATCH",
326
- path="/ontologies/%s/addLabels" % self.id,
327
- json_req=json_req)
328
- if success:
329
- logger.debug("Labels {} has been added successfully".format(json_req))
330
- else:
331
- raise exceptions.PlatformException(response)
332
- return response
333
-
334
- def _base_labels_handler(self, labels, update_ontology=True, mode=LabelHandlerMode.UPSERT):
335
- """
336
- Add a single label to ontology using add label endpoint , nested label is also supported
337
-
338
- :param labels = list of labels
339
- :param update_ontology - return json_req if False
340
- :param mode add, update or upsert, relevant on update_ontology=True only
341
- :return: Ontology updated entire label entity
342
- """
343
- labels_node = list()
344
- if mode not in [LabelHandlerMode.ADD,
345
- LabelHandlerMode.UPDATE,
346
- LabelHandlerMode.UPSERT]:
347
- raise ValueError('mode must be on of: "add", "update", "upsert"')
348
-
349
- if not isinstance(labels, list): # for case that add label get one label
350
- labels = [labels]
351
-
352
- for label in labels:
353
- if isinstance(label, str):
354
- # Generate label from string
355
- label = entities.Label(tag=label)
356
- elif isinstance(label, dict):
357
- # Generate label from dict
358
- label = Label.from_root(label)
359
- elif isinstance(label, entities.Label):
360
- ...
361
- else:
362
- raise ValueError(
363
- 'Unsupported type for `labels`. Expected a list of (str, dict, dl.Label). Got: {}'.format(
364
- type(label)))
365
-
366
- # label entity
367
- label_node = {"tag": label.tag}
368
- if label.color is not None:
369
- label_node["color"] = label.hex
370
- if label.attributes is not None:
371
- label_node["attributes"] = label.attributes
372
- if label.display_label is not None:
373
- label_node["displayLabel"] = label.display_label
374
- if label.display_data is not None:
375
- label_node["displayData"] = label.display_data
376
- labels_node.append(label_node)
377
- children = label.children
378
- self._add_children(label.tag, children, labels_node, mode=mode)
379
-
380
- if not update_ontology or not len(labels_node):
381
- return labels_node
382
-
383
- json_req = {
384
- "labelsNode": labels_node
385
- }
386
-
387
- if mode == LabelHandlerMode.UPDATE:
388
- response = self._labels_handler_update_mode(json_req)
389
- else:
390
- response = self._labels_handler_update_mode(json_req, upsert=True, log_error=False)
391
-
392
- added_label = list()
393
- if "roots" not in response.json():
394
- raise exceptions.PlatformException("error fetching updated labels from server")
395
-
396
- for root in response.json()["roots"]: # to get all labels
397
- added_label.append(entities.Label.from_root(root=root))
398
-
399
- self.labels = added_label
400
- return added_label
401
-
402
- def _add_image_label(self, icon_path):
403
- display_data = dict()
404
- if self.project is not None:
405
- dataset = self.project.datasets._get_binaries_dataset()
406
- elif self.dataset is not None:
407
- dataset = self.dataset.project.datasets._get_binaries_dataset()
408
- else:
409
- raise ValueError('must have project or dataset to create with icon path')
410
- platform_path = "/.dataloop/ontologies/{}/labelDisplayImages/".format(self.id)
411
- basename = os.path.basename(icon_path)
412
- item = dataset.items.upload(local_path=icon_path,
413
- remote_path=platform_path,
414
- remote_name='{}-{}'.format(uuid.uuid4().hex, basename))
415
- display_data['displayImage'] = dict()
416
- display_data['displayImage']['itemId'] = item.id
417
- display_data['displayImage']['datasetId'] = item.dataset_id
418
- return display_data
419
-
420
- def _label_handler(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
421
- add=True, icon_path=None, update_ontology=False, mode=LabelHandlerMode.UPSERT):
422
- """
423
- Add a single label to ontology
424
-
425
- :param label_name: label name
426
- :param color: optional - if not given a random color will be selected
427
- :param children: optional - children
428
- :param attributes: optional - attributes
429
- :param display_label: optional - display_label
430
- :param label: label
431
- :param add:to add or not
432
- :param icon_path: path to image to be display on label
433
- :param update_ontology: update the ontology, default = False for backward compatible
434
- :param mode add, update or upsert, relevant on update_ontology=True only
435
- :return: Label entity
436
- """
437
-
438
- if update_ontology:
439
- if isinstance(label, entities.Label) or isinstance(label, str):
440
- return self._base_labels_handler(labels=label,
441
- update_ontology=update_ontology,
442
- mode=mode)
443
- else:
444
- display_data = dict()
445
- if icon_path is not None:
446
- display_data = self._add_image_label(icon_path=icon_path)
447
- return self._base_labels_handler(labels={"tag": label_name,
448
- "displayLabel": display_label,
449
- "color": color,
450
- "attributes": attributes,
451
- "children": children,
452
- "displayData": display_data
453
- },
454
- update_ontology=update_ontology,
455
- mode=mode)
456
-
457
- if not isinstance(label, entities.Label):
458
- if "." in label_name:
459
- raise PlatformException("400",
460
- "Invalid parameters - nested label can work with update_ontology option only")
461
-
462
- if attributes is None:
463
- attributes = list()
464
- if not isinstance(attributes, list):
465
- attributes = [attributes]
466
-
467
- # get random color if none given
468
- if color is None:
469
- color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
470
-
471
- if children is None:
472
- children = list()
473
- if not isinstance(children, list):
474
- children = [children]
475
-
476
- # add children
477
- added_children = list()
478
- for child in children:
479
- if not isinstance(child, entities.Label):
480
- added_children.append(self._label_handler(**child, add=False))
481
- else:
482
- added_children.append(child)
483
-
484
- if display_label is None:
485
- display_label = ""
486
- if len(label_name.split("_")) == 1:
487
- display_label = label_name[0].upper() + label_name[1:]
488
- else:
489
- for word in label_name.split("_"):
490
- display_label += word[0].upper() + word[1:] + " "
491
- display_label = display_label[0:-1]
492
-
493
- display_data = dict()
494
- if icon_path is not None:
495
- display_data = self._add_image_label(icon_path=icon_path)
496
-
497
- root = {
498
- "value": {
499
- "tag": label_name,
500
- "displayLabel": display_label,
501
- "color": color,
502
- "attributes": attributes,
503
- "displayData": display_data
504
- },
505
- "children": list(),
506
- }
507
- added_label = entities.Label.from_root(root)
508
- added_label.children = added_children
509
- else:
510
- added_label = label
511
- if add and self._validate_label(added_label=added_label, mode=mode, color=color,
512
- children=children, attributes=attributes,
513
- display_label=display_label, display_data=icon_path):
514
- self.labels.append(added_label)
515
- self._base_labels_handler(labels=added_label, update_ontology=True, mode=mode)
516
- return added_label
517
-
518
- def _validate_label(self, added_label, mode=LabelHandlerMode.UPSERT, color=None, children=None, attributes=None,
519
- display_label=None, display_data=None):
520
- """
521
- check if the label is exist
522
- """
523
- for i in range(len(self.labels)):
524
- if self.labels[i].tag == added_label.tag:
525
- if mode == LabelHandlerMode.UPDATE:
526
- if color:
527
- self.labels[i].color = added_label.color
528
- if children:
529
- self.labels[i].children = added_label.children
530
- if attributes:
531
- self.labels[i].attributes = added_label.attributes
532
- if display_label:
533
- self.labels[i].display_label = added_label.display_label
534
- if display_data:
535
- self.labels[i].display_data = added_label.display_data
536
- return False
537
- return True
538
-
539
- def _labels_handler(self, label_list, update_ontology=False, mode=LabelHandlerMode.UPSERT):
540
- """
541
- Adds a list of labels to ontology
542
-
543
- :param list label_list: a list of labels to add to the dataset's ontology. each value should be a dict, dl.Label or a string
544
- if dictionary, should look like this: {"value": {"tag": "name of the label", "displayLabel": "display name on the platform",
545
- "color": "#hex value", "attributes": [attributes]}, "children": [children]}
546
- :param update_ontology: update the ontology, default = False for backward compatible
547
- :param mode add, update or upsert, relevant on update_ontology=True only
548
- :return: List of label entities added
549
- """
550
- if update_ontology:
551
- return self._base_labels_handler(labels=label_list, mode=mode)
552
- labels = list()
553
- for label in label_list:
554
-
555
- if isinstance(label, str):
556
- label = entities.Label(tag=label)
557
-
558
- if isinstance(label, entities.Label):
559
- # label entity
560
- labels.append(label)
561
- else:
562
- # dictionary
563
- labels.append(Label.from_root(label))
564
- added_labels = list()
565
- for label in labels:
566
- added_labels.append(self._label_handler(label.tag, label=label, update_ontology=update_ontology))
567
-
568
- return added_labels
569
-
570
- def delete_labels(self, label_names):
571
- """
572
- Delete labels from ontology
573
-
574
- :param label_names: label object/ label name / list of label objects / list of label names
575
- :return:
576
- """
577
- if not isinstance(label_names, list):
578
- label_names = [label_names]
579
-
580
- if isinstance(label_names[0], entities.Label):
581
- label_names = [label.tag for label in label_names]
582
-
583
- for label in label_names:
584
- self.__delete_label(label)
585
-
586
- self.update()
587
-
588
- def __delete_label(self, label_name):
589
- if label_name in self.instance_map.keys():
590
- labels = self.labels
591
- label_chain = label_name.split('.')
592
- while len(label_chain) > 1:
593
- label_name = label_chain.pop(0)
594
- for i_label, label in enumerate(labels):
595
- if label.tag == label_name:
596
- labels = labels[i_label].children
597
- break
598
- label_name = label_chain[0]
599
- for i_label, label in enumerate(labels):
600
- if label.tag == label_name:
601
- labels.pop(i_label)
602
-
603
- def add_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
604
- add=True, icon_path=None, update_ontology=False):
605
- """
606
- Add a single label to ontology
607
-
608
- :param str label_name: str - label name
609
- :param tuple color: color
610
- :param children: children (sub labels)
611
- :param list attributes: attributes
612
- :param str display_label: display_label
613
- :param dtlpy.entities.label.Label label: label
614
- :param bool add: to add or not
615
- :param str icon_path: path to image to be display on label
616
- :param bool update_ontology: update the ontology, default = False for backward compatible
617
- :return: Label entity
618
- :rtype: dtlpy.entities.label.Label
619
-
620
- **Example**:
621
-
622
- .. code-block:: python
623
-
624
- label = ontology.add_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
625
- """
626
- return self._label_handler(label_name=label_name, color=color, children=children, attributes=attributes,
627
- display_label=display_label, label=label, add=add, icon_path=icon_path,
628
- update_ontology=update_ontology)
629
-
630
- def add_labels(self, label_list, update_ontology=False):
631
- """
632
- Adds a list of labels to ontology
633
-
634
- :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel",
635
- "color": "#color", "attributes": [attributes]}, "children": [children]}]
636
- :param bool update_ontology: update the ontology, default = False for backward compatible
637
- :return: List of label entities added
638
-
639
- **Example**:
640
-
641
- .. code-block:: python
642
-
643
- labels = ontology.add_labels(label_list=label_list)
644
- """
645
- self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=LabelHandlerMode.UPSERT)
646
-
647
- def update_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
648
- add=True, icon_path=None, upsert=False, update_ontology=False):
649
- """
650
- Update a single label to ontology
651
-
652
- :param str label_name: str - label name
653
- :param tuple color: color
654
- :param children: children (sub labels)
655
- :param list attributes: attributes
656
- :param str display_label: display_label
657
- :param dtlpy.entities.label.Label label: label
658
- :param bool add: to add or not
659
- :param str icon_path: path to image to be display on label
660
- :param bool update_ontology: update the ontology, default = False for backward compatible
661
- :param bool upsert: if True will add in case it does not existing
662
- :return: Label entity
663
- :rtype: dtlpy.entities.label.Label
664
-
665
- **Example**:
666
-
667
- .. code-block:: python
668
-
669
- label = ontology.update_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
670
- """
671
- if upsert:
672
- mode = LabelHandlerMode.UPSERT
673
- else:
674
- mode = LabelHandlerMode.UPDATE
675
-
676
- return self._label_handler(label_name=label_name, color=color, children=children,
677
- attributes=attributes, display_label=display_label, label=label,
678
- add=add, icon_path=icon_path, update_ontology=update_ontology, mode=mode)
679
-
680
- def update_labels(self, label_list, upsert=False, update_ontology=False):
681
- """
682
- Update a list of labels to ontology
683
-
684
- :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel", "color": "#color", "attributes": [attributes]}, "children": [children]}]
685
- :param bool upsert: if True will add in case it does not existing
686
- :param bool update_ontology: update the ontology, default = False for backward compatible
687
-
688
- :return: List of label entities added
689
-
690
- **Example**:
691
-
692
- .. code-block:: python
693
-
694
- labels = ontology.update_labels(label_list=label_list)
695
- """
696
-
697
- if upsert:
698
- mode = LabelHandlerMode.UPSERT
699
- else:
700
- mode = LabelHandlerMode.UPDATE
701
- self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=mode)
702
-
703
- def update_attributes(self,
704
- title: str,
705
- key: str,
706
- attribute_type,
707
- scope: list = None,
708
- optional: bool = None,
709
- values: list = None,
710
- attribute_range=None):
711
- """
712
- ADD a new attribute or update if exist
713
-
714
- :param str title: attribute title
715
- :param str key: the key of the attribute must br unique
716
- :param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
717
- :param list scope: list of the labels or * for all labels
718
- :param bool optional: optional attribute
719
- :param list values: list of the attribute values ( for checkbox and radio button)
720
- :param dict or AttributesRange attribute_range: dl.AttributesRange object
721
- :return: true in success
722
- :rtype: bool
723
- """
724
- return self.ontologies.update_attributes(
725
- ontology_id=self.id,
726
- title=title,
727
- key=key,
728
- attribute_type=attribute_type,
729
- scope=scope,
730
- optional=optional,
731
- values=values,
732
- attribute_range=attribute_range)
733
-
734
- def delete_attributes(self, keys: list):
735
- """
736
- Delete a bulk of attributes
737
-
738
- :param list keys: Keys of attributes to delete
739
- :return: True if success
740
- :rtype: bool
741
-
742
- **Example**:
743
-
744
- .. code-block:: python
745
-
746
- success = ontology.delete_attributes(['1'])
747
- """
748
-
749
- return self.ontologies.delete_attributes(ontology_id=self.id, keys=keys)
750
-
751
- def copy_from(self, ontology_json: dict):
752
- """
753
- Import ontology to the platform.\n
754
- Notice: only the following fields will be updated: `labels`, `attributes`, `instance_map` and `color_map`.
755
-
756
- :param dict ontology_json: The source ontology json to copy from
757
- :return: Ontology object: The updated ontology entity
758
- :rtype: dtlpy.entities.ontology.Ontology
759
-
760
- **Example**:
761
-
762
- .. code-block:: python
763
-
764
- ontology = ontology.import_ontology(ontology_json=ontology_json)
765
- """
766
- # TODO: Add support for import from ontology entity in the Future
767
- if not self._use_attributes_2:
768
- raise ValueError("This method is only supported for attributes 2 mode!")
769
- new_ontology = self.from_json(_json=ontology_json, client_api=self._client_api)
770
-
771
- # Update 'labels' and 'attributes'
772
- self.labels = new_ontology.labels
773
- new_attributes = new_ontology.attributes
774
- if isinstance(new_attributes, list):
775
- for new_attribute in new_attributes:
776
- attribute_range = new_attribute.get("range", None)
777
- if attribute_range is not None:
778
- attribute_range = entities.AttributesRange(
779
- min_range=attribute_range.get("min", None),
780
- max_range=attribute_range.get("max", None),
781
- step=attribute_range.get("step", None)
782
- )
783
- script_data = new_attribute.get("scriptData", None)
784
- if script_data is None:
785
- new_attribute_key = new_attribute.get("key", None)
786
- raise Exception(f"Attribute '{new_attribute_key}' scriptData is missing in the ontology json!")
787
- self.update_attributes(
788
- title=script_data.get("title", None),
789
- key=new_attribute.get("key", None),
790
- attribute_type=new_attribute.get("type", None),
791
- scope=new_attribute.get("scope", None),
792
- optional=script_data.get("optional", None),
793
- values=new_attribute.get("values", None),
794
- attribute_range=attribute_range
795
- )
796
-
797
- # Get remote updated 'attributes'
798
- self.metadata["attributes"] = self.ontologies.get(ontology_id=self.id).attributes
799
-
800
- # Update 'instance map' and 'color map'
801
- self._instance_map = new_ontology.instance_map
802
- self._color_map = new_ontology.color_map
803
- return self.update(system_metadata=True)
1
+ from collections import namedtuple
2
+ import traceback
3
+ import logging
4
+ import random
5
+ import uuid
6
+ import attr
7
+ import os
8
+
9
+ from .. import entities, PlatformException, repositories, exceptions
10
+ from ..services.api_client import ApiClient
11
+ from .label import Label
12
+
13
+ logger = logging.getLogger(name='dtlpy')
14
+
15
+
16
+ class AttributesTypes:
17
+ CHECKBOX = "checkbox"
18
+ RADIO_BUTTON = "radio_button"
19
+ SLIDER = "range"
20
+ YES_NO = "boolean"
21
+ FREE_TEXT = "freeText"
22
+
23
+
24
+ class AttributesRange:
25
+ def __init__(self, min_range, max_range, step):
26
+ self.min_range = min_range
27
+ self.max_range = max_range
28
+ self.step = step
29
+
30
+ def to_json(self):
31
+ return {'min': self.min_range, 'max': self.max_range, 'step': self.step}
32
+
33
+
34
+ class LabelHandlerMode:
35
+ ADD = "add"
36
+ UPDATE = "update"
37
+ UPSERT = "upsert"
38
+
39
+
40
+ @attr.s
41
+ class Ontology(entities.BaseEntity):
42
+ """
43
+ Ontology object
44
+ """
45
+ # api
46
+ _client_api = attr.ib(type=ApiClient, repr=False)
47
+
48
+ # params
49
+ id = attr.ib()
50
+ creator = attr.ib()
51
+ url = attr.ib(repr=False)
52
+ title = attr.ib()
53
+ labels = attr.ib(repr=False)
54
+ metadata = attr.ib(repr=False)
55
+ attributes = attr.ib()
56
+
57
+ # entities
58
+ _recipe = attr.ib(repr=False, default=None)
59
+ _dataset = attr.ib(repr=False, default=None)
60
+ _project = attr.ib(repr=False, default=None)
61
+
62
+ # repositories
63
+ _repositories = attr.ib(repr=False)
64
+
65
+ # defaults
66
+ _instance_map = attr.ib(default=None, repr=False)
67
+ _color_map = attr.ib(default=None, repr=False)
68
+
69
+ @_repositories.default
70
+ def set_repositories(self):
71
+ reps = namedtuple('repositories',
72
+ field_names=['ontologies', 'datasets', 'projects'])
73
+
74
+ if self._recipe is None:
75
+ ontologies = repositories.Ontologies(client_api=self._client_api, recipe=self._recipe)
76
+ else:
77
+ ontologies = self.recipe.ontologies
78
+
79
+ r = reps(ontologies=ontologies, datasets=repositories.Datasets(client_api=self._client_api),
80
+ projects=repositories.Projects(client_api=self._client_api))
81
+ return r
82
+
83
+ @property
84
+ def recipe(self):
85
+ if self._recipe is not None:
86
+ assert isinstance(self._recipe, entities.Recipe)
87
+ return self._recipe
88
+
89
+ @property
90
+ def dataset(self):
91
+ if self._dataset is None:
92
+ if self.recipe is not None:
93
+ self._dataset = self.recipe.dataset
94
+ if self._dataset is not None:
95
+ assert isinstance(self._dataset, entities.Dataset)
96
+ return self._dataset
97
+
98
+ @property
99
+ def project(self):
100
+ if self._project is None:
101
+ if 'system' in self.metadata:
102
+ project_id = self.metadata['system'].get('projectIds', None)
103
+ if project_id is not None:
104
+ self._project = self.projects.get(project_id=project_id[0])
105
+ elif self.dataset is not None:
106
+ self._project = self.dataset.project
107
+ if self._project is not None:
108
+ assert isinstance(self._project, entities.Project)
109
+ return self._project
110
+
111
+ @property
112
+ def ontologies(self):
113
+ if self._repositories.ontologies is not None:
114
+ assert isinstance(self._repositories.ontologies, repositories.Ontologies)
115
+ return self._repositories.ontologies
116
+
117
+ @property
118
+ def projects(self):
119
+ if self._repositories.projects is not None:
120
+ assert isinstance(self._repositories.projects, repositories.Projects)
121
+ return self._repositories.projects
122
+
123
+ @property
124
+ def labels_flat_dict(self):
125
+ flatten_dict = dict()
126
+
127
+ def add_to_dict(tag, father):
128
+ flatten_dict[tag] = father
129
+ for child in father.children:
130
+ add_to_dict('{}.{}'.format(tag, child.tag), child)
131
+
132
+ for label in self.labels:
133
+ add_to_dict(label.tag, label)
134
+ return flatten_dict
135
+
136
+ @property
137
+ def instance_map(self):
138
+ """
139
+ instance mapping for creating instance mask
140
+
141
+ :return dictionary {label: map_id}
142
+ :rtype: dict
143
+ """
144
+ if self._instance_map is None:
145
+ labels = [label for label in self.labels_flat_dict]
146
+ labels.sort()
147
+ # each label gets index as instance id
148
+ self._instance_map = {label: (i_label + 1) for i_label, label in enumerate(labels)}
149
+ return self._instance_map
150
+
151
+ @instance_map.setter
152
+ def instance_map(self, value: dict):
153
+ """
154
+ instance mapping for creating instance mask
155
+
156
+ :param value: dictionary {label: map_id}
157
+ :rtype: dict
158
+ """
159
+ if not isinstance(value, dict):
160
+ raise ValueError('input must be a dictionary of {label_name: instance_id}')
161
+ self._instance_map = value
162
+
163
+ @property
164
+ def color_map(self):
165
+ """
166
+ Color mapping of labels, {label: rgb}
167
+
168
+ :return: dict
169
+ :rtype: dict
170
+ """
171
+ if self._color_map is None:
172
+ self._color_map = {k: v.rgb for k, v in self.labels_flat_dict.items()}
173
+ return self._color_map
174
+
175
+ @color_map.setter
176
+ def color_map(self, values):
177
+ """
178
+ Color mapping of labels, {label: rgb}
179
+
180
+ :param values: dict {label: rgb}
181
+ :return:
182
+ """
183
+ if not isinstance(values, dict):
184
+ raise ValueError('input must be a dict. got: {}'.format(type(values)))
185
+ self._color_map = values
186
+
187
+ @staticmethod
188
+ def _protected_from_json(_json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
189
+ """
190
+ Same as from_json but with try-except to catch if error
191
+ :param _json: platform json
192
+ :param client_api: ApiClient entity
193
+ :return:
194
+ """
195
+ try:
196
+ ontology = Ontology.from_json(_json=_json,
197
+ client_api=client_api,
198
+ project=project,
199
+ dataset=dataset,
200
+ recipe=recipe,
201
+ is_fetched=is_fetched)
202
+ status = True
203
+ except Exception:
204
+ ontology = traceback.format_exc()
205
+ status = False
206
+ return status, ontology
207
+
208
+ @property
209
+ def _use_attributes_2(self):
210
+ if isinstance(self.metadata, dict):
211
+ attributes = self.metadata.get("attributes", None)
212
+ if attributes is not None:
213
+ return True
214
+ else:
215
+ if isinstance(self.attributes, list) and len(self.attributes) > 0:
216
+ return False
217
+ return True
218
+
219
+ @classmethod
220
+ def from_json(cls, _json, client_api, recipe=None, dataset=None, project=None, is_fetched=True):
221
+ """
222
+ Build an Ontology entity object from a json
223
+
224
+ :param bool is_fetched: is Entity fetched from Platform
225
+ :param dtlpy.entities.project.Project project: project entity
226
+ :param dtlpy.entities.dataset.Dataset dataset: dataset
227
+ :param dict _json: _json response from host
228
+ :param dtlpy.entities.recipe.Recipe recipe: ontology's recipe
229
+ :param dl.ApiClient client_api: ApiClient entity
230
+ :return: Ontology object
231
+ :rtype: dtlpy.entities.ontology.Ontology
232
+ """
233
+ attributes_v2 = _json.get('metadata', {}).get("attributes", [])
234
+ attributes_v1 = _json.get("attributes", [])
235
+ attributes = attributes_v2 if attributes_v2 else attributes_v1
236
+
237
+ labels = list()
238
+ for root in _json["roots"]:
239
+ labels.append(entities.Label.from_root(root=root))
240
+
241
+ inst = cls(
242
+ metadata=_json.get("metadata", None),
243
+ creator=_json.get("creator", None),
244
+ url=_json.get("url", None),
245
+ id=_json["id"],
246
+ title=_json.get("title", None),
247
+ attributes=attributes,
248
+ client_api=client_api,
249
+ project=project,
250
+ dataset=dataset,
251
+ recipe=recipe,
252
+ labels=labels,
253
+ )
254
+
255
+ inst.is_fetched = is_fetched
256
+ return inst
257
+
258
+ def to_json(self):
259
+ """
260
+ Returns platform _json format of object
261
+
262
+ :return: platform json format of object
263
+ :rtype: dict
264
+ """
265
+ roots = [label.to_root() for label in self.labels]
266
+ _json = attr.asdict(self, filter=attr.filters.exclude(attr.fields(Ontology)._client_api,
267
+ attr.fields(Ontology)._recipe,
268
+ attr.fields(Ontology)._project,
269
+ attr.fields(Ontology)._dataset,
270
+ attr.fields(Ontology)._instance_map,
271
+ attr.fields(Ontology)._color_map,
272
+ attr.fields(Ontology)._repositories))
273
+ _json["roots"] = roots
274
+ return _json
275
+
276
+ def delete(self):
277
+ """
278
+ Delete recipe from platform
279
+
280
+ :return: True
281
+ """
282
+ return self.ontologies.delete(self.id)
283
+
284
+ def update(self, system_metadata=False):
285
+ """
286
+ Update items metadata
287
+
288
+ :param bool system_metadata: bool - True, if you want to change metadata system
289
+ :return: Ontology object
290
+ """
291
+ return self.ontologies.update(self, system_metadata=system_metadata)
292
+
293
+ def _add_children(self, label_name, children, labels_node, mode):
294
+ for child in children:
295
+ if not isinstance(child, entities.Label):
296
+ if isinstance(child, dict):
297
+ if "label_name" in child:
298
+ child = dict(child)
299
+ child["label_name"] = "{}.{}".format(label_name, child["label_name"])
300
+ labels_node += self._base_labels_handler(labels=[child], update_ontology=False, mode=mode)
301
+ else:
302
+ raise PlatformException("400",
303
+ "Invalid parameters - child list must have label name attribute")
304
+ else:
305
+ raise PlatformException("400", "Invalid parameters - child must be a dict type")
306
+ else:
307
+ child.tag = "{}.{}".format(label_name, child.tag)
308
+ labels_node += self._base_labels_handler(labels=child, update_ontology=False, mode=mode)
309
+
310
+ return labels_node
311
+
312
+ def _labels_handler_update_mode(self, json_req, upsert=False, log_error=True):
313
+ json_req['upsert'] = upsert
314
+ success, response = self._client_api.gen_request(req_type="PATCH",
315
+ path="/ontologies/%s/labels" % self.id,
316
+ json_req=json_req,
317
+ log_error=log_error)
318
+ if success:
319
+ logger.debug("Labels {} has been added successfully".format(json_req))
320
+ else:
321
+ raise exceptions.PlatformException(response)
322
+ return response
323
+
324
+ def _labels_handler_add_mode(self, json_req):
325
+ success, response = self._client_api.gen_request(req_type="PATCH",
326
+ path="/ontologies/%s/addLabels" % self.id,
327
+ json_req=json_req)
328
+ if success:
329
+ logger.debug("Labels {} has been added successfully".format(json_req))
330
+ else:
331
+ raise exceptions.PlatformException(response)
332
+ return response
333
+
334
+ def _base_labels_handler(self, labels, update_ontology=True, mode=LabelHandlerMode.UPSERT):
335
+ """
336
+ Add a single label to ontology using add label endpoint , nested label is also supported
337
+
338
+ :param labels = list of labels
339
+ :param update_ontology - return json_req if False
340
+ :param mode add, update or upsert, relevant on update_ontology=True only
341
+ :return: Ontology updated entire label entity
342
+ """
343
+ labels_node = list()
344
+ if mode not in [LabelHandlerMode.ADD,
345
+ LabelHandlerMode.UPDATE,
346
+ LabelHandlerMode.UPSERT]:
347
+ raise ValueError('mode must be on of: "add", "update", "upsert"')
348
+
349
+ if not isinstance(labels, list): # for case that add label get one label
350
+ labels = [labels]
351
+
352
+ for label in labels:
353
+ if isinstance(label, str):
354
+ # Generate label from string
355
+ label = entities.Label(tag=label)
356
+ elif isinstance(label, dict):
357
+ # Generate label from dict
358
+ label = Label.from_root(label)
359
+ elif isinstance(label, entities.Label):
360
+ ...
361
+ else:
362
+ raise ValueError(
363
+ 'Unsupported type for `labels`. Expected a list of (str, dict, dl.Label). Got: {}'.format(
364
+ type(label)))
365
+
366
+ # label entity
367
+ label_node = {"tag": label.tag}
368
+ if label.color is not None:
369
+ label_node["color"] = label.hex
370
+ if label.attributes is not None:
371
+ label_node["attributes"] = label.attributes
372
+ if label.display_label is not None:
373
+ label_node["displayLabel"] = label.display_label
374
+ if label.display_data is not None:
375
+ label_node["displayData"] = label.display_data
376
+ labels_node.append(label_node)
377
+ children = label.children
378
+ self._add_children(label.tag, children, labels_node, mode=mode)
379
+
380
+ if not update_ontology or not len(labels_node):
381
+ return labels_node
382
+
383
+ json_req = {
384
+ "labelsNode": labels_node
385
+ }
386
+
387
+ if mode == LabelHandlerMode.UPDATE:
388
+ response = self._labels_handler_update_mode(json_req)
389
+ else:
390
+ response = self._labels_handler_update_mode(json_req, upsert=True, log_error=False)
391
+
392
+ added_label = list()
393
+ if "roots" not in response.json():
394
+ raise exceptions.PlatformException("error fetching updated labels from server")
395
+
396
+ for root in response.json()["roots"]: # to get all labels
397
+ added_label.append(entities.Label.from_root(root=root))
398
+
399
+ self.labels = added_label
400
+ return added_label
401
+
402
+ def _add_image_label(self, icon_path):
403
+ display_data = dict()
404
+ if self.project is not None:
405
+ dataset = self.project.datasets._get_binaries_dataset()
406
+ elif self.dataset is not None:
407
+ dataset = self.dataset.project.datasets._get_binaries_dataset()
408
+ else:
409
+ raise ValueError('must have project or dataset to create with icon path')
410
+ platform_path = "/.dataloop/ontologies/{}/labelDisplayImages/".format(self.id)
411
+ basename = os.path.basename(icon_path)
412
+ item = dataset.items.upload(local_path=icon_path,
413
+ remote_path=platform_path,
414
+ remote_name='{}-{}'.format(uuid.uuid4().hex, basename))
415
+ display_data['displayImage'] = dict()
416
+ display_data['displayImage']['itemId'] = item.id
417
+ display_data['displayImage']['datasetId'] = item.dataset_id
418
+ return display_data
419
+
420
+ def _label_handler(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
421
+ add=True, icon_path=None, update_ontology=False, mode=LabelHandlerMode.UPSERT):
422
+ """
423
+ Add a single label to ontology
424
+
425
+ :param label_name: label name
426
+ :param color: optional - if not given a random color will be selected
427
+ :param children: optional - children
428
+ :param attributes: optional - attributes
429
+ :param display_label: optional - display_label
430
+ :param label: label
431
+ :param add:to add or not
432
+ :param icon_path: path to image to be display on label
433
+ :param update_ontology: update the ontology, default = False for backward compatible
434
+ :param mode add, update or upsert, relevant on update_ontology=True only
435
+ :return: Label entity
436
+ """
437
+
438
+ if update_ontology:
439
+ if isinstance(label, entities.Label) or isinstance(label, str):
440
+ return self._base_labels_handler(labels=label,
441
+ update_ontology=update_ontology,
442
+ mode=mode)
443
+ else:
444
+ display_data = dict()
445
+ if icon_path is not None:
446
+ display_data = self._add_image_label(icon_path=icon_path)
447
+ return self._base_labels_handler(labels={"tag": label_name,
448
+ "displayLabel": display_label,
449
+ "color": color,
450
+ "attributes": attributes,
451
+ "children": children,
452
+ "displayData": display_data
453
+ },
454
+ update_ontology=update_ontology,
455
+ mode=mode)
456
+
457
+ if not isinstance(label, entities.Label):
458
+ if "." in label_name:
459
+ raise PlatformException("400",
460
+ "Invalid parameters - nested label can work with update_ontology option only")
461
+
462
+ if attributes is None:
463
+ attributes = list()
464
+ if not isinstance(attributes, list):
465
+ attributes = [attributes]
466
+
467
+ # get random color if none given
468
+ if color is None:
469
+ color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
470
+
471
+ if children is None:
472
+ children = list()
473
+ if not isinstance(children, list):
474
+ children = [children]
475
+
476
+ # add children
477
+ added_children = list()
478
+ for child in children:
479
+ if not isinstance(child, entities.Label):
480
+ added_children.append(self._label_handler(**child, add=False))
481
+ else:
482
+ added_children.append(child)
483
+
484
+ if display_label is None:
485
+ display_label = ""
486
+ if len(label_name.split("_")) == 1:
487
+ display_label = label_name[0].upper() + label_name[1:]
488
+ else:
489
+ for word in label_name.split("_"):
490
+ display_label += word[0].upper() + word[1:] + " "
491
+ display_label = display_label[0:-1]
492
+
493
+ display_data = dict()
494
+ if icon_path is not None:
495
+ display_data = self._add_image_label(icon_path=icon_path)
496
+
497
+ root = {
498
+ "value": {
499
+ "tag": label_name,
500
+ "displayLabel": display_label,
501
+ "color": color,
502
+ "attributes": attributes,
503
+ "displayData": display_data
504
+ },
505
+ "children": list(),
506
+ }
507
+ added_label = entities.Label.from_root(root)
508
+ added_label.children = added_children
509
+ else:
510
+ added_label = label
511
+ if add and self._validate_label(added_label=added_label, mode=mode, color=color,
512
+ children=children, attributes=attributes,
513
+ display_label=display_label, display_data=icon_path):
514
+ self.labels.append(added_label)
515
+ self._base_labels_handler(labels=added_label, update_ontology=True, mode=mode)
516
+ return added_label
517
+
518
+ def _validate_label(self, added_label, mode=LabelHandlerMode.UPSERT, color=None, children=None, attributes=None,
519
+ display_label=None, display_data=None):
520
+ """
521
+ check if the label is exist
522
+ """
523
+ for i in range(len(self.labels)):
524
+ if self.labels[i].tag == added_label.tag:
525
+ if mode == LabelHandlerMode.UPDATE:
526
+ if color:
527
+ self.labels[i].color = added_label.color
528
+ if children:
529
+ self.labels[i].children = added_label.children
530
+ if attributes:
531
+ self.labels[i].attributes = added_label.attributes
532
+ if display_label:
533
+ self.labels[i].display_label = added_label.display_label
534
+ if display_data:
535
+ self.labels[i].display_data = added_label.display_data
536
+ return False
537
+ return True
538
+
539
+ def _labels_handler(self, label_list, update_ontology=False, mode=LabelHandlerMode.UPSERT):
540
+ """
541
+ Adds a list of labels to ontology
542
+
543
+ :param list label_list: a list of labels to add to the dataset's ontology. each value should be a dict, dl.Label or a string
544
+ if dictionary, should look like this: {"value": {"tag": "name of the label", "displayLabel": "display name on the platform",
545
+ "color": "#hex value", "attributes": [attributes]}, "children": [children]}
546
+ :param update_ontology: update the ontology, default = False for backward compatible
547
+ :param mode add, update or upsert, relevant on update_ontology=True only
548
+ :return: List of label entities added
549
+ """
550
+ if update_ontology:
551
+ return self._base_labels_handler(labels=label_list, mode=mode)
552
+ labels = list()
553
+ for label in label_list:
554
+
555
+ if isinstance(label, str):
556
+ label = entities.Label(tag=label)
557
+
558
+ if isinstance(label, entities.Label):
559
+ # label entity
560
+ labels.append(label)
561
+ else:
562
+ # dictionary
563
+ labels.append(Label.from_root(label))
564
+ added_labels = list()
565
+ for label in labels:
566
+ added_labels.append(self._label_handler(label.tag, label=label, update_ontology=update_ontology))
567
+
568
+ return added_labels
569
+
570
+ def delete_labels(self, label_names):
571
+ """
572
+ Delete labels from ontology
573
+
574
+ :param label_names: label object/ label name / list of label objects / list of label names
575
+ :return:
576
+ """
577
+ if not isinstance(label_names, list):
578
+ label_names = [label_names]
579
+
580
+ if isinstance(label_names[0], entities.Label):
581
+ label_names = [label.tag for label in label_names]
582
+
583
+ for label in label_names:
584
+ self.__delete_label(label)
585
+
586
+ self.update()
587
+
588
+ def __delete_label(self, label_name):
589
+ if label_name in self.instance_map.keys():
590
+ labels = self.labels
591
+ label_chain = label_name.split('.')
592
+ while len(label_chain) > 1:
593
+ label_name = label_chain.pop(0)
594
+ for i_label, label in enumerate(labels):
595
+ if label.tag == label_name:
596
+ labels = labels[i_label].children
597
+ break
598
+ label_name = label_chain[0]
599
+ for i_label, label in enumerate(labels):
600
+ if label.tag == label_name:
601
+ labels.pop(i_label)
602
+
603
+ def add_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
604
+ add=True, icon_path=None, update_ontology=False):
605
+ """
606
+ Add a single label to ontology
607
+
608
+ :param str label_name: str - label name
609
+ :param tuple color: color
610
+ :param children: children (sub labels)
611
+ :param list attributes: attributes
612
+ :param str display_label: display_label
613
+ :param dtlpy.entities.label.Label label: label
614
+ :param bool add: to add or not
615
+ :param str icon_path: path to image to be display on label
616
+ :param bool update_ontology: update the ontology, default = False for backward compatible
617
+ :return: Label entity
618
+ :rtype: dtlpy.entities.label.Label
619
+
620
+ **Example**:
621
+
622
+ .. code-block:: python
623
+
624
+ label = ontology.add_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
625
+ """
626
+ return self._label_handler(label_name=label_name, color=color, children=children, attributes=attributes,
627
+ display_label=display_label, label=label, add=add, icon_path=icon_path,
628
+ update_ontology=update_ontology)
629
+
630
+ def add_labels(self, label_list, update_ontology=False):
631
+ """
632
+ Adds a list of labels to ontology
633
+
634
+ :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel",
635
+ "color": "#color", "attributes": [attributes]}, "children": [children]}]
636
+ :param bool update_ontology: update the ontology, default = False for backward compatible
637
+ :return: List of label entities added
638
+
639
+ **Example**:
640
+
641
+ .. code-block:: python
642
+
643
+ labels = ontology.add_labels(label_list=label_list)
644
+ """
645
+ self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=LabelHandlerMode.UPSERT)
646
+
647
+ def update_label(self, label_name, color=None, children=None, attributes=None, display_label=None, label=None,
648
+ add=True, icon_path=None, upsert=False, update_ontology=False):
649
+ """
650
+ Update a single label to ontology
651
+
652
+ :param str label_name: str - label name
653
+ :param tuple color: color
654
+ :param children: children (sub labels)
655
+ :param list attributes: attributes
656
+ :param str display_label: display_label
657
+ :param dtlpy.entities.label.Label label: label
658
+ :param bool add: to add or not
659
+ :param str icon_path: path to image to be display on label
660
+ :param bool update_ontology: update the ontology, default = False for backward compatible
661
+ :param bool upsert: if True will add in case it does not existing
662
+ :return: Label entity
663
+ :rtype: dtlpy.entities.label.Label
664
+
665
+ **Example**:
666
+
667
+ .. code-block:: python
668
+
669
+ label = ontology.update_label(label_name='person', color=(34, 6, 231), attributes=['big', 'small'])
670
+ """
671
+ if upsert:
672
+ mode = LabelHandlerMode.UPSERT
673
+ else:
674
+ mode = LabelHandlerMode.UPDATE
675
+
676
+ return self._label_handler(label_name=label_name, color=color, children=children,
677
+ attributes=attributes, display_label=display_label, label=label,
678
+ add=add, icon_path=icon_path, update_ontology=update_ontology, mode=mode)
679
+
680
+ def update_labels(self, label_list, upsert=False, update_ontology=False):
681
+ """
682
+ Update a list of labels to ontology
683
+
684
+ :param list label_list: list of labels [{"value": {"tag": "tag", "displayLabel": "displayLabel", "color": "#color", "attributes": [attributes]}, "children": [children]}]
685
+ :param bool upsert: if True will add in case it does not existing
686
+ :param bool update_ontology: update the ontology, default = False for backward compatible
687
+
688
+ :return: List of label entities added
689
+
690
+ **Example**:
691
+
692
+ .. code-block:: python
693
+
694
+ labels = ontology.update_labels(label_list=label_list)
695
+ """
696
+
697
+ if upsert:
698
+ mode = LabelHandlerMode.UPSERT
699
+ else:
700
+ mode = LabelHandlerMode.UPDATE
701
+ self._labels_handler(label_list=label_list, update_ontology=update_ontology, mode=mode)
702
+
703
+ def update_attributes(self,
704
+ title: str,
705
+ key: str,
706
+ attribute_type,
707
+ scope: list = None,
708
+ optional: bool = None,
709
+ values: list = None,
710
+ attribute_range=None):
711
+ """
712
+ ADD a new attribute or update if exist
713
+
714
+ :param str title: attribute title
715
+ :param str key: the key of the attribute must br unique
716
+ :param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
717
+ :param list scope: list of the labels or * for all labels
718
+ :param bool optional: optional attribute
719
+ :param list values: list of the attribute values ( for checkbox and radio button)
720
+ :param dict or AttributesRange attribute_range: dl.AttributesRange object
721
+ :return: true in success
722
+ :rtype: bool
723
+ """
724
+ return self.ontologies.update_attributes(
725
+ ontology_id=self.id,
726
+ title=title,
727
+ key=key,
728
+ attribute_type=attribute_type,
729
+ scope=scope,
730
+ optional=optional,
731
+ values=values,
732
+ attribute_range=attribute_range)
733
+
734
+ def delete_attributes(self, keys: list):
735
+ """
736
+ Delete a bulk of attributes
737
+
738
+ :param list keys: Keys of attributes to delete
739
+ :return: True if success
740
+ :rtype: bool
741
+
742
+ **Example**:
743
+
744
+ .. code-block:: python
745
+
746
+ success = ontology.delete_attributes(['1'])
747
+ """
748
+
749
+ return self.ontologies.delete_attributes(ontology_id=self.id, keys=keys)
750
+
751
+ def copy_from(self, ontology_json: dict):
752
+ """
753
+ Import ontology to the platform.\n
754
+ Notice: only the following fields will be updated: `labels`, `attributes`, `instance_map` and `color_map`.
755
+
756
+ :param dict ontology_json: The source ontology json to copy from
757
+ :return: Ontology object: The updated ontology entity
758
+ :rtype: dtlpy.entities.ontology.Ontology
759
+
760
+ **Example**:
761
+
762
+ .. code-block:: python
763
+
764
+ ontology = ontology.import_ontology(ontology_json=ontology_json)
765
+ """
766
+ # TODO: Add support for import from ontology entity in the Future
767
+ if not self._use_attributes_2:
768
+ raise ValueError("This method is only supported for attributes 2 mode!")
769
+ new_ontology = self.from_json(_json=ontology_json, client_api=self._client_api)
770
+
771
+ # Update 'labels' and 'attributes'
772
+ self.labels = new_ontology.labels
773
+ new_attributes = new_ontology.attributes
774
+ if isinstance(new_attributes, list):
775
+ for new_attribute in new_attributes:
776
+ attribute_range = new_attribute.get("range", None)
777
+ if attribute_range is not None:
778
+ attribute_range = entities.AttributesRange(
779
+ min_range=attribute_range.get("min", None),
780
+ max_range=attribute_range.get("max", None),
781
+ step=attribute_range.get("step", None)
782
+ )
783
+ script_data = new_attribute.get("scriptData", None)
784
+ if script_data is None:
785
+ new_attribute_key = new_attribute.get("key", None)
786
+ raise Exception(f"Attribute '{new_attribute_key}' scriptData is missing in the ontology json!")
787
+ self.update_attributes(
788
+ title=script_data.get("title", None),
789
+ key=new_attribute.get("key", None),
790
+ attribute_type=new_attribute.get("type", None),
791
+ scope=new_attribute.get("scope", None),
792
+ optional=script_data.get("optional", None),
793
+ values=new_attribute.get("values", None),
794
+ attribute_range=attribute_range
795
+ )
796
+
797
+ # Get remote updated 'attributes'
798
+ self.metadata["attributes"] = self.ontologies.get(ontology_id=self.id).attributes
799
+
800
+ # Update 'instance map' and 'color map'
801
+ self._instance_map = new_ontology.instance_map
802
+ self._color_map = new_ontology.color_map
803
+ return self.update(system_metadata=True)