dtlpy 1.115.44__py3-none-any.whl → 1.117.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +491 -491
- dtlpy/__version__.py +1 -1
- dtlpy/assets/__init__.py +26 -26
- dtlpy/assets/code_server/config.yaml +2 -2
- dtlpy/assets/code_server/installation.sh +24 -24
- dtlpy/assets/code_server/launch.json +13 -13
- dtlpy/assets/code_server/settings.json +2 -2
- dtlpy/assets/main.py +53 -53
- dtlpy/assets/main_partial.py +18 -18
- dtlpy/assets/mock.json +11 -11
- dtlpy/assets/model_adapter.py +83 -83
- dtlpy/assets/package.json +61 -61
- dtlpy/assets/package_catalog.json +29 -29
- dtlpy/assets/package_gitignore +307 -307
- dtlpy/assets/service_runners/__init__.py +33 -33
- dtlpy/assets/service_runners/converter.py +96 -96
- dtlpy/assets/service_runners/multi_method.py +49 -49
- dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
- dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
- dtlpy/assets/service_runners/multi_method_item.py +52 -52
- dtlpy/assets/service_runners/multi_method_json.py +52 -52
- dtlpy/assets/service_runners/single_method.py +37 -37
- dtlpy/assets/service_runners/single_method_annotation.py +43 -43
- dtlpy/assets/service_runners/single_method_dataset.py +43 -43
- dtlpy/assets/service_runners/single_method_item.py +41 -41
- dtlpy/assets/service_runners/single_method_json.py +42 -42
- dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
- dtlpy/assets/voc_annotation_template.xml +23 -23
- dtlpy/caches/base_cache.py +32 -32
- dtlpy/caches/cache.py +473 -473
- dtlpy/caches/dl_cache.py +201 -201
- dtlpy/caches/filesystem_cache.py +89 -89
- dtlpy/caches/redis_cache.py +84 -84
- dtlpy/dlp/__init__.py +20 -20
- dtlpy/dlp/cli_utilities.py +367 -367
- dtlpy/dlp/command_executor.py +764 -764
- dtlpy/dlp/dlp +1 -1
- dtlpy/dlp/dlp.bat +1 -1
- dtlpy/dlp/dlp.py +128 -128
- dtlpy/dlp/parser.py +651 -651
- dtlpy/entities/__init__.py +83 -83
- dtlpy/entities/analytic.py +347 -347
- dtlpy/entities/annotation.py +1879 -1879
- dtlpy/entities/annotation_collection.py +699 -699
- dtlpy/entities/annotation_definitions/__init__.py +20 -20
- dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
- dtlpy/entities/annotation_definitions/box.py +195 -195
- dtlpy/entities/annotation_definitions/classification.py +67 -67
- dtlpy/entities/annotation_definitions/comparison.py +72 -72
- dtlpy/entities/annotation_definitions/cube.py +204 -204
- dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
- dtlpy/entities/annotation_definitions/description.py +32 -32
- dtlpy/entities/annotation_definitions/ellipse.py +124 -124
- dtlpy/entities/annotation_definitions/free_text.py +62 -62
- dtlpy/entities/annotation_definitions/gis.py +69 -69
- dtlpy/entities/annotation_definitions/note.py +139 -139
- dtlpy/entities/annotation_definitions/point.py +117 -117
- dtlpy/entities/annotation_definitions/polygon.py +182 -182
- dtlpy/entities/annotation_definitions/polyline.py +111 -111
- dtlpy/entities/annotation_definitions/pose.py +92 -92
- dtlpy/entities/annotation_definitions/ref_image.py +86 -86
- dtlpy/entities/annotation_definitions/segmentation.py +240 -240
- dtlpy/entities/annotation_definitions/subtitle.py +34 -34
- dtlpy/entities/annotation_definitions/text.py +85 -85
- dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
- dtlpy/entities/app.py +220 -220
- dtlpy/entities/app_module.py +107 -107
- dtlpy/entities/artifact.py +174 -174
- dtlpy/entities/assignment.py +399 -399
- dtlpy/entities/base_entity.py +214 -214
- dtlpy/entities/bot.py +113 -113
- dtlpy/entities/codebase.py +292 -292
- dtlpy/entities/collection.py +38 -38
- dtlpy/entities/command.py +169 -169
- dtlpy/entities/compute.py +449 -449
- dtlpy/entities/dataset.py +1299 -1299
- dtlpy/entities/directory_tree.py +44 -44
- dtlpy/entities/dpk.py +470 -470
- dtlpy/entities/driver.py +235 -235
- dtlpy/entities/execution.py +397 -397
- dtlpy/entities/feature.py +124 -124
- dtlpy/entities/feature_set.py +152 -145
- dtlpy/entities/filters.py +798 -798
- dtlpy/entities/gis_item.py +107 -107
- dtlpy/entities/integration.py +184 -184
- dtlpy/entities/item.py +975 -959
- dtlpy/entities/label.py +123 -123
- dtlpy/entities/links.py +85 -85
- dtlpy/entities/message.py +175 -175
- dtlpy/entities/model.py +684 -684
- dtlpy/entities/node.py +1005 -1005
- dtlpy/entities/ontology.py +810 -803
- dtlpy/entities/organization.py +287 -287
- dtlpy/entities/package.py +657 -657
- dtlpy/entities/package_defaults.py +5 -5
- dtlpy/entities/package_function.py +185 -185
- dtlpy/entities/package_module.py +113 -113
- dtlpy/entities/package_slot.py +118 -118
- dtlpy/entities/paged_entities.py +299 -299
- dtlpy/entities/pipeline.py +624 -624
- dtlpy/entities/pipeline_execution.py +279 -279
- dtlpy/entities/project.py +394 -394
- dtlpy/entities/prompt_item.py +505 -505
- dtlpy/entities/recipe.py +301 -301
- dtlpy/entities/reflect_dict.py +102 -102
- dtlpy/entities/resource_execution.py +138 -138
- dtlpy/entities/service.py +974 -963
- dtlpy/entities/service_driver.py +117 -117
- dtlpy/entities/setting.py +294 -294
- dtlpy/entities/task.py +495 -495
- dtlpy/entities/time_series.py +143 -143
- dtlpy/entities/trigger.py +426 -426
- dtlpy/entities/user.py +118 -118
- dtlpy/entities/webhook.py +124 -124
- dtlpy/examples/__init__.py +19 -19
- dtlpy/examples/add_labels.py +135 -135
- dtlpy/examples/add_metadata_to_item.py +21 -21
- dtlpy/examples/annotate_items_using_model.py +65 -65
- dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
- dtlpy/examples/annotations_convert_to_voc.py +9 -9
- dtlpy/examples/annotations_convert_to_yolo.py +9 -9
- dtlpy/examples/convert_annotation_types.py +51 -51
- dtlpy/examples/converter.py +143 -143
- dtlpy/examples/copy_annotations.py +22 -22
- dtlpy/examples/copy_folder.py +31 -31
- dtlpy/examples/create_annotations.py +51 -51
- dtlpy/examples/create_video_annotations.py +83 -83
- dtlpy/examples/delete_annotations.py +26 -26
- dtlpy/examples/filters.py +113 -113
- dtlpy/examples/move_item.py +23 -23
- dtlpy/examples/play_video_annotation.py +13 -13
- dtlpy/examples/show_item_and_mask.py +53 -53
- dtlpy/examples/triggers.py +49 -49
- dtlpy/examples/upload_batch_of_items.py +20 -20
- dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
- dtlpy/examples/upload_items_with_modalities.py +43 -43
- dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
- dtlpy/examples/upload_yolo_format_annotations.py +70 -70
- dtlpy/exceptions.py +125 -125
- dtlpy/miscellaneous/__init__.py +20 -20
- dtlpy/miscellaneous/dict_differ.py +95 -95
- dtlpy/miscellaneous/git_utils.py +217 -217
- dtlpy/miscellaneous/json_utils.py +14 -14
- dtlpy/miscellaneous/list_print.py +105 -105
- dtlpy/miscellaneous/zipping.py +130 -130
- dtlpy/ml/__init__.py +20 -20
- dtlpy/ml/base_feature_extractor_adapter.py +27 -27
- dtlpy/ml/base_model_adapter.py +1287 -1230
- dtlpy/ml/metrics.py +461 -461
- dtlpy/ml/predictions_utils.py +274 -274
- dtlpy/ml/summary_writer.py +57 -57
- dtlpy/ml/train_utils.py +60 -60
- dtlpy/new_instance.py +252 -252
- dtlpy/repositories/__init__.py +56 -56
- dtlpy/repositories/analytics.py +85 -85
- dtlpy/repositories/annotations.py +916 -916
- dtlpy/repositories/apps.py +383 -383
- dtlpy/repositories/artifacts.py +452 -452
- dtlpy/repositories/assignments.py +599 -599
- dtlpy/repositories/bots.py +213 -213
- dtlpy/repositories/codebases.py +559 -559
- dtlpy/repositories/collections.py +332 -332
- dtlpy/repositories/commands.py +152 -152
- dtlpy/repositories/compositions.py +61 -61
- dtlpy/repositories/computes.py +439 -439
- dtlpy/repositories/datasets.py +1585 -1504
- dtlpy/repositories/downloader.py +1157 -923
- dtlpy/repositories/dpks.py +433 -433
- dtlpy/repositories/drivers.py +482 -482
- dtlpy/repositories/executions.py +815 -815
- dtlpy/repositories/feature_sets.py +256 -226
- dtlpy/repositories/features.py +255 -255
- dtlpy/repositories/integrations.py +484 -484
- dtlpy/repositories/items.py +912 -912
- dtlpy/repositories/messages.py +94 -94
- dtlpy/repositories/models.py +1000 -1000
- dtlpy/repositories/nodes.py +80 -80
- dtlpy/repositories/ontologies.py +511 -511
- dtlpy/repositories/organizations.py +525 -525
- dtlpy/repositories/packages.py +1941 -1941
- dtlpy/repositories/pipeline_executions.py +451 -451
- dtlpy/repositories/pipelines.py +640 -640
- dtlpy/repositories/projects.py +539 -539
- dtlpy/repositories/recipes.py +429 -399
- dtlpy/repositories/resource_executions.py +137 -137
- dtlpy/repositories/schema.py +120 -120
- dtlpy/repositories/service_drivers.py +213 -213
- dtlpy/repositories/services.py +1704 -1704
- dtlpy/repositories/settings.py +339 -339
- dtlpy/repositories/tasks.py +1477 -1477
- dtlpy/repositories/times_series.py +278 -278
- dtlpy/repositories/triggers.py +536 -536
- dtlpy/repositories/upload_element.py +257 -257
- dtlpy/repositories/uploader.py +661 -661
- dtlpy/repositories/webhooks.py +249 -249
- dtlpy/services/__init__.py +22 -22
- dtlpy/services/aihttp_retry.py +131 -131
- dtlpy/services/api_client.py +1786 -1785
- dtlpy/services/api_reference.py +40 -40
- dtlpy/services/async_utils.py +133 -133
- dtlpy/services/calls_counter.py +44 -44
- dtlpy/services/check_sdk.py +68 -68
- dtlpy/services/cookie.py +115 -115
- dtlpy/services/create_logger.py +156 -156
- dtlpy/services/events.py +84 -84
- dtlpy/services/logins.py +235 -235
- dtlpy/services/reporter.py +256 -256
- dtlpy/services/service_defaults.py +91 -91
- dtlpy/utilities/__init__.py +20 -20
- dtlpy/utilities/annotations/__init__.py +16 -16
- dtlpy/utilities/annotations/annotation_converters.py +269 -269
- dtlpy/utilities/base_package_runner.py +285 -264
- dtlpy/utilities/converter.py +1650 -1650
- dtlpy/utilities/dataset_generators/__init__.py +1 -1
- dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
- dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
- dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
- dtlpy/utilities/local_development/__init__.py +1 -1
- dtlpy/utilities/local_development/local_session.py +179 -179
- dtlpy/utilities/reports/__init__.py +2 -2
- dtlpy/utilities/reports/figures.py +343 -343
- dtlpy/utilities/reports/report.py +71 -71
- dtlpy/utilities/videos/__init__.py +17 -17
- dtlpy/utilities/videos/video_player.py +598 -598
- dtlpy/utilities/videos/videos.py +470 -470
- {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp +1 -1
- dtlpy-1.117.6.data/scripts/dlp.bat +2 -0
- {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp.py +128 -128
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/METADATA +186 -186
- dtlpy-1.117.6.dist-info/RECORD +239 -0
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/WHEEL +1 -1
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/licenses/LICENSE +200 -200
- tests/features/environment.py +551 -551
- dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
- dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
- dtlpy-1.115.44.dist-info/RECORD +0 -240
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/top_level.txt +0 -0
dtlpy/repositories/ontologies.py
CHANGED
|
@@ -1,511 +1,511 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import traceback
|
|
3
|
-
|
|
4
|
-
from .. import entities, miscellaneous, exceptions, _api_reference
|
|
5
|
-
from ..services.api_client import ApiClient
|
|
6
|
-
|
|
7
|
-
logger = logging.getLogger(name='dtlpy')
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class Ontologies:
|
|
11
|
-
"""
|
|
12
|
-
Ontologies Repository
|
|
13
|
-
|
|
14
|
-
The Ontologies class allows users to manage ontologies and their properties.
|
|
15
|
-
Read more about ontology in our `developers docs <https://developers.dataloop.ai/tutorials/recipe_and_ontology/ontology/chapter/>`_.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
def __init__(self, client_api: ApiClient,
|
|
19
|
-
recipe: entities.Recipe = None,
|
|
20
|
-
project: entities.Project = None,
|
|
21
|
-
dataset: entities.Dataset = None):
|
|
22
|
-
self._client_api = client_api
|
|
23
|
-
self._recipe = recipe
|
|
24
|
-
self._project = project
|
|
25
|
-
self._dataset = dataset
|
|
26
|
-
|
|
27
|
-
############
|
|
28
|
-
# entities #
|
|
29
|
-
############
|
|
30
|
-
@property
|
|
31
|
-
def recipe(self) -> entities.Recipe:
|
|
32
|
-
if self._recipe is None:
|
|
33
|
-
raise exceptions.PlatformException(
|
|
34
|
-
error='2001',
|
|
35
|
-
message='Missing "recipe". need to set a Recipe entity or use ontology.recipes repository')
|
|
36
|
-
assert isinstance(self._recipe, entities.Recipe)
|
|
37
|
-
return self._recipe
|
|
38
|
-
|
|
39
|
-
@recipe.setter
|
|
40
|
-
def recipe(self, recipe: entities.Recipe):
|
|
41
|
-
if not isinstance(recipe, entities.Recipe):
|
|
42
|
-
raise ValueError('Must input a valid Recipe entity')
|
|
43
|
-
self._recipe = recipe
|
|
44
|
-
|
|
45
|
-
@property
|
|
46
|
-
def project(self) -> entities.Project:
|
|
47
|
-
if self._project is None:
|
|
48
|
-
raise exceptions.PlatformException(
|
|
49
|
-
error='2001',
|
|
50
|
-
message='Missing "project". need to set a Project entity or use project.ontologies repository')
|
|
51
|
-
assert isinstance(self._project, entities.Project)
|
|
52
|
-
return self._project
|
|
53
|
-
|
|
54
|
-
@project.setter
|
|
55
|
-
def project(self, project: entities.Project):
|
|
56
|
-
if not isinstance(project, entities.Project):
|
|
57
|
-
raise ValueError('Must input a valid Project entity')
|
|
58
|
-
self._project = project
|
|
59
|
-
|
|
60
|
-
@property
|
|
61
|
-
def dataset(self) -> entities.Dataset:
|
|
62
|
-
if self._dataset is None:
|
|
63
|
-
raise exceptions.PlatformException(
|
|
64
|
-
error='2001',
|
|
65
|
-
message='Missing "dataset". need to set a Dataset entity or use dataset.ontologies repository')
|
|
66
|
-
assert isinstance(self._dataset, entities.Dataset)
|
|
67
|
-
return self._dataset
|
|
68
|
-
|
|
69
|
-
@dataset.setter
|
|
70
|
-
def dataset(self, dataset: entities.Dataset):
|
|
71
|
-
if not isinstance(dataset, entities.Dataset):
|
|
72
|
-
raise ValueError('Must input a valid Dataset entity')
|
|
73
|
-
self._dataset = dataset
|
|
74
|
-
|
|
75
|
-
def __get_project_ids(self, project_ids):
|
|
76
|
-
if project_ids is not None:
|
|
77
|
-
return project_ids if isinstance(project_ids, list) else [project_ids]
|
|
78
|
-
elif self._recipe is not None:
|
|
79
|
-
return self._recipe.project_ids
|
|
80
|
-
elif self._project is not None:
|
|
81
|
-
return [self._project.id]
|
|
82
|
-
elif self._dataset is not None:
|
|
83
|
-
return self._dataset.projects
|
|
84
|
-
else:
|
|
85
|
-
return project_ids
|
|
86
|
-
|
|
87
|
-
###########
|
|
88
|
-
# methods #
|
|
89
|
-
###########
|
|
90
|
-
@_api_reference.add(path='/ontologies', method='post')
|
|
91
|
-
def create(self,
|
|
92
|
-
labels,
|
|
93
|
-
title=None,
|
|
94
|
-
project_ids=None,
|
|
95
|
-
attributes=None
|
|
96
|
-
) -> entities.Ontology:
|
|
97
|
-
"""
|
|
98
|
-
Create a new ontology.
|
|
99
|
-
|
|
100
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
101
|
-
|
|
102
|
-
:param labels: recipe tags
|
|
103
|
-
:param str title: ontology title, name
|
|
104
|
-
:param list project_ids: recipe project/s
|
|
105
|
-
:param list attributes: recipe attributes
|
|
106
|
-
:return: Ontology object
|
|
107
|
-
:rtype: dtlpy.entities.ontology.Ontology
|
|
108
|
-
|
|
109
|
-
**Example**:
|
|
110
|
-
|
|
111
|
-
.. code-block:: python
|
|
112
|
-
|
|
113
|
-
recipe.ontologies.create(labels='labels_entity',
|
|
114
|
-
title='new_ontology',
|
|
115
|
-
project_ids='project_ids')
|
|
116
|
-
"""
|
|
117
|
-
project_ids = self.__get_project_ids(project_ids=project_ids)
|
|
118
|
-
if attributes is None:
|
|
119
|
-
attributes = list()
|
|
120
|
-
elif not isinstance(project_ids, list):
|
|
121
|
-
project_ids = [project_ids]
|
|
122
|
-
# convert to platform label format (root)
|
|
123
|
-
labels = self.labels_to_roots(labels)
|
|
124
|
-
payload = {"roots": labels,
|
|
125
|
-
"projectIds": project_ids,
|
|
126
|
-
"attributes": attributes}
|
|
127
|
-
if title is not None:
|
|
128
|
-
payload['title'] = title
|
|
129
|
-
success, response = self._client_api.gen_request(req_type="post",
|
|
130
|
-
path="/ontologies",
|
|
131
|
-
json_req=payload)
|
|
132
|
-
if success:
|
|
133
|
-
logger.info("Ontology was created successfully")
|
|
134
|
-
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
135
|
-
client_api=self._client_api,
|
|
136
|
-
recipe=self._recipe)
|
|
137
|
-
if self._recipe:
|
|
138
|
-
self._recipe.ontology_ids.append(ontology.id)
|
|
139
|
-
self._recipe.update()
|
|
140
|
-
else:
|
|
141
|
-
logger.error("Failed to create Ontology")
|
|
142
|
-
raise exceptions.PlatformException(response)
|
|
143
|
-
return ontology
|
|
144
|
-
|
|
145
|
-
def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Ontology]:
|
|
146
|
-
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
147
|
-
jobs = [None for _ in range(len(response_items))]
|
|
148
|
-
# return triggers list
|
|
149
|
-
for i_package, package in enumerate(response_items):
|
|
150
|
-
jobs[i_package] = pool.submit(entities.Ontology._protected_from_json,
|
|
151
|
-
**{'client_api': self._client_api,
|
|
152
|
-
'_json': package,
|
|
153
|
-
'project': self._project,
|
|
154
|
-
'dataset': self._dataset,
|
|
155
|
-
'recipe': self._recipe})
|
|
156
|
-
|
|
157
|
-
# get all results
|
|
158
|
-
results = [j.result() for j in jobs]
|
|
159
|
-
# log errors
|
|
160
|
-
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
161
|
-
# return good jobs
|
|
162
|
-
ontologies = miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
163
|
-
return ontologies
|
|
164
|
-
|
|
165
|
-
def _list(self, filters: entities.Filters):
|
|
166
|
-
url = '/ontologies?pageOffset={}&pageSize={}'.format(filters.page, filters.page_size)
|
|
167
|
-
project_ids = None
|
|
168
|
-
ids = None
|
|
169
|
-
for single_filter in filters.and_filter_list:
|
|
170
|
-
if single_filter.field == 'projects':
|
|
171
|
-
project_ids = single_filter.values
|
|
172
|
-
break
|
|
173
|
-
for single_filter in filters.and_filter_list:
|
|
174
|
-
if single_filter.field == 'ids':
|
|
175
|
-
ids = single_filter.values
|
|
176
|
-
break
|
|
177
|
-
|
|
178
|
-
if project_ids:
|
|
179
|
-
url = '{}&projects={}'.format(url, ','.join(project_ids))
|
|
180
|
-
if ids:
|
|
181
|
-
url = '{}&ids={}'.format(url, ','.join(ids))
|
|
182
|
-
|
|
183
|
-
# request
|
|
184
|
-
success, response = self._client_api.gen_request(req_type='get',
|
|
185
|
-
path=url)
|
|
186
|
-
if not success:
|
|
187
|
-
raise exceptions.PlatformException(response)
|
|
188
|
-
return response.json()
|
|
189
|
-
|
|
190
|
-
def __list(self, filters: entities.Filters) -> entities.PagedEntities:
|
|
191
|
-
"""
|
|
192
|
-
List project ontologies.
|
|
193
|
-
|
|
194
|
-
:param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
|
|
195
|
-
:return:
|
|
196
|
-
"""
|
|
197
|
-
paged = entities.PagedEntities(items_repository=self,
|
|
198
|
-
filters=filters,
|
|
199
|
-
page_offset=filters.page,
|
|
200
|
-
page_size=filters.page_size,
|
|
201
|
-
client_api=self._client_api)
|
|
202
|
-
paged.get_page()
|
|
203
|
-
return paged
|
|
204
|
-
|
|
205
|
-
def list(self, project_ids=None) -> miscellaneous.List[entities.Ontology]:
|
|
206
|
-
"""
|
|
207
|
-
List ontologies for recipe
|
|
208
|
-
|
|
209
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
210
|
-
|
|
211
|
-
:param project_ids:
|
|
212
|
-
:return: list of all the ontologies
|
|
213
|
-
|
|
214
|
-
**Example**:
|
|
215
|
-
|
|
216
|
-
.. code-block:: python
|
|
217
|
-
|
|
218
|
-
recipe.ontologies.list(project_ids='project_ids')
|
|
219
|
-
"""
|
|
220
|
-
if self._recipe is not None:
|
|
221
|
-
ontologies = [ontology_id for ontology_id in self.recipe.ontology_ids]
|
|
222
|
-
|
|
223
|
-
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
224
|
-
jobs = [None for _ in range(len(ontologies))]
|
|
225
|
-
for i_ontology, ontology_id in enumerate(ontologies):
|
|
226
|
-
jobs[i_ontology] = pool.submit(self._protected_get, **{'ontology_id': ontology_id})
|
|
227
|
-
|
|
228
|
-
# get all results
|
|
229
|
-
results = [j.result() for j in jobs]
|
|
230
|
-
# log errors
|
|
231
|
-
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
232
|
-
# return good jobs
|
|
233
|
-
return miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
234
|
-
else:
|
|
235
|
-
filters = entities.Filters(resource=entities.FiltersResource.ONTOLOGY)
|
|
236
|
-
project_ids = self.__get_project_ids(project_ids=project_ids)
|
|
237
|
-
if project_ids:
|
|
238
|
-
filters.add(field='projects', values=self.__get_project_ids(project_ids=project_ids))
|
|
239
|
-
if self._dataset:
|
|
240
|
-
filters.add(field='ids', values=self._dataset.ontology_ids)
|
|
241
|
-
ontologies = list()
|
|
242
|
-
pages = self.__list(filters=filters)
|
|
243
|
-
for page in pages:
|
|
244
|
-
ontologies += page
|
|
245
|
-
return miscellaneous.List(ontologies)
|
|
246
|
-
|
|
247
|
-
def _protected_get(self, ontology_id):
|
|
248
|
-
"""
|
|
249
|
-
Same as get but with try-except to catch if error
|
|
250
|
-
:param ontology_id:
|
|
251
|
-
:return:
|
|
252
|
-
"""
|
|
253
|
-
try:
|
|
254
|
-
ontology = self.get(ontology_id=ontology_id)
|
|
255
|
-
status = True
|
|
256
|
-
except Exception:
|
|
257
|
-
ontology = traceback.format_exc()
|
|
258
|
-
status = False
|
|
259
|
-
return status, ontology
|
|
260
|
-
|
|
261
|
-
@_api_reference.add(path='/ontologies/{id}', method='get')
|
|
262
|
-
def get(self, ontology_id: str) -> entities.Ontology:
|
|
263
|
-
"""
|
|
264
|
-
Get Ontology object to use in your code.
|
|
265
|
-
|
|
266
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
267
|
-
|
|
268
|
-
:param str ontology_id: ontology id
|
|
269
|
-
:return: Ontology object
|
|
270
|
-
:rtype: dtlpy.entities.ontology.Ontology
|
|
271
|
-
|
|
272
|
-
**Example**:
|
|
273
|
-
|
|
274
|
-
.. code-block:: python
|
|
275
|
-
|
|
276
|
-
recipe.ontologies.get(ontology_id='ontology_id')
|
|
277
|
-
"""
|
|
278
|
-
success, response = self._client_api.gen_request(req_type="get",
|
|
279
|
-
path="/ontologies/{}".format(ontology_id))
|
|
280
|
-
if success:
|
|
281
|
-
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
282
|
-
client_api=self._client_api,
|
|
283
|
-
recipe=self._recipe,
|
|
284
|
-
dataset=self._dataset,
|
|
285
|
-
project=self._project)
|
|
286
|
-
else:
|
|
287
|
-
raise exceptions.PlatformException(response)
|
|
288
|
-
return ontology
|
|
289
|
-
|
|
290
|
-
@_api_reference.add(path='/ontologies/{id}', method='delete')
|
|
291
|
-
def delete(self, ontology_id):
|
|
292
|
-
"""
|
|
293
|
-
Delete Ontology from the platform.
|
|
294
|
-
|
|
295
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
296
|
-
|
|
297
|
-
:param ontology_id: ontology id
|
|
298
|
-
:return: True if success
|
|
299
|
-
:rtype: bool
|
|
300
|
-
|
|
301
|
-
**Example**:
|
|
302
|
-
|
|
303
|
-
.. code-block:: python
|
|
304
|
-
|
|
305
|
-
recipe.ontologies.delete(ontology_id='ontology_id')
|
|
306
|
-
"""
|
|
307
|
-
success, response = self._client_api.gen_request(req_type="delete",
|
|
308
|
-
path="/ontologies/%s" % ontology_id)
|
|
309
|
-
if success:
|
|
310
|
-
logger.debug("Ontology was deleted successfully")
|
|
311
|
-
return success
|
|
312
|
-
else:
|
|
313
|
-
raise exceptions.PlatformException(response)
|
|
314
|
-
|
|
315
|
-
@_api_reference.add(path='/ontologies/{id}', method='put')
|
|
316
|
-
def update(self, ontology: entities.Ontology, system_metadata=False) -> entities.Ontology:
|
|
317
|
-
"""
|
|
318
|
-
Update the Ontology metadata.
|
|
319
|
-
|
|
320
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
321
|
-
|
|
322
|
-
:param dtlpy.entities.ontology.Ontology ontology: Ontology object
|
|
323
|
-
:param bool system_metadata: bool - True, if you want to change metadata system
|
|
324
|
-
:return: Ontology object
|
|
325
|
-
:rtype: dtlpy.entities.ontology.Ontology
|
|
326
|
-
|
|
327
|
-
**Example**:
|
|
328
|
-
|
|
329
|
-
.. code-block:: python
|
|
330
|
-
|
|
331
|
-
recipe.ontologies.delete(ontology='ontology_entity')
|
|
332
|
-
"""
|
|
333
|
-
url_path = "/ontologies/%s" % ontology.id
|
|
334
|
-
if system_metadata:
|
|
335
|
-
url_path += "?system=true"
|
|
336
|
-
success, response = self._client_api.gen_request(req_type="put",
|
|
337
|
-
path=url_path,
|
|
338
|
-
json_req=ontology.to_json())
|
|
339
|
-
if success:
|
|
340
|
-
logger.debug("Ontology was updated successfully")
|
|
341
|
-
# update dataset labels
|
|
342
|
-
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
343
|
-
client_api=self._client_api,
|
|
344
|
-
recipe=self._recipe)
|
|
345
|
-
if self._recipe is not None and self._recipe._dataset is not None:
|
|
346
|
-
self.recipe.dataset._labels = ontology.labels
|
|
347
|
-
return ontology
|
|
348
|
-
else:
|
|
349
|
-
logger.error("Failed to update ontology:")
|
|
350
|
-
raise exceptions.PlatformException(response)
|
|
351
|
-
|
|
352
|
-
@staticmethod
|
|
353
|
-
def labels_to_roots(labels):
|
|
354
|
-
"""
|
|
355
|
-
Converts labels dictionary to a list of platform representation of labels.
|
|
356
|
-
|
|
357
|
-
:param dict labels: labels dict
|
|
358
|
-
:return: platform representation of labels
|
|
359
|
-
"""
|
|
360
|
-
roots = list()
|
|
361
|
-
if isinstance(labels, dict):
|
|
362
|
-
for label in labels:
|
|
363
|
-
root = {
|
|
364
|
-
"value": {
|
|
365
|
-
"tag": label,
|
|
366
|
-
"color": labels[label],
|
|
367
|
-
"attributes": list(),
|
|
368
|
-
},
|
|
369
|
-
"children": list(),
|
|
370
|
-
}
|
|
371
|
-
roots.append(root)
|
|
372
|
-
elif isinstance(labels, list):
|
|
373
|
-
for label in labels:
|
|
374
|
-
if isinstance(label, entities.Label):
|
|
375
|
-
root = label.to_root()
|
|
376
|
-
elif "value" in label:
|
|
377
|
-
root = {
|
|
378
|
-
"value": label["value"],
|
|
379
|
-
"children": label.get("children", list()),
|
|
380
|
-
}
|
|
381
|
-
else:
|
|
382
|
-
root = {
|
|
383
|
-
"value": {
|
|
384
|
-
"tag": label.get("tag", None),
|
|
385
|
-
"color": label.get("color", "#FFFFFF"),
|
|
386
|
-
"attributes": label.get("attributes", list()),
|
|
387
|
-
},
|
|
388
|
-
"children": label.get("children", list()),
|
|
389
|
-
}
|
|
390
|
-
roots.append(root)
|
|
391
|
-
for root in roots:
|
|
392
|
-
if not isinstance(root["value"]["color"], str):
|
|
393
|
-
# noinspection PyStringFormat
|
|
394
|
-
root["value"]["color"] = "#%02x%02x%02x" % root["value"]["color"]
|
|
395
|
-
return roots
|
|
396
|
-
|
|
397
|
-
def update_attributes(self,
|
|
398
|
-
ontology_id: str,
|
|
399
|
-
title: str,
|
|
400
|
-
key: str,
|
|
401
|
-
attribute_type: entities.AttributesTypes,
|
|
402
|
-
scope: list = None,
|
|
403
|
-
optional: bool = None,
|
|
404
|
-
values: list = None,
|
|
405
|
-
attribute_range: entities.AttributesRange = None):
|
|
406
|
-
"""
|
|
407
|
-
ADD a new attribute or update if exist
|
|
408
|
-
|
|
409
|
-
:param str ontology_id: ontology_id
|
|
410
|
-
:param str title: attribute title
|
|
411
|
-
:param str key: the key of the attribute must br unique
|
|
412
|
-
:param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
|
|
413
|
-
:param list scope: list of the labels or * for all labels
|
|
414
|
-
:param bool optional: optional attribute
|
|
415
|
-
:param list values: list of the attribute values ( for checkbox and radio button)
|
|
416
|
-
:param dict or AttributesRange attribute_range: dl.AttributesRange object
|
|
417
|
-
:return: true in success
|
|
418
|
-
:rtype: bool
|
|
419
|
-
|
|
420
|
-
**Example**:
|
|
421
|
-
|
|
422
|
-
.. code-block:: python
|
|
423
|
-
|
|
424
|
-
ontology.update_attributes(key='1',
|
|
425
|
-
title='checkbox',
|
|
426
|
-
attribute_type=dl.AttributesTypes.CHECKBOX,
|
|
427
|
-
values=[1,2,3])
|
|
428
|
-
"""
|
|
429
|
-
if not title:
|
|
430
|
-
raise exceptions.PlatformException(400, "title must be provided")
|
|
431
|
-
url_path = '/ontologies/{ontology_id}/attributes'.format(ontology_id=ontology_id)
|
|
432
|
-
|
|
433
|
-
multi = None
|
|
434
|
-
if attribute_type == entities.AttributesTypes.CHECKBOX:
|
|
435
|
-
attribute_type = 'options'
|
|
436
|
-
multi = True
|
|
437
|
-
elif attribute_type == entities.AttributesTypes.RADIO_BUTTON:
|
|
438
|
-
attribute_type = 'options'
|
|
439
|
-
multi = False
|
|
440
|
-
|
|
441
|
-
# build attribute json
|
|
442
|
-
attribute_json = {
|
|
443
|
-
'title': title,
|
|
444
|
-
'key': key,
|
|
445
|
-
'type': attribute_type,
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
if optional is not None:
|
|
449
|
-
attribute_json['optional'] = optional
|
|
450
|
-
|
|
451
|
-
if multi is not None:
|
|
452
|
-
attribute_json['multi'] = multi
|
|
453
|
-
|
|
454
|
-
if values is not None:
|
|
455
|
-
if not isinstance(values, list):
|
|
456
|
-
values = [values]
|
|
457
|
-
for val in values:
|
|
458
|
-
if not isinstance(val, str):
|
|
459
|
-
raise exceptions.PlatformException(400, 'Attributes values type must be list of strings')
|
|
460
|
-
attribute_json['values'] = values
|
|
461
|
-
|
|
462
|
-
if attribute_range is not None:
|
|
463
|
-
attribute_json['range'] = attribute_range.to_json()
|
|
464
|
-
|
|
465
|
-
if scope is not None:
|
|
466
|
-
if not isinstance(scope, list):
|
|
467
|
-
scope = [scope]
|
|
468
|
-
else:
|
|
469
|
-
scope = ['*']
|
|
470
|
-
attribute_json['scope'] = scope
|
|
471
|
-
|
|
472
|
-
json_req = {
|
|
473
|
-
'items': [attribute_json],
|
|
474
|
-
'upsert': True
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
success, response = self._client_api.gen_request(req_type="PATCH",
|
|
478
|
-
path=url_path,
|
|
479
|
-
json_req=json_req)
|
|
480
|
-
if not success:
|
|
481
|
-
raise exceptions.PlatformException(response)
|
|
482
|
-
return True
|
|
483
|
-
|
|
484
|
-
def delete_attributes(self, ontology_id, keys: list):
|
|
485
|
-
"""
|
|
486
|
-
Delete a bulk of attributes
|
|
487
|
-
|
|
488
|
-
:param str ontology_id: ontology id
|
|
489
|
-
:param list keys: Keys of attributes to delete
|
|
490
|
-
:return: True if success
|
|
491
|
-
:rtype: bool
|
|
492
|
-
|
|
493
|
-
**Example**:
|
|
494
|
-
|
|
495
|
-
.. code-block:: python
|
|
496
|
-
|
|
497
|
-
ontology.delete_attributes(['1'])
|
|
498
|
-
"""
|
|
499
|
-
|
|
500
|
-
if not isinstance(keys, list):
|
|
501
|
-
keys = [keys]
|
|
502
|
-
url_path = '/ontologies/{ontology_id}/attributes'.format(ontology_id=ontology_id)
|
|
503
|
-
json_req = {
|
|
504
|
-
'keys': keys
|
|
505
|
-
}
|
|
506
|
-
success, response = self._client_api.gen_request(req_type="DELETE",
|
|
507
|
-
path=url_path,
|
|
508
|
-
json_req=json_req)
|
|
509
|
-
if not success:
|
|
510
|
-
raise exceptions.PlatformException(response)
|
|
511
|
-
return True
|
|
1
|
+
import logging
|
|
2
|
+
import traceback
|
|
3
|
+
|
|
4
|
+
from .. import entities, miscellaneous, exceptions, _api_reference
|
|
5
|
+
from ..services.api_client import ApiClient
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(name='dtlpy')
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Ontologies:
|
|
11
|
+
"""
|
|
12
|
+
Ontologies Repository
|
|
13
|
+
|
|
14
|
+
The Ontologies class allows users to manage ontologies and their properties.
|
|
15
|
+
Read more about ontology in our `developers docs <https://developers.dataloop.ai/tutorials/recipe_and_ontology/ontology/chapter/>`_.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, client_api: ApiClient,
|
|
19
|
+
recipe: entities.Recipe = None,
|
|
20
|
+
project: entities.Project = None,
|
|
21
|
+
dataset: entities.Dataset = None):
|
|
22
|
+
self._client_api = client_api
|
|
23
|
+
self._recipe = recipe
|
|
24
|
+
self._project = project
|
|
25
|
+
self._dataset = dataset
|
|
26
|
+
|
|
27
|
+
############
|
|
28
|
+
# entities #
|
|
29
|
+
############
|
|
30
|
+
@property
|
|
31
|
+
def recipe(self) -> entities.Recipe:
|
|
32
|
+
if self._recipe is None:
|
|
33
|
+
raise exceptions.PlatformException(
|
|
34
|
+
error='2001',
|
|
35
|
+
message='Missing "recipe". need to set a Recipe entity or use ontology.recipes repository')
|
|
36
|
+
assert isinstance(self._recipe, entities.Recipe)
|
|
37
|
+
return self._recipe
|
|
38
|
+
|
|
39
|
+
@recipe.setter
|
|
40
|
+
def recipe(self, recipe: entities.Recipe):
|
|
41
|
+
if not isinstance(recipe, entities.Recipe):
|
|
42
|
+
raise ValueError('Must input a valid Recipe entity')
|
|
43
|
+
self._recipe = recipe
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def project(self) -> entities.Project:
|
|
47
|
+
if self._project is None:
|
|
48
|
+
raise exceptions.PlatformException(
|
|
49
|
+
error='2001',
|
|
50
|
+
message='Missing "project". need to set a Project entity or use project.ontologies repository')
|
|
51
|
+
assert isinstance(self._project, entities.Project)
|
|
52
|
+
return self._project
|
|
53
|
+
|
|
54
|
+
@project.setter
|
|
55
|
+
def project(self, project: entities.Project):
|
|
56
|
+
if not isinstance(project, entities.Project):
|
|
57
|
+
raise ValueError('Must input a valid Project entity')
|
|
58
|
+
self._project = project
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def dataset(self) -> entities.Dataset:
|
|
62
|
+
if self._dataset is None:
|
|
63
|
+
raise exceptions.PlatformException(
|
|
64
|
+
error='2001',
|
|
65
|
+
message='Missing "dataset". need to set a Dataset entity or use dataset.ontologies repository')
|
|
66
|
+
assert isinstance(self._dataset, entities.Dataset)
|
|
67
|
+
return self._dataset
|
|
68
|
+
|
|
69
|
+
@dataset.setter
|
|
70
|
+
def dataset(self, dataset: entities.Dataset):
|
|
71
|
+
if not isinstance(dataset, entities.Dataset):
|
|
72
|
+
raise ValueError('Must input a valid Dataset entity')
|
|
73
|
+
self._dataset = dataset
|
|
74
|
+
|
|
75
|
+
def __get_project_ids(self, project_ids):
|
|
76
|
+
if project_ids is not None:
|
|
77
|
+
return project_ids if isinstance(project_ids, list) else [project_ids]
|
|
78
|
+
elif self._recipe is not None:
|
|
79
|
+
return self._recipe.project_ids
|
|
80
|
+
elif self._project is not None:
|
|
81
|
+
return [self._project.id]
|
|
82
|
+
elif self._dataset is not None:
|
|
83
|
+
return self._dataset.projects
|
|
84
|
+
else:
|
|
85
|
+
return project_ids
|
|
86
|
+
|
|
87
|
+
###########
|
|
88
|
+
# methods #
|
|
89
|
+
###########
|
|
90
|
+
@_api_reference.add(path='/ontologies', method='post')
|
|
91
|
+
def create(self,
|
|
92
|
+
labels,
|
|
93
|
+
title=None,
|
|
94
|
+
project_ids=None,
|
|
95
|
+
attributes=None
|
|
96
|
+
) -> entities.Ontology:
|
|
97
|
+
"""
|
|
98
|
+
Create a new ontology.
|
|
99
|
+
|
|
100
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
101
|
+
|
|
102
|
+
:param labels: recipe tags
|
|
103
|
+
:param str title: ontology title, name
|
|
104
|
+
:param list project_ids: recipe project/s
|
|
105
|
+
:param list attributes: recipe attributes
|
|
106
|
+
:return: Ontology object
|
|
107
|
+
:rtype: dtlpy.entities.ontology.Ontology
|
|
108
|
+
|
|
109
|
+
**Example**:
|
|
110
|
+
|
|
111
|
+
.. code-block:: python
|
|
112
|
+
|
|
113
|
+
recipe.ontologies.create(labels='labels_entity',
|
|
114
|
+
title='new_ontology',
|
|
115
|
+
project_ids='project_ids')
|
|
116
|
+
"""
|
|
117
|
+
project_ids = self.__get_project_ids(project_ids=project_ids)
|
|
118
|
+
if attributes is None:
|
|
119
|
+
attributes = list()
|
|
120
|
+
elif not isinstance(project_ids, list):
|
|
121
|
+
project_ids = [project_ids]
|
|
122
|
+
# convert to platform label format (root)
|
|
123
|
+
labels = self.labels_to_roots(labels)
|
|
124
|
+
payload = {"roots": labels,
|
|
125
|
+
"projectIds": project_ids,
|
|
126
|
+
"attributes": attributes}
|
|
127
|
+
if title is not None:
|
|
128
|
+
payload['title'] = title
|
|
129
|
+
success, response = self._client_api.gen_request(req_type="post",
|
|
130
|
+
path="/ontologies",
|
|
131
|
+
json_req=payload)
|
|
132
|
+
if success:
|
|
133
|
+
logger.info("Ontology was created successfully")
|
|
134
|
+
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
135
|
+
client_api=self._client_api,
|
|
136
|
+
recipe=self._recipe)
|
|
137
|
+
if self._recipe:
|
|
138
|
+
self._recipe.ontology_ids.append(ontology.id)
|
|
139
|
+
self._recipe.update()
|
|
140
|
+
else:
|
|
141
|
+
logger.error("Failed to create Ontology")
|
|
142
|
+
raise exceptions.PlatformException(response)
|
|
143
|
+
return ontology
|
|
144
|
+
|
|
145
|
+
def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Ontology]:
|
|
146
|
+
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
147
|
+
jobs = [None for _ in range(len(response_items))]
|
|
148
|
+
# return triggers list
|
|
149
|
+
for i_package, package in enumerate(response_items):
|
|
150
|
+
jobs[i_package] = pool.submit(entities.Ontology._protected_from_json,
|
|
151
|
+
**{'client_api': self._client_api,
|
|
152
|
+
'_json': package,
|
|
153
|
+
'project': self._project,
|
|
154
|
+
'dataset': self._dataset,
|
|
155
|
+
'recipe': self._recipe})
|
|
156
|
+
|
|
157
|
+
# get all results
|
|
158
|
+
results = [j.result() for j in jobs]
|
|
159
|
+
# log errors
|
|
160
|
+
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
161
|
+
# return good jobs
|
|
162
|
+
ontologies = miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
163
|
+
return ontologies
|
|
164
|
+
|
|
165
|
+
def _list(self, filters: entities.Filters):
|
|
166
|
+
url = '/ontologies?pageOffset={}&pageSize={}'.format(filters.page, filters.page_size)
|
|
167
|
+
project_ids = None
|
|
168
|
+
ids = None
|
|
169
|
+
for single_filter in filters.and_filter_list:
|
|
170
|
+
if single_filter.field == 'projects':
|
|
171
|
+
project_ids = single_filter.values
|
|
172
|
+
break
|
|
173
|
+
for single_filter in filters.and_filter_list:
|
|
174
|
+
if single_filter.field == 'ids':
|
|
175
|
+
ids = single_filter.values
|
|
176
|
+
break
|
|
177
|
+
|
|
178
|
+
if project_ids:
|
|
179
|
+
url = '{}&projects={}'.format(url, ','.join(project_ids))
|
|
180
|
+
if ids:
|
|
181
|
+
url = '{}&ids={}'.format(url, ','.join(ids))
|
|
182
|
+
|
|
183
|
+
# request
|
|
184
|
+
success, response = self._client_api.gen_request(req_type='get',
|
|
185
|
+
path=url)
|
|
186
|
+
if not success:
|
|
187
|
+
raise exceptions.PlatformException(response)
|
|
188
|
+
return response.json()
|
|
189
|
+
|
|
190
|
+
def __list(self, filters: entities.Filters) -> entities.PagedEntities:
|
|
191
|
+
"""
|
|
192
|
+
List project ontologies.
|
|
193
|
+
|
|
194
|
+
:param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
|
|
195
|
+
:return:
|
|
196
|
+
"""
|
|
197
|
+
paged = entities.PagedEntities(items_repository=self,
|
|
198
|
+
filters=filters,
|
|
199
|
+
page_offset=filters.page,
|
|
200
|
+
page_size=filters.page_size,
|
|
201
|
+
client_api=self._client_api)
|
|
202
|
+
paged.get_page()
|
|
203
|
+
return paged
|
|
204
|
+
|
|
205
|
+
def list(self, project_ids=None) -> miscellaneous.List[entities.Ontology]:
|
|
206
|
+
"""
|
|
207
|
+
List ontologies for recipe
|
|
208
|
+
|
|
209
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
210
|
+
|
|
211
|
+
:param project_ids:
|
|
212
|
+
:return: list of all the ontologies
|
|
213
|
+
|
|
214
|
+
**Example**:
|
|
215
|
+
|
|
216
|
+
.. code-block:: python
|
|
217
|
+
|
|
218
|
+
recipe.ontologies.list(project_ids='project_ids')
|
|
219
|
+
"""
|
|
220
|
+
if self._recipe is not None:
|
|
221
|
+
ontologies = [ontology_id for ontology_id in self.recipe.ontology_ids]
|
|
222
|
+
|
|
223
|
+
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
224
|
+
jobs = [None for _ in range(len(ontologies))]
|
|
225
|
+
for i_ontology, ontology_id in enumerate(ontologies):
|
|
226
|
+
jobs[i_ontology] = pool.submit(self._protected_get, **{'ontology_id': ontology_id})
|
|
227
|
+
|
|
228
|
+
# get all results
|
|
229
|
+
results = [j.result() for j in jobs]
|
|
230
|
+
# log errors
|
|
231
|
+
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
232
|
+
# return good jobs
|
|
233
|
+
return miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
234
|
+
else:
|
|
235
|
+
filters = entities.Filters(resource=entities.FiltersResource.ONTOLOGY)
|
|
236
|
+
project_ids = self.__get_project_ids(project_ids=project_ids)
|
|
237
|
+
if project_ids:
|
|
238
|
+
filters.add(field='projects', values=self.__get_project_ids(project_ids=project_ids))
|
|
239
|
+
if self._dataset:
|
|
240
|
+
filters.add(field='ids', values=self._dataset.ontology_ids)
|
|
241
|
+
ontologies = list()
|
|
242
|
+
pages = self.__list(filters=filters)
|
|
243
|
+
for page in pages:
|
|
244
|
+
ontologies += page
|
|
245
|
+
return miscellaneous.List(ontologies)
|
|
246
|
+
|
|
247
|
+
def _protected_get(self, ontology_id):
|
|
248
|
+
"""
|
|
249
|
+
Same as get but with try-except to catch if error
|
|
250
|
+
:param ontology_id:
|
|
251
|
+
:return:
|
|
252
|
+
"""
|
|
253
|
+
try:
|
|
254
|
+
ontology = self.get(ontology_id=ontology_id)
|
|
255
|
+
status = True
|
|
256
|
+
except Exception:
|
|
257
|
+
ontology = traceback.format_exc()
|
|
258
|
+
status = False
|
|
259
|
+
return status, ontology
|
|
260
|
+
|
|
261
|
+
@_api_reference.add(path='/ontologies/{id}', method='get')
|
|
262
|
+
def get(self, ontology_id: str) -> entities.Ontology:
|
|
263
|
+
"""
|
|
264
|
+
Get Ontology object to use in your code.
|
|
265
|
+
|
|
266
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
267
|
+
|
|
268
|
+
:param str ontology_id: ontology id
|
|
269
|
+
:return: Ontology object
|
|
270
|
+
:rtype: dtlpy.entities.ontology.Ontology
|
|
271
|
+
|
|
272
|
+
**Example**:
|
|
273
|
+
|
|
274
|
+
.. code-block:: python
|
|
275
|
+
|
|
276
|
+
recipe.ontologies.get(ontology_id='ontology_id')
|
|
277
|
+
"""
|
|
278
|
+
success, response = self._client_api.gen_request(req_type="get",
|
|
279
|
+
path="/ontologies/{}".format(ontology_id))
|
|
280
|
+
if success:
|
|
281
|
+
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
282
|
+
client_api=self._client_api,
|
|
283
|
+
recipe=self._recipe,
|
|
284
|
+
dataset=self._dataset,
|
|
285
|
+
project=self._project)
|
|
286
|
+
else:
|
|
287
|
+
raise exceptions.PlatformException(response)
|
|
288
|
+
return ontology
|
|
289
|
+
|
|
290
|
+
@_api_reference.add(path='/ontologies/{id}', method='delete')
|
|
291
|
+
def delete(self, ontology_id):
|
|
292
|
+
"""
|
|
293
|
+
Delete Ontology from the platform.
|
|
294
|
+
|
|
295
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
296
|
+
|
|
297
|
+
:param ontology_id: ontology id
|
|
298
|
+
:return: True if success
|
|
299
|
+
:rtype: bool
|
|
300
|
+
|
|
301
|
+
**Example**:
|
|
302
|
+
|
|
303
|
+
.. code-block:: python
|
|
304
|
+
|
|
305
|
+
recipe.ontologies.delete(ontology_id='ontology_id')
|
|
306
|
+
"""
|
|
307
|
+
success, response = self._client_api.gen_request(req_type="delete",
|
|
308
|
+
path="/ontologies/%s" % ontology_id)
|
|
309
|
+
if success:
|
|
310
|
+
logger.debug("Ontology was deleted successfully")
|
|
311
|
+
return success
|
|
312
|
+
else:
|
|
313
|
+
raise exceptions.PlatformException(response)
|
|
314
|
+
|
|
315
|
+
@_api_reference.add(path='/ontologies/{id}', method='put')
|
|
316
|
+
def update(self, ontology: entities.Ontology, system_metadata=False) -> entities.Ontology:
|
|
317
|
+
"""
|
|
318
|
+
Update the Ontology metadata.
|
|
319
|
+
|
|
320
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
321
|
+
|
|
322
|
+
:param dtlpy.entities.ontology.Ontology ontology: Ontology object
|
|
323
|
+
:param bool system_metadata: bool - True, if you want to change metadata system
|
|
324
|
+
:return: Ontology object
|
|
325
|
+
:rtype: dtlpy.entities.ontology.Ontology
|
|
326
|
+
|
|
327
|
+
**Example**:
|
|
328
|
+
|
|
329
|
+
.. code-block:: python
|
|
330
|
+
|
|
331
|
+
recipe.ontologies.delete(ontology='ontology_entity')
|
|
332
|
+
"""
|
|
333
|
+
url_path = "/ontologies/%s" % ontology.id
|
|
334
|
+
if system_metadata:
|
|
335
|
+
url_path += "?system=true"
|
|
336
|
+
success, response = self._client_api.gen_request(req_type="put",
|
|
337
|
+
path=url_path,
|
|
338
|
+
json_req=ontology.to_json())
|
|
339
|
+
if success:
|
|
340
|
+
logger.debug("Ontology was updated successfully")
|
|
341
|
+
# update dataset labels
|
|
342
|
+
ontology = entities.Ontology.from_json(_json=response.json(),
|
|
343
|
+
client_api=self._client_api,
|
|
344
|
+
recipe=self._recipe)
|
|
345
|
+
if self._recipe is not None and self._recipe._dataset is not None:
|
|
346
|
+
self.recipe.dataset._labels = ontology.labels
|
|
347
|
+
return ontology
|
|
348
|
+
else:
|
|
349
|
+
logger.error("Failed to update ontology:")
|
|
350
|
+
raise exceptions.PlatformException(response)
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
def labels_to_roots(labels):
|
|
354
|
+
"""
|
|
355
|
+
Converts labels dictionary to a list of platform representation of labels.
|
|
356
|
+
|
|
357
|
+
:param dict labels: labels dict
|
|
358
|
+
:return: platform representation of labels
|
|
359
|
+
"""
|
|
360
|
+
roots = list()
|
|
361
|
+
if isinstance(labels, dict):
|
|
362
|
+
for label in labels:
|
|
363
|
+
root = {
|
|
364
|
+
"value": {
|
|
365
|
+
"tag": label,
|
|
366
|
+
"color": labels[label],
|
|
367
|
+
"attributes": list(),
|
|
368
|
+
},
|
|
369
|
+
"children": list(),
|
|
370
|
+
}
|
|
371
|
+
roots.append(root)
|
|
372
|
+
elif isinstance(labels, list):
|
|
373
|
+
for label in labels:
|
|
374
|
+
if isinstance(label, entities.Label):
|
|
375
|
+
root = label.to_root()
|
|
376
|
+
elif "value" in label:
|
|
377
|
+
root = {
|
|
378
|
+
"value": label["value"],
|
|
379
|
+
"children": label.get("children", list()),
|
|
380
|
+
}
|
|
381
|
+
else:
|
|
382
|
+
root = {
|
|
383
|
+
"value": {
|
|
384
|
+
"tag": label.get("tag", None),
|
|
385
|
+
"color": label.get("color", "#FFFFFF"),
|
|
386
|
+
"attributes": label.get("attributes", list()),
|
|
387
|
+
},
|
|
388
|
+
"children": label.get("children", list()),
|
|
389
|
+
}
|
|
390
|
+
roots.append(root)
|
|
391
|
+
for root in roots:
|
|
392
|
+
if not isinstance(root["value"]["color"], str):
|
|
393
|
+
# noinspection PyStringFormat
|
|
394
|
+
root["value"]["color"] = "#%02x%02x%02x" % root["value"]["color"]
|
|
395
|
+
return roots
|
|
396
|
+
|
|
397
|
+
def update_attributes(self,
|
|
398
|
+
ontology_id: str,
|
|
399
|
+
title: str,
|
|
400
|
+
key: str,
|
|
401
|
+
attribute_type: entities.AttributesTypes,
|
|
402
|
+
scope: list = None,
|
|
403
|
+
optional: bool = None,
|
|
404
|
+
values: list = None,
|
|
405
|
+
attribute_range: entities.AttributesRange = None):
|
|
406
|
+
"""
|
|
407
|
+
ADD a new attribute or update if exist
|
|
408
|
+
|
|
409
|
+
:param str ontology_id: ontology_id
|
|
410
|
+
:param str title: attribute title
|
|
411
|
+
:param str key: the key of the attribute must br unique
|
|
412
|
+
:param AttributesTypes attribute_type: dl.AttributesTypes your attribute type
|
|
413
|
+
:param list scope: list of the labels or * for all labels
|
|
414
|
+
:param bool optional: optional attribute
|
|
415
|
+
:param list values: list of the attribute values ( for checkbox and radio button)
|
|
416
|
+
:param dict or AttributesRange attribute_range: dl.AttributesRange object
|
|
417
|
+
:return: true in success
|
|
418
|
+
:rtype: bool
|
|
419
|
+
|
|
420
|
+
**Example**:
|
|
421
|
+
|
|
422
|
+
.. code-block:: python
|
|
423
|
+
|
|
424
|
+
ontology.update_attributes(key='1',
|
|
425
|
+
title='checkbox',
|
|
426
|
+
attribute_type=dl.AttributesTypes.CHECKBOX,
|
|
427
|
+
values=[1,2,3])
|
|
428
|
+
"""
|
|
429
|
+
if not title:
|
|
430
|
+
raise exceptions.PlatformException(400, "title must be provided")
|
|
431
|
+
url_path = '/ontologies/{ontology_id}/attributes'.format(ontology_id=ontology_id)
|
|
432
|
+
|
|
433
|
+
multi = None
|
|
434
|
+
if attribute_type == entities.AttributesTypes.CHECKBOX:
|
|
435
|
+
attribute_type = 'options'
|
|
436
|
+
multi = True
|
|
437
|
+
elif attribute_type == entities.AttributesTypes.RADIO_BUTTON:
|
|
438
|
+
attribute_type = 'options'
|
|
439
|
+
multi = False
|
|
440
|
+
|
|
441
|
+
# build attribute json
|
|
442
|
+
attribute_json = {
|
|
443
|
+
'title': title,
|
|
444
|
+
'key': key,
|
|
445
|
+
'type': attribute_type,
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
if optional is not None:
|
|
449
|
+
attribute_json['optional'] = optional
|
|
450
|
+
|
|
451
|
+
if multi is not None:
|
|
452
|
+
attribute_json['multi'] = multi
|
|
453
|
+
|
|
454
|
+
if values is not None:
|
|
455
|
+
if not isinstance(values, list):
|
|
456
|
+
values = [values]
|
|
457
|
+
for val in values:
|
|
458
|
+
if not isinstance(val, str):
|
|
459
|
+
raise exceptions.PlatformException(400, 'Attributes values type must be list of strings')
|
|
460
|
+
attribute_json['values'] = values
|
|
461
|
+
|
|
462
|
+
if attribute_range is not None:
|
|
463
|
+
attribute_json['range'] = attribute_range.to_json()
|
|
464
|
+
|
|
465
|
+
if scope is not None:
|
|
466
|
+
if not isinstance(scope, list):
|
|
467
|
+
scope = [scope]
|
|
468
|
+
else:
|
|
469
|
+
scope = ['*']
|
|
470
|
+
attribute_json['scope'] = scope
|
|
471
|
+
|
|
472
|
+
json_req = {
|
|
473
|
+
'items': [attribute_json],
|
|
474
|
+
'upsert': True
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
success, response = self._client_api.gen_request(req_type="PATCH",
|
|
478
|
+
path=url_path,
|
|
479
|
+
json_req=json_req)
|
|
480
|
+
if not success:
|
|
481
|
+
raise exceptions.PlatformException(response)
|
|
482
|
+
return True
|
|
483
|
+
|
|
484
|
+
def delete_attributes(self, ontology_id, keys: list):
|
|
485
|
+
"""
|
|
486
|
+
Delete a bulk of attributes
|
|
487
|
+
|
|
488
|
+
:param str ontology_id: ontology id
|
|
489
|
+
:param list keys: Keys of attributes to delete
|
|
490
|
+
:return: True if success
|
|
491
|
+
:rtype: bool
|
|
492
|
+
|
|
493
|
+
**Example**:
|
|
494
|
+
|
|
495
|
+
.. code-block:: python
|
|
496
|
+
|
|
497
|
+
ontology.delete_attributes(['1'])
|
|
498
|
+
"""
|
|
499
|
+
|
|
500
|
+
if not isinstance(keys, list):
|
|
501
|
+
keys = [keys]
|
|
502
|
+
url_path = '/ontologies/{ontology_id}/attributes'.format(ontology_id=ontology_id)
|
|
503
|
+
json_req = {
|
|
504
|
+
'keys': keys
|
|
505
|
+
}
|
|
506
|
+
success, response = self._client_api.gen_request(req_type="DELETE",
|
|
507
|
+
path=url_path,
|
|
508
|
+
json_req=json_req)
|
|
509
|
+
if not success:
|
|
510
|
+
raise exceptions.PlatformException(response)
|
|
511
|
+
return True
|