dtlpy 1.115.44__py3-none-any.whl → 1.117.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +152 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +975 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +974 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1287 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1585 -1504
  167. dtlpy/repositories/downloader.py +1157 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +256 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +429 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1786 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.117.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.117.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.117.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.117.6.dist-info}/top_level.txt +0 -0
@@ -1,399 +1,429 @@
1
- import logging
2
- import traceback
3
- import urllib.parse
4
-
5
- from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
- from ..services.api_client import ApiClient
7
-
8
- logger = logging.getLogger(name='dtlpy')
9
-
10
-
11
- class Recipes:
12
- """
13
- Recipes Repository
14
-
15
- The Recipes class allows you to manage recipes and their properties.
16
- For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
- """
18
-
19
- def __init__(self,
20
- client_api: ApiClient,
21
- dataset: entities.Dataset = None,
22
- project: entities.Project = None,
23
- project_id: str = None):
24
- self._client_api = client_api
25
- self._dataset = dataset
26
- self._project = project
27
- self._project_id = project_id
28
- if project_id is None and project is not None:
29
- self._project_id = project.id
30
-
31
- ############
32
- # entities #
33
- ############
34
- @property
35
- def platform_url(self):
36
- if self._project_id is None:
37
- project_id = self.dataset.project.id
38
- else:
39
- project_id = self._project_id
40
- return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
-
42
- @property
43
- def dataset(self) -> entities.Dataset:
44
- if self._dataset is None:
45
- raise exceptions.PlatformException(
46
- error='2001',
47
- message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
48
- assert isinstance(self._dataset, entities.Dataset)
49
- return self._dataset
50
-
51
- @dataset.setter
52
- def dataset(self, dataset: entities.Dataset):
53
- if not isinstance(dataset, entities.Dataset):
54
- raise ValueError('Must input a valid Dataset entity')
55
- self._dataset = dataset
56
-
57
- ###########
58
- # methods #
59
- ###########
60
- @_api_reference.add(path='/recipes', method='post')
61
- def create(self,
62
- project_ids=None,
63
- ontology_ids=None,
64
- labels=None,
65
- recipe_name=None,
66
- attributes=None,
67
- annotation_instruction_file=None
68
- ) -> entities.Recipe:
69
- """
70
- Create a new Recipe.
71
- Note: If the param ontology_ids is None, an ontology will be created first.
72
-
73
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
74
-
75
- :param str project_ids: project ids
76
- :param str or list ontology_ids: ontology ids
77
- :param labels: labels
78
- :param str recipe_name: recipe name
79
- :param attributes: attributes
80
- :param str annotation_instruction_file: file path or url of the recipe instruction
81
- :return: Recipe entity
82
- :rtype: dtlpy.entities.recipe.Recipe
83
-
84
- **Example**:
85
-
86
- .. code-block:: python
87
-
88
- dataset.recipes.create(recipe_name='My Recipe', labels=labels))
89
- """
90
- if labels is None:
91
- labels = list()
92
- if attributes is None:
93
- attributes = list()
94
- if project_ids is None:
95
- if self._dataset is not None:
96
- project_ids = [self._dataset.project.id]
97
- else:
98
- # get from cache
99
- project = self._client_api.state_io.get('project')
100
- if project is not None:
101
- # build entity from json
102
- p = entities.Project.from_json(_json=project, client_api=self._client_api)
103
- project_ids = [p.id]
104
- else:
105
- raise exceptions.PlatformException('Must provide project_ids')
106
- if ontology_ids is None:
107
- ontolgies = repositories.Ontologies(client_api=self._client_api,
108
- recipe=None)
109
- ontology = ontolgies.create(labels=labels,
110
- project_ids=project_ids,
111
- attributes=attributes)
112
- ontology_ids = [ontology.id]
113
- elif not isinstance(ontology_ids, list):
114
- ontology_ids = [ontology_ids]
115
- if recipe_name is None:
116
- recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
117
- payload = {'title': recipe_name,
118
- 'projectIds': project_ids,
119
- 'ontologyIds': ontology_ids,
120
- 'uiSettings': {
121
- "allowObjectIdAutoAssign": True,
122
- "studioV2App": True
123
- }}
124
- success, response = self._client_api.gen_request(req_type='post',
125
- path='/recipes',
126
- json_req=payload)
127
- if success:
128
- recipe = entities.Recipe.from_json(client_api=self._client_api,
129
- _json=response.json(),
130
- dataset=self._dataset)
131
- if annotation_instruction_file:
132
- recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
133
- else:
134
- logger.error('Failed to create Recipe')
135
- raise exceptions.PlatformException(response)
136
-
137
- if self._dataset is not None:
138
- self._dataset.switch_recipe(recipe.id)
139
- return recipe
140
-
141
- def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
142
- """
143
- List recipes for a dataset.
144
-
145
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
146
-
147
- :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
148
- :return: list of all recipes
149
- :retype: list
150
-
151
- **Example**:
152
-
153
- .. code-block:: python
154
-
155
- dataset.recipes.list()
156
- """
157
- if self._dataset is not None:
158
- try:
159
- recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
160
- except KeyError:
161
- recipes = list()
162
-
163
- pool = self._client_api.thread_pools(pool_name='entity.create')
164
- jobs = [None for _ in range(len(recipes))]
165
- for i_recipe, recipe_id in enumerate(recipes):
166
- jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
167
-
168
- # get all results
169
- results = [j.result() for j in jobs]
170
- # log errors
171
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
172
- # return good jobs
173
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
174
- elif self._project_id is not None:
175
- if filters is None:
176
- filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
177
- # assert type filters
178
- elif not isinstance(filters, entities.Filters):
179
- raise exceptions.PlatformException(error='400',
180
- message='Unknown filters type: {!r}'.format(type(filters)))
181
- if filters.resource != entities.FiltersResource.RECIPE:
182
- raise exceptions.PlatformException(
183
- error='400',
184
- message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
185
- if not filters.has_field('projects'):
186
- filters.add(field='projects', values=[self._project_id])
187
-
188
- recipes = entities.PagedEntities(items_repository=self,
189
- filters=filters,
190
- page_offset=filters.page,
191
- page_size=filters.page_size,
192
- project_id=self._project_id,
193
- client_api=self._client_api)
194
- recipes.get_page()
195
- else:
196
- raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
197
-
198
- return recipes
199
-
200
- def _list(self, filters: entities.Filters):
201
- url = filters.generate_url_query_params('/recipes')
202
- encoded_url = urllib.parse.quote(url, safe='/:?=&')
203
- # request
204
- success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
205
- if not success:
206
- raise exceptions.PlatformException(response)
207
- return response.json()
208
-
209
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
210
- pool = self._client_api.thread_pools(pool_name='entity.create')
211
- jobs = [None for _ in range(len(response_items))]
212
- # return triggers list
213
- for i_rec, rec in enumerate(response_items):
214
- jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
215
- **{'client_api': self._client_api,
216
- '_json': rec,
217
- 'project': self._project,
218
- 'dataset': self._dataset})
219
-
220
- # get all results
221
- results = [j.result() for j in jobs]
222
- # log errors
223
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
224
- # return good jobs
225
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
226
- return recipes
227
-
228
- def _protected_get(self, recipe_id):
229
- """
230
- Same as get but with try-except to catch if error
231
- :param recipe_id:
232
- :return:
233
- """
234
- try:
235
- recipe = self.get(recipe_id=recipe_id)
236
- status = True
237
- except Exception:
238
- recipe = traceback.format_exc()
239
- status = False
240
- return status, recipe
241
-
242
- @_api_reference.add(path='/recipes/{id}', method='get')
243
- def get(self, recipe_id: str) -> entities.Recipe:
244
- """
245
- Get a Recipe object to use in your code.
246
-
247
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
248
-
249
- :param str recipe_id: recipe id
250
- :return: Recipe object
251
- :rtype: dtlpy.entities.recipe.Recipe
252
-
253
- **Example**:
254
-
255
- .. code-block:: python
256
-
257
- dataset.recipes.get(recipe_id='recipe_id')
258
- """
259
- success, response = self._client_api.gen_request(req_type='get',
260
- path='/recipes/%s' % recipe_id)
261
- if success:
262
- recipe = entities.Recipe.from_json(client_api=self._client_api,
263
- _json=response.json(),
264
- project=self._project,
265
- dataset=self._dataset)
266
- else:
267
- logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
268
- raise exceptions.PlatformException(response)
269
-
270
- return recipe
271
-
272
- def open_in_web(self,
273
- recipe: entities.Recipe = None,
274
- recipe_id: str = None):
275
- """
276
- Open the recipe in web platform.
277
-
278
- **Prerequisites**: All users.
279
-
280
- :param dtlpy.entities.recipe.Recipe recipe: recipe entity
281
- :param str recipe_id: recipe id
282
-
283
- **Example**:
284
-
285
- .. code-block:: python
286
-
287
- dataset.recipes.open_in_web(recipe_id='recipe_id')
288
- """
289
- if recipe is not None:
290
- recipe.open_in_web()
291
- elif recipe_id is not None:
292
- self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
293
- else:
294
- self._client_api._open_in_web(url=self.platform_url)
295
-
296
- @_api_reference.add(path='/recipes/{id}', method='delete')
297
- def delete(self, recipe_id: str, force: bool = False):
298
- """
299
- Delete recipe from platform.
300
-
301
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
302
-
303
- :param str recipe_id: recipe id
304
- :param bool force: force delete recipe
305
- :return: True if success
306
- :rtype: bool
307
-
308
- **Example**:
309
-
310
- .. code-block:: python
311
-
312
- dataset.recipes.delete(recipe_id='recipe_id')
313
- """
314
- path = '/recipes/{}'.format(recipe_id)
315
- if force:
316
- path += '?force=true'
317
- success, response = self._client_api.gen_request(req_type='delete',
318
- path=path)
319
- if not success:
320
- raise exceptions.PlatformException(response)
321
- logger.info('Recipe id {} deleted successfully'.format(recipe_id))
322
- return True
323
-
324
- @_api_reference.add(path='/recipes/{id}', method='patch')
325
- def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
326
- """
327
- Update recipe.
328
-
329
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
330
-
331
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
332
- :param bool system_metadata: True, if you want to change metadata system
333
- :return: Recipe object
334
- :rtype: dtlpy.entities.recipe.Recipe
335
-
336
- **Example**:
337
-
338
- .. code-block:: python
339
-
340
- dataset.recipes.update(recipe='recipe_entity')
341
- """
342
- url_path = '/recipes/%s' % recipe.id
343
- if system_metadata:
344
- url_path += '?system=true'
345
- success, response = self._client_api.gen_request(req_type='patch',
346
- path=url_path,
347
- json_req=recipe.to_json())
348
- if success:
349
- return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
350
- else:
351
- logger.error('Error while updating item:')
352
- raise exceptions.PlatformException(response)
353
-
354
- @_api_reference.add(path='/recipes/{id}/clone', method='post')
355
- def clone(self,
356
- recipe: entities.Recipe = None,
357
- recipe_id: str = None,
358
- shallow: bool = False):
359
- """
360
- Clone recipe.
361
-
362
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
363
-
364
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
365
- :param str recipe_id: Recipe id
366
- :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
367
- :return: Cloned ontology object
368
- :rtype: dtlpy.entities.recipe.Recipe
369
-
370
- **Example**:
371
-
372
- .. code-block:: python
373
-
374
- dataset.recipes.clone(recipe_id='recipe_id')
375
- """
376
- if recipe is None and recipe_id is None:
377
- raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
378
- if recipe_id is None:
379
- if not isinstance(recipe, entities.Recipe):
380
- raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
381
- else:
382
- recipe_id = recipe.id
383
-
384
- payload = {'shallow': shallow}
385
-
386
- success, response = self._client_api.gen_request(req_type='post',
387
- path='/recipes/{}/clone'.format(recipe_id),
388
- json_req=payload)
389
- if success:
390
- recipe = entities.Recipe.from_json(client_api=self._client_api,
391
- _json=response.json())
392
- else:
393
- logger.error('Failed to clone Recipe')
394
- raise exceptions.PlatformException(response)
395
-
396
- assert isinstance(recipe, entities.Recipe)
397
- logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
398
-
399
- return recipe
1
+ import logging
2
+ import traceback
3
+ import urllib.parse
4
+
5
+ from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
+ from ..services.api_client import ApiClient
7
+
8
+ logger = logging.getLogger(name='dtlpy')
9
+
10
+
11
+ class Recipes:
12
+ """
13
+ Recipes Repository
14
+
15
+ The Recipes class allows you to manage recipes and their properties.
16
+ For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
+ """
18
+
19
+ def __init__(self,
20
+ client_api: ApiClient,
21
+ dataset: entities.Dataset = None,
22
+ project: entities.Project = None,
23
+ project_id: str = None):
24
+ self._client_api = client_api
25
+ self._dataset = dataset
26
+ self._project = project
27
+ self._project_id = project_id
28
+ if project_id is None and project is not None:
29
+ self._project_id = project.id
30
+
31
+ ############
32
+ # entities #
33
+ ############
34
+ @property
35
+ def platform_url(self):
36
+ if self._project_id is None:
37
+ project_id = self.dataset.project.id
38
+ else:
39
+ project_id = self._project_id
40
+ return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
+
42
+ @property
43
+ def project(self) -> entities.Project:
44
+ if self._project is None:
45
+ if self._project_id is None:
46
+ if self._dataset is None:
47
+ raise exceptions.PlatformException(
48
+ error='2001',
49
+ message='Missing "Project". need to set a Project entity or use project.recipes repository'
50
+ )
51
+ else:
52
+ self._project = self._dataset.project
53
+ self._project_id = self._project.id
54
+ else:
55
+ self._project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
56
+ self._project_id = self._project.id
57
+
58
+ assert isinstance(self._project, entities.Project)
59
+ return self._project
60
+
61
+ @project.setter
62
+ def project(self, project: entities.Project):
63
+ if not isinstance(project, entities.Project):
64
+ raise ValueError('Must input a valid Project entity')
65
+ self._project = project
66
+ self._project_id = project.id
67
+
68
+ @property
69
+ def dataset(self) -> entities.Dataset:
70
+ if self._dataset is None:
71
+ raise exceptions.PlatformException(
72
+ error='2001',
73
+ message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
74
+ assert isinstance(self._dataset, entities.Dataset)
75
+ return self._dataset
76
+
77
+ @dataset.setter
78
+ def dataset(self, dataset: entities.Dataset):
79
+ if not isinstance(dataset, entities.Dataset):
80
+ raise ValueError('Must input a valid Dataset entity')
81
+ self._dataset = dataset
82
+
83
+ ###########
84
+ # methods #
85
+ ###########
86
+ @_api_reference.add(path='/recipes', method='post')
87
+ def create(self,
88
+ project_ids=None,
89
+ ontology_ids=None,
90
+ labels=None,
91
+ recipe_name=None,
92
+ attributes=None,
93
+ annotation_instruction_file=None
94
+ ) -> entities.Recipe:
95
+ """
96
+ Create a new Recipe.
97
+ Note: If the param ontology_ids is None, an ontology will be created first.
98
+
99
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
100
+
101
+ :param str project_ids: project ids
102
+ :param str or list ontology_ids: ontology ids
103
+ :param labels: labels
104
+ :param str recipe_name: recipe name
105
+ :param attributes: attributes
106
+ :param str annotation_instruction_file: file path or url of the recipe instruction
107
+ :return: Recipe entity
108
+ :rtype: dtlpy.entities.recipe.Recipe
109
+
110
+ **Example**:
111
+
112
+ .. code-block:: python
113
+
114
+ dataset.recipes.create(recipe_name='My Recipe', labels=labels))
115
+ """
116
+ if labels is None:
117
+ labels = list()
118
+ if attributes is None:
119
+ attributes = list()
120
+ if project_ids is None:
121
+ if self._dataset is not None:
122
+ project_ids = [self._dataset.project.id]
123
+ else:
124
+ # get from cache
125
+ project = self._client_api.state_io.get('project')
126
+ if project is not None:
127
+ # build entity from json
128
+ p = entities.Project.from_json(_json=project, client_api=self._client_api)
129
+ project_ids = [p.id]
130
+ else:
131
+ # get from self.project property
132
+ try:
133
+ project_ids = [self.project.id]
134
+ except exceptions.PlatformException:
135
+ raise exceptions.PlatformException('Must provide project_ids')
136
+ if ontology_ids is None:
137
+ ontolgies = repositories.Ontologies(client_api=self._client_api,
138
+ recipe=None)
139
+ ontology = ontolgies.create(labels=labels,
140
+ project_ids=project_ids,
141
+ attributes=attributes)
142
+ ontology_ids = [ontology.id]
143
+ elif not isinstance(ontology_ids, list):
144
+ ontology_ids = [ontology_ids]
145
+ if recipe_name is None:
146
+ recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
147
+ payload = {'title': recipe_name,
148
+ 'projectIds': project_ids,
149
+ 'ontologyIds': ontology_ids,
150
+ 'uiSettings': {
151
+ "allowObjectIdAutoAssign": True,
152
+ "studioV2App": True
153
+ }}
154
+ success, response = self._client_api.gen_request(req_type='post',
155
+ path='/recipes',
156
+ json_req=payload)
157
+ if success:
158
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
159
+ _json=response.json(),
160
+ dataset=self._dataset)
161
+ if annotation_instruction_file:
162
+ recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
163
+ else:
164
+ logger.error('Failed to create Recipe')
165
+ raise exceptions.PlatformException(response)
166
+
167
+ if self._dataset is not None:
168
+ self._dataset.switch_recipe(recipe.id)
169
+ return recipe
170
+
171
+ def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
172
+ """
173
+ List recipes for a dataset.
174
+
175
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
176
+
177
+ :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
178
+ :return: list of all recipes
179
+ :retype: list
180
+
181
+ **Example**:
182
+
183
+ .. code-block:: python
184
+
185
+ dataset.recipes.list()
186
+ """
187
+ if self._dataset is not None:
188
+ try:
189
+ recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
190
+ except KeyError:
191
+ recipes = list()
192
+
193
+ pool = self._client_api.thread_pools(pool_name='entity.create')
194
+ jobs = [None for _ in range(len(recipes))]
195
+ for i_recipe, recipe_id in enumerate(recipes):
196
+ jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
197
+
198
+ # get all results
199
+ results = [j.result() for j in jobs]
200
+ # log errors
201
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
202
+ # return good jobs
203
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
204
+ elif self._project_id is not None:
205
+ if filters is None:
206
+ filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
207
+ # assert type filters
208
+ elif not isinstance(filters, entities.Filters):
209
+ raise exceptions.PlatformException(error='400',
210
+ message='Unknown filters type: {!r}'.format(type(filters)))
211
+ if filters.resource != entities.FiltersResource.RECIPE:
212
+ raise exceptions.PlatformException(
213
+ error='400',
214
+ message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
215
+ if not filters.has_field('projects'):
216
+ filters.add(field='projects', values=[self._project_id])
217
+
218
+ recipes = entities.PagedEntities(items_repository=self,
219
+ filters=filters,
220
+ page_offset=filters.page,
221
+ page_size=filters.page_size,
222
+ project_id=self._project_id,
223
+ client_api=self._client_api)
224
+ recipes.get_page()
225
+ else:
226
+ raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
227
+
228
+ return recipes
229
+
230
+ def _list(self, filters: entities.Filters):
231
+ url = filters.generate_url_query_params('/recipes')
232
+ encoded_url = urllib.parse.quote(url, safe='/:?=&')
233
+ # request
234
+ success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
235
+ if not success:
236
+ raise exceptions.PlatformException(response)
237
+ return response.json()
238
+
239
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
240
+ pool = self._client_api.thread_pools(pool_name='entity.create')
241
+ jobs = [None for _ in range(len(response_items))]
242
+ # return triggers list
243
+ for i_rec, rec in enumerate(response_items):
244
+ jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
245
+ **{'client_api': self._client_api,
246
+ '_json': rec,
247
+ 'project': self._project,
248
+ 'dataset': self._dataset})
249
+
250
+ # get all results
251
+ results = [j.result() for j in jobs]
252
+ # log errors
253
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
254
+ # return good jobs
255
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
256
+ return recipes
257
+
258
+ def _protected_get(self, recipe_id):
259
+ """
260
+ Same as get but with try-except to catch if error
261
+ :param recipe_id:
262
+ :return:
263
+ """
264
+ try:
265
+ recipe = self.get(recipe_id=recipe_id)
266
+ status = True
267
+ except Exception:
268
+ recipe = traceback.format_exc()
269
+ status = False
270
+ return status, recipe
271
+
272
+ @_api_reference.add(path='/recipes/{id}', method='get')
273
+ def get(self, recipe_id: str) -> entities.Recipe:
274
+ """
275
+ Get a Recipe object to use in your code.
276
+
277
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
278
+
279
+ :param str recipe_id: recipe id
280
+ :return: Recipe object
281
+ :rtype: dtlpy.entities.recipe.Recipe
282
+
283
+ **Example**:
284
+
285
+ .. code-block:: python
286
+
287
+ dataset.recipes.get(recipe_id='recipe_id')
288
+ """
289
+ success, response = self._client_api.gen_request(req_type='get',
290
+ path='/recipes/%s' % recipe_id)
291
+ if success:
292
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
293
+ _json=response.json(),
294
+ project=self._project,
295
+ dataset=self._dataset)
296
+ else:
297
+ logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
298
+ raise exceptions.PlatformException(response)
299
+
300
+ return recipe
301
+
302
+ def open_in_web(self,
303
+ recipe: entities.Recipe = None,
304
+ recipe_id: str = None):
305
+ """
306
+ Open the recipe in web platform.
307
+
308
+ **Prerequisites**: All users.
309
+
310
+ :param dtlpy.entities.recipe.Recipe recipe: recipe entity
311
+ :param str recipe_id: recipe id
312
+
313
+ **Example**:
314
+
315
+ .. code-block:: python
316
+
317
+ dataset.recipes.open_in_web(recipe_id='recipe_id')
318
+ """
319
+ if recipe is not None:
320
+ recipe.open_in_web()
321
+ elif recipe_id is not None:
322
+ self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
323
+ else:
324
+ self._client_api._open_in_web(url=self.platform_url)
325
+
326
+ @_api_reference.add(path='/recipes/{id}', method='delete')
327
+ def delete(self, recipe_id: str, force: bool = False):
328
+ """
329
+ Delete recipe from platform.
330
+
331
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
332
+
333
+ :param str recipe_id: recipe id
334
+ :param bool force: force delete recipe
335
+ :return: True if success
336
+ :rtype: bool
337
+
338
+ **Example**:
339
+
340
+ .. code-block:: python
341
+
342
+ dataset.recipes.delete(recipe_id='recipe_id')
343
+ """
344
+ path = '/recipes/{}'.format(recipe_id)
345
+ if force:
346
+ path += '?force=true'
347
+ success, response = self._client_api.gen_request(req_type='delete',
348
+ path=path)
349
+ if not success:
350
+ raise exceptions.PlatformException(response)
351
+ logger.info('Recipe id {} deleted successfully'.format(recipe_id))
352
+ return True
353
+
354
+ @_api_reference.add(path='/recipes/{id}', method='patch')
355
+ def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
356
+ """
357
+ Update recipe.
358
+
359
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
360
+
361
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
362
+ :param bool system_metadata: True, if you want to change metadata system
363
+ :return: Recipe object
364
+ :rtype: dtlpy.entities.recipe.Recipe
365
+
366
+ **Example**:
367
+
368
+ .. code-block:: python
369
+
370
+ dataset.recipes.update(recipe='recipe_entity')
371
+ """
372
+ url_path = '/recipes/%s' % recipe.id
373
+ if system_metadata:
374
+ url_path += '?system=true'
375
+ success, response = self._client_api.gen_request(req_type='patch',
376
+ path=url_path,
377
+ json_req=recipe.to_json())
378
+ if success:
379
+ return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
380
+ else:
381
+ logger.error('Error while updating item:')
382
+ raise exceptions.PlatformException(response)
383
+
384
+ @_api_reference.add(path='/recipes/{id}/clone', method='post')
385
+ def clone(self,
386
+ recipe: entities.Recipe = None,
387
+ recipe_id: str = None,
388
+ shallow: bool = False):
389
+ """
390
+ Clone recipe.
391
+
392
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
393
+
394
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
395
+ :param str recipe_id: Recipe id
396
+ :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
397
+ :return: Cloned ontology object
398
+ :rtype: dtlpy.entities.recipe.Recipe
399
+
400
+ **Example**:
401
+
402
+ .. code-block:: python
403
+
404
+ dataset.recipes.clone(recipe_id='recipe_id')
405
+ """
406
+ if recipe is None and recipe_id is None:
407
+ raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
408
+ if recipe_id is None:
409
+ if not isinstance(recipe, entities.Recipe):
410
+ raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
411
+ else:
412
+ recipe_id = recipe.id
413
+
414
+ payload = {'shallow': shallow}
415
+
416
+ success, response = self._client_api.gen_request(req_type='post',
417
+ path='/recipes/{}/clone'.format(recipe_id),
418
+ json_req=payload)
419
+ if success:
420
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
421
+ _json=response.json())
422
+ else:
423
+ logger.error('Failed to clone Recipe')
424
+ raise exceptions.PlatformException(response)
425
+
426
+ assert isinstance(recipe, entities.Recipe)
427
+ logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
428
+
429
+ return recipe