dtlpy 1.115.44__py3-none-any.whl → 1.116.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +145 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +959 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +963 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1257 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1504 -1504
  167. dtlpy/repositories/downloader.py +976 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +226 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +419 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1785 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.116.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
@@ -1,399 +1,419 @@
1
- import logging
2
- import traceback
3
- import urllib.parse
4
-
5
- from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
- from ..services.api_client import ApiClient
7
-
8
- logger = logging.getLogger(name='dtlpy')
9
-
10
-
11
- class Recipes:
12
- """
13
- Recipes Repository
14
-
15
- The Recipes class allows you to manage recipes and their properties.
16
- For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
- """
18
-
19
- def __init__(self,
20
- client_api: ApiClient,
21
- dataset: entities.Dataset = None,
22
- project: entities.Project = None,
23
- project_id: str = None):
24
- self._client_api = client_api
25
- self._dataset = dataset
26
- self._project = project
27
- self._project_id = project_id
28
- if project_id is None and project is not None:
29
- self._project_id = project.id
30
-
31
- ############
32
- # entities #
33
- ############
34
- @property
35
- def platform_url(self):
36
- if self._project_id is None:
37
- project_id = self.dataset.project.id
38
- else:
39
- project_id = self._project_id
40
- return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
-
42
- @property
43
- def dataset(self) -> entities.Dataset:
44
- if self._dataset is None:
45
- raise exceptions.PlatformException(
46
- error='2001',
47
- message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
48
- assert isinstance(self._dataset, entities.Dataset)
49
- return self._dataset
50
-
51
- @dataset.setter
52
- def dataset(self, dataset: entities.Dataset):
53
- if not isinstance(dataset, entities.Dataset):
54
- raise ValueError('Must input a valid Dataset entity')
55
- self._dataset = dataset
56
-
57
- ###########
58
- # methods #
59
- ###########
60
- @_api_reference.add(path='/recipes', method='post')
61
- def create(self,
62
- project_ids=None,
63
- ontology_ids=None,
64
- labels=None,
65
- recipe_name=None,
66
- attributes=None,
67
- annotation_instruction_file=None
68
- ) -> entities.Recipe:
69
- """
70
- Create a new Recipe.
71
- Note: If the param ontology_ids is None, an ontology will be created first.
72
-
73
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
74
-
75
- :param str project_ids: project ids
76
- :param str or list ontology_ids: ontology ids
77
- :param labels: labels
78
- :param str recipe_name: recipe name
79
- :param attributes: attributes
80
- :param str annotation_instruction_file: file path or url of the recipe instruction
81
- :return: Recipe entity
82
- :rtype: dtlpy.entities.recipe.Recipe
83
-
84
- **Example**:
85
-
86
- .. code-block:: python
87
-
88
- dataset.recipes.create(recipe_name='My Recipe', labels=labels))
89
- """
90
- if labels is None:
91
- labels = list()
92
- if attributes is None:
93
- attributes = list()
94
- if project_ids is None:
95
- if self._dataset is not None:
96
- project_ids = [self._dataset.project.id]
97
- else:
98
- # get from cache
99
- project = self._client_api.state_io.get('project')
100
- if project is not None:
101
- # build entity from json
102
- p = entities.Project.from_json(_json=project, client_api=self._client_api)
103
- project_ids = [p.id]
104
- else:
105
- raise exceptions.PlatformException('Must provide project_ids')
106
- if ontology_ids is None:
107
- ontolgies = repositories.Ontologies(client_api=self._client_api,
108
- recipe=None)
109
- ontology = ontolgies.create(labels=labels,
110
- project_ids=project_ids,
111
- attributes=attributes)
112
- ontology_ids = [ontology.id]
113
- elif not isinstance(ontology_ids, list):
114
- ontology_ids = [ontology_ids]
115
- if recipe_name is None:
116
- recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
117
- payload = {'title': recipe_name,
118
- 'projectIds': project_ids,
119
- 'ontologyIds': ontology_ids,
120
- 'uiSettings': {
121
- "allowObjectIdAutoAssign": True,
122
- "studioV2App": True
123
- }}
124
- success, response = self._client_api.gen_request(req_type='post',
125
- path='/recipes',
126
- json_req=payload)
127
- if success:
128
- recipe = entities.Recipe.from_json(client_api=self._client_api,
129
- _json=response.json(),
130
- dataset=self._dataset)
131
- if annotation_instruction_file:
132
- recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
133
- else:
134
- logger.error('Failed to create Recipe')
135
- raise exceptions.PlatformException(response)
136
-
137
- if self._dataset is not None:
138
- self._dataset.switch_recipe(recipe.id)
139
- return recipe
140
-
141
- def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
142
- """
143
- List recipes for a dataset.
144
-
145
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
146
-
147
- :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
148
- :return: list of all recipes
149
- :retype: list
150
-
151
- **Example**:
152
-
153
- .. code-block:: python
154
-
155
- dataset.recipes.list()
156
- """
157
- if self._dataset is not None:
158
- try:
159
- recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
160
- except KeyError:
161
- recipes = list()
162
-
163
- pool = self._client_api.thread_pools(pool_name='entity.create')
164
- jobs = [None for _ in range(len(recipes))]
165
- for i_recipe, recipe_id in enumerate(recipes):
166
- jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
167
-
168
- # get all results
169
- results = [j.result() for j in jobs]
170
- # log errors
171
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
172
- # return good jobs
173
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
174
- elif self._project_id is not None:
175
- if filters is None:
176
- filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
177
- # assert type filters
178
- elif not isinstance(filters, entities.Filters):
179
- raise exceptions.PlatformException(error='400',
180
- message='Unknown filters type: {!r}'.format(type(filters)))
181
- if filters.resource != entities.FiltersResource.RECIPE:
182
- raise exceptions.PlatformException(
183
- error='400',
184
- message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
185
- if not filters.has_field('projects'):
186
- filters.add(field='projects', values=[self._project_id])
187
-
188
- recipes = entities.PagedEntities(items_repository=self,
189
- filters=filters,
190
- page_offset=filters.page,
191
- page_size=filters.page_size,
192
- project_id=self._project_id,
193
- client_api=self._client_api)
194
- recipes.get_page()
195
- else:
196
- raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
197
-
198
- return recipes
199
-
200
- def _list(self, filters: entities.Filters):
201
- url = filters.generate_url_query_params('/recipes')
202
- encoded_url = urllib.parse.quote(url, safe='/:?=&')
203
- # request
204
- success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
205
- if not success:
206
- raise exceptions.PlatformException(response)
207
- return response.json()
208
-
209
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
210
- pool = self._client_api.thread_pools(pool_name='entity.create')
211
- jobs = [None for _ in range(len(response_items))]
212
- # return triggers list
213
- for i_rec, rec in enumerate(response_items):
214
- jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
215
- **{'client_api': self._client_api,
216
- '_json': rec,
217
- 'project': self._project,
218
- 'dataset': self._dataset})
219
-
220
- # get all results
221
- results = [j.result() for j in jobs]
222
- # log errors
223
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
224
- # return good jobs
225
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
226
- return recipes
227
-
228
- def _protected_get(self, recipe_id):
229
- """
230
- Same as get but with try-except to catch if error
231
- :param recipe_id:
232
- :return:
233
- """
234
- try:
235
- recipe = self.get(recipe_id=recipe_id)
236
- status = True
237
- except Exception:
238
- recipe = traceback.format_exc()
239
- status = False
240
- return status, recipe
241
-
242
- @_api_reference.add(path='/recipes/{id}', method='get')
243
- def get(self, recipe_id: str) -> entities.Recipe:
244
- """
245
- Get a Recipe object to use in your code.
246
-
247
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
248
-
249
- :param str recipe_id: recipe id
250
- :return: Recipe object
251
- :rtype: dtlpy.entities.recipe.Recipe
252
-
253
- **Example**:
254
-
255
- .. code-block:: python
256
-
257
- dataset.recipes.get(recipe_id='recipe_id')
258
- """
259
- success, response = self._client_api.gen_request(req_type='get',
260
- path='/recipes/%s' % recipe_id)
261
- if success:
262
- recipe = entities.Recipe.from_json(client_api=self._client_api,
263
- _json=response.json(),
264
- project=self._project,
265
- dataset=self._dataset)
266
- else:
267
- logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
268
- raise exceptions.PlatformException(response)
269
-
270
- return recipe
271
-
272
- def open_in_web(self,
273
- recipe: entities.Recipe = None,
274
- recipe_id: str = None):
275
- """
276
- Open the recipe in web platform.
277
-
278
- **Prerequisites**: All users.
279
-
280
- :param dtlpy.entities.recipe.Recipe recipe: recipe entity
281
- :param str recipe_id: recipe id
282
-
283
- **Example**:
284
-
285
- .. code-block:: python
286
-
287
- dataset.recipes.open_in_web(recipe_id='recipe_id')
288
- """
289
- if recipe is not None:
290
- recipe.open_in_web()
291
- elif recipe_id is not None:
292
- self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
293
- else:
294
- self._client_api._open_in_web(url=self.platform_url)
295
-
296
- @_api_reference.add(path='/recipes/{id}', method='delete')
297
- def delete(self, recipe_id: str, force: bool = False):
298
- """
299
- Delete recipe from platform.
300
-
301
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
302
-
303
- :param str recipe_id: recipe id
304
- :param bool force: force delete recipe
305
- :return: True if success
306
- :rtype: bool
307
-
308
- **Example**:
309
-
310
- .. code-block:: python
311
-
312
- dataset.recipes.delete(recipe_id='recipe_id')
313
- """
314
- path = '/recipes/{}'.format(recipe_id)
315
- if force:
316
- path += '?force=true'
317
- success, response = self._client_api.gen_request(req_type='delete',
318
- path=path)
319
- if not success:
320
- raise exceptions.PlatformException(response)
321
- logger.info('Recipe id {} deleted successfully'.format(recipe_id))
322
- return True
323
-
324
- @_api_reference.add(path='/recipes/{id}', method='patch')
325
- def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
326
- """
327
- Update recipe.
328
-
329
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
330
-
331
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
332
- :param bool system_metadata: True, if you want to change metadata system
333
- :return: Recipe object
334
- :rtype: dtlpy.entities.recipe.Recipe
335
-
336
- **Example**:
337
-
338
- .. code-block:: python
339
-
340
- dataset.recipes.update(recipe='recipe_entity')
341
- """
342
- url_path = '/recipes/%s' % recipe.id
343
- if system_metadata:
344
- url_path += '?system=true'
345
- success, response = self._client_api.gen_request(req_type='patch',
346
- path=url_path,
347
- json_req=recipe.to_json())
348
- if success:
349
- return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
350
- else:
351
- logger.error('Error while updating item:')
352
- raise exceptions.PlatformException(response)
353
-
354
- @_api_reference.add(path='/recipes/{id}/clone', method='post')
355
- def clone(self,
356
- recipe: entities.Recipe = None,
357
- recipe_id: str = None,
358
- shallow: bool = False):
359
- """
360
- Clone recipe.
361
-
362
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
363
-
364
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
365
- :param str recipe_id: Recipe id
366
- :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
367
- :return: Cloned ontology object
368
- :rtype: dtlpy.entities.recipe.Recipe
369
-
370
- **Example**:
371
-
372
- .. code-block:: python
373
-
374
- dataset.recipes.clone(recipe_id='recipe_id')
375
- """
376
- if recipe is None and recipe_id is None:
377
- raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
378
- if recipe_id is None:
379
- if not isinstance(recipe, entities.Recipe):
380
- raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
381
- else:
382
- recipe_id = recipe.id
383
-
384
- payload = {'shallow': shallow}
385
-
386
- success, response = self._client_api.gen_request(req_type='post',
387
- path='/recipes/{}/clone'.format(recipe_id),
388
- json_req=payload)
389
- if success:
390
- recipe = entities.Recipe.from_json(client_api=self._client_api,
391
- _json=response.json())
392
- else:
393
- logger.error('Failed to clone Recipe')
394
- raise exceptions.PlatformException(response)
395
-
396
- assert isinstance(recipe, entities.Recipe)
397
- logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
398
-
399
- return recipe
1
+ import logging
2
+ import traceback
3
+ import urllib.parse
4
+
5
+ from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
+ from ..services.api_client import ApiClient
7
+
8
+ logger = logging.getLogger(name='dtlpy')
9
+
10
+
11
+ class Recipes:
12
+ """
13
+ Recipes Repository
14
+
15
+ The Recipes class allows you to manage recipes and their properties.
16
+ For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
+ """
18
+
19
+ def __init__(self,
20
+ client_api: ApiClient,
21
+ dataset: entities.Dataset = None,
22
+ project: entities.Project = None,
23
+ project_id: str = None):
24
+ self._client_api = client_api
25
+ self._dataset = dataset
26
+ self._project = project
27
+ self._project_id = project_id
28
+ if project_id is None and project is not None:
29
+ self._project_id = project.id
30
+
31
+ ############
32
+ # entities #
33
+ ############
34
+ @property
35
+ def platform_url(self):
36
+ if self._project_id is None:
37
+ project_id = self.dataset.project.id
38
+ else:
39
+ project_id = self._project_id
40
+ return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
+
42
+ @property
43
+ def project(self) -> entities.Project:
44
+ if self._project is None:
45
+ project = self._client_api.state_io.get('project')
46
+ if project is not None:
47
+ self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
48
+ self._project_id = self._project.id
49
+ if self._project_id is None:
50
+ if self._dataset is None:
51
+ raise exceptions.PlatformException(
52
+ error='2001',
53
+ message='Missing "Project". need to set a Project entity or use project.recipes repository'
54
+ )
55
+ else:
56
+ self._project = self._dataset.project
57
+ self._project_id = self._project.id
58
+ else:
59
+ self._project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
60
+ self._project_id = self._project.id
61
+
62
+ assert isinstance(self._project, entities.Project)
63
+ return self._project
64
+
65
+ @project.setter
66
+ def project(self, project: entities.Project):
67
+ if not isinstance(project, entities.Project):
68
+ raise ValueError('Must input a valid Project entity')
69
+ self._project = project
70
+ self._project_id = project.id
71
+
72
+ @property
73
+ def dataset(self) -> entities.Dataset:
74
+ if self._dataset is None:
75
+ raise exceptions.PlatformException(
76
+ error='2001',
77
+ message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
78
+ assert isinstance(self._dataset, entities.Dataset)
79
+ return self._dataset
80
+
81
+ @dataset.setter
82
+ def dataset(self, dataset: entities.Dataset):
83
+ if not isinstance(dataset, entities.Dataset):
84
+ raise ValueError('Must input a valid Dataset entity')
85
+ self._dataset = dataset
86
+
87
+ ###########
88
+ # methods #
89
+ ###########
90
+ @_api_reference.add(path='/recipes', method='post')
91
+ def create(self,
92
+ project_ids=None,
93
+ ontology_ids=None,
94
+ labels=None,
95
+ recipe_name=None,
96
+ attributes=None,
97
+ annotation_instruction_file=None
98
+ ) -> entities.Recipe:
99
+ """
100
+ Create a new Recipe.
101
+ Note: If the param ontology_ids is None, an ontology will be created first.
102
+
103
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
104
+
105
+ :param str project_ids: project ids
106
+ :param str or list ontology_ids: ontology ids
107
+ :param labels: labels
108
+ :param str recipe_name: recipe name
109
+ :param attributes: attributes
110
+ :param str annotation_instruction_file: file path or url of the recipe instruction
111
+ :return: Recipe entity
112
+ :rtype: dtlpy.entities.recipe.Recipe
113
+
114
+ **Example**:
115
+
116
+ .. code-block:: python
117
+
118
+ dataset.recipes.create(recipe_name='My Recipe', labels=labels))
119
+ """
120
+ if labels is None:
121
+ labels = list()
122
+ if attributes is None:
123
+ attributes = list()
124
+ if project_ids is None:
125
+ project_ids = [self.project.id]
126
+ if ontology_ids is None:
127
+ ontolgies = repositories.Ontologies(client_api=self._client_api,
128
+ recipe=None)
129
+ ontology = ontolgies.create(labels=labels,
130
+ project_ids=project_ids,
131
+ attributes=attributes)
132
+ ontology_ids = [ontology.id]
133
+ elif not isinstance(ontology_ids, list):
134
+ ontology_ids = [ontology_ids]
135
+ if recipe_name is None:
136
+ recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
137
+ payload = {'title': recipe_name,
138
+ 'projectIds': project_ids,
139
+ 'ontologyIds': ontology_ids,
140
+ 'uiSettings': {
141
+ "allowObjectIdAutoAssign": True,
142
+ "studioV2App": True
143
+ }}
144
+ success, response = self._client_api.gen_request(req_type='post',
145
+ path='/recipes',
146
+ json_req=payload)
147
+ if success:
148
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
149
+ _json=response.json(),
150
+ dataset=self._dataset)
151
+ if annotation_instruction_file:
152
+ recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
153
+ else:
154
+ logger.error('Failed to create Recipe')
155
+ raise exceptions.PlatformException(response)
156
+
157
+ if self._dataset is not None:
158
+ self._dataset.switch_recipe(recipe.id)
159
+ return recipe
160
+
161
+ def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
162
+ """
163
+ List recipes for a dataset.
164
+
165
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
166
+
167
+ :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
168
+ :return: list of all recipes
169
+ :retype: list
170
+
171
+ **Example**:
172
+
173
+ .. code-block:: python
174
+
175
+ dataset.recipes.list()
176
+ """
177
+ if self._dataset is not None:
178
+ try:
179
+ recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
180
+ except KeyError:
181
+ recipes = list()
182
+
183
+ pool = self._client_api.thread_pools(pool_name='entity.create')
184
+ jobs = [None for _ in range(len(recipes))]
185
+ for i_recipe, recipe_id in enumerate(recipes):
186
+ jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
187
+
188
+ # get all results
189
+ results = [j.result() for j in jobs]
190
+ # log errors
191
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
192
+ # return good jobs
193
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
194
+ elif self._project_id is not None:
195
+ if filters is None:
196
+ filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
197
+ # assert type filters
198
+ elif not isinstance(filters, entities.Filters):
199
+ raise exceptions.PlatformException(error='400',
200
+ message='Unknown filters type: {!r}'.format(type(filters)))
201
+ if filters.resource != entities.FiltersResource.RECIPE:
202
+ raise exceptions.PlatformException(
203
+ error='400',
204
+ message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
205
+ if not filters.has_field('projects'):
206
+ filters.add(field='projects', values=[self._project_id])
207
+
208
+ recipes = entities.PagedEntities(items_repository=self,
209
+ filters=filters,
210
+ page_offset=filters.page,
211
+ page_size=filters.page_size,
212
+ project_id=self._project_id,
213
+ client_api=self._client_api)
214
+ recipes.get_page()
215
+ else:
216
+ raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
217
+
218
+ return recipes
219
+
220
+ def _list(self, filters: entities.Filters):
221
+ url = filters.generate_url_query_params('/recipes')
222
+ encoded_url = urllib.parse.quote(url, safe='/:?=&')
223
+ # request
224
+ success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
225
+ if not success:
226
+ raise exceptions.PlatformException(response)
227
+ return response.json()
228
+
229
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
230
+ pool = self._client_api.thread_pools(pool_name='entity.create')
231
+ jobs = [None for _ in range(len(response_items))]
232
+ # return triggers list
233
+ for i_rec, rec in enumerate(response_items):
234
+ jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
235
+ **{'client_api': self._client_api,
236
+ '_json': rec,
237
+ 'project': self._project,
238
+ 'dataset': self._dataset})
239
+
240
+ # get all results
241
+ results = [j.result() for j in jobs]
242
+ # log errors
243
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
244
+ # return good jobs
245
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
246
+ return recipes
247
+
248
+ def _protected_get(self, recipe_id):
249
+ """
250
+ Same as get but with try-except to catch if error
251
+ :param recipe_id:
252
+ :return:
253
+ """
254
+ try:
255
+ recipe = self.get(recipe_id=recipe_id)
256
+ status = True
257
+ except Exception:
258
+ recipe = traceback.format_exc()
259
+ status = False
260
+ return status, recipe
261
+
262
+ @_api_reference.add(path='/recipes/{id}', method='get')
263
+ def get(self, recipe_id: str) -> entities.Recipe:
264
+ """
265
+ Get a Recipe object to use in your code.
266
+
267
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
268
+
269
+ :param str recipe_id: recipe id
270
+ :return: Recipe object
271
+ :rtype: dtlpy.entities.recipe.Recipe
272
+
273
+ **Example**:
274
+
275
+ .. code-block:: python
276
+
277
+ dataset.recipes.get(recipe_id='recipe_id')
278
+ """
279
+ success, response = self._client_api.gen_request(req_type='get',
280
+ path='/recipes/%s' % recipe_id)
281
+ if success:
282
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
283
+ _json=response.json(),
284
+ project=self._project,
285
+ dataset=self._dataset)
286
+ else:
287
+ logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
288
+ raise exceptions.PlatformException(response)
289
+
290
+ return recipe
291
+
292
+ def open_in_web(self,
293
+ recipe: entities.Recipe = None,
294
+ recipe_id: str = None):
295
+ """
296
+ Open the recipe in web platform.
297
+
298
+ **Prerequisites**: All users.
299
+
300
+ :param dtlpy.entities.recipe.Recipe recipe: recipe entity
301
+ :param str recipe_id: recipe id
302
+
303
+ **Example**:
304
+
305
+ .. code-block:: python
306
+
307
+ dataset.recipes.open_in_web(recipe_id='recipe_id')
308
+ """
309
+ if recipe is not None:
310
+ recipe.open_in_web()
311
+ elif recipe_id is not None:
312
+ self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
313
+ else:
314
+ self._client_api._open_in_web(url=self.platform_url)
315
+
316
+ @_api_reference.add(path='/recipes/{id}', method='delete')
317
+ def delete(self, recipe_id: str, force: bool = False):
318
+ """
319
+ Delete recipe from platform.
320
+
321
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
322
+
323
+ :param str recipe_id: recipe id
324
+ :param bool force: force delete recipe
325
+ :return: True if success
326
+ :rtype: bool
327
+
328
+ **Example**:
329
+
330
+ .. code-block:: python
331
+
332
+ dataset.recipes.delete(recipe_id='recipe_id')
333
+ """
334
+ path = '/recipes/{}'.format(recipe_id)
335
+ if force:
336
+ path += '?force=true'
337
+ success, response = self._client_api.gen_request(req_type='delete',
338
+ path=path)
339
+ if not success:
340
+ raise exceptions.PlatformException(response)
341
+ logger.info('Recipe id {} deleted successfully'.format(recipe_id))
342
+ return True
343
+
344
+ @_api_reference.add(path='/recipes/{id}', method='patch')
345
+ def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
346
+ """
347
+ Update recipe.
348
+
349
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
350
+
351
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
352
+ :param bool system_metadata: True, if you want to change metadata system
353
+ :return: Recipe object
354
+ :rtype: dtlpy.entities.recipe.Recipe
355
+
356
+ **Example**:
357
+
358
+ .. code-block:: python
359
+
360
+ dataset.recipes.update(recipe='recipe_entity')
361
+ """
362
+ url_path = '/recipes/%s' % recipe.id
363
+ if system_metadata:
364
+ url_path += '?system=true'
365
+ success, response = self._client_api.gen_request(req_type='patch',
366
+ path=url_path,
367
+ json_req=recipe.to_json())
368
+ if success:
369
+ return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
370
+ else:
371
+ logger.error('Error while updating item:')
372
+ raise exceptions.PlatformException(response)
373
+
374
+ @_api_reference.add(path='/recipes/{id}/clone', method='post')
375
+ def clone(self,
376
+ recipe: entities.Recipe = None,
377
+ recipe_id: str = None,
378
+ shallow: bool = False):
379
+ """
380
+ Clone recipe.
381
+
382
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
383
+
384
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
385
+ :param str recipe_id: Recipe id
386
+ :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
387
+ :return: Cloned ontology object
388
+ :rtype: dtlpy.entities.recipe.Recipe
389
+
390
+ **Example**:
391
+
392
+ .. code-block:: python
393
+
394
+ dataset.recipes.clone(recipe_id='recipe_id')
395
+ """
396
+ if recipe is None and recipe_id is None:
397
+ raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
398
+ if recipe_id is None:
399
+ if not isinstance(recipe, entities.Recipe):
400
+ raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
401
+ else:
402
+ recipe_id = recipe.id
403
+
404
+ payload = {'shallow': shallow}
405
+
406
+ success, response = self._client_api.gen_request(req_type='post',
407
+ path='/recipes/{}/clone'.format(recipe_id),
408
+ json_req=payload)
409
+ if success:
410
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
411
+ _json=response.json())
412
+ else:
413
+ logger.error('Failed to clone Recipe')
414
+ raise exceptions.PlatformException(response)
415
+
416
+ assert isinstance(recipe, entities.Recipe)
417
+ logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
418
+
419
+ return recipe