dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,399 +1,399 @@
1
- import logging
2
- import traceback
3
- import urllib.parse
4
-
5
- from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
- from ..services.api_client import ApiClient
7
-
8
- logger = logging.getLogger(name='dtlpy')
9
-
10
-
11
- class Recipes:
12
- """
13
- Recipes Repository
14
-
15
- The Recipes class allows you to manage recipes and their properties.
16
- For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
- """
18
-
19
- def __init__(self,
20
- client_api: ApiClient,
21
- dataset: entities.Dataset = None,
22
- project: entities.Project = None,
23
- project_id: str = None):
24
- self._client_api = client_api
25
- self._dataset = dataset
26
- self._project = project
27
- self._project_id = project_id
28
- if project_id is None and project is not None:
29
- self._project_id = project.id
30
-
31
- ############
32
- # entities #
33
- ############
34
- @property
35
- def platform_url(self):
36
- if self._project_id is None:
37
- project_id = self.dataset.project.id
38
- else:
39
- project_id = self._project_id
40
- return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
-
42
- @property
43
- def dataset(self) -> entities.Dataset:
44
- if self._dataset is None:
45
- raise exceptions.PlatformException(
46
- error='2001',
47
- message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
48
- assert isinstance(self._dataset, entities.Dataset)
49
- return self._dataset
50
-
51
- @dataset.setter
52
- def dataset(self, dataset: entities.Dataset):
53
- if not isinstance(dataset, entities.Dataset):
54
- raise ValueError('Must input a valid Dataset entity')
55
- self._dataset = dataset
56
-
57
- ###########
58
- # methods #
59
- ###########
60
- @_api_reference.add(path='/recipes', method='post')
61
- def create(self,
62
- project_ids=None,
63
- ontology_ids=None,
64
- labels=None,
65
- recipe_name=None,
66
- attributes=None,
67
- annotation_instruction_file=None
68
- ) -> entities.Recipe:
69
- """
70
- Create a new Recipe.
71
- Note: If the param ontology_ids is None, an ontology will be created first.
72
-
73
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
74
-
75
- :param str project_ids: project ids
76
- :param str or list ontology_ids: ontology ids
77
- :param labels: labels
78
- :param str recipe_name: recipe name
79
- :param attributes: attributes
80
- :param str annotation_instruction_file: file path or url of the recipe instruction
81
- :return: Recipe entity
82
- :rtype: dtlpy.entities.recipe.Recipe
83
-
84
- **Example**:
85
-
86
- .. code-block:: python
87
-
88
- dataset.recipes.create(recipe_name='My Recipe', labels=labels))
89
- """
90
- if labels is None:
91
- labels = list()
92
- if attributes is None:
93
- attributes = list()
94
- if project_ids is None:
95
- if self._dataset is not None:
96
- project_ids = [self._dataset.project.id]
97
- else:
98
- # get from cache
99
- project = self._client_api.state_io.get('project')
100
- if project is not None:
101
- # build entity from json
102
- p = entities.Project.from_json(_json=project, client_api=self._client_api)
103
- project_ids = [p.id]
104
- else:
105
- raise exceptions.PlatformException('Must provide project_ids')
106
- if ontology_ids is None:
107
- ontolgies = repositories.Ontologies(client_api=self._client_api,
108
- recipe=None)
109
- ontology = ontolgies.create(labels=labels,
110
- project_ids=project_ids,
111
- attributes=attributes)
112
- ontology_ids = [ontology.id]
113
- elif not isinstance(ontology_ids, list):
114
- ontology_ids = [ontology_ids]
115
- if recipe_name is None:
116
- recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
117
- payload = {'title': recipe_name,
118
- 'projectIds': project_ids,
119
- 'ontologyIds': ontology_ids,
120
- 'uiSettings': {
121
- "allowObjectIdAutoAssign": True,
122
- "studioV2App": True
123
- }}
124
- success, response = self._client_api.gen_request(req_type='post',
125
- path='/recipes',
126
- json_req=payload)
127
- if success:
128
- recipe = entities.Recipe.from_json(client_api=self._client_api,
129
- _json=response.json(),
130
- dataset=self._dataset)
131
- if annotation_instruction_file:
132
- recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
133
- else:
134
- logger.error('Failed to create Recipe')
135
- raise exceptions.PlatformException(response)
136
-
137
- if self._dataset is not None:
138
- self._dataset.switch_recipe(recipe.id)
139
- return recipe
140
-
141
- def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
142
- """
143
- List recipes for a dataset.
144
-
145
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
146
-
147
- :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
148
- :return: list of all recipes
149
- :retype: list
150
-
151
- **Example**:
152
-
153
- .. code-block:: python
154
-
155
- dataset.recipes.list()
156
- """
157
- if self._dataset is not None:
158
- try:
159
- recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
160
- except KeyError:
161
- recipes = list()
162
-
163
- pool = self._client_api.thread_pools(pool_name='entity.create')
164
- jobs = [None for _ in range(len(recipes))]
165
- for i_recipe, recipe_id in enumerate(recipes):
166
- jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
167
-
168
- # get all results
169
- results = [j.result() for j in jobs]
170
- # log errors
171
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
172
- # return good jobs
173
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
174
- elif self._project_id is not None:
175
- if filters is None:
176
- filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
177
- # assert type filters
178
- elif not isinstance(filters, entities.Filters):
179
- raise exceptions.PlatformException(error='400',
180
- message='Unknown filters type: {!r}'.format(type(filters)))
181
- if filters.resource != entities.FiltersResource.RECIPE:
182
- raise exceptions.PlatformException(
183
- error='400',
184
- message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
185
- if not filters.has_field('projects'):
186
- filters.add(field='projects', values=[self._project_id])
187
-
188
- recipes = entities.PagedEntities(items_repository=self,
189
- filters=filters,
190
- page_offset=filters.page,
191
- page_size=filters.page_size,
192
- project_id=self._project_id,
193
- client_api=self._client_api)
194
- recipes.get_page()
195
- else:
196
- raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
197
-
198
- return recipes
199
-
200
- def _list(self, filters: entities.Filters):
201
- url = filters.generate_url_query_params('/recipes')
202
- encoded_url = urllib.parse.quote(url, safe='/:?=&')
203
- # request
204
- success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
205
- if not success:
206
- raise exceptions.PlatformException(response)
207
- return response.json()
208
-
209
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
210
- pool = self._client_api.thread_pools(pool_name='entity.create')
211
- jobs = [None for _ in range(len(response_items))]
212
- # return triggers list
213
- for i_rec, rec in enumerate(response_items):
214
- jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
215
- **{'client_api': self._client_api,
216
- '_json': rec,
217
- 'project': self._project,
218
- 'dataset': self._dataset})
219
-
220
- # get all results
221
- results = [j.result() for j in jobs]
222
- # log errors
223
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
224
- # return good jobs
225
- recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
226
- return recipes
227
-
228
- def _protected_get(self, recipe_id):
229
- """
230
- Same as get but with try-except to catch if error
231
- :param recipe_id:
232
- :return:
233
- """
234
- try:
235
- recipe = self.get(recipe_id=recipe_id)
236
- status = True
237
- except Exception:
238
- recipe = traceback.format_exc()
239
- status = False
240
- return status, recipe
241
-
242
- @_api_reference.add(path='/recipes/{id}', method='get')
243
- def get(self, recipe_id: str) -> entities.Recipe:
244
- """
245
- Get a Recipe object to use in your code.
246
-
247
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
248
-
249
- :param str recipe_id: recipe id
250
- :return: Recipe object
251
- :rtype: dtlpy.entities.recipe.Recipe
252
-
253
- **Example**:
254
-
255
- .. code-block:: python
256
-
257
- dataset.recipes.get(recipe_id='recipe_id')
258
- """
259
- success, response = self._client_api.gen_request(req_type='get',
260
- path='/recipes/%s' % recipe_id)
261
- if success:
262
- recipe = entities.Recipe.from_json(client_api=self._client_api,
263
- _json=response.json(),
264
- project=self._project,
265
- dataset=self._dataset)
266
- else:
267
- logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
268
- raise exceptions.PlatformException(response)
269
-
270
- return recipe
271
-
272
- def open_in_web(self,
273
- recipe: entities.Recipe = None,
274
- recipe_id: str = None):
275
- """
276
- Open the recipe in web platform.
277
-
278
- **Prerequisites**: All users.
279
-
280
- :param dtlpy.entities.recipe.Recipe recipe: recipe entity
281
- :param str recipe_id: recipe id
282
-
283
- **Example**:
284
-
285
- .. code-block:: python
286
-
287
- dataset.recipes.open_in_web(recipe_id='recipe_id')
288
- """
289
- if recipe is not None:
290
- recipe.open_in_web()
291
- elif recipe_id is not None:
292
- self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
293
- else:
294
- self._client_api._open_in_web(url=self.platform_url)
295
-
296
- @_api_reference.add(path='/recipes/{id}', method='delete')
297
- def delete(self, recipe_id: str, force: bool = False):
298
- """
299
- Delete recipe from platform.
300
-
301
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
302
-
303
- :param str recipe_id: recipe id
304
- :param bool force: force delete recipe
305
- :return: True if success
306
- :rtype: bool
307
-
308
- **Example**:
309
-
310
- .. code-block:: python
311
-
312
- dataset.recipes.delete(recipe_id='recipe_id')
313
- """
314
- path = '/recipes/{}'.format(recipe_id)
315
- if force:
316
- path += '?force=true'
317
- success, response = self._client_api.gen_request(req_type='delete',
318
- path=path)
319
- if not success:
320
- raise exceptions.PlatformException(response)
321
- logger.info('Recipe id {} deleted successfully'.format(recipe_id))
322
- return True
323
-
324
- @_api_reference.add(path='/recipes/{id}', method='patch')
325
- def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
326
- """
327
- Update recipe.
328
-
329
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
330
-
331
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
332
- :param bool system_metadata: True, if you want to change metadata system
333
- :return: Recipe object
334
- :rtype: dtlpy.entities.recipe.Recipe
335
-
336
- **Example**:
337
-
338
- .. code-block:: python
339
-
340
- dataset.recipes.update(recipe='recipe_entity')
341
- """
342
- url_path = '/recipes/%s' % recipe.id
343
- if system_metadata:
344
- url_path += '?system=true'
345
- success, response = self._client_api.gen_request(req_type='patch',
346
- path=url_path,
347
- json_req=recipe.to_json())
348
- if success:
349
- return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
350
- else:
351
- logger.error('Error while updating item:')
352
- raise exceptions.PlatformException(response)
353
-
354
- @_api_reference.add(path='/recipes/{id}/clone', method='post')
355
- def clone(self,
356
- recipe: entities.Recipe = None,
357
- recipe_id: str = None,
358
- shallow: bool = False):
359
- """
360
- Clone recipe.
361
-
362
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
363
-
364
- :param dtlpy.entities.recipe.Recipe recipe: Recipe object
365
- :param str recipe_id: Recipe id
366
- :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
367
- :return: Cloned ontology object
368
- :rtype: dtlpy.entities.recipe.Recipe
369
-
370
- **Example**:
371
-
372
- .. code-block:: python
373
-
374
- dataset.recipes.clone(recipe_id='recipe_id')
375
- """
376
- if recipe is None and recipe_id is None:
377
- raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
378
- if recipe_id is None:
379
- if not isinstance(recipe, entities.Recipe):
380
- raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
381
- else:
382
- recipe_id = recipe.id
383
-
384
- payload = {'shallow': shallow}
385
-
386
- success, response = self._client_api.gen_request(req_type='post',
387
- path='/recipes/{}/clone'.format(recipe_id),
388
- json_req=payload)
389
- if success:
390
- recipe = entities.Recipe.from_json(client_api=self._client_api,
391
- _json=response.json())
392
- else:
393
- logger.error('Failed to clone Recipe')
394
- raise exceptions.PlatformException(response)
395
-
396
- assert isinstance(recipe, entities.Recipe)
397
- logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
398
-
399
- return recipe
1
+ import logging
2
+ import traceback
3
+ import urllib.parse
4
+
5
+ from .. import entities, miscellaneous, repositories, exceptions, _api_reference
6
+ from ..services.api_client import ApiClient
7
+
8
+ logger = logging.getLogger(name='dtlpy')
9
+
10
+
11
+ class Recipes:
12
+ """
13
+ Recipes Repository
14
+
15
+ The Recipes class allows you to manage recipes and their properties.
16
+ For more information on Recipes, see our `documentation <https://dataloop.ai/docs/ontology>`_ and `developers' documentation <https://developers.dataloop.ai/tutorials/recipe_and_ontology/recipe/chapter/>`_.
17
+ """
18
+
19
+ def __init__(self,
20
+ client_api: ApiClient,
21
+ dataset: entities.Dataset = None,
22
+ project: entities.Project = None,
23
+ project_id: str = None):
24
+ self._client_api = client_api
25
+ self._dataset = dataset
26
+ self._project = project
27
+ self._project_id = project_id
28
+ if project_id is None and project is not None:
29
+ self._project_id = project.id
30
+
31
+ ############
32
+ # entities #
33
+ ############
34
+ @property
35
+ def platform_url(self):
36
+ if self._project_id is None:
37
+ project_id = self.dataset.project.id
38
+ else:
39
+ project_id = self._project_id
40
+ return self._client_api._get_resource_url("projects/{}/recipes".format(project_id))
41
+
42
+ @property
43
+ def dataset(self) -> entities.Dataset:
44
+ if self._dataset is None:
45
+ raise exceptions.PlatformException(
46
+ error='2001',
47
+ message='Missing "dataset". need to set a Dataset entity or use dataset.recipes repository')
48
+ assert isinstance(self._dataset, entities.Dataset)
49
+ return self._dataset
50
+
51
+ @dataset.setter
52
+ def dataset(self, dataset: entities.Dataset):
53
+ if not isinstance(dataset, entities.Dataset):
54
+ raise ValueError('Must input a valid Dataset entity')
55
+ self._dataset = dataset
56
+
57
+ ###########
58
+ # methods #
59
+ ###########
60
+ @_api_reference.add(path='/recipes', method='post')
61
+ def create(self,
62
+ project_ids=None,
63
+ ontology_ids=None,
64
+ labels=None,
65
+ recipe_name=None,
66
+ attributes=None,
67
+ annotation_instruction_file=None
68
+ ) -> entities.Recipe:
69
+ """
70
+ Create a new Recipe.
71
+ Note: If the param ontology_ids is None, an ontology will be created first.
72
+
73
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
74
+
75
+ :param str project_ids: project ids
76
+ :param str or list ontology_ids: ontology ids
77
+ :param labels: labels
78
+ :param str recipe_name: recipe name
79
+ :param attributes: attributes
80
+ :param str annotation_instruction_file: file path or url of the recipe instruction
81
+ :return: Recipe entity
82
+ :rtype: dtlpy.entities.recipe.Recipe
83
+
84
+ **Example**:
85
+
86
+ .. code-block:: python
87
+
88
+ dataset.recipes.create(recipe_name='My Recipe', labels=labels))
89
+ """
90
+ if labels is None:
91
+ labels = list()
92
+ if attributes is None:
93
+ attributes = list()
94
+ if project_ids is None:
95
+ if self._dataset is not None:
96
+ project_ids = [self._dataset.project.id]
97
+ else:
98
+ # get from cache
99
+ project = self._client_api.state_io.get('project')
100
+ if project is not None:
101
+ # build entity from json
102
+ p = entities.Project.from_json(_json=project, client_api=self._client_api)
103
+ project_ids = [p.id]
104
+ else:
105
+ raise exceptions.PlatformException('Must provide project_ids')
106
+ if ontology_ids is None:
107
+ ontolgies = repositories.Ontologies(client_api=self._client_api,
108
+ recipe=None)
109
+ ontology = ontolgies.create(labels=labels,
110
+ project_ids=project_ids,
111
+ attributes=attributes)
112
+ ontology_ids = [ontology.id]
113
+ elif not isinstance(ontology_ids, list):
114
+ ontology_ids = [ontology_ids]
115
+ if recipe_name is None:
116
+ recipe_name = self._dataset.name + " Default Recipe" if self._dataset is not None else "Default Recipe"
117
+ payload = {'title': recipe_name,
118
+ 'projectIds': project_ids,
119
+ 'ontologyIds': ontology_ids,
120
+ 'uiSettings': {
121
+ "allowObjectIdAutoAssign": True,
122
+ "studioV2App": True
123
+ }}
124
+ success, response = self._client_api.gen_request(req_type='post',
125
+ path='/recipes',
126
+ json_req=payload)
127
+ if success:
128
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
129
+ _json=response.json(),
130
+ dataset=self._dataset)
131
+ if annotation_instruction_file:
132
+ recipe.add_instruction(annotation_instruction_file=annotation_instruction_file)
133
+ else:
134
+ logger.error('Failed to create Recipe')
135
+ raise exceptions.PlatformException(response)
136
+
137
+ if self._dataset is not None:
138
+ self._dataset.switch_recipe(recipe.id)
139
+ return recipe
140
+
141
+ def list(self, filters: entities.Filters = None) -> miscellaneous.List[entities.Recipe]:
142
+ """
143
+ List recipes for a dataset.
144
+
145
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
146
+
147
+ :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
148
+ :return: list of all recipes
149
+ :retype: list
150
+
151
+ **Example**:
152
+
153
+ .. code-block:: python
154
+
155
+ dataset.recipes.list()
156
+ """
157
+ if self._dataset is not None:
158
+ try:
159
+ recipes = [recipe_id for recipe_id in self._dataset.metadata['system']['recipes']]
160
+ except KeyError:
161
+ recipes = list()
162
+
163
+ pool = self._client_api.thread_pools(pool_name='entity.create')
164
+ jobs = [None for _ in range(len(recipes))]
165
+ for i_recipe, recipe_id in enumerate(recipes):
166
+ jobs[i_recipe] = pool.submit(self._protected_get, **{'recipe_id': recipe_id})
167
+
168
+ # get all results
169
+ results = [j.result() for j in jobs]
170
+ # log errors
171
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
172
+ # return good jobs
173
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
174
+ elif self._project_id is not None:
175
+ if filters is None:
176
+ filters = entities.Filters(resource=entities.FiltersResource.RECIPE)
177
+ # assert type filters
178
+ elif not isinstance(filters, entities.Filters):
179
+ raise exceptions.PlatformException(error='400',
180
+ message='Unknown filters type: {!r}'.format(type(filters)))
181
+ if filters.resource != entities.FiltersResource.RECIPE:
182
+ raise exceptions.PlatformException(
183
+ error='400',
184
+ message='Filters resource must to be FiltersResource.RECIPE. Got: {!r}'.format(filters.resource))
185
+ if not filters.has_field('projects'):
186
+ filters.add(field='projects', values=[self._project_id])
187
+
188
+ recipes = entities.PagedEntities(items_repository=self,
189
+ filters=filters,
190
+ page_offset=filters.page,
191
+ page_size=filters.page_size,
192
+ project_id=self._project_id,
193
+ client_api=self._client_api)
194
+ recipes.get_page()
195
+ else:
196
+ raise exceptions.PlatformException('400', 'Must have project or dataset entity in repository')
197
+
198
+ return recipes
199
+
200
+ def _list(self, filters: entities.Filters):
201
+ url = filters.generate_url_query_params('/recipes')
202
+ encoded_url = urllib.parse.quote(url, safe='/:?=&')
203
+ # request
204
+ success, response = self._client_api.gen_request(req_type='get', path=encoded_url)
205
+ if not success:
206
+ raise exceptions.PlatformException(response)
207
+ return response.json()
208
+
209
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Recipe]:
210
+ pool = self._client_api.thread_pools(pool_name='entity.create')
211
+ jobs = [None for _ in range(len(response_items))]
212
+ # return triggers list
213
+ for i_rec, rec in enumerate(response_items):
214
+ jobs[i_rec] = pool.submit(entities.Recipe._protected_from_json,
215
+ **{'client_api': self._client_api,
216
+ '_json': rec,
217
+ 'project': self._project,
218
+ 'dataset': self._dataset})
219
+
220
+ # get all results
221
+ results = [j.result() for j in jobs]
222
+ # log errors
223
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
224
+ # return good jobs
225
+ recipes = miscellaneous.List([r[1] for r in results if r[0] is True])
226
+ return recipes
227
+
228
+ def _protected_get(self, recipe_id):
229
+ """
230
+ Same as get but with try-except to catch if error
231
+ :param recipe_id:
232
+ :return:
233
+ """
234
+ try:
235
+ recipe = self.get(recipe_id=recipe_id)
236
+ status = True
237
+ except Exception:
238
+ recipe = traceback.format_exc()
239
+ status = False
240
+ return status, recipe
241
+
242
+ @_api_reference.add(path='/recipes/{id}', method='get')
243
+ def get(self, recipe_id: str) -> entities.Recipe:
244
+ """
245
+ Get a Recipe object to use in your code.
246
+
247
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
248
+
249
+ :param str recipe_id: recipe id
250
+ :return: Recipe object
251
+ :rtype: dtlpy.entities.recipe.Recipe
252
+
253
+ **Example**:
254
+
255
+ .. code-block:: python
256
+
257
+ dataset.recipes.get(recipe_id='recipe_id')
258
+ """
259
+ success, response = self._client_api.gen_request(req_type='get',
260
+ path='/recipes/%s' % recipe_id)
261
+ if success:
262
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
263
+ _json=response.json(),
264
+ project=self._project,
265
+ dataset=self._dataset)
266
+ else:
267
+ logger.error('Unable to get info from recipe. Recipe_id id: {}'.format(recipe_id))
268
+ raise exceptions.PlatformException(response)
269
+
270
+ return recipe
271
+
272
+ def open_in_web(self,
273
+ recipe: entities.Recipe = None,
274
+ recipe_id: str = None):
275
+ """
276
+ Open the recipe in web platform.
277
+
278
+ **Prerequisites**: All users.
279
+
280
+ :param dtlpy.entities.recipe.Recipe recipe: recipe entity
281
+ :param str recipe_id: recipe id
282
+
283
+ **Example**:
284
+
285
+ .. code-block:: python
286
+
287
+ dataset.recipes.open_in_web(recipe_id='recipe_id')
288
+ """
289
+ if recipe is not None:
290
+ recipe.open_in_web()
291
+ elif recipe_id is not None:
292
+ self._client_api._open_in_web(url=self.platform_url + '/' + str(recipe_id))
293
+ else:
294
+ self._client_api._open_in_web(url=self.platform_url)
295
+
296
+ @_api_reference.add(path='/recipes/{id}', method='delete')
297
+ def delete(self, recipe_id: str, force: bool = False):
298
+ """
299
+ Delete recipe from platform.
300
+
301
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
302
+
303
+ :param str recipe_id: recipe id
304
+ :param bool force: force delete recipe
305
+ :return: True if success
306
+ :rtype: bool
307
+
308
+ **Example**:
309
+
310
+ .. code-block:: python
311
+
312
+ dataset.recipes.delete(recipe_id='recipe_id')
313
+ """
314
+ path = '/recipes/{}'.format(recipe_id)
315
+ if force:
316
+ path += '?force=true'
317
+ success, response = self._client_api.gen_request(req_type='delete',
318
+ path=path)
319
+ if not success:
320
+ raise exceptions.PlatformException(response)
321
+ logger.info('Recipe id {} deleted successfully'.format(recipe_id))
322
+ return True
323
+
324
+ @_api_reference.add(path='/recipes/{id}', method='patch')
325
+ def update(self, recipe: entities.Recipe, system_metadata=False) -> entities.Recipe:
326
+ """
327
+ Update recipe.
328
+
329
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
330
+
331
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
332
+ :param bool system_metadata: True, if you want to change metadata system
333
+ :return: Recipe object
334
+ :rtype: dtlpy.entities.recipe.Recipe
335
+
336
+ **Example**:
337
+
338
+ .. code-block:: python
339
+
340
+ dataset.recipes.update(recipe='recipe_entity')
341
+ """
342
+ url_path = '/recipes/%s' % recipe.id
343
+ if system_metadata:
344
+ url_path += '?system=true'
345
+ success, response = self._client_api.gen_request(req_type='patch',
346
+ path=url_path,
347
+ json_req=recipe.to_json())
348
+ if success:
349
+ return entities.Recipe.from_json(client_api=self._client_api, _json=response.json(), dataset=self._dataset)
350
+ else:
351
+ logger.error('Error while updating item:')
352
+ raise exceptions.PlatformException(response)
353
+
354
+ @_api_reference.add(path='/recipes/{id}/clone', method='post')
355
+ def clone(self,
356
+ recipe: entities.Recipe = None,
357
+ recipe_id: str = None,
358
+ shallow: bool = False):
359
+ """
360
+ Clone recipe.
361
+
362
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
363
+
364
+ :param dtlpy.entities.recipe.Recipe recipe: Recipe object
365
+ :param str recipe_id: Recipe id
366
+ :param bool shallow: If True, link to existing ontology, clones all ontologies that are linked to the recipe as well
367
+ :return: Cloned ontology object
368
+ :rtype: dtlpy.entities.recipe.Recipe
369
+
370
+ **Example**:
371
+
372
+ .. code-block:: python
373
+
374
+ dataset.recipes.clone(recipe_id='recipe_id')
375
+ """
376
+ if recipe is None and recipe_id is None:
377
+ raise exceptions.PlatformException('400', 'Must provide recipe or recipe_id')
378
+ if recipe_id is None:
379
+ if not isinstance(recipe, entities.Recipe):
380
+ raise exceptions.PlatformException('400', 'Recipe must me entities.Recipe type')
381
+ else:
382
+ recipe_id = recipe.id
383
+
384
+ payload = {'shallow': shallow}
385
+
386
+ success, response = self._client_api.gen_request(req_type='post',
387
+ path='/recipes/{}/clone'.format(recipe_id),
388
+ json_req=payload)
389
+ if success:
390
+ recipe = entities.Recipe.from_json(client_api=self._client_api,
391
+ _json=response.json())
392
+ else:
393
+ logger.error('Failed to clone Recipe')
394
+ raise exceptions.PlatformException(response)
395
+
396
+ assert isinstance(recipe, entities.Recipe)
397
+ logger.debug('Recipe has been cloned successfully. recipe id: {}'.format(recipe.id))
398
+
399
+ return recipe