dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,817 +1,817 @@
1
- import threading
2
- import logging
3
- import time
4
- from copy import deepcopy
5
-
6
- import numpy as np
7
-
8
- from .. import exceptions, entities, repositories, miscellaneous, _api_reference
9
- from ..services.api_client import ApiClient
10
-
11
- logger = logging.getLogger(name='dtlpy')
12
- MAX_SLEEP_TIME = 30
13
-
14
-
15
- class Executions:
16
- """
17
- Service Executions Repository
18
-
19
- The Executions class allows the users to manage executions (executions of services) and their properties.
20
- See our documentation for more information about `executions <https://developers.dataloop.ai/tutorials/faas/execution_control/chapter/>`_.
21
- """
22
-
23
- def __init__(self,
24
- client_api: ApiClient,
25
- service: entities.Service = None,
26
- project: entities.Project = None):
27
- self._client_api = client_api
28
- self._service = service
29
- self._project = project
30
-
31
- ############
32
- # entities #
33
- ############
34
- @property
35
- def service(self) -> entities.Service:
36
- if self._service is None:
37
- raise exceptions.PlatformException(
38
- error='2001',
39
- message='Missing "service". need to set a Service entity or use service.executions repository')
40
- assert isinstance(self._service, entities.Service)
41
- return self._service
42
-
43
- @service.setter
44
- def service(self, service: entities.Service):
45
- if not isinstance(service, entities.Service):
46
- raise ValueError('Must input a valid Service entity')
47
- self._service = service
48
-
49
- @property
50
- def project(self) -> entities.Project:
51
- if self._project is None:
52
- if self._service is not None:
53
- self._project = self._service._project
54
- if self._project is None:
55
- raise exceptions.PlatformException(
56
- error='2001',
57
- message='Missing "project". need to set a Project entity or use Project.executions repository')
58
- assert isinstance(self._project, entities.Project)
59
- return self._project
60
-
61
- @project.setter
62
- def project(self, project: entities.Project):
63
- if not isinstance(project, entities.Project):
64
- raise ValueError('Must input a valid Project entity')
65
- self._project = project
66
-
67
- def __get_from_entity(self, name, value):
68
- project_id = None
69
- try:
70
- from_dataset = False
71
- entity_obj = None
72
- for input_type in entities.FunctionIO.INPUT_TYPES:
73
- input_type = input_type.value
74
- entity = input_type.lower()
75
- param = '{}_id'.format(entity)
76
- if isinstance(value, dict):
77
- if param in value:
78
- repo = getattr(repositories, '{}s'.format(input_type))(client_api=self._client_api)
79
- entity_obj = repo.get(**{param: value[param]})
80
- if param in ['annotation_id', 'item_id']:
81
- from_dataset = True
82
- if param == 'item_id':
83
- entity_obj = entity_obj.dataset
84
- else:
85
- entity_obj = repositories.Datasets(client_api=self._client_api).get(
86
- dataset_id=entity_obj.dataset_id)
87
- elif param in ['dataset_id']:
88
- from_dataset = True
89
- break
90
- elif isinstance(value, str):
91
- if entity == name:
92
- repo = getattr(repositories, '{}s'.format(input_type))(client_api=self._client_api)
93
- entity_obj = repo.get(**{param: value})
94
- if name in ['annotation', 'item']:
95
- from_dataset = True
96
- if param == 'item_id':
97
- entity_obj = entity_obj.dataset
98
- else:
99
- entity_obj = repositories.Datasets(client_api=self._client_api).get(
100
- dataset_id=entity_obj.dataset_id)
101
- elif name in ['dataset']:
102
- from_dataset = True
103
- break
104
-
105
- if entity_obj is not None:
106
- if isinstance(entity_obj, entities.Project):
107
- project_id = entity_obj.id
108
- elif from_dataset:
109
- project_id = entity_obj.projects[0]
110
- else:
111
- project_id = entity_obj.project_id
112
- except Exception:
113
- project_id = None
114
-
115
- return project_id
116
-
117
- def __get_project_id(self, project_id=None, payload=None):
118
- if project_id is None:
119
- inputs = payload.get('input', dict())
120
- try:
121
- for key, val in inputs.items():
122
- project_id = self.__get_from_entity(name=key, value=val)
123
- if project_id is not None:
124
- break
125
- except Exception:
126
- project_id = None
127
-
128
- if project_id is None:
129
- # if still None - get from current repository
130
- if self._project is not None:
131
- project_id = self._project.id
132
- else:
133
- raise exceptions.PlatformException('400', 'Please provide project_id')
134
-
135
- return project_id
136
-
137
- ###########
138
- # methods #
139
- ###########
140
- @_api_reference.add(path='/executions/{serviceId}', method='post')
141
- def create(self,
142
- # executions info
143
- service_id: str = None,
144
- execution_input: list = None,
145
- function_name: str = None,
146
- # inputs info
147
- resource: entities.PackageInputType = None,
148
- item_id: str = None,
149
- dataset_id: str = None,
150
- annotation_id: str = None,
151
- project_id: str = None,
152
- # execution config
153
- sync: bool = False,
154
- stream_logs: bool = False,
155
- return_output: bool = False,
156
- # misc
157
- return_curl_only: bool = False,
158
- timeout: int = None) -> entities.Execution:
159
- """
160
- Execute a function on an existing service
161
-
162
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
163
-
164
- :param str service_id: service id to execute on
165
- :param List[FunctionIO] or dict execution_input: input dictionary or list of FunctionIO entities
166
- :param str function_name: function name to run
167
- :param str resource: input type.
168
- :param str item_id: optional - item id as input to function
169
- :param str dataset_id: optional - dataset id as input to function
170
- :param str annotation_id: optional - annotation id as input to function
171
- :param str project_id: resource's project
172
- :param bool sync: if true, wait for function to end
173
- :param bool stream_logs: prints logs of the new execution. only works with sync=True
174
- :param bool return_output: if True and sync is True - will return the output directly
175
- :param bool return_curl_only: return the cURL of the creation WITHOUT actually do it
176
- :param int timeout: int, seconds to wait until TimeoutError is raised. if <=0 - wait until done -
177
- by default wait take the service timeout
178
- :return: execution object
179
- :rtype: dtlpy.entities.execution.Execution
180
-
181
- **Example**:
182
-
183
- .. code-block:: python
184
-
185
- service.executions.create(function_name='function_name', item_id='item_id', project_id='project_id')
186
- """
187
- if service_id is None:
188
- if self._service is None:
189
- raise exceptions.PlatformException('400', 'Please provide service id')
190
- service_id = self._service.id
191
-
192
- if resource is None:
193
- if annotation_id is not None:
194
- resource = entities.PackageInputType.ANNOTATION
195
- elif item_id is not None:
196
- resource = entities.PackageInputType.ITEM
197
- elif dataset_id is not None:
198
- resource = entities.PackageInputType.DATASET
199
-
200
- # payload
201
- payload = dict()
202
- if execution_input is None:
203
- if resource is not None:
204
- inputs = {resource.lower(): {
205
- 'dataset_id': dataset_id}
206
- }
207
- if item_id is not None:
208
- inputs[resource.lower()]['item_id'] = item_id
209
- if annotation_id is not None:
210
- inputs[resource.lower()]['annotation_id'] = annotation_id
211
- payload['input'] = inputs
212
- else:
213
- if isinstance(execution_input, dict):
214
- payload['input'] = execution_input
215
- else:
216
- if not isinstance(execution_input, list):
217
- execution_input = [execution_input]
218
- if len(execution_input) > 0 and isinstance(execution_input[0], entities.FunctionIO):
219
- payload['input'] = dict()
220
- for single_input in execution_input:
221
- payload['input'].update(single_input.to_json(resource='execution'))
222
- else:
223
- raise exceptions.PlatformException('400', 'Unknown input type')
224
-
225
- payload['projectId'] = self.__get_project_id(project_id=project_id, payload=payload)
226
-
227
- if function_name is not None:
228
- payload['functionName'] = function_name
229
- else:
230
- payload['functionName'] = entities.package_defaults.DEFAULT_PACKAGE_FUNCTION_NAME
231
-
232
- # request url
233
- url_path = '/executions/{service_id}'.format(service_id=service_id)
234
-
235
- if return_curl_only:
236
- curl = self._client_api.export_curl_request(req_type='post',
237
- path=url_path,
238
- json_req=payload)
239
- logger.warning(msg='Execution was NOT created. Exporting cURL only.')
240
- return curl
241
- success, response = self._client_api.gen_request(req_type='post',
242
- path=url_path,
243
- json_req=payload)
244
- # exception handling
245
- if not success:
246
- raise exceptions.PlatformException(response)
247
-
248
- # return entity
249
- execution = entities.Execution.from_json(_json=response.json(),
250
- client_api=self._client_api,
251
- project=self._project,
252
- service=self._service)
253
-
254
- if sync and not return_output and not stream_logs:
255
- execution = self.wait(execution_id=execution.id, timeout=timeout)
256
-
257
- if sync and (stream_logs or return_output):
258
- thread = None
259
- if stream_logs:
260
- thread = threading.Thread(target=self.logs,
261
- kwargs={'execution_id': execution.id,
262
- 'follow': True,
263
- 'until_completed': True})
264
- thread.setDaemon(True)
265
- thread.start()
266
- execution = self.get(execution_id=execution.id,
267
- sync=True)
268
- # stream logs
269
- if stream_logs and thread is not None:
270
- thread.join()
271
- if sync and return_output:
272
- return execution.output
273
- return execution
274
-
275
- @_api_reference.add(path='/executions/{serviceId}', method='post')
276
- def create_batch(self,
277
- service_id: str,
278
- filters,
279
- function_name: str = None,
280
- execution_inputs: list = None,
281
- wait=True
282
- ):
283
- """
284
- Execute a function on an existing service
285
-
286
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
287
-
288
- :param str service_id: service id to execute on
289
- :param filters: Filters entity for a filtering before execute
290
- :param str function_name: function name to run
291
- :param List[FunctionIO] or dict execution_inputs: input dictionary or list of FunctionIO entities, that represent the extra inputs of the function
292
- :param bool wait: wait until create task finish
293
- :return: execution object
294
- :rtype: dtlpy.entities.execution.Execution
295
-
296
- **Example**:
297
-
298
- .. code-block:: python
299
-
300
- command = service.executions.create_batch(
301
- execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
302
- filters=dl.Filters(field='dir', values='/test'),
303
- function_name='run')
304
- """
305
- if service_id is None:
306
- if self._service is None:
307
- raise exceptions.PlatformException('400', 'Please provide service id')
308
- service_id = self._service.id
309
-
310
- if filters is None:
311
- raise exceptions.PlatformException('400', 'Please provide filter')
312
-
313
- if execution_inputs is None:
314
- execution_inputs = dict()
315
-
316
- if isinstance(execution_inputs, dict):
317
- extra_inputs = execution_inputs
318
- else:
319
- if not isinstance(execution_inputs, list):
320
- execution_inputs = [execution_inputs]
321
- if len(execution_inputs) > 0 and isinstance(execution_inputs[0], entities.FunctionIO):
322
- extra_inputs = dict()
323
- for single_input in execution_inputs:
324
- extra_inputs.update(single_input.to_json(resource='execution'))
325
- else:
326
- raise exceptions.PlatformException('400', 'Unknown input type')
327
-
328
- # payload
329
- payload = dict()
330
- payload['batch'] = dict()
331
- payload['batch']['query'] = filters.prepare()
332
- payload['batch']['args'] = extra_inputs
333
-
334
- if function_name is not None:
335
- payload['functionName'] = function_name
336
- else:
337
- payload['functionName'] = entities.package_defaults.DEFAULT_PACKAGE_FUNCTION_NAME
338
-
339
- # request url
340
- url_path = '/executions/{service_id}'.format(service_id=service_id)
341
-
342
- success, response = self._client_api.gen_request(req_type='post',
343
- path=url_path,
344
- json_req=payload)
345
- # exception handling
346
- if not success:
347
- raise exceptions.PlatformException(response)
348
-
349
- response_json = response.json()
350
- command = entities.Command.from_json(_json=response_json,
351
- client_api=self._client_api)
352
- if wait:
353
- command = command.wait(timeout=0)
354
- return command
355
-
356
- @_api_reference.add(path='/executions/rerun', method='post')
357
- def rerun_batch(self,
358
- filters,
359
- service_id: str = None,
360
- wait=True
361
- ):
362
- """
363
- rerun a executions on an existing service
364
-
365
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a Filter.
366
-
367
- :param filters: Filters entity for a filtering before rerun
368
- :param str service_id: service id to rerun on
369
- :param bool wait: wait until create task finish
370
- :return: rerun command
371
- :rtype: dtlpy.entities.command.Command
372
-
373
- **Example**:
374
-
375
- .. code-block:: python
376
-
377
- command = service.executions.rerun_batch(
378
- filters=dl.Filters(field='id', values=['executionId'], operator=dl.FiltersOperations.IN, resource=dl.FiltersResource.EXECUTION))
379
- """
380
- url_path = '/executions/rerun'
381
-
382
- if filters is None:
383
- raise exceptions.PlatformException('400', 'Please provide filter')
384
-
385
- if filters.resource != entities.FiltersResource.EXECUTION:
386
- raise exceptions.PlatformException(
387
- error='400',
388
- message='Filters resource must to be FiltersResource.EXECUTION. Got: {!r}'.format(filters.resource))
389
-
390
- if service_id is not None and not filters.has_field('serviceId'):
391
- filters = deepcopy(filters)
392
- filters.add(field='serviceId', values=service_id, method=entities.FiltersMethod.AND)
393
-
394
- success, response = self._client_api.gen_request(req_type='post',
395
- path=url_path,
396
- json_req={'query': filters.prepare()['filter']})
397
- # exception handling
398
- if not success:
399
- raise exceptions.PlatformException(response)
400
-
401
- response_json = response.json()
402
- command = entities.Command.from_json(_json=response_json,
403
- client_api=self._client_api)
404
- if wait:
405
- command = command.wait(timeout=0)
406
- return command
407
-
408
- def _list(self, filters: entities.Filters):
409
- """
410
- List service executions
411
-
412
- :param dtlpy.entities.filters.Filters filters: dl.Filters entity to filters items
413
- :return:
414
- """
415
- url = '/query/faas'
416
-
417
- # request
418
- success, response = self._client_api.gen_request(req_type='POST',
419
- path=url,
420
- json_req=filters.prepare())
421
- if not success:
422
- raise exceptions.PlatformException(response)
423
-
424
- return response.json()
425
-
426
- @_api_reference.add(path='/query/faas', method='post')
427
- def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
428
- """
429
- List service executions
430
-
431
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
432
-
433
- :param dtlpy.entities.filters.Filters filters: dl.Filters entity to filters items
434
- :return: Paged entity
435
- :rtype: dtlpy.entities.paged_entities.PagedEntities
436
-
437
- **Example**:
438
-
439
- .. code-block:: python
440
-
441
- service.executions.list()
442
- """
443
- # default filtersf
444
- if filters is None:
445
- filters = entities.Filters(resource=entities.FiltersResource.EXECUTION)
446
- # assert type filters
447
- elif not isinstance(filters, entities.Filters):
448
- raise exceptions.PlatformException(
449
- error='400',
450
- message='Unknown filters type: {!r}'.format(type(filters)))
451
- if filters.resource != entities.FiltersResource.EXECUTION:
452
- raise exceptions.PlatformException(
453
- error='400',
454
- message='Filters resource must to be FiltersResource.EXECUTION. Got: {!r}'.format(filters.resource))
455
- if self._project is not None:
456
- filters.add(field='projectId', values=self._project.id)
457
- if self._service is not None:
458
- filters.add(field='serviceId', values=self._service.id)
459
-
460
- paged = entities.PagedEntities(items_repository=self,
461
- filters=filters,
462
- page_offset=filters.page,
463
- page_size=filters.page_size,
464
- client_api=self._client_api)
465
- paged.get_page()
466
- return paged
467
-
468
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Execution]:
469
- pool = self._client_api.thread_pools(pool_name='entity.create')
470
- jobs = [None for _ in range(len(response_items))]
471
- # return execution list
472
- for i_item, item in enumerate(response_items):
473
- jobs[i_item] = pool.submit(entities.Execution._protected_from_json,
474
- **{'client_api': self._client_api,
475
- '_json': item,
476
- 'project': self._project,
477
- 'service': self._service})
478
-
479
- # get results
480
- results = [j.result() for j in jobs]
481
- # log errors
482
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
483
- # return good jobs
484
- return miscellaneous.List([r[1] for r in results if r[0] is True])
485
-
486
- @_api_reference.add(path='/executions/{id}', method='get')
487
- def get(self,
488
- execution_id: str = None,
489
- sync: bool = False
490
- ) -> entities.Execution:
491
- """
492
- Get Service execution object
493
-
494
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
495
-
496
- :param str execution_id: execution id
497
- :param bool sync: if true, wait for the execution to finish
498
- :return: Service execution object
499
- :rtype: dtlpy.entities.execution.Execution
500
-
501
- **Example**:
502
-
503
- .. code-block:: python
504
-
505
- service.executions.get(execution_id='execution_id')
506
- """
507
- url_path = "/executions/{}".format(execution_id)
508
- if sync:
509
- return self.wait(execution_id=execution_id)
510
-
511
- success, response = self._client_api.gen_request(req_type="get",
512
- path=url_path)
513
-
514
- # exception handling
515
- if not success:
516
- raise exceptions.PlatformException(response)
517
-
518
- # return entity
519
- return entities.Execution.from_json(client_api=self._client_api,
520
- _json=response.json(),
521
- project=self._project,
522
- service=self._service)
523
-
524
- def logs(self,
525
- execution_id: str,
526
- follow: bool = True,
527
- until_completed: bool = True):
528
- """
529
- executions logs
530
-
531
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
532
-
533
- :param str execution_id: execution id
534
- :param bool follow: if true, keep stream future logs
535
- :param bool until_completed: if true, wait until completed
536
- :return: executions logs
537
-
538
- **Example**:
539
-
540
- .. code-block:: python
541
-
542
- service.executions.logs(execution_id='execution_id')
543
- """
544
- return self.service.log(execution_id=execution_id,
545
- follow=follow,
546
- until_completed=until_completed,
547
- view=True)
548
-
549
- def increment(self, execution: entities.Execution):
550
- """
551
- Increment the number of attempts that an execution is allowed to attempt to run a service that is not responding.
552
-
553
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
554
-
555
- :param dtlpy.entities.execution.Execution execution:
556
- :return: int
557
- :rtype: int
558
-
559
- **Example**:
560
-
561
- .. code-block:: python
562
-
563
- service.executions.increment(execution='execution_entity')
564
- """
565
- # request
566
- success, response = self._client_api.gen_request(
567
- req_type='post',
568
- path='/executions/{}/attempts'.format(execution.id)
569
- )
570
-
571
- # exception handling
572
- if not success:
573
- raise exceptions.PlatformException(response)
574
-
575
- # exception handling
576
- if not success:
577
- raise exceptions.PlatformException(response)
578
- else:
579
- return response.json()
580
-
581
- @_api_reference.add(path='/executions/{executionId}/rerun', method='post')
582
- def rerun(self, execution: entities.Execution, sync: bool = False):
583
- """
584
- Rerun execution
585
-
586
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
587
-
588
- :param dtlpy.entities.execution.Execution execution:
589
- :param bool sync: wait for the execution to finish
590
- :return: Execution object
591
- :rtype: dtlpy.entities.execution.Execution
592
-
593
- **Example**:
594
-
595
- .. code-block:: python
596
-
597
- service.executions.rerun(execution='execution_entity')
598
- """
599
-
600
- url_path = "/executions/{}/rerun".format(execution.id)
601
- # request
602
- success, response = self._client_api.gen_request(req_type='post',
603
- path=url_path)
604
-
605
- # exception handling
606
- if not success:
607
- raise exceptions.PlatformException(response)
608
- else:
609
- execution = entities.Execution.from_json(
610
- client_api=self._client_api,
611
- _json=response.json(),
612
- project=self._project,
613
- service=self._service
614
- )
615
- if sync:
616
- execution = self.wait(execution_id=execution.id)
617
- return execution
618
-
619
- def wait(self,
620
- execution_id: str = None,
621
- execution: entities.Execution = None,
622
- timeout: int = None,
623
- backoff_factor=1):
624
- """
625
- Get Service execution object.
626
-
627
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
628
-
629
- :param str execution_id: execution id
630
- :param str execution: dl.Execution, optional. must input one of execution or execution_id
631
- :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
632
- :param float backoff_factor: A backoff factor to apply between attempts after the second try
633
- :return: Service execution object
634
- :rtype: dtlpy.entities.execution.Execution
635
-
636
- **Example**:
637
-
638
- .. code-block:: python
639
-
640
- service.executions.wait(execution_id='execution_id')
641
- """
642
- if execution is None:
643
- if execution_id is None:
644
- raise ValueError('Must input at least one: [execution, execution_id]')
645
- else:
646
- execution = self.get(execution_id=execution_id)
647
- elapsed = 0
648
- start = int(time.time())
649
- if timeout is None or timeout <= 0:
650
- timeout = np.inf
651
-
652
- num_tries = 1
653
- while elapsed < timeout:
654
- execution = self.get(execution_id=execution.id)
655
- if not execution.in_progress():
656
- break
657
- elapsed = time.time() - start
658
- if elapsed >= timeout:
659
- raise TimeoutError(
660
- f"execution wait() got timeout. id: {execution.id!r}, status: {execution.latest_status}")
661
- sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
662
- num_tries += 1
663
- logger.debug("Execution {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(execution.id,
664
- elapsed,
665
- sleep_time))
666
- time.sleep(sleep_time)
667
-
668
- return execution
669
-
670
- @_api_reference.add(path='/executions/{id}/terminate', method='post')
671
- def terminate(self, execution: entities.Execution):
672
- """
673
- Terminate Execution
674
-
675
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
676
-
677
- :param dtlpy.entities.execution.Execution execution:
678
- :return: execution object
679
- :rtype: dtlpy.entities.execution.Execution
680
-
681
- **Example**:
682
-
683
- .. code-block:: python
684
-
685
- service.executions.terminate(execution='execution_entity')
686
- """
687
- # request
688
- success, response = self._client_api.gen_request(req_type='post',
689
- path='/executions/{}/terminate'.format(execution.id))
690
-
691
- # exception handling
692
- if not success:
693
- raise exceptions.PlatformException(response)
694
- else:
695
- return entities.Execution.from_json(_json=response.json(),
696
- service=self._service,
697
- project=self._project,
698
- client_api=self._client_api)
699
-
700
- @_api_reference.add(path='/executions/{id}', method='patch')
701
- def update(self, execution: entities.Execution) -> entities.Execution:
702
- """
703
- Update execution changes to platform
704
-
705
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
706
-
707
- :param dtlpy.entities.execution.Execution execution: execution entity
708
- :return: Service execution object
709
- :rtype: dtlpy.entities.execution.Execution
710
-
711
- **Example**:
712
-
713
- .. code-block:: python
714
-
715
- service.executions.update(execution='execution_entity')
716
- """
717
- # payload
718
- payload = execution.to_json()
719
-
720
- # request
721
- success, response = self._client_api.gen_request(req_type='patch',
722
- path='/executions/{}'.format(execution.id),
723
- json_req=payload)
724
-
725
- # exception handling
726
- if not success:
727
- raise exceptions.PlatformException(response)
728
-
729
- # return entity
730
- if self._project is not None:
731
- project = self._project
732
- else:
733
- project = execution._project
734
-
735
- # return
736
- if self._service is not None:
737
- service = self._service
738
- else:
739
- service = execution._service
740
-
741
- return entities.Execution.from_json(_json=response.json(),
742
- service=service,
743
- project=self._project,
744
- client_api=self._client_api)
745
-
746
- def progress_update(
747
- self,
748
- execution_id: str,
749
- status: entities.ExecutionStatus = None,
750
- percent_complete: int = None,
751
- message: str = None,
752
- output: str = None,
753
- service_version: str = None
754
- ):
755
- """
756
- Update Execution Progress.
757
-
758
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
759
-
760
- :param str execution_id: execution id
761
- :param str status: ExecutionStatus
762
- :param int percent_complete: percent work done
763
- :param str message: message
764
- :param str output: the output of the execution
765
- :param str service_version: service version
766
- :return: Service execution object
767
- :rtype: dtlpy.entities.execution.Execution
768
-
769
- **Example**:
770
-
771
- .. code-block:: python
772
-
773
- service.executions.progress_update(execution_id='execution_id', status='complete', percent_complete=100)
774
- """
775
- # create payload
776
- payload = dict()
777
-
778
- if status is not None:
779
- payload['status'] = status
780
- else:
781
- if percent_complete is not None and isinstance(percent_complete, int):
782
- if percent_complete < 100:
783
- payload['status'] = 'inProgress'
784
- else:
785
- payload['status'] = 'completed'
786
- elif output is not None:
787
- payload['status'] = 'completed'
788
- else:
789
- payload['status'] = 'inProgress'
790
-
791
- if percent_complete is not None:
792
- payload['percentComplete'] = percent_complete
793
-
794
- if message is not None:
795
- payload['message'] = message
796
-
797
- if output is not None:
798
- payload['output'] = output
799
-
800
- if service_version is not None:
801
- payload['serviceVersion'] = service_version
802
-
803
- # request
804
- success, response = self._client_api.gen_request(
805
- req_type="post",
806
- path="/executions/{}/progress".format(execution_id),
807
- json_req=payload
808
- )
809
-
810
- # exception handling
811
- if success:
812
- return entities.Execution.from_json(_json=response.json(),
813
- client_api=self._client_api,
814
- project=self._project,
815
- service=self._service)
816
- else:
817
- raise exceptions.PlatformException(response)
1
+ import threading
2
+ import logging
3
+ import time
4
+ from copy import deepcopy
5
+
6
+ import numpy as np
7
+
8
+ from .. import exceptions, entities, repositories, miscellaneous, _api_reference
9
+ from ..services.api_client import ApiClient
10
+
11
+ logger = logging.getLogger(name='dtlpy')
12
+ MAX_SLEEP_TIME = 30
13
+
14
+
15
+ class Executions:
16
+ """
17
+ Service Executions Repository
18
+
19
+ The Executions class allows the users to manage executions (executions of services) and their properties.
20
+ See our documentation for more information about `executions <https://developers.dataloop.ai/tutorials/faas/execution_control/chapter/>`_.
21
+ """
22
+
23
+ def __init__(self,
24
+ client_api: ApiClient,
25
+ service: entities.Service = None,
26
+ project: entities.Project = None):
27
+ self._client_api = client_api
28
+ self._service = service
29
+ self._project = project
30
+
31
+ ############
32
+ # entities #
33
+ ############
34
+ @property
35
+ def service(self) -> entities.Service:
36
+ if self._service is None:
37
+ raise exceptions.PlatformException(
38
+ error='2001',
39
+ message='Missing "service". need to set a Service entity or use service.executions repository')
40
+ assert isinstance(self._service, entities.Service)
41
+ return self._service
42
+
43
+ @service.setter
44
+ def service(self, service: entities.Service):
45
+ if not isinstance(service, entities.Service):
46
+ raise ValueError('Must input a valid Service entity')
47
+ self._service = service
48
+
49
+ @property
50
+ def project(self) -> entities.Project:
51
+ if self._project is None:
52
+ if self._service is not None:
53
+ self._project = self._service._project
54
+ if self._project is None:
55
+ raise exceptions.PlatformException(
56
+ error='2001',
57
+ message='Missing "project". need to set a Project entity or use Project.executions repository')
58
+ assert isinstance(self._project, entities.Project)
59
+ return self._project
60
+
61
+ @project.setter
62
+ def project(self, project: entities.Project):
63
+ if not isinstance(project, entities.Project):
64
+ raise ValueError('Must input a valid Project entity')
65
+ self._project = project
66
+
67
+ def __get_from_entity(self, name, value):
68
+ project_id = None
69
+ try:
70
+ from_dataset = False
71
+ entity_obj = None
72
+ for input_type in entities.FunctionIO.INPUT_TYPES:
73
+ input_type = input_type.value
74
+ entity = input_type.lower()
75
+ param = '{}_id'.format(entity)
76
+ if isinstance(value, dict):
77
+ if param in value:
78
+ repo = getattr(repositories, '{}s'.format(input_type))(client_api=self._client_api)
79
+ entity_obj = repo.get(**{param: value[param]})
80
+ if param in ['annotation_id', 'item_id']:
81
+ from_dataset = True
82
+ if param == 'item_id':
83
+ entity_obj = entity_obj.dataset
84
+ else:
85
+ entity_obj = repositories.Datasets(client_api=self._client_api).get(
86
+ dataset_id=entity_obj.dataset_id)
87
+ elif param in ['dataset_id']:
88
+ from_dataset = True
89
+ break
90
+ elif isinstance(value, str):
91
+ if entity == name:
92
+ repo = getattr(repositories, '{}s'.format(input_type))(client_api=self._client_api)
93
+ entity_obj = repo.get(**{param: value})
94
+ if name in ['annotation', 'item']:
95
+ from_dataset = True
96
+ if param == 'item_id':
97
+ entity_obj = entity_obj.dataset
98
+ else:
99
+ entity_obj = repositories.Datasets(client_api=self._client_api).get(
100
+ dataset_id=entity_obj.dataset_id)
101
+ elif name in ['dataset']:
102
+ from_dataset = True
103
+ break
104
+
105
+ if entity_obj is not None:
106
+ if isinstance(entity_obj, entities.Project):
107
+ project_id = entity_obj.id
108
+ elif from_dataset:
109
+ project_id = entity_obj.projects[0]
110
+ else:
111
+ project_id = entity_obj.project_id
112
+ except Exception:
113
+ project_id = None
114
+
115
+ return project_id
116
+
117
+ def __get_project_id(self, project_id=None, payload=None):
118
+ if project_id is None:
119
+ inputs = payload.get('input', dict())
120
+ try:
121
+ for key, val in inputs.items():
122
+ project_id = self.__get_from_entity(name=key, value=val)
123
+ if project_id is not None:
124
+ break
125
+ except Exception:
126
+ project_id = None
127
+
128
+ if project_id is None:
129
+ # if still None - get from current repository
130
+ if self._project is not None:
131
+ project_id = self._project.id
132
+ else:
133
+ raise exceptions.PlatformException('400', 'Please provide project_id')
134
+
135
+ return project_id
136
+
137
+ ###########
138
+ # methods #
139
+ ###########
140
+ @_api_reference.add(path='/executions/{serviceId}', method='post')
141
+ def create(self,
142
+ # executions info
143
+ service_id: str = None,
144
+ execution_input: list = None,
145
+ function_name: str = None,
146
+ # inputs info
147
+ resource: entities.PackageInputType = None,
148
+ item_id: str = None,
149
+ dataset_id: str = None,
150
+ annotation_id: str = None,
151
+ project_id: str = None,
152
+ # execution config
153
+ sync: bool = False,
154
+ stream_logs: bool = False,
155
+ return_output: bool = False,
156
+ # misc
157
+ return_curl_only: bool = False,
158
+ timeout: int = None) -> entities.Execution:
159
+ """
160
+ Execute a function on an existing service
161
+
162
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
163
+
164
+ :param str service_id: service id to execute on
165
+ :param List[FunctionIO] or dict execution_input: input dictionary or list of FunctionIO entities
166
+ :param str function_name: function name to run
167
+ :param str resource: input type.
168
+ :param str item_id: optional - item id as input to function
169
+ :param str dataset_id: optional - dataset id as input to function
170
+ :param str annotation_id: optional - annotation id as input to function
171
+ :param str project_id: resource's project
172
+ :param bool sync: if true, wait for function to end
173
+ :param bool stream_logs: prints logs of the new execution. only works with sync=True
174
+ :param bool return_output: if True and sync is True - will return the output directly
175
+ :param bool return_curl_only: return the cURL of the creation WITHOUT actually do it
176
+ :param int timeout: int, seconds to wait until TimeoutError is raised. if <=0 - wait until done -
177
+ by default wait take the service timeout
178
+ :return: execution object
179
+ :rtype: dtlpy.entities.execution.Execution
180
+
181
+ **Example**:
182
+
183
+ .. code-block:: python
184
+
185
+ service.executions.create(function_name='function_name', item_id='item_id', project_id='project_id')
186
+ """
187
+ if service_id is None:
188
+ if self._service is None:
189
+ raise exceptions.PlatformException('400', 'Please provide service id')
190
+ service_id = self._service.id
191
+
192
+ if resource is None:
193
+ if annotation_id is not None:
194
+ resource = entities.PackageInputType.ANNOTATION
195
+ elif item_id is not None:
196
+ resource = entities.PackageInputType.ITEM
197
+ elif dataset_id is not None:
198
+ resource = entities.PackageInputType.DATASET
199
+
200
+ # payload
201
+ payload = dict()
202
+ if execution_input is None:
203
+ if resource is not None:
204
+ inputs = {resource.lower(): {
205
+ 'dataset_id': dataset_id}
206
+ }
207
+ if item_id is not None:
208
+ inputs[resource.lower()]['item_id'] = item_id
209
+ if annotation_id is not None:
210
+ inputs[resource.lower()]['annotation_id'] = annotation_id
211
+ payload['input'] = inputs
212
+ else:
213
+ if isinstance(execution_input, dict):
214
+ payload['input'] = execution_input
215
+ else:
216
+ if not isinstance(execution_input, list):
217
+ execution_input = [execution_input]
218
+ if len(execution_input) > 0 and isinstance(execution_input[0], entities.FunctionIO):
219
+ payload['input'] = dict()
220
+ for single_input in execution_input:
221
+ payload['input'].update(single_input.to_json(resource='execution'))
222
+ else:
223
+ raise exceptions.PlatformException('400', 'Unknown input type')
224
+
225
+ payload['projectId'] = self.__get_project_id(project_id=project_id, payload=payload)
226
+
227
+ if function_name is not None:
228
+ payload['functionName'] = function_name
229
+ else:
230
+ payload['functionName'] = entities.package_defaults.DEFAULT_PACKAGE_FUNCTION_NAME
231
+
232
+ # request url
233
+ url_path = '/executions/{service_id}'.format(service_id=service_id)
234
+
235
+ if return_curl_only:
236
+ curl = self._client_api.export_curl_request(req_type='post',
237
+ path=url_path,
238
+ json_req=payload)
239
+ logger.warning(msg='Execution was NOT created. Exporting cURL only.')
240
+ return curl
241
+ success, response = self._client_api.gen_request(req_type='post',
242
+ path=url_path,
243
+ json_req=payload)
244
+ # exception handling
245
+ if not success:
246
+ raise exceptions.PlatformException(response)
247
+
248
+ # return entity
249
+ execution = entities.Execution.from_json(_json=response.json(),
250
+ client_api=self._client_api,
251
+ project=self._project,
252
+ service=self._service)
253
+
254
+ if sync and not return_output and not stream_logs:
255
+ execution = self.wait(execution_id=execution.id, timeout=timeout)
256
+
257
+ if sync and (stream_logs or return_output):
258
+ thread = None
259
+ if stream_logs:
260
+ thread = threading.Thread(target=self.logs,
261
+ kwargs={'execution_id': execution.id,
262
+ 'follow': True,
263
+ 'until_completed': True})
264
+ thread.setDaemon(True)
265
+ thread.start()
266
+ execution = self.get(execution_id=execution.id,
267
+ sync=True)
268
+ # stream logs
269
+ if stream_logs and thread is not None:
270
+ thread.join()
271
+ if sync and return_output:
272
+ return execution.output
273
+ return execution
274
+
275
+ @_api_reference.add(path='/executions/{serviceId}', method='post')
276
+ def create_batch(self,
277
+ service_id: str,
278
+ filters,
279
+ function_name: str = None,
280
+ execution_inputs: list = None,
281
+ wait=True
282
+ ):
283
+ """
284
+ Execute a function on an existing service
285
+
286
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
287
+
288
+ :param str service_id: service id to execute on
289
+ :param filters: Filters entity for a filtering before execute
290
+ :param str function_name: function name to run
291
+ :param List[FunctionIO] or dict execution_inputs: input dictionary or list of FunctionIO entities, that represent the extra inputs of the function
292
+ :param bool wait: wait until create task finish
293
+ :return: execution object
294
+ :rtype: dtlpy.entities.execution.Execution
295
+
296
+ **Example**:
297
+
298
+ .. code-block:: python
299
+
300
+ command = service.executions.create_batch(
301
+ execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
302
+ filters=dl.Filters(field='dir', values='/test'),
303
+ function_name='run')
304
+ """
305
+ if service_id is None:
306
+ if self._service is None:
307
+ raise exceptions.PlatformException('400', 'Please provide service id')
308
+ service_id = self._service.id
309
+
310
+ if filters is None:
311
+ raise exceptions.PlatformException('400', 'Please provide filter')
312
+
313
+ if execution_inputs is None:
314
+ execution_inputs = dict()
315
+
316
+ if isinstance(execution_inputs, dict):
317
+ extra_inputs = execution_inputs
318
+ else:
319
+ if not isinstance(execution_inputs, list):
320
+ execution_inputs = [execution_inputs]
321
+ if len(execution_inputs) > 0 and isinstance(execution_inputs[0], entities.FunctionIO):
322
+ extra_inputs = dict()
323
+ for single_input in execution_inputs:
324
+ extra_inputs.update(single_input.to_json(resource='execution'))
325
+ else:
326
+ raise exceptions.PlatformException('400', 'Unknown input type')
327
+
328
+ # payload
329
+ payload = dict()
330
+ payload['batch'] = dict()
331
+ payload['batch']['query'] = filters.prepare()
332
+ payload['batch']['args'] = extra_inputs
333
+
334
+ if function_name is not None:
335
+ payload['functionName'] = function_name
336
+ else:
337
+ payload['functionName'] = entities.package_defaults.DEFAULT_PACKAGE_FUNCTION_NAME
338
+
339
+ # request url
340
+ url_path = '/executions/{service_id}'.format(service_id=service_id)
341
+
342
+ success, response = self._client_api.gen_request(req_type='post',
343
+ path=url_path,
344
+ json_req=payload)
345
+ # exception handling
346
+ if not success:
347
+ raise exceptions.PlatformException(response)
348
+
349
+ response_json = response.json()
350
+ command = entities.Command.from_json(_json=response_json,
351
+ client_api=self._client_api)
352
+ if wait:
353
+ command = command.wait(timeout=0)
354
+ return command
355
+
356
+ @_api_reference.add(path='/executions/rerun', method='post')
357
+ def rerun_batch(self,
358
+ filters,
359
+ service_id: str = None,
360
+ wait=True
361
+ ):
362
+ """
363
+ rerun a executions on an existing service
364
+
365
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a Filter.
366
+
367
+ :param filters: Filters entity for a filtering before rerun
368
+ :param str service_id: service id to rerun on
369
+ :param bool wait: wait until create task finish
370
+ :return: rerun command
371
+ :rtype: dtlpy.entities.command.Command
372
+
373
+ **Example**:
374
+
375
+ .. code-block:: python
376
+
377
+ command = service.executions.rerun_batch(
378
+ filters=dl.Filters(field='id', values=['executionId'], operator=dl.FiltersOperations.IN, resource=dl.FiltersResource.EXECUTION))
379
+ """
380
+ url_path = '/executions/rerun'
381
+
382
+ if filters is None:
383
+ raise exceptions.PlatformException('400', 'Please provide filter')
384
+
385
+ if filters.resource != entities.FiltersResource.EXECUTION:
386
+ raise exceptions.PlatformException(
387
+ error='400',
388
+ message='Filters resource must to be FiltersResource.EXECUTION. Got: {!r}'.format(filters.resource))
389
+
390
+ if service_id is not None and not filters.has_field('serviceId'):
391
+ filters = deepcopy(filters)
392
+ filters.add(field='serviceId', values=service_id, method=entities.FiltersMethod.AND)
393
+
394
+ success, response = self._client_api.gen_request(req_type='post',
395
+ path=url_path,
396
+ json_req={'query': filters.prepare()['filter']})
397
+ # exception handling
398
+ if not success:
399
+ raise exceptions.PlatformException(response)
400
+
401
+ response_json = response.json()
402
+ command = entities.Command.from_json(_json=response_json,
403
+ client_api=self._client_api)
404
+ if wait:
405
+ command = command.wait(timeout=0)
406
+ return command
407
+
408
+ def _list(self, filters: entities.Filters):
409
+ """
410
+ List service executions
411
+
412
+ :param dtlpy.entities.filters.Filters filters: dl.Filters entity to filters items
413
+ :return:
414
+ """
415
+ url = '/query/faas'
416
+
417
+ # request
418
+ success, response = self._client_api.gen_request(req_type='POST',
419
+ path=url,
420
+ json_req=filters.prepare())
421
+ if not success:
422
+ raise exceptions.PlatformException(response)
423
+
424
+ return response.json()
425
+
426
+ @_api_reference.add(path='/query/faas', method='post')
427
+ def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
428
+ """
429
+ List service executions
430
+
431
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
432
+
433
+ :param dtlpy.entities.filters.Filters filters: dl.Filters entity to filters items
434
+ :return: Paged entity
435
+ :rtype: dtlpy.entities.paged_entities.PagedEntities
436
+
437
+ **Example**:
438
+
439
+ .. code-block:: python
440
+
441
+ service.executions.list()
442
+ """
443
+ # default filtersf
444
+ if filters is None:
445
+ filters = entities.Filters(resource=entities.FiltersResource.EXECUTION)
446
+ # assert type filters
447
+ elif not isinstance(filters, entities.Filters):
448
+ raise exceptions.PlatformException(
449
+ error='400',
450
+ message='Unknown filters type: {!r}'.format(type(filters)))
451
+ if filters.resource != entities.FiltersResource.EXECUTION:
452
+ raise exceptions.PlatformException(
453
+ error='400',
454
+ message='Filters resource must to be FiltersResource.EXECUTION. Got: {!r}'.format(filters.resource))
455
+ if self._project is not None:
456
+ filters.add(field='projectId', values=self._project.id)
457
+ if self._service is not None:
458
+ filters.add(field='serviceId', values=self._service.id)
459
+
460
+ paged = entities.PagedEntities(items_repository=self,
461
+ filters=filters,
462
+ page_offset=filters.page,
463
+ page_size=filters.page_size,
464
+ client_api=self._client_api)
465
+ paged.get_page()
466
+ return paged
467
+
468
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Execution]:
469
+ pool = self._client_api.thread_pools(pool_name='entity.create')
470
+ jobs = [None for _ in range(len(response_items))]
471
+ # return execution list
472
+ for i_item, item in enumerate(response_items):
473
+ jobs[i_item] = pool.submit(entities.Execution._protected_from_json,
474
+ **{'client_api': self._client_api,
475
+ '_json': item,
476
+ 'project': self._project,
477
+ 'service': self._service})
478
+
479
+ # get results
480
+ results = [j.result() for j in jobs]
481
+ # log errors
482
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
483
+ # return good jobs
484
+ return miscellaneous.List([r[1] for r in results if r[0] is True])
485
+
486
+ @_api_reference.add(path='/executions/{id}', method='get')
487
+ def get(self,
488
+ execution_id: str = None,
489
+ sync: bool = False
490
+ ) -> entities.Execution:
491
+ """
492
+ Get Service execution object
493
+
494
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
495
+
496
+ :param str execution_id: execution id
497
+ :param bool sync: if true, wait for the execution to finish
498
+ :return: Service execution object
499
+ :rtype: dtlpy.entities.execution.Execution
500
+
501
+ **Example**:
502
+
503
+ .. code-block:: python
504
+
505
+ service.executions.get(execution_id='execution_id')
506
+ """
507
+ url_path = "/executions/{}".format(execution_id)
508
+ if sync:
509
+ return self.wait(execution_id=execution_id)
510
+
511
+ success, response = self._client_api.gen_request(req_type="get",
512
+ path=url_path)
513
+
514
+ # exception handling
515
+ if not success:
516
+ raise exceptions.PlatformException(response)
517
+
518
+ # return entity
519
+ return entities.Execution.from_json(client_api=self._client_api,
520
+ _json=response.json(),
521
+ project=self._project,
522
+ service=self._service)
523
+
524
+ def logs(self,
525
+ execution_id: str,
526
+ follow: bool = True,
527
+ until_completed: bool = True):
528
+ """
529
+ executions logs
530
+
531
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
532
+
533
+ :param str execution_id: execution id
534
+ :param bool follow: if true, keep stream future logs
535
+ :param bool until_completed: if true, wait until completed
536
+ :return: executions logs
537
+
538
+ **Example**:
539
+
540
+ .. code-block:: python
541
+
542
+ service.executions.logs(execution_id='execution_id')
543
+ """
544
+ return self.service.log(execution_id=execution_id,
545
+ follow=follow,
546
+ until_completed=until_completed,
547
+ view=True)
548
+
549
+ def increment(self, execution: entities.Execution):
550
+ """
551
+ Increment the number of attempts that an execution is allowed to attempt to run a service that is not responding.
552
+
553
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
554
+
555
+ :param dtlpy.entities.execution.Execution execution:
556
+ :return: int
557
+ :rtype: int
558
+
559
+ **Example**:
560
+
561
+ .. code-block:: python
562
+
563
+ service.executions.increment(execution='execution_entity')
564
+ """
565
+ # request
566
+ success, response = self._client_api.gen_request(
567
+ req_type='post',
568
+ path='/executions/{}/attempts'.format(execution.id)
569
+ )
570
+
571
+ # exception handling
572
+ if not success:
573
+ raise exceptions.PlatformException(response)
574
+
575
+ # exception handling
576
+ if not success:
577
+ raise exceptions.PlatformException(response)
578
+ else:
579
+ return response.json()
580
+
581
+ @_api_reference.add(path='/executions/{executionId}/rerun', method='post')
582
+ def rerun(self, execution: entities.Execution, sync: bool = False):
583
+ """
584
+ Rerun execution
585
+
586
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
587
+
588
+ :param dtlpy.entities.execution.Execution execution:
589
+ :param bool sync: wait for the execution to finish
590
+ :return: Execution object
591
+ :rtype: dtlpy.entities.execution.Execution
592
+
593
+ **Example**:
594
+
595
+ .. code-block:: python
596
+
597
+ service.executions.rerun(execution='execution_entity')
598
+ """
599
+
600
+ url_path = "/executions/{}/rerun".format(execution.id)
601
+ # request
602
+ success, response = self._client_api.gen_request(req_type='post',
603
+ path=url_path)
604
+
605
+ # exception handling
606
+ if not success:
607
+ raise exceptions.PlatformException(response)
608
+ else:
609
+ execution = entities.Execution.from_json(
610
+ client_api=self._client_api,
611
+ _json=response.json(),
612
+ project=self._project,
613
+ service=self._service
614
+ )
615
+ if sync:
616
+ execution = self.wait(execution_id=execution.id)
617
+ return execution
618
+
619
+ def wait(self,
620
+ execution_id: str = None,
621
+ execution: entities.Execution = None,
622
+ timeout: int = None,
623
+ backoff_factor=1):
624
+ """
625
+ Get Service execution object.
626
+
627
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
628
+
629
+ :param str execution_id: execution id
630
+ :param str execution: dl.Execution, optional. must input one of execution or execution_id
631
+ :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
632
+ :param float backoff_factor: A backoff factor to apply between attempts after the second try
633
+ :return: Service execution object
634
+ :rtype: dtlpy.entities.execution.Execution
635
+
636
+ **Example**:
637
+
638
+ .. code-block:: python
639
+
640
+ service.executions.wait(execution_id='execution_id')
641
+ """
642
+ if execution is None:
643
+ if execution_id is None:
644
+ raise ValueError('Must input at least one: [execution, execution_id]')
645
+ else:
646
+ execution = self.get(execution_id=execution_id)
647
+ elapsed = 0
648
+ start = int(time.time())
649
+ if timeout is None or timeout <= 0:
650
+ timeout = np.inf
651
+
652
+ num_tries = 1
653
+ while elapsed < timeout:
654
+ execution = self.get(execution_id=execution.id)
655
+ if not execution.in_progress():
656
+ break
657
+ elapsed = time.time() - start
658
+ if elapsed >= timeout:
659
+ raise TimeoutError(
660
+ f"execution wait() got timeout. id: {execution.id!r}, status: {execution.latest_status}")
661
+ sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
662
+ num_tries += 1
663
+ logger.debug("Execution {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(execution.id,
664
+ elapsed,
665
+ sleep_time))
666
+ time.sleep(sleep_time)
667
+
668
+ return execution
669
+
670
+ @_api_reference.add(path='/executions/{id}/terminate', method='post')
671
+ def terminate(self, execution: entities.Execution):
672
+ """
673
+ Terminate Execution
674
+
675
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
676
+
677
+ :param dtlpy.entities.execution.Execution execution:
678
+ :return: execution object
679
+ :rtype: dtlpy.entities.execution.Execution
680
+
681
+ **Example**:
682
+
683
+ .. code-block:: python
684
+
685
+ service.executions.terminate(execution='execution_entity')
686
+ """
687
+ # request
688
+ success, response = self._client_api.gen_request(req_type='post',
689
+ path='/executions/{}/terminate'.format(execution.id))
690
+
691
+ # exception handling
692
+ if not success:
693
+ raise exceptions.PlatformException(response)
694
+ else:
695
+ return entities.Execution.from_json(_json=response.json(),
696
+ service=self._service,
697
+ project=self._project,
698
+ client_api=self._client_api)
699
+
700
+ @_api_reference.add(path='/executions/{id}', method='patch')
701
+ def update(self, execution: entities.Execution) -> entities.Execution:
702
+ """
703
+ Update execution changes to platform
704
+
705
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
706
+
707
+ :param dtlpy.entities.execution.Execution execution: execution entity
708
+ :return: Service execution object
709
+ :rtype: dtlpy.entities.execution.Execution
710
+
711
+ **Example**:
712
+
713
+ .. code-block:: python
714
+
715
+ service.executions.update(execution='execution_entity')
716
+ """
717
+ # payload
718
+ payload = execution.to_json()
719
+
720
+ # request
721
+ success, response = self._client_api.gen_request(req_type='patch',
722
+ path='/executions/{}'.format(execution.id),
723
+ json_req=payload)
724
+
725
+ # exception handling
726
+ if not success:
727
+ raise exceptions.PlatformException(response)
728
+
729
+ # return entity
730
+ if self._project is not None:
731
+ project = self._project
732
+ else:
733
+ project = execution._project
734
+
735
+ # return
736
+ if self._service is not None:
737
+ service = self._service
738
+ else:
739
+ service = execution._service
740
+
741
+ return entities.Execution.from_json(_json=response.json(),
742
+ service=service,
743
+ project=self._project,
744
+ client_api=self._client_api)
745
+
746
+ def progress_update(
747
+ self,
748
+ execution_id: str,
749
+ status: entities.ExecutionStatus = None,
750
+ percent_complete: int = None,
751
+ message: str = None,
752
+ output: str = None,
753
+ service_version: str = None
754
+ ):
755
+ """
756
+ Update Execution Progress.
757
+
758
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
759
+
760
+ :param str execution_id: execution id
761
+ :param str status: ExecutionStatus
762
+ :param int percent_complete: percent work done
763
+ :param str message: message
764
+ :param str output: the output of the execution
765
+ :param str service_version: service version
766
+ :return: Service execution object
767
+ :rtype: dtlpy.entities.execution.Execution
768
+
769
+ **Example**:
770
+
771
+ .. code-block:: python
772
+
773
+ service.executions.progress_update(execution_id='execution_id', status='complete', percent_complete=100)
774
+ """
775
+ # create payload
776
+ payload = dict()
777
+
778
+ if status is not None:
779
+ payload['status'] = status
780
+ else:
781
+ if percent_complete is not None and isinstance(percent_complete, int):
782
+ if percent_complete < 100:
783
+ payload['status'] = 'inProgress'
784
+ else:
785
+ payload['status'] = 'completed'
786
+ elif output is not None:
787
+ payload['status'] = 'completed'
788
+ else:
789
+ payload['status'] = 'inProgress'
790
+
791
+ if percent_complete is not None:
792
+ payload['percentComplete'] = percent_complete
793
+
794
+ if message is not None:
795
+ payload['message'] = message
796
+
797
+ if output is not None:
798
+ payload['output'] = output
799
+
800
+ if service_version is not None:
801
+ payload['serviceVersion'] = service_version
802
+
803
+ # request
804
+ success, response = self._client_api.gen_request(
805
+ req_type="post",
806
+ path="/executions/{}/progress".format(execution_id),
807
+ json_req=payload
808
+ )
809
+
810
+ # exception handling
811
+ if success:
812
+ return entities.Execution.from_json(_json=response.json(),
813
+ client_api=self._client_api,
814
+ project=self._project,
815
+ service=self._service)
816
+ else:
817
+ raise exceptions.PlatformException(response)