dtlpy 1.113.10__py3-none-any.whl → 1.114.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. dtlpy/__init__.py +488 -488
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/__pycache__/__init__.cpython-38.pyc +0 -0
  5. dtlpy/assets/code_server/config.yaml +2 -2
  6. dtlpy/assets/code_server/installation.sh +24 -24
  7. dtlpy/assets/code_server/launch.json +13 -13
  8. dtlpy/assets/code_server/settings.json +2 -2
  9. dtlpy/assets/main.py +53 -53
  10. dtlpy/assets/main_partial.py +18 -18
  11. dtlpy/assets/mock.json +11 -11
  12. dtlpy/assets/model_adapter.py +83 -83
  13. dtlpy/assets/package.json +61 -61
  14. dtlpy/assets/package_catalog.json +29 -29
  15. dtlpy/assets/package_gitignore +307 -307
  16. dtlpy/assets/service_runners/__init__.py +33 -33
  17. dtlpy/assets/service_runners/converter.py +96 -96
  18. dtlpy/assets/service_runners/multi_method.py +49 -49
  19. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  20. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  21. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  22. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  23. dtlpy/assets/service_runners/single_method.py +37 -37
  24. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  25. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  26. dtlpy/assets/service_runners/single_method_item.py +41 -41
  27. dtlpy/assets/service_runners/single_method_json.py +42 -42
  28. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  29. dtlpy/assets/voc_annotation_template.xml +23 -23
  30. dtlpy/caches/base_cache.py +32 -32
  31. dtlpy/caches/cache.py +473 -473
  32. dtlpy/caches/dl_cache.py +201 -201
  33. dtlpy/caches/filesystem_cache.py +89 -89
  34. dtlpy/caches/redis_cache.py +84 -84
  35. dtlpy/dlp/__init__.py +20 -20
  36. dtlpy/dlp/cli_utilities.py +367 -367
  37. dtlpy/dlp/command_executor.py +764 -764
  38. dtlpy/dlp/dlp +1 -1
  39. dtlpy/dlp/dlp.bat +1 -1
  40. dtlpy/dlp/dlp.py +128 -128
  41. dtlpy/dlp/parser.py +651 -651
  42. dtlpy/entities/__init__.py +83 -83
  43. dtlpy/entities/analytic.py +311 -311
  44. dtlpy/entities/annotation.py +1879 -1879
  45. dtlpy/entities/annotation_collection.py +699 -699
  46. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  47. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  48. dtlpy/entities/annotation_definitions/box.py +195 -195
  49. dtlpy/entities/annotation_definitions/classification.py +67 -67
  50. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  51. dtlpy/entities/annotation_definitions/cube.py +204 -204
  52. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  53. dtlpy/entities/annotation_definitions/description.py +32 -32
  54. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  55. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  56. dtlpy/entities/annotation_definitions/gis.py +69 -69
  57. dtlpy/entities/annotation_definitions/note.py +139 -139
  58. dtlpy/entities/annotation_definitions/point.py +117 -117
  59. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  60. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  61. dtlpy/entities/annotation_definitions/pose.py +92 -92
  62. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  63. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  64. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  65. dtlpy/entities/annotation_definitions/text.py +85 -85
  66. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  67. dtlpy/entities/app.py +220 -220
  68. dtlpy/entities/app_module.py +107 -107
  69. dtlpy/entities/artifact.py +174 -174
  70. dtlpy/entities/assignment.py +399 -399
  71. dtlpy/entities/base_entity.py +214 -214
  72. dtlpy/entities/bot.py +113 -113
  73. dtlpy/entities/codebase.py +296 -296
  74. dtlpy/entities/collection.py +38 -38
  75. dtlpy/entities/command.py +169 -169
  76. dtlpy/entities/compute.py +442 -442
  77. dtlpy/entities/dataset.py +1285 -1285
  78. dtlpy/entities/directory_tree.py +44 -44
  79. dtlpy/entities/dpk.py +470 -470
  80. dtlpy/entities/driver.py +222 -222
  81. dtlpy/entities/execution.py +397 -397
  82. dtlpy/entities/feature.py +124 -124
  83. dtlpy/entities/feature_set.py +145 -145
  84. dtlpy/entities/filters.py +641 -641
  85. dtlpy/entities/gis_item.py +107 -107
  86. dtlpy/entities/integration.py +184 -184
  87. dtlpy/entities/item.py +953 -953
  88. dtlpy/entities/label.py +123 -123
  89. dtlpy/entities/links.py +85 -85
  90. dtlpy/entities/message.py +175 -175
  91. dtlpy/entities/model.py +694 -691
  92. dtlpy/entities/node.py +1005 -1005
  93. dtlpy/entities/ontology.py +803 -803
  94. dtlpy/entities/organization.py +287 -287
  95. dtlpy/entities/package.py +657 -657
  96. dtlpy/entities/package_defaults.py +5 -5
  97. dtlpy/entities/package_function.py +185 -185
  98. dtlpy/entities/package_module.py +113 -113
  99. dtlpy/entities/package_slot.py +118 -118
  100. dtlpy/entities/paged_entities.py +290 -267
  101. dtlpy/entities/pipeline.py +593 -593
  102. dtlpy/entities/pipeline_execution.py +279 -279
  103. dtlpy/entities/project.py +394 -394
  104. dtlpy/entities/prompt_item.py +499 -499
  105. dtlpy/entities/recipe.py +301 -301
  106. dtlpy/entities/reflect_dict.py +102 -102
  107. dtlpy/entities/resource_execution.py +138 -138
  108. dtlpy/entities/service.py +958 -958
  109. dtlpy/entities/service_driver.py +117 -117
  110. dtlpy/entities/setting.py +294 -294
  111. dtlpy/entities/task.py +491 -491
  112. dtlpy/entities/time_series.py +143 -143
  113. dtlpy/entities/trigger.py +426 -426
  114. dtlpy/entities/user.py +118 -118
  115. dtlpy/entities/webhook.py +124 -124
  116. dtlpy/examples/__init__.py +19 -19
  117. dtlpy/examples/add_labels.py +135 -135
  118. dtlpy/examples/add_metadata_to_item.py +21 -21
  119. dtlpy/examples/annotate_items_using_model.py +65 -65
  120. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  121. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  122. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  123. dtlpy/examples/convert_annotation_types.py +51 -51
  124. dtlpy/examples/converter.py +143 -143
  125. dtlpy/examples/copy_annotations.py +22 -22
  126. dtlpy/examples/copy_folder.py +31 -31
  127. dtlpy/examples/create_annotations.py +51 -51
  128. dtlpy/examples/create_video_annotations.py +83 -83
  129. dtlpy/examples/delete_annotations.py +26 -26
  130. dtlpy/examples/filters.py +113 -113
  131. dtlpy/examples/move_item.py +23 -23
  132. dtlpy/examples/play_video_annotation.py +13 -13
  133. dtlpy/examples/show_item_and_mask.py +53 -53
  134. dtlpy/examples/triggers.py +49 -49
  135. dtlpy/examples/upload_batch_of_items.py +20 -20
  136. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  137. dtlpy/examples/upload_items_with_modalities.py +43 -43
  138. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  139. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  140. dtlpy/exceptions.py +125 -125
  141. dtlpy/miscellaneous/__init__.py +20 -20
  142. dtlpy/miscellaneous/dict_differ.py +95 -95
  143. dtlpy/miscellaneous/git_utils.py +217 -217
  144. dtlpy/miscellaneous/json_utils.py +14 -14
  145. dtlpy/miscellaneous/list_print.py +105 -105
  146. dtlpy/miscellaneous/zipping.py +130 -130
  147. dtlpy/ml/__init__.py +20 -20
  148. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  149. dtlpy/ml/base_model_adapter.py +945 -940
  150. dtlpy/ml/metrics.py +461 -461
  151. dtlpy/ml/predictions_utils.py +274 -274
  152. dtlpy/ml/summary_writer.py +57 -57
  153. dtlpy/ml/train_utils.py +60 -60
  154. dtlpy/new_instance.py +252 -252
  155. dtlpy/repositories/__init__.py +56 -56
  156. dtlpy/repositories/analytics.py +85 -85
  157. dtlpy/repositories/annotations.py +916 -916
  158. dtlpy/repositories/apps.py +383 -383
  159. dtlpy/repositories/artifacts.py +452 -452
  160. dtlpy/repositories/assignments.py +599 -599
  161. dtlpy/repositories/bots.py +213 -213
  162. dtlpy/repositories/codebases.py +559 -559
  163. dtlpy/repositories/collections.py +332 -348
  164. dtlpy/repositories/commands.py +158 -158
  165. dtlpy/repositories/compositions.py +61 -61
  166. dtlpy/repositories/computes.py +434 -406
  167. dtlpy/repositories/datasets.py +1291 -1291
  168. dtlpy/repositories/downloader.py +895 -895
  169. dtlpy/repositories/dpks.py +433 -433
  170. dtlpy/repositories/drivers.py +266 -266
  171. dtlpy/repositories/executions.py +817 -817
  172. dtlpy/repositories/feature_sets.py +226 -226
  173. dtlpy/repositories/features.py +238 -238
  174. dtlpy/repositories/integrations.py +484 -484
  175. dtlpy/repositories/items.py +909 -915
  176. dtlpy/repositories/messages.py +94 -94
  177. dtlpy/repositories/models.py +877 -867
  178. dtlpy/repositories/nodes.py +80 -80
  179. dtlpy/repositories/ontologies.py +511 -511
  180. dtlpy/repositories/organizations.py +525 -525
  181. dtlpy/repositories/packages.py +1941 -1941
  182. dtlpy/repositories/pipeline_executions.py +448 -448
  183. dtlpy/repositories/pipelines.py +642 -642
  184. dtlpy/repositories/projects.py +539 -539
  185. dtlpy/repositories/recipes.py +399 -399
  186. dtlpy/repositories/resource_executions.py +137 -137
  187. dtlpy/repositories/schema.py +120 -120
  188. dtlpy/repositories/service_drivers.py +213 -213
  189. dtlpy/repositories/services.py +1704 -1704
  190. dtlpy/repositories/settings.py +339 -339
  191. dtlpy/repositories/tasks.py +1124 -1124
  192. dtlpy/repositories/times_series.py +278 -278
  193. dtlpy/repositories/triggers.py +536 -536
  194. dtlpy/repositories/upload_element.py +257 -257
  195. dtlpy/repositories/uploader.py +651 -651
  196. dtlpy/repositories/webhooks.py +249 -249
  197. dtlpy/services/__init__.py +22 -22
  198. dtlpy/services/aihttp_retry.py +131 -131
  199. dtlpy/services/api_client.py +1782 -1782
  200. dtlpy/services/api_reference.py +40 -40
  201. dtlpy/services/async_utils.py +133 -133
  202. dtlpy/services/calls_counter.py +44 -44
  203. dtlpy/services/check_sdk.py +68 -68
  204. dtlpy/services/cookie.py +115 -115
  205. dtlpy/services/create_logger.py +156 -156
  206. dtlpy/services/events.py +84 -84
  207. dtlpy/services/logins.py +235 -235
  208. dtlpy/services/reporter.py +256 -256
  209. dtlpy/services/service_defaults.py +91 -91
  210. dtlpy/utilities/__init__.py +20 -20
  211. dtlpy/utilities/annotations/__init__.py +16 -16
  212. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  213. dtlpy/utilities/base_package_runner.py +264 -264
  214. dtlpy/utilities/converter.py +1650 -1650
  215. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  216. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  217. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  218. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  219. dtlpy/utilities/local_development/__init__.py +1 -1
  220. dtlpy/utilities/local_development/local_session.py +179 -179
  221. dtlpy/utilities/reports/__init__.py +2 -2
  222. dtlpy/utilities/reports/figures.py +343 -343
  223. dtlpy/utilities/reports/report.py +71 -71
  224. dtlpy/utilities/videos/__init__.py +17 -17
  225. dtlpy/utilities/videos/video_player.py +598 -598
  226. dtlpy/utilities/videos/videos.py +470 -470
  227. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp +1 -1
  228. dtlpy-1.114.13.data/scripts/dlp.bat +2 -0
  229. {dtlpy-1.113.10.data → dtlpy-1.114.13.data}/scripts/dlp.py +128 -128
  230. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/LICENSE +200 -200
  231. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/METADATA +172 -172
  232. dtlpy-1.114.13.dist-info/RECORD +240 -0
  233. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/WHEEL +1 -1
  234. tests/features/environment.py +551 -550
  235. dtlpy-1.113.10.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.113.10.dist-info/RECORD +0 -244
  237. tests/assets/__init__.py +0 -0
  238. tests/assets/models_flow/__init__.py +0 -0
  239. tests/assets/models_flow/failedmain.py +0 -52
  240. tests/assets/models_flow/main.py +0 -62
  241. tests/assets/models_flow/main_model.py +0 -54
  242. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/entry_points.txt +0 -0
  243. {dtlpy-1.113.10.dist-info → dtlpy-1.114.13.dist-info}/top_level.txt +0 -0
@@ -1,448 +1,448 @@
1
- import logging
2
- import time
3
- import numpy as np
4
-
5
- from .. import entities, repositories, exceptions, miscellaneous, services, _api_reference
6
- from ..services.api_client import ApiClient
7
-
8
- logger = logging.getLogger(name='dtlpy')
9
- MAX_SLEEP_TIME = 30
10
-
11
-
12
- class PipelineExecutions:
13
- """
14
- PipelineExecutions Repository
15
-
16
- The PipelineExecutions class allows users to manage pipeline executions. See our documentation for more information on `pipelines <https://dataloop.ai/docs/pipelines-overview>`_.
17
- """
18
-
19
- def __init__(
20
- self,
21
- client_api: ApiClient,
22
- project: entities.Project = None,
23
- pipeline: entities.Pipeline = None
24
- ):
25
- self._client_api = client_api
26
- self._project = project
27
- self._pipeline = pipeline
28
-
29
- ############
30
- # entities #
31
- ############
32
- @property
33
- def project(self) -> entities.Project:
34
- if self._project is None:
35
- try:
36
- self._project = repositories.Projects(client_api=self._client_api).get()
37
- except exceptions.NotFound:
38
- raise exceptions.PlatformException(
39
- error='2001',
40
- message='Missing "project". need to set a Project entity or use project.pipelines repository')
41
- return self._project
42
-
43
- @project.setter
44
- def project(self, project: entities.Project):
45
- if not isinstance(project, entities.Project):
46
- raise ValueError('Must input a valid Project entity')
47
- self._project = project
48
-
49
- @property
50
- def pipeline(self) -> entities.Pipeline:
51
- assert isinstance(self._pipeline, entities.Pipeline)
52
- return self._pipeline
53
-
54
- @pipeline.setter
55
- def pipeline(self, pipeline: entities.Pipeline):
56
- if not isinstance(pipeline, entities.Pipeline):
57
- raise ValueError('Must input a valid pipeline entity')
58
- self._pipeline = pipeline
59
-
60
- ###########
61
- # methods #
62
- ###########
63
- # @_api_reference.add(path='/pipelines/{pipelineId}/executions/{executionId}', method='get')
64
- def get(self,
65
- pipeline_execution_id: str,
66
- pipeline_id: str = None
67
- ) -> entities.PipelineExecution:
68
- """
69
- Get Pipeline Execution object
70
-
71
- **prerequisites**: You must be an *owner* or *developer* to use this method.
72
-
73
- :param str pipeline_execution_id: pipeline execution id
74
- :param str pipeline_id: pipeline id
75
- :return: PipelineExecution object
76
- :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
77
-
78
- **Example**:
79
-
80
- .. code-block:: python
81
-
82
- pipeline_executions = pipeline.pipeline_executions.get(pipeline_id='pipeline_id')
83
- """
84
-
85
- if pipeline_id is None and self._pipeline is None:
86
- raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
87
- elif pipeline_id is None:
88
- pipeline_id = self._pipeline.id
89
-
90
- success, response = self._client_api.gen_request(
91
- req_type="get",
92
- path="/pipelines/{pipeline_id}/executions/{pipeline_execution_id}".format(
93
- pipeline_id=pipeline_id,
94
- pipeline_execution_id=pipeline_execution_id
95
- )
96
- )
97
- if not success:
98
- raise exceptions.PlatformException(response)
99
-
100
- pipeline_execution = entities.PipelineExecution.from_json(
101
- client_api=self._client_api,
102
- _json=response.json(),
103
- pipeline=self._pipeline
104
- )
105
-
106
- return pipeline_execution
107
-
108
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.PipelineExecution]:
109
- pool = self._client_api.thread_pools(pool_name='entity.create')
110
- jobs = [None for _ in range(len(response_items))]
111
-
112
- for i_pipeline_execution, pipeline_execution in enumerate(response_items):
113
- jobs[i_pipeline_execution] = pool.submit(
114
- entities.PipelineExecution._protected_from_json,
115
- **{
116
- 'client_api': self._client_api,
117
- '_json': pipeline_execution,
118
- 'pipeline': self._pipeline
119
- }
120
- )
121
-
122
- # get all results
123
- # noinspection PyUnresolvedReferences
124
- results = [j.result() for j in jobs]
125
- # log errors
126
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
127
- # return good jobs
128
- pipeline_executions = miscellaneous.List([r[1] for r in results if r[0] is True])
129
- return pipeline_executions
130
-
131
- def _list(self, filters: entities.Filters):
132
- url = '/pipelines/query'
133
-
134
- # request
135
- success, response = self._client_api.gen_request(
136
- req_type='post',
137
- path=url,
138
- json_req=filters.prepare()
139
- )
140
- if not success:
141
- raise exceptions.PlatformException(response)
142
- return response.json()
143
-
144
- @_api_reference.add(path='/pipelines/query', method='post')
145
- def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
146
- """
147
- List project pipeline executions.
148
-
149
- **prerequisites**: You must be an *owner* or *developer* to use this method.
150
-
151
- :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
152
- :return: Paged entity
153
- :rtype: dtlpy.entities.paged_entities.PagedEntities
154
-
155
- **Example**:
156
-
157
- .. code-block:: python
158
-
159
- pipeline_executions = pipeline.pipeline_executions.list()
160
- """
161
- if filters is None:
162
- filters = entities.Filters(resource=entities.FiltersResource.PIPELINE_EXECUTION)
163
- # assert type filters
164
- elif not isinstance(filters, entities.Filters):
165
- raise exceptions.PlatformException(error='400',
166
- message='Unknown filters type: {!r}'.format(type(filters)))
167
- if filters.resource != entities.FiltersResource.PIPELINE_EXECUTION:
168
- raise exceptions.PlatformException(
169
- error='400',
170
- message='Filters resource must to be FiltersResource.PIPELINE_EXECUTION. Got: {!r}'.format(
171
- filters.resource))
172
-
173
- project_id = None
174
- if self._project is not None:
175
- project_id = self._project.id
176
-
177
- # TODO - uncomment this after DAT-24496 is done and cycles have projectId
178
- # if self._project is not None:
179
- # filters.add(field='projectId', values=self.project.id)
180
-
181
- if self._pipeline is not None:
182
- filters.add(field='pipelineId', values=self._pipeline.id)
183
-
184
- paged = entities.PagedEntities(
185
- items_repository=self,
186
- filters=filters,
187
- page_offset=filters.page,
188
- page_size=filters.page_size,
189
- project_id=project_id,
190
- client_api=self._client_api
191
- )
192
-
193
- paged.get_page()
194
- return paged
195
-
196
- @_api_reference.add(path='/pipelines/{pipelineId}/execute', method='post')
197
- def create(
198
- self,
199
- pipeline_id: str = None,
200
- execution_input=None,
201
- node_id: str = None
202
- ):
203
- """
204
- Execute a pipeline.
205
-
206
- **prerequisites**: You must be an *owner* or *developer* to use this method.
207
-
208
- :param pipeline_id: pipeline id
209
- :param execution_input: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}
210
- :param node_id: node id to start from
211
-
212
- :return: entities.PipelineExecution object
213
- :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
214
-
215
- **Example**:
216
-
217
- .. code-block:: python
218
-
219
- pipeline_execution = pipeline.pipeline_executions.create(pipeline_id='pipeline_id', execution_input={'item': 'item_id'})
220
- """
221
- if pipeline_id is None:
222
- if self._pipeline is None:
223
- raise exceptions.PlatformException('400', 'Please provide pipeline id')
224
- pipeline_id = self._pipeline.id
225
-
226
- payload = dict()
227
- if execution_input is None:
228
- # support pipeline executions without any input
229
- pass
230
- elif isinstance(execution_input, dict):
231
- payload['input'] = execution_input
232
- else:
233
- if not isinstance(execution_input, list):
234
- execution_input = [execution_input]
235
- if len(execution_input) > 0 and isinstance(execution_input[0], entities.FunctionIO):
236
- payload['input'] = dict()
237
- for single_input in execution_input:
238
- payload['input'].update(single_input.to_json(resource='execution'))
239
- else:
240
- raise exceptions.PlatformException('400', 'Unknown input type')
241
-
242
- if node_id is not None:
243
- payload['nodeId'] = node_id
244
-
245
- success, response = self._client_api.gen_request(
246
- path='/pipelines/{}/execute'.format(pipeline_id),
247
- req_type='POST',
248
- json_req=payload
249
- )
250
-
251
- if not success:
252
- raise exceptions.PlatformException(response)
253
-
254
- execution = entities.PipelineExecution.from_json(_json=response.json(),
255
- client_api=self._client_api,
256
- pipeline=self._pipeline)
257
- return execution
258
-
259
- @_api_reference.add(path='/pipelines/{pipelineId}/execute', method='post')
260
- def create_batch(
261
- self,
262
- pipeline_id: str,
263
- filters,
264
- execution_inputs=None,
265
- wait=True,
266
- node_id: str = None
267
- ):
268
- """
269
- Create batch executions of a pipeline.
270
-
271
- **prerequisites**: You must be an *owner* or *developer* to use this method.
272
-
273
- :param pipeline_id: pipeline id
274
- :param filters: Filters entity for a filtering before execute
275
- :param execution_inputs: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}, that represent the extra inputs of the function
276
- :param bool wait: wait until create task finish
277
- :return: entities.PipelineExecution object
278
- :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
279
-
280
- **Example**:
281
-
282
- .. code-block:: python
283
-
284
- command = pipeline.pipeline_executions.create_batch(
285
- execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
286
- filters=dl.Filters(field='dir', values='/test'))
287
- """
288
- if pipeline_id is None:
289
- if self._pipeline is None:
290
- raise exceptions.PlatformException('400', 'Please provide pipeline id')
291
- pipeline_id = self._pipeline.id
292
-
293
- if filters is None:
294
- raise exceptions.PlatformException('400', 'Please provide filter')
295
- extra_input = dict()
296
-
297
- if execution_inputs is None:
298
- execution_inputs = {}
299
-
300
- if isinstance(execution_inputs, dict):
301
- extra_input = execution_inputs
302
- else:
303
- if not isinstance(execution_inputs, list):
304
- execution_inputs = [execution_inputs]
305
- if len(execution_inputs) > 0 and isinstance(execution_inputs[0], entities.FunctionIO):
306
- for single_input in execution_inputs:
307
- extra_input.update(single_input.to_json(resource='execution'))
308
- else:
309
- raise exceptions.PlatformException('400', 'Unknown input type')
310
- payload = dict()
311
- payload['batch'] = dict()
312
- payload['batch']['query'] = filters.prepare()
313
- payload['batch']['args'] = extra_input
314
-
315
- if node_id is not None:
316
- payload['nodeId'] = node_id
317
-
318
- success, response = self._client_api.gen_request(
319
- path='/pipelines/{}/execute'.format(pipeline_id),
320
- req_type='POST',
321
- json_req=payload
322
- )
323
-
324
- if not success:
325
- raise exceptions.PlatformException(response)
326
-
327
- response_json = response.json()
328
- command = entities.Command.from_json(_json=response_json,
329
- client_api=self._client_api)
330
- if wait:
331
- command = command.wait(timeout=0)
332
- return command
333
-
334
- @_api_reference.add(path='/pipelines/{pipelineId}/executions/rerun', method='post')
335
- def rerun(self,
336
- pipeline_id: str = None,
337
- method: str = None,
338
- start_nodes_ids: list = None,
339
- filters: entities.Filters = None,
340
- wait: bool = True
341
- ) -> bool:
342
- """
343
- Get Pipeline Execution object
344
-
345
- **prerequisites**: You must be an *owner* or *developer* to use this method.
346
-
347
- :param str pipeline_id: pipeline id
348
- :param str method: method to run
349
- :param list start_nodes_ids: list of start nodes ids
350
- :param filters: Filters entity for a filtering before execute
351
- :param bool wait: wait until rerun finish
352
- :return: True if success
353
- :rtype: bool
354
-
355
- **Example**:
356
-
357
- .. code-block:: python
358
-
359
- pipeline.pipeline_executions.rerun(pipeline_id='pipeline_id', method=dl.CycleRerunMethod.START_FROM_BEGINNING)
360
- """
361
-
362
- if pipeline_id is None and self._pipeline is None:
363
- raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
364
- elif pipeline_id is None:
365
- pipeline_id = self._pipeline.id
366
-
367
- if filters is None:
368
- filters = entities.Filters(resource=entities.FiltersResource.PIPELINE_EXECUTION)
369
-
370
- success, response = self._client_api.gen_request(
371
- req_type="post",
372
- path="/pipelines/{pipeline_id}/executions/rerun".format(
373
- pipeline_id=pipeline_id,
374
- ),
375
- json_req={
376
- 'pipeline': {
377
- "method": method,
378
- "startNodeIds": start_nodes_ids,
379
- },
380
- "batch": {
381
- "query": filters.prepare()
382
- }
383
- }
384
- )
385
-
386
- if not success:
387
- raise exceptions.PlatformException(response)
388
-
389
- command = entities.Command.from_json(_json=response.json(),
390
- client_api=self._client_api)
391
- if not wait:
392
- return command
393
- command = command.wait(timeout=0)
394
-
395
- if 'cycleOptions' not in command.spec:
396
- raise exceptions.PlatformException(error='400',
397
- message="cycleOptions key is missing in command response: {!r}"
398
- .format(response))
399
- return True
400
-
401
- def wait(self,
402
- pipeline_execution_id: str = None,
403
- pipeline_execution: entities.PipelineExecution = None,
404
- timeout: int = None,
405
- backoff_factor=1):
406
- """
407
- Get Service execution object.
408
-
409
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
410
-
411
- :param str pipeline_execution_id: pipeline execution id
412
- :param str pipeline_execution: dl.PipelineExecution, optional. must input one of pipeline execution or pipeline_execution_id
413
- :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
414
- :param float backoff_factor: A backoff factor to apply between attempts after the second try
415
- :return: Service execution object
416
- :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
417
-
418
- **Example**:
419
-
420
- .. code-block:: python
421
-
422
- pipeline.pipeline_executions.wait(pipeline_execution_id='pipeline_execution_id')
423
- """
424
- if pipeline_execution is None:
425
- if pipeline_execution_id is None:
426
- raise ValueError('Must input at least one: [pipeline_execution, pipeline_execution_id]')
427
- else:
428
- pipeline_execution_id = pipeline_execution.id
429
- elapsed = 0
430
- start = time.time()
431
- if timeout is None or timeout <= 0:
432
- timeout = np.inf
433
-
434
- num_tries = 1
435
- while elapsed < timeout:
436
- pipeline_execution = self.get(pipeline_execution_id=pipeline_execution_id)
437
- if not pipeline_execution.in_progress():
438
- break
439
- elapsed = time.time() - start
440
- if elapsed >= timeout:
441
- raise TimeoutError(
442
- f"Pipeline execution wait() function timed out. id: {pipeline_execution.id!r}, status: {pipeline_execution.status}.")
443
- sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
444
- num_tries += 1
445
- logger.debug(
446
- f"Pipeline execution {pipeline_execution.id!r} has been running for {elapsed:.2f}[s]. Sleeping for {sleep_time:.2f}[s]")
447
- time.sleep(sleep_time)
448
- return pipeline_execution
1
+ import logging
2
+ import time
3
+ import numpy as np
4
+
5
+ from .. import entities, repositories, exceptions, miscellaneous, services, _api_reference
6
+ from ..services.api_client import ApiClient
7
+
8
+ logger = logging.getLogger(name='dtlpy')
9
+ MAX_SLEEP_TIME = 30
10
+
11
+
12
+ class PipelineExecutions:
13
+ """
14
+ PipelineExecutions Repository
15
+
16
+ The PipelineExecutions class allows users to manage pipeline executions. See our documentation for more information on `pipelines <https://dataloop.ai/docs/pipelines-overview>`_.
17
+ """
18
+
19
+ def __init__(
20
+ self,
21
+ client_api: ApiClient,
22
+ project: entities.Project = None,
23
+ pipeline: entities.Pipeline = None
24
+ ):
25
+ self._client_api = client_api
26
+ self._project = project
27
+ self._pipeline = pipeline
28
+
29
+ ############
30
+ # entities #
31
+ ############
32
+ @property
33
+ def project(self) -> entities.Project:
34
+ if self._project is None:
35
+ try:
36
+ self._project = repositories.Projects(client_api=self._client_api).get()
37
+ except exceptions.NotFound:
38
+ raise exceptions.PlatformException(
39
+ error='2001',
40
+ message='Missing "project". need to set a Project entity or use project.pipelines repository')
41
+ return self._project
42
+
43
+ @project.setter
44
+ def project(self, project: entities.Project):
45
+ if not isinstance(project, entities.Project):
46
+ raise ValueError('Must input a valid Project entity')
47
+ self._project = project
48
+
49
+ @property
50
+ def pipeline(self) -> entities.Pipeline:
51
+ assert isinstance(self._pipeline, entities.Pipeline)
52
+ return self._pipeline
53
+
54
+ @pipeline.setter
55
+ def pipeline(self, pipeline: entities.Pipeline):
56
+ if not isinstance(pipeline, entities.Pipeline):
57
+ raise ValueError('Must input a valid pipeline entity')
58
+ self._pipeline = pipeline
59
+
60
+ ###########
61
+ # methods #
62
+ ###########
63
+ # @_api_reference.add(path='/pipelines/{pipelineId}/executions/{executionId}', method='get')
64
+ def get(self,
65
+ pipeline_execution_id: str,
66
+ pipeline_id: str = None
67
+ ) -> entities.PipelineExecution:
68
+ """
69
+ Get Pipeline Execution object
70
+
71
+ **prerequisites**: You must be an *owner* or *developer* to use this method.
72
+
73
+ :param str pipeline_execution_id: pipeline execution id
74
+ :param str pipeline_id: pipeline id
75
+ :return: PipelineExecution object
76
+ :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
77
+
78
+ **Example**:
79
+
80
+ .. code-block:: python
81
+
82
+ pipeline_executions = pipeline.pipeline_executions.get(pipeline_id='pipeline_id')
83
+ """
84
+
85
+ if pipeline_id is None and self._pipeline is None:
86
+ raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
87
+ elif pipeline_id is None:
88
+ pipeline_id = self._pipeline.id
89
+
90
+ success, response = self._client_api.gen_request(
91
+ req_type="get",
92
+ path="/pipelines/{pipeline_id}/executions/{pipeline_execution_id}".format(
93
+ pipeline_id=pipeline_id,
94
+ pipeline_execution_id=pipeline_execution_id
95
+ )
96
+ )
97
+ if not success:
98
+ raise exceptions.PlatformException(response)
99
+
100
+ pipeline_execution = entities.PipelineExecution.from_json(
101
+ client_api=self._client_api,
102
+ _json=response.json(),
103
+ pipeline=self._pipeline
104
+ )
105
+
106
+ return pipeline_execution
107
+
108
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.PipelineExecution]:
109
+ pool = self._client_api.thread_pools(pool_name='entity.create')
110
+ jobs = [None for _ in range(len(response_items))]
111
+
112
+ for i_pipeline_execution, pipeline_execution in enumerate(response_items):
113
+ jobs[i_pipeline_execution] = pool.submit(
114
+ entities.PipelineExecution._protected_from_json,
115
+ **{
116
+ 'client_api': self._client_api,
117
+ '_json': pipeline_execution,
118
+ 'pipeline': self._pipeline
119
+ }
120
+ )
121
+
122
+ # get all results
123
+ # noinspection PyUnresolvedReferences
124
+ results = [j.result() for j in jobs]
125
+ # log errors
126
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
127
+ # return good jobs
128
+ pipeline_executions = miscellaneous.List([r[1] for r in results if r[0] is True])
129
+ return pipeline_executions
130
+
131
+ def _list(self, filters: entities.Filters):
132
+ url = '/pipelines/query'
133
+
134
+ # request
135
+ success, response = self._client_api.gen_request(
136
+ req_type='post',
137
+ path=url,
138
+ json_req=filters.prepare()
139
+ )
140
+ if not success:
141
+ raise exceptions.PlatformException(response)
142
+ return response.json()
143
+
144
+ @_api_reference.add(path='/pipelines/query', method='post')
145
+ def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
146
+ """
147
+ List project pipeline executions.
148
+
149
+ **prerequisites**: You must be an *owner* or *developer* to use this method.
150
+
151
+ :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
152
+ :return: Paged entity
153
+ :rtype: dtlpy.entities.paged_entities.PagedEntities
154
+
155
+ **Example**:
156
+
157
+ .. code-block:: python
158
+
159
+ pipeline_executions = pipeline.pipeline_executions.list()
160
+ """
161
+ if filters is None:
162
+ filters = entities.Filters(resource=entities.FiltersResource.PIPELINE_EXECUTION)
163
+ # assert type filters
164
+ elif not isinstance(filters, entities.Filters):
165
+ raise exceptions.PlatformException(error='400',
166
+ message='Unknown filters type: {!r}'.format(type(filters)))
167
+ if filters.resource != entities.FiltersResource.PIPELINE_EXECUTION:
168
+ raise exceptions.PlatformException(
169
+ error='400',
170
+ message='Filters resource must to be FiltersResource.PIPELINE_EXECUTION. Got: {!r}'.format(
171
+ filters.resource))
172
+
173
+ project_id = None
174
+ if self._project is not None:
175
+ project_id = self._project.id
176
+
177
+ # TODO - uncomment this after DAT-24496 is done and cycles have projectId
178
+ # if self._project is not None:
179
+ # filters.add(field='projectId', values=self.project.id)
180
+
181
+ if self._pipeline is not None:
182
+ filters.add(field='pipelineId', values=self._pipeline.id)
183
+
184
+ paged = entities.PagedEntities(
185
+ items_repository=self,
186
+ filters=filters,
187
+ page_offset=filters.page,
188
+ page_size=filters.page_size,
189
+ project_id=project_id,
190
+ client_api=self._client_api
191
+ )
192
+
193
+ paged.get_page()
194
+ return paged
195
+
196
+ @_api_reference.add(path='/pipelines/{pipelineId}/execute', method='post')
197
+ def create(
198
+ self,
199
+ pipeline_id: str = None,
200
+ execution_input=None,
201
+ node_id: str = None
202
+ ):
203
+ """
204
+ Execute a pipeline.
205
+
206
+ **prerequisites**: You must be an *owner* or *developer* to use this method.
207
+
208
+ :param pipeline_id: pipeline id
209
+ :param execution_input: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}
210
+ :param node_id: node id to start from
211
+
212
+ :return: entities.PipelineExecution object
213
+ :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
214
+
215
+ **Example**:
216
+
217
+ .. code-block:: python
218
+
219
+ pipeline_execution = pipeline.pipeline_executions.create(pipeline_id='pipeline_id', execution_input={'item': 'item_id'})
220
+ """
221
+ if pipeline_id is None:
222
+ if self._pipeline is None:
223
+ raise exceptions.PlatformException('400', 'Please provide pipeline id')
224
+ pipeline_id = self._pipeline.id
225
+
226
+ payload = dict()
227
+ if execution_input is None:
228
+ # support pipeline executions without any input
229
+ pass
230
+ elif isinstance(execution_input, dict):
231
+ payload['input'] = execution_input
232
+ else:
233
+ if not isinstance(execution_input, list):
234
+ execution_input = [execution_input]
235
+ if len(execution_input) > 0 and isinstance(execution_input[0], entities.FunctionIO):
236
+ payload['input'] = dict()
237
+ for single_input in execution_input:
238
+ payload['input'].update(single_input.to_json(resource='execution'))
239
+ else:
240
+ raise exceptions.PlatformException('400', 'Unknown input type')
241
+
242
+ if node_id is not None:
243
+ payload['nodeId'] = node_id
244
+
245
+ success, response = self._client_api.gen_request(
246
+ path='/pipelines/{}/execute'.format(pipeline_id),
247
+ req_type='POST',
248
+ json_req=payload
249
+ )
250
+
251
+ if not success:
252
+ raise exceptions.PlatformException(response)
253
+
254
+ execution = entities.PipelineExecution.from_json(_json=response.json(),
255
+ client_api=self._client_api,
256
+ pipeline=self._pipeline)
257
+ return execution
258
+
259
+ @_api_reference.add(path='/pipelines/{pipelineId}/execute', method='post')
260
+ def create_batch(
261
+ self,
262
+ pipeline_id: str,
263
+ filters,
264
+ execution_inputs=None,
265
+ wait=True,
266
+ node_id: str = None
267
+ ):
268
+ """
269
+ Create batch executions of a pipeline.
270
+
271
+ **prerequisites**: You must be an *owner* or *developer* to use this method.
272
+
273
+ :param pipeline_id: pipeline id
274
+ :param filters: Filters entity for a filtering before execute
275
+ :param execution_inputs: list of the dl.FunctionIO or dict of pipeline input - example {'item': 'item_id'}, that represent the extra inputs of the function
276
+ :param bool wait: wait until create task finish
277
+ :return: entities.PipelineExecution object
278
+ :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
279
+
280
+ **Example**:
281
+
282
+ .. code-block:: python
283
+
284
+ command = pipeline.pipeline_executions.create_batch(
285
+ execution_inputs=dl.FunctionIO(type=dl.PackageInputType.STRING, value='test', name='string'),
286
+ filters=dl.Filters(field='dir', values='/test'))
287
+ """
288
+ if pipeline_id is None:
289
+ if self._pipeline is None:
290
+ raise exceptions.PlatformException('400', 'Please provide pipeline id')
291
+ pipeline_id = self._pipeline.id
292
+
293
+ if filters is None:
294
+ raise exceptions.PlatformException('400', 'Please provide filter')
295
+ extra_input = dict()
296
+
297
+ if execution_inputs is None:
298
+ execution_inputs = {}
299
+
300
+ if isinstance(execution_inputs, dict):
301
+ extra_input = execution_inputs
302
+ else:
303
+ if not isinstance(execution_inputs, list):
304
+ execution_inputs = [execution_inputs]
305
+ if len(execution_inputs) > 0 and isinstance(execution_inputs[0], entities.FunctionIO):
306
+ for single_input in execution_inputs:
307
+ extra_input.update(single_input.to_json(resource='execution'))
308
+ else:
309
+ raise exceptions.PlatformException('400', 'Unknown input type')
310
+ payload = dict()
311
+ payload['batch'] = dict()
312
+ payload['batch']['query'] = filters.prepare()
313
+ payload['batch']['args'] = extra_input
314
+
315
+ if node_id is not None:
316
+ payload['nodeId'] = node_id
317
+
318
+ success, response = self._client_api.gen_request(
319
+ path='/pipelines/{}/execute'.format(pipeline_id),
320
+ req_type='POST',
321
+ json_req=payload
322
+ )
323
+
324
+ if not success:
325
+ raise exceptions.PlatformException(response)
326
+
327
+ response_json = response.json()
328
+ command = entities.Command.from_json(_json=response_json,
329
+ client_api=self._client_api)
330
+ if wait:
331
+ command = command.wait(timeout=0)
332
+ return command
333
+
334
+ @_api_reference.add(path='/pipelines/{pipelineId}/executions/rerun', method='post')
335
+ def rerun(self,
336
+ pipeline_id: str = None,
337
+ method: str = None,
338
+ start_nodes_ids: list = None,
339
+ filters: entities.Filters = None,
340
+ wait: bool = True
341
+ ) -> bool:
342
+ """
343
+ Get Pipeline Execution object
344
+
345
+ **prerequisites**: You must be an *owner* or *developer* to use this method.
346
+
347
+ :param str pipeline_id: pipeline id
348
+ :param str method: method to run
349
+ :param list start_nodes_ids: list of start nodes ids
350
+ :param filters: Filters entity for a filtering before execute
351
+ :param bool wait: wait until rerun finish
352
+ :return: True if success
353
+ :rtype: bool
354
+
355
+ **Example**:
356
+
357
+ .. code-block:: python
358
+
359
+ pipeline.pipeline_executions.rerun(pipeline_id='pipeline_id', method=dl.CycleRerunMethod.START_FROM_BEGINNING)
360
+ """
361
+
362
+ if pipeline_id is None and self._pipeline is None:
363
+ raise exceptions.PlatformException('400', 'Must provide param pipeline_id')
364
+ elif pipeline_id is None:
365
+ pipeline_id = self._pipeline.id
366
+
367
+ if filters is None:
368
+ filters = entities.Filters(resource=entities.FiltersResource.PIPELINE_EXECUTION)
369
+
370
+ success, response = self._client_api.gen_request(
371
+ req_type="post",
372
+ path="/pipelines/{pipeline_id}/executions/rerun".format(
373
+ pipeline_id=pipeline_id,
374
+ ),
375
+ json_req={
376
+ 'pipeline': {
377
+ "method": method,
378
+ "startNodeIds": start_nodes_ids,
379
+ },
380
+ "batch": {
381
+ "query": filters.prepare()
382
+ }
383
+ }
384
+ )
385
+
386
+ if not success:
387
+ raise exceptions.PlatformException(response)
388
+
389
+ command = entities.Command.from_json(_json=response.json(),
390
+ client_api=self._client_api)
391
+ if not wait:
392
+ return command
393
+ command = command.wait(timeout=0)
394
+
395
+ if 'cycleOptions' not in command.spec:
396
+ raise exceptions.PlatformException(error='400',
397
+ message="cycleOptions key is missing in command response: {!r}"
398
+ .format(response))
399
+ return True
400
+
401
+ def wait(self,
402
+ pipeline_execution_id: str = None,
403
+ pipeline_execution: entities.PipelineExecution = None,
404
+ timeout: int = None,
405
+ backoff_factor=1):
406
+ """
407
+ Get Service execution object.
408
+
409
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a service.
410
+
411
+ :param str pipeline_execution_id: pipeline execution id
412
+ :param str pipeline_execution: dl.PipelineExecution, optional. must input one of pipeline execution or pipeline_execution_id
413
+ :param int timeout: seconds to wait until TimeoutError is raised. if <=0 - wait until done - by default wait take the service timeout
414
+ :param float backoff_factor: A backoff factor to apply between attempts after the second try
415
+ :return: Service execution object
416
+ :rtype: dtlpy.entities.pipeline_execution.PipelineExecution
417
+
418
+ **Example**:
419
+
420
+ .. code-block:: python
421
+
422
+ pipeline.pipeline_executions.wait(pipeline_execution_id='pipeline_execution_id')
423
+ """
424
+ if pipeline_execution is None:
425
+ if pipeline_execution_id is None:
426
+ raise ValueError('Must input at least one: [pipeline_execution, pipeline_execution_id]')
427
+ else:
428
+ pipeline_execution_id = pipeline_execution.id
429
+ elapsed = 0
430
+ start = time.time()
431
+ if timeout is None or timeout <= 0:
432
+ timeout = np.inf
433
+
434
+ num_tries = 1
435
+ while elapsed < timeout:
436
+ pipeline_execution = self.get(pipeline_execution_id=pipeline_execution_id)
437
+ if not pipeline_execution.in_progress():
438
+ break
439
+ elapsed = time.time() - start
440
+ if elapsed >= timeout:
441
+ raise TimeoutError(
442
+ f"Pipeline execution wait() function timed out. id: {pipeline_execution.id!r}, status: {pipeline_execution.status}.")
443
+ sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
444
+ num_tries += 1
445
+ logger.debug(
446
+ f"Pipeline execution {pipeline_execution.id!r} has been running for {elapsed:.2f}[s]. Sleeping for {sleep_time:.2f}[s]")
447
+ time.sleep(sleep_time)
448
+ return pipeline_execution