dtlpy 1.115.44__py3-none-any.whl → 1.116.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dtlpy/__init__.py +491 -491
  2. dtlpy/__version__.py +1 -1
  3. dtlpy/assets/__init__.py +26 -26
  4. dtlpy/assets/code_server/config.yaml +2 -2
  5. dtlpy/assets/code_server/installation.sh +24 -24
  6. dtlpy/assets/code_server/launch.json +13 -13
  7. dtlpy/assets/code_server/settings.json +2 -2
  8. dtlpy/assets/main.py +53 -53
  9. dtlpy/assets/main_partial.py +18 -18
  10. dtlpy/assets/mock.json +11 -11
  11. dtlpy/assets/model_adapter.py +83 -83
  12. dtlpy/assets/package.json +61 -61
  13. dtlpy/assets/package_catalog.json +29 -29
  14. dtlpy/assets/package_gitignore +307 -307
  15. dtlpy/assets/service_runners/__init__.py +33 -33
  16. dtlpy/assets/service_runners/converter.py +96 -96
  17. dtlpy/assets/service_runners/multi_method.py +49 -49
  18. dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
  19. dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
  20. dtlpy/assets/service_runners/multi_method_item.py +52 -52
  21. dtlpy/assets/service_runners/multi_method_json.py +52 -52
  22. dtlpy/assets/service_runners/single_method.py +37 -37
  23. dtlpy/assets/service_runners/single_method_annotation.py +43 -43
  24. dtlpy/assets/service_runners/single_method_dataset.py +43 -43
  25. dtlpy/assets/service_runners/single_method_item.py +41 -41
  26. dtlpy/assets/service_runners/single_method_json.py +42 -42
  27. dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
  28. dtlpy/assets/voc_annotation_template.xml +23 -23
  29. dtlpy/caches/base_cache.py +32 -32
  30. dtlpy/caches/cache.py +473 -473
  31. dtlpy/caches/dl_cache.py +201 -201
  32. dtlpy/caches/filesystem_cache.py +89 -89
  33. dtlpy/caches/redis_cache.py +84 -84
  34. dtlpy/dlp/__init__.py +20 -20
  35. dtlpy/dlp/cli_utilities.py +367 -367
  36. dtlpy/dlp/command_executor.py +764 -764
  37. dtlpy/dlp/dlp +1 -1
  38. dtlpy/dlp/dlp.bat +1 -1
  39. dtlpy/dlp/dlp.py +128 -128
  40. dtlpy/dlp/parser.py +651 -651
  41. dtlpy/entities/__init__.py +83 -83
  42. dtlpy/entities/analytic.py +347 -347
  43. dtlpy/entities/annotation.py +1879 -1879
  44. dtlpy/entities/annotation_collection.py +699 -699
  45. dtlpy/entities/annotation_definitions/__init__.py +20 -20
  46. dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
  47. dtlpy/entities/annotation_definitions/box.py +195 -195
  48. dtlpy/entities/annotation_definitions/classification.py +67 -67
  49. dtlpy/entities/annotation_definitions/comparison.py +72 -72
  50. dtlpy/entities/annotation_definitions/cube.py +204 -204
  51. dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
  52. dtlpy/entities/annotation_definitions/description.py +32 -32
  53. dtlpy/entities/annotation_definitions/ellipse.py +124 -124
  54. dtlpy/entities/annotation_definitions/free_text.py +62 -62
  55. dtlpy/entities/annotation_definitions/gis.py +69 -69
  56. dtlpy/entities/annotation_definitions/note.py +139 -139
  57. dtlpy/entities/annotation_definitions/point.py +117 -117
  58. dtlpy/entities/annotation_definitions/polygon.py +182 -182
  59. dtlpy/entities/annotation_definitions/polyline.py +111 -111
  60. dtlpy/entities/annotation_definitions/pose.py +92 -92
  61. dtlpy/entities/annotation_definitions/ref_image.py +86 -86
  62. dtlpy/entities/annotation_definitions/segmentation.py +240 -240
  63. dtlpy/entities/annotation_definitions/subtitle.py +34 -34
  64. dtlpy/entities/annotation_definitions/text.py +85 -85
  65. dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
  66. dtlpy/entities/app.py +220 -220
  67. dtlpy/entities/app_module.py +107 -107
  68. dtlpy/entities/artifact.py +174 -174
  69. dtlpy/entities/assignment.py +399 -399
  70. dtlpy/entities/base_entity.py +214 -214
  71. dtlpy/entities/bot.py +113 -113
  72. dtlpy/entities/codebase.py +292 -292
  73. dtlpy/entities/collection.py +38 -38
  74. dtlpy/entities/command.py +169 -169
  75. dtlpy/entities/compute.py +449 -449
  76. dtlpy/entities/dataset.py +1299 -1299
  77. dtlpy/entities/directory_tree.py +44 -44
  78. dtlpy/entities/dpk.py +470 -470
  79. dtlpy/entities/driver.py +235 -235
  80. dtlpy/entities/execution.py +397 -397
  81. dtlpy/entities/feature.py +124 -124
  82. dtlpy/entities/feature_set.py +145 -145
  83. dtlpy/entities/filters.py +798 -798
  84. dtlpy/entities/gis_item.py +107 -107
  85. dtlpy/entities/integration.py +184 -184
  86. dtlpy/entities/item.py +959 -959
  87. dtlpy/entities/label.py +123 -123
  88. dtlpy/entities/links.py +85 -85
  89. dtlpy/entities/message.py +175 -175
  90. dtlpy/entities/model.py +684 -684
  91. dtlpy/entities/node.py +1005 -1005
  92. dtlpy/entities/ontology.py +810 -803
  93. dtlpy/entities/organization.py +287 -287
  94. dtlpy/entities/package.py +657 -657
  95. dtlpy/entities/package_defaults.py +5 -5
  96. dtlpy/entities/package_function.py +185 -185
  97. dtlpy/entities/package_module.py +113 -113
  98. dtlpy/entities/package_slot.py +118 -118
  99. dtlpy/entities/paged_entities.py +299 -299
  100. dtlpy/entities/pipeline.py +624 -624
  101. dtlpy/entities/pipeline_execution.py +279 -279
  102. dtlpy/entities/project.py +394 -394
  103. dtlpy/entities/prompt_item.py +505 -505
  104. dtlpy/entities/recipe.py +301 -301
  105. dtlpy/entities/reflect_dict.py +102 -102
  106. dtlpy/entities/resource_execution.py +138 -138
  107. dtlpy/entities/service.py +963 -963
  108. dtlpy/entities/service_driver.py +117 -117
  109. dtlpy/entities/setting.py +294 -294
  110. dtlpy/entities/task.py +495 -495
  111. dtlpy/entities/time_series.py +143 -143
  112. dtlpy/entities/trigger.py +426 -426
  113. dtlpy/entities/user.py +118 -118
  114. dtlpy/entities/webhook.py +124 -124
  115. dtlpy/examples/__init__.py +19 -19
  116. dtlpy/examples/add_labels.py +135 -135
  117. dtlpy/examples/add_metadata_to_item.py +21 -21
  118. dtlpy/examples/annotate_items_using_model.py +65 -65
  119. dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
  120. dtlpy/examples/annotations_convert_to_voc.py +9 -9
  121. dtlpy/examples/annotations_convert_to_yolo.py +9 -9
  122. dtlpy/examples/convert_annotation_types.py +51 -51
  123. dtlpy/examples/converter.py +143 -143
  124. dtlpy/examples/copy_annotations.py +22 -22
  125. dtlpy/examples/copy_folder.py +31 -31
  126. dtlpy/examples/create_annotations.py +51 -51
  127. dtlpy/examples/create_video_annotations.py +83 -83
  128. dtlpy/examples/delete_annotations.py +26 -26
  129. dtlpy/examples/filters.py +113 -113
  130. dtlpy/examples/move_item.py +23 -23
  131. dtlpy/examples/play_video_annotation.py +13 -13
  132. dtlpy/examples/show_item_and_mask.py +53 -53
  133. dtlpy/examples/triggers.py +49 -49
  134. dtlpy/examples/upload_batch_of_items.py +20 -20
  135. dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
  136. dtlpy/examples/upload_items_with_modalities.py +43 -43
  137. dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
  138. dtlpy/examples/upload_yolo_format_annotations.py +70 -70
  139. dtlpy/exceptions.py +125 -125
  140. dtlpy/miscellaneous/__init__.py +20 -20
  141. dtlpy/miscellaneous/dict_differ.py +95 -95
  142. dtlpy/miscellaneous/git_utils.py +217 -217
  143. dtlpy/miscellaneous/json_utils.py +14 -14
  144. dtlpy/miscellaneous/list_print.py +105 -105
  145. dtlpy/miscellaneous/zipping.py +130 -130
  146. dtlpy/ml/__init__.py +20 -20
  147. dtlpy/ml/base_feature_extractor_adapter.py +27 -27
  148. dtlpy/ml/base_model_adapter.py +1257 -1230
  149. dtlpy/ml/metrics.py +461 -461
  150. dtlpy/ml/predictions_utils.py +274 -274
  151. dtlpy/ml/summary_writer.py +57 -57
  152. dtlpy/ml/train_utils.py +60 -60
  153. dtlpy/new_instance.py +252 -252
  154. dtlpy/repositories/__init__.py +56 -56
  155. dtlpy/repositories/analytics.py +85 -85
  156. dtlpy/repositories/annotations.py +916 -916
  157. dtlpy/repositories/apps.py +383 -383
  158. dtlpy/repositories/artifacts.py +452 -452
  159. dtlpy/repositories/assignments.py +599 -599
  160. dtlpy/repositories/bots.py +213 -213
  161. dtlpy/repositories/codebases.py +559 -559
  162. dtlpy/repositories/collections.py +332 -332
  163. dtlpy/repositories/commands.py +152 -152
  164. dtlpy/repositories/compositions.py +61 -61
  165. dtlpy/repositories/computes.py +439 -439
  166. dtlpy/repositories/datasets.py +1504 -1504
  167. dtlpy/repositories/downloader.py +976 -923
  168. dtlpy/repositories/dpks.py +433 -433
  169. dtlpy/repositories/drivers.py +482 -482
  170. dtlpy/repositories/executions.py +815 -815
  171. dtlpy/repositories/feature_sets.py +226 -226
  172. dtlpy/repositories/features.py +255 -255
  173. dtlpy/repositories/integrations.py +484 -484
  174. dtlpy/repositories/items.py +912 -912
  175. dtlpy/repositories/messages.py +94 -94
  176. dtlpy/repositories/models.py +1000 -1000
  177. dtlpy/repositories/nodes.py +80 -80
  178. dtlpy/repositories/ontologies.py +511 -511
  179. dtlpy/repositories/organizations.py +525 -525
  180. dtlpy/repositories/packages.py +1941 -1941
  181. dtlpy/repositories/pipeline_executions.py +451 -451
  182. dtlpy/repositories/pipelines.py +640 -640
  183. dtlpy/repositories/projects.py +539 -539
  184. dtlpy/repositories/recipes.py +419 -399
  185. dtlpy/repositories/resource_executions.py +137 -137
  186. dtlpy/repositories/schema.py +120 -120
  187. dtlpy/repositories/service_drivers.py +213 -213
  188. dtlpy/repositories/services.py +1704 -1704
  189. dtlpy/repositories/settings.py +339 -339
  190. dtlpy/repositories/tasks.py +1477 -1477
  191. dtlpy/repositories/times_series.py +278 -278
  192. dtlpy/repositories/triggers.py +536 -536
  193. dtlpy/repositories/upload_element.py +257 -257
  194. dtlpy/repositories/uploader.py +661 -661
  195. dtlpy/repositories/webhooks.py +249 -249
  196. dtlpy/services/__init__.py +22 -22
  197. dtlpy/services/aihttp_retry.py +131 -131
  198. dtlpy/services/api_client.py +1785 -1785
  199. dtlpy/services/api_reference.py +40 -40
  200. dtlpy/services/async_utils.py +133 -133
  201. dtlpy/services/calls_counter.py +44 -44
  202. dtlpy/services/check_sdk.py +68 -68
  203. dtlpy/services/cookie.py +115 -115
  204. dtlpy/services/create_logger.py +156 -156
  205. dtlpy/services/events.py +84 -84
  206. dtlpy/services/logins.py +235 -235
  207. dtlpy/services/reporter.py +256 -256
  208. dtlpy/services/service_defaults.py +91 -91
  209. dtlpy/utilities/__init__.py +20 -20
  210. dtlpy/utilities/annotations/__init__.py +16 -16
  211. dtlpy/utilities/annotations/annotation_converters.py +269 -269
  212. dtlpy/utilities/base_package_runner.py +285 -264
  213. dtlpy/utilities/converter.py +1650 -1650
  214. dtlpy/utilities/dataset_generators/__init__.py +1 -1
  215. dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
  216. dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
  217. dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
  218. dtlpy/utilities/local_development/__init__.py +1 -1
  219. dtlpy/utilities/local_development/local_session.py +179 -179
  220. dtlpy/utilities/reports/__init__.py +2 -2
  221. dtlpy/utilities/reports/figures.py +343 -343
  222. dtlpy/utilities/reports/report.py +71 -71
  223. dtlpy/utilities/videos/__init__.py +17 -17
  224. dtlpy/utilities/videos/video_player.py +598 -598
  225. dtlpy/utilities/videos/videos.py +470 -470
  226. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
  227. dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
  228. {dtlpy-1.115.44.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
  229. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -186
  230. dtlpy-1.116.6.dist-info/RECORD +239 -0
  231. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
  232. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
  233. tests/features/environment.py +551 -551
  234. dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
  235. dtlpy-1.115.44.data/scripts/dlp.bat +0 -2
  236. dtlpy-1.115.44.dist-info/RECORD +0 -240
  237. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
  238. {dtlpy-1.115.44.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
@@ -1,1704 +1,1704 @@
1
- import datetime
2
- import inspect
3
- import logging
4
- import json
5
- import os
6
- import tempfile
7
- import time
8
- from typing import Union, List, Callable
9
- from .. import miscellaneous, exceptions, entities, repositories, assets, ApiClient, _api_reference
10
- from ..__version__ import version as __version__
11
-
12
- logger = logging.getLogger(name='dtlpy')
13
- FUNCTION_END_LINE = '[Done] Executing function.'
14
- MAX_WAIT_TIME = 8
15
-
16
-
17
- class Services:
18
- """
19
- Services Repository
20
-
21
- The Services class allows the user to manage services and their properties. Services are created from the packages users create.
22
- See our documentation for more information about `services <https://developers.dataloop.ai/tutorials/faas/advance/chapter/>`_.
23
- """
24
-
25
- def __init__(self,
26
- client_api: ApiClient,
27
- project: entities.Project = None,
28
- package: Union[entities.Package, entities.Dpk] = None,
29
- project_id=None,
30
- model_id=None,
31
- model: entities.Model = None):
32
- self._client_api = client_api
33
- self._package = package
34
- self._project = project
35
- self._model = model
36
- if model_id is None:
37
- if model is not None:
38
- model_id = model.id
39
- self._model_id = model_id
40
- if project_id is None:
41
- if project is not None:
42
- project_id = project.id
43
- self._project_id = project_id
44
- self._settings = repositories.Settings(project=project, client_api=client_api)
45
-
46
- ############
47
- # entities #
48
- ############
49
- @property
50
- def package(self) -> entities.Package:
51
- if self._package is None:
52
- raise exceptions.PlatformException(
53
- error='2001',
54
- message='Cannot perform action WITHOUT package entity in services repository. Please set a package')
55
- assert (isinstance(self._package, entities.Package) or isinstance(self._package, entities.Dpk))
56
- return self._package
57
-
58
- @package.setter
59
- def package(self, package: Union[entities.Package, entities.Dpk]):
60
- if not isinstance(package, entities.Package) and not isinstance(package, entities.Dpk):
61
- raise ValueError('Must input a valid package entity')
62
- self._package = package
63
-
64
- @property
65
- def project(self) -> entities.Project:
66
- if self._project is None:
67
- # try to get from package
68
- if self._package is not None:
69
- self._project = self._package._project
70
-
71
- if self._project is None:
72
- # try to get checked out project
73
- project = self._client_api.state_io.get('project')
74
- if project is not None:
75
- self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
76
- if self._project is None:
77
- raise exceptions.PlatformException(
78
- error='2001',
79
- message='Cannot perform action WITHOUT Project entity in services repository. Please set a project')
80
- return self._project
81
-
82
- @project.setter
83
- def project(self, project: entities.Project):
84
- if not isinstance(project, entities.Project):
85
- raise ValueError('Must input a valid Project entity')
86
- self._project = project
87
- self._project_id = project.id
88
-
89
- @property
90
- def model(self) -> entities.Model:
91
- if self._model is None:
92
- if self._model_id is not None:
93
- self._model = self.project.models.get(model_id=self._model_id)
94
- else:
95
- raise exceptions.PlatformException(
96
- error='2001',
97
- message='Cannot perform action WITHOUT model entity in services repository. Please set a model')
98
- assert isinstance(self._model, entities.Model)
99
- return self._model
100
-
101
- @model.setter
102
- def model(self, model: entities.Model):
103
- if not isinstance(model, entities.Model):
104
- raise ValueError('Must input a valid model entity')
105
- self._model = model
106
- self._model_id = model.id
107
-
108
- @property
109
- def platform_url(self):
110
- return self._client_api._get_resource_url("projects/{}/services".format(self.project.id))
111
-
112
- def open_in_web(self,
113
- service: entities.Service = None,
114
- service_id: str = None,
115
- service_name: str = None):
116
- """
117
- Open the service in web platform
118
-
119
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
120
-
121
- :param str service_name: service name
122
- :param str service_id: service id
123
- :param dtlpy.entities.service.Service service: service entity
124
-
125
- **Example**:
126
-
127
- .. code-block:: python
128
-
129
- package.services.open_in_web(service_id='service_id')
130
- """
131
- if service_name is not None:
132
- service = self.get(service_name=service_name)
133
- if service is not None:
134
- service.open_in_web()
135
- elif service_id is not None:
136
- self._client_api._open_in_web(url=self.platform_url + '/' + str(service_id) + '/main')
137
- else:
138
- self._client_api._open_in_web(url=self.platform_url)
139
-
140
- def __get_from_cache(self) -> entities.Service:
141
- service = self._client_api.state_io.get('service')
142
- if service is not None:
143
- service = entities.Service.from_json(_json=service,
144
- client_api=self._client_api,
145
- project=self._project,
146
- package=self._package)
147
- return service
148
-
149
- def checkout(self,
150
- service: entities.Service = None,
151
- service_name: str = None,
152
- service_id: str = None):
153
- """
154
- Checkout (switch) to a service.
155
-
156
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
157
-
158
- :param dtlpy.entities.service.Service service: Service entity
159
- :param str service_name: service name
160
- :param str service_id: service id
161
-
162
- **Example**:
163
-
164
- .. code-block:: python
165
-
166
- package.services.checkout(service_id='service_id')
167
- """
168
- if service is None:
169
- service = self.get(service_name=service_name, service_id=service_id)
170
- self._client_api.state_io.put('service', service.to_json())
171
- logger.info('Checked out to service {}'.format(service.name))
172
-
173
- ###########
174
- # methods #
175
- ###########
176
- @_api_reference.add(path='/services/{id}/revisions', method='get')
177
- def revisions(self,
178
- service: entities.Service = None,
179
- service_id: str = None):
180
- """
181
- Get service revisions history.
182
-
183
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
184
-
185
- You must provide at leats ONE of the following params: service, service_id
186
-
187
- :param dtlpy.entities.service.Service service: Service entity
188
- :param str service_id: service id
189
-
190
- **Example**:
191
-
192
- .. code-block:: python
193
-
194
- service_revision = package.services.revisions(service_id='service_id')
195
- """
196
- if service is None and service_id is None:
197
- raise exceptions.PlatformException(
198
- error='400',
199
- message='must provide an identifier in inputs: "service" or "service_id"')
200
- if service is not None:
201
- service_id = service.id
202
-
203
- success, response = self._client_api.gen_request(
204
- req_type="get",
205
- path="/services/{}/revisions".format(service_id))
206
- if not success:
207
- raise exceptions.PlatformException(response)
208
- return response.json()
209
-
210
- @_api_reference.add(path='/services/{id}', method='get')
211
- def get(self,
212
- service_name=None,
213
- service_id=None,
214
- checkout=False,
215
- fetch=None
216
- ) -> entities.Service:
217
- """
218
- Get service to use in your code.
219
-
220
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
221
-
222
- :param str service_name: optional - search by name
223
- :param str service_id: optional - search by id
224
- :param bool checkout: if true, checkout (switch) to service
225
- :param fetch: optional - fetch entity from platform, default taken from cookie
226
- :return: Service object
227
- :rtype: dtlpy.entities.service.Service
228
-
229
- **Example**:
230
-
231
- .. code-block:: python
232
-
233
- service = package.services.get(service_id='service_id')
234
- """
235
- if fetch is None:
236
- fetch = self._client_api.fetch_entities
237
-
238
- if service_name is None and service_id is None:
239
- service = self.__get_from_cache()
240
- if service is None:
241
- raise exceptions.PlatformException(
242
- error='400',
243
- message='No checked-out Service was found, must checkout or provide an identifier in inputs')
244
-
245
- elif fetch:
246
- if service_id is not None:
247
- success, response = self._client_api.gen_request(
248
- req_type="get",
249
- path="/services/{}".format(service_id)
250
- )
251
- if not success:
252
- raise exceptions.PlatformException(response)
253
- service = entities.Service.from_json(client_api=self._client_api,
254
- _json=response.json(),
255
- package=self._package,
256
- project=self._project)
257
- # verify input service name is same as the given id
258
- if service_name is not None and service.name != service_name:
259
- logger.warning(
260
- "Mismatch found in services.get: service_name is different then service.name:"
261
- " {!r} != {!r}".format(
262
- service_name,
263
- service.name))
264
- elif service_name is not None:
265
- filters = entities.Filters(resource=entities.FiltersResource.SERVICE,
266
- field='name',
267
- values=service_name,
268
- use_defaults=False)
269
- if self._project_id is not None:
270
- filters.add(field='projectId', values=self._project_id)
271
- if self._package is not None:
272
- filters.add(field='packageId', values=self._package.id)
273
- services = self.list(filters=filters)
274
- if services.items_count > 1:
275
- raise exceptions.PlatformException('404', 'More than one service with same name. '
276
- 'Please get services from package/project entity')
277
- elif services.items_count == 0:
278
- raise exceptions.PlatformException('404', 'Service not found: {}.'.format(service_name))
279
- service = services.items[0]
280
- else:
281
- raise exceptions.PlatformException(
282
- error='400',
283
- message='No checked-out Service was found, must checkout or provide an identifier in inputs')
284
- else:
285
- service = entities.Service.from_json(_json={'id': service_id,
286
- 'name': service_name},
287
- client_api=self._client_api,
288
- project=self._project,
289
- package=self._package,
290
- is_fetched=False)
291
-
292
- assert isinstance(service, entities.Service)
293
- if checkout:
294
- self.checkout(service=service)
295
- return service
296
-
297
- def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Service]:
298
- jobs = [None for _ in range(len(response_items))]
299
- pool = self._client_api.thread_pools(pool_name='entity.create')
300
-
301
- # return triggers list
302
- for i_service, service in enumerate(response_items):
303
- jobs[i_service] = pool.submit(entities.Service._protected_from_json,
304
- **{'client_api': self._client_api,
305
- '_json': service,
306
- 'package': self._package,
307
- 'project': self._project})
308
-
309
- # get all results
310
- results = [j.result() for j in jobs]
311
- # log errors
312
- _ = [logger.warning(r[1]) for r in results if r[0] is False]
313
- # return good jobs
314
- return miscellaneous.List([r[1] for r in results if r[0] is True])
315
-
316
- def _list(self, filters: entities.Filters):
317
- url = '/query/faas'
318
- success, response = self._client_api.gen_request(req_type='POST',
319
- path=url,
320
- json_req=filters.prepare())
321
- if not success:
322
- raise exceptions.PlatformException(response)
323
-
324
- return response.json()
325
-
326
- @_api_reference.add(path='/query/faas', method='post')
327
- def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
328
- """
329
- List all services (services can be listed for a package or for a project).
330
-
331
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
332
-
333
- :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
334
- :return: Paged entity
335
- :rtype: dtlpy.entities.paged_entities.PagedEntities
336
-
337
- **Example**:
338
-
339
- .. code-block:: python
340
-
341
- services = package.services.list()
342
- """
343
- # default filters
344
- if filters is None:
345
- filters = entities.Filters(resource=entities.FiltersResource.SERVICE)
346
- # assert type filters
347
- elif not isinstance(filters, entities.Filters):
348
- raise exceptions.PlatformException(error='400',
349
- message='Unknown filters type: {!r}'.format(type(filters)))
350
- if filters.resource != entities.FiltersResource.SERVICE:
351
- raise exceptions.PlatformException(
352
- error='400',
353
- message='Filters resource must to be FiltersResource.SERVICE. Got: {!r}'.format(filters.resource))
354
- if self._project_id is not None:
355
- filters.add(field='projectId', values=self._project_id)
356
- if self._model_id is not None:
357
- filters.add(field='metadata.ml.modelId', values=self._model_id)
358
- if self._package is not None:
359
- filters.add(field='packageId', values=self._package.id)
360
-
361
- # assert type filters
362
- if not isinstance(filters, entities.Filters):
363
- raise exceptions.PlatformException('400', 'Unknown filters type')
364
-
365
- paged = entities.PagedEntities(items_repository=self,
366
- filters=filters,
367
- page_offset=filters.page,
368
- page_size=filters.page_size,
369
- client_api=self._client_api)
370
- paged.get_page()
371
- return paged
372
-
373
- @_api_reference.add(path='/services/{id}/status', method='post')
374
- def status(self, service_name=None, service_id=None):
375
- """
376
- Get service status.
377
-
378
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
379
-
380
- You must provide at least ONE of the following params: service_id, service_name
381
-
382
- :param str service_name: service name
383
- :param str service_id: service id
384
- :return: status json
385
- :rtype: dict
386
-
387
- **Example**:
388
-
389
- .. code-block:: python
390
-
391
- status_json = package.services.status(service_id='service_id')
392
- """
393
- if service_id is None:
394
- if service_name is None:
395
- raise exceptions.PlatformException(error='400',
396
- message='must input "service_name" or "service_id" to get status')
397
- service = self.get(service_name=service_name)
398
- service_id = service.id
399
- # request
400
- success, response = self._client_api.gen_request(req_type="get",
401
- path="/services/{}/status".format(service_id))
402
- if not success:
403
- raise exceptions.PlatformException(response)
404
- return response.json()
405
-
406
- @_api_reference.add(path='/services/{id}/stop', method='post')
407
- def pause(self,
408
- service_name: str = None,
409
- service_id: str = None,
410
- force: bool = False):
411
- """
412
- Pause service.
413
-
414
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
415
-
416
- You must provide at least ONE of the following params: service_id, service_name
417
-
418
- :param str service_name: service name
419
- :param str service_id: service id
420
- :param bool force: optional - terminate old replicas immediately
421
- :return: True if success
422
- :rtype: bool
423
-
424
- **Example**:
425
-
426
- .. code-block:: python
427
-
428
- success = package.services.pause(service_id='service_id')
429
- """
430
- if service_id is None:
431
- if service_name is None:
432
- raise exceptions.PlatformException(error='400',
433
- message='must input "service_name" or "service_id" to pause service')
434
- service = self.get(service_name=service_name)
435
- service_id = service.id
436
- # request
437
- url = "/services/{}/stop".format(service_id)
438
- if force:
439
- url = '{}?force=true'
440
- success, response = self._client_api.gen_request(req_type="post",
441
- path=url)
442
- if not success:
443
- raise exceptions.PlatformException(response)
444
- return success
445
-
446
- def _notify(
447
- self,
448
- service_id: str,
449
- message: str,
450
- name: str,
451
- action: str = 'created',
452
- support: str = None,
453
- docs: str = None,
454
- agent_info: dict = None
455
- ):
456
- url = "/services/{}/notify".format(service_id)
457
- payload = {
458
- 'action': action,
459
- 'message': message,
460
- 'notificationName': name
461
- }
462
- if agent_info is not None:
463
- payload['agentInfo'] = agent_info
464
-
465
- if support:
466
- payload['support'] = support
467
-
468
- if docs:
469
- payload['docs'] = docs
470
-
471
- success, response = self._client_api.gen_request(
472
- req_type="post",
473
- path=url,
474
- json_req=payload
475
- )
476
- if not success:
477
- raise exceptions.PlatformException(response)
478
- return success
479
-
480
- @_api_reference.add(path='/services/{id}/resume', method='post')
481
- def resume(self,
482
- service_name: str = None,
483
- service_id: str = None,
484
- force: bool = False):
485
- """
486
- Resume service.
487
-
488
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
489
-
490
- You must provide at least ONE of the following params: service_id, service_name.
491
-
492
- :param str service_name: service name
493
- :param str service_id: service id
494
- :param bool force: optional - terminate old replicas immediately
495
- :return: json of the service
496
- :rtype: dict
497
-
498
- **Example**:
499
-
500
- .. code-block:: python
501
-
502
- service_json = package.services.resume(service_id='service_id')
503
- """
504
- if service_id is None:
505
- if service_name is None:
506
- raise exceptions.PlatformException(error='400',
507
- message='must input "service_name" or "service_id" to resume')
508
- service = self.get(service_name=service_name)
509
- service_id = service.id
510
- # request
511
- url = "/services/{}/resume".format(service_id)
512
- if force:
513
- url = '{}?force=true'
514
- success, response = self._client_api.gen_request(req_type="post",
515
- path=url)
516
- if not success:
517
- raise exceptions.PlatformException(response)
518
- return response.json()
519
-
520
- def _get_bot_email(self, bot=None):
521
-
522
- if bot is None:
523
- project = self._project
524
- if project is None and self._project_id is not None:
525
- project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
526
-
527
- if project is None:
528
- raise exceptions.PlatformException(error='2001',
529
- message='Need project entity or bot to perform this action')
530
- bots = project.bots.list()
531
- if len(bots) == 0:
532
- logger.info('Bot not found for project. Creating a default bot')
533
- bot = project.bots.create(name='default')
534
- else:
535
- bot = bots[0]
536
- if len(bots) > 1:
537
- logger.debug('More than one bot users. Choosing first. email: {}'.format(bots[0].email))
538
-
539
- if isinstance(bot, str):
540
- bot_email = bot
541
- elif isinstance(bot, entities.Bot) or isinstance(bot, entities.User):
542
- bot_email = bot.email
543
- else:
544
- raise ValueError('input "bot" must be a str or a Bot type, got: {}'.format(type(bot)))
545
-
546
- return bot_email
547
-
548
- @staticmethod
549
- def _parse_init_input(init_input):
550
- if not isinstance(init_input, dict):
551
- if init_input is None:
552
- init_input = dict()
553
- else:
554
- init_params = dict()
555
- if not isinstance(init_input, list):
556
- init_input = [init_input]
557
- for param in init_input:
558
- if isinstance(param, entities.FunctionIO):
559
- init_params.update(param.to_json(resource='service'))
560
- else:
561
- raise exceptions.PlatformException('400', 'Unknown type of init params')
562
- init_input = init_params
563
-
564
- return init_input
565
-
566
- def name_validation(self, name: str):
567
- """
568
- Validation service name.
569
-
570
- **Prerequisites**: You must be in the role of an *owner* or *developer*.
571
-
572
- :param str name: service name
573
-
574
- **Example**:
575
-
576
- .. code-block:: python
577
-
578
- package.services.name_validation(name='name')
579
- """
580
- url = '/piper-misc/naming/services/{}'.format(name)
581
-
582
- # request
583
- success, response = self._client_api.gen_request(req_type='get',
584
- path=url)
585
- if not success:
586
- raise exceptions.PlatformException(response)
587
-
588
- def _create(self,
589
- service_name: str = None,
590
- package: entities.Package = None,
591
- module_name: str = None,
592
- bot: Union[entities.Bot, str] = None,
593
- revision: str or int = None,
594
- init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
595
- runtime: Union[entities.KubernetesRuntime, dict] = None,
596
- pod_type: entities.InstanceCatalog = None,
597
- project_id: str = None,
598
- sdk_version: str = None,
599
- agent_versions: dict = None,
600
- verify: bool = True,
601
- driver_id: str = None,
602
- run_execution_as_process: bool = None,
603
- execution_timeout: int = None,
604
- drain_time: int = None,
605
- on_reset: str = None,
606
- max_attempts: int = None,
607
- secrets=None,
608
- integrations=None,
609
- active: bool = True,
610
- **kwargs
611
- ) -> entities.Service:
612
- """
613
- Create service entity.
614
-
615
-
616
- :param str service_name: service name
617
- :param dtlpy.entities.package.Package package: package entity
618
- :param str module_name: module name
619
- :param str bot: bot email
620
- :param str revision: package revision - default=latest
621
- :param init_input: config to run at startup
622
- :param dict runtime: runtime resources
623
- :param str pod_type: pod type dl.InstanceCatalog
624
- :param str project_id: project id
625
- :param str sdk_version: - optional - string - sdk version
626
- :param dict agent_versions: - dictionary - - optional -versions of sdk, agent runner and agent proxy
627
- :param bool verify: verify the inputs
628
- :param str driver_id: driver id
629
- :param bool run_execution_as_process: run execution as process
630
- :param int execution_timeout: execution timeout
631
- :param int drain_time: drain time
632
- :param str on_reset: on reset
633
- :param int max_attempts: Maximum execution retries in-case of a service reset
634
- :param bool force: optional - terminate old replicas immediately
635
- :param list secrets: list of the integrations ids
636
- :param list integrations: list of the integrations
637
- :param bool active: optional - if true, service will be active
638
- :param kwargs:
639
- :return: Service object
640
- :rtype: dtlpy.entities.service.Service
641
- """
642
- if package is None:
643
- if self._package is None:
644
- raise exceptions.PlatformException('400', 'Please provide param package')
645
- package = self._package
646
-
647
- if verify is not None:
648
- logger.warning('verify attribute has been deprecated and will be ignored')
649
-
650
- is_global = kwargs.get('is_global', None)
651
- jwt_forward = kwargs.get('jwt_forward', None)
652
-
653
- if is_global is not None or jwt_forward is not None:
654
- logger.warning(
655
- 'Params jwt_forward and is_global are restricted to superuser. '
656
- 'If you are not a superuser this action will not work'
657
- )
658
-
659
- service_config = dict()
660
- if package is not None and package.service_config is not None:
661
- service_config = package.service_config
662
-
663
- if agent_versions is None:
664
- if sdk_version is None:
665
- sdk_version = service_config.get('versions', dict()).get('dtlpy', __version__)
666
- agent_versions = {
667
- "dtlpy": sdk_version
668
- }
669
-
670
- if project_id is None:
671
- if self._project is None and self._project_id is None:
672
- raise exceptions.PlatformException('400', 'Please provide project id')
673
- elif self._project_id is not None:
674
- project_id = self._project_id
675
- elif self._project is not None:
676
- project_id = self._project.id
677
-
678
- if service_name is None:
679
- service_name = 'default-service'
680
-
681
- # payload
682
- payload = {
683
- 'name': service_name,
684
- 'projectId': project_id,
685
- 'packageId': package.id,
686
- 'initParams': self._parse_init_input(init_input=init_input),
687
- 'botUserName': self._get_bot_email(bot=bot),
688
- 'versions': agent_versions,
689
- 'packageRevision': revision if revision is not None else package.version,
690
- 'driverId': driver_id,
691
- 'active': active
692
- }
693
-
694
- if secrets is not None:
695
- if not isinstance(secrets, list):
696
- secrets = [secrets]
697
- payload['secrets'] = secrets
698
-
699
- if integrations is not None:
700
- if not isinstance(integrations, list):
701
- integrations = [integrations]
702
- payload['integrations'] = integrations
703
-
704
- if runtime is not None:
705
- if isinstance(runtime, entities.KubernetesRuntime):
706
- runtime = runtime.to_json()
707
-
708
- if pod_type is not None:
709
- if runtime is None:
710
- runtime = {'podType': pod_type}
711
- else:
712
- runtime['podType'] = pod_type
713
-
714
- if runtime is not None:
715
- payload['runtime'] = runtime
716
-
717
- if module_name is not None:
718
- payload['moduleName'] = module_name
719
-
720
- if is_global is not None:
721
- payload['global'] = is_global
722
-
723
- if max_attempts is not None:
724
- payload['maxAttempts'] = max_attempts
725
-
726
- if jwt_forward is not None:
727
- payload['useUserJwt'] = jwt_forward
728
-
729
- if run_execution_as_process is not None:
730
- payload['runExecutionAsProcess'] = run_execution_as_process
731
-
732
- if drain_time is not None:
733
- payload['drainTime'] = drain_time
734
-
735
- if on_reset is not None:
736
- payload['onReset'] = on_reset
737
-
738
- if execution_timeout is not None:
739
- payload['executionTimeout'] = execution_timeout
740
-
741
- # request
742
- success, response = self._client_api.gen_request(
743
- req_type='post',
744
- path='/services',
745
- json_req=payload
746
- )
747
-
748
- # exception handling
749
- if not success:
750
- raise exceptions.PlatformException(response)
751
-
752
- # return entity
753
- return entities.Service.from_json(
754
- _json=response.json(),
755
- client_api=self._client_api,
756
- package=package,
757
- project=self._project
758
- )
759
-
760
- @_api_reference.add(path='/services/{id}', method='delete')
761
- def delete(self, service_name: str = None, service_id: str = None, force=False):
762
- """
763
- Delete Service object
764
-
765
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
766
-
767
- You must provide at least ONE of the following params: service_id, service_name.
768
-
769
- :param force:
770
- :param str service_name: by name
771
- :param str service_id: by id
772
- :return: True
773
- :rtype: bool
774
-
775
- **Example**:
776
-
777
- .. code-block:: python
778
-
779
- is_deleted = package.services.delete(service_id='service_id')
780
- """
781
- # get bby name
782
- if service_id is None:
783
- if service_name is None:
784
- raise exceptions.PlatformException('400', 'Must provide either service id or service name')
785
- else:
786
- service_id = self.get(service_name=service_name).id
787
-
788
- path = "/services/{}".format(service_id)
789
- if force:
790
- path = '{}?force=true'.format(path)
791
-
792
- # request
793
- success, response = self._client_api.gen_request(
794
- req_type="delete",
795
- path=path
796
- )
797
- if not success:
798
- raise exceptions.PlatformException(response)
799
- return True
800
-
801
- @_api_reference.add(path='/services/{id}', method='patch')
802
- def update(self, service: entities.Service, force: bool = False) -> entities.Service:
803
- """
804
- Update service changes to platform.
805
-
806
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
807
-
808
- :param dtlpy.entities.service.Service service: Service entity
809
- :param bool force: optional - terminate old replicas immediately
810
- :return: Service entity
811
- :rtype: dtlpy.entities.service.Service
812
-
813
- **Example**:
814
-
815
- .. code-block:: python
816
-
817
- service = package.services.update(service='service_entity')
818
- """
819
- assert isinstance(service, entities.Service)
820
-
821
- # payload
822
- payload = service.to_json()
823
-
824
- # request
825
- url = '/services/{}'.format(service.id)
826
- if force:
827
- url = '{}?force=true'.format(url)
828
- success, response = self._client_api.gen_request(req_type='patch',
829
- path=url,
830
- json_req=payload)
831
-
832
- # exception handling
833
- if not success:
834
- raise exceptions.PlatformException(response)
835
-
836
- # return entity
837
- if self._package is not None:
838
- package = self._package
839
- else:
840
- package = service._package
841
-
842
- return entities.Service.from_json(_json=response.json(),
843
- client_api=self._client_api,
844
- package=package,
845
- project=self._project)
846
-
847
- def activate_slots(
848
- self,
849
- service: entities.Service,
850
- project_id: str = None,
851
- task_id: str = None,
852
- dataset_id: str = None,
853
- org_id: str = None,
854
- user_email: str = None,
855
- slots: List[entities.PackageSlot] = None,
856
- role=None,
857
- prevent_override: bool = True,
858
- visible: bool = True,
859
- icon: str = 'fas fa-magic',
860
- **kwargs
861
- ):
862
- """
863
- Activate service slots (creates buttons in the UI that activate services).
864
-
865
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
866
-
867
- :param dtlpy.entities.service.Service service: service entity
868
- :param str project_id: project id
869
- :param str task_id: task id
870
- :param str dataset_id: dataset id
871
- :param str org_id: org id
872
- :param str user_email: user email
873
- :param list slots: list of entities.PackageSlot
874
- :param str role: user role MemberOrgRole.ADMIN, MemberOrgRole.owner, MemberOrgRole.MEMBER, MemberOrgRole.WORKER
875
- :param bool prevent_override: True to prevent override
876
- :param bool visible: visible
877
- :param str icon: icon
878
- :param kwargs: all additional arguments
879
- :return: list of user setting for activated slots
880
- :rtype: list
881
-
882
- **Example**:
883
-
884
- .. code-block:: python
885
-
886
- setting = package.services.activate_slots(service='service_entity',
887
- project_id='project_id',
888
- slots=List[entities.PackageSlot],
889
- icon='fas fa-magic')
890
- """
891
- package = service.package
892
- if not isinstance(package.slots, list) or len(package.slots) == 0:
893
- raise exceptions.PlatformException('400', "Service's package has no slots")
894
-
895
- if kwargs.get('is_global', False):
896
- project_id = '*'
897
- scope_ids = [project_id]
898
- else:
899
- scope_ids = [s_id for s_id in [project_id, task_id, org_id, dataset_id, user_email] if s_id is not None]
900
- if len(scope_ids) == 0:
901
- raise exceptions.PlatformException('400', "Must provide scope resource ID")
902
-
903
- settings = list()
904
-
905
- if role is None:
906
- role = entities.Role.ALL
907
-
908
- if not slots:
909
- slots = [s.to_json() for s in service.package.slots]
910
- elif isinstance(slots, list) and isinstance(slots[0], entities.PackageSlot):
911
- slots = [s.to_json() for s in slots]
912
- else:
913
- raise exceptions.PlatformException('400', "Slots param must be a list of PackageSlot objects")
914
-
915
- for scope_id in scope_ids:
916
-
917
- if kwargs.get('is_global', False):
918
- scope_type = entities.PlatformEntityType.DATALOOP
919
- elif scope_id == project_id:
920
- scope_type = entities.PlatformEntityType.PROJECT
921
- elif scope_id == task_id:
922
- scope_type = entities.PlatformEntityType.TASK
923
- elif scope_id == dataset_id:
924
- scope_type = entities.PlatformEntityType.DATASET
925
- elif scope_id == user_email:
926
- scope_type = entities.PlatformEntityType.USER
927
- elif scope_id == org_id:
928
- scope_type = entities.PlatformEntityType.ORG
929
- else:
930
- raise exceptions.PlatformException('400', "Unknown resource id")
931
-
932
- setting = entities.Setting(
933
- default_value=True,
934
- value=True,
935
- inputs=None,
936
- name=service.name,
937
- value_type=entities.SettingsValueTypes.BOOLEAN,
938
- scope=entities.SettingScope(
939
- type=scope_type,
940
- id=scope_id,
941
- role=role,
942
- prevent_override=prevent_override,
943
- visible=visible
944
- ),
945
- metadata={
946
- 'serviceId': service.id,
947
- 'serviceName': service.name,
948
- 'projectId': service.project_id,
949
- 'slots': slots
950
- },
951
- description=service.name,
952
- icon=icon,
953
- section_name=entities.SettingsSectionNames.APPLICATIONS,
954
- sub_section_name=None,
955
- hint=None
956
- )
957
-
958
- try:
959
- settings.append(self._settings.create(setting=setting))
960
- except exceptions.BadRequest as err:
961
- if 'settings already exist' in err.message:
962
- old_setting = self._settings.get(setting_name=setting.name)
963
- setting.id = old_setting.id
964
- settings.append(self._settings.update(setting=setting))
965
- else:
966
- raise err
967
-
968
- return settings
969
-
970
- @_api_reference.add(path='/services/logs', method='post')
971
- def log(self,
972
- service,
973
- size=100,
974
- checkpoint=None,
975
- start=None,
976
- end=None,
977
- follow=False,
978
- text=None,
979
- execution_id=None,
980
- function_name=None,
981
- replica_id=None,
982
- system=False,
983
- view=True,
984
- until_completed=True,
985
- log_level='DEBUG',
986
- model_id: str = None,
987
- model_operation: str = None,
988
- project_id: str = None
989
- ):
990
- """
991
- Get service logs.
992
-
993
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
994
-
995
- :param dtlpy.entities.service.Service service: service object
996
- :param int size: size
997
- :param dict checkpoint: the information from the lst point checked in the service
998
- :param str start: iso format time
999
- :param str end: iso format time
1000
- :param bool follow: if true, keep stream future logs
1001
- :param str text: text
1002
- :param str execution_id: execution id
1003
- :param str function_name: function name
1004
- :param str replica_id: replica id
1005
- :param bool system: system
1006
- :param bool view: if true, print out all the logs
1007
- :param bool until_completed: wait until completed
1008
- :param str log_level: the log level to display dl.LoggingLevel
1009
- :param str model_id: model id
1010
- :param str model_operation: model operation action
1011
- :param str project_id: project id
1012
- :return: ServiceLog entity
1013
- :rtype: ServiceLog
1014
-
1015
- **Example**:
1016
-
1017
- .. code-block:: python
1018
-
1019
- service_logs = package.services.log(service='service_entity')
1020
- """
1021
- if not isinstance(service, entities.Service) and model_id is None:
1022
- raise exceptions.PlatformException('400', 'Must provide service or model_id')
1023
- if isinstance(log_level, str):
1024
- log_level = log_level.upper()
1025
-
1026
- payload = {
1027
- 'direction': 'asc',
1028
- 'follow': follow,
1029
- 'system': system,
1030
- 'logLevel': log_level
1031
- }
1032
-
1033
- if service is not None:
1034
- payload['serviceId'] = service.id
1035
-
1036
- if size is not None:
1037
- payload['size'] = size
1038
-
1039
- if execution_id is not None:
1040
- payload['executionId'] = [execution_id]
1041
-
1042
- if function_name is not None:
1043
- payload['functionName'] = function_name
1044
-
1045
- if text is not None:
1046
- payload['text'] = text
1047
-
1048
- if replica_id is not None:
1049
- payload['replicaId'] = replica_id
1050
-
1051
- if checkpoint is not None:
1052
- payload['checkpoint'] = checkpoint
1053
-
1054
- if start is not None:
1055
- payload['start'] = start
1056
- else:
1057
- payload['start'] = datetime.datetime(datetime.date.today().year,
1058
- datetime.date.today().month,
1059
- datetime.date.today().day,
1060
- 0,
1061
- 0,
1062
- 0).isoformat()
1063
-
1064
- if end is not None:
1065
- payload['end'] = end
1066
-
1067
- if model_id is not None:
1068
- payload['modelId'] = model_id
1069
-
1070
- if model_operation is not None:
1071
- payload['modelOperation'] = model_operation
1072
-
1073
- if project_id is not None:
1074
- payload['projectId'] = project_id
1075
- else:
1076
- payload['projectId'] = service.project_id
1077
-
1078
- # request
1079
- success, response = self._client_api.gen_request(req_type='post',
1080
- path='/services/logs',
1081
- json_req=payload)
1082
-
1083
- # exception handling
1084
- if not success:
1085
- raise exceptions.PlatformException(response)
1086
-
1087
- log = ServiceLog(_json=response.json(),
1088
- service=service,
1089
- services=self,
1090
- start=payload['start'],
1091
- follow=follow,
1092
- execution_id=execution_id,
1093
- function_name=function_name,
1094
- replica_id=replica_id,
1095
- system=system,
1096
- model_id=model_id,
1097
- model_operation=model_operation,
1098
- project_id=project_id)
1099
-
1100
- if view:
1101
- log.view(until_completed=until_completed)
1102
- else:
1103
- return log
1104
-
1105
- def execute(self,
1106
- service: entities.Service = None,
1107
- service_id: str = None,
1108
- service_name: str = None,
1109
- sync: bool = False,
1110
- function_name: str = None,
1111
- stream_logs: bool = False,
1112
- execution_input=None,
1113
- resource=None,
1114
- item_id=None,
1115
- dataset_id=None,
1116
- annotation_id=None,
1117
- project_id=None,
1118
- ) -> entities.Execution:
1119
- """
1120
- Execute a function on an existing service.
1121
-
1122
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1123
-
1124
- :param dtlpy.entities.service.Service service: service entity
1125
- :param str service_id: service id
1126
- :param str service_name: service name
1127
- :param bool sync: wait for function to end
1128
- :param str function_name: function name to run
1129
- :param bool stream_logs: prints logs of the new execution. only works with sync=True
1130
- :param execution_input: input dictionary or list of FunctionIO entities
1131
- :param str resource: dl.PackageInputType - input type.
1132
- :param str item_id: str - optional - input to function
1133
- :param str dataset_id: str - optional - input to function
1134
- :param str annotation_id: str - optional - input to function
1135
- :param str project_id: str - resource's project
1136
- :return: entities.Execution
1137
- :rtype: dtlpy.entities.execution.Execution
1138
-
1139
- **Example**:
1140
-
1141
- .. code-block:: python
1142
-
1143
- execution = package.services.execute(service='service_entity',
1144
- function_name='run',
1145
- item_id='item_id',
1146
- project_id='project_id')
1147
- """
1148
- if service is None:
1149
- service = self.get(service_id=service_id, service_name=service_name)
1150
- execution = repositories.Executions(service=service,
1151
- client_api=self._client_api,
1152
- project=self._project).create(service_id=service.id,
1153
- sync=sync,
1154
- execution_input=execution_input,
1155
- function_name=function_name,
1156
- resource=resource,
1157
- item_id=item_id,
1158
- dataset_id=dataset_id,
1159
- annotation_id=annotation_id,
1160
- project_id=project_id,
1161
- stream_logs=stream_logs)
1162
- return execution
1163
-
1164
- @_api_reference.add(path='/services', method='post')
1165
- def deploy(self,
1166
- service_name: str = None,
1167
- package: entities.Package = None,
1168
- bot: Union[entities.Bot, str] = None,
1169
- revision: str or int = None,
1170
- init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
1171
- runtime: Union[entities.KubernetesRuntime, dict] = None,
1172
- pod_type: entities.InstanceCatalog = None,
1173
- sdk_version: str = None,
1174
- agent_versions: dict = None,
1175
- verify: bool = True,
1176
- checkout: bool = False,
1177
- module_name: str = None,
1178
- project_id: str = None,
1179
- driver_id: str = None,
1180
- func: Callable = None,
1181
- run_execution_as_process: bool = None,
1182
- execution_timeout: int = None,
1183
- drain_time: int = None,
1184
- max_attempts: int = None,
1185
- on_reset: str = None,
1186
- force: bool = False,
1187
- secrets: list = None,
1188
- integrations: list = None,
1189
- active: bool = True,
1190
- **kwargs) -> entities.Service:
1191
- """
1192
- Deploy service.
1193
-
1194
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1195
-
1196
- :param str service_name: name
1197
- :param dtlpy.entities.package.Package package: package entity
1198
- :param str bot: bot email
1199
- :param str revision: package revision of version
1200
- :param init_input: config to run at startup
1201
- :param dict runtime: runtime resources
1202
- :param str pod_type: pod type dl.InstanceCatalog
1203
- :param str sdk_version: - optional - string - sdk version
1204
- :param str agent_versions: - dictionary - - optional -versions of sdk
1205
- :param bool verify: if true, verify the inputs
1206
- :param bool checkout: if true, checkout (switch) to service
1207
- :param str module_name: module name
1208
- :param str project_id: project id
1209
- :param str driver_id: driver id
1210
- :param Callable func: function to deploy
1211
- :param bool run_execution_as_process: if true, run execution as process
1212
- :param int execution_timeout: execution timeout in seconds
1213
- :param int drain_time: drain time in seconds
1214
- :param int max_attempts: maximum execution retries in-case of a service reset
1215
- :param str on_reset: what happens on reset
1216
- :param bool force: optional - if true, terminate old replicas immediately
1217
- :param list secrets: list of the integrations ids
1218
- :param list integrations: list of the integrations
1219
- :param bool active: if true, activate the service
1220
- :param kwargs: list of additional arguments
1221
- :return: Service object
1222
- :rtype: dtlpy.entities.service.Service
1223
-
1224
- **Example**:
1225
-
1226
- .. code-block:: python
1227
-
1228
- service = package.services.deploy(service_name=package_name,
1229
- execution_timeout=3 * 60 * 60,
1230
- module_name=module.name,
1231
- runtime=dl.KubernetesRuntime(
1232
- concurrency=10,
1233
- pod_type=dl.InstanceCatalog.REGULAR_S,
1234
- autoscaler=dl.KubernetesRabbitmqAutoscaler(
1235
- min_replicas=1,
1236
- max_replicas=20,
1237
- queue_length=20
1238
- )
1239
- )
1240
- )
1241
- """
1242
- if package is None and isinstance(package, entities.Dpk):
1243
- raise exceptions.PlatformException('400', 'cannot deploy dpk package. Please install the app')
1244
- package = package if package is not None else self._package
1245
- if service_name is None:
1246
- get_name = False
1247
- if package is not None and package.service_config is not None and 'name' in package.service_config:
1248
- service_name = package.service_config['name']
1249
- get_name = True
1250
- else:
1251
- if package is not None:
1252
- service_name = package.name
1253
- else:
1254
- service_name = 'default-service'
1255
- if not get_name:
1256
- logger.warning('service_name not provided, using: {} by default'.format(service_name))
1257
-
1258
- if isinstance(revision, int):
1259
- logger.warning('Deprecation Warning - Package/service versions have been refactored'
1260
- 'The version you provided has type: int, it will be converted to: 1.0.{}'
1261
- 'Next time use a 3-level semver for package/service versions'.format(revision))
1262
-
1263
- if func is not None:
1264
- package = self.__deploy_function(name=service_name, project=self._project, func=func)
1265
-
1266
- if init_input is not None and not isinstance(init_input, dict):
1267
- if not isinstance(init_input, list):
1268
- init_input = [init_input]
1269
-
1270
- if len(init_input) > 0 and isinstance(init_input[0], entities.FunctionIO):
1271
- params = dict()
1272
- for i_param, param in enumerate(init_input):
1273
- params[param.name] = param.value
1274
- init_input = params
1275
- elif len(init_input) == 0:
1276
- init_input = None
1277
- else:
1278
- raise exceptions.PlatformException(
1279
- error='400',
1280
- message='Unknown init_input type. expecting list or dict, got: {}'.format(type(init_input))
1281
- )
1282
-
1283
- if project_id is None:
1284
- if self._project is not None:
1285
- project_id = self._project.id
1286
- else:
1287
- project_id = self._project_id
1288
-
1289
- filters = entities.Filters(resource=entities.FiltersResource.SERVICE, use_defaults=False)
1290
- filters.add(field='name', values=service_name)
1291
- if project_id is not None:
1292
- filters.add(field='projectId', values=project_id)
1293
- services = self.list(filters=filters)
1294
- if services.items_count > 1:
1295
- raise exceptions.PlatformException('400',
1296
- 'More than 1 service by this name are associated with this user. '
1297
- 'Please provide project_id')
1298
- elif services.items_count > 0:
1299
- service = services.items[0]
1300
- if runtime is not None:
1301
- service.runtime = runtime
1302
- if init_input is not None:
1303
- service.init_input = init_input
1304
- if revision is not None:
1305
- service.package_revision = revision
1306
- if agent_versions is not None:
1307
- service.versions = agent_versions
1308
- elif sdk_version:
1309
- service.versions = {'dtlpy': sdk_version}
1310
- if driver_id is not None:
1311
- service.driver_id = driver_id
1312
- if secrets is not None:
1313
- if not isinstance(secrets, list):
1314
- secrets = [secrets]
1315
- service.secrets = secrets
1316
- if integrations is not None:
1317
- if not isinstance(integrations, list):
1318
- integrations = [integrations]
1319
- service.integrations = integrations
1320
- service = self.update(service=service, force=force)
1321
- else:
1322
- service = self._create(service_name=service_name,
1323
- package=package,
1324
- project_id=project_id,
1325
- bot=bot,
1326
- revision=revision,
1327
- init_input=init_input,
1328
- runtime=runtime,
1329
- pod_type=pod_type,
1330
- sdk_version=sdk_version,
1331
- agent_versions=agent_versions,
1332
- verify=verify,
1333
- module_name=module_name,
1334
- driver_id=driver_id,
1335
- jwt_forward=kwargs.get('jwt_forward', None),
1336
- is_global=kwargs.get('is_global', None),
1337
- run_execution_as_process=run_execution_as_process,
1338
- execution_timeout=execution_timeout,
1339
- drain_time=drain_time,
1340
- max_attempts=max_attempts,
1341
- on_reset=on_reset,
1342
- secrets=secrets,
1343
- integrations=integrations,
1344
- active=active
1345
- )
1346
- if checkout:
1347
- self.checkout(service=service)
1348
- return service
1349
-
1350
- @staticmethod
1351
- def __get_import_string(imports: List[str]):
1352
- import_string = ''
1353
- if imports is not None:
1354
- import_string = '\n'.join(imports)
1355
- return import_string
1356
-
1357
- @staticmethod
1358
- def __get_inputs(func):
1359
- method = inspect.signature(func)
1360
- params = list(method.parameters)
1361
- inpts = list()
1362
- inputs_types = {i.name.lower(): i.value for i in list(entities.PackageInputType)}
1363
- for arg in params:
1364
- if arg in inputs_types:
1365
- inpt_type = inputs_types[arg]
1366
- else:
1367
- inpt_type = entities.PackageInputType.JSON
1368
- inpts.append(entities.FunctionIO(type=inpt_type, name=arg))
1369
- return inpts
1370
-
1371
- def __deploy_function(self,
1372
- name: str,
1373
- func: Callable,
1374
- project: entities.Project) -> entities.Package:
1375
- package_dir = tempfile.mkdtemp()
1376
- # imports_string = self.__get_import_string()
1377
- imports_string = ''
1378
-
1379
- main_file = os.path.join(package_dir, entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
1380
- with open(assets.paths.PARTIAL_MAIN_FILEPATH, 'r') as f:
1381
- main_string = f.read()
1382
- lines = inspect.getsourcelines(func)
1383
- tabs_diff = lines[0][0].count(' ') - 1
1384
- for line_index in range(len(lines[0])):
1385
- line_tabs = lines[0][line_index].count(' ') - tabs_diff
1386
- lines[0][line_index] = (' ' * line_tabs) + lines[0][line_index].strip() + '\n'
1387
-
1388
- method_func_string = "".join(lines[0])
1389
-
1390
- with open(main_file, 'w') as f:
1391
- f.write('{}\n{}\n @staticmethod\n{}'.format(imports_string, main_string,
1392
- method_func_string))
1393
-
1394
- function = entities.PackageFunction(name=func.__name__, inputs=self.__get_inputs(func=func))
1395
- module = entities.PackageModule(functions=[function],
1396
- entry_point=entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
1397
- packages = repositories.Packages(client_api=self._client_api, project=project)
1398
- return packages.push(src_path=package_dir,
1399
- package_name=name,
1400
- checkout=True,
1401
- modules=[module])
1402
-
1403
- def deploy_from_local_folder(self,
1404
- cwd=None,
1405
- service_file=None,
1406
- bot=None,
1407
- checkout=False,
1408
- force=False
1409
- ) -> entities.Service:
1410
- """
1411
- Deploy from local folder in local environment.
1412
-
1413
- **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1414
-
1415
- :param str cwd: optional - package working directory. Default=cwd
1416
- :param str service_file: optional - service file. Default=None
1417
- :param str bot: bot
1418
- :param checkout: checkout
1419
- :param bool force: optional - terminate old replicas immediately
1420
- :return: Service object
1421
- :rtype: dtlpy.entities.service.Service
1422
-
1423
- **Example**:
1424
-
1425
- .. code-block:: python
1426
-
1427
- service = package.services.deploy_from_local_folder(cwd='file_path',
1428
- service_file='service_file')
1429
- """
1430
- # get cwd and service.json path
1431
- if cwd is None:
1432
- cwd = os.getcwd()
1433
- if service_file is None:
1434
- service_file = os.path.join(cwd, assets.paths.SERVICE_FILENAME)
1435
-
1436
- # load service json
1437
- if os.path.isfile(service_file):
1438
- with open(service_file, 'r') as f:
1439
- service_json = json.load(f)
1440
- service_triggers = service_json.get('triggers', list())
1441
- else:
1442
- raise exceptions.PlatformException(error='400',
1443
- message='Could not find service.json in path: {}'.format(cwd))
1444
-
1445
- # get package
1446
- package_name = service_json.get('packageName', None)
1447
- packages = repositories.Packages(client_api=self._client_api, project=self._project)
1448
-
1449
- if package_name is None:
1450
- package = packages.get()
1451
- else:
1452
- package = packages.get(package_name=package_name)
1453
-
1454
- name = service_json.get('name', None)
1455
- revision = service_json.get('revision', package.version)
1456
- init_input = service_json.get('initParams', dict())
1457
- runtime = service_json.get('runtime', dict())
1458
- sdk_version = service_json.get('version', None)
1459
- agent_versions = service_json.get('versions', None)
1460
- verify = service_json.get('verify', True)
1461
- module_name = service_json.get('module_name', None)
1462
- run_execution_as_process = service_json.get('run_execution_as_process', None)
1463
- execution_timeout = service_json.get('execution_timeout', None)
1464
- drain_time = service_json.get('drain_time', None)
1465
- on_reset = service_json.get('on_reset', None)
1466
- max_attempts = service_json.get('maxAttempts', None)
1467
-
1468
- service = self.deploy(bot=bot,
1469
- service_name=name,
1470
- package=package,
1471
- revision=revision,
1472
- runtime=runtime,
1473
- init_input=init_input,
1474
- sdk_version=sdk_version,
1475
- agent_versions=agent_versions,
1476
- verify=verify,
1477
- checkout=checkout,
1478
- run_execution_as_process=run_execution_as_process,
1479
- execution_timeout=execution_timeout,
1480
- drain_time=drain_time,
1481
- max_attempts=max_attempts,
1482
- on_reset=on_reset,
1483
- module_name=module_name,
1484
- force=force
1485
- )
1486
-
1487
- logger.info('Service was deployed successfully. Service id: {}'.format(service.id))
1488
-
1489
- if len(service_triggers) > 0:
1490
- logger.info('Creating triggers...')
1491
- triggers = repositories.Triggers(client_api=self._client_api, project=self._project)
1492
-
1493
- for trigger in service_triggers:
1494
- name = trigger.get('name', None)
1495
- filters = trigger.get('filter', dict())
1496
- resource = trigger['resource']
1497
- actions = trigger.get('actions', list())
1498
- active = trigger.get('active', True)
1499
- execution_mode = trigger.get('executionMode', None)
1500
- function_name = trigger.get('function', None)
1501
-
1502
- trigger = triggers.create(service_id=service.id,
1503
- name=name,
1504
- filters=filters,
1505
- resource=resource,
1506
- actions=actions,
1507
- active=active,
1508
- execution_mode=execution_mode,
1509
- function_name=function_name)
1510
-
1511
- logger.info('Trigger was created successfully. Service id: {}'.format(trigger.id))
1512
-
1513
- logging.info('Successfully deployed!')
1514
- return service
1515
-
1516
- def __enable_cache(self,
1517
- url,
1518
- organization: entities.Organization,
1519
- pod_type=entities.PodType.SMALL):
1520
- payload = {
1521
- "org": {
1522
- "name": organization.name,
1523
- "id": organization.id
1524
- },
1525
- "runner": {
1526
- "podType": pod_type # small, medium, high
1527
- }
1528
- }
1529
-
1530
- return self._client_api.gen_request(req_type='post',
1531
- path=url,
1532
- json_req=payload)
1533
-
1534
- def __polling_wait(self, organization, pod_type, backoff_factor=1):
1535
- fs_url_path = '/services/fs-cache?mode={}'.format('get')
1536
- i = 1
1537
- while True:
1538
- success, response = self.__enable_cache(url=fs_url_path, organization=organization, pod_type=pod_type)
1539
- if response.json().get('state', None) == 'READY':
1540
- break
1541
- sleep_time = min(backoff_factor * (2 ** (i - 1)), MAX_WAIT_TIME)
1542
- logger.debug("Going to sleep {:.2f}[s]".format(sleep_time))
1543
- time.sleep(sleep_time)
1544
- i += 1
1545
- return success
1546
-
1547
- def _cache_action(self,
1548
- organization: entities.Organization = None,
1549
- mode=entities.CacheAction.APPLY,
1550
- pod_type=entities.PodType.SMALL):
1551
- """
1552
- Add or remove Cache for the org
1553
-
1554
- **Prerequisites**: You must be an organization *owner*
1555
-
1556
- You must provide at least ONE of the following params: organization, organization_name, or organization_id.
1557
-
1558
- :param entities.Organization organization: Organization object
1559
- :param str mode: dl.CacheAction.APPLY or dl.CacheAction.DESTROY
1560
- :param entities.PodType pod_type: dl.PodType.SMALL, dl.PodType.MEDIUM, dl.PodType.HIGH
1561
- :return: True if success
1562
- :rtype: bool
1563
-
1564
- **Example**:
1565
-
1566
- .. code-block:: python
1567
-
1568
- success = dl.organizations.enable_cache(organization='organization',
1569
- mode=dl.CacheAction.APPLY)
1570
- """
1571
- if organization is None:
1572
- raise exceptions.PlatformException(
1573
- error='400',
1574
- message='Must provide an identifier in inputs')
1575
-
1576
- fs_mode = mode if mode != entities.CacheAction.APPLY else '{}-filestore'.format(mode)
1577
- apply_fs_url_path = '/services/fs-cache?mode={}'.format(fs_mode)
1578
- apply_volume_url_path = '/services/fs-cache?mode={}'.format(mode)
1579
- cache_url_path = '/services/cache?mode={}'.format(mode)
1580
-
1581
- success, response = self.__enable_cache(url=apply_fs_url_path, organization=organization, pod_type=pod_type)
1582
- if not success:
1583
- raise exceptions.PlatformException(response)
1584
-
1585
- if mode == entities.CacheAction.APPLY:
1586
- self.__polling_wait(organization=organization, pod_type=pod_type)
1587
- success, response = self.__enable_cache(url=apply_volume_url_path, organization=organization,
1588
- pod_type=pod_type)
1589
- if not success:
1590
- raise exceptions.PlatformException(response)
1591
-
1592
- success, response = self.__enable_cache(url=cache_url_path, organization=organization, pod_type=pod_type)
1593
- if not success:
1594
- raise exceptions.PlatformException(response)
1595
-
1596
- return True
1597
-
1598
- def restart(self, service: entities.Service, replica_name: str = None):
1599
- """
1600
- Restart service replica
1601
-
1602
- **Prerequisites**: You must be in the role of a *developer*.
1603
-
1604
- :param dtlpy.entities.service.Service service: service entity
1605
- :param str replica_name: replica name
1606
- :return: True
1607
- :rtype: bool
1608
-
1609
- **Example**:
1610
-
1611
- .. code-block:: python
1612
-
1613
- is_restarted = dl.services.restart(service='service_entity',
1614
- replica_name='replica_name')
1615
- """
1616
- payload = {}
1617
-
1618
- if replica_name is not None:
1619
- payload['replicaName'] = replica_name
1620
-
1621
- # request
1622
- success, response = self._client_api.gen_request(req_type='post',
1623
- path='/services/{}/restart'.format(service.id),
1624
- json_req=payload)
1625
-
1626
- # exception handling
1627
- if not success:
1628
- raise exceptions.PlatformException(response)
1629
-
1630
- return True
1631
-
1632
-
1633
- class ServiceLog:
1634
- """
1635
- Service Log
1636
- """
1637
-
1638
- def __init__(self,
1639
- _json: dict,
1640
- service: entities.Service,
1641
- services: Services,
1642
- start=None,
1643
- follow=None,
1644
- execution_id=None,
1645
- function_name=None,
1646
- replica_id=None,
1647
- system=False,
1648
- model_id=None,
1649
- model_operation=None,
1650
- project_id=None):
1651
-
1652
- self.logs = _json.get('logs', dict())
1653
- self.checkpoint = _json.get('checkpoint', None)
1654
- self.stop = _json.get('stop', False)
1655
- self.service = service
1656
- self.services = services
1657
- self.start = start
1658
- self.follow = follow
1659
- self.execution_id = execution_id
1660
- self.function_name = function_name
1661
- self.replica_id = replica_id
1662
- self.system = system
1663
- self.model_id = model_id
1664
- self.model_operation = model_operation
1665
- self.project_id = project_id
1666
-
1667
- def get_next_log(self):
1668
- log = self.services.log(service=self.service,
1669
- checkpoint=self.checkpoint,
1670
- start=self.start,
1671
- follow=self.follow,
1672
- execution_id=self.execution_id,
1673
- function_name=self.function_name,
1674
- replica_id=self.replica_id,
1675
- system=self.system,
1676
- view=False,
1677
- model_id=self.model_id,
1678
- model_operation=self.model_operation,
1679
- project_id=self.project_id)
1680
-
1681
- self.logs = log.logs
1682
- self.checkpoint = log.checkpoint
1683
- self.stop = log.stop
1684
-
1685
- def view(self, until_completed):
1686
- """
1687
- View logs
1688
-
1689
- :param until_completed:
1690
- """
1691
- try:
1692
- for log in self:
1693
- print(log)
1694
- if until_completed and FUNCTION_END_LINE in log:
1695
- break
1696
- except KeyboardInterrupt:
1697
- return
1698
-
1699
- def __iter__(self):
1700
- while not self.stop:
1701
- for log in self.logs:
1702
- yield '{}: {}'.format(log.get('timestamp', self.start), log.get('message', '').strip())
1703
- self.get_next_log()
1704
-
1
+ import datetime
2
+ import inspect
3
+ import logging
4
+ import json
5
+ import os
6
+ import tempfile
7
+ import time
8
+ from typing import Union, List, Callable
9
+ from .. import miscellaneous, exceptions, entities, repositories, assets, ApiClient, _api_reference
10
+ from ..__version__ import version as __version__
11
+
12
+ logger = logging.getLogger(name='dtlpy')
13
+ FUNCTION_END_LINE = '[Done] Executing function.'
14
+ MAX_WAIT_TIME = 8
15
+
16
+
17
+ class Services:
18
+ """
19
+ Services Repository
20
+
21
+ The Services class allows the user to manage services and their properties. Services are created from the packages users create.
22
+ See our documentation for more information about `services <https://developers.dataloop.ai/tutorials/faas/advance/chapter/>`_.
23
+ """
24
+
25
+ def __init__(self,
26
+ client_api: ApiClient,
27
+ project: entities.Project = None,
28
+ package: Union[entities.Package, entities.Dpk] = None,
29
+ project_id=None,
30
+ model_id=None,
31
+ model: entities.Model = None):
32
+ self._client_api = client_api
33
+ self._package = package
34
+ self._project = project
35
+ self._model = model
36
+ if model_id is None:
37
+ if model is not None:
38
+ model_id = model.id
39
+ self._model_id = model_id
40
+ if project_id is None:
41
+ if project is not None:
42
+ project_id = project.id
43
+ self._project_id = project_id
44
+ self._settings = repositories.Settings(project=project, client_api=client_api)
45
+
46
+ ############
47
+ # entities #
48
+ ############
49
+ @property
50
+ def package(self) -> entities.Package:
51
+ if self._package is None:
52
+ raise exceptions.PlatformException(
53
+ error='2001',
54
+ message='Cannot perform action WITHOUT package entity in services repository. Please set a package')
55
+ assert (isinstance(self._package, entities.Package) or isinstance(self._package, entities.Dpk))
56
+ return self._package
57
+
58
+ @package.setter
59
+ def package(self, package: Union[entities.Package, entities.Dpk]):
60
+ if not isinstance(package, entities.Package) and not isinstance(package, entities.Dpk):
61
+ raise ValueError('Must input a valid package entity')
62
+ self._package = package
63
+
64
+ @property
65
+ def project(self) -> entities.Project:
66
+ if self._project is None:
67
+ # try to get from package
68
+ if self._package is not None:
69
+ self._project = self._package._project
70
+
71
+ if self._project is None:
72
+ # try to get checked out project
73
+ project = self._client_api.state_io.get('project')
74
+ if project is not None:
75
+ self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
76
+ if self._project is None:
77
+ raise exceptions.PlatformException(
78
+ error='2001',
79
+ message='Cannot perform action WITHOUT Project entity in services repository. Please set a project')
80
+ return self._project
81
+
82
+ @project.setter
83
+ def project(self, project: entities.Project):
84
+ if not isinstance(project, entities.Project):
85
+ raise ValueError('Must input a valid Project entity')
86
+ self._project = project
87
+ self._project_id = project.id
88
+
89
+ @property
90
+ def model(self) -> entities.Model:
91
+ if self._model is None:
92
+ if self._model_id is not None:
93
+ self._model = self.project.models.get(model_id=self._model_id)
94
+ else:
95
+ raise exceptions.PlatformException(
96
+ error='2001',
97
+ message='Cannot perform action WITHOUT model entity in services repository. Please set a model')
98
+ assert isinstance(self._model, entities.Model)
99
+ return self._model
100
+
101
+ @model.setter
102
+ def model(self, model: entities.Model):
103
+ if not isinstance(model, entities.Model):
104
+ raise ValueError('Must input a valid model entity')
105
+ self._model = model
106
+ self._model_id = model.id
107
+
108
+ @property
109
+ def platform_url(self):
110
+ return self._client_api._get_resource_url("projects/{}/services".format(self.project.id))
111
+
112
+ def open_in_web(self,
113
+ service: entities.Service = None,
114
+ service_id: str = None,
115
+ service_name: str = None):
116
+ """
117
+ Open the service in web platform
118
+
119
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
120
+
121
+ :param str service_name: service name
122
+ :param str service_id: service id
123
+ :param dtlpy.entities.service.Service service: service entity
124
+
125
+ **Example**:
126
+
127
+ .. code-block:: python
128
+
129
+ package.services.open_in_web(service_id='service_id')
130
+ """
131
+ if service_name is not None:
132
+ service = self.get(service_name=service_name)
133
+ if service is not None:
134
+ service.open_in_web()
135
+ elif service_id is not None:
136
+ self._client_api._open_in_web(url=self.platform_url + '/' + str(service_id) + '/main')
137
+ else:
138
+ self._client_api._open_in_web(url=self.platform_url)
139
+
140
+ def __get_from_cache(self) -> entities.Service:
141
+ service = self._client_api.state_io.get('service')
142
+ if service is not None:
143
+ service = entities.Service.from_json(_json=service,
144
+ client_api=self._client_api,
145
+ project=self._project,
146
+ package=self._package)
147
+ return service
148
+
149
+ def checkout(self,
150
+ service: entities.Service = None,
151
+ service_name: str = None,
152
+ service_id: str = None):
153
+ """
154
+ Checkout (switch) to a service.
155
+
156
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
157
+
158
+ :param dtlpy.entities.service.Service service: Service entity
159
+ :param str service_name: service name
160
+ :param str service_id: service id
161
+
162
+ **Example**:
163
+
164
+ .. code-block:: python
165
+
166
+ package.services.checkout(service_id='service_id')
167
+ """
168
+ if service is None:
169
+ service = self.get(service_name=service_name, service_id=service_id)
170
+ self._client_api.state_io.put('service', service.to_json())
171
+ logger.info('Checked out to service {}'.format(service.name))
172
+
173
+ ###########
174
+ # methods #
175
+ ###########
176
+ @_api_reference.add(path='/services/{id}/revisions', method='get')
177
+ def revisions(self,
178
+ service: entities.Service = None,
179
+ service_id: str = None):
180
+ """
181
+ Get service revisions history.
182
+
183
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
184
+
185
+ You must provide at leats ONE of the following params: service, service_id
186
+
187
+ :param dtlpy.entities.service.Service service: Service entity
188
+ :param str service_id: service id
189
+
190
+ **Example**:
191
+
192
+ .. code-block:: python
193
+
194
+ service_revision = package.services.revisions(service_id='service_id')
195
+ """
196
+ if service is None and service_id is None:
197
+ raise exceptions.PlatformException(
198
+ error='400',
199
+ message='must provide an identifier in inputs: "service" or "service_id"')
200
+ if service is not None:
201
+ service_id = service.id
202
+
203
+ success, response = self._client_api.gen_request(
204
+ req_type="get",
205
+ path="/services/{}/revisions".format(service_id))
206
+ if not success:
207
+ raise exceptions.PlatformException(response)
208
+ return response.json()
209
+
210
+ @_api_reference.add(path='/services/{id}', method='get')
211
+ def get(self,
212
+ service_name=None,
213
+ service_id=None,
214
+ checkout=False,
215
+ fetch=None
216
+ ) -> entities.Service:
217
+ """
218
+ Get service to use in your code.
219
+
220
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
221
+
222
+ :param str service_name: optional - search by name
223
+ :param str service_id: optional - search by id
224
+ :param bool checkout: if true, checkout (switch) to service
225
+ :param fetch: optional - fetch entity from platform, default taken from cookie
226
+ :return: Service object
227
+ :rtype: dtlpy.entities.service.Service
228
+
229
+ **Example**:
230
+
231
+ .. code-block:: python
232
+
233
+ service = package.services.get(service_id='service_id')
234
+ """
235
+ if fetch is None:
236
+ fetch = self._client_api.fetch_entities
237
+
238
+ if service_name is None and service_id is None:
239
+ service = self.__get_from_cache()
240
+ if service is None:
241
+ raise exceptions.PlatformException(
242
+ error='400',
243
+ message='No checked-out Service was found, must checkout or provide an identifier in inputs')
244
+
245
+ elif fetch:
246
+ if service_id is not None:
247
+ success, response = self._client_api.gen_request(
248
+ req_type="get",
249
+ path="/services/{}".format(service_id)
250
+ )
251
+ if not success:
252
+ raise exceptions.PlatformException(response)
253
+ service = entities.Service.from_json(client_api=self._client_api,
254
+ _json=response.json(),
255
+ package=self._package,
256
+ project=self._project)
257
+ # verify input service name is same as the given id
258
+ if service_name is not None and service.name != service_name:
259
+ logger.warning(
260
+ "Mismatch found in services.get: service_name is different then service.name:"
261
+ " {!r} != {!r}".format(
262
+ service_name,
263
+ service.name))
264
+ elif service_name is not None:
265
+ filters = entities.Filters(resource=entities.FiltersResource.SERVICE,
266
+ field='name',
267
+ values=service_name,
268
+ use_defaults=False)
269
+ if self._project_id is not None:
270
+ filters.add(field='projectId', values=self._project_id)
271
+ if self._package is not None:
272
+ filters.add(field='packageId', values=self._package.id)
273
+ services = self.list(filters=filters)
274
+ if services.items_count > 1:
275
+ raise exceptions.PlatformException('404', 'More than one service with same name. '
276
+ 'Please get services from package/project entity')
277
+ elif services.items_count == 0:
278
+ raise exceptions.PlatformException('404', 'Service not found: {}.'.format(service_name))
279
+ service = services.items[0]
280
+ else:
281
+ raise exceptions.PlatformException(
282
+ error='400',
283
+ message='No checked-out Service was found, must checkout or provide an identifier in inputs')
284
+ else:
285
+ service = entities.Service.from_json(_json={'id': service_id,
286
+ 'name': service_name},
287
+ client_api=self._client_api,
288
+ project=self._project,
289
+ package=self._package,
290
+ is_fetched=False)
291
+
292
+ assert isinstance(service, entities.Service)
293
+ if checkout:
294
+ self.checkout(service=service)
295
+ return service
296
+
297
+ def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Service]:
298
+ jobs = [None for _ in range(len(response_items))]
299
+ pool = self._client_api.thread_pools(pool_name='entity.create')
300
+
301
+ # return triggers list
302
+ for i_service, service in enumerate(response_items):
303
+ jobs[i_service] = pool.submit(entities.Service._protected_from_json,
304
+ **{'client_api': self._client_api,
305
+ '_json': service,
306
+ 'package': self._package,
307
+ 'project': self._project})
308
+
309
+ # get all results
310
+ results = [j.result() for j in jobs]
311
+ # log errors
312
+ _ = [logger.warning(r[1]) for r in results if r[0] is False]
313
+ # return good jobs
314
+ return miscellaneous.List([r[1] for r in results if r[0] is True])
315
+
316
+ def _list(self, filters: entities.Filters):
317
+ url = '/query/faas'
318
+ success, response = self._client_api.gen_request(req_type='POST',
319
+ path=url,
320
+ json_req=filters.prepare())
321
+ if not success:
322
+ raise exceptions.PlatformException(response)
323
+
324
+ return response.json()
325
+
326
+ @_api_reference.add(path='/query/faas', method='post')
327
+ def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
328
+ """
329
+ List all services (services can be listed for a package or for a project).
330
+
331
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
332
+
333
+ :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
334
+ :return: Paged entity
335
+ :rtype: dtlpy.entities.paged_entities.PagedEntities
336
+
337
+ **Example**:
338
+
339
+ .. code-block:: python
340
+
341
+ services = package.services.list()
342
+ """
343
+ # default filters
344
+ if filters is None:
345
+ filters = entities.Filters(resource=entities.FiltersResource.SERVICE)
346
+ # assert type filters
347
+ elif not isinstance(filters, entities.Filters):
348
+ raise exceptions.PlatformException(error='400',
349
+ message='Unknown filters type: {!r}'.format(type(filters)))
350
+ if filters.resource != entities.FiltersResource.SERVICE:
351
+ raise exceptions.PlatformException(
352
+ error='400',
353
+ message='Filters resource must to be FiltersResource.SERVICE. Got: {!r}'.format(filters.resource))
354
+ if self._project_id is not None:
355
+ filters.add(field='projectId', values=self._project_id)
356
+ if self._model_id is not None:
357
+ filters.add(field='metadata.ml.modelId', values=self._model_id)
358
+ if self._package is not None:
359
+ filters.add(field='packageId', values=self._package.id)
360
+
361
+ # assert type filters
362
+ if not isinstance(filters, entities.Filters):
363
+ raise exceptions.PlatformException('400', 'Unknown filters type')
364
+
365
+ paged = entities.PagedEntities(items_repository=self,
366
+ filters=filters,
367
+ page_offset=filters.page,
368
+ page_size=filters.page_size,
369
+ client_api=self._client_api)
370
+ paged.get_page()
371
+ return paged
372
+
373
+ @_api_reference.add(path='/services/{id}/status', method='post')
374
+ def status(self, service_name=None, service_id=None):
375
+ """
376
+ Get service status.
377
+
378
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
379
+
380
+ You must provide at least ONE of the following params: service_id, service_name
381
+
382
+ :param str service_name: service name
383
+ :param str service_id: service id
384
+ :return: status json
385
+ :rtype: dict
386
+
387
+ **Example**:
388
+
389
+ .. code-block:: python
390
+
391
+ status_json = package.services.status(service_id='service_id')
392
+ """
393
+ if service_id is None:
394
+ if service_name is None:
395
+ raise exceptions.PlatformException(error='400',
396
+ message='must input "service_name" or "service_id" to get status')
397
+ service = self.get(service_name=service_name)
398
+ service_id = service.id
399
+ # request
400
+ success, response = self._client_api.gen_request(req_type="get",
401
+ path="/services/{}/status".format(service_id))
402
+ if not success:
403
+ raise exceptions.PlatformException(response)
404
+ return response.json()
405
+
406
+ @_api_reference.add(path='/services/{id}/stop', method='post')
407
+ def pause(self,
408
+ service_name: str = None,
409
+ service_id: str = None,
410
+ force: bool = False):
411
+ """
412
+ Pause service.
413
+
414
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
415
+
416
+ You must provide at least ONE of the following params: service_id, service_name
417
+
418
+ :param str service_name: service name
419
+ :param str service_id: service id
420
+ :param bool force: optional - terminate old replicas immediately
421
+ :return: True if success
422
+ :rtype: bool
423
+
424
+ **Example**:
425
+
426
+ .. code-block:: python
427
+
428
+ success = package.services.pause(service_id='service_id')
429
+ """
430
+ if service_id is None:
431
+ if service_name is None:
432
+ raise exceptions.PlatformException(error='400',
433
+ message='must input "service_name" or "service_id" to pause service')
434
+ service = self.get(service_name=service_name)
435
+ service_id = service.id
436
+ # request
437
+ url = "/services/{}/stop".format(service_id)
438
+ if force:
439
+ url = '{}?force=true'
440
+ success, response = self._client_api.gen_request(req_type="post",
441
+ path=url)
442
+ if not success:
443
+ raise exceptions.PlatformException(response)
444
+ return success
445
+
446
+ def _notify(
447
+ self,
448
+ service_id: str,
449
+ message: str,
450
+ name: str,
451
+ action: str = 'created',
452
+ support: str = None,
453
+ docs: str = None,
454
+ agent_info: dict = None
455
+ ):
456
+ url = "/services/{}/notify".format(service_id)
457
+ payload = {
458
+ 'action': action,
459
+ 'message': message,
460
+ 'notificationName': name
461
+ }
462
+ if agent_info is not None:
463
+ payload['agentInfo'] = agent_info
464
+
465
+ if support:
466
+ payload['support'] = support
467
+
468
+ if docs:
469
+ payload['docs'] = docs
470
+
471
+ success, response = self._client_api.gen_request(
472
+ req_type="post",
473
+ path=url,
474
+ json_req=payload
475
+ )
476
+ if not success:
477
+ raise exceptions.PlatformException(response)
478
+ return success
479
+
480
+ @_api_reference.add(path='/services/{id}/resume', method='post')
481
+ def resume(self,
482
+ service_name: str = None,
483
+ service_id: str = None,
484
+ force: bool = False):
485
+ """
486
+ Resume service.
487
+
488
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
489
+
490
+ You must provide at least ONE of the following params: service_id, service_name.
491
+
492
+ :param str service_name: service name
493
+ :param str service_id: service id
494
+ :param bool force: optional - terminate old replicas immediately
495
+ :return: json of the service
496
+ :rtype: dict
497
+
498
+ **Example**:
499
+
500
+ .. code-block:: python
501
+
502
+ service_json = package.services.resume(service_id='service_id')
503
+ """
504
+ if service_id is None:
505
+ if service_name is None:
506
+ raise exceptions.PlatformException(error='400',
507
+ message='must input "service_name" or "service_id" to resume')
508
+ service = self.get(service_name=service_name)
509
+ service_id = service.id
510
+ # request
511
+ url = "/services/{}/resume".format(service_id)
512
+ if force:
513
+ url = '{}?force=true'
514
+ success, response = self._client_api.gen_request(req_type="post",
515
+ path=url)
516
+ if not success:
517
+ raise exceptions.PlatformException(response)
518
+ return response.json()
519
+
520
+ def _get_bot_email(self, bot=None):
521
+
522
+ if bot is None:
523
+ project = self._project
524
+ if project is None and self._project_id is not None:
525
+ project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
526
+
527
+ if project is None:
528
+ raise exceptions.PlatformException(error='2001',
529
+ message='Need project entity or bot to perform this action')
530
+ bots = project.bots.list()
531
+ if len(bots) == 0:
532
+ logger.info('Bot not found for project. Creating a default bot')
533
+ bot = project.bots.create(name='default')
534
+ else:
535
+ bot = bots[0]
536
+ if len(bots) > 1:
537
+ logger.debug('More than one bot users. Choosing first. email: {}'.format(bots[0].email))
538
+
539
+ if isinstance(bot, str):
540
+ bot_email = bot
541
+ elif isinstance(bot, entities.Bot) or isinstance(bot, entities.User):
542
+ bot_email = bot.email
543
+ else:
544
+ raise ValueError('input "bot" must be a str or a Bot type, got: {}'.format(type(bot)))
545
+
546
+ return bot_email
547
+
548
+ @staticmethod
549
+ def _parse_init_input(init_input):
550
+ if not isinstance(init_input, dict):
551
+ if init_input is None:
552
+ init_input = dict()
553
+ else:
554
+ init_params = dict()
555
+ if not isinstance(init_input, list):
556
+ init_input = [init_input]
557
+ for param in init_input:
558
+ if isinstance(param, entities.FunctionIO):
559
+ init_params.update(param.to_json(resource='service'))
560
+ else:
561
+ raise exceptions.PlatformException('400', 'Unknown type of init params')
562
+ init_input = init_params
563
+
564
+ return init_input
565
+
566
+ def name_validation(self, name: str):
567
+ """
568
+ Validation service name.
569
+
570
+ **Prerequisites**: You must be in the role of an *owner* or *developer*.
571
+
572
+ :param str name: service name
573
+
574
+ **Example**:
575
+
576
+ .. code-block:: python
577
+
578
+ package.services.name_validation(name='name')
579
+ """
580
+ url = '/piper-misc/naming/services/{}'.format(name)
581
+
582
+ # request
583
+ success, response = self._client_api.gen_request(req_type='get',
584
+ path=url)
585
+ if not success:
586
+ raise exceptions.PlatformException(response)
587
+
588
+ def _create(self,
589
+ service_name: str = None,
590
+ package: entities.Package = None,
591
+ module_name: str = None,
592
+ bot: Union[entities.Bot, str] = None,
593
+ revision: str or int = None,
594
+ init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
595
+ runtime: Union[entities.KubernetesRuntime, dict] = None,
596
+ pod_type: entities.InstanceCatalog = None,
597
+ project_id: str = None,
598
+ sdk_version: str = None,
599
+ agent_versions: dict = None,
600
+ verify: bool = True,
601
+ driver_id: str = None,
602
+ run_execution_as_process: bool = None,
603
+ execution_timeout: int = None,
604
+ drain_time: int = None,
605
+ on_reset: str = None,
606
+ max_attempts: int = None,
607
+ secrets=None,
608
+ integrations=None,
609
+ active: bool = True,
610
+ **kwargs
611
+ ) -> entities.Service:
612
+ """
613
+ Create service entity.
614
+
615
+
616
+ :param str service_name: service name
617
+ :param dtlpy.entities.package.Package package: package entity
618
+ :param str module_name: module name
619
+ :param str bot: bot email
620
+ :param str revision: package revision - default=latest
621
+ :param init_input: config to run at startup
622
+ :param dict runtime: runtime resources
623
+ :param str pod_type: pod type dl.InstanceCatalog
624
+ :param str project_id: project id
625
+ :param str sdk_version: - optional - string - sdk version
626
+ :param dict agent_versions: - dictionary - - optional -versions of sdk, agent runner and agent proxy
627
+ :param bool verify: verify the inputs
628
+ :param str driver_id: driver id
629
+ :param bool run_execution_as_process: run execution as process
630
+ :param int execution_timeout: execution timeout
631
+ :param int drain_time: drain time
632
+ :param str on_reset: on reset
633
+ :param int max_attempts: Maximum execution retries in-case of a service reset
634
+ :param bool force: optional - terminate old replicas immediately
635
+ :param list secrets: list of the integrations ids
636
+ :param list integrations: list of the integrations
637
+ :param bool active: optional - if true, service will be active
638
+ :param kwargs:
639
+ :return: Service object
640
+ :rtype: dtlpy.entities.service.Service
641
+ """
642
+ if package is None:
643
+ if self._package is None:
644
+ raise exceptions.PlatformException('400', 'Please provide param package')
645
+ package = self._package
646
+
647
+ if verify is not None:
648
+ logger.warning('verify attribute has been deprecated and will be ignored')
649
+
650
+ is_global = kwargs.get('is_global', None)
651
+ jwt_forward = kwargs.get('jwt_forward', None)
652
+
653
+ if is_global is not None or jwt_forward is not None:
654
+ logger.warning(
655
+ 'Params jwt_forward and is_global are restricted to superuser. '
656
+ 'If you are not a superuser this action will not work'
657
+ )
658
+
659
+ service_config = dict()
660
+ if package is not None and package.service_config is not None:
661
+ service_config = package.service_config
662
+
663
+ if agent_versions is None:
664
+ if sdk_version is None:
665
+ sdk_version = service_config.get('versions', dict()).get('dtlpy', __version__)
666
+ agent_versions = {
667
+ "dtlpy": sdk_version
668
+ }
669
+
670
+ if project_id is None:
671
+ if self._project is None and self._project_id is None:
672
+ raise exceptions.PlatformException('400', 'Please provide project id')
673
+ elif self._project_id is not None:
674
+ project_id = self._project_id
675
+ elif self._project is not None:
676
+ project_id = self._project.id
677
+
678
+ if service_name is None:
679
+ service_name = 'default-service'
680
+
681
+ # payload
682
+ payload = {
683
+ 'name': service_name,
684
+ 'projectId': project_id,
685
+ 'packageId': package.id,
686
+ 'initParams': self._parse_init_input(init_input=init_input),
687
+ 'botUserName': self._get_bot_email(bot=bot),
688
+ 'versions': agent_versions,
689
+ 'packageRevision': revision if revision is not None else package.version,
690
+ 'driverId': driver_id,
691
+ 'active': active
692
+ }
693
+
694
+ if secrets is not None:
695
+ if not isinstance(secrets, list):
696
+ secrets = [secrets]
697
+ payload['secrets'] = secrets
698
+
699
+ if integrations is not None:
700
+ if not isinstance(integrations, list):
701
+ integrations = [integrations]
702
+ payload['integrations'] = integrations
703
+
704
+ if runtime is not None:
705
+ if isinstance(runtime, entities.KubernetesRuntime):
706
+ runtime = runtime.to_json()
707
+
708
+ if pod_type is not None:
709
+ if runtime is None:
710
+ runtime = {'podType': pod_type}
711
+ else:
712
+ runtime['podType'] = pod_type
713
+
714
+ if runtime is not None:
715
+ payload['runtime'] = runtime
716
+
717
+ if module_name is not None:
718
+ payload['moduleName'] = module_name
719
+
720
+ if is_global is not None:
721
+ payload['global'] = is_global
722
+
723
+ if max_attempts is not None:
724
+ payload['maxAttempts'] = max_attempts
725
+
726
+ if jwt_forward is not None:
727
+ payload['useUserJwt'] = jwt_forward
728
+
729
+ if run_execution_as_process is not None:
730
+ payload['runExecutionAsProcess'] = run_execution_as_process
731
+
732
+ if drain_time is not None:
733
+ payload['drainTime'] = drain_time
734
+
735
+ if on_reset is not None:
736
+ payload['onReset'] = on_reset
737
+
738
+ if execution_timeout is not None:
739
+ payload['executionTimeout'] = execution_timeout
740
+
741
+ # request
742
+ success, response = self._client_api.gen_request(
743
+ req_type='post',
744
+ path='/services',
745
+ json_req=payload
746
+ )
747
+
748
+ # exception handling
749
+ if not success:
750
+ raise exceptions.PlatformException(response)
751
+
752
+ # return entity
753
+ return entities.Service.from_json(
754
+ _json=response.json(),
755
+ client_api=self._client_api,
756
+ package=package,
757
+ project=self._project
758
+ )
759
+
760
+ @_api_reference.add(path='/services/{id}', method='delete')
761
+ def delete(self, service_name: str = None, service_id: str = None, force=False):
762
+ """
763
+ Delete Service object
764
+
765
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
766
+
767
+ You must provide at least ONE of the following params: service_id, service_name.
768
+
769
+ :param force:
770
+ :param str service_name: by name
771
+ :param str service_id: by id
772
+ :return: True
773
+ :rtype: bool
774
+
775
+ **Example**:
776
+
777
+ .. code-block:: python
778
+
779
+ is_deleted = package.services.delete(service_id='service_id')
780
+ """
781
+ # get bby name
782
+ if service_id is None:
783
+ if service_name is None:
784
+ raise exceptions.PlatformException('400', 'Must provide either service id or service name')
785
+ else:
786
+ service_id = self.get(service_name=service_name).id
787
+
788
+ path = "/services/{}".format(service_id)
789
+ if force:
790
+ path = '{}?force=true'.format(path)
791
+
792
+ # request
793
+ success, response = self._client_api.gen_request(
794
+ req_type="delete",
795
+ path=path
796
+ )
797
+ if not success:
798
+ raise exceptions.PlatformException(response)
799
+ return True
800
+
801
+ @_api_reference.add(path='/services/{id}', method='patch')
802
+ def update(self, service: entities.Service, force: bool = False) -> entities.Service:
803
+ """
804
+ Update service changes to platform.
805
+
806
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
807
+
808
+ :param dtlpy.entities.service.Service service: Service entity
809
+ :param bool force: optional - terminate old replicas immediately
810
+ :return: Service entity
811
+ :rtype: dtlpy.entities.service.Service
812
+
813
+ **Example**:
814
+
815
+ .. code-block:: python
816
+
817
+ service = package.services.update(service='service_entity')
818
+ """
819
+ assert isinstance(service, entities.Service)
820
+
821
+ # payload
822
+ payload = service.to_json()
823
+
824
+ # request
825
+ url = '/services/{}'.format(service.id)
826
+ if force:
827
+ url = '{}?force=true'.format(url)
828
+ success, response = self._client_api.gen_request(req_type='patch',
829
+ path=url,
830
+ json_req=payload)
831
+
832
+ # exception handling
833
+ if not success:
834
+ raise exceptions.PlatformException(response)
835
+
836
+ # return entity
837
+ if self._package is not None:
838
+ package = self._package
839
+ else:
840
+ package = service._package
841
+
842
+ return entities.Service.from_json(_json=response.json(),
843
+ client_api=self._client_api,
844
+ package=package,
845
+ project=self._project)
846
+
847
+ def activate_slots(
848
+ self,
849
+ service: entities.Service,
850
+ project_id: str = None,
851
+ task_id: str = None,
852
+ dataset_id: str = None,
853
+ org_id: str = None,
854
+ user_email: str = None,
855
+ slots: List[entities.PackageSlot] = None,
856
+ role=None,
857
+ prevent_override: bool = True,
858
+ visible: bool = True,
859
+ icon: str = 'fas fa-magic',
860
+ **kwargs
861
+ ):
862
+ """
863
+ Activate service slots (creates buttons in the UI that activate services).
864
+
865
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
866
+
867
+ :param dtlpy.entities.service.Service service: service entity
868
+ :param str project_id: project id
869
+ :param str task_id: task id
870
+ :param str dataset_id: dataset id
871
+ :param str org_id: org id
872
+ :param str user_email: user email
873
+ :param list slots: list of entities.PackageSlot
874
+ :param str role: user role MemberOrgRole.ADMIN, MemberOrgRole.owner, MemberOrgRole.MEMBER, MemberOrgRole.WORKER
875
+ :param bool prevent_override: True to prevent override
876
+ :param bool visible: visible
877
+ :param str icon: icon
878
+ :param kwargs: all additional arguments
879
+ :return: list of user setting for activated slots
880
+ :rtype: list
881
+
882
+ **Example**:
883
+
884
+ .. code-block:: python
885
+
886
+ setting = package.services.activate_slots(service='service_entity',
887
+ project_id='project_id',
888
+ slots=List[entities.PackageSlot],
889
+ icon='fas fa-magic')
890
+ """
891
+ package = service.package
892
+ if not isinstance(package.slots, list) or len(package.slots) == 0:
893
+ raise exceptions.PlatformException('400', "Service's package has no slots")
894
+
895
+ if kwargs.get('is_global', False):
896
+ project_id = '*'
897
+ scope_ids = [project_id]
898
+ else:
899
+ scope_ids = [s_id for s_id in [project_id, task_id, org_id, dataset_id, user_email] if s_id is not None]
900
+ if len(scope_ids) == 0:
901
+ raise exceptions.PlatformException('400', "Must provide scope resource ID")
902
+
903
+ settings = list()
904
+
905
+ if role is None:
906
+ role = entities.Role.ALL
907
+
908
+ if not slots:
909
+ slots = [s.to_json() for s in service.package.slots]
910
+ elif isinstance(slots, list) and isinstance(slots[0], entities.PackageSlot):
911
+ slots = [s.to_json() for s in slots]
912
+ else:
913
+ raise exceptions.PlatformException('400', "Slots param must be a list of PackageSlot objects")
914
+
915
+ for scope_id in scope_ids:
916
+
917
+ if kwargs.get('is_global', False):
918
+ scope_type = entities.PlatformEntityType.DATALOOP
919
+ elif scope_id == project_id:
920
+ scope_type = entities.PlatformEntityType.PROJECT
921
+ elif scope_id == task_id:
922
+ scope_type = entities.PlatformEntityType.TASK
923
+ elif scope_id == dataset_id:
924
+ scope_type = entities.PlatformEntityType.DATASET
925
+ elif scope_id == user_email:
926
+ scope_type = entities.PlatformEntityType.USER
927
+ elif scope_id == org_id:
928
+ scope_type = entities.PlatformEntityType.ORG
929
+ else:
930
+ raise exceptions.PlatformException('400', "Unknown resource id")
931
+
932
+ setting = entities.Setting(
933
+ default_value=True,
934
+ value=True,
935
+ inputs=None,
936
+ name=service.name,
937
+ value_type=entities.SettingsValueTypes.BOOLEAN,
938
+ scope=entities.SettingScope(
939
+ type=scope_type,
940
+ id=scope_id,
941
+ role=role,
942
+ prevent_override=prevent_override,
943
+ visible=visible
944
+ ),
945
+ metadata={
946
+ 'serviceId': service.id,
947
+ 'serviceName': service.name,
948
+ 'projectId': service.project_id,
949
+ 'slots': slots
950
+ },
951
+ description=service.name,
952
+ icon=icon,
953
+ section_name=entities.SettingsSectionNames.APPLICATIONS,
954
+ sub_section_name=None,
955
+ hint=None
956
+ )
957
+
958
+ try:
959
+ settings.append(self._settings.create(setting=setting))
960
+ except exceptions.BadRequest as err:
961
+ if 'settings already exist' in err.message:
962
+ old_setting = self._settings.get(setting_name=setting.name)
963
+ setting.id = old_setting.id
964
+ settings.append(self._settings.update(setting=setting))
965
+ else:
966
+ raise err
967
+
968
+ return settings
969
+
970
+ @_api_reference.add(path='/services/logs', method='post')
971
+ def log(self,
972
+ service,
973
+ size=100,
974
+ checkpoint=None,
975
+ start=None,
976
+ end=None,
977
+ follow=False,
978
+ text=None,
979
+ execution_id=None,
980
+ function_name=None,
981
+ replica_id=None,
982
+ system=False,
983
+ view=True,
984
+ until_completed=True,
985
+ log_level='DEBUG',
986
+ model_id: str = None,
987
+ model_operation: str = None,
988
+ project_id: str = None
989
+ ):
990
+ """
991
+ Get service logs.
992
+
993
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
994
+
995
+ :param dtlpy.entities.service.Service service: service object
996
+ :param int size: size
997
+ :param dict checkpoint: the information from the lst point checked in the service
998
+ :param str start: iso format time
999
+ :param str end: iso format time
1000
+ :param bool follow: if true, keep stream future logs
1001
+ :param str text: text
1002
+ :param str execution_id: execution id
1003
+ :param str function_name: function name
1004
+ :param str replica_id: replica id
1005
+ :param bool system: system
1006
+ :param bool view: if true, print out all the logs
1007
+ :param bool until_completed: wait until completed
1008
+ :param str log_level: the log level to display dl.LoggingLevel
1009
+ :param str model_id: model id
1010
+ :param str model_operation: model operation action
1011
+ :param str project_id: project id
1012
+ :return: ServiceLog entity
1013
+ :rtype: ServiceLog
1014
+
1015
+ **Example**:
1016
+
1017
+ .. code-block:: python
1018
+
1019
+ service_logs = package.services.log(service='service_entity')
1020
+ """
1021
+ if not isinstance(service, entities.Service) and model_id is None:
1022
+ raise exceptions.PlatformException('400', 'Must provide service or model_id')
1023
+ if isinstance(log_level, str):
1024
+ log_level = log_level.upper()
1025
+
1026
+ payload = {
1027
+ 'direction': 'asc',
1028
+ 'follow': follow,
1029
+ 'system': system,
1030
+ 'logLevel': log_level
1031
+ }
1032
+
1033
+ if service is not None:
1034
+ payload['serviceId'] = service.id
1035
+
1036
+ if size is not None:
1037
+ payload['size'] = size
1038
+
1039
+ if execution_id is not None:
1040
+ payload['executionId'] = [execution_id]
1041
+
1042
+ if function_name is not None:
1043
+ payload['functionName'] = function_name
1044
+
1045
+ if text is not None:
1046
+ payload['text'] = text
1047
+
1048
+ if replica_id is not None:
1049
+ payload['replicaId'] = replica_id
1050
+
1051
+ if checkpoint is not None:
1052
+ payload['checkpoint'] = checkpoint
1053
+
1054
+ if start is not None:
1055
+ payload['start'] = start
1056
+ else:
1057
+ payload['start'] = datetime.datetime(datetime.date.today().year,
1058
+ datetime.date.today().month,
1059
+ datetime.date.today().day,
1060
+ 0,
1061
+ 0,
1062
+ 0).isoformat()
1063
+
1064
+ if end is not None:
1065
+ payload['end'] = end
1066
+
1067
+ if model_id is not None:
1068
+ payload['modelId'] = model_id
1069
+
1070
+ if model_operation is not None:
1071
+ payload['modelOperation'] = model_operation
1072
+
1073
+ if project_id is not None:
1074
+ payload['projectId'] = project_id
1075
+ else:
1076
+ payload['projectId'] = service.project_id
1077
+
1078
+ # request
1079
+ success, response = self._client_api.gen_request(req_type='post',
1080
+ path='/services/logs',
1081
+ json_req=payload)
1082
+
1083
+ # exception handling
1084
+ if not success:
1085
+ raise exceptions.PlatformException(response)
1086
+
1087
+ log = ServiceLog(_json=response.json(),
1088
+ service=service,
1089
+ services=self,
1090
+ start=payload['start'],
1091
+ follow=follow,
1092
+ execution_id=execution_id,
1093
+ function_name=function_name,
1094
+ replica_id=replica_id,
1095
+ system=system,
1096
+ model_id=model_id,
1097
+ model_operation=model_operation,
1098
+ project_id=project_id)
1099
+
1100
+ if view:
1101
+ log.view(until_completed=until_completed)
1102
+ else:
1103
+ return log
1104
+
1105
+ def execute(self,
1106
+ service: entities.Service = None,
1107
+ service_id: str = None,
1108
+ service_name: str = None,
1109
+ sync: bool = False,
1110
+ function_name: str = None,
1111
+ stream_logs: bool = False,
1112
+ execution_input=None,
1113
+ resource=None,
1114
+ item_id=None,
1115
+ dataset_id=None,
1116
+ annotation_id=None,
1117
+ project_id=None,
1118
+ ) -> entities.Execution:
1119
+ """
1120
+ Execute a function on an existing service.
1121
+
1122
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1123
+
1124
+ :param dtlpy.entities.service.Service service: service entity
1125
+ :param str service_id: service id
1126
+ :param str service_name: service name
1127
+ :param bool sync: wait for function to end
1128
+ :param str function_name: function name to run
1129
+ :param bool stream_logs: prints logs of the new execution. only works with sync=True
1130
+ :param execution_input: input dictionary or list of FunctionIO entities
1131
+ :param str resource: dl.PackageInputType - input type.
1132
+ :param str item_id: str - optional - input to function
1133
+ :param str dataset_id: str - optional - input to function
1134
+ :param str annotation_id: str - optional - input to function
1135
+ :param str project_id: str - resource's project
1136
+ :return: entities.Execution
1137
+ :rtype: dtlpy.entities.execution.Execution
1138
+
1139
+ **Example**:
1140
+
1141
+ .. code-block:: python
1142
+
1143
+ execution = package.services.execute(service='service_entity',
1144
+ function_name='run',
1145
+ item_id='item_id',
1146
+ project_id='project_id')
1147
+ """
1148
+ if service is None:
1149
+ service = self.get(service_id=service_id, service_name=service_name)
1150
+ execution = repositories.Executions(service=service,
1151
+ client_api=self._client_api,
1152
+ project=self._project).create(service_id=service.id,
1153
+ sync=sync,
1154
+ execution_input=execution_input,
1155
+ function_name=function_name,
1156
+ resource=resource,
1157
+ item_id=item_id,
1158
+ dataset_id=dataset_id,
1159
+ annotation_id=annotation_id,
1160
+ project_id=project_id,
1161
+ stream_logs=stream_logs)
1162
+ return execution
1163
+
1164
+ @_api_reference.add(path='/services', method='post')
1165
+ def deploy(self,
1166
+ service_name: str = None,
1167
+ package: entities.Package = None,
1168
+ bot: Union[entities.Bot, str] = None,
1169
+ revision: str or int = None,
1170
+ init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
1171
+ runtime: Union[entities.KubernetesRuntime, dict] = None,
1172
+ pod_type: entities.InstanceCatalog = None,
1173
+ sdk_version: str = None,
1174
+ agent_versions: dict = None,
1175
+ verify: bool = True,
1176
+ checkout: bool = False,
1177
+ module_name: str = None,
1178
+ project_id: str = None,
1179
+ driver_id: str = None,
1180
+ func: Callable = None,
1181
+ run_execution_as_process: bool = None,
1182
+ execution_timeout: int = None,
1183
+ drain_time: int = None,
1184
+ max_attempts: int = None,
1185
+ on_reset: str = None,
1186
+ force: bool = False,
1187
+ secrets: list = None,
1188
+ integrations: list = None,
1189
+ active: bool = True,
1190
+ **kwargs) -> entities.Service:
1191
+ """
1192
+ Deploy service.
1193
+
1194
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1195
+
1196
+ :param str service_name: name
1197
+ :param dtlpy.entities.package.Package package: package entity
1198
+ :param str bot: bot email
1199
+ :param str revision: package revision of version
1200
+ :param init_input: config to run at startup
1201
+ :param dict runtime: runtime resources
1202
+ :param str pod_type: pod type dl.InstanceCatalog
1203
+ :param str sdk_version: - optional - string - sdk version
1204
+ :param str agent_versions: - dictionary - - optional -versions of sdk
1205
+ :param bool verify: if true, verify the inputs
1206
+ :param bool checkout: if true, checkout (switch) to service
1207
+ :param str module_name: module name
1208
+ :param str project_id: project id
1209
+ :param str driver_id: driver id
1210
+ :param Callable func: function to deploy
1211
+ :param bool run_execution_as_process: if true, run execution as process
1212
+ :param int execution_timeout: execution timeout in seconds
1213
+ :param int drain_time: drain time in seconds
1214
+ :param int max_attempts: maximum execution retries in-case of a service reset
1215
+ :param str on_reset: what happens on reset
1216
+ :param bool force: optional - if true, terminate old replicas immediately
1217
+ :param list secrets: list of the integrations ids
1218
+ :param list integrations: list of the integrations
1219
+ :param bool active: if true, activate the service
1220
+ :param kwargs: list of additional arguments
1221
+ :return: Service object
1222
+ :rtype: dtlpy.entities.service.Service
1223
+
1224
+ **Example**:
1225
+
1226
+ .. code-block:: python
1227
+
1228
+ service = package.services.deploy(service_name=package_name,
1229
+ execution_timeout=3 * 60 * 60,
1230
+ module_name=module.name,
1231
+ runtime=dl.KubernetesRuntime(
1232
+ concurrency=10,
1233
+ pod_type=dl.InstanceCatalog.REGULAR_S,
1234
+ autoscaler=dl.KubernetesRabbitmqAutoscaler(
1235
+ min_replicas=1,
1236
+ max_replicas=20,
1237
+ queue_length=20
1238
+ )
1239
+ )
1240
+ )
1241
+ """
1242
+ if package is None and isinstance(package, entities.Dpk):
1243
+ raise exceptions.PlatformException('400', 'cannot deploy dpk package. Please install the app')
1244
+ package = package if package is not None else self._package
1245
+ if service_name is None:
1246
+ get_name = False
1247
+ if package is not None and package.service_config is not None and 'name' in package.service_config:
1248
+ service_name = package.service_config['name']
1249
+ get_name = True
1250
+ else:
1251
+ if package is not None:
1252
+ service_name = package.name
1253
+ else:
1254
+ service_name = 'default-service'
1255
+ if not get_name:
1256
+ logger.warning('service_name not provided, using: {} by default'.format(service_name))
1257
+
1258
+ if isinstance(revision, int):
1259
+ logger.warning('Deprecation Warning - Package/service versions have been refactored'
1260
+ 'The version you provided has type: int, it will be converted to: 1.0.{}'
1261
+ 'Next time use a 3-level semver for package/service versions'.format(revision))
1262
+
1263
+ if func is not None:
1264
+ package = self.__deploy_function(name=service_name, project=self._project, func=func)
1265
+
1266
+ if init_input is not None and not isinstance(init_input, dict):
1267
+ if not isinstance(init_input, list):
1268
+ init_input = [init_input]
1269
+
1270
+ if len(init_input) > 0 and isinstance(init_input[0], entities.FunctionIO):
1271
+ params = dict()
1272
+ for i_param, param in enumerate(init_input):
1273
+ params[param.name] = param.value
1274
+ init_input = params
1275
+ elif len(init_input) == 0:
1276
+ init_input = None
1277
+ else:
1278
+ raise exceptions.PlatformException(
1279
+ error='400',
1280
+ message='Unknown init_input type. expecting list or dict, got: {}'.format(type(init_input))
1281
+ )
1282
+
1283
+ if project_id is None:
1284
+ if self._project is not None:
1285
+ project_id = self._project.id
1286
+ else:
1287
+ project_id = self._project_id
1288
+
1289
+ filters = entities.Filters(resource=entities.FiltersResource.SERVICE, use_defaults=False)
1290
+ filters.add(field='name', values=service_name)
1291
+ if project_id is not None:
1292
+ filters.add(field='projectId', values=project_id)
1293
+ services = self.list(filters=filters)
1294
+ if services.items_count > 1:
1295
+ raise exceptions.PlatformException('400',
1296
+ 'More than 1 service by this name are associated with this user. '
1297
+ 'Please provide project_id')
1298
+ elif services.items_count > 0:
1299
+ service = services.items[0]
1300
+ if runtime is not None:
1301
+ service.runtime = runtime
1302
+ if init_input is not None:
1303
+ service.init_input = init_input
1304
+ if revision is not None:
1305
+ service.package_revision = revision
1306
+ if agent_versions is not None:
1307
+ service.versions = agent_versions
1308
+ elif sdk_version:
1309
+ service.versions = {'dtlpy': sdk_version}
1310
+ if driver_id is not None:
1311
+ service.driver_id = driver_id
1312
+ if secrets is not None:
1313
+ if not isinstance(secrets, list):
1314
+ secrets = [secrets]
1315
+ service.secrets = secrets
1316
+ if integrations is not None:
1317
+ if not isinstance(integrations, list):
1318
+ integrations = [integrations]
1319
+ service.integrations = integrations
1320
+ service = self.update(service=service, force=force)
1321
+ else:
1322
+ service = self._create(service_name=service_name,
1323
+ package=package,
1324
+ project_id=project_id,
1325
+ bot=bot,
1326
+ revision=revision,
1327
+ init_input=init_input,
1328
+ runtime=runtime,
1329
+ pod_type=pod_type,
1330
+ sdk_version=sdk_version,
1331
+ agent_versions=agent_versions,
1332
+ verify=verify,
1333
+ module_name=module_name,
1334
+ driver_id=driver_id,
1335
+ jwt_forward=kwargs.get('jwt_forward', None),
1336
+ is_global=kwargs.get('is_global', None),
1337
+ run_execution_as_process=run_execution_as_process,
1338
+ execution_timeout=execution_timeout,
1339
+ drain_time=drain_time,
1340
+ max_attempts=max_attempts,
1341
+ on_reset=on_reset,
1342
+ secrets=secrets,
1343
+ integrations=integrations,
1344
+ active=active
1345
+ )
1346
+ if checkout:
1347
+ self.checkout(service=service)
1348
+ return service
1349
+
1350
+ @staticmethod
1351
+ def __get_import_string(imports: List[str]):
1352
+ import_string = ''
1353
+ if imports is not None:
1354
+ import_string = '\n'.join(imports)
1355
+ return import_string
1356
+
1357
+ @staticmethod
1358
+ def __get_inputs(func):
1359
+ method = inspect.signature(func)
1360
+ params = list(method.parameters)
1361
+ inpts = list()
1362
+ inputs_types = {i.name.lower(): i.value for i in list(entities.PackageInputType)}
1363
+ for arg in params:
1364
+ if arg in inputs_types:
1365
+ inpt_type = inputs_types[arg]
1366
+ else:
1367
+ inpt_type = entities.PackageInputType.JSON
1368
+ inpts.append(entities.FunctionIO(type=inpt_type, name=arg))
1369
+ return inpts
1370
+
1371
+ def __deploy_function(self,
1372
+ name: str,
1373
+ func: Callable,
1374
+ project: entities.Project) -> entities.Package:
1375
+ package_dir = tempfile.mkdtemp()
1376
+ # imports_string = self.__get_import_string()
1377
+ imports_string = ''
1378
+
1379
+ main_file = os.path.join(package_dir, entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
1380
+ with open(assets.paths.PARTIAL_MAIN_FILEPATH, 'r') as f:
1381
+ main_string = f.read()
1382
+ lines = inspect.getsourcelines(func)
1383
+ tabs_diff = lines[0][0].count(' ') - 1
1384
+ for line_index in range(len(lines[0])):
1385
+ line_tabs = lines[0][line_index].count(' ') - tabs_diff
1386
+ lines[0][line_index] = (' ' * line_tabs) + lines[0][line_index].strip() + '\n'
1387
+
1388
+ method_func_string = "".join(lines[0])
1389
+
1390
+ with open(main_file, 'w') as f:
1391
+ f.write('{}\n{}\n @staticmethod\n{}'.format(imports_string, main_string,
1392
+ method_func_string))
1393
+
1394
+ function = entities.PackageFunction(name=func.__name__, inputs=self.__get_inputs(func=func))
1395
+ module = entities.PackageModule(functions=[function],
1396
+ entry_point=entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
1397
+ packages = repositories.Packages(client_api=self._client_api, project=project)
1398
+ return packages.push(src_path=package_dir,
1399
+ package_name=name,
1400
+ checkout=True,
1401
+ modules=[module])
1402
+
1403
+ def deploy_from_local_folder(self,
1404
+ cwd=None,
1405
+ service_file=None,
1406
+ bot=None,
1407
+ checkout=False,
1408
+ force=False
1409
+ ) -> entities.Service:
1410
+ """
1411
+ Deploy from local folder in local environment.
1412
+
1413
+ **Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
1414
+
1415
+ :param str cwd: optional - package working directory. Default=cwd
1416
+ :param str service_file: optional - service file. Default=None
1417
+ :param str bot: bot
1418
+ :param checkout: checkout
1419
+ :param bool force: optional - terminate old replicas immediately
1420
+ :return: Service object
1421
+ :rtype: dtlpy.entities.service.Service
1422
+
1423
+ **Example**:
1424
+
1425
+ .. code-block:: python
1426
+
1427
+ service = package.services.deploy_from_local_folder(cwd='file_path',
1428
+ service_file='service_file')
1429
+ """
1430
+ # get cwd and service.json path
1431
+ if cwd is None:
1432
+ cwd = os.getcwd()
1433
+ if service_file is None:
1434
+ service_file = os.path.join(cwd, assets.paths.SERVICE_FILENAME)
1435
+
1436
+ # load service json
1437
+ if os.path.isfile(service_file):
1438
+ with open(service_file, 'r') as f:
1439
+ service_json = json.load(f)
1440
+ service_triggers = service_json.get('triggers', list())
1441
+ else:
1442
+ raise exceptions.PlatformException(error='400',
1443
+ message='Could not find service.json in path: {}'.format(cwd))
1444
+
1445
+ # get package
1446
+ package_name = service_json.get('packageName', None)
1447
+ packages = repositories.Packages(client_api=self._client_api, project=self._project)
1448
+
1449
+ if package_name is None:
1450
+ package = packages.get()
1451
+ else:
1452
+ package = packages.get(package_name=package_name)
1453
+
1454
+ name = service_json.get('name', None)
1455
+ revision = service_json.get('revision', package.version)
1456
+ init_input = service_json.get('initParams', dict())
1457
+ runtime = service_json.get('runtime', dict())
1458
+ sdk_version = service_json.get('version', None)
1459
+ agent_versions = service_json.get('versions', None)
1460
+ verify = service_json.get('verify', True)
1461
+ module_name = service_json.get('module_name', None)
1462
+ run_execution_as_process = service_json.get('run_execution_as_process', None)
1463
+ execution_timeout = service_json.get('execution_timeout', None)
1464
+ drain_time = service_json.get('drain_time', None)
1465
+ on_reset = service_json.get('on_reset', None)
1466
+ max_attempts = service_json.get('maxAttempts', None)
1467
+
1468
+ service = self.deploy(bot=bot,
1469
+ service_name=name,
1470
+ package=package,
1471
+ revision=revision,
1472
+ runtime=runtime,
1473
+ init_input=init_input,
1474
+ sdk_version=sdk_version,
1475
+ agent_versions=agent_versions,
1476
+ verify=verify,
1477
+ checkout=checkout,
1478
+ run_execution_as_process=run_execution_as_process,
1479
+ execution_timeout=execution_timeout,
1480
+ drain_time=drain_time,
1481
+ max_attempts=max_attempts,
1482
+ on_reset=on_reset,
1483
+ module_name=module_name,
1484
+ force=force
1485
+ )
1486
+
1487
+ logger.info('Service was deployed successfully. Service id: {}'.format(service.id))
1488
+
1489
+ if len(service_triggers) > 0:
1490
+ logger.info('Creating triggers...')
1491
+ triggers = repositories.Triggers(client_api=self._client_api, project=self._project)
1492
+
1493
+ for trigger in service_triggers:
1494
+ name = trigger.get('name', None)
1495
+ filters = trigger.get('filter', dict())
1496
+ resource = trigger['resource']
1497
+ actions = trigger.get('actions', list())
1498
+ active = trigger.get('active', True)
1499
+ execution_mode = trigger.get('executionMode', None)
1500
+ function_name = trigger.get('function', None)
1501
+
1502
+ trigger = triggers.create(service_id=service.id,
1503
+ name=name,
1504
+ filters=filters,
1505
+ resource=resource,
1506
+ actions=actions,
1507
+ active=active,
1508
+ execution_mode=execution_mode,
1509
+ function_name=function_name)
1510
+
1511
+ logger.info('Trigger was created successfully. Service id: {}'.format(trigger.id))
1512
+
1513
+ logging.info('Successfully deployed!')
1514
+ return service
1515
+
1516
+ def __enable_cache(self,
1517
+ url,
1518
+ organization: entities.Organization,
1519
+ pod_type=entities.PodType.SMALL):
1520
+ payload = {
1521
+ "org": {
1522
+ "name": organization.name,
1523
+ "id": organization.id
1524
+ },
1525
+ "runner": {
1526
+ "podType": pod_type # small, medium, high
1527
+ }
1528
+ }
1529
+
1530
+ return self._client_api.gen_request(req_type='post',
1531
+ path=url,
1532
+ json_req=payload)
1533
+
1534
+ def __polling_wait(self, organization, pod_type, backoff_factor=1):
1535
+ fs_url_path = '/services/fs-cache?mode={}'.format('get')
1536
+ i = 1
1537
+ while True:
1538
+ success, response = self.__enable_cache(url=fs_url_path, organization=organization, pod_type=pod_type)
1539
+ if response.json().get('state', None) == 'READY':
1540
+ break
1541
+ sleep_time = min(backoff_factor * (2 ** (i - 1)), MAX_WAIT_TIME)
1542
+ logger.debug("Going to sleep {:.2f}[s]".format(sleep_time))
1543
+ time.sleep(sleep_time)
1544
+ i += 1
1545
+ return success
1546
+
1547
+ def _cache_action(self,
1548
+ organization: entities.Organization = None,
1549
+ mode=entities.CacheAction.APPLY,
1550
+ pod_type=entities.PodType.SMALL):
1551
+ """
1552
+ Add or remove Cache for the org
1553
+
1554
+ **Prerequisites**: You must be an organization *owner*
1555
+
1556
+ You must provide at least ONE of the following params: organization, organization_name, or organization_id.
1557
+
1558
+ :param entities.Organization organization: Organization object
1559
+ :param str mode: dl.CacheAction.APPLY or dl.CacheAction.DESTROY
1560
+ :param entities.PodType pod_type: dl.PodType.SMALL, dl.PodType.MEDIUM, dl.PodType.HIGH
1561
+ :return: True if success
1562
+ :rtype: bool
1563
+
1564
+ **Example**:
1565
+
1566
+ .. code-block:: python
1567
+
1568
+ success = dl.organizations.enable_cache(organization='organization',
1569
+ mode=dl.CacheAction.APPLY)
1570
+ """
1571
+ if organization is None:
1572
+ raise exceptions.PlatformException(
1573
+ error='400',
1574
+ message='Must provide an identifier in inputs')
1575
+
1576
+ fs_mode = mode if mode != entities.CacheAction.APPLY else '{}-filestore'.format(mode)
1577
+ apply_fs_url_path = '/services/fs-cache?mode={}'.format(fs_mode)
1578
+ apply_volume_url_path = '/services/fs-cache?mode={}'.format(mode)
1579
+ cache_url_path = '/services/cache?mode={}'.format(mode)
1580
+
1581
+ success, response = self.__enable_cache(url=apply_fs_url_path, organization=organization, pod_type=pod_type)
1582
+ if not success:
1583
+ raise exceptions.PlatformException(response)
1584
+
1585
+ if mode == entities.CacheAction.APPLY:
1586
+ self.__polling_wait(organization=organization, pod_type=pod_type)
1587
+ success, response = self.__enable_cache(url=apply_volume_url_path, organization=organization,
1588
+ pod_type=pod_type)
1589
+ if not success:
1590
+ raise exceptions.PlatformException(response)
1591
+
1592
+ success, response = self.__enable_cache(url=cache_url_path, organization=organization, pod_type=pod_type)
1593
+ if not success:
1594
+ raise exceptions.PlatformException(response)
1595
+
1596
+ return True
1597
+
1598
+ def restart(self, service: entities.Service, replica_name: str = None):
1599
+ """
1600
+ Restart service replica
1601
+
1602
+ **Prerequisites**: You must be in the role of a *developer*.
1603
+
1604
+ :param dtlpy.entities.service.Service service: service entity
1605
+ :param str replica_name: replica name
1606
+ :return: True
1607
+ :rtype: bool
1608
+
1609
+ **Example**:
1610
+
1611
+ .. code-block:: python
1612
+
1613
+ is_restarted = dl.services.restart(service='service_entity',
1614
+ replica_name='replica_name')
1615
+ """
1616
+ payload = {}
1617
+
1618
+ if replica_name is not None:
1619
+ payload['replicaName'] = replica_name
1620
+
1621
+ # request
1622
+ success, response = self._client_api.gen_request(req_type='post',
1623
+ path='/services/{}/restart'.format(service.id),
1624
+ json_req=payload)
1625
+
1626
+ # exception handling
1627
+ if not success:
1628
+ raise exceptions.PlatformException(response)
1629
+
1630
+ return True
1631
+
1632
+
1633
+ class ServiceLog:
1634
+ """
1635
+ Service Log
1636
+ """
1637
+
1638
+ def __init__(self,
1639
+ _json: dict,
1640
+ service: entities.Service,
1641
+ services: Services,
1642
+ start=None,
1643
+ follow=None,
1644
+ execution_id=None,
1645
+ function_name=None,
1646
+ replica_id=None,
1647
+ system=False,
1648
+ model_id=None,
1649
+ model_operation=None,
1650
+ project_id=None):
1651
+
1652
+ self.logs = _json.get('logs', dict())
1653
+ self.checkpoint = _json.get('checkpoint', None)
1654
+ self.stop = _json.get('stop', False)
1655
+ self.service = service
1656
+ self.services = services
1657
+ self.start = start
1658
+ self.follow = follow
1659
+ self.execution_id = execution_id
1660
+ self.function_name = function_name
1661
+ self.replica_id = replica_id
1662
+ self.system = system
1663
+ self.model_id = model_id
1664
+ self.model_operation = model_operation
1665
+ self.project_id = project_id
1666
+
1667
+ def get_next_log(self):
1668
+ log = self.services.log(service=self.service,
1669
+ checkpoint=self.checkpoint,
1670
+ start=self.start,
1671
+ follow=self.follow,
1672
+ execution_id=self.execution_id,
1673
+ function_name=self.function_name,
1674
+ replica_id=self.replica_id,
1675
+ system=self.system,
1676
+ view=False,
1677
+ model_id=self.model_id,
1678
+ model_operation=self.model_operation,
1679
+ project_id=self.project_id)
1680
+
1681
+ self.logs = log.logs
1682
+ self.checkpoint = log.checkpoint
1683
+ self.stop = log.stop
1684
+
1685
+ def view(self, until_completed):
1686
+ """
1687
+ View logs
1688
+
1689
+ :param until_completed:
1690
+ """
1691
+ try:
1692
+ for log in self:
1693
+ print(log)
1694
+ if until_completed and FUNCTION_END_LINE in log:
1695
+ break
1696
+ except KeyboardInterrupt:
1697
+ return
1698
+
1699
+ def __iter__(self):
1700
+ while not self.stop:
1701
+ for log in self.logs:
1702
+ yield '{}: {}'.format(log.get('timestamp', self.start), log.get('message', '').strip())
1703
+ self.get_next_log()
1704
+