dtlpy 1.114.17__py3-none-any.whl → 1.116.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtlpy/__init__.py +491 -491
- dtlpy/__version__.py +1 -1
- dtlpy/assets/__init__.py +26 -26
- dtlpy/assets/code_server/config.yaml +2 -2
- dtlpy/assets/code_server/installation.sh +24 -24
- dtlpy/assets/code_server/launch.json +13 -13
- dtlpy/assets/code_server/settings.json +2 -2
- dtlpy/assets/main.py +53 -53
- dtlpy/assets/main_partial.py +18 -18
- dtlpy/assets/mock.json +11 -11
- dtlpy/assets/model_adapter.py +83 -83
- dtlpy/assets/package.json +61 -61
- dtlpy/assets/package_catalog.json +29 -29
- dtlpy/assets/package_gitignore +307 -307
- dtlpy/assets/service_runners/__init__.py +33 -33
- dtlpy/assets/service_runners/converter.py +96 -96
- dtlpy/assets/service_runners/multi_method.py +49 -49
- dtlpy/assets/service_runners/multi_method_annotation.py +54 -54
- dtlpy/assets/service_runners/multi_method_dataset.py +55 -55
- dtlpy/assets/service_runners/multi_method_item.py +52 -52
- dtlpy/assets/service_runners/multi_method_json.py +52 -52
- dtlpy/assets/service_runners/single_method.py +37 -37
- dtlpy/assets/service_runners/single_method_annotation.py +43 -43
- dtlpy/assets/service_runners/single_method_dataset.py +43 -43
- dtlpy/assets/service_runners/single_method_item.py +41 -41
- dtlpy/assets/service_runners/single_method_json.py +42 -42
- dtlpy/assets/service_runners/single_method_multi_input.py +45 -45
- dtlpy/assets/voc_annotation_template.xml +23 -23
- dtlpy/caches/base_cache.py +32 -32
- dtlpy/caches/cache.py +473 -473
- dtlpy/caches/dl_cache.py +201 -201
- dtlpy/caches/filesystem_cache.py +89 -89
- dtlpy/caches/redis_cache.py +84 -84
- dtlpy/dlp/__init__.py +20 -20
- dtlpy/dlp/cli_utilities.py +367 -367
- dtlpy/dlp/command_executor.py +764 -764
- dtlpy/dlp/dlp +1 -1
- dtlpy/dlp/dlp.bat +1 -1
- dtlpy/dlp/dlp.py +128 -128
- dtlpy/dlp/parser.py +651 -651
- dtlpy/entities/__init__.py +83 -83
- dtlpy/entities/analytic.py +347 -311
- dtlpy/entities/annotation.py +1879 -1879
- dtlpy/entities/annotation_collection.py +699 -699
- dtlpy/entities/annotation_definitions/__init__.py +20 -20
- dtlpy/entities/annotation_definitions/base_annotation_definition.py +100 -100
- dtlpy/entities/annotation_definitions/box.py +195 -195
- dtlpy/entities/annotation_definitions/classification.py +67 -67
- dtlpy/entities/annotation_definitions/comparison.py +72 -72
- dtlpy/entities/annotation_definitions/cube.py +204 -204
- dtlpy/entities/annotation_definitions/cube_3d.py +149 -149
- dtlpy/entities/annotation_definitions/description.py +32 -32
- dtlpy/entities/annotation_definitions/ellipse.py +124 -124
- dtlpy/entities/annotation_definitions/free_text.py +62 -62
- dtlpy/entities/annotation_definitions/gis.py +69 -69
- dtlpy/entities/annotation_definitions/note.py +139 -139
- dtlpy/entities/annotation_definitions/point.py +117 -117
- dtlpy/entities/annotation_definitions/polygon.py +182 -182
- dtlpy/entities/annotation_definitions/polyline.py +111 -111
- dtlpy/entities/annotation_definitions/pose.py +92 -92
- dtlpy/entities/annotation_definitions/ref_image.py +86 -86
- dtlpy/entities/annotation_definitions/segmentation.py +240 -240
- dtlpy/entities/annotation_definitions/subtitle.py +34 -34
- dtlpy/entities/annotation_definitions/text.py +85 -85
- dtlpy/entities/annotation_definitions/undefined_annotation.py +74 -74
- dtlpy/entities/app.py +220 -220
- dtlpy/entities/app_module.py +107 -107
- dtlpy/entities/artifact.py +174 -174
- dtlpy/entities/assignment.py +399 -399
- dtlpy/entities/base_entity.py +214 -214
- dtlpy/entities/bot.py +113 -113
- dtlpy/entities/codebase.py +292 -296
- dtlpy/entities/collection.py +38 -38
- dtlpy/entities/command.py +169 -169
- dtlpy/entities/compute.py +449 -442
- dtlpy/entities/dataset.py +1299 -1285
- dtlpy/entities/directory_tree.py +44 -44
- dtlpy/entities/dpk.py +470 -470
- dtlpy/entities/driver.py +235 -223
- dtlpy/entities/execution.py +397 -397
- dtlpy/entities/feature.py +124 -124
- dtlpy/entities/feature_set.py +145 -145
- dtlpy/entities/filters.py +798 -645
- dtlpy/entities/gis_item.py +107 -107
- dtlpy/entities/integration.py +184 -184
- dtlpy/entities/item.py +959 -953
- dtlpy/entities/label.py +123 -123
- dtlpy/entities/links.py +85 -85
- dtlpy/entities/message.py +175 -175
- dtlpy/entities/model.py +684 -684
- dtlpy/entities/node.py +1005 -1005
- dtlpy/entities/ontology.py +810 -803
- dtlpy/entities/organization.py +287 -287
- dtlpy/entities/package.py +657 -657
- dtlpy/entities/package_defaults.py +5 -5
- dtlpy/entities/package_function.py +185 -185
- dtlpy/entities/package_module.py +113 -113
- dtlpy/entities/package_slot.py +118 -118
- dtlpy/entities/paged_entities.py +299 -299
- dtlpy/entities/pipeline.py +624 -624
- dtlpy/entities/pipeline_execution.py +279 -279
- dtlpy/entities/project.py +394 -394
- dtlpy/entities/prompt_item.py +505 -499
- dtlpy/entities/recipe.py +301 -301
- dtlpy/entities/reflect_dict.py +102 -102
- dtlpy/entities/resource_execution.py +138 -138
- dtlpy/entities/service.py +963 -958
- dtlpy/entities/service_driver.py +117 -117
- dtlpy/entities/setting.py +294 -294
- dtlpy/entities/task.py +495 -495
- dtlpy/entities/time_series.py +143 -143
- dtlpy/entities/trigger.py +426 -426
- dtlpy/entities/user.py +118 -118
- dtlpy/entities/webhook.py +124 -124
- dtlpy/examples/__init__.py +19 -19
- dtlpy/examples/add_labels.py +135 -135
- dtlpy/examples/add_metadata_to_item.py +21 -21
- dtlpy/examples/annotate_items_using_model.py +65 -65
- dtlpy/examples/annotate_video_using_model_and_tracker.py +75 -75
- dtlpy/examples/annotations_convert_to_voc.py +9 -9
- dtlpy/examples/annotations_convert_to_yolo.py +9 -9
- dtlpy/examples/convert_annotation_types.py +51 -51
- dtlpy/examples/converter.py +143 -143
- dtlpy/examples/copy_annotations.py +22 -22
- dtlpy/examples/copy_folder.py +31 -31
- dtlpy/examples/create_annotations.py +51 -51
- dtlpy/examples/create_video_annotations.py +83 -83
- dtlpy/examples/delete_annotations.py +26 -26
- dtlpy/examples/filters.py +113 -113
- dtlpy/examples/move_item.py +23 -23
- dtlpy/examples/play_video_annotation.py +13 -13
- dtlpy/examples/show_item_and_mask.py +53 -53
- dtlpy/examples/triggers.py +49 -49
- dtlpy/examples/upload_batch_of_items.py +20 -20
- dtlpy/examples/upload_items_and_custom_format_annotations.py +55 -55
- dtlpy/examples/upload_items_with_modalities.py +43 -43
- dtlpy/examples/upload_segmentation_annotations_from_mask_image.py +44 -44
- dtlpy/examples/upload_yolo_format_annotations.py +70 -70
- dtlpy/exceptions.py +125 -125
- dtlpy/miscellaneous/__init__.py +20 -20
- dtlpy/miscellaneous/dict_differ.py +95 -95
- dtlpy/miscellaneous/git_utils.py +217 -217
- dtlpy/miscellaneous/json_utils.py +14 -14
- dtlpy/miscellaneous/list_print.py +105 -105
- dtlpy/miscellaneous/zipping.py +130 -130
- dtlpy/ml/__init__.py +20 -20
- dtlpy/ml/base_feature_extractor_adapter.py +27 -27
- dtlpy/ml/base_model_adapter.py +1257 -1086
- dtlpy/ml/metrics.py +461 -461
- dtlpy/ml/predictions_utils.py +274 -274
- dtlpy/ml/summary_writer.py +57 -57
- dtlpy/ml/train_utils.py +60 -60
- dtlpy/new_instance.py +252 -252
- dtlpy/repositories/__init__.py +56 -56
- dtlpy/repositories/analytics.py +85 -85
- dtlpy/repositories/annotations.py +916 -916
- dtlpy/repositories/apps.py +383 -383
- dtlpy/repositories/artifacts.py +452 -452
- dtlpy/repositories/assignments.py +599 -599
- dtlpy/repositories/bots.py +213 -213
- dtlpy/repositories/codebases.py +559 -559
- dtlpy/repositories/collections.py +332 -332
- dtlpy/repositories/commands.py +152 -158
- dtlpy/repositories/compositions.py +61 -61
- dtlpy/repositories/computes.py +439 -435
- dtlpy/repositories/datasets.py +1504 -1291
- dtlpy/repositories/downloader.py +976 -903
- dtlpy/repositories/dpks.py +433 -433
- dtlpy/repositories/drivers.py +482 -470
- dtlpy/repositories/executions.py +815 -817
- dtlpy/repositories/feature_sets.py +226 -226
- dtlpy/repositories/features.py +255 -238
- dtlpy/repositories/integrations.py +484 -484
- dtlpy/repositories/items.py +912 -909
- dtlpy/repositories/messages.py +94 -94
- dtlpy/repositories/models.py +1000 -988
- dtlpy/repositories/nodes.py +80 -80
- dtlpy/repositories/ontologies.py +511 -511
- dtlpy/repositories/organizations.py +525 -525
- dtlpy/repositories/packages.py +1941 -1941
- dtlpy/repositories/pipeline_executions.py +451 -451
- dtlpy/repositories/pipelines.py +640 -640
- dtlpy/repositories/projects.py +539 -539
- dtlpy/repositories/recipes.py +419 -399
- dtlpy/repositories/resource_executions.py +137 -137
- dtlpy/repositories/schema.py +120 -120
- dtlpy/repositories/service_drivers.py +213 -213
- dtlpy/repositories/services.py +1704 -1704
- dtlpy/repositories/settings.py +339 -339
- dtlpy/repositories/tasks.py +1477 -1477
- dtlpy/repositories/times_series.py +278 -278
- dtlpy/repositories/triggers.py +536 -536
- dtlpy/repositories/upload_element.py +257 -257
- dtlpy/repositories/uploader.py +661 -651
- dtlpy/repositories/webhooks.py +249 -249
- dtlpy/services/__init__.py +22 -22
- dtlpy/services/aihttp_retry.py +131 -131
- dtlpy/services/api_client.py +1785 -1782
- dtlpy/services/api_reference.py +40 -40
- dtlpy/services/async_utils.py +133 -133
- dtlpy/services/calls_counter.py +44 -44
- dtlpy/services/check_sdk.py +68 -68
- dtlpy/services/cookie.py +115 -115
- dtlpy/services/create_logger.py +156 -156
- dtlpy/services/events.py +84 -84
- dtlpy/services/logins.py +235 -235
- dtlpy/services/reporter.py +256 -256
- dtlpy/services/service_defaults.py +91 -91
- dtlpy/utilities/__init__.py +20 -20
- dtlpy/utilities/annotations/__init__.py +16 -16
- dtlpy/utilities/annotations/annotation_converters.py +269 -269
- dtlpy/utilities/base_package_runner.py +285 -264
- dtlpy/utilities/converter.py +1650 -1650
- dtlpy/utilities/dataset_generators/__init__.py +1 -1
- dtlpy/utilities/dataset_generators/dataset_generator.py +670 -670
- dtlpy/utilities/dataset_generators/dataset_generator_tensorflow.py +23 -23
- dtlpy/utilities/dataset_generators/dataset_generator_torch.py +21 -21
- dtlpy/utilities/local_development/__init__.py +1 -1
- dtlpy/utilities/local_development/local_session.py +179 -179
- dtlpy/utilities/reports/__init__.py +2 -2
- dtlpy/utilities/reports/figures.py +343 -343
- dtlpy/utilities/reports/report.py +71 -71
- dtlpy/utilities/videos/__init__.py +17 -17
- dtlpy/utilities/videos/video_player.py +598 -598
- dtlpy/utilities/videos/videos.py +470 -470
- {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp +1 -1
- dtlpy-1.116.6.data/scripts/dlp.bat +2 -0
- {dtlpy-1.114.17.data → dtlpy-1.116.6.data}/scripts/dlp.py +128 -128
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/METADATA +186 -183
- dtlpy-1.116.6.dist-info/RECORD +239 -0
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/WHEEL +1 -1
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/licenses/LICENSE +200 -200
- tests/features/environment.py +551 -551
- dtlpy/assets/__pycache__/__init__.cpython-310.pyc +0 -0
- dtlpy-1.114.17.data/scripts/dlp.bat +0 -2
- dtlpy-1.114.17.dist-info/RECORD +0 -240
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/entry_points.txt +0 -0
- {dtlpy-1.114.17.dist-info → dtlpy-1.116.6.dist-info}/top_level.txt +0 -0
dtlpy/repositories/services.py
CHANGED
|
@@ -1,1704 +1,1704 @@
|
|
|
1
|
-
import datetime
|
|
2
|
-
import inspect
|
|
3
|
-
import logging
|
|
4
|
-
import json
|
|
5
|
-
import os
|
|
6
|
-
import tempfile
|
|
7
|
-
import time
|
|
8
|
-
from typing import Union, List, Callable
|
|
9
|
-
from .. import miscellaneous, exceptions, entities, repositories, assets, ApiClient, _api_reference
|
|
10
|
-
from ..__version__ import version as __version__
|
|
11
|
-
|
|
12
|
-
logger = logging.getLogger(name='dtlpy')
|
|
13
|
-
FUNCTION_END_LINE = '[Done] Executing function.'
|
|
14
|
-
MAX_WAIT_TIME = 8
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class Services:
|
|
18
|
-
"""
|
|
19
|
-
Services Repository
|
|
20
|
-
|
|
21
|
-
The Services class allows the user to manage services and their properties. Services are created from the packages users create.
|
|
22
|
-
See our documentation for more information about `services <https://developers.dataloop.ai/tutorials/faas/advance/chapter/>`_.
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
def __init__(self,
|
|
26
|
-
client_api: ApiClient,
|
|
27
|
-
project: entities.Project = None,
|
|
28
|
-
package: Union[entities.Package, entities.Dpk] = None,
|
|
29
|
-
project_id=None,
|
|
30
|
-
model_id=None,
|
|
31
|
-
model: entities.Model = None):
|
|
32
|
-
self._client_api = client_api
|
|
33
|
-
self._package = package
|
|
34
|
-
self._project = project
|
|
35
|
-
self._model = model
|
|
36
|
-
if model_id is None:
|
|
37
|
-
if model is not None:
|
|
38
|
-
model_id = model.id
|
|
39
|
-
self._model_id = model_id
|
|
40
|
-
if project_id is None:
|
|
41
|
-
if project is not None:
|
|
42
|
-
project_id = project.id
|
|
43
|
-
self._project_id = project_id
|
|
44
|
-
self._settings = repositories.Settings(project=project, client_api=client_api)
|
|
45
|
-
|
|
46
|
-
############
|
|
47
|
-
# entities #
|
|
48
|
-
############
|
|
49
|
-
@property
|
|
50
|
-
def package(self) -> entities.Package:
|
|
51
|
-
if self._package is None:
|
|
52
|
-
raise exceptions.PlatformException(
|
|
53
|
-
error='2001',
|
|
54
|
-
message='Cannot perform action WITHOUT package entity in services repository. Please set a package')
|
|
55
|
-
assert (isinstance(self._package, entities.Package) or isinstance(self._package, entities.Dpk))
|
|
56
|
-
return self._package
|
|
57
|
-
|
|
58
|
-
@package.setter
|
|
59
|
-
def package(self, package: Union[entities.Package, entities.Dpk]):
|
|
60
|
-
if not isinstance(package, entities.Package) and not isinstance(package, entities.Dpk):
|
|
61
|
-
raise ValueError('Must input a valid package entity')
|
|
62
|
-
self._package = package
|
|
63
|
-
|
|
64
|
-
@property
|
|
65
|
-
def project(self) -> entities.Project:
|
|
66
|
-
if self._project is None:
|
|
67
|
-
# try to get from package
|
|
68
|
-
if self._package is not None:
|
|
69
|
-
self._project = self._package._project
|
|
70
|
-
|
|
71
|
-
if self._project is None:
|
|
72
|
-
# try to get checked out project
|
|
73
|
-
project = self._client_api.state_io.get('project')
|
|
74
|
-
if project is not None:
|
|
75
|
-
self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
|
|
76
|
-
if self._project is None:
|
|
77
|
-
raise exceptions.PlatformException(
|
|
78
|
-
error='2001',
|
|
79
|
-
message='Cannot perform action WITHOUT Project entity in services repository. Please set a project')
|
|
80
|
-
return self._project
|
|
81
|
-
|
|
82
|
-
@project.setter
|
|
83
|
-
def project(self, project: entities.Project):
|
|
84
|
-
if not isinstance(project, entities.Project):
|
|
85
|
-
raise ValueError('Must input a valid Project entity')
|
|
86
|
-
self._project = project
|
|
87
|
-
self._project_id = project.id
|
|
88
|
-
|
|
89
|
-
@property
|
|
90
|
-
def model(self) -> entities.Model:
|
|
91
|
-
if self._model is None:
|
|
92
|
-
if self._model_id is not None:
|
|
93
|
-
self._model = self.project.models.get(model_id=self._model_id)
|
|
94
|
-
else:
|
|
95
|
-
raise exceptions.PlatformException(
|
|
96
|
-
error='2001',
|
|
97
|
-
message='Cannot perform action WITHOUT model entity in services repository. Please set a model')
|
|
98
|
-
assert isinstance(self._model, entities.Model)
|
|
99
|
-
return self._model
|
|
100
|
-
|
|
101
|
-
@model.setter
|
|
102
|
-
def model(self, model: entities.Model):
|
|
103
|
-
if not isinstance(model, entities.Model):
|
|
104
|
-
raise ValueError('Must input a valid model entity')
|
|
105
|
-
self._model = model
|
|
106
|
-
self._model_id = model.id
|
|
107
|
-
|
|
108
|
-
@property
|
|
109
|
-
def platform_url(self):
|
|
110
|
-
return self._client_api._get_resource_url("projects/{}/services".format(self.project.id))
|
|
111
|
-
|
|
112
|
-
def open_in_web(self,
|
|
113
|
-
service: entities.Service = None,
|
|
114
|
-
service_id: str = None,
|
|
115
|
-
service_name: str = None):
|
|
116
|
-
"""
|
|
117
|
-
Open the service in web platform
|
|
118
|
-
|
|
119
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
120
|
-
|
|
121
|
-
:param str service_name: service name
|
|
122
|
-
:param str service_id: service id
|
|
123
|
-
:param dtlpy.entities.service.Service service: service entity
|
|
124
|
-
|
|
125
|
-
**Example**:
|
|
126
|
-
|
|
127
|
-
.. code-block:: python
|
|
128
|
-
|
|
129
|
-
package.services.open_in_web(service_id='service_id')
|
|
130
|
-
"""
|
|
131
|
-
if service_name is not None:
|
|
132
|
-
service = self.get(service_name=service_name)
|
|
133
|
-
if service is not None:
|
|
134
|
-
service.open_in_web()
|
|
135
|
-
elif service_id is not None:
|
|
136
|
-
self._client_api._open_in_web(url=self.platform_url + '/' + str(service_id) + '/main')
|
|
137
|
-
else:
|
|
138
|
-
self._client_api._open_in_web(url=self.platform_url)
|
|
139
|
-
|
|
140
|
-
def __get_from_cache(self) -> entities.Service:
|
|
141
|
-
service = self._client_api.state_io.get('service')
|
|
142
|
-
if service is not None:
|
|
143
|
-
service = entities.Service.from_json(_json=service,
|
|
144
|
-
client_api=self._client_api,
|
|
145
|
-
project=self._project,
|
|
146
|
-
package=self._package)
|
|
147
|
-
return service
|
|
148
|
-
|
|
149
|
-
def checkout(self,
|
|
150
|
-
service: entities.Service = None,
|
|
151
|
-
service_name: str = None,
|
|
152
|
-
service_id: str = None):
|
|
153
|
-
"""
|
|
154
|
-
Checkout (switch) to a service.
|
|
155
|
-
|
|
156
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
157
|
-
|
|
158
|
-
:param dtlpy.entities.service.Service service: Service entity
|
|
159
|
-
:param str service_name: service name
|
|
160
|
-
:param str service_id: service id
|
|
161
|
-
|
|
162
|
-
**Example**:
|
|
163
|
-
|
|
164
|
-
.. code-block:: python
|
|
165
|
-
|
|
166
|
-
package.services.checkout(service_id='service_id')
|
|
167
|
-
"""
|
|
168
|
-
if service is None:
|
|
169
|
-
service = self.get(service_name=service_name, service_id=service_id)
|
|
170
|
-
self._client_api.state_io.put('service', service.to_json())
|
|
171
|
-
logger.info('Checked out to service {}'.format(service.name))
|
|
172
|
-
|
|
173
|
-
###########
|
|
174
|
-
# methods #
|
|
175
|
-
###########
|
|
176
|
-
@_api_reference.add(path='/services/{id}/revisions', method='get')
|
|
177
|
-
def revisions(self,
|
|
178
|
-
service: entities.Service = None,
|
|
179
|
-
service_id: str = None):
|
|
180
|
-
"""
|
|
181
|
-
Get service revisions history.
|
|
182
|
-
|
|
183
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
184
|
-
|
|
185
|
-
You must provide at leats ONE of the following params: service, service_id
|
|
186
|
-
|
|
187
|
-
:param dtlpy.entities.service.Service service: Service entity
|
|
188
|
-
:param str service_id: service id
|
|
189
|
-
|
|
190
|
-
**Example**:
|
|
191
|
-
|
|
192
|
-
.. code-block:: python
|
|
193
|
-
|
|
194
|
-
service_revision = package.services.revisions(service_id='service_id')
|
|
195
|
-
"""
|
|
196
|
-
if service is None and service_id is None:
|
|
197
|
-
raise exceptions.PlatformException(
|
|
198
|
-
error='400',
|
|
199
|
-
message='must provide an identifier in inputs: "service" or "service_id"')
|
|
200
|
-
if service is not None:
|
|
201
|
-
service_id = service.id
|
|
202
|
-
|
|
203
|
-
success, response = self._client_api.gen_request(
|
|
204
|
-
req_type="get",
|
|
205
|
-
path="/services/{}/revisions".format(service_id))
|
|
206
|
-
if not success:
|
|
207
|
-
raise exceptions.PlatformException(response)
|
|
208
|
-
return response.json()
|
|
209
|
-
|
|
210
|
-
@_api_reference.add(path='/services/{id}', method='get')
|
|
211
|
-
def get(self,
|
|
212
|
-
service_name=None,
|
|
213
|
-
service_id=None,
|
|
214
|
-
checkout=False,
|
|
215
|
-
fetch=None
|
|
216
|
-
) -> entities.Service:
|
|
217
|
-
"""
|
|
218
|
-
Get service to use in your code.
|
|
219
|
-
|
|
220
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
221
|
-
|
|
222
|
-
:param str service_name: optional - search by name
|
|
223
|
-
:param str service_id: optional - search by id
|
|
224
|
-
:param bool checkout: if true, checkout (switch) to service
|
|
225
|
-
:param fetch: optional - fetch entity from platform, default taken from cookie
|
|
226
|
-
:return: Service object
|
|
227
|
-
:rtype: dtlpy.entities.service.Service
|
|
228
|
-
|
|
229
|
-
**Example**:
|
|
230
|
-
|
|
231
|
-
.. code-block:: python
|
|
232
|
-
|
|
233
|
-
service = package.services.get(service_id='service_id')
|
|
234
|
-
"""
|
|
235
|
-
if fetch is None:
|
|
236
|
-
fetch = self._client_api.fetch_entities
|
|
237
|
-
|
|
238
|
-
if service_name is None and service_id is None:
|
|
239
|
-
service = self.__get_from_cache()
|
|
240
|
-
if service is None:
|
|
241
|
-
raise exceptions.PlatformException(
|
|
242
|
-
error='400',
|
|
243
|
-
message='No checked-out Service was found, must checkout or provide an identifier in inputs')
|
|
244
|
-
|
|
245
|
-
elif fetch:
|
|
246
|
-
if service_id is not None:
|
|
247
|
-
success, response = self._client_api.gen_request(
|
|
248
|
-
req_type="get",
|
|
249
|
-
path="/services/{}".format(service_id)
|
|
250
|
-
)
|
|
251
|
-
if not success:
|
|
252
|
-
raise exceptions.PlatformException(response)
|
|
253
|
-
service = entities.Service.from_json(client_api=self._client_api,
|
|
254
|
-
_json=response.json(),
|
|
255
|
-
package=self._package,
|
|
256
|
-
project=self._project)
|
|
257
|
-
# verify input service name is same as the given id
|
|
258
|
-
if service_name is not None and service.name != service_name:
|
|
259
|
-
logger.warning(
|
|
260
|
-
"Mismatch found in services.get: service_name is different then service.name:"
|
|
261
|
-
" {!r} != {!r}".format(
|
|
262
|
-
service_name,
|
|
263
|
-
service.name))
|
|
264
|
-
elif service_name is not None:
|
|
265
|
-
filters = entities.Filters(resource=entities.FiltersResource.SERVICE,
|
|
266
|
-
field='name',
|
|
267
|
-
values=service_name,
|
|
268
|
-
use_defaults=False)
|
|
269
|
-
if self._project_id is not None:
|
|
270
|
-
filters.add(field='projectId', values=self._project_id)
|
|
271
|
-
if self._package is not None:
|
|
272
|
-
filters.add(field='packageId', values=self._package.id)
|
|
273
|
-
services = self.list(filters=filters)
|
|
274
|
-
if services.items_count > 1:
|
|
275
|
-
raise exceptions.PlatformException('404', 'More than one service with same name. '
|
|
276
|
-
'Please get services from package/project entity')
|
|
277
|
-
elif services.items_count == 0:
|
|
278
|
-
raise exceptions.PlatformException('404', 'Service not found: {}.'.format(service_name))
|
|
279
|
-
service = services.items[0]
|
|
280
|
-
else:
|
|
281
|
-
raise exceptions.PlatformException(
|
|
282
|
-
error='400',
|
|
283
|
-
message='No checked-out Service was found, must checkout or provide an identifier in inputs')
|
|
284
|
-
else:
|
|
285
|
-
service = entities.Service.from_json(_json={'id': service_id,
|
|
286
|
-
'name': service_name},
|
|
287
|
-
client_api=self._client_api,
|
|
288
|
-
project=self._project,
|
|
289
|
-
package=self._package,
|
|
290
|
-
is_fetched=False)
|
|
291
|
-
|
|
292
|
-
assert isinstance(service, entities.Service)
|
|
293
|
-
if checkout:
|
|
294
|
-
self.checkout(service=service)
|
|
295
|
-
return service
|
|
296
|
-
|
|
297
|
-
def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Service]:
|
|
298
|
-
jobs = [None for _ in range(len(response_items))]
|
|
299
|
-
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
300
|
-
|
|
301
|
-
# return triggers list
|
|
302
|
-
for i_service, service in enumerate(response_items):
|
|
303
|
-
jobs[i_service] = pool.submit(entities.Service._protected_from_json,
|
|
304
|
-
**{'client_api': self._client_api,
|
|
305
|
-
'_json': service,
|
|
306
|
-
'package': self._package,
|
|
307
|
-
'project': self._project})
|
|
308
|
-
|
|
309
|
-
# get all results
|
|
310
|
-
results = [j.result() for j in jobs]
|
|
311
|
-
# log errors
|
|
312
|
-
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
313
|
-
# return good jobs
|
|
314
|
-
return miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
315
|
-
|
|
316
|
-
def _list(self, filters: entities.Filters):
|
|
317
|
-
url = '/query/faas'
|
|
318
|
-
success, response = self._client_api.gen_request(req_type='POST',
|
|
319
|
-
path=url,
|
|
320
|
-
json_req=filters.prepare())
|
|
321
|
-
if not success:
|
|
322
|
-
raise exceptions.PlatformException(response)
|
|
323
|
-
|
|
324
|
-
return response.json()
|
|
325
|
-
|
|
326
|
-
@_api_reference.add(path='/query/faas', method='post')
|
|
327
|
-
def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
|
|
328
|
-
"""
|
|
329
|
-
List all services (services can be listed for a package or for a project).
|
|
330
|
-
|
|
331
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
332
|
-
|
|
333
|
-
:param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
|
|
334
|
-
:return: Paged entity
|
|
335
|
-
:rtype: dtlpy.entities.paged_entities.PagedEntities
|
|
336
|
-
|
|
337
|
-
**Example**:
|
|
338
|
-
|
|
339
|
-
.. code-block:: python
|
|
340
|
-
|
|
341
|
-
services = package.services.list()
|
|
342
|
-
"""
|
|
343
|
-
# default filters
|
|
344
|
-
if filters is None:
|
|
345
|
-
filters = entities.Filters(resource=entities.FiltersResource.SERVICE)
|
|
346
|
-
# assert type filters
|
|
347
|
-
elif not isinstance(filters, entities.Filters):
|
|
348
|
-
raise exceptions.PlatformException(error='400',
|
|
349
|
-
message='Unknown filters type: {!r}'.format(type(filters)))
|
|
350
|
-
if filters.resource != entities.FiltersResource.SERVICE:
|
|
351
|
-
raise exceptions.PlatformException(
|
|
352
|
-
error='400',
|
|
353
|
-
message='Filters resource must to be FiltersResource.SERVICE. Got: {!r}'.format(filters.resource))
|
|
354
|
-
if self._project_id is not None:
|
|
355
|
-
filters.add(field='projectId', values=self._project_id)
|
|
356
|
-
if self._model_id is not None:
|
|
357
|
-
filters.add(field='metadata.ml.modelId', values=self._model_id)
|
|
358
|
-
if self._package is not None:
|
|
359
|
-
filters.add(field='packageId', values=self._package.id)
|
|
360
|
-
|
|
361
|
-
# assert type filters
|
|
362
|
-
if not isinstance(filters, entities.Filters):
|
|
363
|
-
raise exceptions.PlatformException('400', 'Unknown filters type')
|
|
364
|
-
|
|
365
|
-
paged = entities.PagedEntities(items_repository=self,
|
|
366
|
-
filters=filters,
|
|
367
|
-
page_offset=filters.page,
|
|
368
|
-
page_size=filters.page_size,
|
|
369
|
-
client_api=self._client_api)
|
|
370
|
-
paged.get_page()
|
|
371
|
-
return paged
|
|
372
|
-
|
|
373
|
-
@_api_reference.add(path='/services/{id}/status', method='post')
|
|
374
|
-
def status(self, service_name=None, service_id=None):
|
|
375
|
-
"""
|
|
376
|
-
Get service status.
|
|
377
|
-
|
|
378
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
379
|
-
|
|
380
|
-
You must provide at least ONE of the following params: service_id, service_name
|
|
381
|
-
|
|
382
|
-
:param str service_name: service name
|
|
383
|
-
:param str service_id: service id
|
|
384
|
-
:return: status json
|
|
385
|
-
:rtype: dict
|
|
386
|
-
|
|
387
|
-
**Example**:
|
|
388
|
-
|
|
389
|
-
.. code-block:: python
|
|
390
|
-
|
|
391
|
-
status_json = package.services.status(service_id='service_id')
|
|
392
|
-
"""
|
|
393
|
-
if service_id is None:
|
|
394
|
-
if service_name is None:
|
|
395
|
-
raise exceptions.PlatformException(error='400',
|
|
396
|
-
message='must input "service_name" or "service_id" to get status')
|
|
397
|
-
service = self.get(service_name=service_name)
|
|
398
|
-
service_id = service.id
|
|
399
|
-
# request
|
|
400
|
-
success, response = self._client_api.gen_request(req_type="get",
|
|
401
|
-
path="/services/{}/status".format(service_id))
|
|
402
|
-
if not success:
|
|
403
|
-
raise exceptions.PlatformException(response)
|
|
404
|
-
return response.json()
|
|
405
|
-
|
|
406
|
-
@_api_reference.add(path='/services/{id}/stop', method='post')
|
|
407
|
-
def pause(self,
|
|
408
|
-
service_name: str = None,
|
|
409
|
-
service_id: str = None,
|
|
410
|
-
force: bool = False):
|
|
411
|
-
"""
|
|
412
|
-
Pause service.
|
|
413
|
-
|
|
414
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
415
|
-
|
|
416
|
-
You must provide at least ONE of the following params: service_id, service_name
|
|
417
|
-
|
|
418
|
-
:param str service_name: service name
|
|
419
|
-
:param str service_id: service id
|
|
420
|
-
:param bool force: optional - terminate old replicas immediately
|
|
421
|
-
:return: True if success
|
|
422
|
-
:rtype: bool
|
|
423
|
-
|
|
424
|
-
**Example**:
|
|
425
|
-
|
|
426
|
-
.. code-block:: python
|
|
427
|
-
|
|
428
|
-
success = package.services.pause(service_id='service_id')
|
|
429
|
-
"""
|
|
430
|
-
if service_id is None:
|
|
431
|
-
if service_name is None:
|
|
432
|
-
raise exceptions.PlatformException(error='400',
|
|
433
|
-
message='must input "service_name" or "service_id" to pause service')
|
|
434
|
-
service = self.get(service_name=service_name)
|
|
435
|
-
service_id = service.id
|
|
436
|
-
# request
|
|
437
|
-
url = "/services/{}/stop".format(service_id)
|
|
438
|
-
if force:
|
|
439
|
-
url = '{}?force=true'
|
|
440
|
-
success, response = self._client_api.gen_request(req_type="post",
|
|
441
|
-
path=url)
|
|
442
|
-
if not success:
|
|
443
|
-
raise exceptions.PlatformException(response)
|
|
444
|
-
return success
|
|
445
|
-
|
|
446
|
-
def _notify(
|
|
447
|
-
self,
|
|
448
|
-
service_id: str,
|
|
449
|
-
message: str,
|
|
450
|
-
name: str,
|
|
451
|
-
action: str = 'created',
|
|
452
|
-
support: str = None,
|
|
453
|
-
docs: str = None,
|
|
454
|
-
agent_info: dict = None
|
|
455
|
-
):
|
|
456
|
-
url = "/services/{}/notify".format(service_id)
|
|
457
|
-
payload = {
|
|
458
|
-
'action': action,
|
|
459
|
-
'message': message,
|
|
460
|
-
'notificationName': name
|
|
461
|
-
}
|
|
462
|
-
if agent_info is not None:
|
|
463
|
-
payload['agentInfo'] = agent_info
|
|
464
|
-
|
|
465
|
-
if support:
|
|
466
|
-
payload['support'] = support
|
|
467
|
-
|
|
468
|
-
if docs:
|
|
469
|
-
payload['docs'] = docs
|
|
470
|
-
|
|
471
|
-
success, response = self._client_api.gen_request(
|
|
472
|
-
req_type="post",
|
|
473
|
-
path=url,
|
|
474
|
-
json_req=payload
|
|
475
|
-
)
|
|
476
|
-
if not success:
|
|
477
|
-
raise exceptions.PlatformException(response)
|
|
478
|
-
return success
|
|
479
|
-
|
|
480
|
-
@_api_reference.add(path='/services/{id}/resume', method='post')
|
|
481
|
-
def resume(self,
|
|
482
|
-
service_name: str = None,
|
|
483
|
-
service_id: str = None,
|
|
484
|
-
force: bool = False):
|
|
485
|
-
"""
|
|
486
|
-
Resume service.
|
|
487
|
-
|
|
488
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
489
|
-
|
|
490
|
-
You must provide at least ONE of the following params: service_id, service_name.
|
|
491
|
-
|
|
492
|
-
:param str service_name: service name
|
|
493
|
-
:param str service_id: service id
|
|
494
|
-
:param bool force: optional - terminate old replicas immediately
|
|
495
|
-
:return: json of the service
|
|
496
|
-
:rtype: dict
|
|
497
|
-
|
|
498
|
-
**Example**:
|
|
499
|
-
|
|
500
|
-
.. code-block:: python
|
|
501
|
-
|
|
502
|
-
service_json = package.services.resume(service_id='service_id')
|
|
503
|
-
"""
|
|
504
|
-
if service_id is None:
|
|
505
|
-
if service_name is None:
|
|
506
|
-
raise exceptions.PlatformException(error='400',
|
|
507
|
-
message='must input "service_name" or "service_id" to resume')
|
|
508
|
-
service = self.get(service_name=service_name)
|
|
509
|
-
service_id = service.id
|
|
510
|
-
# request
|
|
511
|
-
url = "/services/{}/resume".format(service_id)
|
|
512
|
-
if force:
|
|
513
|
-
url = '{}?force=true'
|
|
514
|
-
success, response = self._client_api.gen_request(req_type="post",
|
|
515
|
-
path=url)
|
|
516
|
-
if not success:
|
|
517
|
-
raise exceptions.PlatformException(response)
|
|
518
|
-
return response.json()
|
|
519
|
-
|
|
520
|
-
def _get_bot_email(self, bot=None):
|
|
521
|
-
|
|
522
|
-
if bot is None:
|
|
523
|
-
project = self._project
|
|
524
|
-
if project is None and self._project_id is not None:
|
|
525
|
-
project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
|
|
526
|
-
|
|
527
|
-
if project is None:
|
|
528
|
-
raise exceptions.PlatformException(error='2001',
|
|
529
|
-
message='Need project entity or bot to perform this action')
|
|
530
|
-
bots = project.bots.list()
|
|
531
|
-
if len(bots) == 0:
|
|
532
|
-
logger.info('Bot not found for project. Creating a default bot')
|
|
533
|
-
bot = project.bots.create(name='default')
|
|
534
|
-
else:
|
|
535
|
-
bot = bots[0]
|
|
536
|
-
if len(bots) > 1:
|
|
537
|
-
logger.debug('More than one bot users. Choosing first. email: {}'.format(bots[0].email))
|
|
538
|
-
|
|
539
|
-
if isinstance(bot, str):
|
|
540
|
-
bot_email = bot
|
|
541
|
-
elif isinstance(bot, entities.Bot) or isinstance(bot, entities.User):
|
|
542
|
-
bot_email = bot.email
|
|
543
|
-
else:
|
|
544
|
-
raise ValueError('input "bot" must be a str or a Bot type, got: {}'.format(type(bot)))
|
|
545
|
-
|
|
546
|
-
return bot_email
|
|
547
|
-
|
|
548
|
-
@staticmethod
|
|
549
|
-
def _parse_init_input(init_input):
|
|
550
|
-
if not isinstance(init_input, dict):
|
|
551
|
-
if init_input is None:
|
|
552
|
-
init_input = dict()
|
|
553
|
-
else:
|
|
554
|
-
init_params = dict()
|
|
555
|
-
if not isinstance(init_input, list):
|
|
556
|
-
init_input = [init_input]
|
|
557
|
-
for param in init_input:
|
|
558
|
-
if isinstance(param, entities.FunctionIO):
|
|
559
|
-
init_params.update(param.to_json(resource='service'))
|
|
560
|
-
else:
|
|
561
|
-
raise exceptions.PlatformException('400', 'Unknown type of init params')
|
|
562
|
-
init_input = init_params
|
|
563
|
-
|
|
564
|
-
return init_input
|
|
565
|
-
|
|
566
|
-
def name_validation(self, name: str):
|
|
567
|
-
"""
|
|
568
|
-
Validation service name.
|
|
569
|
-
|
|
570
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
571
|
-
|
|
572
|
-
:param str name: service name
|
|
573
|
-
|
|
574
|
-
**Example**:
|
|
575
|
-
|
|
576
|
-
.. code-block:: python
|
|
577
|
-
|
|
578
|
-
package.services.name_validation(name='name')
|
|
579
|
-
"""
|
|
580
|
-
url = '/piper-misc/naming/services/{}'.format(name)
|
|
581
|
-
|
|
582
|
-
# request
|
|
583
|
-
success, response = self._client_api.gen_request(req_type='get',
|
|
584
|
-
path=url)
|
|
585
|
-
if not success:
|
|
586
|
-
raise exceptions.PlatformException(response)
|
|
587
|
-
|
|
588
|
-
def _create(self,
|
|
589
|
-
service_name: str = None,
|
|
590
|
-
package: entities.Package = None,
|
|
591
|
-
module_name: str = None,
|
|
592
|
-
bot: Union[entities.Bot, str] = None,
|
|
593
|
-
revision: str or int = None,
|
|
594
|
-
init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
|
|
595
|
-
runtime: Union[entities.KubernetesRuntime, dict] = None,
|
|
596
|
-
pod_type: entities.InstanceCatalog = None,
|
|
597
|
-
project_id: str = None,
|
|
598
|
-
sdk_version: str = None,
|
|
599
|
-
agent_versions: dict = None,
|
|
600
|
-
verify: bool = True,
|
|
601
|
-
driver_id: str = None,
|
|
602
|
-
run_execution_as_process: bool = None,
|
|
603
|
-
execution_timeout: int = None,
|
|
604
|
-
drain_time: int = None,
|
|
605
|
-
on_reset: str = None,
|
|
606
|
-
max_attempts: int = None,
|
|
607
|
-
secrets=None,
|
|
608
|
-
integrations=None,
|
|
609
|
-
active: bool = True,
|
|
610
|
-
**kwargs
|
|
611
|
-
) -> entities.Service:
|
|
612
|
-
"""
|
|
613
|
-
Create service entity.
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
:param str service_name: service name
|
|
617
|
-
:param dtlpy.entities.package.Package package: package entity
|
|
618
|
-
:param str module_name: module name
|
|
619
|
-
:param str bot: bot email
|
|
620
|
-
:param str revision: package revision - default=latest
|
|
621
|
-
:param init_input: config to run at startup
|
|
622
|
-
:param dict runtime: runtime resources
|
|
623
|
-
:param str pod_type: pod type dl.InstanceCatalog
|
|
624
|
-
:param str project_id: project id
|
|
625
|
-
:param str sdk_version: - optional - string - sdk version
|
|
626
|
-
:param dict agent_versions: - dictionary - - optional -versions of sdk, agent runner and agent proxy
|
|
627
|
-
:param bool verify: verify the inputs
|
|
628
|
-
:param str driver_id: driver id
|
|
629
|
-
:param bool run_execution_as_process: run execution as process
|
|
630
|
-
:param int execution_timeout: execution timeout
|
|
631
|
-
:param int drain_time: drain time
|
|
632
|
-
:param str on_reset: on reset
|
|
633
|
-
:param int max_attempts: Maximum execution retries in-case of a service reset
|
|
634
|
-
:param bool force: optional - terminate old replicas immediately
|
|
635
|
-
:param list secrets: list of the integrations ids
|
|
636
|
-
:param list integrations: list of the integrations
|
|
637
|
-
:param bool active: optional - if true, service will be active
|
|
638
|
-
:param kwargs:
|
|
639
|
-
:return: Service object
|
|
640
|
-
:rtype: dtlpy.entities.service.Service
|
|
641
|
-
"""
|
|
642
|
-
if package is None:
|
|
643
|
-
if self._package is None:
|
|
644
|
-
raise exceptions.PlatformException('400', 'Please provide param package')
|
|
645
|
-
package = self._package
|
|
646
|
-
|
|
647
|
-
if verify is not None:
|
|
648
|
-
logger.warning('verify attribute has been deprecated and will be ignored')
|
|
649
|
-
|
|
650
|
-
is_global = kwargs.get('is_global', None)
|
|
651
|
-
jwt_forward = kwargs.get('jwt_forward', None)
|
|
652
|
-
|
|
653
|
-
if is_global is not None or jwt_forward is not None:
|
|
654
|
-
logger.warning(
|
|
655
|
-
'Params jwt_forward and is_global are restricted to superuser. '
|
|
656
|
-
'If you are not a superuser this action will not work'
|
|
657
|
-
)
|
|
658
|
-
|
|
659
|
-
service_config = dict()
|
|
660
|
-
if package is not None and package.service_config is not None:
|
|
661
|
-
service_config = package.service_config
|
|
662
|
-
|
|
663
|
-
if agent_versions is None:
|
|
664
|
-
if sdk_version is None:
|
|
665
|
-
sdk_version = service_config.get('versions', dict()).get('dtlpy', __version__)
|
|
666
|
-
agent_versions = {
|
|
667
|
-
"dtlpy": sdk_version
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
if project_id is None:
|
|
671
|
-
if self._project is None and self._project_id is None:
|
|
672
|
-
raise exceptions.PlatformException('400', 'Please provide project id')
|
|
673
|
-
elif self._project_id is not None:
|
|
674
|
-
project_id = self._project_id
|
|
675
|
-
elif self._project is not None:
|
|
676
|
-
project_id = self._project.id
|
|
677
|
-
|
|
678
|
-
if service_name is None:
|
|
679
|
-
service_name = 'default-service'
|
|
680
|
-
|
|
681
|
-
# payload
|
|
682
|
-
payload = {
|
|
683
|
-
'name': service_name,
|
|
684
|
-
'projectId': project_id,
|
|
685
|
-
'packageId': package.id,
|
|
686
|
-
'initParams': self._parse_init_input(init_input=init_input),
|
|
687
|
-
'botUserName': self._get_bot_email(bot=bot),
|
|
688
|
-
'versions': agent_versions,
|
|
689
|
-
'packageRevision': revision if revision is not None else package.version,
|
|
690
|
-
'driverId': driver_id,
|
|
691
|
-
'active': active
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
if secrets is not None:
|
|
695
|
-
if not isinstance(secrets, list):
|
|
696
|
-
secrets = [secrets]
|
|
697
|
-
payload['secrets'] = secrets
|
|
698
|
-
|
|
699
|
-
if integrations is not None:
|
|
700
|
-
if not isinstance(integrations, list):
|
|
701
|
-
integrations = [integrations]
|
|
702
|
-
payload['integrations'] = integrations
|
|
703
|
-
|
|
704
|
-
if runtime is not None:
|
|
705
|
-
if isinstance(runtime, entities.KubernetesRuntime):
|
|
706
|
-
runtime = runtime.to_json()
|
|
707
|
-
|
|
708
|
-
if pod_type is not None:
|
|
709
|
-
if runtime is None:
|
|
710
|
-
runtime = {'podType': pod_type}
|
|
711
|
-
else:
|
|
712
|
-
runtime['podType'] = pod_type
|
|
713
|
-
|
|
714
|
-
if runtime is not None:
|
|
715
|
-
payload['runtime'] = runtime
|
|
716
|
-
|
|
717
|
-
if module_name is not None:
|
|
718
|
-
payload['moduleName'] = module_name
|
|
719
|
-
|
|
720
|
-
if is_global is not None:
|
|
721
|
-
payload['global'] = is_global
|
|
722
|
-
|
|
723
|
-
if max_attempts is not None:
|
|
724
|
-
payload['maxAttempts'] = max_attempts
|
|
725
|
-
|
|
726
|
-
if jwt_forward is not None:
|
|
727
|
-
payload['useUserJwt'] = jwt_forward
|
|
728
|
-
|
|
729
|
-
if run_execution_as_process is not None:
|
|
730
|
-
payload['runExecutionAsProcess'] = run_execution_as_process
|
|
731
|
-
|
|
732
|
-
if drain_time is not None:
|
|
733
|
-
payload['drainTime'] = drain_time
|
|
734
|
-
|
|
735
|
-
if on_reset is not None:
|
|
736
|
-
payload['onReset'] = on_reset
|
|
737
|
-
|
|
738
|
-
if execution_timeout is not None:
|
|
739
|
-
payload['executionTimeout'] = execution_timeout
|
|
740
|
-
|
|
741
|
-
# request
|
|
742
|
-
success, response = self._client_api.gen_request(
|
|
743
|
-
req_type='post',
|
|
744
|
-
path='/services',
|
|
745
|
-
json_req=payload
|
|
746
|
-
)
|
|
747
|
-
|
|
748
|
-
# exception handling
|
|
749
|
-
if not success:
|
|
750
|
-
raise exceptions.PlatformException(response)
|
|
751
|
-
|
|
752
|
-
# return entity
|
|
753
|
-
return entities.Service.from_json(
|
|
754
|
-
_json=response.json(),
|
|
755
|
-
client_api=self._client_api,
|
|
756
|
-
package=package,
|
|
757
|
-
project=self._project
|
|
758
|
-
)
|
|
759
|
-
|
|
760
|
-
@_api_reference.add(path='/services/{id}', method='delete')
|
|
761
|
-
def delete(self, service_name: str = None, service_id: str = None, force=False):
|
|
762
|
-
"""
|
|
763
|
-
Delete Service object
|
|
764
|
-
|
|
765
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
766
|
-
|
|
767
|
-
You must provide at least ONE of the following params: service_id, service_name.
|
|
768
|
-
|
|
769
|
-
:param force:
|
|
770
|
-
:param str service_name: by name
|
|
771
|
-
:param str service_id: by id
|
|
772
|
-
:return: True
|
|
773
|
-
:rtype: bool
|
|
774
|
-
|
|
775
|
-
**Example**:
|
|
776
|
-
|
|
777
|
-
.. code-block:: python
|
|
778
|
-
|
|
779
|
-
is_deleted = package.services.delete(service_id='service_id')
|
|
780
|
-
"""
|
|
781
|
-
# get bby name
|
|
782
|
-
if service_id is None:
|
|
783
|
-
if service_name is None:
|
|
784
|
-
raise exceptions.PlatformException('400', 'Must provide either service id or service name')
|
|
785
|
-
else:
|
|
786
|
-
service_id = self.get(service_name=service_name).id
|
|
787
|
-
|
|
788
|
-
path = "/services/{}".format(service_id)
|
|
789
|
-
if force:
|
|
790
|
-
path = '{}?force=true'.format(path)
|
|
791
|
-
|
|
792
|
-
# request
|
|
793
|
-
success, response = self._client_api.gen_request(
|
|
794
|
-
req_type="delete",
|
|
795
|
-
path=path
|
|
796
|
-
)
|
|
797
|
-
if not success:
|
|
798
|
-
raise exceptions.PlatformException(response)
|
|
799
|
-
return True
|
|
800
|
-
|
|
801
|
-
@_api_reference.add(path='/services/{id}', method='patch')
|
|
802
|
-
def update(self, service: entities.Service, force: bool = False) -> entities.Service:
|
|
803
|
-
"""
|
|
804
|
-
Update service changes to platform.
|
|
805
|
-
|
|
806
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
807
|
-
|
|
808
|
-
:param dtlpy.entities.service.Service service: Service entity
|
|
809
|
-
:param bool force: optional - terminate old replicas immediately
|
|
810
|
-
:return: Service entity
|
|
811
|
-
:rtype: dtlpy.entities.service.Service
|
|
812
|
-
|
|
813
|
-
**Example**:
|
|
814
|
-
|
|
815
|
-
.. code-block:: python
|
|
816
|
-
|
|
817
|
-
service = package.services.update(service='service_entity')
|
|
818
|
-
"""
|
|
819
|
-
assert isinstance(service, entities.Service)
|
|
820
|
-
|
|
821
|
-
# payload
|
|
822
|
-
payload = service.to_json()
|
|
823
|
-
|
|
824
|
-
# request
|
|
825
|
-
url = '/services/{}'.format(service.id)
|
|
826
|
-
if force:
|
|
827
|
-
url = '{}?force=true'.format(url)
|
|
828
|
-
success, response = self._client_api.gen_request(req_type='patch',
|
|
829
|
-
path=url,
|
|
830
|
-
json_req=payload)
|
|
831
|
-
|
|
832
|
-
# exception handling
|
|
833
|
-
if not success:
|
|
834
|
-
raise exceptions.PlatformException(response)
|
|
835
|
-
|
|
836
|
-
# return entity
|
|
837
|
-
if self._package is not None:
|
|
838
|
-
package = self._package
|
|
839
|
-
else:
|
|
840
|
-
package = service._package
|
|
841
|
-
|
|
842
|
-
return entities.Service.from_json(_json=response.json(),
|
|
843
|
-
client_api=self._client_api,
|
|
844
|
-
package=package,
|
|
845
|
-
project=self._project)
|
|
846
|
-
|
|
847
|
-
def activate_slots(
|
|
848
|
-
self,
|
|
849
|
-
service: entities.Service,
|
|
850
|
-
project_id: str = None,
|
|
851
|
-
task_id: str = None,
|
|
852
|
-
dataset_id: str = None,
|
|
853
|
-
org_id: str = None,
|
|
854
|
-
user_email: str = None,
|
|
855
|
-
slots: List[entities.PackageSlot] = None,
|
|
856
|
-
role=None,
|
|
857
|
-
prevent_override: bool = True,
|
|
858
|
-
visible: bool = True,
|
|
859
|
-
icon: str = 'fas fa-magic',
|
|
860
|
-
**kwargs
|
|
861
|
-
):
|
|
862
|
-
"""
|
|
863
|
-
Activate service slots (creates buttons in the UI that activate services).
|
|
864
|
-
|
|
865
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
866
|
-
|
|
867
|
-
:param dtlpy.entities.service.Service service: service entity
|
|
868
|
-
:param str project_id: project id
|
|
869
|
-
:param str task_id: task id
|
|
870
|
-
:param str dataset_id: dataset id
|
|
871
|
-
:param str org_id: org id
|
|
872
|
-
:param str user_email: user email
|
|
873
|
-
:param list slots: list of entities.PackageSlot
|
|
874
|
-
:param str role: user role MemberOrgRole.ADMIN, MemberOrgRole.owner, MemberOrgRole.MEMBER, MemberOrgRole.WORKER
|
|
875
|
-
:param bool prevent_override: True to prevent override
|
|
876
|
-
:param bool visible: visible
|
|
877
|
-
:param str icon: icon
|
|
878
|
-
:param kwargs: all additional arguments
|
|
879
|
-
:return: list of user setting for activated slots
|
|
880
|
-
:rtype: list
|
|
881
|
-
|
|
882
|
-
**Example**:
|
|
883
|
-
|
|
884
|
-
.. code-block:: python
|
|
885
|
-
|
|
886
|
-
setting = package.services.activate_slots(service='service_entity',
|
|
887
|
-
project_id='project_id',
|
|
888
|
-
slots=List[entities.PackageSlot],
|
|
889
|
-
icon='fas fa-magic')
|
|
890
|
-
"""
|
|
891
|
-
package = service.package
|
|
892
|
-
if not isinstance(package.slots, list) or len(package.slots) == 0:
|
|
893
|
-
raise exceptions.PlatformException('400', "Service's package has no slots")
|
|
894
|
-
|
|
895
|
-
if kwargs.get('is_global', False):
|
|
896
|
-
project_id = '*'
|
|
897
|
-
scope_ids = [project_id]
|
|
898
|
-
else:
|
|
899
|
-
scope_ids = [s_id for s_id in [project_id, task_id, org_id, dataset_id, user_email] if s_id is not None]
|
|
900
|
-
if len(scope_ids) == 0:
|
|
901
|
-
raise exceptions.PlatformException('400', "Must provide scope resource ID")
|
|
902
|
-
|
|
903
|
-
settings = list()
|
|
904
|
-
|
|
905
|
-
if role is None:
|
|
906
|
-
role = entities.Role.ALL
|
|
907
|
-
|
|
908
|
-
if not slots:
|
|
909
|
-
slots = [s.to_json() for s in service.package.slots]
|
|
910
|
-
elif isinstance(slots, list) and isinstance(slots[0], entities.PackageSlot):
|
|
911
|
-
slots = [s.to_json() for s in slots]
|
|
912
|
-
else:
|
|
913
|
-
raise exceptions.PlatformException('400', "Slots param must be a list of PackageSlot objects")
|
|
914
|
-
|
|
915
|
-
for scope_id in scope_ids:
|
|
916
|
-
|
|
917
|
-
if kwargs.get('is_global', False):
|
|
918
|
-
scope_type = entities.PlatformEntityType.DATALOOP
|
|
919
|
-
elif scope_id == project_id:
|
|
920
|
-
scope_type = entities.PlatformEntityType.PROJECT
|
|
921
|
-
elif scope_id == task_id:
|
|
922
|
-
scope_type = entities.PlatformEntityType.TASK
|
|
923
|
-
elif scope_id == dataset_id:
|
|
924
|
-
scope_type = entities.PlatformEntityType.DATASET
|
|
925
|
-
elif scope_id == user_email:
|
|
926
|
-
scope_type = entities.PlatformEntityType.USER
|
|
927
|
-
elif scope_id == org_id:
|
|
928
|
-
scope_type = entities.PlatformEntityType.ORG
|
|
929
|
-
else:
|
|
930
|
-
raise exceptions.PlatformException('400', "Unknown resource id")
|
|
931
|
-
|
|
932
|
-
setting = entities.Setting(
|
|
933
|
-
default_value=True,
|
|
934
|
-
value=True,
|
|
935
|
-
inputs=None,
|
|
936
|
-
name=service.name,
|
|
937
|
-
value_type=entities.SettingsValueTypes.BOOLEAN,
|
|
938
|
-
scope=entities.SettingScope(
|
|
939
|
-
type=scope_type,
|
|
940
|
-
id=scope_id,
|
|
941
|
-
role=role,
|
|
942
|
-
prevent_override=prevent_override,
|
|
943
|
-
visible=visible
|
|
944
|
-
),
|
|
945
|
-
metadata={
|
|
946
|
-
'serviceId': service.id,
|
|
947
|
-
'serviceName': service.name,
|
|
948
|
-
'projectId': service.project_id,
|
|
949
|
-
'slots': slots
|
|
950
|
-
},
|
|
951
|
-
description=service.name,
|
|
952
|
-
icon=icon,
|
|
953
|
-
section_name=entities.SettingsSectionNames.APPLICATIONS,
|
|
954
|
-
sub_section_name=None,
|
|
955
|
-
hint=None
|
|
956
|
-
)
|
|
957
|
-
|
|
958
|
-
try:
|
|
959
|
-
settings.append(self._settings.create(setting=setting))
|
|
960
|
-
except exceptions.BadRequest as err:
|
|
961
|
-
if 'settings already exist' in err.message:
|
|
962
|
-
old_setting = self._settings.get(setting_name=setting.name)
|
|
963
|
-
setting.id = old_setting.id
|
|
964
|
-
settings.append(self._settings.update(setting=setting))
|
|
965
|
-
else:
|
|
966
|
-
raise err
|
|
967
|
-
|
|
968
|
-
return settings
|
|
969
|
-
|
|
970
|
-
@_api_reference.add(path='/services/logs', method='post')
|
|
971
|
-
def log(self,
|
|
972
|
-
service,
|
|
973
|
-
size=100,
|
|
974
|
-
checkpoint=None,
|
|
975
|
-
start=None,
|
|
976
|
-
end=None,
|
|
977
|
-
follow=False,
|
|
978
|
-
text=None,
|
|
979
|
-
execution_id=None,
|
|
980
|
-
function_name=None,
|
|
981
|
-
replica_id=None,
|
|
982
|
-
system=False,
|
|
983
|
-
view=True,
|
|
984
|
-
until_completed=True,
|
|
985
|
-
log_level='DEBUG',
|
|
986
|
-
model_id: str = None,
|
|
987
|
-
model_operation: str = None,
|
|
988
|
-
project_id: str = None
|
|
989
|
-
):
|
|
990
|
-
"""
|
|
991
|
-
Get service logs.
|
|
992
|
-
|
|
993
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
994
|
-
|
|
995
|
-
:param dtlpy.entities.service.Service service: service object
|
|
996
|
-
:param int size: size
|
|
997
|
-
:param dict checkpoint: the information from the lst point checked in the service
|
|
998
|
-
:param str start: iso format time
|
|
999
|
-
:param str end: iso format time
|
|
1000
|
-
:param bool follow: if true, keep stream future logs
|
|
1001
|
-
:param str text: text
|
|
1002
|
-
:param str execution_id: execution id
|
|
1003
|
-
:param str function_name: function name
|
|
1004
|
-
:param str replica_id: replica id
|
|
1005
|
-
:param bool system: system
|
|
1006
|
-
:param bool view: if true, print out all the logs
|
|
1007
|
-
:param bool until_completed: wait until completed
|
|
1008
|
-
:param str log_level: the log level to display dl.LoggingLevel
|
|
1009
|
-
:param str model_id: model id
|
|
1010
|
-
:param str model_operation: model operation action
|
|
1011
|
-
:param str project_id: project id
|
|
1012
|
-
:return: ServiceLog entity
|
|
1013
|
-
:rtype: ServiceLog
|
|
1014
|
-
|
|
1015
|
-
**Example**:
|
|
1016
|
-
|
|
1017
|
-
.. code-block:: python
|
|
1018
|
-
|
|
1019
|
-
service_logs = package.services.log(service='service_entity')
|
|
1020
|
-
"""
|
|
1021
|
-
if not isinstance(service, entities.Service) and model_id is None:
|
|
1022
|
-
raise exceptions.PlatformException('400', 'Must provide service or model_id')
|
|
1023
|
-
if isinstance(log_level, str):
|
|
1024
|
-
log_level = log_level.upper()
|
|
1025
|
-
|
|
1026
|
-
payload = {
|
|
1027
|
-
'direction': 'asc',
|
|
1028
|
-
'follow': follow,
|
|
1029
|
-
'system': system,
|
|
1030
|
-
'logLevel': log_level
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
if service is not None:
|
|
1034
|
-
payload['serviceId'] = service.id
|
|
1035
|
-
|
|
1036
|
-
if size is not None:
|
|
1037
|
-
payload['size'] = size
|
|
1038
|
-
|
|
1039
|
-
if execution_id is not None:
|
|
1040
|
-
payload['executionId'] = [execution_id]
|
|
1041
|
-
|
|
1042
|
-
if function_name is not None:
|
|
1043
|
-
payload['functionName'] = function_name
|
|
1044
|
-
|
|
1045
|
-
if text is not None:
|
|
1046
|
-
payload['text'] = text
|
|
1047
|
-
|
|
1048
|
-
if replica_id is not None:
|
|
1049
|
-
payload['replicaId'] = replica_id
|
|
1050
|
-
|
|
1051
|
-
if checkpoint is not None:
|
|
1052
|
-
payload['checkpoint'] = checkpoint
|
|
1053
|
-
|
|
1054
|
-
if start is not None:
|
|
1055
|
-
payload['start'] = start
|
|
1056
|
-
else:
|
|
1057
|
-
payload['start'] = datetime.datetime(datetime.date.today().year,
|
|
1058
|
-
datetime.date.today().month,
|
|
1059
|
-
datetime.date.today().day,
|
|
1060
|
-
0,
|
|
1061
|
-
0,
|
|
1062
|
-
0).isoformat()
|
|
1063
|
-
|
|
1064
|
-
if end is not None:
|
|
1065
|
-
payload['end'] = end
|
|
1066
|
-
|
|
1067
|
-
if model_id is not None:
|
|
1068
|
-
payload['modelId'] = model_id
|
|
1069
|
-
|
|
1070
|
-
if model_operation is not None:
|
|
1071
|
-
payload['modelOperation'] = model_operation
|
|
1072
|
-
|
|
1073
|
-
if project_id is not None:
|
|
1074
|
-
payload['projectId'] = project_id
|
|
1075
|
-
else:
|
|
1076
|
-
payload['projectId'] = service.project_id
|
|
1077
|
-
|
|
1078
|
-
# request
|
|
1079
|
-
success, response = self._client_api.gen_request(req_type='post',
|
|
1080
|
-
path='/services/logs',
|
|
1081
|
-
json_req=payload)
|
|
1082
|
-
|
|
1083
|
-
# exception handling
|
|
1084
|
-
if not success:
|
|
1085
|
-
raise exceptions.PlatformException(response)
|
|
1086
|
-
|
|
1087
|
-
log = ServiceLog(_json=response.json(),
|
|
1088
|
-
service=service,
|
|
1089
|
-
services=self,
|
|
1090
|
-
start=payload['start'],
|
|
1091
|
-
follow=follow,
|
|
1092
|
-
execution_id=execution_id,
|
|
1093
|
-
function_name=function_name,
|
|
1094
|
-
replica_id=replica_id,
|
|
1095
|
-
system=system,
|
|
1096
|
-
model_id=model_id,
|
|
1097
|
-
model_operation=model_operation,
|
|
1098
|
-
project_id=project_id)
|
|
1099
|
-
|
|
1100
|
-
if view:
|
|
1101
|
-
log.view(until_completed=until_completed)
|
|
1102
|
-
else:
|
|
1103
|
-
return log
|
|
1104
|
-
|
|
1105
|
-
def execute(self,
|
|
1106
|
-
service: entities.Service = None,
|
|
1107
|
-
service_id: str = None,
|
|
1108
|
-
service_name: str = None,
|
|
1109
|
-
sync: bool = False,
|
|
1110
|
-
function_name: str = None,
|
|
1111
|
-
stream_logs: bool = False,
|
|
1112
|
-
execution_input=None,
|
|
1113
|
-
resource=None,
|
|
1114
|
-
item_id=None,
|
|
1115
|
-
dataset_id=None,
|
|
1116
|
-
annotation_id=None,
|
|
1117
|
-
project_id=None,
|
|
1118
|
-
) -> entities.Execution:
|
|
1119
|
-
"""
|
|
1120
|
-
Execute a function on an existing service.
|
|
1121
|
-
|
|
1122
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1123
|
-
|
|
1124
|
-
:param dtlpy.entities.service.Service service: service entity
|
|
1125
|
-
:param str service_id: service id
|
|
1126
|
-
:param str service_name: service name
|
|
1127
|
-
:param bool sync: wait for function to end
|
|
1128
|
-
:param str function_name: function name to run
|
|
1129
|
-
:param bool stream_logs: prints logs of the new execution. only works with sync=True
|
|
1130
|
-
:param execution_input: input dictionary or list of FunctionIO entities
|
|
1131
|
-
:param str resource: dl.PackageInputType - input type.
|
|
1132
|
-
:param str item_id: str - optional - input to function
|
|
1133
|
-
:param str dataset_id: str - optional - input to function
|
|
1134
|
-
:param str annotation_id: str - optional - input to function
|
|
1135
|
-
:param str project_id: str - resource's project
|
|
1136
|
-
:return: entities.Execution
|
|
1137
|
-
:rtype: dtlpy.entities.execution.Execution
|
|
1138
|
-
|
|
1139
|
-
**Example**:
|
|
1140
|
-
|
|
1141
|
-
.. code-block:: python
|
|
1142
|
-
|
|
1143
|
-
execution = package.services.execute(service='service_entity',
|
|
1144
|
-
function_name='run',
|
|
1145
|
-
item_id='item_id',
|
|
1146
|
-
project_id='project_id')
|
|
1147
|
-
"""
|
|
1148
|
-
if service is None:
|
|
1149
|
-
service = self.get(service_id=service_id, service_name=service_name)
|
|
1150
|
-
execution = repositories.Executions(service=service,
|
|
1151
|
-
client_api=self._client_api,
|
|
1152
|
-
project=self._project).create(service_id=service.id,
|
|
1153
|
-
sync=sync,
|
|
1154
|
-
execution_input=execution_input,
|
|
1155
|
-
function_name=function_name,
|
|
1156
|
-
resource=resource,
|
|
1157
|
-
item_id=item_id,
|
|
1158
|
-
dataset_id=dataset_id,
|
|
1159
|
-
annotation_id=annotation_id,
|
|
1160
|
-
project_id=project_id,
|
|
1161
|
-
stream_logs=stream_logs)
|
|
1162
|
-
return execution
|
|
1163
|
-
|
|
1164
|
-
@_api_reference.add(path='/services', method='post')
|
|
1165
|
-
def deploy(self,
|
|
1166
|
-
service_name: str = None,
|
|
1167
|
-
package: entities.Package = None,
|
|
1168
|
-
bot: Union[entities.Bot, str] = None,
|
|
1169
|
-
revision: str or int = None,
|
|
1170
|
-
init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
|
|
1171
|
-
runtime: Union[entities.KubernetesRuntime, dict] = None,
|
|
1172
|
-
pod_type: entities.InstanceCatalog = None,
|
|
1173
|
-
sdk_version: str = None,
|
|
1174
|
-
agent_versions: dict = None,
|
|
1175
|
-
verify: bool = True,
|
|
1176
|
-
checkout: bool = False,
|
|
1177
|
-
module_name: str = None,
|
|
1178
|
-
project_id: str = None,
|
|
1179
|
-
driver_id: str = None,
|
|
1180
|
-
func: Callable = None,
|
|
1181
|
-
run_execution_as_process: bool = None,
|
|
1182
|
-
execution_timeout: int = None,
|
|
1183
|
-
drain_time: int = None,
|
|
1184
|
-
max_attempts: int = None,
|
|
1185
|
-
on_reset: str = None,
|
|
1186
|
-
force: bool = False,
|
|
1187
|
-
secrets: list = None,
|
|
1188
|
-
integrations: list = None,
|
|
1189
|
-
active: bool = True,
|
|
1190
|
-
**kwargs) -> entities.Service:
|
|
1191
|
-
"""
|
|
1192
|
-
Deploy service.
|
|
1193
|
-
|
|
1194
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1195
|
-
|
|
1196
|
-
:param str service_name: name
|
|
1197
|
-
:param dtlpy.entities.package.Package package: package entity
|
|
1198
|
-
:param str bot: bot email
|
|
1199
|
-
:param str revision: package revision of version
|
|
1200
|
-
:param init_input: config to run at startup
|
|
1201
|
-
:param dict runtime: runtime resources
|
|
1202
|
-
:param str pod_type: pod type dl.InstanceCatalog
|
|
1203
|
-
:param str sdk_version: - optional - string - sdk version
|
|
1204
|
-
:param str agent_versions: - dictionary - - optional -versions of sdk
|
|
1205
|
-
:param bool verify: if true, verify the inputs
|
|
1206
|
-
:param bool checkout: if true, checkout (switch) to service
|
|
1207
|
-
:param str module_name: module name
|
|
1208
|
-
:param str project_id: project id
|
|
1209
|
-
:param str driver_id: driver id
|
|
1210
|
-
:param Callable func: function to deploy
|
|
1211
|
-
:param bool run_execution_as_process: if true, run execution as process
|
|
1212
|
-
:param int execution_timeout: execution timeout in seconds
|
|
1213
|
-
:param int drain_time: drain time in seconds
|
|
1214
|
-
:param int max_attempts: maximum execution retries in-case of a service reset
|
|
1215
|
-
:param str on_reset: what happens on reset
|
|
1216
|
-
:param bool force: optional - if true, terminate old replicas immediately
|
|
1217
|
-
:param list secrets: list of the integrations ids
|
|
1218
|
-
:param list integrations: list of the integrations
|
|
1219
|
-
:param bool active: if true, activate the service
|
|
1220
|
-
:param kwargs: list of additional arguments
|
|
1221
|
-
:return: Service object
|
|
1222
|
-
:rtype: dtlpy.entities.service.Service
|
|
1223
|
-
|
|
1224
|
-
**Example**:
|
|
1225
|
-
|
|
1226
|
-
.. code-block:: python
|
|
1227
|
-
|
|
1228
|
-
service = package.services.deploy(service_name=package_name,
|
|
1229
|
-
execution_timeout=3 * 60 * 60,
|
|
1230
|
-
module_name=module.name,
|
|
1231
|
-
runtime=dl.KubernetesRuntime(
|
|
1232
|
-
concurrency=10,
|
|
1233
|
-
pod_type=dl.InstanceCatalog.REGULAR_S,
|
|
1234
|
-
autoscaler=dl.KubernetesRabbitmqAutoscaler(
|
|
1235
|
-
min_replicas=1,
|
|
1236
|
-
max_replicas=20,
|
|
1237
|
-
queue_length=20
|
|
1238
|
-
)
|
|
1239
|
-
)
|
|
1240
|
-
)
|
|
1241
|
-
"""
|
|
1242
|
-
if package is None and isinstance(package, entities.Dpk):
|
|
1243
|
-
raise exceptions.PlatformException('400', 'cannot deploy dpk package. Please install the app')
|
|
1244
|
-
package = package if package is not None else self._package
|
|
1245
|
-
if service_name is None:
|
|
1246
|
-
get_name = False
|
|
1247
|
-
if package is not None and package.service_config is not None and 'name' in package.service_config:
|
|
1248
|
-
service_name = package.service_config['name']
|
|
1249
|
-
get_name = True
|
|
1250
|
-
else:
|
|
1251
|
-
if package is not None:
|
|
1252
|
-
service_name = package.name
|
|
1253
|
-
else:
|
|
1254
|
-
service_name = 'default-service'
|
|
1255
|
-
if not get_name:
|
|
1256
|
-
logger.warning('service_name not provided, using: {} by default'.format(service_name))
|
|
1257
|
-
|
|
1258
|
-
if isinstance(revision, int):
|
|
1259
|
-
logger.warning('Deprecation Warning - Package/service versions have been refactored'
|
|
1260
|
-
'The version you provided has type: int, it will be converted to: 1.0.{}'
|
|
1261
|
-
'Next time use a 3-level semver for package/service versions'.format(revision))
|
|
1262
|
-
|
|
1263
|
-
if func is not None:
|
|
1264
|
-
package = self.__deploy_function(name=service_name, project=self._project, func=func)
|
|
1265
|
-
|
|
1266
|
-
if init_input is not None and not isinstance(init_input, dict):
|
|
1267
|
-
if not isinstance(init_input, list):
|
|
1268
|
-
init_input = [init_input]
|
|
1269
|
-
|
|
1270
|
-
if len(init_input) > 0 and isinstance(init_input[0], entities.FunctionIO):
|
|
1271
|
-
params = dict()
|
|
1272
|
-
for i_param, param in enumerate(init_input):
|
|
1273
|
-
params[param.name] = param.value
|
|
1274
|
-
init_input = params
|
|
1275
|
-
elif len(init_input) == 0:
|
|
1276
|
-
init_input = None
|
|
1277
|
-
else:
|
|
1278
|
-
raise exceptions.PlatformException(
|
|
1279
|
-
error='400',
|
|
1280
|
-
message='Unknown init_input type. expecting list or dict, got: {}'.format(type(init_input))
|
|
1281
|
-
)
|
|
1282
|
-
|
|
1283
|
-
if project_id is None:
|
|
1284
|
-
if self._project is not None:
|
|
1285
|
-
project_id = self._project.id
|
|
1286
|
-
else:
|
|
1287
|
-
project_id = self._project_id
|
|
1288
|
-
|
|
1289
|
-
filters = entities.Filters(resource=entities.FiltersResource.SERVICE, use_defaults=False)
|
|
1290
|
-
filters.add(field='name', values=service_name)
|
|
1291
|
-
if project_id is not None:
|
|
1292
|
-
filters.add(field='projectId', values=project_id)
|
|
1293
|
-
services = self.list(filters=filters)
|
|
1294
|
-
if services.items_count > 1:
|
|
1295
|
-
raise exceptions.PlatformException('400',
|
|
1296
|
-
'More than 1 service by this name are associated with this user. '
|
|
1297
|
-
'Please provide project_id')
|
|
1298
|
-
elif services.items_count > 0:
|
|
1299
|
-
service = services.items[0]
|
|
1300
|
-
if runtime is not None:
|
|
1301
|
-
service.runtime = runtime
|
|
1302
|
-
if init_input is not None:
|
|
1303
|
-
service.init_input = init_input
|
|
1304
|
-
if revision is not None:
|
|
1305
|
-
service.package_revision = revision
|
|
1306
|
-
if agent_versions is not None:
|
|
1307
|
-
service.versions = agent_versions
|
|
1308
|
-
elif sdk_version:
|
|
1309
|
-
service.versions = {'dtlpy': sdk_version}
|
|
1310
|
-
if driver_id is not None:
|
|
1311
|
-
service.driver_id = driver_id
|
|
1312
|
-
if secrets is not None:
|
|
1313
|
-
if not isinstance(secrets, list):
|
|
1314
|
-
secrets = [secrets]
|
|
1315
|
-
service.secrets = secrets
|
|
1316
|
-
if integrations is not None:
|
|
1317
|
-
if not isinstance(integrations, list):
|
|
1318
|
-
integrations = [integrations]
|
|
1319
|
-
service.integrations = integrations
|
|
1320
|
-
service = self.update(service=service, force=force)
|
|
1321
|
-
else:
|
|
1322
|
-
service = self._create(service_name=service_name,
|
|
1323
|
-
package=package,
|
|
1324
|
-
project_id=project_id,
|
|
1325
|
-
bot=bot,
|
|
1326
|
-
revision=revision,
|
|
1327
|
-
init_input=init_input,
|
|
1328
|
-
runtime=runtime,
|
|
1329
|
-
pod_type=pod_type,
|
|
1330
|
-
sdk_version=sdk_version,
|
|
1331
|
-
agent_versions=agent_versions,
|
|
1332
|
-
verify=verify,
|
|
1333
|
-
module_name=module_name,
|
|
1334
|
-
driver_id=driver_id,
|
|
1335
|
-
jwt_forward=kwargs.get('jwt_forward', None),
|
|
1336
|
-
is_global=kwargs.get('is_global', None),
|
|
1337
|
-
run_execution_as_process=run_execution_as_process,
|
|
1338
|
-
execution_timeout=execution_timeout,
|
|
1339
|
-
drain_time=drain_time,
|
|
1340
|
-
max_attempts=max_attempts,
|
|
1341
|
-
on_reset=on_reset,
|
|
1342
|
-
secrets=secrets,
|
|
1343
|
-
integrations=integrations,
|
|
1344
|
-
active=active
|
|
1345
|
-
)
|
|
1346
|
-
if checkout:
|
|
1347
|
-
self.checkout(service=service)
|
|
1348
|
-
return service
|
|
1349
|
-
|
|
1350
|
-
@staticmethod
|
|
1351
|
-
def __get_import_string(imports: List[str]):
|
|
1352
|
-
import_string = ''
|
|
1353
|
-
if imports is not None:
|
|
1354
|
-
import_string = '\n'.join(imports)
|
|
1355
|
-
return import_string
|
|
1356
|
-
|
|
1357
|
-
@staticmethod
|
|
1358
|
-
def __get_inputs(func):
|
|
1359
|
-
method = inspect.signature(func)
|
|
1360
|
-
params = list(method.parameters)
|
|
1361
|
-
inpts = list()
|
|
1362
|
-
inputs_types = {i.name.lower(): i.value for i in list(entities.PackageInputType)}
|
|
1363
|
-
for arg in params:
|
|
1364
|
-
if arg in inputs_types:
|
|
1365
|
-
inpt_type = inputs_types[arg]
|
|
1366
|
-
else:
|
|
1367
|
-
inpt_type = entities.PackageInputType.JSON
|
|
1368
|
-
inpts.append(entities.FunctionIO(type=inpt_type, name=arg))
|
|
1369
|
-
return inpts
|
|
1370
|
-
|
|
1371
|
-
def __deploy_function(self,
|
|
1372
|
-
name: str,
|
|
1373
|
-
func: Callable,
|
|
1374
|
-
project: entities.Project) -> entities.Package:
|
|
1375
|
-
package_dir = tempfile.mkdtemp()
|
|
1376
|
-
# imports_string = self.__get_import_string()
|
|
1377
|
-
imports_string = ''
|
|
1378
|
-
|
|
1379
|
-
main_file = os.path.join(package_dir, entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
|
|
1380
|
-
with open(assets.paths.PARTIAL_MAIN_FILEPATH, 'r') as f:
|
|
1381
|
-
main_string = f.read()
|
|
1382
|
-
lines = inspect.getsourcelines(func)
|
|
1383
|
-
tabs_diff = lines[0][0].count(' ') - 1
|
|
1384
|
-
for line_index in range(len(lines[0])):
|
|
1385
|
-
line_tabs = lines[0][line_index].count(' ') - tabs_diff
|
|
1386
|
-
lines[0][line_index] = (' ' * line_tabs) + lines[0][line_index].strip() + '\n'
|
|
1387
|
-
|
|
1388
|
-
method_func_string = "".join(lines[0])
|
|
1389
|
-
|
|
1390
|
-
with open(main_file, 'w') as f:
|
|
1391
|
-
f.write('{}\n{}\n @staticmethod\n{}'.format(imports_string, main_string,
|
|
1392
|
-
method_func_string))
|
|
1393
|
-
|
|
1394
|
-
function = entities.PackageFunction(name=func.__name__, inputs=self.__get_inputs(func=func))
|
|
1395
|
-
module = entities.PackageModule(functions=[function],
|
|
1396
|
-
entry_point=entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
|
|
1397
|
-
packages = repositories.Packages(client_api=self._client_api, project=project)
|
|
1398
|
-
return packages.push(src_path=package_dir,
|
|
1399
|
-
package_name=name,
|
|
1400
|
-
checkout=True,
|
|
1401
|
-
modules=[module])
|
|
1402
|
-
|
|
1403
|
-
def deploy_from_local_folder(self,
|
|
1404
|
-
cwd=None,
|
|
1405
|
-
service_file=None,
|
|
1406
|
-
bot=None,
|
|
1407
|
-
checkout=False,
|
|
1408
|
-
force=False
|
|
1409
|
-
) -> entities.Service:
|
|
1410
|
-
"""
|
|
1411
|
-
Deploy from local folder in local environment.
|
|
1412
|
-
|
|
1413
|
-
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1414
|
-
|
|
1415
|
-
:param str cwd: optional - package working directory. Default=cwd
|
|
1416
|
-
:param str service_file: optional - service file. Default=None
|
|
1417
|
-
:param str bot: bot
|
|
1418
|
-
:param checkout: checkout
|
|
1419
|
-
:param bool force: optional - terminate old replicas immediately
|
|
1420
|
-
:return: Service object
|
|
1421
|
-
:rtype: dtlpy.entities.service.Service
|
|
1422
|
-
|
|
1423
|
-
**Example**:
|
|
1424
|
-
|
|
1425
|
-
.. code-block:: python
|
|
1426
|
-
|
|
1427
|
-
service = package.services.deploy_from_local_folder(cwd='file_path',
|
|
1428
|
-
service_file='service_file')
|
|
1429
|
-
"""
|
|
1430
|
-
# get cwd and service.json path
|
|
1431
|
-
if cwd is None:
|
|
1432
|
-
cwd = os.getcwd()
|
|
1433
|
-
if service_file is None:
|
|
1434
|
-
service_file = os.path.join(cwd, assets.paths.SERVICE_FILENAME)
|
|
1435
|
-
|
|
1436
|
-
# load service json
|
|
1437
|
-
if os.path.isfile(service_file):
|
|
1438
|
-
with open(service_file, 'r') as f:
|
|
1439
|
-
service_json = json.load(f)
|
|
1440
|
-
service_triggers = service_json.get('triggers', list())
|
|
1441
|
-
else:
|
|
1442
|
-
raise exceptions.PlatformException(error='400',
|
|
1443
|
-
message='Could not find service.json in path: {}'.format(cwd))
|
|
1444
|
-
|
|
1445
|
-
# get package
|
|
1446
|
-
package_name = service_json.get('packageName', None)
|
|
1447
|
-
packages = repositories.Packages(client_api=self._client_api, project=self._project)
|
|
1448
|
-
|
|
1449
|
-
if package_name is None:
|
|
1450
|
-
package = packages.get()
|
|
1451
|
-
else:
|
|
1452
|
-
package = packages.get(package_name=package_name)
|
|
1453
|
-
|
|
1454
|
-
name = service_json.get('name', None)
|
|
1455
|
-
revision = service_json.get('revision', package.version)
|
|
1456
|
-
init_input = service_json.get('initParams', dict())
|
|
1457
|
-
runtime = service_json.get('runtime', dict())
|
|
1458
|
-
sdk_version = service_json.get('version', None)
|
|
1459
|
-
agent_versions = service_json.get('versions', None)
|
|
1460
|
-
verify = service_json.get('verify', True)
|
|
1461
|
-
module_name = service_json.get('module_name', None)
|
|
1462
|
-
run_execution_as_process = service_json.get('run_execution_as_process', None)
|
|
1463
|
-
execution_timeout = service_json.get('execution_timeout', None)
|
|
1464
|
-
drain_time = service_json.get('drain_time', None)
|
|
1465
|
-
on_reset = service_json.get('on_reset', None)
|
|
1466
|
-
max_attempts = service_json.get('maxAttempts', None)
|
|
1467
|
-
|
|
1468
|
-
service = self.deploy(bot=bot,
|
|
1469
|
-
service_name=name,
|
|
1470
|
-
package=package,
|
|
1471
|
-
revision=revision,
|
|
1472
|
-
runtime=runtime,
|
|
1473
|
-
init_input=init_input,
|
|
1474
|
-
sdk_version=sdk_version,
|
|
1475
|
-
agent_versions=agent_versions,
|
|
1476
|
-
verify=verify,
|
|
1477
|
-
checkout=checkout,
|
|
1478
|
-
run_execution_as_process=run_execution_as_process,
|
|
1479
|
-
execution_timeout=execution_timeout,
|
|
1480
|
-
drain_time=drain_time,
|
|
1481
|
-
max_attempts=max_attempts,
|
|
1482
|
-
on_reset=on_reset,
|
|
1483
|
-
module_name=module_name,
|
|
1484
|
-
force=force
|
|
1485
|
-
)
|
|
1486
|
-
|
|
1487
|
-
logger.info('Service was deployed successfully. Service id: {}'.format(service.id))
|
|
1488
|
-
|
|
1489
|
-
if len(service_triggers) > 0:
|
|
1490
|
-
logger.info('Creating triggers...')
|
|
1491
|
-
triggers = repositories.Triggers(client_api=self._client_api, project=self._project)
|
|
1492
|
-
|
|
1493
|
-
for trigger in service_triggers:
|
|
1494
|
-
name = trigger.get('name', None)
|
|
1495
|
-
filters = trigger.get('filter', dict())
|
|
1496
|
-
resource = trigger['resource']
|
|
1497
|
-
actions = trigger.get('actions', list())
|
|
1498
|
-
active = trigger.get('active', True)
|
|
1499
|
-
execution_mode = trigger.get('executionMode', None)
|
|
1500
|
-
function_name = trigger.get('function', None)
|
|
1501
|
-
|
|
1502
|
-
trigger = triggers.create(service_id=service.id,
|
|
1503
|
-
name=name,
|
|
1504
|
-
filters=filters,
|
|
1505
|
-
resource=resource,
|
|
1506
|
-
actions=actions,
|
|
1507
|
-
active=active,
|
|
1508
|
-
execution_mode=execution_mode,
|
|
1509
|
-
function_name=function_name)
|
|
1510
|
-
|
|
1511
|
-
logger.info('Trigger was created successfully. Service id: {}'.format(trigger.id))
|
|
1512
|
-
|
|
1513
|
-
logging.info('Successfully deployed!')
|
|
1514
|
-
return service
|
|
1515
|
-
|
|
1516
|
-
def __enable_cache(self,
|
|
1517
|
-
url,
|
|
1518
|
-
organization: entities.Organization,
|
|
1519
|
-
pod_type=entities.PodType.SMALL):
|
|
1520
|
-
payload = {
|
|
1521
|
-
"org": {
|
|
1522
|
-
"name": organization.name,
|
|
1523
|
-
"id": organization.id
|
|
1524
|
-
},
|
|
1525
|
-
"runner": {
|
|
1526
|
-
"podType": pod_type # small, medium, high
|
|
1527
|
-
}
|
|
1528
|
-
}
|
|
1529
|
-
|
|
1530
|
-
return self._client_api.gen_request(req_type='post',
|
|
1531
|
-
path=url,
|
|
1532
|
-
json_req=payload)
|
|
1533
|
-
|
|
1534
|
-
def __polling_wait(self, organization, pod_type, backoff_factor=1):
|
|
1535
|
-
fs_url_path = '/services/fs-cache?mode={}'.format('get')
|
|
1536
|
-
i = 1
|
|
1537
|
-
while True:
|
|
1538
|
-
success, response = self.__enable_cache(url=fs_url_path, organization=organization, pod_type=pod_type)
|
|
1539
|
-
if response.json().get('state', None) == 'READY':
|
|
1540
|
-
break
|
|
1541
|
-
sleep_time = min(backoff_factor * (2 ** (i - 1)), MAX_WAIT_TIME)
|
|
1542
|
-
logger.debug("Going to sleep {:.2f}[s]".format(sleep_time))
|
|
1543
|
-
time.sleep(sleep_time)
|
|
1544
|
-
i += 1
|
|
1545
|
-
return success
|
|
1546
|
-
|
|
1547
|
-
def _cache_action(self,
|
|
1548
|
-
organization: entities.Organization = None,
|
|
1549
|
-
mode=entities.CacheAction.APPLY,
|
|
1550
|
-
pod_type=entities.PodType.SMALL):
|
|
1551
|
-
"""
|
|
1552
|
-
Add or remove Cache for the org
|
|
1553
|
-
|
|
1554
|
-
**Prerequisites**: You must be an organization *owner*
|
|
1555
|
-
|
|
1556
|
-
You must provide at least ONE of the following params: organization, organization_name, or organization_id.
|
|
1557
|
-
|
|
1558
|
-
:param entities.Organization organization: Organization object
|
|
1559
|
-
:param str mode: dl.CacheAction.APPLY or dl.CacheAction.DESTROY
|
|
1560
|
-
:param entities.PodType pod_type: dl.PodType.SMALL, dl.PodType.MEDIUM, dl.PodType.HIGH
|
|
1561
|
-
:return: True if success
|
|
1562
|
-
:rtype: bool
|
|
1563
|
-
|
|
1564
|
-
**Example**:
|
|
1565
|
-
|
|
1566
|
-
.. code-block:: python
|
|
1567
|
-
|
|
1568
|
-
success = dl.organizations.enable_cache(organization='organization',
|
|
1569
|
-
mode=dl.CacheAction.APPLY)
|
|
1570
|
-
"""
|
|
1571
|
-
if organization is None:
|
|
1572
|
-
raise exceptions.PlatformException(
|
|
1573
|
-
error='400',
|
|
1574
|
-
message='Must provide an identifier in inputs')
|
|
1575
|
-
|
|
1576
|
-
fs_mode = mode if mode != entities.CacheAction.APPLY else '{}-filestore'.format(mode)
|
|
1577
|
-
apply_fs_url_path = '/services/fs-cache?mode={}'.format(fs_mode)
|
|
1578
|
-
apply_volume_url_path = '/services/fs-cache?mode={}'.format(mode)
|
|
1579
|
-
cache_url_path = '/services/cache?mode={}'.format(mode)
|
|
1580
|
-
|
|
1581
|
-
success, response = self.__enable_cache(url=apply_fs_url_path, organization=organization, pod_type=pod_type)
|
|
1582
|
-
if not success:
|
|
1583
|
-
raise exceptions.PlatformException(response)
|
|
1584
|
-
|
|
1585
|
-
if mode == entities.CacheAction.APPLY:
|
|
1586
|
-
self.__polling_wait(organization=organization, pod_type=pod_type)
|
|
1587
|
-
success, response = self.__enable_cache(url=apply_volume_url_path, organization=organization,
|
|
1588
|
-
pod_type=pod_type)
|
|
1589
|
-
if not success:
|
|
1590
|
-
raise exceptions.PlatformException(response)
|
|
1591
|
-
|
|
1592
|
-
success, response = self.__enable_cache(url=cache_url_path, organization=organization, pod_type=pod_type)
|
|
1593
|
-
if not success:
|
|
1594
|
-
raise exceptions.PlatformException(response)
|
|
1595
|
-
|
|
1596
|
-
return True
|
|
1597
|
-
|
|
1598
|
-
def restart(self, service: entities.Service, replica_name: str = None):
|
|
1599
|
-
"""
|
|
1600
|
-
Restart service replica
|
|
1601
|
-
|
|
1602
|
-
**Prerequisites**: You must be in the role of a *developer*.
|
|
1603
|
-
|
|
1604
|
-
:param dtlpy.entities.service.Service service: service entity
|
|
1605
|
-
:param str replica_name: replica name
|
|
1606
|
-
:return: True
|
|
1607
|
-
:rtype: bool
|
|
1608
|
-
|
|
1609
|
-
**Example**:
|
|
1610
|
-
|
|
1611
|
-
.. code-block:: python
|
|
1612
|
-
|
|
1613
|
-
is_restarted = dl.services.restart(service='service_entity',
|
|
1614
|
-
replica_name='replica_name')
|
|
1615
|
-
"""
|
|
1616
|
-
payload = {}
|
|
1617
|
-
|
|
1618
|
-
if replica_name is not None:
|
|
1619
|
-
payload['replicaName'] = replica_name
|
|
1620
|
-
|
|
1621
|
-
# request
|
|
1622
|
-
success, response = self._client_api.gen_request(req_type='post',
|
|
1623
|
-
path='/services/{}/restart'.format(service.id),
|
|
1624
|
-
json_req=payload)
|
|
1625
|
-
|
|
1626
|
-
# exception handling
|
|
1627
|
-
if not success:
|
|
1628
|
-
raise exceptions.PlatformException(response)
|
|
1629
|
-
|
|
1630
|
-
return True
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
class ServiceLog:
|
|
1634
|
-
"""
|
|
1635
|
-
Service Log
|
|
1636
|
-
"""
|
|
1637
|
-
|
|
1638
|
-
def __init__(self,
|
|
1639
|
-
_json: dict,
|
|
1640
|
-
service: entities.Service,
|
|
1641
|
-
services: Services,
|
|
1642
|
-
start=None,
|
|
1643
|
-
follow=None,
|
|
1644
|
-
execution_id=None,
|
|
1645
|
-
function_name=None,
|
|
1646
|
-
replica_id=None,
|
|
1647
|
-
system=False,
|
|
1648
|
-
model_id=None,
|
|
1649
|
-
model_operation=None,
|
|
1650
|
-
project_id=None):
|
|
1651
|
-
|
|
1652
|
-
self.logs = _json.get('logs', dict())
|
|
1653
|
-
self.checkpoint = _json.get('checkpoint', None)
|
|
1654
|
-
self.stop = _json.get('stop', False)
|
|
1655
|
-
self.service = service
|
|
1656
|
-
self.services = services
|
|
1657
|
-
self.start = start
|
|
1658
|
-
self.follow = follow
|
|
1659
|
-
self.execution_id = execution_id
|
|
1660
|
-
self.function_name = function_name
|
|
1661
|
-
self.replica_id = replica_id
|
|
1662
|
-
self.system = system
|
|
1663
|
-
self.model_id = model_id
|
|
1664
|
-
self.model_operation = model_operation
|
|
1665
|
-
self.project_id = project_id
|
|
1666
|
-
|
|
1667
|
-
def get_next_log(self):
|
|
1668
|
-
log = self.services.log(service=self.service,
|
|
1669
|
-
checkpoint=self.checkpoint,
|
|
1670
|
-
start=self.start,
|
|
1671
|
-
follow=self.follow,
|
|
1672
|
-
execution_id=self.execution_id,
|
|
1673
|
-
function_name=self.function_name,
|
|
1674
|
-
replica_id=self.replica_id,
|
|
1675
|
-
system=self.system,
|
|
1676
|
-
view=False,
|
|
1677
|
-
model_id=self.model_id,
|
|
1678
|
-
model_operation=self.model_operation,
|
|
1679
|
-
project_id=self.project_id)
|
|
1680
|
-
|
|
1681
|
-
self.logs = log.logs
|
|
1682
|
-
self.checkpoint = log.checkpoint
|
|
1683
|
-
self.stop = log.stop
|
|
1684
|
-
|
|
1685
|
-
def view(self, until_completed):
|
|
1686
|
-
"""
|
|
1687
|
-
View logs
|
|
1688
|
-
|
|
1689
|
-
:param until_completed:
|
|
1690
|
-
"""
|
|
1691
|
-
try:
|
|
1692
|
-
for log in self:
|
|
1693
|
-
print(log)
|
|
1694
|
-
if until_completed and FUNCTION_END_LINE in log:
|
|
1695
|
-
break
|
|
1696
|
-
except KeyboardInterrupt:
|
|
1697
|
-
return
|
|
1698
|
-
|
|
1699
|
-
def __iter__(self):
|
|
1700
|
-
while not self.stop:
|
|
1701
|
-
for log in self.logs:
|
|
1702
|
-
yield '{}: {}'.format(log.get('timestamp', self.start), log.get('message', '').strip())
|
|
1703
|
-
self.get_next_log()
|
|
1704
|
-
|
|
1
|
+
import datetime
|
|
2
|
+
import inspect
|
|
3
|
+
import logging
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import tempfile
|
|
7
|
+
import time
|
|
8
|
+
from typing import Union, List, Callable
|
|
9
|
+
from .. import miscellaneous, exceptions, entities, repositories, assets, ApiClient, _api_reference
|
|
10
|
+
from ..__version__ import version as __version__
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(name='dtlpy')
|
|
13
|
+
FUNCTION_END_LINE = '[Done] Executing function.'
|
|
14
|
+
MAX_WAIT_TIME = 8
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Services:
|
|
18
|
+
"""
|
|
19
|
+
Services Repository
|
|
20
|
+
|
|
21
|
+
The Services class allows the user to manage services and their properties. Services are created from the packages users create.
|
|
22
|
+
See our documentation for more information about `services <https://developers.dataloop.ai/tutorials/faas/advance/chapter/>`_.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self,
|
|
26
|
+
client_api: ApiClient,
|
|
27
|
+
project: entities.Project = None,
|
|
28
|
+
package: Union[entities.Package, entities.Dpk] = None,
|
|
29
|
+
project_id=None,
|
|
30
|
+
model_id=None,
|
|
31
|
+
model: entities.Model = None):
|
|
32
|
+
self._client_api = client_api
|
|
33
|
+
self._package = package
|
|
34
|
+
self._project = project
|
|
35
|
+
self._model = model
|
|
36
|
+
if model_id is None:
|
|
37
|
+
if model is not None:
|
|
38
|
+
model_id = model.id
|
|
39
|
+
self._model_id = model_id
|
|
40
|
+
if project_id is None:
|
|
41
|
+
if project is not None:
|
|
42
|
+
project_id = project.id
|
|
43
|
+
self._project_id = project_id
|
|
44
|
+
self._settings = repositories.Settings(project=project, client_api=client_api)
|
|
45
|
+
|
|
46
|
+
############
|
|
47
|
+
# entities #
|
|
48
|
+
############
|
|
49
|
+
@property
|
|
50
|
+
def package(self) -> entities.Package:
|
|
51
|
+
if self._package is None:
|
|
52
|
+
raise exceptions.PlatformException(
|
|
53
|
+
error='2001',
|
|
54
|
+
message='Cannot perform action WITHOUT package entity in services repository. Please set a package')
|
|
55
|
+
assert (isinstance(self._package, entities.Package) or isinstance(self._package, entities.Dpk))
|
|
56
|
+
return self._package
|
|
57
|
+
|
|
58
|
+
@package.setter
|
|
59
|
+
def package(self, package: Union[entities.Package, entities.Dpk]):
|
|
60
|
+
if not isinstance(package, entities.Package) and not isinstance(package, entities.Dpk):
|
|
61
|
+
raise ValueError('Must input a valid package entity')
|
|
62
|
+
self._package = package
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def project(self) -> entities.Project:
|
|
66
|
+
if self._project is None:
|
|
67
|
+
# try to get from package
|
|
68
|
+
if self._package is not None:
|
|
69
|
+
self._project = self._package._project
|
|
70
|
+
|
|
71
|
+
if self._project is None:
|
|
72
|
+
# try to get checked out project
|
|
73
|
+
project = self._client_api.state_io.get('project')
|
|
74
|
+
if project is not None:
|
|
75
|
+
self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
|
|
76
|
+
if self._project is None:
|
|
77
|
+
raise exceptions.PlatformException(
|
|
78
|
+
error='2001',
|
|
79
|
+
message='Cannot perform action WITHOUT Project entity in services repository. Please set a project')
|
|
80
|
+
return self._project
|
|
81
|
+
|
|
82
|
+
@project.setter
|
|
83
|
+
def project(self, project: entities.Project):
|
|
84
|
+
if not isinstance(project, entities.Project):
|
|
85
|
+
raise ValueError('Must input a valid Project entity')
|
|
86
|
+
self._project = project
|
|
87
|
+
self._project_id = project.id
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def model(self) -> entities.Model:
|
|
91
|
+
if self._model is None:
|
|
92
|
+
if self._model_id is not None:
|
|
93
|
+
self._model = self.project.models.get(model_id=self._model_id)
|
|
94
|
+
else:
|
|
95
|
+
raise exceptions.PlatformException(
|
|
96
|
+
error='2001',
|
|
97
|
+
message='Cannot perform action WITHOUT model entity in services repository. Please set a model')
|
|
98
|
+
assert isinstance(self._model, entities.Model)
|
|
99
|
+
return self._model
|
|
100
|
+
|
|
101
|
+
@model.setter
|
|
102
|
+
def model(self, model: entities.Model):
|
|
103
|
+
if not isinstance(model, entities.Model):
|
|
104
|
+
raise ValueError('Must input a valid model entity')
|
|
105
|
+
self._model = model
|
|
106
|
+
self._model_id = model.id
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def platform_url(self):
|
|
110
|
+
return self._client_api._get_resource_url("projects/{}/services".format(self.project.id))
|
|
111
|
+
|
|
112
|
+
def open_in_web(self,
|
|
113
|
+
service: entities.Service = None,
|
|
114
|
+
service_id: str = None,
|
|
115
|
+
service_name: str = None):
|
|
116
|
+
"""
|
|
117
|
+
Open the service in web platform
|
|
118
|
+
|
|
119
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
120
|
+
|
|
121
|
+
:param str service_name: service name
|
|
122
|
+
:param str service_id: service id
|
|
123
|
+
:param dtlpy.entities.service.Service service: service entity
|
|
124
|
+
|
|
125
|
+
**Example**:
|
|
126
|
+
|
|
127
|
+
.. code-block:: python
|
|
128
|
+
|
|
129
|
+
package.services.open_in_web(service_id='service_id')
|
|
130
|
+
"""
|
|
131
|
+
if service_name is not None:
|
|
132
|
+
service = self.get(service_name=service_name)
|
|
133
|
+
if service is not None:
|
|
134
|
+
service.open_in_web()
|
|
135
|
+
elif service_id is not None:
|
|
136
|
+
self._client_api._open_in_web(url=self.platform_url + '/' + str(service_id) + '/main')
|
|
137
|
+
else:
|
|
138
|
+
self._client_api._open_in_web(url=self.platform_url)
|
|
139
|
+
|
|
140
|
+
def __get_from_cache(self) -> entities.Service:
|
|
141
|
+
service = self._client_api.state_io.get('service')
|
|
142
|
+
if service is not None:
|
|
143
|
+
service = entities.Service.from_json(_json=service,
|
|
144
|
+
client_api=self._client_api,
|
|
145
|
+
project=self._project,
|
|
146
|
+
package=self._package)
|
|
147
|
+
return service
|
|
148
|
+
|
|
149
|
+
def checkout(self,
|
|
150
|
+
service: entities.Service = None,
|
|
151
|
+
service_name: str = None,
|
|
152
|
+
service_id: str = None):
|
|
153
|
+
"""
|
|
154
|
+
Checkout (switch) to a service.
|
|
155
|
+
|
|
156
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
157
|
+
|
|
158
|
+
:param dtlpy.entities.service.Service service: Service entity
|
|
159
|
+
:param str service_name: service name
|
|
160
|
+
:param str service_id: service id
|
|
161
|
+
|
|
162
|
+
**Example**:
|
|
163
|
+
|
|
164
|
+
.. code-block:: python
|
|
165
|
+
|
|
166
|
+
package.services.checkout(service_id='service_id')
|
|
167
|
+
"""
|
|
168
|
+
if service is None:
|
|
169
|
+
service = self.get(service_name=service_name, service_id=service_id)
|
|
170
|
+
self._client_api.state_io.put('service', service.to_json())
|
|
171
|
+
logger.info('Checked out to service {}'.format(service.name))
|
|
172
|
+
|
|
173
|
+
###########
|
|
174
|
+
# methods #
|
|
175
|
+
###########
|
|
176
|
+
@_api_reference.add(path='/services/{id}/revisions', method='get')
|
|
177
|
+
def revisions(self,
|
|
178
|
+
service: entities.Service = None,
|
|
179
|
+
service_id: str = None):
|
|
180
|
+
"""
|
|
181
|
+
Get service revisions history.
|
|
182
|
+
|
|
183
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
184
|
+
|
|
185
|
+
You must provide at leats ONE of the following params: service, service_id
|
|
186
|
+
|
|
187
|
+
:param dtlpy.entities.service.Service service: Service entity
|
|
188
|
+
:param str service_id: service id
|
|
189
|
+
|
|
190
|
+
**Example**:
|
|
191
|
+
|
|
192
|
+
.. code-block:: python
|
|
193
|
+
|
|
194
|
+
service_revision = package.services.revisions(service_id='service_id')
|
|
195
|
+
"""
|
|
196
|
+
if service is None and service_id is None:
|
|
197
|
+
raise exceptions.PlatformException(
|
|
198
|
+
error='400',
|
|
199
|
+
message='must provide an identifier in inputs: "service" or "service_id"')
|
|
200
|
+
if service is not None:
|
|
201
|
+
service_id = service.id
|
|
202
|
+
|
|
203
|
+
success, response = self._client_api.gen_request(
|
|
204
|
+
req_type="get",
|
|
205
|
+
path="/services/{}/revisions".format(service_id))
|
|
206
|
+
if not success:
|
|
207
|
+
raise exceptions.PlatformException(response)
|
|
208
|
+
return response.json()
|
|
209
|
+
|
|
210
|
+
@_api_reference.add(path='/services/{id}', method='get')
|
|
211
|
+
def get(self,
|
|
212
|
+
service_name=None,
|
|
213
|
+
service_id=None,
|
|
214
|
+
checkout=False,
|
|
215
|
+
fetch=None
|
|
216
|
+
) -> entities.Service:
|
|
217
|
+
"""
|
|
218
|
+
Get service to use in your code.
|
|
219
|
+
|
|
220
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
221
|
+
|
|
222
|
+
:param str service_name: optional - search by name
|
|
223
|
+
:param str service_id: optional - search by id
|
|
224
|
+
:param bool checkout: if true, checkout (switch) to service
|
|
225
|
+
:param fetch: optional - fetch entity from platform, default taken from cookie
|
|
226
|
+
:return: Service object
|
|
227
|
+
:rtype: dtlpy.entities.service.Service
|
|
228
|
+
|
|
229
|
+
**Example**:
|
|
230
|
+
|
|
231
|
+
.. code-block:: python
|
|
232
|
+
|
|
233
|
+
service = package.services.get(service_id='service_id')
|
|
234
|
+
"""
|
|
235
|
+
if fetch is None:
|
|
236
|
+
fetch = self._client_api.fetch_entities
|
|
237
|
+
|
|
238
|
+
if service_name is None and service_id is None:
|
|
239
|
+
service = self.__get_from_cache()
|
|
240
|
+
if service is None:
|
|
241
|
+
raise exceptions.PlatformException(
|
|
242
|
+
error='400',
|
|
243
|
+
message='No checked-out Service was found, must checkout or provide an identifier in inputs')
|
|
244
|
+
|
|
245
|
+
elif fetch:
|
|
246
|
+
if service_id is not None:
|
|
247
|
+
success, response = self._client_api.gen_request(
|
|
248
|
+
req_type="get",
|
|
249
|
+
path="/services/{}".format(service_id)
|
|
250
|
+
)
|
|
251
|
+
if not success:
|
|
252
|
+
raise exceptions.PlatformException(response)
|
|
253
|
+
service = entities.Service.from_json(client_api=self._client_api,
|
|
254
|
+
_json=response.json(),
|
|
255
|
+
package=self._package,
|
|
256
|
+
project=self._project)
|
|
257
|
+
# verify input service name is same as the given id
|
|
258
|
+
if service_name is not None and service.name != service_name:
|
|
259
|
+
logger.warning(
|
|
260
|
+
"Mismatch found in services.get: service_name is different then service.name:"
|
|
261
|
+
" {!r} != {!r}".format(
|
|
262
|
+
service_name,
|
|
263
|
+
service.name))
|
|
264
|
+
elif service_name is not None:
|
|
265
|
+
filters = entities.Filters(resource=entities.FiltersResource.SERVICE,
|
|
266
|
+
field='name',
|
|
267
|
+
values=service_name,
|
|
268
|
+
use_defaults=False)
|
|
269
|
+
if self._project_id is not None:
|
|
270
|
+
filters.add(field='projectId', values=self._project_id)
|
|
271
|
+
if self._package is not None:
|
|
272
|
+
filters.add(field='packageId', values=self._package.id)
|
|
273
|
+
services = self.list(filters=filters)
|
|
274
|
+
if services.items_count > 1:
|
|
275
|
+
raise exceptions.PlatformException('404', 'More than one service with same name. '
|
|
276
|
+
'Please get services from package/project entity')
|
|
277
|
+
elif services.items_count == 0:
|
|
278
|
+
raise exceptions.PlatformException('404', 'Service not found: {}.'.format(service_name))
|
|
279
|
+
service = services.items[0]
|
|
280
|
+
else:
|
|
281
|
+
raise exceptions.PlatformException(
|
|
282
|
+
error='400',
|
|
283
|
+
message='No checked-out Service was found, must checkout or provide an identifier in inputs')
|
|
284
|
+
else:
|
|
285
|
+
service = entities.Service.from_json(_json={'id': service_id,
|
|
286
|
+
'name': service_name},
|
|
287
|
+
client_api=self._client_api,
|
|
288
|
+
project=self._project,
|
|
289
|
+
package=self._package,
|
|
290
|
+
is_fetched=False)
|
|
291
|
+
|
|
292
|
+
assert isinstance(service, entities.Service)
|
|
293
|
+
if checkout:
|
|
294
|
+
self.checkout(service=service)
|
|
295
|
+
return service
|
|
296
|
+
|
|
297
|
+
def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Service]:
|
|
298
|
+
jobs = [None for _ in range(len(response_items))]
|
|
299
|
+
pool = self._client_api.thread_pools(pool_name='entity.create')
|
|
300
|
+
|
|
301
|
+
# return triggers list
|
|
302
|
+
for i_service, service in enumerate(response_items):
|
|
303
|
+
jobs[i_service] = pool.submit(entities.Service._protected_from_json,
|
|
304
|
+
**{'client_api': self._client_api,
|
|
305
|
+
'_json': service,
|
|
306
|
+
'package': self._package,
|
|
307
|
+
'project': self._project})
|
|
308
|
+
|
|
309
|
+
# get all results
|
|
310
|
+
results = [j.result() for j in jobs]
|
|
311
|
+
# log errors
|
|
312
|
+
_ = [logger.warning(r[1]) for r in results if r[0] is False]
|
|
313
|
+
# return good jobs
|
|
314
|
+
return miscellaneous.List([r[1] for r in results if r[0] is True])
|
|
315
|
+
|
|
316
|
+
def _list(self, filters: entities.Filters):
|
|
317
|
+
url = '/query/faas'
|
|
318
|
+
success, response = self._client_api.gen_request(req_type='POST',
|
|
319
|
+
path=url,
|
|
320
|
+
json_req=filters.prepare())
|
|
321
|
+
if not success:
|
|
322
|
+
raise exceptions.PlatformException(response)
|
|
323
|
+
|
|
324
|
+
return response.json()
|
|
325
|
+
|
|
326
|
+
@_api_reference.add(path='/query/faas', method='post')
|
|
327
|
+
def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
|
|
328
|
+
"""
|
|
329
|
+
List all services (services can be listed for a package or for a project).
|
|
330
|
+
|
|
331
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
332
|
+
|
|
333
|
+
:param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
|
|
334
|
+
:return: Paged entity
|
|
335
|
+
:rtype: dtlpy.entities.paged_entities.PagedEntities
|
|
336
|
+
|
|
337
|
+
**Example**:
|
|
338
|
+
|
|
339
|
+
.. code-block:: python
|
|
340
|
+
|
|
341
|
+
services = package.services.list()
|
|
342
|
+
"""
|
|
343
|
+
# default filters
|
|
344
|
+
if filters is None:
|
|
345
|
+
filters = entities.Filters(resource=entities.FiltersResource.SERVICE)
|
|
346
|
+
# assert type filters
|
|
347
|
+
elif not isinstance(filters, entities.Filters):
|
|
348
|
+
raise exceptions.PlatformException(error='400',
|
|
349
|
+
message='Unknown filters type: {!r}'.format(type(filters)))
|
|
350
|
+
if filters.resource != entities.FiltersResource.SERVICE:
|
|
351
|
+
raise exceptions.PlatformException(
|
|
352
|
+
error='400',
|
|
353
|
+
message='Filters resource must to be FiltersResource.SERVICE. Got: {!r}'.format(filters.resource))
|
|
354
|
+
if self._project_id is not None:
|
|
355
|
+
filters.add(field='projectId', values=self._project_id)
|
|
356
|
+
if self._model_id is not None:
|
|
357
|
+
filters.add(field='metadata.ml.modelId', values=self._model_id)
|
|
358
|
+
if self._package is not None:
|
|
359
|
+
filters.add(field='packageId', values=self._package.id)
|
|
360
|
+
|
|
361
|
+
# assert type filters
|
|
362
|
+
if not isinstance(filters, entities.Filters):
|
|
363
|
+
raise exceptions.PlatformException('400', 'Unknown filters type')
|
|
364
|
+
|
|
365
|
+
paged = entities.PagedEntities(items_repository=self,
|
|
366
|
+
filters=filters,
|
|
367
|
+
page_offset=filters.page,
|
|
368
|
+
page_size=filters.page_size,
|
|
369
|
+
client_api=self._client_api)
|
|
370
|
+
paged.get_page()
|
|
371
|
+
return paged
|
|
372
|
+
|
|
373
|
+
@_api_reference.add(path='/services/{id}/status', method='post')
|
|
374
|
+
def status(self, service_name=None, service_id=None):
|
|
375
|
+
"""
|
|
376
|
+
Get service status.
|
|
377
|
+
|
|
378
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
379
|
+
|
|
380
|
+
You must provide at least ONE of the following params: service_id, service_name
|
|
381
|
+
|
|
382
|
+
:param str service_name: service name
|
|
383
|
+
:param str service_id: service id
|
|
384
|
+
:return: status json
|
|
385
|
+
:rtype: dict
|
|
386
|
+
|
|
387
|
+
**Example**:
|
|
388
|
+
|
|
389
|
+
.. code-block:: python
|
|
390
|
+
|
|
391
|
+
status_json = package.services.status(service_id='service_id')
|
|
392
|
+
"""
|
|
393
|
+
if service_id is None:
|
|
394
|
+
if service_name is None:
|
|
395
|
+
raise exceptions.PlatformException(error='400',
|
|
396
|
+
message='must input "service_name" or "service_id" to get status')
|
|
397
|
+
service = self.get(service_name=service_name)
|
|
398
|
+
service_id = service.id
|
|
399
|
+
# request
|
|
400
|
+
success, response = self._client_api.gen_request(req_type="get",
|
|
401
|
+
path="/services/{}/status".format(service_id))
|
|
402
|
+
if not success:
|
|
403
|
+
raise exceptions.PlatformException(response)
|
|
404
|
+
return response.json()
|
|
405
|
+
|
|
406
|
+
@_api_reference.add(path='/services/{id}/stop', method='post')
|
|
407
|
+
def pause(self,
|
|
408
|
+
service_name: str = None,
|
|
409
|
+
service_id: str = None,
|
|
410
|
+
force: bool = False):
|
|
411
|
+
"""
|
|
412
|
+
Pause service.
|
|
413
|
+
|
|
414
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
415
|
+
|
|
416
|
+
You must provide at least ONE of the following params: service_id, service_name
|
|
417
|
+
|
|
418
|
+
:param str service_name: service name
|
|
419
|
+
:param str service_id: service id
|
|
420
|
+
:param bool force: optional - terminate old replicas immediately
|
|
421
|
+
:return: True if success
|
|
422
|
+
:rtype: bool
|
|
423
|
+
|
|
424
|
+
**Example**:
|
|
425
|
+
|
|
426
|
+
.. code-block:: python
|
|
427
|
+
|
|
428
|
+
success = package.services.pause(service_id='service_id')
|
|
429
|
+
"""
|
|
430
|
+
if service_id is None:
|
|
431
|
+
if service_name is None:
|
|
432
|
+
raise exceptions.PlatformException(error='400',
|
|
433
|
+
message='must input "service_name" or "service_id" to pause service')
|
|
434
|
+
service = self.get(service_name=service_name)
|
|
435
|
+
service_id = service.id
|
|
436
|
+
# request
|
|
437
|
+
url = "/services/{}/stop".format(service_id)
|
|
438
|
+
if force:
|
|
439
|
+
url = '{}?force=true'
|
|
440
|
+
success, response = self._client_api.gen_request(req_type="post",
|
|
441
|
+
path=url)
|
|
442
|
+
if not success:
|
|
443
|
+
raise exceptions.PlatformException(response)
|
|
444
|
+
return success
|
|
445
|
+
|
|
446
|
+
def _notify(
|
|
447
|
+
self,
|
|
448
|
+
service_id: str,
|
|
449
|
+
message: str,
|
|
450
|
+
name: str,
|
|
451
|
+
action: str = 'created',
|
|
452
|
+
support: str = None,
|
|
453
|
+
docs: str = None,
|
|
454
|
+
agent_info: dict = None
|
|
455
|
+
):
|
|
456
|
+
url = "/services/{}/notify".format(service_id)
|
|
457
|
+
payload = {
|
|
458
|
+
'action': action,
|
|
459
|
+
'message': message,
|
|
460
|
+
'notificationName': name
|
|
461
|
+
}
|
|
462
|
+
if agent_info is not None:
|
|
463
|
+
payload['agentInfo'] = agent_info
|
|
464
|
+
|
|
465
|
+
if support:
|
|
466
|
+
payload['support'] = support
|
|
467
|
+
|
|
468
|
+
if docs:
|
|
469
|
+
payload['docs'] = docs
|
|
470
|
+
|
|
471
|
+
success, response = self._client_api.gen_request(
|
|
472
|
+
req_type="post",
|
|
473
|
+
path=url,
|
|
474
|
+
json_req=payload
|
|
475
|
+
)
|
|
476
|
+
if not success:
|
|
477
|
+
raise exceptions.PlatformException(response)
|
|
478
|
+
return success
|
|
479
|
+
|
|
480
|
+
@_api_reference.add(path='/services/{id}/resume', method='post')
|
|
481
|
+
def resume(self,
|
|
482
|
+
service_name: str = None,
|
|
483
|
+
service_id: str = None,
|
|
484
|
+
force: bool = False):
|
|
485
|
+
"""
|
|
486
|
+
Resume service.
|
|
487
|
+
|
|
488
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
489
|
+
|
|
490
|
+
You must provide at least ONE of the following params: service_id, service_name.
|
|
491
|
+
|
|
492
|
+
:param str service_name: service name
|
|
493
|
+
:param str service_id: service id
|
|
494
|
+
:param bool force: optional - terminate old replicas immediately
|
|
495
|
+
:return: json of the service
|
|
496
|
+
:rtype: dict
|
|
497
|
+
|
|
498
|
+
**Example**:
|
|
499
|
+
|
|
500
|
+
.. code-block:: python
|
|
501
|
+
|
|
502
|
+
service_json = package.services.resume(service_id='service_id')
|
|
503
|
+
"""
|
|
504
|
+
if service_id is None:
|
|
505
|
+
if service_name is None:
|
|
506
|
+
raise exceptions.PlatformException(error='400',
|
|
507
|
+
message='must input "service_name" or "service_id" to resume')
|
|
508
|
+
service = self.get(service_name=service_name)
|
|
509
|
+
service_id = service.id
|
|
510
|
+
# request
|
|
511
|
+
url = "/services/{}/resume".format(service_id)
|
|
512
|
+
if force:
|
|
513
|
+
url = '{}?force=true'
|
|
514
|
+
success, response = self._client_api.gen_request(req_type="post",
|
|
515
|
+
path=url)
|
|
516
|
+
if not success:
|
|
517
|
+
raise exceptions.PlatformException(response)
|
|
518
|
+
return response.json()
|
|
519
|
+
|
|
520
|
+
def _get_bot_email(self, bot=None):
|
|
521
|
+
|
|
522
|
+
if bot is None:
|
|
523
|
+
project = self._project
|
|
524
|
+
if project is None and self._project_id is not None:
|
|
525
|
+
project = repositories.Projects(client_api=self._client_api).get(project_id=self._project_id)
|
|
526
|
+
|
|
527
|
+
if project is None:
|
|
528
|
+
raise exceptions.PlatformException(error='2001',
|
|
529
|
+
message='Need project entity or bot to perform this action')
|
|
530
|
+
bots = project.bots.list()
|
|
531
|
+
if len(bots) == 0:
|
|
532
|
+
logger.info('Bot not found for project. Creating a default bot')
|
|
533
|
+
bot = project.bots.create(name='default')
|
|
534
|
+
else:
|
|
535
|
+
bot = bots[0]
|
|
536
|
+
if len(bots) > 1:
|
|
537
|
+
logger.debug('More than one bot users. Choosing first. email: {}'.format(bots[0].email))
|
|
538
|
+
|
|
539
|
+
if isinstance(bot, str):
|
|
540
|
+
bot_email = bot
|
|
541
|
+
elif isinstance(bot, entities.Bot) or isinstance(bot, entities.User):
|
|
542
|
+
bot_email = bot.email
|
|
543
|
+
else:
|
|
544
|
+
raise ValueError('input "bot" must be a str or a Bot type, got: {}'.format(type(bot)))
|
|
545
|
+
|
|
546
|
+
return bot_email
|
|
547
|
+
|
|
548
|
+
@staticmethod
|
|
549
|
+
def _parse_init_input(init_input):
|
|
550
|
+
if not isinstance(init_input, dict):
|
|
551
|
+
if init_input is None:
|
|
552
|
+
init_input = dict()
|
|
553
|
+
else:
|
|
554
|
+
init_params = dict()
|
|
555
|
+
if not isinstance(init_input, list):
|
|
556
|
+
init_input = [init_input]
|
|
557
|
+
for param in init_input:
|
|
558
|
+
if isinstance(param, entities.FunctionIO):
|
|
559
|
+
init_params.update(param.to_json(resource='service'))
|
|
560
|
+
else:
|
|
561
|
+
raise exceptions.PlatformException('400', 'Unknown type of init params')
|
|
562
|
+
init_input = init_params
|
|
563
|
+
|
|
564
|
+
return init_input
|
|
565
|
+
|
|
566
|
+
def name_validation(self, name: str):
|
|
567
|
+
"""
|
|
568
|
+
Validation service name.
|
|
569
|
+
|
|
570
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*.
|
|
571
|
+
|
|
572
|
+
:param str name: service name
|
|
573
|
+
|
|
574
|
+
**Example**:
|
|
575
|
+
|
|
576
|
+
.. code-block:: python
|
|
577
|
+
|
|
578
|
+
package.services.name_validation(name='name')
|
|
579
|
+
"""
|
|
580
|
+
url = '/piper-misc/naming/services/{}'.format(name)
|
|
581
|
+
|
|
582
|
+
# request
|
|
583
|
+
success, response = self._client_api.gen_request(req_type='get',
|
|
584
|
+
path=url)
|
|
585
|
+
if not success:
|
|
586
|
+
raise exceptions.PlatformException(response)
|
|
587
|
+
|
|
588
|
+
def _create(self,
|
|
589
|
+
service_name: str = None,
|
|
590
|
+
package: entities.Package = None,
|
|
591
|
+
module_name: str = None,
|
|
592
|
+
bot: Union[entities.Bot, str] = None,
|
|
593
|
+
revision: str or int = None,
|
|
594
|
+
init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
|
|
595
|
+
runtime: Union[entities.KubernetesRuntime, dict] = None,
|
|
596
|
+
pod_type: entities.InstanceCatalog = None,
|
|
597
|
+
project_id: str = None,
|
|
598
|
+
sdk_version: str = None,
|
|
599
|
+
agent_versions: dict = None,
|
|
600
|
+
verify: bool = True,
|
|
601
|
+
driver_id: str = None,
|
|
602
|
+
run_execution_as_process: bool = None,
|
|
603
|
+
execution_timeout: int = None,
|
|
604
|
+
drain_time: int = None,
|
|
605
|
+
on_reset: str = None,
|
|
606
|
+
max_attempts: int = None,
|
|
607
|
+
secrets=None,
|
|
608
|
+
integrations=None,
|
|
609
|
+
active: bool = True,
|
|
610
|
+
**kwargs
|
|
611
|
+
) -> entities.Service:
|
|
612
|
+
"""
|
|
613
|
+
Create service entity.
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
:param str service_name: service name
|
|
617
|
+
:param dtlpy.entities.package.Package package: package entity
|
|
618
|
+
:param str module_name: module name
|
|
619
|
+
:param str bot: bot email
|
|
620
|
+
:param str revision: package revision - default=latest
|
|
621
|
+
:param init_input: config to run at startup
|
|
622
|
+
:param dict runtime: runtime resources
|
|
623
|
+
:param str pod_type: pod type dl.InstanceCatalog
|
|
624
|
+
:param str project_id: project id
|
|
625
|
+
:param str sdk_version: - optional - string - sdk version
|
|
626
|
+
:param dict agent_versions: - dictionary - - optional -versions of sdk, agent runner and agent proxy
|
|
627
|
+
:param bool verify: verify the inputs
|
|
628
|
+
:param str driver_id: driver id
|
|
629
|
+
:param bool run_execution_as_process: run execution as process
|
|
630
|
+
:param int execution_timeout: execution timeout
|
|
631
|
+
:param int drain_time: drain time
|
|
632
|
+
:param str on_reset: on reset
|
|
633
|
+
:param int max_attempts: Maximum execution retries in-case of a service reset
|
|
634
|
+
:param bool force: optional - terminate old replicas immediately
|
|
635
|
+
:param list secrets: list of the integrations ids
|
|
636
|
+
:param list integrations: list of the integrations
|
|
637
|
+
:param bool active: optional - if true, service will be active
|
|
638
|
+
:param kwargs:
|
|
639
|
+
:return: Service object
|
|
640
|
+
:rtype: dtlpy.entities.service.Service
|
|
641
|
+
"""
|
|
642
|
+
if package is None:
|
|
643
|
+
if self._package is None:
|
|
644
|
+
raise exceptions.PlatformException('400', 'Please provide param package')
|
|
645
|
+
package = self._package
|
|
646
|
+
|
|
647
|
+
if verify is not None:
|
|
648
|
+
logger.warning('verify attribute has been deprecated and will be ignored')
|
|
649
|
+
|
|
650
|
+
is_global = kwargs.get('is_global', None)
|
|
651
|
+
jwt_forward = kwargs.get('jwt_forward', None)
|
|
652
|
+
|
|
653
|
+
if is_global is not None or jwt_forward is not None:
|
|
654
|
+
logger.warning(
|
|
655
|
+
'Params jwt_forward and is_global are restricted to superuser. '
|
|
656
|
+
'If you are not a superuser this action will not work'
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
service_config = dict()
|
|
660
|
+
if package is not None and package.service_config is not None:
|
|
661
|
+
service_config = package.service_config
|
|
662
|
+
|
|
663
|
+
if agent_versions is None:
|
|
664
|
+
if sdk_version is None:
|
|
665
|
+
sdk_version = service_config.get('versions', dict()).get('dtlpy', __version__)
|
|
666
|
+
agent_versions = {
|
|
667
|
+
"dtlpy": sdk_version
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
if project_id is None:
|
|
671
|
+
if self._project is None and self._project_id is None:
|
|
672
|
+
raise exceptions.PlatformException('400', 'Please provide project id')
|
|
673
|
+
elif self._project_id is not None:
|
|
674
|
+
project_id = self._project_id
|
|
675
|
+
elif self._project is not None:
|
|
676
|
+
project_id = self._project.id
|
|
677
|
+
|
|
678
|
+
if service_name is None:
|
|
679
|
+
service_name = 'default-service'
|
|
680
|
+
|
|
681
|
+
# payload
|
|
682
|
+
payload = {
|
|
683
|
+
'name': service_name,
|
|
684
|
+
'projectId': project_id,
|
|
685
|
+
'packageId': package.id,
|
|
686
|
+
'initParams': self._parse_init_input(init_input=init_input),
|
|
687
|
+
'botUserName': self._get_bot_email(bot=bot),
|
|
688
|
+
'versions': agent_versions,
|
|
689
|
+
'packageRevision': revision if revision is not None else package.version,
|
|
690
|
+
'driverId': driver_id,
|
|
691
|
+
'active': active
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
if secrets is not None:
|
|
695
|
+
if not isinstance(secrets, list):
|
|
696
|
+
secrets = [secrets]
|
|
697
|
+
payload['secrets'] = secrets
|
|
698
|
+
|
|
699
|
+
if integrations is not None:
|
|
700
|
+
if not isinstance(integrations, list):
|
|
701
|
+
integrations = [integrations]
|
|
702
|
+
payload['integrations'] = integrations
|
|
703
|
+
|
|
704
|
+
if runtime is not None:
|
|
705
|
+
if isinstance(runtime, entities.KubernetesRuntime):
|
|
706
|
+
runtime = runtime.to_json()
|
|
707
|
+
|
|
708
|
+
if pod_type is not None:
|
|
709
|
+
if runtime is None:
|
|
710
|
+
runtime = {'podType': pod_type}
|
|
711
|
+
else:
|
|
712
|
+
runtime['podType'] = pod_type
|
|
713
|
+
|
|
714
|
+
if runtime is not None:
|
|
715
|
+
payload['runtime'] = runtime
|
|
716
|
+
|
|
717
|
+
if module_name is not None:
|
|
718
|
+
payload['moduleName'] = module_name
|
|
719
|
+
|
|
720
|
+
if is_global is not None:
|
|
721
|
+
payload['global'] = is_global
|
|
722
|
+
|
|
723
|
+
if max_attempts is not None:
|
|
724
|
+
payload['maxAttempts'] = max_attempts
|
|
725
|
+
|
|
726
|
+
if jwt_forward is not None:
|
|
727
|
+
payload['useUserJwt'] = jwt_forward
|
|
728
|
+
|
|
729
|
+
if run_execution_as_process is not None:
|
|
730
|
+
payload['runExecutionAsProcess'] = run_execution_as_process
|
|
731
|
+
|
|
732
|
+
if drain_time is not None:
|
|
733
|
+
payload['drainTime'] = drain_time
|
|
734
|
+
|
|
735
|
+
if on_reset is not None:
|
|
736
|
+
payload['onReset'] = on_reset
|
|
737
|
+
|
|
738
|
+
if execution_timeout is not None:
|
|
739
|
+
payload['executionTimeout'] = execution_timeout
|
|
740
|
+
|
|
741
|
+
# request
|
|
742
|
+
success, response = self._client_api.gen_request(
|
|
743
|
+
req_type='post',
|
|
744
|
+
path='/services',
|
|
745
|
+
json_req=payload
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
# exception handling
|
|
749
|
+
if not success:
|
|
750
|
+
raise exceptions.PlatformException(response)
|
|
751
|
+
|
|
752
|
+
# return entity
|
|
753
|
+
return entities.Service.from_json(
|
|
754
|
+
_json=response.json(),
|
|
755
|
+
client_api=self._client_api,
|
|
756
|
+
package=package,
|
|
757
|
+
project=self._project
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
@_api_reference.add(path='/services/{id}', method='delete')
|
|
761
|
+
def delete(self, service_name: str = None, service_id: str = None, force=False):
|
|
762
|
+
"""
|
|
763
|
+
Delete Service object
|
|
764
|
+
|
|
765
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
766
|
+
|
|
767
|
+
You must provide at least ONE of the following params: service_id, service_name.
|
|
768
|
+
|
|
769
|
+
:param force:
|
|
770
|
+
:param str service_name: by name
|
|
771
|
+
:param str service_id: by id
|
|
772
|
+
:return: True
|
|
773
|
+
:rtype: bool
|
|
774
|
+
|
|
775
|
+
**Example**:
|
|
776
|
+
|
|
777
|
+
.. code-block:: python
|
|
778
|
+
|
|
779
|
+
is_deleted = package.services.delete(service_id='service_id')
|
|
780
|
+
"""
|
|
781
|
+
# get bby name
|
|
782
|
+
if service_id is None:
|
|
783
|
+
if service_name is None:
|
|
784
|
+
raise exceptions.PlatformException('400', 'Must provide either service id or service name')
|
|
785
|
+
else:
|
|
786
|
+
service_id = self.get(service_name=service_name).id
|
|
787
|
+
|
|
788
|
+
path = "/services/{}".format(service_id)
|
|
789
|
+
if force:
|
|
790
|
+
path = '{}?force=true'.format(path)
|
|
791
|
+
|
|
792
|
+
# request
|
|
793
|
+
success, response = self._client_api.gen_request(
|
|
794
|
+
req_type="delete",
|
|
795
|
+
path=path
|
|
796
|
+
)
|
|
797
|
+
if not success:
|
|
798
|
+
raise exceptions.PlatformException(response)
|
|
799
|
+
return True
|
|
800
|
+
|
|
801
|
+
@_api_reference.add(path='/services/{id}', method='patch')
|
|
802
|
+
def update(self, service: entities.Service, force: bool = False) -> entities.Service:
|
|
803
|
+
"""
|
|
804
|
+
Update service changes to platform.
|
|
805
|
+
|
|
806
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
807
|
+
|
|
808
|
+
:param dtlpy.entities.service.Service service: Service entity
|
|
809
|
+
:param bool force: optional - terminate old replicas immediately
|
|
810
|
+
:return: Service entity
|
|
811
|
+
:rtype: dtlpy.entities.service.Service
|
|
812
|
+
|
|
813
|
+
**Example**:
|
|
814
|
+
|
|
815
|
+
.. code-block:: python
|
|
816
|
+
|
|
817
|
+
service = package.services.update(service='service_entity')
|
|
818
|
+
"""
|
|
819
|
+
assert isinstance(service, entities.Service)
|
|
820
|
+
|
|
821
|
+
# payload
|
|
822
|
+
payload = service.to_json()
|
|
823
|
+
|
|
824
|
+
# request
|
|
825
|
+
url = '/services/{}'.format(service.id)
|
|
826
|
+
if force:
|
|
827
|
+
url = '{}?force=true'.format(url)
|
|
828
|
+
success, response = self._client_api.gen_request(req_type='patch',
|
|
829
|
+
path=url,
|
|
830
|
+
json_req=payload)
|
|
831
|
+
|
|
832
|
+
# exception handling
|
|
833
|
+
if not success:
|
|
834
|
+
raise exceptions.PlatformException(response)
|
|
835
|
+
|
|
836
|
+
# return entity
|
|
837
|
+
if self._package is not None:
|
|
838
|
+
package = self._package
|
|
839
|
+
else:
|
|
840
|
+
package = service._package
|
|
841
|
+
|
|
842
|
+
return entities.Service.from_json(_json=response.json(),
|
|
843
|
+
client_api=self._client_api,
|
|
844
|
+
package=package,
|
|
845
|
+
project=self._project)
|
|
846
|
+
|
|
847
|
+
def activate_slots(
|
|
848
|
+
self,
|
|
849
|
+
service: entities.Service,
|
|
850
|
+
project_id: str = None,
|
|
851
|
+
task_id: str = None,
|
|
852
|
+
dataset_id: str = None,
|
|
853
|
+
org_id: str = None,
|
|
854
|
+
user_email: str = None,
|
|
855
|
+
slots: List[entities.PackageSlot] = None,
|
|
856
|
+
role=None,
|
|
857
|
+
prevent_override: bool = True,
|
|
858
|
+
visible: bool = True,
|
|
859
|
+
icon: str = 'fas fa-magic',
|
|
860
|
+
**kwargs
|
|
861
|
+
):
|
|
862
|
+
"""
|
|
863
|
+
Activate service slots (creates buttons in the UI that activate services).
|
|
864
|
+
|
|
865
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
866
|
+
|
|
867
|
+
:param dtlpy.entities.service.Service service: service entity
|
|
868
|
+
:param str project_id: project id
|
|
869
|
+
:param str task_id: task id
|
|
870
|
+
:param str dataset_id: dataset id
|
|
871
|
+
:param str org_id: org id
|
|
872
|
+
:param str user_email: user email
|
|
873
|
+
:param list slots: list of entities.PackageSlot
|
|
874
|
+
:param str role: user role MemberOrgRole.ADMIN, MemberOrgRole.owner, MemberOrgRole.MEMBER, MemberOrgRole.WORKER
|
|
875
|
+
:param bool prevent_override: True to prevent override
|
|
876
|
+
:param bool visible: visible
|
|
877
|
+
:param str icon: icon
|
|
878
|
+
:param kwargs: all additional arguments
|
|
879
|
+
:return: list of user setting for activated slots
|
|
880
|
+
:rtype: list
|
|
881
|
+
|
|
882
|
+
**Example**:
|
|
883
|
+
|
|
884
|
+
.. code-block:: python
|
|
885
|
+
|
|
886
|
+
setting = package.services.activate_slots(service='service_entity',
|
|
887
|
+
project_id='project_id',
|
|
888
|
+
slots=List[entities.PackageSlot],
|
|
889
|
+
icon='fas fa-magic')
|
|
890
|
+
"""
|
|
891
|
+
package = service.package
|
|
892
|
+
if not isinstance(package.slots, list) or len(package.slots) == 0:
|
|
893
|
+
raise exceptions.PlatformException('400', "Service's package has no slots")
|
|
894
|
+
|
|
895
|
+
if kwargs.get('is_global', False):
|
|
896
|
+
project_id = '*'
|
|
897
|
+
scope_ids = [project_id]
|
|
898
|
+
else:
|
|
899
|
+
scope_ids = [s_id for s_id in [project_id, task_id, org_id, dataset_id, user_email] if s_id is not None]
|
|
900
|
+
if len(scope_ids) == 0:
|
|
901
|
+
raise exceptions.PlatformException('400', "Must provide scope resource ID")
|
|
902
|
+
|
|
903
|
+
settings = list()
|
|
904
|
+
|
|
905
|
+
if role is None:
|
|
906
|
+
role = entities.Role.ALL
|
|
907
|
+
|
|
908
|
+
if not slots:
|
|
909
|
+
slots = [s.to_json() for s in service.package.slots]
|
|
910
|
+
elif isinstance(slots, list) and isinstance(slots[0], entities.PackageSlot):
|
|
911
|
+
slots = [s.to_json() for s in slots]
|
|
912
|
+
else:
|
|
913
|
+
raise exceptions.PlatformException('400', "Slots param must be a list of PackageSlot objects")
|
|
914
|
+
|
|
915
|
+
for scope_id in scope_ids:
|
|
916
|
+
|
|
917
|
+
if kwargs.get('is_global', False):
|
|
918
|
+
scope_type = entities.PlatformEntityType.DATALOOP
|
|
919
|
+
elif scope_id == project_id:
|
|
920
|
+
scope_type = entities.PlatformEntityType.PROJECT
|
|
921
|
+
elif scope_id == task_id:
|
|
922
|
+
scope_type = entities.PlatformEntityType.TASK
|
|
923
|
+
elif scope_id == dataset_id:
|
|
924
|
+
scope_type = entities.PlatformEntityType.DATASET
|
|
925
|
+
elif scope_id == user_email:
|
|
926
|
+
scope_type = entities.PlatformEntityType.USER
|
|
927
|
+
elif scope_id == org_id:
|
|
928
|
+
scope_type = entities.PlatformEntityType.ORG
|
|
929
|
+
else:
|
|
930
|
+
raise exceptions.PlatformException('400', "Unknown resource id")
|
|
931
|
+
|
|
932
|
+
setting = entities.Setting(
|
|
933
|
+
default_value=True,
|
|
934
|
+
value=True,
|
|
935
|
+
inputs=None,
|
|
936
|
+
name=service.name,
|
|
937
|
+
value_type=entities.SettingsValueTypes.BOOLEAN,
|
|
938
|
+
scope=entities.SettingScope(
|
|
939
|
+
type=scope_type,
|
|
940
|
+
id=scope_id,
|
|
941
|
+
role=role,
|
|
942
|
+
prevent_override=prevent_override,
|
|
943
|
+
visible=visible
|
|
944
|
+
),
|
|
945
|
+
metadata={
|
|
946
|
+
'serviceId': service.id,
|
|
947
|
+
'serviceName': service.name,
|
|
948
|
+
'projectId': service.project_id,
|
|
949
|
+
'slots': slots
|
|
950
|
+
},
|
|
951
|
+
description=service.name,
|
|
952
|
+
icon=icon,
|
|
953
|
+
section_name=entities.SettingsSectionNames.APPLICATIONS,
|
|
954
|
+
sub_section_name=None,
|
|
955
|
+
hint=None
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
try:
|
|
959
|
+
settings.append(self._settings.create(setting=setting))
|
|
960
|
+
except exceptions.BadRequest as err:
|
|
961
|
+
if 'settings already exist' in err.message:
|
|
962
|
+
old_setting = self._settings.get(setting_name=setting.name)
|
|
963
|
+
setting.id = old_setting.id
|
|
964
|
+
settings.append(self._settings.update(setting=setting))
|
|
965
|
+
else:
|
|
966
|
+
raise err
|
|
967
|
+
|
|
968
|
+
return settings
|
|
969
|
+
|
|
970
|
+
@_api_reference.add(path='/services/logs', method='post')
|
|
971
|
+
def log(self,
|
|
972
|
+
service,
|
|
973
|
+
size=100,
|
|
974
|
+
checkpoint=None,
|
|
975
|
+
start=None,
|
|
976
|
+
end=None,
|
|
977
|
+
follow=False,
|
|
978
|
+
text=None,
|
|
979
|
+
execution_id=None,
|
|
980
|
+
function_name=None,
|
|
981
|
+
replica_id=None,
|
|
982
|
+
system=False,
|
|
983
|
+
view=True,
|
|
984
|
+
until_completed=True,
|
|
985
|
+
log_level='DEBUG',
|
|
986
|
+
model_id: str = None,
|
|
987
|
+
model_operation: str = None,
|
|
988
|
+
project_id: str = None
|
|
989
|
+
):
|
|
990
|
+
"""
|
|
991
|
+
Get service logs.
|
|
992
|
+
|
|
993
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
994
|
+
|
|
995
|
+
:param dtlpy.entities.service.Service service: service object
|
|
996
|
+
:param int size: size
|
|
997
|
+
:param dict checkpoint: the information from the lst point checked in the service
|
|
998
|
+
:param str start: iso format time
|
|
999
|
+
:param str end: iso format time
|
|
1000
|
+
:param bool follow: if true, keep stream future logs
|
|
1001
|
+
:param str text: text
|
|
1002
|
+
:param str execution_id: execution id
|
|
1003
|
+
:param str function_name: function name
|
|
1004
|
+
:param str replica_id: replica id
|
|
1005
|
+
:param bool system: system
|
|
1006
|
+
:param bool view: if true, print out all the logs
|
|
1007
|
+
:param bool until_completed: wait until completed
|
|
1008
|
+
:param str log_level: the log level to display dl.LoggingLevel
|
|
1009
|
+
:param str model_id: model id
|
|
1010
|
+
:param str model_operation: model operation action
|
|
1011
|
+
:param str project_id: project id
|
|
1012
|
+
:return: ServiceLog entity
|
|
1013
|
+
:rtype: ServiceLog
|
|
1014
|
+
|
|
1015
|
+
**Example**:
|
|
1016
|
+
|
|
1017
|
+
.. code-block:: python
|
|
1018
|
+
|
|
1019
|
+
service_logs = package.services.log(service='service_entity')
|
|
1020
|
+
"""
|
|
1021
|
+
if not isinstance(service, entities.Service) and model_id is None:
|
|
1022
|
+
raise exceptions.PlatformException('400', 'Must provide service or model_id')
|
|
1023
|
+
if isinstance(log_level, str):
|
|
1024
|
+
log_level = log_level.upper()
|
|
1025
|
+
|
|
1026
|
+
payload = {
|
|
1027
|
+
'direction': 'asc',
|
|
1028
|
+
'follow': follow,
|
|
1029
|
+
'system': system,
|
|
1030
|
+
'logLevel': log_level
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
if service is not None:
|
|
1034
|
+
payload['serviceId'] = service.id
|
|
1035
|
+
|
|
1036
|
+
if size is not None:
|
|
1037
|
+
payload['size'] = size
|
|
1038
|
+
|
|
1039
|
+
if execution_id is not None:
|
|
1040
|
+
payload['executionId'] = [execution_id]
|
|
1041
|
+
|
|
1042
|
+
if function_name is not None:
|
|
1043
|
+
payload['functionName'] = function_name
|
|
1044
|
+
|
|
1045
|
+
if text is not None:
|
|
1046
|
+
payload['text'] = text
|
|
1047
|
+
|
|
1048
|
+
if replica_id is not None:
|
|
1049
|
+
payload['replicaId'] = replica_id
|
|
1050
|
+
|
|
1051
|
+
if checkpoint is not None:
|
|
1052
|
+
payload['checkpoint'] = checkpoint
|
|
1053
|
+
|
|
1054
|
+
if start is not None:
|
|
1055
|
+
payload['start'] = start
|
|
1056
|
+
else:
|
|
1057
|
+
payload['start'] = datetime.datetime(datetime.date.today().year,
|
|
1058
|
+
datetime.date.today().month,
|
|
1059
|
+
datetime.date.today().day,
|
|
1060
|
+
0,
|
|
1061
|
+
0,
|
|
1062
|
+
0).isoformat()
|
|
1063
|
+
|
|
1064
|
+
if end is not None:
|
|
1065
|
+
payload['end'] = end
|
|
1066
|
+
|
|
1067
|
+
if model_id is not None:
|
|
1068
|
+
payload['modelId'] = model_id
|
|
1069
|
+
|
|
1070
|
+
if model_operation is not None:
|
|
1071
|
+
payload['modelOperation'] = model_operation
|
|
1072
|
+
|
|
1073
|
+
if project_id is not None:
|
|
1074
|
+
payload['projectId'] = project_id
|
|
1075
|
+
else:
|
|
1076
|
+
payload['projectId'] = service.project_id
|
|
1077
|
+
|
|
1078
|
+
# request
|
|
1079
|
+
success, response = self._client_api.gen_request(req_type='post',
|
|
1080
|
+
path='/services/logs',
|
|
1081
|
+
json_req=payload)
|
|
1082
|
+
|
|
1083
|
+
# exception handling
|
|
1084
|
+
if not success:
|
|
1085
|
+
raise exceptions.PlatformException(response)
|
|
1086
|
+
|
|
1087
|
+
log = ServiceLog(_json=response.json(),
|
|
1088
|
+
service=service,
|
|
1089
|
+
services=self,
|
|
1090
|
+
start=payload['start'],
|
|
1091
|
+
follow=follow,
|
|
1092
|
+
execution_id=execution_id,
|
|
1093
|
+
function_name=function_name,
|
|
1094
|
+
replica_id=replica_id,
|
|
1095
|
+
system=system,
|
|
1096
|
+
model_id=model_id,
|
|
1097
|
+
model_operation=model_operation,
|
|
1098
|
+
project_id=project_id)
|
|
1099
|
+
|
|
1100
|
+
if view:
|
|
1101
|
+
log.view(until_completed=until_completed)
|
|
1102
|
+
else:
|
|
1103
|
+
return log
|
|
1104
|
+
|
|
1105
|
+
def execute(self,
|
|
1106
|
+
service: entities.Service = None,
|
|
1107
|
+
service_id: str = None,
|
|
1108
|
+
service_name: str = None,
|
|
1109
|
+
sync: bool = False,
|
|
1110
|
+
function_name: str = None,
|
|
1111
|
+
stream_logs: bool = False,
|
|
1112
|
+
execution_input=None,
|
|
1113
|
+
resource=None,
|
|
1114
|
+
item_id=None,
|
|
1115
|
+
dataset_id=None,
|
|
1116
|
+
annotation_id=None,
|
|
1117
|
+
project_id=None,
|
|
1118
|
+
) -> entities.Execution:
|
|
1119
|
+
"""
|
|
1120
|
+
Execute a function on an existing service.
|
|
1121
|
+
|
|
1122
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1123
|
+
|
|
1124
|
+
:param dtlpy.entities.service.Service service: service entity
|
|
1125
|
+
:param str service_id: service id
|
|
1126
|
+
:param str service_name: service name
|
|
1127
|
+
:param bool sync: wait for function to end
|
|
1128
|
+
:param str function_name: function name to run
|
|
1129
|
+
:param bool stream_logs: prints logs of the new execution. only works with sync=True
|
|
1130
|
+
:param execution_input: input dictionary or list of FunctionIO entities
|
|
1131
|
+
:param str resource: dl.PackageInputType - input type.
|
|
1132
|
+
:param str item_id: str - optional - input to function
|
|
1133
|
+
:param str dataset_id: str - optional - input to function
|
|
1134
|
+
:param str annotation_id: str - optional - input to function
|
|
1135
|
+
:param str project_id: str - resource's project
|
|
1136
|
+
:return: entities.Execution
|
|
1137
|
+
:rtype: dtlpy.entities.execution.Execution
|
|
1138
|
+
|
|
1139
|
+
**Example**:
|
|
1140
|
+
|
|
1141
|
+
.. code-block:: python
|
|
1142
|
+
|
|
1143
|
+
execution = package.services.execute(service='service_entity',
|
|
1144
|
+
function_name='run',
|
|
1145
|
+
item_id='item_id',
|
|
1146
|
+
project_id='project_id')
|
|
1147
|
+
"""
|
|
1148
|
+
if service is None:
|
|
1149
|
+
service = self.get(service_id=service_id, service_name=service_name)
|
|
1150
|
+
execution = repositories.Executions(service=service,
|
|
1151
|
+
client_api=self._client_api,
|
|
1152
|
+
project=self._project).create(service_id=service.id,
|
|
1153
|
+
sync=sync,
|
|
1154
|
+
execution_input=execution_input,
|
|
1155
|
+
function_name=function_name,
|
|
1156
|
+
resource=resource,
|
|
1157
|
+
item_id=item_id,
|
|
1158
|
+
dataset_id=dataset_id,
|
|
1159
|
+
annotation_id=annotation_id,
|
|
1160
|
+
project_id=project_id,
|
|
1161
|
+
stream_logs=stream_logs)
|
|
1162
|
+
return execution
|
|
1163
|
+
|
|
1164
|
+
@_api_reference.add(path='/services', method='post')
|
|
1165
|
+
def deploy(self,
|
|
1166
|
+
service_name: str = None,
|
|
1167
|
+
package: entities.Package = None,
|
|
1168
|
+
bot: Union[entities.Bot, str] = None,
|
|
1169
|
+
revision: str or int = None,
|
|
1170
|
+
init_input: Union[List[entities.FunctionIO], entities.FunctionIO, dict] = None,
|
|
1171
|
+
runtime: Union[entities.KubernetesRuntime, dict] = None,
|
|
1172
|
+
pod_type: entities.InstanceCatalog = None,
|
|
1173
|
+
sdk_version: str = None,
|
|
1174
|
+
agent_versions: dict = None,
|
|
1175
|
+
verify: bool = True,
|
|
1176
|
+
checkout: bool = False,
|
|
1177
|
+
module_name: str = None,
|
|
1178
|
+
project_id: str = None,
|
|
1179
|
+
driver_id: str = None,
|
|
1180
|
+
func: Callable = None,
|
|
1181
|
+
run_execution_as_process: bool = None,
|
|
1182
|
+
execution_timeout: int = None,
|
|
1183
|
+
drain_time: int = None,
|
|
1184
|
+
max_attempts: int = None,
|
|
1185
|
+
on_reset: str = None,
|
|
1186
|
+
force: bool = False,
|
|
1187
|
+
secrets: list = None,
|
|
1188
|
+
integrations: list = None,
|
|
1189
|
+
active: bool = True,
|
|
1190
|
+
**kwargs) -> entities.Service:
|
|
1191
|
+
"""
|
|
1192
|
+
Deploy service.
|
|
1193
|
+
|
|
1194
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1195
|
+
|
|
1196
|
+
:param str service_name: name
|
|
1197
|
+
:param dtlpy.entities.package.Package package: package entity
|
|
1198
|
+
:param str bot: bot email
|
|
1199
|
+
:param str revision: package revision of version
|
|
1200
|
+
:param init_input: config to run at startup
|
|
1201
|
+
:param dict runtime: runtime resources
|
|
1202
|
+
:param str pod_type: pod type dl.InstanceCatalog
|
|
1203
|
+
:param str sdk_version: - optional - string - sdk version
|
|
1204
|
+
:param str agent_versions: - dictionary - - optional -versions of sdk
|
|
1205
|
+
:param bool verify: if true, verify the inputs
|
|
1206
|
+
:param bool checkout: if true, checkout (switch) to service
|
|
1207
|
+
:param str module_name: module name
|
|
1208
|
+
:param str project_id: project id
|
|
1209
|
+
:param str driver_id: driver id
|
|
1210
|
+
:param Callable func: function to deploy
|
|
1211
|
+
:param bool run_execution_as_process: if true, run execution as process
|
|
1212
|
+
:param int execution_timeout: execution timeout in seconds
|
|
1213
|
+
:param int drain_time: drain time in seconds
|
|
1214
|
+
:param int max_attempts: maximum execution retries in-case of a service reset
|
|
1215
|
+
:param str on_reset: what happens on reset
|
|
1216
|
+
:param bool force: optional - if true, terminate old replicas immediately
|
|
1217
|
+
:param list secrets: list of the integrations ids
|
|
1218
|
+
:param list integrations: list of the integrations
|
|
1219
|
+
:param bool active: if true, activate the service
|
|
1220
|
+
:param kwargs: list of additional arguments
|
|
1221
|
+
:return: Service object
|
|
1222
|
+
:rtype: dtlpy.entities.service.Service
|
|
1223
|
+
|
|
1224
|
+
**Example**:
|
|
1225
|
+
|
|
1226
|
+
.. code-block:: python
|
|
1227
|
+
|
|
1228
|
+
service = package.services.deploy(service_name=package_name,
|
|
1229
|
+
execution_timeout=3 * 60 * 60,
|
|
1230
|
+
module_name=module.name,
|
|
1231
|
+
runtime=dl.KubernetesRuntime(
|
|
1232
|
+
concurrency=10,
|
|
1233
|
+
pod_type=dl.InstanceCatalog.REGULAR_S,
|
|
1234
|
+
autoscaler=dl.KubernetesRabbitmqAutoscaler(
|
|
1235
|
+
min_replicas=1,
|
|
1236
|
+
max_replicas=20,
|
|
1237
|
+
queue_length=20
|
|
1238
|
+
)
|
|
1239
|
+
)
|
|
1240
|
+
)
|
|
1241
|
+
"""
|
|
1242
|
+
if package is None and isinstance(package, entities.Dpk):
|
|
1243
|
+
raise exceptions.PlatformException('400', 'cannot deploy dpk package. Please install the app')
|
|
1244
|
+
package = package if package is not None else self._package
|
|
1245
|
+
if service_name is None:
|
|
1246
|
+
get_name = False
|
|
1247
|
+
if package is not None and package.service_config is not None and 'name' in package.service_config:
|
|
1248
|
+
service_name = package.service_config['name']
|
|
1249
|
+
get_name = True
|
|
1250
|
+
else:
|
|
1251
|
+
if package is not None:
|
|
1252
|
+
service_name = package.name
|
|
1253
|
+
else:
|
|
1254
|
+
service_name = 'default-service'
|
|
1255
|
+
if not get_name:
|
|
1256
|
+
logger.warning('service_name not provided, using: {} by default'.format(service_name))
|
|
1257
|
+
|
|
1258
|
+
if isinstance(revision, int):
|
|
1259
|
+
logger.warning('Deprecation Warning - Package/service versions have been refactored'
|
|
1260
|
+
'The version you provided has type: int, it will be converted to: 1.0.{}'
|
|
1261
|
+
'Next time use a 3-level semver for package/service versions'.format(revision))
|
|
1262
|
+
|
|
1263
|
+
if func is not None:
|
|
1264
|
+
package = self.__deploy_function(name=service_name, project=self._project, func=func)
|
|
1265
|
+
|
|
1266
|
+
if init_input is not None and not isinstance(init_input, dict):
|
|
1267
|
+
if not isinstance(init_input, list):
|
|
1268
|
+
init_input = [init_input]
|
|
1269
|
+
|
|
1270
|
+
if len(init_input) > 0 and isinstance(init_input[0], entities.FunctionIO):
|
|
1271
|
+
params = dict()
|
|
1272
|
+
for i_param, param in enumerate(init_input):
|
|
1273
|
+
params[param.name] = param.value
|
|
1274
|
+
init_input = params
|
|
1275
|
+
elif len(init_input) == 0:
|
|
1276
|
+
init_input = None
|
|
1277
|
+
else:
|
|
1278
|
+
raise exceptions.PlatformException(
|
|
1279
|
+
error='400',
|
|
1280
|
+
message='Unknown init_input type. expecting list or dict, got: {}'.format(type(init_input))
|
|
1281
|
+
)
|
|
1282
|
+
|
|
1283
|
+
if project_id is None:
|
|
1284
|
+
if self._project is not None:
|
|
1285
|
+
project_id = self._project.id
|
|
1286
|
+
else:
|
|
1287
|
+
project_id = self._project_id
|
|
1288
|
+
|
|
1289
|
+
filters = entities.Filters(resource=entities.FiltersResource.SERVICE, use_defaults=False)
|
|
1290
|
+
filters.add(field='name', values=service_name)
|
|
1291
|
+
if project_id is not None:
|
|
1292
|
+
filters.add(field='projectId', values=project_id)
|
|
1293
|
+
services = self.list(filters=filters)
|
|
1294
|
+
if services.items_count > 1:
|
|
1295
|
+
raise exceptions.PlatformException('400',
|
|
1296
|
+
'More than 1 service by this name are associated with this user. '
|
|
1297
|
+
'Please provide project_id')
|
|
1298
|
+
elif services.items_count > 0:
|
|
1299
|
+
service = services.items[0]
|
|
1300
|
+
if runtime is not None:
|
|
1301
|
+
service.runtime = runtime
|
|
1302
|
+
if init_input is not None:
|
|
1303
|
+
service.init_input = init_input
|
|
1304
|
+
if revision is not None:
|
|
1305
|
+
service.package_revision = revision
|
|
1306
|
+
if agent_versions is not None:
|
|
1307
|
+
service.versions = agent_versions
|
|
1308
|
+
elif sdk_version:
|
|
1309
|
+
service.versions = {'dtlpy': sdk_version}
|
|
1310
|
+
if driver_id is not None:
|
|
1311
|
+
service.driver_id = driver_id
|
|
1312
|
+
if secrets is not None:
|
|
1313
|
+
if not isinstance(secrets, list):
|
|
1314
|
+
secrets = [secrets]
|
|
1315
|
+
service.secrets = secrets
|
|
1316
|
+
if integrations is not None:
|
|
1317
|
+
if not isinstance(integrations, list):
|
|
1318
|
+
integrations = [integrations]
|
|
1319
|
+
service.integrations = integrations
|
|
1320
|
+
service = self.update(service=service, force=force)
|
|
1321
|
+
else:
|
|
1322
|
+
service = self._create(service_name=service_name,
|
|
1323
|
+
package=package,
|
|
1324
|
+
project_id=project_id,
|
|
1325
|
+
bot=bot,
|
|
1326
|
+
revision=revision,
|
|
1327
|
+
init_input=init_input,
|
|
1328
|
+
runtime=runtime,
|
|
1329
|
+
pod_type=pod_type,
|
|
1330
|
+
sdk_version=sdk_version,
|
|
1331
|
+
agent_versions=agent_versions,
|
|
1332
|
+
verify=verify,
|
|
1333
|
+
module_name=module_name,
|
|
1334
|
+
driver_id=driver_id,
|
|
1335
|
+
jwt_forward=kwargs.get('jwt_forward', None),
|
|
1336
|
+
is_global=kwargs.get('is_global', None),
|
|
1337
|
+
run_execution_as_process=run_execution_as_process,
|
|
1338
|
+
execution_timeout=execution_timeout,
|
|
1339
|
+
drain_time=drain_time,
|
|
1340
|
+
max_attempts=max_attempts,
|
|
1341
|
+
on_reset=on_reset,
|
|
1342
|
+
secrets=secrets,
|
|
1343
|
+
integrations=integrations,
|
|
1344
|
+
active=active
|
|
1345
|
+
)
|
|
1346
|
+
if checkout:
|
|
1347
|
+
self.checkout(service=service)
|
|
1348
|
+
return service
|
|
1349
|
+
|
|
1350
|
+
@staticmethod
|
|
1351
|
+
def __get_import_string(imports: List[str]):
|
|
1352
|
+
import_string = ''
|
|
1353
|
+
if imports is not None:
|
|
1354
|
+
import_string = '\n'.join(imports)
|
|
1355
|
+
return import_string
|
|
1356
|
+
|
|
1357
|
+
@staticmethod
|
|
1358
|
+
def __get_inputs(func):
|
|
1359
|
+
method = inspect.signature(func)
|
|
1360
|
+
params = list(method.parameters)
|
|
1361
|
+
inpts = list()
|
|
1362
|
+
inputs_types = {i.name.lower(): i.value for i in list(entities.PackageInputType)}
|
|
1363
|
+
for arg in params:
|
|
1364
|
+
if arg in inputs_types:
|
|
1365
|
+
inpt_type = inputs_types[arg]
|
|
1366
|
+
else:
|
|
1367
|
+
inpt_type = entities.PackageInputType.JSON
|
|
1368
|
+
inpts.append(entities.FunctionIO(type=inpt_type, name=arg))
|
|
1369
|
+
return inpts
|
|
1370
|
+
|
|
1371
|
+
def __deploy_function(self,
|
|
1372
|
+
name: str,
|
|
1373
|
+
func: Callable,
|
|
1374
|
+
project: entities.Project) -> entities.Package:
|
|
1375
|
+
package_dir = tempfile.mkdtemp()
|
|
1376
|
+
# imports_string = self.__get_import_string()
|
|
1377
|
+
imports_string = ''
|
|
1378
|
+
|
|
1379
|
+
main_file = os.path.join(package_dir, entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
|
|
1380
|
+
with open(assets.paths.PARTIAL_MAIN_FILEPATH, 'r') as f:
|
|
1381
|
+
main_string = f.read()
|
|
1382
|
+
lines = inspect.getsourcelines(func)
|
|
1383
|
+
tabs_diff = lines[0][0].count(' ') - 1
|
|
1384
|
+
for line_index in range(len(lines[0])):
|
|
1385
|
+
line_tabs = lines[0][line_index].count(' ') - tabs_diff
|
|
1386
|
+
lines[0][line_index] = (' ' * line_tabs) + lines[0][line_index].strip() + '\n'
|
|
1387
|
+
|
|
1388
|
+
method_func_string = "".join(lines[0])
|
|
1389
|
+
|
|
1390
|
+
with open(main_file, 'w') as f:
|
|
1391
|
+
f.write('{}\n{}\n @staticmethod\n{}'.format(imports_string, main_string,
|
|
1392
|
+
method_func_string))
|
|
1393
|
+
|
|
1394
|
+
function = entities.PackageFunction(name=func.__name__, inputs=self.__get_inputs(func=func))
|
|
1395
|
+
module = entities.PackageModule(functions=[function],
|
|
1396
|
+
entry_point=entities.package_defaults.DEFAULT_PACKAGE_ENTRY_POINT)
|
|
1397
|
+
packages = repositories.Packages(client_api=self._client_api, project=project)
|
|
1398
|
+
return packages.push(src_path=package_dir,
|
|
1399
|
+
package_name=name,
|
|
1400
|
+
checkout=True,
|
|
1401
|
+
modules=[module])
|
|
1402
|
+
|
|
1403
|
+
def deploy_from_local_folder(self,
|
|
1404
|
+
cwd=None,
|
|
1405
|
+
service_file=None,
|
|
1406
|
+
bot=None,
|
|
1407
|
+
checkout=False,
|
|
1408
|
+
force=False
|
|
1409
|
+
) -> entities.Service:
|
|
1410
|
+
"""
|
|
1411
|
+
Deploy from local folder in local environment.
|
|
1412
|
+
|
|
1413
|
+
**Prerequisites**: You must be in the role of an *owner* or *developer*. You must have a package.
|
|
1414
|
+
|
|
1415
|
+
:param str cwd: optional - package working directory. Default=cwd
|
|
1416
|
+
:param str service_file: optional - service file. Default=None
|
|
1417
|
+
:param str bot: bot
|
|
1418
|
+
:param checkout: checkout
|
|
1419
|
+
:param bool force: optional - terminate old replicas immediately
|
|
1420
|
+
:return: Service object
|
|
1421
|
+
:rtype: dtlpy.entities.service.Service
|
|
1422
|
+
|
|
1423
|
+
**Example**:
|
|
1424
|
+
|
|
1425
|
+
.. code-block:: python
|
|
1426
|
+
|
|
1427
|
+
service = package.services.deploy_from_local_folder(cwd='file_path',
|
|
1428
|
+
service_file='service_file')
|
|
1429
|
+
"""
|
|
1430
|
+
# get cwd and service.json path
|
|
1431
|
+
if cwd is None:
|
|
1432
|
+
cwd = os.getcwd()
|
|
1433
|
+
if service_file is None:
|
|
1434
|
+
service_file = os.path.join(cwd, assets.paths.SERVICE_FILENAME)
|
|
1435
|
+
|
|
1436
|
+
# load service json
|
|
1437
|
+
if os.path.isfile(service_file):
|
|
1438
|
+
with open(service_file, 'r') as f:
|
|
1439
|
+
service_json = json.load(f)
|
|
1440
|
+
service_triggers = service_json.get('triggers', list())
|
|
1441
|
+
else:
|
|
1442
|
+
raise exceptions.PlatformException(error='400',
|
|
1443
|
+
message='Could not find service.json in path: {}'.format(cwd))
|
|
1444
|
+
|
|
1445
|
+
# get package
|
|
1446
|
+
package_name = service_json.get('packageName', None)
|
|
1447
|
+
packages = repositories.Packages(client_api=self._client_api, project=self._project)
|
|
1448
|
+
|
|
1449
|
+
if package_name is None:
|
|
1450
|
+
package = packages.get()
|
|
1451
|
+
else:
|
|
1452
|
+
package = packages.get(package_name=package_name)
|
|
1453
|
+
|
|
1454
|
+
name = service_json.get('name', None)
|
|
1455
|
+
revision = service_json.get('revision', package.version)
|
|
1456
|
+
init_input = service_json.get('initParams', dict())
|
|
1457
|
+
runtime = service_json.get('runtime', dict())
|
|
1458
|
+
sdk_version = service_json.get('version', None)
|
|
1459
|
+
agent_versions = service_json.get('versions', None)
|
|
1460
|
+
verify = service_json.get('verify', True)
|
|
1461
|
+
module_name = service_json.get('module_name', None)
|
|
1462
|
+
run_execution_as_process = service_json.get('run_execution_as_process', None)
|
|
1463
|
+
execution_timeout = service_json.get('execution_timeout', None)
|
|
1464
|
+
drain_time = service_json.get('drain_time', None)
|
|
1465
|
+
on_reset = service_json.get('on_reset', None)
|
|
1466
|
+
max_attempts = service_json.get('maxAttempts', None)
|
|
1467
|
+
|
|
1468
|
+
service = self.deploy(bot=bot,
|
|
1469
|
+
service_name=name,
|
|
1470
|
+
package=package,
|
|
1471
|
+
revision=revision,
|
|
1472
|
+
runtime=runtime,
|
|
1473
|
+
init_input=init_input,
|
|
1474
|
+
sdk_version=sdk_version,
|
|
1475
|
+
agent_versions=agent_versions,
|
|
1476
|
+
verify=verify,
|
|
1477
|
+
checkout=checkout,
|
|
1478
|
+
run_execution_as_process=run_execution_as_process,
|
|
1479
|
+
execution_timeout=execution_timeout,
|
|
1480
|
+
drain_time=drain_time,
|
|
1481
|
+
max_attempts=max_attempts,
|
|
1482
|
+
on_reset=on_reset,
|
|
1483
|
+
module_name=module_name,
|
|
1484
|
+
force=force
|
|
1485
|
+
)
|
|
1486
|
+
|
|
1487
|
+
logger.info('Service was deployed successfully. Service id: {}'.format(service.id))
|
|
1488
|
+
|
|
1489
|
+
if len(service_triggers) > 0:
|
|
1490
|
+
logger.info('Creating triggers...')
|
|
1491
|
+
triggers = repositories.Triggers(client_api=self._client_api, project=self._project)
|
|
1492
|
+
|
|
1493
|
+
for trigger in service_triggers:
|
|
1494
|
+
name = trigger.get('name', None)
|
|
1495
|
+
filters = trigger.get('filter', dict())
|
|
1496
|
+
resource = trigger['resource']
|
|
1497
|
+
actions = trigger.get('actions', list())
|
|
1498
|
+
active = trigger.get('active', True)
|
|
1499
|
+
execution_mode = trigger.get('executionMode', None)
|
|
1500
|
+
function_name = trigger.get('function', None)
|
|
1501
|
+
|
|
1502
|
+
trigger = triggers.create(service_id=service.id,
|
|
1503
|
+
name=name,
|
|
1504
|
+
filters=filters,
|
|
1505
|
+
resource=resource,
|
|
1506
|
+
actions=actions,
|
|
1507
|
+
active=active,
|
|
1508
|
+
execution_mode=execution_mode,
|
|
1509
|
+
function_name=function_name)
|
|
1510
|
+
|
|
1511
|
+
logger.info('Trigger was created successfully. Service id: {}'.format(trigger.id))
|
|
1512
|
+
|
|
1513
|
+
logging.info('Successfully deployed!')
|
|
1514
|
+
return service
|
|
1515
|
+
|
|
1516
|
+
def __enable_cache(self,
|
|
1517
|
+
url,
|
|
1518
|
+
organization: entities.Organization,
|
|
1519
|
+
pod_type=entities.PodType.SMALL):
|
|
1520
|
+
payload = {
|
|
1521
|
+
"org": {
|
|
1522
|
+
"name": organization.name,
|
|
1523
|
+
"id": organization.id
|
|
1524
|
+
},
|
|
1525
|
+
"runner": {
|
|
1526
|
+
"podType": pod_type # small, medium, high
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
return self._client_api.gen_request(req_type='post',
|
|
1531
|
+
path=url,
|
|
1532
|
+
json_req=payload)
|
|
1533
|
+
|
|
1534
|
+
def __polling_wait(self, organization, pod_type, backoff_factor=1):
|
|
1535
|
+
fs_url_path = '/services/fs-cache?mode={}'.format('get')
|
|
1536
|
+
i = 1
|
|
1537
|
+
while True:
|
|
1538
|
+
success, response = self.__enable_cache(url=fs_url_path, organization=organization, pod_type=pod_type)
|
|
1539
|
+
if response.json().get('state', None) == 'READY':
|
|
1540
|
+
break
|
|
1541
|
+
sleep_time = min(backoff_factor * (2 ** (i - 1)), MAX_WAIT_TIME)
|
|
1542
|
+
logger.debug("Going to sleep {:.2f}[s]".format(sleep_time))
|
|
1543
|
+
time.sleep(sleep_time)
|
|
1544
|
+
i += 1
|
|
1545
|
+
return success
|
|
1546
|
+
|
|
1547
|
+
def _cache_action(self,
|
|
1548
|
+
organization: entities.Organization = None,
|
|
1549
|
+
mode=entities.CacheAction.APPLY,
|
|
1550
|
+
pod_type=entities.PodType.SMALL):
|
|
1551
|
+
"""
|
|
1552
|
+
Add or remove Cache for the org
|
|
1553
|
+
|
|
1554
|
+
**Prerequisites**: You must be an organization *owner*
|
|
1555
|
+
|
|
1556
|
+
You must provide at least ONE of the following params: organization, organization_name, or organization_id.
|
|
1557
|
+
|
|
1558
|
+
:param entities.Organization organization: Organization object
|
|
1559
|
+
:param str mode: dl.CacheAction.APPLY or dl.CacheAction.DESTROY
|
|
1560
|
+
:param entities.PodType pod_type: dl.PodType.SMALL, dl.PodType.MEDIUM, dl.PodType.HIGH
|
|
1561
|
+
:return: True if success
|
|
1562
|
+
:rtype: bool
|
|
1563
|
+
|
|
1564
|
+
**Example**:
|
|
1565
|
+
|
|
1566
|
+
.. code-block:: python
|
|
1567
|
+
|
|
1568
|
+
success = dl.organizations.enable_cache(organization='organization',
|
|
1569
|
+
mode=dl.CacheAction.APPLY)
|
|
1570
|
+
"""
|
|
1571
|
+
if organization is None:
|
|
1572
|
+
raise exceptions.PlatformException(
|
|
1573
|
+
error='400',
|
|
1574
|
+
message='Must provide an identifier in inputs')
|
|
1575
|
+
|
|
1576
|
+
fs_mode = mode if mode != entities.CacheAction.APPLY else '{}-filestore'.format(mode)
|
|
1577
|
+
apply_fs_url_path = '/services/fs-cache?mode={}'.format(fs_mode)
|
|
1578
|
+
apply_volume_url_path = '/services/fs-cache?mode={}'.format(mode)
|
|
1579
|
+
cache_url_path = '/services/cache?mode={}'.format(mode)
|
|
1580
|
+
|
|
1581
|
+
success, response = self.__enable_cache(url=apply_fs_url_path, organization=organization, pod_type=pod_type)
|
|
1582
|
+
if not success:
|
|
1583
|
+
raise exceptions.PlatformException(response)
|
|
1584
|
+
|
|
1585
|
+
if mode == entities.CacheAction.APPLY:
|
|
1586
|
+
self.__polling_wait(organization=organization, pod_type=pod_type)
|
|
1587
|
+
success, response = self.__enable_cache(url=apply_volume_url_path, organization=organization,
|
|
1588
|
+
pod_type=pod_type)
|
|
1589
|
+
if not success:
|
|
1590
|
+
raise exceptions.PlatformException(response)
|
|
1591
|
+
|
|
1592
|
+
success, response = self.__enable_cache(url=cache_url_path, organization=organization, pod_type=pod_type)
|
|
1593
|
+
if not success:
|
|
1594
|
+
raise exceptions.PlatformException(response)
|
|
1595
|
+
|
|
1596
|
+
return True
|
|
1597
|
+
|
|
1598
|
+
def restart(self, service: entities.Service, replica_name: str = None):
|
|
1599
|
+
"""
|
|
1600
|
+
Restart service replica
|
|
1601
|
+
|
|
1602
|
+
**Prerequisites**: You must be in the role of a *developer*.
|
|
1603
|
+
|
|
1604
|
+
:param dtlpy.entities.service.Service service: service entity
|
|
1605
|
+
:param str replica_name: replica name
|
|
1606
|
+
:return: True
|
|
1607
|
+
:rtype: bool
|
|
1608
|
+
|
|
1609
|
+
**Example**:
|
|
1610
|
+
|
|
1611
|
+
.. code-block:: python
|
|
1612
|
+
|
|
1613
|
+
is_restarted = dl.services.restart(service='service_entity',
|
|
1614
|
+
replica_name='replica_name')
|
|
1615
|
+
"""
|
|
1616
|
+
payload = {}
|
|
1617
|
+
|
|
1618
|
+
if replica_name is not None:
|
|
1619
|
+
payload['replicaName'] = replica_name
|
|
1620
|
+
|
|
1621
|
+
# request
|
|
1622
|
+
success, response = self._client_api.gen_request(req_type='post',
|
|
1623
|
+
path='/services/{}/restart'.format(service.id),
|
|
1624
|
+
json_req=payload)
|
|
1625
|
+
|
|
1626
|
+
# exception handling
|
|
1627
|
+
if not success:
|
|
1628
|
+
raise exceptions.PlatformException(response)
|
|
1629
|
+
|
|
1630
|
+
return True
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
class ServiceLog:
|
|
1634
|
+
"""
|
|
1635
|
+
Service Log
|
|
1636
|
+
"""
|
|
1637
|
+
|
|
1638
|
+
def __init__(self,
|
|
1639
|
+
_json: dict,
|
|
1640
|
+
service: entities.Service,
|
|
1641
|
+
services: Services,
|
|
1642
|
+
start=None,
|
|
1643
|
+
follow=None,
|
|
1644
|
+
execution_id=None,
|
|
1645
|
+
function_name=None,
|
|
1646
|
+
replica_id=None,
|
|
1647
|
+
system=False,
|
|
1648
|
+
model_id=None,
|
|
1649
|
+
model_operation=None,
|
|
1650
|
+
project_id=None):
|
|
1651
|
+
|
|
1652
|
+
self.logs = _json.get('logs', dict())
|
|
1653
|
+
self.checkpoint = _json.get('checkpoint', None)
|
|
1654
|
+
self.stop = _json.get('stop', False)
|
|
1655
|
+
self.service = service
|
|
1656
|
+
self.services = services
|
|
1657
|
+
self.start = start
|
|
1658
|
+
self.follow = follow
|
|
1659
|
+
self.execution_id = execution_id
|
|
1660
|
+
self.function_name = function_name
|
|
1661
|
+
self.replica_id = replica_id
|
|
1662
|
+
self.system = system
|
|
1663
|
+
self.model_id = model_id
|
|
1664
|
+
self.model_operation = model_operation
|
|
1665
|
+
self.project_id = project_id
|
|
1666
|
+
|
|
1667
|
+
def get_next_log(self):
|
|
1668
|
+
log = self.services.log(service=self.service,
|
|
1669
|
+
checkpoint=self.checkpoint,
|
|
1670
|
+
start=self.start,
|
|
1671
|
+
follow=self.follow,
|
|
1672
|
+
execution_id=self.execution_id,
|
|
1673
|
+
function_name=self.function_name,
|
|
1674
|
+
replica_id=self.replica_id,
|
|
1675
|
+
system=self.system,
|
|
1676
|
+
view=False,
|
|
1677
|
+
model_id=self.model_id,
|
|
1678
|
+
model_operation=self.model_operation,
|
|
1679
|
+
project_id=self.project_id)
|
|
1680
|
+
|
|
1681
|
+
self.logs = log.logs
|
|
1682
|
+
self.checkpoint = log.checkpoint
|
|
1683
|
+
self.stop = log.stop
|
|
1684
|
+
|
|
1685
|
+
def view(self, until_completed):
|
|
1686
|
+
"""
|
|
1687
|
+
View logs
|
|
1688
|
+
|
|
1689
|
+
:param until_completed:
|
|
1690
|
+
"""
|
|
1691
|
+
try:
|
|
1692
|
+
for log in self:
|
|
1693
|
+
print(log)
|
|
1694
|
+
if until_completed and FUNCTION_END_LINE in log:
|
|
1695
|
+
break
|
|
1696
|
+
except KeyboardInterrupt:
|
|
1697
|
+
return
|
|
1698
|
+
|
|
1699
|
+
def __iter__(self):
|
|
1700
|
+
while not self.stop:
|
|
1701
|
+
for log in self.logs:
|
|
1702
|
+
yield '{}: {}'.format(log.get('timestamp', self.start), log.get('message', '').strip())
|
|
1703
|
+
self.get_next_log()
|
|
1704
|
+
|