zenml-nightly 0.60.0.dev20240627__py3-none-any.whl → 0.61.0.dev20240711__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +30 -9
- RELEASE_NOTES.md +34 -0
- zenml/VERSION +1 -1
- zenml/analytics/enums.py +3 -0
- zenml/cli/__init__.py +28 -0
- zenml/cli/artifact.py +1 -2
- zenml/cli/integration.py +9 -8
- zenml/cli/server.py +6 -0
- zenml/cli/stack.py +797 -39
- zenml/cli/stack_components.py +7 -0
- zenml/cli/text_utils.py +35 -1
- zenml/cli/utils.py +127 -10
- zenml/client.py +23 -14
- zenml/config/docker_settings.py +8 -5
- zenml/constants.py +5 -1
- zenml/container_registries/base_container_registry.py +1 -0
- zenml/enums.py +6 -0
- zenml/event_hub/event_hub.py +5 -8
- zenml/integrations/aws/__init__.py +1 -0
- zenml/integrations/azure/__init__.py +1 -0
- zenml/integrations/deepchecks/__init__.py +1 -0
- zenml/integrations/discord/__init__.py +1 -0
- zenml/integrations/evidently/__init__.py +1 -0
- zenml/integrations/facets/__init__.py +1 -0
- zenml/integrations/feast/__init__.py +1 -0
- zenml/integrations/gcp/__init__.py +3 -1
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +203 -44
- zenml/integrations/huggingface/__init__.py +1 -0
- zenml/integrations/integration.py +24 -0
- zenml/integrations/kubeflow/__init__.py +3 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +1 -1
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +0 -1
- zenml/integrations/kubernetes/__init__.py +3 -1
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +4 -1
- zenml/integrations/label_studio/annotators/label_studio_annotator.py +1 -0
- zenml/integrations/langchain/__init__.py +1 -0
- zenml/integrations/mlflow/__init__.py +3 -1
- zenml/integrations/neural_prophet/__init__.py +1 -0
- zenml/integrations/polars/__init__.py +1 -0
- zenml/integrations/prodigy/__init__.py +1 -0
- zenml/integrations/pycaret/__init__.py +6 -0
- zenml/integrations/registry.py +37 -0
- zenml/integrations/s3/artifact_stores/s3_artifact_store.py +17 -6
- zenml/integrations/seldon/__init__.py +1 -0
- zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +1 -0
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +2 -2
- zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +1 -1
- zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
- zenml/integrations/skypilot_aws/__init__.py +2 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/skypilot_gcp/__init__.py +1 -1
- zenml/integrations/skypilot_lambda/__init__.py +1 -1
- zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +1 -1
- zenml/integrations/slack/__init__.py +1 -0
- zenml/integrations/tekton/__init__.py +1 -0
- zenml/integrations/tensorboard/__init__.py +0 -1
- zenml/integrations/tensorflow/__init__.py +18 -6
- zenml/integrations/wandb/__init__.py +1 -0
- zenml/models/__init__.py +9 -0
- zenml/models/v2/core/component.py +18 -0
- zenml/models/v2/core/model.py +1 -2
- zenml/models/v2/core/service_connector.py +17 -0
- zenml/models/v2/core/stack.py +31 -0
- zenml/models/v2/misc/full_stack.py +97 -0
- zenml/models/v2/misc/stack_deployment.py +66 -0
- zenml/new/pipelines/pipeline.py +1 -1
- zenml/orchestrators/input_utils.py +3 -6
- zenml/stack/stack.py +3 -6
- zenml/stack_deployments/__init__.py +14 -0
- zenml/stack_deployments/aws_stack_deployment.py +289 -0
- zenml/stack_deployments/stack_deployment.py +130 -0
- zenml/stack_deployments/utils.py +40 -0
- zenml/utils/function_utils.py +1 -1
- zenml/utils/pagination_utils.py +7 -5
- zenml/utils/pipeline_docker_image_builder.py +97 -68
- zenml/utils/pydantic_utils.py +6 -5
- zenml/zen_server/cloud_utils.py +18 -3
- zenml/zen_server/dashboard/assets/{404-C1mcUujL.js → 404-DpJaNHKF.js} +1 -1
- zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +85 -0
- zenml/zen_server/dashboard/assets/{@react-router-DYovave8.js → @react-router-CO-OsFwI.js} +2 -2
- zenml/zen_server/dashboard/assets/{@reactflow-DYIyhCfd.js → @reactflow-DJfzkHO1.js} +2 -2
- zenml/zen_server/dashboard/assets/@tanstack-DYiOyJUL.js +22 -0
- zenml/zen_server/dashboard/assets/AwarenessChannel-BYDLT2xC.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-WEzpO0az.js → CodeSnippet-BkOuRmyq.js} +2 -2
- zenml/zen_server/dashboard/assets/Commands-ZvWR1BRs.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-DVwLkafa.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-Bx931j4U.js → CsvVizualization-C2IiqX4I.js} +7 -7
- zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +1 -0
- zenml/zen_server/dashboard/assets/EmptyState-BMLnFVlB.js +1 -0
- zenml/zen_server/dashboard/assets/Error-CqX0VqW_.js +1 -0
- zenml/zen_server/dashboard/assets/ExecutionStatus-BoLUXR9t.js +1 -0
- zenml/zen_server/dashboard/assets/Helpbox-LFydyVwh.js +1 -0
- zenml/zen_server/dashboard/assets/Infobox-DnENC0sh.js +1 -0
- zenml/zen_server/dashboard/assets/InlineAvatar-CbJtYr0t.js +1 -0
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-DsB2QZiK.js → MarkdownVisualization-xp3hhULl.js} +2 -2
- zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +1 -0
- zenml/zen_server/dashboard/assets/PasswordChecker-DUveqlva.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-BYBdbQDo.js +1 -0
- zenml/zen_server/dashboard/assets/SuccessStep-Nx743hll.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CKrd3UZz.js → UpdatePasswordSchemas-DF9gSzE0.js} +1 -1
- zenml/zen_server/dashboard/assets/{aws-t0gKCj_R.js → aws-BgKTfTfx.js} +1 -1
- zenml/zen_server/dashboard/assets/{check-circle-BVvhm5dy.js → check-circle-i56092KI.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-zcvCWmyP.js → chevron-down-D_ZlKMqH.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-CJ50E9Gr.js → chevron-right-double-BiEMg7rd.js} +1 -1
- zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +1 -0
- zenml/zen_server/dashboard/assets/{copy-BRhQz3j-.js → copy-BXNk6BjL.js} +1 -1
- zenml/zen_server/dashboard/assets/{database-CRRnyFWh.js → database-1xWSgZfO.js} +1 -1
- zenml/zen_server/dashboard/assets/{docker-BAonhm6G.js → docker-CQMVm_4d.js} +1 -1
- zenml/zen_server/dashboard/assets/{file-text-CbVERUON.js → file-text-CqD_iu6l.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-B8rqCvqn.js → help-bu_DgLKI.js} +1 -1
- zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +1 -0
- zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +55 -0
- zenml/zen_server/dashboard/assets/index-inApY3KQ.css +1 -0
- zenml/zen_server/dashboard/assets/index-rK_Wuy2W.js +1 -0
- zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-Bk2tn324.js → login-mutation-BUnVASxp.js} +1 -1
- zenml/zen_server/dashboard/assets/not-found-B4VnX8gK.js +1 -0
- zenml/zen_server/dashboard/assets/package-CsUhPmou.js +1 -0
- zenml/zen_server/dashboard/assets/{page-D12Rvf0j.js → page-3efNCDeb.js} +2 -2
- zenml/zen_server/dashboard/assets/page-7zTHbhhI.js +1 -0
- zenml/zen_server/dashboard/assets/page-BEs6jK71.js +1 -0
- zenml/zen_server/dashboard/assets/page-BpSqIf4B.js +1 -0
- zenml/zen_server/dashboard/assets/{page-8vRWJ5b8.js → page-Bx6o0ARS.js} +1 -1
- zenml/zen_server/dashboard/assets/page-C43QGHTt.js +9 -0
- zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CBuSUrE9.js → page-CRTJ0UuR.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CUZIGO-3.js +1 -0
- zenml/zen_server/dashboard/assets/page-CaopxiU1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CCtCgG-x.js → page-Cx67M0QT.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D7Z399xy.js +1 -0
- zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +1 -0
- zenml/zen_server/dashboard/assets/{page-Dw9-aJV6.js → page-DKlIdAe5.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-COafKNbw.js → page-DMOYZppS.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +2 -0
- zenml/zen_server/dashboard/assets/{page-C6v3o0Qj.js → page-Dc_7KMQE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DvCvroOM.js +1 -0
- zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +1 -0
- zenml/zen_server/dashboard/assets/page-JyfeDUfu.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CH26py0a.js → page-Sxn82W-5.js} +1 -1
- zenml/zen_server/dashboard/assets/page-TKXERe16.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xu8JEjSU.js +1 -0
- zenml/zen_server/dashboard/assets/{play-circle-DK5QMJyp.js → play-circle-CNtZKDnW.js} +1 -1
- zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +1 -0
- zenml/zen_server/dashboard/assets/{terminal-B2ovgWuz.js → terminal-By9cErXc.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-bKxf7U9h.js → update-server-settings-mutation-CR8e3Sir.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-CgvM-IVM.js → url-DuQMeqYA.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-DrZvVLjd.js → zod-BhoGpZ63.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.e7c29295aae591541ef59d1734d79387.js → precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js → main.382439a7.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js.map → main.382439a7.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +11 -5
- zenml/zen_server/pipeline_deployment/utils.py +57 -44
- zenml/zen_server/rbac/zenml_cloud_rbac.py +11 -5
- zenml/zen_server/routers/stack_deployment_endpoints.py +144 -0
- zenml/zen_server/routers/workspaces_endpoints.py +64 -0
- zenml/zen_server/zen_server_api.py +2 -0
- zenml/zen_stores/migrations/utils.py +1 -1
- zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
- zenml/zen_stores/rest_zen_store.py +117 -0
- zenml/zen_stores/schemas/stack_schemas.py +10 -0
- zenml/zen_stores/schemas/step_run_schemas.py +27 -11
- zenml/zen_stores/sql_zen_store.py +283 -0
- zenml/zen_stores/zen_store_interface.py +79 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/METADATA +32 -10
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/RECORD +175 -161
- zenml/zen_server/dashboard/assets/@radix-C9DBgJhe.js +0 -77
- zenml/zen_server/dashboard/assets/@tanstack-CEbkxrhX.js +0 -30
- zenml/zen_server/dashboard/assets/AwarenessChannel-B2KR83Tr.js +0 -1
- zenml/zen_server/dashboard/assets/Cards-DSEdjsk8.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-CTlhyic5.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-CTrzKmUO.js +0 -2
- zenml/zen_server/dashboard/assets/DisplayDate-BdguISQF.js +0 -1
- zenml/zen_server/dashboard/assets/EmptyState-BkooiGtL.js +0 -1
- zenml/zen_server/dashboard/assets/Error-4sKxHad4.js +0 -1
- zenml/zen_server/dashboard/assets/Helpbox-DW21i5LD.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-C7bf70VS.js +0 -1
- zenml/zen_server/dashboard/assets/InlineAvatar-Dxrtafpg.js +0 -1
- zenml/zen_server/dashboard/assets/PageHeader-B0pUife2.js +0 -1
- zenml/zen_server/dashboard/assets/Pagination-B9WG_9cJ.js +0 -1
- zenml/zen_server/dashboard/assets/PasswordChecker-DSLBp7Vl.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-CiNhT15a.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-CykrFndS.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-Bkawp7CJ.js +0 -1
- zenml/zen_server/dashboard/assets/index-BawkpTlr.js +0 -55
- zenml/zen_server/dashboard/assets/index-CRmm7QhS.css +0 -1
- zenml/zen_server/dashboard/assets/index.esm-F7nqy9zY.js +0 -1
- zenml/zen_server/dashboard/assets/not-found-BAuhP4Jb.js +0 -1
- zenml/zen_server/dashboard/assets/page--5YvAHg3.js +0 -1
- zenml/zen_server/dashboard/assets/page-B0RAq4s_.js +0 -1
- zenml/zen_server/dashboard/assets/page-BePtEPHl.js +0 -1
- zenml/zen_server/dashboard/assets/page-C1pra1Bc.js +0 -9
- zenml/zen_server/dashboard/assets/page-CSs4C9jL.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cf2XSej0.js +0 -1
- zenml/zen_server/dashboard/assets/page-ClPUAE_f.js +0 -1
- zenml/zen_server/dashboard/assets/page-D8pf2vis.js +0 -1
- zenml/zen_server/dashboard/assets/page-DHKMmIQH.js +0 -1
- zenml/zen_server/dashboard/assets/page-DMZ0VOda.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dcg-yQv_.js +0 -1
- zenml/zen_server/dashboard/assets/page-DoAK5FSB.js +0 -1
- zenml/zen_server/dashboard/assets/page-iXiDqE0J.js +0 -2
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/entry_points.txt +0 -0
zenml/cli/stack.py
CHANGED
@@ -15,17 +15,34 @@
|
|
15
15
|
|
16
16
|
import getpass
|
17
17
|
import os
|
18
|
+
import re
|
19
|
+
import time
|
20
|
+
import webbrowser
|
21
|
+
from datetime import datetime
|
18
22
|
from pathlib import Path
|
19
|
-
from typing import
|
23
|
+
from typing import (
|
24
|
+
TYPE_CHECKING,
|
25
|
+
Any,
|
26
|
+
Dict,
|
27
|
+
List,
|
28
|
+
Optional,
|
29
|
+
Set,
|
30
|
+
Union,
|
31
|
+
)
|
20
32
|
from uuid import UUID
|
21
33
|
|
22
34
|
import click
|
35
|
+
from rich.console import Console
|
36
|
+
from rich.markdown import Markdown
|
37
|
+
from rich.prompt import Confirm
|
38
|
+
from rich.syntax import Syntax
|
23
39
|
|
24
40
|
import zenml
|
25
41
|
from zenml.analytics.enums import AnalyticsEvent
|
26
42
|
from zenml.analytics.utils import track_handler
|
27
43
|
from zenml.cli import utils as cli_utils
|
28
44
|
from zenml.cli.cli import TagGroup, cli
|
45
|
+
from zenml.cli.text_utils import OldSchoolMarkdownHeading
|
29
46
|
from zenml.cli.utils import (
|
30
47
|
_component_display_name,
|
31
48
|
confirmation,
|
@@ -45,7 +62,11 @@ from zenml.constants import (
|
|
45
62
|
MLSTACKS_SUPPORTED_STACK_COMPONENTS,
|
46
63
|
STACK_RECIPE_MODULAR_RECIPES,
|
47
64
|
)
|
48
|
-
from zenml.enums import
|
65
|
+
from zenml.enums import (
|
66
|
+
CliCategories,
|
67
|
+
StackComponentType,
|
68
|
+
StackDeploymentProvider,
|
69
|
+
)
|
49
70
|
from zenml.exceptions import (
|
50
71
|
IllegalOperationError,
|
51
72
|
ProvisioningError,
|
@@ -53,7 +74,19 @@ from zenml.exceptions import (
|
|
53
74
|
from zenml.io.fileio import rmtree
|
54
75
|
from zenml.logger import get_logger
|
55
76
|
from zenml.models import StackFilter
|
56
|
-
from zenml.
|
77
|
+
from zenml.models.v2.core.service_connector import (
|
78
|
+
ServiceConnectorRequest,
|
79
|
+
ServiceConnectorResponse,
|
80
|
+
)
|
81
|
+
from zenml.models.v2.misc.full_stack import (
|
82
|
+
ComponentInfo,
|
83
|
+
FullStackRequest,
|
84
|
+
ServiceConnectorInfo,
|
85
|
+
)
|
86
|
+
from zenml.models.v2.misc.service_connector_type import (
|
87
|
+
ServiceConnectorTypedResourcesModel,
|
88
|
+
)
|
89
|
+
from zenml.utils.dashboard_utils import get_component_url, get_stack_url
|
57
90
|
from zenml.utils.io_utils import create_dir_recursive_if_not_exists
|
58
91
|
from zenml.utils.mlstacks_utils import (
|
59
92
|
convert_click_params_to_mlstacks_primitives,
|
@@ -93,7 +126,7 @@ def stack() -> None:
|
|
93
126
|
"artifact_store",
|
94
127
|
help="Name of the artifact store for this stack.",
|
95
128
|
type=str,
|
96
|
-
required=
|
129
|
+
required=False,
|
97
130
|
)
|
98
131
|
@click.option(
|
99
132
|
"-o",
|
@@ -101,7 +134,7 @@ def stack() -> None:
|
|
101
134
|
"orchestrator",
|
102
135
|
help="Name of the orchestrator for this stack.",
|
103
136
|
type=str,
|
104
|
-
required=
|
137
|
+
required=False,
|
105
138
|
)
|
106
139
|
@click.option(
|
107
140
|
"-c",
|
@@ -190,10 +223,24 @@ def stack() -> None:
|
|
190
223
|
help="Immediately set this stack as active.",
|
191
224
|
type=click.BOOL,
|
192
225
|
)
|
226
|
+
@click.option(
|
227
|
+
"-p",
|
228
|
+
"--provider",
|
229
|
+
help="Name of the cloud provider for this stack.",
|
230
|
+
type=click.Choice(["aws", "azure", "gcp"]),
|
231
|
+
required=False,
|
232
|
+
)
|
233
|
+
@click.option(
|
234
|
+
"-sc",
|
235
|
+
"--connector",
|
236
|
+
help="Name of the service connector for this stack.",
|
237
|
+
type=str,
|
238
|
+
required=False,
|
239
|
+
)
|
193
240
|
def register_stack(
|
194
241
|
stack_name: str,
|
195
|
-
artifact_store: str,
|
196
|
-
orchestrator: str,
|
242
|
+
artifact_store: Optional[str] = None,
|
243
|
+
orchestrator: Optional[str] = None,
|
197
244
|
container_registry: Optional[str] = None,
|
198
245
|
model_registry: Optional[str] = None,
|
199
246
|
step_operator: Optional[str] = None,
|
@@ -205,6 +252,8 @@ def register_stack(
|
|
205
252
|
data_validator: Optional[str] = None,
|
206
253
|
image_builder: Optional[str] = None,
|
207
254
|
set_stack: bool = False,
|
255
|
+
provider: Optional[str] = None,
|
256
|
+
connector: Optional[str] = None,
|
208
257
|
) -> None:
|
209
258
|
"""Register a stack.
|
210
259
|
|
@@ -223,44 +272,240 @@ def register_stack(
|
|
223
272
|
data_validator: Name of the data validator for this stack.
|
224
273
|
image_builder: Name of the new image builder for this stack.
|
225
274
|
set_stack: Immediately set this stack as active.
|
275
|
+
provider: Name of the cloud provider for this stack.
|
276
|
+
connector: Name of the service connector for this stack.
|
226
277
|
"""
|
227
|
-
|
228
|
-
|
278
|
+
if (provider is None and connector is None) and (
|
279
|
+
artifact_store is None or orchestrator is None
|
280
|
+
):
|
281
|
+
cli_utils.error(
|
282
|
+
"Only stack using service connector can be registered "
|
283
|
+
"without specifying an artifact store and an orchestrator. "
|
284
|
+
"Please specify the artifact store and the orchestrator or "
|
285
|
+
"the service connector or cloud type settings."
|
286
|
+
)
|
229
287
|
|
230
|
-
|
288
|
+
client = Client()
|
231
289
|
|
232
|
-
|
233
|
-
|
290
|
+
try:
|
291
|
+
client.get_stack(
|
292
|
+
name_id_or_prefix=stack_name,
|
293
|
+
allow_name_prefix_match=False,
|
294
|
+
)
|
295
|
+
cli_utils.error(
|
296
|
+
f"A stack with name `{stack_name}` already exists, "
|
297
|
+
"please use a different name."
|
298
|
+
)
|
299
|
+
except KeyError:
|
300
|
+
pass
|
234
301
|
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
components[StackComponentType.STEP_OPERATOR] = step_operator
|
251
|
-
if experiment_tracker:
|
252
|
-
components[StackComponentType.EXPERIMENT_TRACKER] = (
|
253
|
-
experiment_tracker
|
302
|
+
labels: Dict[str, str] = {}
|
303
|
+
components: Dict[StackComponentType, Union[UUID, ComponentInfo]] = {}
|
304
|
+
# cloud flow
|
305
|
+
created_objects: Set[str] = set()
|
306
|
+
service_connector: Optional[Union[UUID, ServiceConnectorInfo]] = None
|
307
|
+
if provider is not None and connector is None:
|
308
|
+
service_connector_response = None
|
309
|
+
use_auto_configure = False
|
310
|
+
try:
|
311
|
+
service_connector_response, _ = client.create_service_connector(
|
312
|
+
name=stack_name,
|
313
|
+
connector_type=provider,
|
314
|
+
register=False,
|
315
|
+
auto_configure=True,
|
316
|
+
verify=False,
|
254
317
|
)
|
255
|
-
|
256
|
-
|
257
|
-
|
318
|
+
except Exception:
|
319
|
+
pass
|
320
|
+
if service_connector_response:
|
321
|
+
use_auto_configure = Confirm.ask(
|
322
|
+
f"[bold]{provider.upper()} cloud service connector[/bold] "
|
323
|
+
"has detected connection credentials in your environment.\n"
|
324
|
+
"Would you like to use these credentials or create a new "
|
325
|
+
"configuration by providing connection details?",
|
326
|
+
default=True,
|
327
|
+
show_choices=True,
|
328
|
+
show_default=True,
|
329
|
+
)
|
330
|
+
|
331
|
+
connector_selected: Optional[int] = None
|
332
|
+
if not use_auto_configure:
|
333
|
+
service_connector_response = None
|
334
|
+
existing_connectors = client.list_service_connectors(
|
335
|
+
connector_type=provider, size=100
|
336
|
+
)
|
337
|
+
if existing_connectors.total:
|
338
|
+
connector_selected = cli_utils.multi_choice_prompt(
|
339
|
+
object_type=f"{provider.upper()} service connectors",
|
340
|
+
choices=[
|
341
|
+
[connector.name]
|
342
|
+
for connector in existing_connectors.items
|
343
|
+
],
|
344
|
+
headers=["Name"],
|
345
|
+
prompt_text=f"We found these {provider.upper()} service connectors. "
|
346
|
+
"Do you want to create a new one or use one of the existing ones?",
|
347
|
+
default_choice="0",
|
348
|
+
allow_zero_be_a_new_object=True,
|
349
|
+
)
|
350
|
+
if use_auto_configure or connector_selected is None:
|
351
|
+
service_connector = _get_service_connector_info(
|
352
|
+
cloud_provider=provider,
|
353
|
+
connector_details=service_connector_response,
|
258
354
|
)
|
355
|
+
created_objects.add("service_connector")
|
356
|
+
else:
|
357
|
+
selected_connector = existing_connectors.items[connector_selected]
|
358
|
+
service_connector = selected_connector.id
|
359
|
+
connector = selected_connector.name
|
360
|
+
if isinstance(selected_connector.connector_type, str):
|
361
|
+
provider = selected_connector.connector_type
|
362
|
+
else:
|
363
|
+
provider = selected_connector.connector_type.connector_type
|
364
|
+
elif connector is not None:
|
365
|
+
service_connector_response = client.get_service_connector(connector)
|
366
|
+
service_connector = service_connector_response.id
|
367
|
+
if provider:
|
368
|
+
if service_connector_response.type != provider:
|
369
|
+
cli_utils.warning(
|
370
|
+
f"The service connector `{connector}` is not of type `{provider}`."
|
371
|
+
)
|
372
|
+
else:
|
373
|
+
provider = service_connector_response.type
|
374
|
+
|
375
|
+
if service_connector:
|
376
|
+
labels["zenml:wizard"] = "true"
|
377
|
+
if provider:
|
378
|
+
labels["zenml:provider"] = provider
|
379
|
+
service_connector_resource_model = None
|
380
|
+
# create components
|
381
|
+
needed_components = (
|
382
|
+
(StackComponentType.ARTIFACT_STORE, artifact_store),
|
383
|
+
(StackComponentType.ORCHESTRATOR, orchestrator),
|
384
|
+
(StackComponentType.CONTAINER_REGISTRY, container_registry),
|
385
|
+
)
|
386
|
+
for component_type, preset_name in needed_components:
|
387
|
+
component_info: Optional[Union[UUID, ComponentInfo]] = None
|
388
|
+
if preset_name is not None:
|
389
|
+
component_response = client.get_stack_component(
|
390
|
+
component_type, preset_name
|
391
|
+
)
|
392
|
+
component_info = component_response.id
|
393
|
+
else:
|
394
|
+
if isinstance(service_connector, UUID):
|
395
|
+
# find existing components under same connector
|
396
|
+
existing_components = client.list_stack_components(
|
397
|
+
type=component_type.value,
|
398
|
+
connector_id=service_connector,
|
399
|
+
size=100,
|
400
|
+
)
|
401
|
+
# if some existing components are found - prompt user what to do
|
402
|
+
component_selected: Optional[int] = None
|
403
|
+
if existing_components.total > 0:
|
404
|
+
component_selected = cli_utils.multi_choice_prompt(
|
405
|
+
object_type=component_type.value.replace("_", " "),
|
406
|
+
choices=[
|
407
|
+
[component.name]
|
408
|
+
for component in existing_components.items
|
409
|
+
],
|
410
|
+
headers=["Name"],
|
411
|
+
prompt_text=f"We found these {component_type.value.replace('_', ' ')} "
|
412
|
+
"connected using the current service connector. Do you "
|
413
|
+
"want to create a new one or use existing one?",
|
414
|
+
default_choice="0",
|
415
|
+
allow_zero_be_a_new_object=True,
|
416
|
+
)
|
417
|
+
else:
|
418
|
+
component_selected = None
|
419
|
+
|
420
|
+
if component_selected is None:
|
421
|
+
if service_connector_resource_model is None:
|
422
|
+
if isinstance(service_connector, UUID):
|
423
|
+
service_connector_resource_model = (
|
424
|
+
client.verify_service_connector(
|
425
|
+
service_connector
|
426
|
+
)
|
427
|
+
)
|
428
|
+
else:
|
429
|
+
_, service_connector_resource_model = (
|
430
|
+
client.create_service_connector(
|
431
|
+
name=stack_name,
|
432
|
+
connector_type=service_connector.type,
|
433
|
+
auth_method=service_connector.auth_method,
|
434
|
+
configuration=service_connector.configuration,
|
435
|
+
register=False,
|
436
|
+
)
|
437
|
+
)
|
438
|
+
if service_connector_resource_model is None:
|
439
|
+
cli_utils.error(
|
440
|
+
f"Failed to validate service connector {service_connector}..."
|
441
|
+
)
|
442
|
+
if provider is None:
|
443
|
+
if isinstance(
|
444
|
+
service_connector_resource_model.connector_type,
|
445
|
+
str,
|
446
|
+
):
|
447
|
+
provider = (
|
448
|
+
service_connector_resource_model.connector_type
|
449
|
+
)
|
450
|
+
else:
|
451
|
+
provider = service_connector_resource_model.connector_type.connector_type
|
452
|
+
|
453
|
+
component_info = _get_stack_component_info(
|
454
|
+
component_type=component_type.value,
|
455
|
+
cloud_provider=provider,
|
456
|
+
service_connector_resource_models=service_connector_resource_model.resources,
|
457
|
+
service_connector_index=0,
|
458
|
+
)
|
459
|
+
component_name = stack_name
|
460
|
+
created_objects.add(component_type.value)
|
461
|
+
else:
|
462
|
+
selected_component = existing_components.items[
|
463
|
+
component_selected
|
464
|
+
]
|
465
|
+
component_info = selected_component.id
|
466
|
+
component_name = selected_component.name
|
467
|
+
|
468
|
+
components[component_type] = component_info
|
469
|
+
if component_type == StackComponentType.ARTIFACT_STORE:
|
470
|
+
artifact_store = component_name
|
471
|
+
if component_type == StackComponentType.ORCHESTRATOR:
|
472
|
+
orchestrator = component_name
|
473
|
+
if component_type == StackComponentType.CONTAINER_REGISTRY:
|
474
|
+
container_registry = component_name
|
475
|
+
|
476
|
+
# normal flow once all components are defined
|
477
|
+
with console.status(f"Registering stack '{stack_name}'...\n"):
|
478
|
+
for component_type_, component_name_ in [
|
479
|
+
(StackComponentType.ARTIFACT_STORE, artifact_store),
|
480
|
+
(StackComponentType.ORCHESTRATOR, orchestrator),
|
481
|
+
(StackComponentType.ALERTER, alerter),
|
482
|
+
(StackComponentType.ANNOTATOR, annotator),
|
483
|
+
(StackComponentType.DATA_VALIDATOR, data_validator),
|
484
|
+
(StackComponentType.FEATURE_STORE, feature_store),
|
485
|
+
(StackComponentType.IMAGE_BUILDER, image_builder),
|
486
|
+
(StackComponentType.MODEL_DEPLOYER, model_deployer),
|
487
|
+
(StackComponentType.MODEL_REGISTRY, model_registry),
|
488
|
+
(StackComponentType.STEP_OPERATOR, step_operator),
|
489
|
+
(StackComponentType.EXPERIMENT_TRACKER, experiment_tracker),
|
490
|
+
(StackComponentType.CONTAINER_REGISTRY, container_registry),
|
491
|
+
]:
|
492
|
+
if component_name_ and component_type_ not in components:
|
493
|
+
components[component_type_] = client.get_stack_component(
|
494
|
+
component_type_, component_name_
|
495
|
+
).id
|
259
496
|
|
260
497
|
try:
|
261
|
-
created_stack = client.
|
262
|
-
|
263
|
-
|
498
|
+
created_stack = client.zen_store.create_full_stack(
|
499
|
+
full_stack=FullStackRequest(
|
500
|
+
user=client.active_user.id,
|
501
|
+
workspace=client.active_workspace.id,
|
502
|
+
name=stack_name,
|
503
|
+
components=components,
|
504
|
+
service_connectors=[service_connector]
|
505
|
+
if service_connector
|
506
|
+
else [],
|
507
|
+
labels=labels,
|
508
|
+
)
|
264
509
|
)
|
265
510
|
except (KeyError, IllegalOperationError) as err:
|
266
511
|
cli_utils.error(str(err))
|
@@ -268,6 +513,10 @@ def register_stack(
|
|
268
513
|
cli_utils.declare(
|
269
514
|
f"Stack '{created_stack.name}' successfully registered!"
|
270
515
|
)
|
516
|
+
cli_utils.print_stack_configuration(
|
517
|
+
stack=created_stack,
|
518
|
+
active=created_stack.id == client.active_stack_model.id,
|
519
|
+
)
|
271
520
|
|
272
521
|
if set_stack:
|
273
522
|
client.activate_stack(created_stack.id)
|
@@ -277,6 +526,30 @@ def register_stack(
|
|
277
526
|
f"Active {scope} stack set to:'{created_stack.name}'"
|
278
527
|
)
|
279
528
|
|
529
|
+
delete_commands = []
|
530
|
+
if "service_connector" in created_objects:
|
531
|
+
created_objects.remove("service_connector")
|
532
|
+
connectors = set()
|
533
|
+
for each in created_objects:
|
534
|
+
if comps_ := created_stack.components[StackComponentType(each)]:
|
535
|
+
if conn_ := comps_[0].connector:
|
536
|
+
connectors.add(conn_.name)
|
537
|
+
for connector in connectors:
|
538
|
+
delete_commands.append(
|
539
|
+
"zenml service-connector delete " + connector
|
540
|
+
)
|
541
|
+
for each in created_objects:
|
542
|
+
if comps_ := created_stack.components[StackComponentType(each)]:
|
543
|
+
delete_commands.append(
|
544
|
+
f"zenml {each.replace('_', '-')} delete {comps_[0].name}"
|
545
|
+
)
|
546
|
+
delete_commands.append("zenml stack delete -y " + created_stack.name)
|
547
|
+
|
548
|
+
Console().print(
|
549
|
+
"To delete the objects created by this command run, please run in a sequence:\n"
|
550
|
+
)
|
551
|
+
Console().print(Syntax("\n".join(delete_commands[::-1]), "bash"))
|
552
|
+
|
280
553
|
print_model_url(get_stack_url(created_stack))
|
281
554
|
|
282
555
|
|
@@ -1286,7 +1559,226 @@ def _get_deployment_params_interactively(
|
|
1286
1559
|
return deployment_values
|
1287
1560
|
|
1288
1561
|
|
1289
|
-
|
1562
|
+
def validate_name(ctx: click.Context, param: str, value: str) -> str:
|
1563
|
+
"""Validate the name of the stack.
|
1564
|
+
|
1565
|
+
Args:
|
1566
|
+
ctx: The click context.
|
1567
|
+
param: The parameter name.
|
1568
|
+
value: The value of the parameter.
|
1569
|
+
|
1570
|
+
Returns:
|
1571
|
+
The validated value.
|
1572
|
+
|
1573
|
+
Raises:
|
1574
|
+
BadParameter: If the name is invalid.
|
1575
|
+
"""
|
1576
|
+
if not value:
|
1577
|
+
return value
|
1578
|
+
|
1579
|
+
if not re.match(r"^[a-zA-Z0-9-]*$", value):
|
1580
|
+
raise click.BadParameter(
|
1581
|
+
"Stack name must contain only alphanumeric characters and hyphens."
|
1582
|
+
)
|
1583
|
+
|
1584
|
+
if len(value) > 16:
|
1585
|
+
raise click.BadParameter(
|
1586
|
+
"Stack name must have a maximum length of 16 characters."
|
1587
|
+
)
|
1588
|
+
|
1589
|
+
return value
|
1590
|
+
|
1591
|
+
|
1592
|
+
@stack.command(
|
1593
|
+
help="""Deploy a fully functional ZenML stack in one of the cloud providers.
|
1594
|
+
|
1595
|
+
Running this command will initiate an assisted process that will walk you
|
1596
|
+
through automatically provisioning all the cloud infrastructure resources
|
1597
|
+
necessary for a fully functional ZenML stack in the cloud provider of your
|
1598
|
+
choice. A corresponding ZenML stack will also be automatically registered along
|
1599
|
+
with all the necessary components and properly authenticated through service
|
1600
|
+
connectors.
|
1601
|
+
"""
|
1602
|
+
)
|
1603
|
+
@click.option(
|
1604
|
+
"--provider",
|
1605
|
+
"-p",
|
1606
|
+
"provider",
|
1607
|
+
required=True,
|
1608
|
+
type=click.Choice(StackDeploymentProvider.values()),
|
1609
|
+
)
|
1610
|
+
@click.option(
|
1611
|
+
"--name",
|
1612
|
+
"-n",
|
1613
|
+
"stack_name",
|
1614
|
+
type=click.STRING,
|
1615
|
+
required=False,
|
1616
|
+
help="Custom string to use as a prefix to generate names for the ZenML "
|
1617
|
+
"stack, its components service connectors as well as provisioned cloud "
|
1618
|
+
"infrastructure resources. May only contain alphanumeric characters and "
|
1619
|
+
"hyphens and have a maximum length of 16 characters.",
|
1620
|
+
callback=validate_name,
|
1621
|
+
)
|
1622
|
+
@click.option(
|
1623
|
+
"--location",
|
1624
|
+
"-l",
|
1625
|
+
type=click.STRING,
|
1626
|
+
required=False,
|
1627
|
+
help="The location to deploy the stack to.",
|
1628
|
+
)
|
1629
|
+
@click.option(
|
1630
|
+
"--set",
|
1631
|
+
"set_stack",
|
1632
|
+
is_flag=True,
|
1633
|
+
help="Immediately set this stack as active.",
|
1634
|
+
type=click.BOOL,
|
1635
|
+
)
|
1636
|
+
@click.pass_context
|
1637
|
+
def deploy(
|
1638
|
+
ctx: click.Context,
|
1639
|
+
provider: str,
|
1640
|
+
stack_name: Optional[str] = None,
|
1641
|
+
location: Optional[str] = None,
|
1642
|
+
set_stack: bool = False,
|
1643
|
+
) -> None:
|
1644
|
+
"""Deploy and register a fully functional cloud ZenML stack.
|
1645
|
+
|
1646
|
+
Args:
|
1647
|
+
ctx: The click context.
|
1648
|
+
provider: The cloud provider to deploy the stack to.
|
1649
|
+
stack_name: A name for the ZenML stack that gets imported as a result
|
1650
|
+
of the recipe deployment.
|
1651
|
+
location: The location to deploy the stack to.
|
1652
|
+
set_stack: Immediately set the deployed stack as active.
|
1653
|
+
|
1654
|
+
Raises:
|
1655
|
+
Abort: If the user aborts the deployment.
|
1656
|
+
KeyboardInterrupt: If the user interrupts the deployment.
|
1657
|
+
"""
|
1658
|
+
stack_name = stack_name or f"zenml-{provider}-stack"
|
1659
|
+
|
1660
|
+
# Set up the markdown renderer to use the old-school markdown heading
|
1661
|
+
Markdown.elements.update(
|
1662
|
+
{
|
1663
|
+
"heading_open": OldSchoolMarkdownHeading,
|
1664
|
+
}
|
1665
|
+
)
|
1666
|
+
|
1667
|
+
client = Client()
|
1668
|
+
if client.zen_store.is_local_store():
|
1669
|
+
cli_utils.error(
|
1670
|
+
"This feature cannot be used with a local ZenML deployment. "
|
1671
|
+
"ZenML needs to be accessible from the cloud provider to allow the "
|
1672
|
+
"stack and its components to be registered automatically. "
|
1673
|
+
"Please deploy ZenML in a remote environment as described in the "
|
1674
|
+
"documentation: https://docs.zenml.io/getting-started/deploying-zenml "
|
1675
|
+
"or use a managed ZenML Pro server instance for quick access to "
|
1676
|
+
"this feature and more: https://www.zenml.io/pro"
|
1677
|
+
)
|
1678
|
+
|
1679
|
+
with track_handler(
|
1680
|
+
event=AnalyticsEvent.DEPLOY_FULL_STACK,
|
1681
|
+
) as analytics_handler:
|
1682
|
+
analytics_handler.metadata = {
|
1683
|
+
"provider": provider,
|
1684
|
+
}
|
1685
|
+
|
1686
|
+
deployment = client.zen_store.get_stack_deployment_info(
|
1687
|
+
provider=StackDeploymentProvider(provider),
|
1688
|
+
)
|
1689
|
+
|
1690
|
+
console.print(
|
1691
|
+
Markdown(
|
1692
|
+
f"# {provider.upper()} ZenML Cloud Stack Deployment\n"
|
1693
|
+
+ deployment.description
|
1694
|
+
)
|
1695
|
+
)
|
1696
|
+
console.print(Markdown("## Instructions\n" + deployment.instructions))
|
1697
|
+
|
1698
|
+
if not cli_utils.confirmation(
|
1699
|
+
"\n\nProceed to continue with the deployment. You will be "
|
1700
|
+
f"automatically redirected to {provider.upper()} in your browser.",
|
1701
|
+
):
|
1702
|
+
raise click.Abort()
|
1703
|
+
|
1704
|
+
deployment_url, deployment_url_title = (
|
1705
|
+
client.zen_store.get_stack_deployment_url(
|
1706
|
+
provider=StackDeploymentProvider(provider),
|
1707
|
+
stack_name=stack_name,
|
1708
|
+
location=location,
|
1709
|
+
)
|
1710
|
+
)
|
1711
|
+
|
1712
|
+
date_start = datetime.utcnow()
|
1713
|
+
|
1714
|
+
webbrowser.open(deployment_url)
|
1715
|
+
console.print(
|
1716
|
+
Markdown(
|
1717
|
+
f"If your browser did not open automatically, please open "
|
1718
|
+
f"the following URL into your browser to deploy the stack to "
|
1719
|
+
f"{provider.upper()}: "
|
1720
|
+
f"[{deployment_url_title}]({deployment_url}).\n\n"
|
1721
|
+
)
|
1722
|
+
)
|
1723
|
+
|
1724
|
+
try:
|
1725
|
+
with console.status(
|
1726
|
+
"Waiting for the deployment to complete and the stack to be "
|
1727
|
+
"registered. Press CTRL+C to abort...\n"
|
1728
|
+
):
|
1729
|
+
while True:
|
1730
|
+
deployed_stack = (
|
1731
|
+
client.zen_store.get_stack_deployment_stack(
|
1732
|
+
provider=StackDeploymentProvider(provider),
|
1733
|
+
stack_name=stack_name,
|
1734
|
+
location=location,
|
1735
|
+
date_start=date_start,
|
1736
|
+
)
|
1737
|
+
)
|
1738
|
+
if deployed_stack:
|
1739
|
+
break
|
1740
|
+
time.sleep(10)
|
1741
|
+
|
1742
|
+
analytics_handler.metadata.update(
|
1743
|
+
{
|
1744
|
+
"stack_id": deployed_stack.stack.id,
|
1745
|
+
}
|
1746
|
+
)
|
1747
|
+
|
1748
|
+
except KeyboardInterrupt:
|
1749
|
+
cli_utils.declare("Stack deployment aborted.")
|
1750
|
+
raise
|
1751
|
+
|
1752
|
+
stack_desc = f"""## Stack successfully registered! 🚀
|
1753
|
+
Stack [{deployed_stack.stack.name}]({get_stack_url(deployed_stack.stack)}):\n"""
|
1754
|
+
|
1755
|
+
for component_type, components in deployed_stack.stack.components.items():
|
1756
|
+
if components:
|
1757
|
+
component = components[0]
|
1758
|
+
stack_desc += (
|
1759
|
+
f" * `{component.flavor}` {component_type.value}: "
|
1760
|
+
f"[{component.name}]({get_component_url(component)})\n"
|
1761
|
+
)
|
1762
|
+
|
1763
|
+
if deployed_stack.service_connector:
|
1764
|
+
stack_desc += (
|
1765
|
+
f" * Service Connector: {deployed_stack.service_connector.name}\n"
|
1766
|
+
)
|
1767
|
+
|
1768
|
+
console.print(Markdown(stack_desc))
|
1769
|
+
|
1770
|
+
console.print(
|
1771
|
+
Markdown("## Follow-up\n" + deployment.post_deploy_instructions)
|
1772
|
+
)
|
1773
|
+
|
1774
|
+
if set_stack:
|
1775
|
+
client.activate_stack(deployed_stack.stack.id)
|
1776
|
+
cli_utils.declare(
|
1777
|
+
f"\nStack `{deployed_stack.stack.name}` set as active"
|
1778
|
+
)
|
1779
|
+
|
1780
|
+
|
1781
|
+
@stack.command(help="[DEPRECATED] Deploy a stack using mlstacks.")
|
1290
1782
|
@click.option(
|
1291
1783
|
"--provider",
|
1292
1784
|
"-p",
|
@@ -1426,7 +1918,7 @@ def _get_deployment_params_interactively(
|
|
1426
1918
|
help="Deploy the stack interactively.",
|
1427
1919
|
)
|
1428
1920
|
@click.pass_context
|
1429
|
-
def
|
1921
|
+
def deploy_mlstack(
|
1430
1922
|
ctx: click.Context,
|
1431
1923
|
provider: str,
|
1432
1924
|
stack_name: str,
|
@@ -1476,6 +1968,13 @@ def deploy(
|
|
1476
1968
|
region: The region to deploy the stack to.
|
1477
1969
|
interactive: Deploy the stack interactively.
|
1478
1970
|
"""
|
1971
|
+
cli_utils.warning(
|
1972
|
+
"The `zenml stack deploy-mlstack` (former `zenml stack deploy`) CLI "
|
1973
|
+
"command has been deprecated and will be removed in a future release. "
|
1974
|
+
"Please use `zenml stack deploy` instead for a simplified "
|
1975
|
+
"experience."
|
1976
|
+
)
|
1977
|
+
|
1479
1978
|
with track_handler(
|
1480
1979
|
event=AnalyticsEvent.DEPLOY_STACK,
|
1481
1980
|
) as analytics_handler:
|
@@ -1727,3 +2226,262 @@ def connect_stack(
|
|
1727
2226
|
interactive=interactive,
|
1728
2227
|
no_verify=no_verify,
|
1729
2228
|
)
|
2229
|
+
|
2230
|
+
|
2231
|
+
def _get_service_connector_info(
|
2232
|
+
cloud_provider: str,
|
2233
|
+
connector_details: Optional[
|
2234
|
+
Union[ServiceConnectorResponse, ServiceConnectorRequest]
|
2235
|
+
],
|
2236
|
+
) -> ServiceConnectorInfo:
|
2237
|
+
"""Get a service connector info with given cloud provider.
|
2238
|
+
|
2239
|
+
Args:
|
2240
|
+
cloud_provider: The cloud provider to use.
|
2241
|
+
connector_details: Whether to use implicit credentials.
|
2242
|
+
|
2243
|
+
Returns:
|
2244
|
+
The info model of the created service connector.
|
2245
|
+
|
2246
|
+
Raises:
|
2247
|
+
ValueError: If the cloud provider is not supported.
|
2248
|
+
"""
|
2249
|
+
from rich.prompt import Prompt
|
2250
|
+
|
2251
|
+
if cloud_provider not in {"aws"}:
|
2252
|
+
raise ValueError(f"Unknown cloud provider {cloud_provider}")
|
2253
|
+
|
2254
|
+
client = Client()
|
2255
|
+
auth_methods = client.get_service_connector_type(
|
2256
|
+
cloud_provider
|
2257
|
+
).auth_method_dict
|
2258
|
+
if not connector_details:
|
2259
|
+
fixed_auth_methods = list(
|
2260
|
+
[
|
2261
|
+
(key, value)
|
2262
|
+
for key, value in auth_methods.items()
|
2263
|
+
if key != "implicit"
|
2264
|
+
]
|
2265
|
+
)
|
2266
|
+
choices = []
|
2267
|
+
headers = ["Name", "Required"]
|
2268
|
+
for _, value in fixed_auth_methods:
|
2269
|
+
schema = value.config_schema
|
2270
|
+
required = ""
|
2271
|
+
for each_req in schema["required"]:
|
2272
|
+
field = schema["properties"][each_req]
|
2273
|
+
required += f"[bold]{each_req}[/bold] [italic]({field.get('title','no description')})[/italic]\n"
|
2274
|
+
choices.append([value.name, required])
|
2275
|
+
|
2276
|
+
selected_auth_idx = cli_utils.multi_choice_prompt(
|
2277
|
+
object_type=f"authentication methods for {cloud_provider}",
|
2278
|
+
choices=choices,
|
2279
|
+
headers=headers,
|
2280
|
+
prompt_text="Please choose one of the authentication option above.",
|
2281
|
+
)
|
2282
|
+
if selected_auth_idx is None:
|
2283
|
+
cli_utils.error("No authentication method selected.")
|
2284
|
+
auth_type = fixed_auth_methods[selected_auth_idx][0]
|
2285
|
+
else:
|
2286
|
+
auth_type = connector_details.auth_method
|
2287
|
+
|
2288
|
+
selected_auth_model = auth_methods[auth_type]
|
2289
|
+
|
2290
|
+
required_fields = selected_auth_model.config_schema["required"]
|
2291
|
+
properties = selected_auth_model.config_schema["properties"]
|
2292
|
+
|
2293
|
+
answers = {}
|
2294
|
+
for req_field in required_fields:
|
2295
|
+
if connector_details:
|
2296
|
+
if conf_value := connector_details.configuration.get(
|
2297
|
+
req_field, None
|
2298
|
+
):
|
2299
|
+
answers[req_field] = conf_value
|
2300
|
+
elif secret_value := connector_details.secrets.get(
|
2301
|
+
req_field, None
|
2302
|
+
):
|
2303
|
+
answers[req_field] = secret_value.get_secret_value()
|
2304
|
+
if req_field not in answers:
|
2305
|
+
answers[req_field] = Prompt.ask(
|
2306
|
+
f"Please enter value for `{req_field}`:",
|
2307
|
+
password="format" in properties[req_field]
|
2308
|
+
and properties[req_field]["format"] == "password",
|
2309
|
+
)
|
2310
|
+
Console().print("All mandatory configuration parameters received!")
|
2311
|
+
|
2312
|
+
return ServiceConnectorInfo(
|
2313
|
+
type=cloud_provider,
|
2314
|
+
auth_method=auth_type,
|
2315
|
+
configuration=answers,
|
2316
|
+
)
|
2317
|
+
|
2318
|
+
|
2319
|
+
def _get_stack_component_info(
|
2320
|
+
component_type: str,
|
2321
|
+
cloud_provider: str,
|
2322
|
+
service_connector_resource_models: List[
|
2323
|
+
ServiceConnectorTypedResourcesModel
|
2324
|
+
],
|
2325
|
+
service_connector_index: Optional[int] = None,
|
2326
|
+
) -> ComponentInfo:
|
2327
|
+
"""Get a stack component info with given type and service connector.
|
2328
|
+
|
2329
|
+
Args:
|
2330
|
+
component_type: The type of component to create.
|
2331
|
+
cloud_provider: The cloud provider to use.
|
2332
|
+
service_connector_resource_models: The list of the available service connector resource models.
|
2333
|
+
service_connector_index: The index of the service connector to use.
|
2334
|
+
|
2335
|
+
Returns:
|
2336
|
+
The info model of the stack component.
|
2337
|
+
|
2338
|
+
Raises:
|
2339
|
+
ValueError: If the cloud provider is not supported.
|
2340
|
+
ValueError: If the component type is not supported.
|
2341
|
+
"""
|
2342
|
+
from rich.prompt import Prompt
|
2343
|
+
|
2344
|
+
if cloud_provider not in {"aws", "azure", "gcp"}:
|
2345
|
+
raise ValueError(f"Unknown cloud provider {cloud_provider}")
|
2346
|
+
|
2347
|
+
AWS_DOCS = (
|
2348
|
+
"https://docs.zenml.io/how-to/auth-management/aws-service-connector"
|
2349
|
+
)
|
2350
|
+
|
2351
|
+
flavor = "undefined"
|
2352
|
+
service_connector_resource_id = None
|
2353
|
+
config = {}
|
2354
|
+
if component_type == "artifact_store":
|
2355
|
+
available_storages: List[str] = []
|
2356
|
+
if cloud_provider == "aws":
|
2357
|
+
for each in service_connector_resource_models:
|
2358
|
+
if each.resource_type == "s3-bucket":
|
2359
|
+
available_storages = each.resource_ids or []
|
2360
|
+
flavor = "s3"
|
2361
|
+
if not available_storages:
|
2362
|
+
cli_utils.error(
|
2363
|
+
"We were unable to find any S3 buckets available "
|
2364
|
+
"to configured service connector. Please, verify "
|
2365
|
+
"that needed permission are granted for the "
|
2366
|
+
"service connector.\nDocumentation for the S3 "
|
2367
|
+
"Buckets configuration can be found at "
|
2368
|
+
f"{AWS_DOCS}#s3-bucket"
|
2369
|
+
)
|
2370
|
+
elif cloud_provider == "azure":
|
2371
|
+
flavor = "azure"
|
2372
|
+
elif cloud_provider == "gcp":
|
2373
|
+
flavor = "gcs"
|
2374
|
+
|
2375
|
+
selected_storage_idx = cli_utils.multi_choice_prompt(
|
2376
|
+
object_type=f"{cloud_provider.upper()} storages",
|
2377
|
+
choices=[[st] for st in available_storages],
|
2378
|
+
headers=["Storage"],
|
2379
|
+
prompt_text="Please choose one of the storages for the new artifact store:",
|
2380
|
+
)
|
2381
|
+
if selected_storage_idx is None:
|
2382
|
+
cli_utils.error("No storage selected.")
|
2383
|
+
|
2384
|
+
selected_storage = available_storages[selected_storage_idx]
|
2385
|
+
|
2386
|
+
config = {"path": selected_storage}
|
2387
|
+
service_connector_resource_id = selected_storage
|
2388
|
+
elif component_type == "orchestrator":
|
2389
|
+
if cloud_provider == "aws":
|
2390
|
+
available_orchestrators = []
|
2391
|
+
for each in service_connector_resource_models:
|
2392
|
+
types = []
|
2393
|
+
if each.resource_type == "aws-generic":
|
2394
|
+
types = ["Sagemaker", "Skypilot (EC2)"]
|
2395
|
+
if each.resource_type == "kubernetes-cluster":
|
2396
|
+
types = ["Kubernetes"]
|
2397
|
+
|
2398
|
+
if each.resource_ids:
|
2399
|
+
for orchestrator in each.resource_ids:
|
2400
|
+
for t in types:
|
2401
|
+
available_orchestrators.append([t, orchestrator])
|
2402
|
+
if not available_orchestrators:
|
2403
|
+
cli_utils.error(
|
2404
|
+
"We were unable to find any orchestrator engines "
|
2405
|
+
"available to the service connector. Please, verify "
|
2406
|
+
"that needed permission are granted for the "
|
2407
|
+
"service connector.\nDocumentation for the Generic "
|
2408
|
+
"AWS resource configuration can be found at "
|
2409
|
+
f"{AWS_DOCS}#generic-aws-resource\n"
|
2410
|
+
"Documentation for the Kubernetes resource "
|
2411
|
+
"configuration can be found at "
|
2412
|
+
f"{AWS_DOCS}#eks-kubernetes-cluster"
|
2413
|
+
)
|
2414
|
+
elif cloud_provider == "gcp":
|
2415
|
+
pass
|
2416
|
+
elif cloud_provider == "azure":
|
2417
|
+
pass
|
2418
|
+
|
2419
|
+
selected_orchestrator_idx = cli_utils.multi_choice_prompt(
|
2420
|
+
object_type=f"orchestrators on {cloud_provider.upper()}",
|
2421
|
+
choices=available_orchestrators,
|
2422
|
+
headers=["Orchestrator Type", "Orchestrator details"],
|
2423
|
+
prompt_text="Please choose one of the orchestrators for the new orchestrator:",
|
2424
|
+
)
|
2425
|
+
if selected_orchestrator_idx is None:
|
2426
|
+
cli_utils.error("No orchestrator selected.")
|
2427
|
+
|
2428
|
+
selected_orchestrator = available_orchestrators[
|
2429
|
+
selected_orchestrator_idx
|
2430
|
+
]
|
2431
|
+
|
2432
|
+
if selected_orchestrator[0] == "Sagemaker":
|
2433
|
+
flavor = "sagemaker"
|
2434
|
+
execution_role = Prompt.ask("Please enter an execution role ARN:")
|
2435
|
+
config = {"execution_role": execution_role}
|
2436
|
+
elif selected_orchestrator[0] == "Skypilot (EC2)":
|
2437
|
+
flavor = "vm_aws"
|
2438
|
+
config = {"region": selected_orchestrator[1]}
|
2439
|
+
elif selected_orchestrator[0] == "Kubernetes":
|
2440
|
+
flavor = "kubernetes"
|
2441
|
+
config = {}
|
2442
|
+
else:
|
2443
|
+
raise ValueError(
|
2444
|
+
f"Unknown orchestrator type {selected_orchestrator[0]}"
|
2445
|
+
)
|
2446
|
+
service_connector_resource_id = selected_orchestrator[1]
|
2447
|
+
elif component_type == "container_registry":
|
2448
|
+
available_registries: List[str] = []
|
2449
|
+
if cloud_provider == "aws":
|
2450
|
+
flavor = "aws"
|
2451
|
+
for each in service_connector_resource_models:
|
2452
|
+
if each.resource_type == "docker-registry":
|
2453
|
+
available_registries = each.resource_ids or []
|
2454
|
+
if not available_registries:
|
2455
|
+
cli_utils.error(
|
2456
|
+
"We were unable to find any container registries "
|
2457
|
+
"available to the service connector. Please, verify "
|
2458
|
+
"that needed permission are granted for the "
|
2459
|
+
"service connector.\nDocumentation for the ECR "
|
2460
|
+
"container registry resource configuration can "
|
2461
|
+
f"be found at {AWS_DOCS}#ecr-container-registry"
|
2462
|
+
)
|
2463
|
+
elif cloud_provider == "azure":
|
2464
|
+
flavor = "azure"
|
2465
|
+
elif cloud_provider == "gcp":
|
2466
|
+
flavor = "gcp"
|
2467
|
+
|
2468
|
+
selected_registry_idx = cli_utils.multi_choice_prompt(
|
2469
|
+
object_type=f"{cloud_provider.upper()} registries",
|
2470
|
+
choices=[[st] for st in available_registries],
|
2471
|
+
headers=["Container Registry"],
|
2472
|
+
prompt_text="Please choose one of the registries for the new container registry:",
|
2473
|
+
)
|
2474
|
+
if selected_registry_idx is None:
|
2475
|
+
cli_utils.error("No container registry selected.")
|
2476
|
+
selected_registry = available_registries[selected_registry_idx]
|
2477
|
+
config = {"uri": selected_registry}
|
2478
|
+
service_connector_resource_id = selected_registry
|
2479
|
+
else:
|
2480
|
+
raise ValueError(f"Unknown component type {component_type}")
|
2481
|
+
|
2482
|
+
return ComponentInfo(
|
2483
|
+
flavor=flavor,
|
2484
|
+
configuration=config,
|
2485
|
+
service_connector_index=service_connector_index,
|
2486
|
+
service_connector_resource_id=service_connector_resource_id,
|
2487
|
+
)
|