zenml-nightly 0.61.0.dev20240712__py3-none-any.whl → 0.62.0.dev20240727__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +2 -2
- RELEASE_NOTES.md +40 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/cli/stack.py +114 -248
- zenml/cli/stack_components.py +5 -3
- zenml/config/pipeline_spec.py +2 -2
- zenml/config/step_configurations.py +3 -3
- zenml/constants.py +3 -0
- zenml/enums.py +16 -0
- zenml/integrations/__init__.py +1 -0
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/constants.py +1 -0
- zenml/integrations/databricks/__init__.py +52 -0
- zenml/integrations/databricks/flavors/__init__.py +30 -0
- zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +147 -0
- zenml/integrations/databricks/model_deployers/__init__.py +20 -0
- zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
- zenml/integrations/databricks/orchestrators/__init__.py +20 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +497 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
- zenml/integrations/databricks/services/__init__.py +19 -0
- zenml/integrations/databricks/services/databricks_deployment.py +407 -0
- zenml/integrations/databricks/utils/__init__.py +14 -0
- zenml/integrations/databricks/utils/databricks_utils.py +87 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +44 -28
- zenml/integrations/great_expectations/data_validators/ge_data_validator.py +12 -8
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +88 -3
- zenml/integrations/huggingface/steps/accelerate_runner.py +1 -7
- zenml/integrations/kubernetes/__init__.py +3 -2
- zenml/integrations/kubernetes/flavors/__init__.py +8 -0
- zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +1 -13
- zenml/integrations/kubernetes/orchestrators/manifest_utils.py +22 -4
- zenml/integrations/kubernetes/pod_settings.py +4 -0
- zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
- zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +235 -0
- zenml/integrations/lightgbm/__init__.py +1 -0
- zenml/integrations/mlflow/__init__.py +1 -1
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +6 -2
- zenml/integrations/mlflow/services/mlflow_deployment.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -3
- zenml/integrations/skypilot_lambda/__init__.py +1 -1
- zenml/logging/step_logging.py +34 -35
- zenml/materializers/built_in_materializer.py +1 -1
- zenml/materializers/cloudpickle_materializer.py +1 -1
- zenml/model/model.py +1 -1
- zenml/models/v2/core/code_repository.py +2 -2
- zenml/models/v2/core/component.py +29 -0
- zenml/models/v2/core/server_settings.py +0 -20
- zenml/models/v2/misc/full_stack.py +32 -0
- zenml/models/v2/misc/stack_deployment.py +5 -0
- zenml/new/pipelines/run_utils.py +1 -1
- zenml/orchestrators/__init__.py +4 -0
- zenml/orchestrators/step_launcher.py +1 -0
- zenml/orchestrators/wheeled_orchestrator.py +147 -0
- zenml/service_connectors/service_connector_utils.py +408 -0
- zenml/stack_deployments/azure_stack_deployment.py +179 -0
- zenml/stack_deployments/gcp_stack_deployment.py +13 -4
- zenml/stack_deployments/stack_deployment.py +10 -0
- zenml/stack_deployments/utils.py +4 -0
- zenml/steps/base_step.py +7 -5
- zenml/utils/function_utils.py +1 -1
- zenml/utils/pipeline_docker_image_builder.py +8 -0
- zenml/utils/source_utils.py +4 -1
- zenml/zen_server/dashboard/assets/{404-DpJaNHKF.js → 404-B_YdvmwS.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-DJfzkHO1.js → @reactflow-l_1hUr1S.js} +1 -1
- zenml/zen_server/dashboard/assets/{AwarenessChannel-BYDLT2xC.js → AwarenessChannel-CFg5iX4Z.js} +1 -1
- zenml/zen_server/dashboard/assets/{CodeSnippet-BkOuRmyq.js → CodeSnippet-Dvkx_82E.js} +1 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-opiuBHHc.js +1 -0
- zenml/zen_server/dashboard/assets/{Commands-ZvWR1BRs.js → Commands-DoN1xrEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{CopyButton-DVwLkafa.js → CopyButton-Cr7xYEPb.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-C2IiqX4I.js → CsvVizualization-Ck-nZ43m.js} +3 -3
- zenml/zen_server/dashboard/assets/{Error-CqX0VqW_.js → Error-kLtljEOM.js} +1 -1
- zenml/zen_server/dashboard/assets/{ExecutionStatus-BoLUXR9t.js → ExecutionStatus-DguLLgTK.js} +1 -1
- zenml/zen_server/dashboard/assets/{Helpbox-LFydyVwh.js → Helpbox-BXUMP21n.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-DnENC0sh.js → Infobox-DSt0O-dm.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-CbJtYr0t.js → InlineAvatar-xsrsIGE-.js} +1 -1
- zenml/zen_server/dashboard/assets/Pagination-C6X-mifw.js +1 -0
- zenml/zen_server/dashboard/assets/{SetPassword-BYBdbQDo.js → SetPassword-BXGTWiwj.js} +1 -1
- zenml/zen_server/dashboard/assets/{SuccessStep-Nx743hll.js → SuccessStep-DZC60t0x.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DF9gSzE0.js → UpdatePasswordSchemas-DGvwFWO1.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-BiEMg7rd.js → chevron-right-double-CZBOf6JM.js} +1 -1
- zenml/zen_server/dashboard/assets/cloud-only-C_yFCAkP.js +1 -0
- zenml/zen_server/dashboard/assets/index-BczVOqUf.js +55 -0
- zenml/zen_server/dashboard/assets/index-EpMIKgrI.css +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-BUnVASxp.js → login-mutation-CrHrndTI.js} +1 -1
- zenml/zen_server/dashboard/assets/logs-D8k8BVFf.js +1 -0
- zenml/zen_server/dashboard/assets/{not-found-B4VnX8gK.js → not-found-DYa4pC-C.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-CsUhPmou.js → package-B3fWP-Dh.js} +1 -1
- zenml/zen_server/dashboard/assets/page-1h_sD1jz.js +1 -0
- zenml/zen_server/dashboard/assets/{page-Sxn82W-5.js → page-1iL8aMqs.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DMOYZppS.js → page-2grKx_MY.js} +1 -1
- zenml/zen_server/dashboard/assets/page-5NCOHOsy.js +1 -0
- zenml/zen_server/dashboard/assets/{page-JyfeDUfu.js → page-8a4UMKXZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-Bx6o0ARS.js → page-B6h3iaHJ.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDns21Iz.js +1 -0
- zenml/zen_server/dashboard/assets/{page-3efNCDeb.js → page-BhgCDInH.js} +2 -2
- zenml/zen_server/dashboard/assets/{page-DKlIdAe5.js → page-Bi-wtWiO.js} +2 -2
- zenml/zen_server/dashboard/assets/{page-7zTHbhhI.js → page-BkeAAYwp.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-CRTJ0UuR.js → page-BkuQDIf-.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BnaevhnB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BEs6jK71.js → page-Bq0YxkLV.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Bs2F4eoD.js +2 -0
- zenml/zen_server/dashboard/assets/{page-CUZIGO-3.js → page-C6-UGEbH.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-Xu8JEjSU.js → page-CCNRIt_f.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DvCvroOM.js → page-CHNxpz3n.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BpSqIf4B.js → page-DgorQFqi.js} +1 -1
- zenml/zen_server/dashboard/assets/page-K8ebxVIs.js +1 -0
- zenml/zen_server/dashboard/assets/{page-Cx67M0QT.js → page-MFQyIJd3.js} +1 -1
- zenml/zen_server/dashboard/assets/page-TgCF0P_U.js +1 -0
- zenml/zen_server/dashboard/assets/page-ZnCEe-eK.js +9 -0
- zenml/zen_server/dashboard/assets/{page-Dc_7KMQE.js → page-uA5prJGY.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-D7HJNBWx.js +1 -0
- zenml/zen_server/dashboard/assets/plus-C8WOyCzt.js +1 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-Cficsl6d.js +1 -0
- zenml/zen_server/dashboard/assets/update-server-settings-mutation-7d8xi1tS.js +1 -0
- zenml/zen_server/dashboard/assets/{url-DuQMeqYA.js → url-D7mAQGUM.js} +1 -1
- zenml/zen_server/dashboard/index.html +4 -4
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js → precache-manifest.12246c7548e71e2c4438e496360de80c.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/main.3b27024b.chunk.js +2 -0
- zenml/zen_server/dashboard_legacy/static/js/{main.382439a7.chunk.js.map → main.3b27024b.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/rbac/utils.py +10 -2
- zenml/zen_server/routers/devices_endpoints.py +4 -1
- zenml/zen_server/routers/server_endpoints.py +29 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +57 -0
- zenml/zen_server/routers/steps_endpoints.py +2 -1
- zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
- zenml/zen_stores/rest_zen_store.py +4 -0
- zenml/zen_stores/schemas/component_schemas.py +14 -0
- zenml/zen_stores/schemas/server_settings_schemas.py +23 -11
- zenml/zen_stores/sql_zen_store.py +151 -1
- {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/METADATA +5 -5
- {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/RECORD +144 -121
- zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +0 -1
- zenml/zen_server/dashboard/assets/chevron-down-D_ZlKMqH.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +0 -1
- zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +0 -1
- zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +0 -55
- zenml/zen_server/dashboard/assets/index-inApY3KQ.css +0 -1
- zenml/zen_server/dashboard/assets/page-C43QGHTt.js +0 -9
- zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +0 -1
- zenml/zen_server/dashboard/assets/page-CaopxiU1.js +0 -1
- zenml/zen_server/dashboard/assets/page-D7Z399xy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +0 -1
- zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +0 -2
- zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +0 -1
- zenml/zen_server/dashboard/assets/page-TKXERe16.js +0 -1
- zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +0 -1
- zenml/zen_server/dashboard/assets/update-server-settings-mutation-CR8e3Sir.js +0 -1
- zenml/zen_server/dashboard_legacy/static/js/main.382439a7.chunk.js +0 -2
- {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,408 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Utility methods for Service Connectors."""
|
15
|
+
|
16
|
+
from typing import Dict, List, Union
|
17
|
+
from uuid import UUID
|
18
|
+
|
19
|
+
from zenml.client import Client
|
20
|
+
from zenml.enums import StackComponentType
|
21
|
+
from zenml.models.v2.core.service_connector import ServiceConnectorRequest
|
22
|
+
from zenml.models.v2.misc.full_stack import (
|
23
|
+
ResourcesInfo,
|
24
|
+
ServiceConnectorInfo,
|
25
|
+
ServiceConnectorResourcesInfo,
|
26
|
+
)
|
27
|
+
from zenml.utils.pagination_utils import depaginate
|
28
|
+
|
29
|
+
|
30
|
+
def _prepare_resource_info(
|
31
|
+
connector_details: Union[UUID, ServiceConnectorInfo],
|
32
|
+
resource_ids: List[str],
|
33
|
+
stack_component_type: StackComponentType,
|
34
|
+
flavor: str,
|
35
|
+
required_configuration: Dict[str, str],
|
36
|
+
flavor_display_name: str,
|
37
|
+
use_resource_value_as_fixed_config: bool = False,
|
38
|
+
) -> ResourcesInfo:
|
39
|
+
existing_components = []
|
40
|
+
if isinstance(connector_details, UUID):
|
41
|
+
existing_components = depaginate(
|
42
|
+
Client().list_stack_components,
|
43
|
+
type=stack_component_type.value,
|
44
|
+
connector_id=connector_details,
|
45
|
+
flavor=flavor,
|
46
|
+
)
|
47
|
+
return ResourcesInfo(
|
48
|
+
flavor=flavor,
|
49
|
+
required_configuration=required_configuration,
|
50
|
+
flavor_display_name=flavor_display_name,
|
51
|
+
use_resource_value_as_fixed_config=use_resource_value_as_fixed_config,
|
52
|
+
accessible_by_service_connector=resource_ids,
|
53
|
+
connected_through_service_connector=existing_components,
|
54
|
+
)
|
55
|
+
|
56
|
+
|
57
|
+
def _raise_specific_cloud_exception_if_needed(
|
58
|
+
cloud_provider: str,
|
59
|
+
artifact_stores: List[ResourcesInfo],
|
60
|
+
orchestrators: List[ResourcesInfo],
|
61
|
+
container_registries: List[ResourcesInfo],
|
62
|
+
) -> None:
|
63
|
+
AWS_DOCS = (
|
64
|
+
"https://docs.zenml.io/how-to/auth-management/aws-service-connector"
|
65
|
+
)
|
66
|
+
GCP_DOCS = (
|
67
|
+
"https://docs.zenml.io/how-to/auth-management/gcp-service-connector"
|
68
|
+
)
|
69
|
+
AZURE_DOCS = (
|
70
|
+
"https://docs.zenml.io/how-to/auth-management/azure-service-connector"
|
71
|
+
)
|
72
|
+
|
73
|
+
if not artifact_stores:
|
74
|
+
error_msg = (
|
75
|
+
"We were unable to find any {obj_name} available "
|
76
|
+
"to configured service connector. Please, verify "
|
77
|
+
"that needed permission are granted for the "
|
78
|
+
"service connector.\nDocumentation for the "
|
79
|
+
"{obj_name} configuration can be found at "
|
80
|
+
"{docs}"
|
81
|
+
)
|
82
|
+
if cloud_provider == "aws":
|
83
|
+
raise ValueError(
|
84
|
+
error_msg.format(
|
85
|
+
obj_name="S3 Bucket", docs=f"{AWS_DOCS}#s3-bucket"
|
86
|
+
)
|
87
|
+
)
|
88
|
+
elif cloud_provider == "gcp":
|
89
|
+
raise ValueError(
|
90
|
+
error_msg.format(
|
91
|
+
obj_name="GCS Bucket", docs=f"{GCP_DOCS}#gcs-bucket"
|
92
|
+
)
|
93
|
+
)
|
94
|
+
elif cloud_provider == "azure":
|
95
|
+
raise ValueError(
|
96
|
+
error_msg.format(
|
97
|
+
obj_name="Blob Container",
|
98
|
+
docs=f"{AZURE_DOCS}#azure-blob-storage-container",
|
99
|
+
)
|
100
|
+
)
|
101
|
+
if not orchestrators:
|
102
|
+
error_msg = (
|
103
|
+
"We were unable to find any orchestrator engines "
|
104
|
+
"available to the service connector. Please, verify "
|
105
|
+
"that needed permission are granted for the "
|
106
|
+
"service connector.\nDocumentation for the Generic "
|
107
|
+
"{cloud_name} resource configuration can be found at "
|
108
|
+
"{gen_docs}\n Documentation for the {k8s_name} resource "
|
109
|
+
"configuration can be found at {k8s_docs}"
|
110
|
+
)
|
111
|
+
if cloud_provider == "aws":
|
112
|
+
raise ValueError(
|
113
|
+
error_msg.format(
|
114
|
+
cloud_name="AWS",
|
115
|
+
gen_docs=f"{AWS_DOCS}#generic-aws-resource",
|
116
|
+
k8s_name="EKS",
|
117
|
+
k8s_docs=f"{AWS_DOCS}#eks-kubernetes-cluster",
|
118
|
+
)
|
119
|
+
)
|
120
|
+
|
121
|
+
elif cloud_provider == "gcp":
|
122
|
+
raise ValueError(
|
123
|
+
error_msg.format(
|
124
|
+
cloud_name="GCP",
|
125
|
+
gen_docs=f"{GCP_DOCS}#generic-gcp-resource",
|
126
|
+
k8s_name="GKE",
|
127
|
+
k8s_docs=f"{GCP_DOCS}#gke-kubernetes-cluster",
|
128
|
+
)
|
129
|
+
)
|
130
|
+
elif cloud_provider == "azure":
|
131
|
+
raise ValueError(
|
132
|
+
error_msg.format(
|
133
|
+
cloud_name="Azure",
|
134
|
+
gen_docs=f"{AZURE_DOCS}#generic-azure-resource",
|
135
|
+
k8s_name="AKS",
|
136
|
+
k8s_docs=f"{AZURE_DOCS}#aks-kubernetes-cluster",
|
137
|
+
)
|
138
|
+
)
|
139
|
+
if not container_registries:
|
140
|
+
error_msg = (
|
141
|
+
"We were unable to find any container registries "
|
142
|
+
"available to the service connector. Please, verify "
|
143
|
+
"that needed permission are granted for the "
|
144
|
+
"service connector.\nDocumentation for the {registry_name} "
|
145
|
+
"container registry resource configuration can "
|
146
|
+
"be found at {docs_link}"
|
147
|
+
)
|
148
|
+
if cloud_provider == "aws":
|
149
|
+
raise ValueError(
|
150
|
+
error_msg.format(
|
151
|
+
registry_name="ECR",
|
152
|
+
docs_link=f"{AWS_DOCS}#ecr-container-registry",
|
153
|
+
)
|
154
|
+
)
|
155
|
+
elif cloud_provider == "gcp":
|
156
|
+
raise ValueError(
|
157
|
+
error_msg.format(
|
158
|
+
registry_name="GCR",
|
159
|
+
docs_link=f"{GCP_DOCS}#gcr-container-registry",
|
160
|
+
)
|
161
|
+
)
|
162
|
+
elif cloud_provider == "azure":
|
163
|
+
raise ValueError(
|
164
|
+
error_msg.format(
|
165
|
+
registry_name="ACR",
|
166
|
+
docs_link=f"{AZURE_DOCS}#acr-container-registry",
|
167
|
+
)
|
168
|
+
)
|
169
|
+
|
170
|
+
|
171
|
+
def get_resources_options_from_resource_model_for_full_stack(
|
172
|
+
connector_details: Union[UUID, ServiceConnectorInfo],
|
173
|
+
) -> ServiceConnectorResourcesInfo:
|
174
|
+
"""Get the resource options from the resource model for the full stack.
|
175
|
+
|
176
|
+
Args:
|
177
|
+
connector_details: The service connector details (UUID or Info).
|
178
|
+
|
179
|
+
Returns:
|
180
|
+
All available service connector resource options.
|
181
|
+
"""
|
182
|
+
client = Client()
|
183
|
+
zen_store = client.zen_store
|
184
|
+
|
185
|
+
if isinstance(connector_details, UUID):
|
186
|
+
resource_model = zen_store.verify_service_connector(
|
187
|
+
connector_details,
|
188
|
+
list_resources=True,
|
189
|
+
)
|
190
|
+
else:
|
191
|
+
resource_model = zen_store.verify_service_connector_config(
|
192
|
+
service_connector=ServiceConnectorRequest(
|
193
|
+
user=client.active_user.id,
|
194
|
+
workspace=client.active_workspace.id,
|
195
|
+
name="fake",
|
196
|
+
connector_type=connector_details.type,
|
197
|
+
auth_method=connector_details.auth_method,
|
198
|
+
configuration=connector_details.configuration,
|
199
|
+
secrets={},
|
200
|
+
labels={},
|
201
|
+
),
|
202
|
+
list_resources=True,
|
203
|
+
)
|
204
|
+
|
205
|
+
resources = resource_model.resources
|
206
|
+
|
207
|
+
if isinstance(
|
208
|
+
resource_model.connector_type,
|
209
|
+
str,
|
210
|
+
):
|
211
|
+
connector_type = resource_model.connector_type
|
212
|
+
else:
|
213
|
+
connector_type = resource_model.connector_type.connector_type
|
214
|
+
|
215
|
+
artifact_stores: List[ResourcesInfo] = []
|
216
|
+
orchestrators: List[ResourcesInfo] = []
|
217
|
+
container_registries: List[ResourcesInfo] = []
|
218
|
+
|
219
|
+
if connector_type == "aws":
|
220
|
+
for each in resources:
|
221
|
+
if each.resource_ids:
|
222
|
+
if each.resource_type == "s3-bucket":
|
223
|
+
artifact_stores.append(
|
224
|
+
_prepare_resource_info(
|
225
|
+
connector_details=connector_details,
|
226
|
+
resource_ids=each.resource_ids,
|
227
|
+
stack_component_type=StackComponentType.ARTIFACT_STORE,
|
228
|
+
flavor="s3",
|
229
|
+
required_configuration={"path": "Path"},
|
230
|
+
use_resource_value_as_fixed_config=True,
|
231
|
+
flavor_display_name="S3 Bucket",
|
232
|
+
)
|
233
|
+
)
|
234
|
+
if each.resource_type == "aws-generic":
|
235
|
+
orchestrators.append(
|
236
|
+
_prepare_resource_info(
|
237
|
+
connector_details=connector_details,
|
238
|
+
resource_ids=each.resource_ids,
|
239
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
240
|
+
flavor="sagemaker",
|
241
|
+
required_configuration={
|
242
|
+
"execution_role": "execution role ARN"
|
243
|
+
},
|
244
|
+
flavor_display_name="AWS Sagemaker",
|
245
|
+
)
|
246
|
+
)
|
247
|
+
orchestrators.append(
|
248
|
+
_prepare_resource_info(
|
249
|
+
connector_details=connector_details,
|
250
|
+
resource_ids=each.resource_ids,
|
251
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
252
|
+
flavor="vm_aws",
|
253
|
+
required_configuration={"region": "region"},
|
254
|
+
use_resource_value_as_fixed_config=True,
|
255
|
+
flavor_display_name="Skypilot (EC2)",
|
256
|
+
)
|
257
|
+
)
|
258
|
+
|
259
|
+
if each.resource_type == "kubernetes-cluster":
|
260
|
+
orchestrators.append(
|
261
|
+
_prepare_resource_info(
|
262
|
+
connector_details=connector_details,
|
263
|
+
resource_ids=each.resource_ids,
|
264
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
265
|
+
flavor="kubernetes",
|
266
|
+
required_configuration={},
|
267
|
+
flavor_display_name="Kubernetes",
|
268
|
+
)
|
269
|
+
)
|
270
|
+
if each.resource_type == "docker-registry":
|
271
|
+
container_registries.append(
|
272
|
+
_prepare_resource_info(
|
273
|
+
connector_details=connector_details,
|
274
|
+
resource_ids=each.resource_ids,
|
275
|
+
stack_component_type=StackComponentType.CONTAINER_REGISTRY,
|
276
|
+
flavor="aws",
|
277
|
+
required_configuration={"uri": "URI"},
|
278
|
+
use_resource_value_as_fixed_config=True,
|
279
|
+
flavor_display_name="ECR",
|
280
|
+
)
|
281
|
+
)
|
282
|
+
|
283
|
+
elif connector_type == "gcp":
|
284
|
+
for each in resources:
|
285
|
+
if each.resource_ids:
|
286
|
+
if each.resource_type == "gcs-bucket":
|
287
|
+
artifact_stores.append(
|
288
|
+
_prepare_resource_info(
|
289
|
+
connector_details=connector_details,
|
290
|
+
resource_ids=each.resource_ids,
|
291
|
+
stack_component_type=StackComponentType.ARTIFACT_STORE,
|
292
|
+
flavor="gcp",
|
293
|
+
required_configuration={},
|
294
|
+
flavor_display_name="GCS Bucket",
|
295
|
+
)
|
296
|
+
)
|
297
|
+
if each.resource_type == "gcp-generic":
|
298
|
+
orchestrators.append(
|
299
|
+
_prepare_resource_info(
|
300
|
+
connector_details=connector_details,
|
301
|
+
resource_ids=each.resource_ids,
|
302
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
303
|
+
flavor="vertex",
|
304
|
+
required_configuration={"location": "region name"},
|
305
|
+
flavor_display_name="Vertex AI",
|
306
|
+
)
|
307
|
+
)
|
308
|
+
orchestrators.append(
|
309
|
+
_prepare_resource_info(
|
310
|
+
connector_details=connector_details,
|
311
|
+
resource_ids=each.resource_ids,
|
312
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
313
|
+
flavor="vm_gcp",
|
314
|
+
required_configuration={"region": "region name"},
|
315
|
+
flavor_display_name="Skypilot (Compute)",
|
316
|
+
)
|
317
|
+
)
|
318
|
+
|
319
|
+
if each.resource_type == "kubernetes-cluster":
|
320
|
+
orchestrators.append(
|
321
|
+
_prepare_resource_info(
|
322
|
+
connector_details=connector_details,
|
323
|
+
resource_ids=each.resource_ids,
|
324
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
325
|
+
flavor="kubernetes",
|
326
|
+
required_configuration={},
|
327
|
+
flavor_display_name="Kubernetes",
|
328
|
+
)
|
329
|
+
)
|
330
|
+
if each.resource_type == "docker-registry":
|
331
|
+
container_registries.append(
|
332
|
+
_prepare_resource_info(
|
333
|
+
connector_details=connector_details,
|
334
|
+
resource_ids=each.resource_ids,
|
335
|
+
stack_component_type=StackComponentType.CONTAINER_REGISTRY,
|
336
|
+
flavor="gcp",
|
337
|
+
required_configuration={"uri": "URI"},
|
338
|
+
use_resource_value_as_fixed_config=True,
|
339
|
+
flavor_display_name="GCR",
|
340
|
+
)
|
341
|
+
)
|
342
|
+
|
343
|
+
elif connector_type == "azure":
|
344
|
+
for each in resources:
|
345
|
+
if each.resource_ids:
|
346
|
+
if each.resource_type == "blob-container":
|
347
|
+
artifact_stores.append(
|
348
|
+
_prepare_resource_info(
|
349
|
+
connector_details=connector_details,
|
350
|
+
resource_ids=each.resource_ids,
|
351
|
+
stack_component_type=StackComponentType.ARTIFACT_STORE,
|
352
|
+
flavor="azure",
|
353
|
+
required_configuration={},
|
354
|
+
flavor_display_name="Blob container",
|
355
|
+
)
|
356
|
+
)
|
357
|
+
if each.resource_type == "azure-generic":
|
358
|
+
# No native orchestrator ATM
|
359
|
+
orchestrators.append(
|
360
|
+
_prepare_resource_info(
|
361
|
+
connector_details=connector_details,
|
362
|
+
resource_ids=each.resource_ids,
|
363
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
364
|
+
flavor="vm_azure",
|
365
|
+
required_configuration={"region": "region name"},
|
366
|
+
flavor_display_name="Skypilot (VM)",
|
367
|
+
)
|
368
|
+
)
|
369
|
+
|
370
|
+
if each.resource_type == "kubernetes-cluster":
|
371
|
+
orchestrators.append(
|
372
|
+
_prepare_resource_info(
|
373
|
+
connector_details=connector_details,
|
374
|
+
resource_ids=each.resource_ids,
|
375
|
+
stack_component_type=StackComponentType.ORCHESTRATOR,
|
376
|
+
flavor="kubernetes",
|
377
|
+
required_configuration={},
|
378
|
+
flavor_display_name="Kubernetes",
|
379
|
+
)
|
380
|
+
)
|
381
|
+
if each.resource_type == "docker-registry":
|
382
|
+
container_registries.append(
|
383
|
+
_prepare_resource_info(
|
384
|
+
connector_details=connector_details,
|
385
|
+
resource_ids=each.resource_ids,
|
386
|
+
stack_component_type=StackComponentType.CONTAINER_REGISTRY,
|
387
|
+
flavor="azure",
|
388
|
+
required_configuration={"uri": "URI"},
|
389
|
+
use_resource_value_as_fixed_config=True,
|
390
|
+
flavor_display_name="ACR",
|
391
|
+
)
|
392
|
+
)
|
393
|
+
|
394
|
+
_raise_specific_cloud_exception_if_needed(
|
395
|
+
cloud_provider=connector_type,
|
396
|
+
artifact_stores=artifact_stores,
|
397
|
+
orchestrators=orchestrators,
|
398
|
+
container_registries=container_registries,
|
399
|
+
)
|
400
|
+
|
401
|
+
return ServiceConnectorResourcesInfo(
|
402
|
+
connector_type=connector_type,
|
403
|
+
components_resources_info={
|
404
|
+
StackComponentType.ARTIFACT_STORE: artifact_stores,
|
405
|
+
StackComponentType.ORCHESTRATOR: orchestrators,
|
406
|
+
StackComponentType.CONTAINER_REGISTRY: container_registries,
|
407
|
+
},
|
408
|
+
)
|
@@ -0,0 +1,179 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Functionality to deploy a ZenML stack to Azure."""
|
15
|
+
|
16
|
+
import re
|
17
|
+
from typing import ClassVar, Dict, List
|
18
|
+
|
19
|
+
from zenml.enums import StackDeploymentProvider
|
20
|
+
from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
|
21
|
+
|
22
|
+
|
23
|
+
# TODO: this class just implements the regions list, and is not suitable for other
|
24
|
+
# deployment tasks.
|
25
|
+
class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
|
26
|
+
"""Azure ZenML Cloud Stack Deployment."""
|
27
|
+
|
28
|
+
provider: ClassVar[StackDeploymentProvider] = StackDeploymentProvider.AZURE
|
29
|
+
|
30
|
+
@classmethod
|
31
|
+
def description(cls) -> str:
|
32
|
+
"""Return a description of the ZenML Cloud Stack Deployment.
|
33
|
+
|
34
|
+
This will be displayed when the user is prompted to deploy
|
35
|
+
the ZenML stack.
|
36
|
+
|
37
|
+
Returns:
|
38
|
+
A MarkDown description of the ZenML Cloud Stack Deployment.
|
39
|
+
"""
|
40
|
+
# TODO: Implement this
|
41
|
+
return ""
|
42
|
+
|
43
|
+
@classmethod
|
44
|
+
def instructions(cls) -> str:
|
45
|
+
"""Return instructions on how to deploy the ZenML stack to the specified cloud provider.
|
46
|
+
|
47
|
+
This will be displayed before the user is prompted to deploy the ZenML
|
48
|
+
stack.
|
49
|
+
|
50
|
+
Returns:
|
51
|
+
MarkDown instructions on how to deploy the ZenML stack to the
|
52
|
+
specified cloud provider.
|
53
|
+
"""
|
54
|
+
# TODO: Implement this
|
55
|
+
return ""
|
56
|
+
|
57
|
+
@classmethod
|
58
|
+
def post_deploy_instructions(cls) -> str:
|
59
|
+
"""Return instructions on what to do after the deployment is complete.
|
60
|
+
|
61
|
+
This will be displayed after the deployment is complete.
|
62
|
+
|
63
|
+
Returns:
|
64
|
+
MarkDown instructions on what to do after the deployment is
|
65
|
+
complete.
|
66
|
+
"""
|
67
|
+
# TODO: Implement this
|
68
|
+
return ""
|
69
|
+
|
70
|
+
@classmethod
|
71
|
+
def integrations(cls) -> List[str]:
|
72
|
+
"""Return the ZenML integrations required for the stack.
|
73
|
+
|
74
|
+
Returns:
|
75
|
+
The list of ZenML integrations that need to be installed for the
|
76
|
+
stack to be usable.
|
77
|
+
"""
|
78
|
+
return ["azure"]
|
79
|
+
|
80
|
+
@classmethod
|
81
|
+
def permissions(cls) -> Dict[str, List[str]]:
|
82
|
+
"""Return the permissions granted to ZenML to access the cloud resources.
|
83
|
+
|
84
|
+
Returns:
|
85
|
+
The permissions granted to ZenML to access the cloud resources, as
|
86
|
+
a dictionary grouping permissions by resource.
|
87
|
+
"""
|
88
|
+
# TODO: Implement this
|
89
|
+
return {}
|
90
|
+
|
91
|
+
@classmethod
|
92
|
+
def locations(cls) -> Dict[str, str]:
|
93
|
+
"""Return the locations where the ZenML stack can be deployed.
|
94
|
+
|
95
|
+
Returns:
|
96
|
+
The regions where the ZenML stack can be deployed as a map of region
|
97
|
+
names to region descriptions.
|
98
|
+
"""
|
99
|
+
# Based on `az account list-locations -o table` on 16.07.2024
|
100
|
+
return {
|
101
|
+
"(US) East US": "eastus",
|
102
|
+
"(US) South Central US": "southcentralus",
|
103
|
+
"(US) West US 2": "westus2",
|
104
|
+
"(US) West US 3": "westus3",
|
105
|
+
"(Asia Pacific) Australia East": "australiaeast",
|
106
|
+
"(Asia Pacific) Southeast Asia": "southeastasia",
|
107
|
+
"(Europe) North Europe": "northeurope",
|
108
|
+
"(Europe) Sweden Central": "swedencentral",
|
109
|
+
"(Europe) UK South": "uksouth",
|
110
|
+
"(Europe) West Europe": "westeurope",
|
111
|
+
"(US) Central US": "centralus",
|
112
|
+
"(Africa) South Africa North": "southafricanorth",
|
113
|
+
"(Asia Pacific) Central India": "centralindia",
|
114
|
+
"(Asia Pacific) East Asia": "eastasia",
|
115
|
+
"(Asia Pacific) Japan East": "japaneast",
|
116
|
+
"(Asia Pacific) Korea Central": "koreacentral",
|
117
|
+
"(Canada) Canada Central": "canadacentral",
|
118
|
+
"(Europe) France Central": "francecentral",
|
119
|
+
"(Europe) Germany West Central": "germanywestcentral",
|
120
|
+
"(Europe) Italy North": "italynorth",
|
121
|
+
"(Europe) Norway East": "norwayeast",
|
122
|
+
"(Europe) Poland Central": "polandcentral",
|
123
|
+
"(Europe) Spain Central": "spaincentral",
|
124
|
+
"(Europe) Switzerland North": "switzerlandnorth",
|
125
|
+
"(Mexico) Mexico Central": "mexicocentral",
|
126
|
+
"(Middle East) UAE North": "uaenorth",
|
127
|
+
"(South America) Brazil South": "brazilsouth",
|
128
|
+
"(Middle East) Israel Central": "israelcentral",
|
129
|
+
"(Middle East) Qatar Central": "qatarcentral",
|
130
|
+
"(US) Central US (Stage)": "centralusstage",
|
131
|
+
"(US) East US (Stage)": "eastusstage",
|
132
|
+
"(US) East US 2 (Stage)": "eastus2stage",
|
133
|
+
"(US) North Central US (Stage)": "northcentralusstage",
|
134
|
+
"(US) South Central US (Stage)": "southcentralusstage",
|
135
|
+
"(US) West US (Stage)": "westusstage",
|
136
|
+
"(US) West US 2 (Stage)": "westus2stage",
|
137
|
+
"(Asia Pacific) East Asia (Stage)": "eastasiastage",
|
138
|
+
"(Asia Pacific) Southeast Asia (Stage)": "southeastasiastage",
|
139
|
+
"(South America) Brazil US": "brazilus",
|
140
|
+
"(US) East US 2": "eastus2",
|
141
|
+
"(US) East US STG": "eastusstg",
|
142
|
+
"(US) North Central US": "northcentralus",
|
143
|
+
"(US) West US": "westus",
|
144
|
+
"(Asia Pacific) Japan West": "japanwest",
|
145
|
+
"(Asia Pacific) Jio India West": "jioindiawest",
|
146
|
+
"(US) Central US EUAP": "centraluseuap",
|
147
|
+
"(US) East US 2 EUAP": "eastus2euap",
|
148
|
+
"(US) West Central US": "westcentralus",
|
149
|
+
"(Africa) South Africa West": "southafricawest",
|
150
|
+
"(Asia Pacific) Australia Central": "australiacentral",
|
151
|
+
"(Asia Pacific) Australia Central 2": "australiacentral2",
|
152
|
+
"(Asia Pacific) Australia Southeast": "australiasoutheast",
|
153
|
+
"(Asia Pacific) Jio India Central": "jioindiacentral",
|
154
|
+
"(Asia Pacific) Korea South": "koreasouth",
|
155
|
+
"(Asia Pacific) South India": "southindia",
|
156
|
+
"(Asia Pacific) West India": "westindia",
|
157
|
+
"(Canada) Canada East": "canadaeast",
|
158
|
+
"(Europe) France South": "francesouth",
|
159
|
+
"(Europe) Germany North": "germanynorth",
|
160
|
+
"(Europe) Norway West": "norwaywest",
|
161
|
+
"(Europe) Switzerland West": "switzerlandwest",
|
162
|
+
"(Europe) UK West": "ukwest",
|
163
|
+
"(Middle East) UAE Central": "uaecentral",
|
164
|
+
"(South America) Brazil Southeast": "brazilsoutheast",
|
165
|
+
}
|
166
|
+
|
167
|
+
@classmethod
|
168
|
+
def skypilot_default_regions(cls) -> Dict[str, str]:
|
169
|
+
"""Returns the regions supported by default for the Skypilot.
|
170
|
+
|
171
|
+
Returns:
|
172
|
+
The regions supported by default for the Skypilot.
|
173
|
+
"""
|
174
|
+
matcher = re.compile(r".*us\d*( |$)")
|
175
|
+
return {
|
176
|
+
k: v
|
177
|
+
for k, v in cls.locations().items()
|
178
|
+
if "(US)" in k and matcher.match(v)
|
179
|
+
}
|
@@ -13,6 +13,7 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Functionality to deploy a ZenML stack to GCP."""
|
15
15
|
|
16
|
+
import re
|
16
17
|
from typing import ClassVar, Dict, List
|
17
18
|
|
18
19
|
from zenml.enums import StackDeploymentProvider
|
@@ -208,6 +209,16 @@ GCP project and to clean up the resources created by the stack by using
|
|
208
209
|
"US West (Las Vegas)": "us-west4",
|
209
210
|
}
|
210
211
|
|
212
|
+
@classmethod
|
213
|
+
def skypilot_default_regions(cls) -> Dict[str, str]:
|
214
|
+
"""Returns the regions supported by default for the Skypilot.
|
215
|
+
|
216
|
+
Returns:
|
217
|
+
The regions supported by default for the Skypilot.
|
218
|
+
"""
|
219
|
+
matcher = re.compile(r"us-.*")
|
220
|
+
return {k: v for k, v in cls.locations().items() if matcher.match(v)}
|
221
|
+
|
211
222
|
def get_deployment_config(
|
212
223
|
self,
|
213
224
|
) -> StackDeploymentConfig:
|
@@ -236,8 +247,6 @@ GCP project and to clean up the resources created by the stack by using
|
|
236
247
|
cloudshell_open_in_editor="gcp-gar-gcs-vertex.jinja,gcp-gar-gcs-vertex-deploy.sh",
|
237
248
|
cloudshell_tutorial="gcp-gar-gcs-vertex.md",
|
238
249
|
ephemeral="true",
|
239
|
-
# TODO: remove this before the branch is merged
|
240
|
-
cloudshell_git_branch="feature/prd-482-gcp-stack-deployment",
|
241
250
|
)
|
242
251
|
# Encode the parameters as URL query parameters
|
243
252
|
query_params = "&".join([f"{k}={v}" for k, v in params.items()])
|
@@ -251,8 +260,8 @@ ZENML_STACK_NAME={self.stack_name}
|
|
251
260
|
ZENML_STACK_REGION={self.location or "europe-west3"}
|
252
261
|
ZENML_SERVER_URL={self.zenml_server_url}
|
253
262
|
ZENML_SERVER_API_TOKEN={self.zenml_server_api_token}
|
254
|
-
### END CONFIGURATION ###
|
255
|
-
|
263
|
+
### END CONFIGURATION ###"""
|
264
|
+
|
256
265
|
return StackDeploymentConfig(
|
257
266
|
deployment_url=url,
|
258
267
|
deployment_url_text="GCP Cloud Shell Console",
|
@@ -105,6 +105,15 @@ class ZenMLCloudStackDeployment(BaseModel):
|
|
105
105
|
names to region descriptions.
|
106
106
|
"""
|
107
107
|
|
108
|
+
@classmethod
|
109
|
+
def skypilot_default_regions(cls) -> Dict[str, str]:
|
110
|
+
"""Returns the regions supported by default for the Skypilot.
|
111
|
+
|
112
|
+
Returns:
|
113
|
+
The regions supported by default for the Skypilot.
|
114
|
+
"""
|
115
|
+
return cls.locations()
|
116
|
+
|
108
117
|
@classmethod
|
109
118
|
def get_deployment_info(cls) -> StackDeploymentInfo:
|
110
119
|
"""Return information about the ZenML Cloud Stack Deployment.
|
@@ -120,6 +129,7 @@ class ZenMLCloudStackDeployment(BaseModel):
|
|
120
129
|
integrations=cls.integrations(),
|
121
130
|
permissions=cls.permissions(),
|
122
131
|
locations=cls.locations(),
|
132
|
+
skypilot_default_regions=cls.skypilot_default_regions(),
|
123
133
|
)
|
124
134
|
|
125
135
|
@abstractmethod
|
zenml/stack_deployments/utils.py
CHANGED
@@ -19,6 +19,9 @@ from zenml.enums import StackDeploymentProvider
|
|
19
19
|
from zenml.stack_deployments.aws_stack_deployment import (
|
20
20
|
AWSZenMLCloudStackDeployment,
|
21
21
|
)
|
22
|
+
from zenml.stack_deployments.azure_stack_deployment import (
|
23
|
+
AZUREZenMLCloudStackDeployment,
|
24
|
+
)
|
22
25
|
from zenml.stack_deployments.gcp_stack_deployment import (
|
23
26
|
GCPZenMLCloudStackDeployment,
|
24
27
|
)
|
@@ -27,6 +30,7 @@ from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
|
|
27
30
|
STACK_DEPLOYMENT_PROVIDERS = {
|
28
31
|
StackDeploymentProvider.AWS: AWSZenMLCloudStackDeployment,
|
29
32
|
StackDeploymentProvider.GCP: GCPZenMLCloudStackDeployment,
|
33
|
+
StackDeploymentProvider.AZURE: AZUREZenMLCloudStackDeployment,
|
30
34
|
}
|
31
35
|
|
32
36
|
|