zenml-nightly 0.61.0.dev20240712__py3-none-any.whl → 0.62.0.dev20240727__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. README.md +2 -2
  2. RELEASE_NOTES.md +40 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/cli/stack.py +114 -248
  6. zenml/cli/stack_components.py +5 -3
  7. zenml/config/pipeline_spec.py +2 -2
  8. zenml/config/step_configurations.py +3 -3
  9. zenml/constants.py +3 -0
  10. zenml/enums.py +16 -0
  11. zenml/integrations/__init__.py +1 -0
  12. zenml/integrations/azure/__init__.py +2 -2
  13. zenml/integrations/constants.py +1 -0
  14. zenml/integrations/databricks/__init__.py +52 -0
  15. zenml/integrations/databricks/flavors/__init__.py +30 -0
  16. zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
  17. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +147 -0
  18. zenml/integrations/databricks/model_deployers/__init__.py +20 -0
  19. zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
  20. zenml/integrations/databricks/orchestrators/__init__.py +20 -0
  21. zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +497 -0
  22. zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
  23. zenml/integrations/databricks/services/__init__.py +19 -0
  24. zenml/integrations/databricks/services/databricks_deployment.py +407 -0
  25. zenml/integrations/databricks/utils/__init__.py +14 -0
  26. zenml/integrations/databricks/utils/databricks_utils.py +87 -0
  27. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +44 -28
  28. zenml/integrations/great_expectations/data_validators/ge_data_validator.py +12 -8
  29. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +88 -3
  30. zenml/integrations/huggingface/steps/accelerate_runner.py +1 -7
  31. zenml/integrations/kubernetes/__init__.py +3 -2
  32. zenml/integrations/kubernetes/flavors/__init__.py +8 -0
  33. zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
  34. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +1 -13
  35. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +22 -4
  36. zenml/integrations/kubernetes/pod_settings.py +4 -0
  37. zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
  38. zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +235 -0
  39. zenml/integrations/lightgbm/__init__.py +1 -0
  40. zenml/integrations/mlflow/__init__.py +1 -1
  41. zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +6 -2
  42. zenml/integrations/mlflow/services/mlflow_deployment.py +1 -1
  43. zenml/integrations/skypilot_azure/__init__.py +1 -3
  44. zenml/integrations/skypilot_lambda/__init__.py +1 -1
  45. zenml/logging/step_logging.py +34 -35
  46. zenml/materializers/built_in_materializer.py +1 -1
  47. zenml/materializers/cloudpickle_materializer.py +1 -1
  48. zenml/model/model.py +1 -1
  49. zenml/models/v2/core/code_repository.py +2 -2
  50. zenml/models/v2/core/component.py +29 -0
  51. zenml/models/v2/core/server_settings.py +0 -20
  52. zenml/models/v2/misc/full_stack.py +32 -0
  53. zenml/models/v2/misc/stack_deployment.py +5 -0
  54. zenml/new/pipelines/run_utils.py +1 -1
  55. zenml/orchestrators/__init__.py +4 -0
  56. zenml/orchestrators/step_launcher.py +1 -0
  57. zenml/orchestrators/wheeled_orchestrator.py +147 -0
  58. zenml/service_connectors/service_connector_utils.py +408 -0
  59. zenml/stack_deployments/azure_stack_deployment.py +179 -0
  60. zenml/stack_deployments/gcp_stack_deployment.py +13 -4
  61. zenml/stack_deployments/stack_deployment.py +10 -0
  62. zenml/stack_deployments/utils.py +4 -0
  63. zenml/steps/base_step.py +7 -5
  64. zenml/utils/function_utils.py +1 -1
  65. zenml/utils/pipeline_docker_image_builder.py +8 -0
  66. zenml/utils/source_utils.py +4 -1
  67. zenml/zen_server/dashboard/assets/{404-DpJaNHKF.js → 404-B_YdvmwS.js} +1 -1
  68. zenml/zen_server/dashboard/assets/{@reactflow-DJfzkHO1.js → @reactflow-l_1hUr1S.js} +1 -1
  69. zenml/zen_server/dashboard/assets/{AwarenessChannel-BYDLT2xC.js → AwarenessChannel-CFg5iX4Z.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{CodeSnippet-BkOuRmyq.js → CodeSnippet-Dvkx_82E.js} +1 -1
  71. zenml/zen_server/dashboard/assets/CollapsibleCard-opiuBHHc.js +1 -0
  72. zenml/zen_server/dashboard/assets/{Commands-ZvWR1BRs.js → Commands-DoN1xrEq.js} +1 -1
  73. zenml/zen_server/dashboard/assets/{CopyButton-DVwLkafa.js → CopyButton-Cr7xYEPb.js} +1 -1
  74. zenml/zen_server/dashboard/assets/{CsvVizualization-C2IiqX4I.js → CsvVizualization-Ck-nZ43m.js} +3 -3
  75. zenml/zen_server/dashboard/assets/{Error-CqX0VqW_.js → Error-kLtljEOM.js} +1 -1
  76. zenml/zen_server/dashboard/assets/{ExecutionStatus-BoLUXR9t.js → ExecutionStatus-DguLLgTK.js} +1 -1
  77. zenml/zen_server/dashboard/assets/{Helpbox-LFydyVwh.js → Helpbox-BXUMP21n.js} +1 -1
  78. zenml/zen_server/dashboard/assets/{Infobox-DnENC0sh.js → Infobox-DSt0O-dm.js} +1 -1
  79. zenml/zen_server/dashboard/assets/{InlineAvatar-CbJtYr0t.js → InlineAvatar-xsrsIGE-.js} +1 -1
  80. zenml/zen_server/dashboard/assets/Pagination-C6X-mifw.js +1 -0
  81. zenml/zen_server/dashboard/assets/{SetPassword-BYBdbQDo.js → SetPassword-BXGTWiwj.js} +1 -1
  82. zenml/zen_server/dashboard/assets/{SuccessStep-Nx743hll.js → SuccessStep-DZC60t0x.js} +1 -1
  83. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DF9gSzE0.js → UpdatePasswordSchemas-DGvwFWO1.js} +1 -1
  84. zenml/zen_server/dashboard/assets/{chevron-right-double-BiEMg7rd.js → chevron-right-double-CZBOf6JM.js} +1 -1
  85. zenml/zen_server/dashboard/assets/cloud-only-C_yFCAkP.js +1 -0
  86. zenml/zen_server/dashboard/assets/index-BczVOqUf.js +55 -0
  87. zenml/zen_server/dashboard/assets/index-EpMIKgrI.css +1 -0
  88. zenml/zen_server/dashboard/assets/{login-mutation-BUnVASxp.js → login-mutation-CrHrndTI.js} +1 -1
  89. zenml/zen_server/dashboard/assets/logs-D8k8BVFf.js +1 -0
  90. zenml/zen_server/dashboard/assets/{not-found-B4VnX8gK.js → not-found-DYa4pC-C.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{package-CsUhPmou.js → package-B3fWP-Dh.js} +1 -1
  92. zenml/zen_server/dashboard/assets/page-1h_sD1jz.js +1 -0
  93. zenml/zen_server/dashboard/assets/{page-Sxn82W-5.js → page-1iL8aMqs.js} +1 -1
  94. zenml/zen_server/dashboard/assets/{page-DMOYZppS.js → page-2grKx_MY.js} +1 -1
  95. zenml/zen_server/dashboard/assets/page-5NCOHOsy.js +1 -0
  96. zenml/zen_server/dashboard/assets/{page-JyfeDUfu.js → page-8a4UMKXZ.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{page-Bx6o0ARS.js → page-B6h3iaHJ.js} +1 -1
  98. zenml/zen_server/dashboard/assets/page-BDns21Iz.js +1 -0
  99. zenml/zen_server/dashboard/assets/{page-3efNCDeb.js → page-BhgCDInH.js} +2 -2
  100. zenml/zen_server/dashboard/assets/{page-DKlIdAe5.js → page-Bi-wtWiO.js} +2 -2
  101. zenml/zen_server/dashboard/assets/{page-7zTHbhhI.js → page-BkeAAYwp.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{page-CRTJ0UuR.js → page-BkuQDIf-.js} +1 -1
  103. zenml/zen_server/dashboard/assets/page-BnaevhnB.js +1 -0
  104. zenml/zen_server/dashboard/assets/{page-BEs6jK71.js → page-Bq0YxkLV.js} +1 -1
  105. zenml/zen_server/dashboard/assets/page-Bs2F4eoD.js +2 -0
  106. zenml/zen_server/dashboard/assets/{page-CUZIGO-3.js → page-C6-UGEbH.js} +1 -1
  107. zenml/zen_server/dashboard/assets/{page-Xu8JEjSU.js → page-CCNRIt_f.js} +1 -1
  108. zenml/zen_server/dashboard/assets/{page-DvCvroOM.js → page-CHNxpz3n.js} +1 -1
  109. zenml/zen_server/dashboard/assets/{page-BpSqIf4B.js → page-DgorQFqi.js} +1 -1
  110. zenml/zen_server/dashboard/assets/page-K8ebxVIs.js +1 -0
  111. zenml/zen_server/dashboard/assets/{page-Cx67M0QT.js → page-MFQyIJd3.js} +1 -1
  112. zenml/zen_server/dashboard/assets/page-TgCF0P_U.js +1 -0
  113. zenml/zen_server/dashboard/assets/page-ZnCEe-eK.js +9 -0
  114. zenml/zen_server/dashboard/assets/{page-Dc_7KMQE.js → page-uA5prJGY.js} +1 -1
  115. zenml/zen_server/dashboard/assets/persist-D7HJNBWx.js +1 -0
  116. zenml/zen_server/dashboard/assets/plus-C8WOyCzt.js +1 -0
  117. zenml/zen_server/dashboard/assets/stack-detail-query-Cficsl6d.js +1 -0
  118. zenml/zen_server/dashboard/assets/update-server-settings-mutation-7d8xi1tS.js +1 -0
  119. zenml/zen_server/dashboard/assets/{url-DuQMeqYA.js → url-D7mAQGUM.js} +1 -1
  120. zenml/zen_server/dashboard/index.html +4 -4
  121. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  122. zenml/zen_server/dashboard_legacy/index.html +1 -1
  123. zenml/zen_server/dashboard_legacy/{precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js → precache-manifest.12246c7548e71e2c4438e496360de80c.js} +4 -4
  124. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  125. zenml/zen_server/dashboard_legacy/static/js/main.3b27024b.chunk.js +2 -0
  126. zenml/zen_server/dashboard_legacy/static/js/{main.382439a7.chunk.js.map → main.3b27024b.chunk.js.map} +1 -1
  127. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  128. zenml/zen_server/deploy/helm/README.md +2 -2
  129. zenml/zen_server/rbac/utils.py +10 -2
  130. zenml/zen_server/routers/devices_endpoints.py +4 -1
  131. zenml/zen_server/routers/server_endpoints.py +29 -2
  132. zenml/zen_server/routers/service_connectors_endpoints.py +57 -0
  133. zenml/zen_server/routers/steps_endpoints.py +2 -1
  134. zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
  135. zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
  136. zenml/zen_stores/rest_zen_store.py +4 -0
  137. zenml/zen_stores/schemas/component_schemas.py +14 -0
  138. zenml/zen_stores/schemas/server_settings_schemas.py +23 -11
  139. zenml/zen_stores/sql_zen_store.py +151 -1
  140. {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/METADATA +5 -5
  141. {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/RECORD +144 -121
  142. zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +0 -1
  143. zenml/zen_server/dashboard/assets/chevron-down-D_ZlKMqH.js +0 -1
  144. zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +0 -1
  145. zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +0 -1
  146. zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +0 -55
  147. zenml/zen_server/dashboard/assets/index-inApY3KQ.css +0 -1
  148. zenml/zen_server/dashboard/assets/page-C43QGHTt.js +0 -9
  149. zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +0 -1
  150. zenml/zen_server/dashboard/assets/page-CaopxiU1.js +0 -1
  151. zenml/zen_server/dashboard/assets/page-D7Z399xy.js +0 -1
  152. zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +0 -1
  153. zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +0 -2
  154. zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +0 -1
  155. zenml/zen_server/dashboard/assets/page-TKXERe16.js +0 -1
  156. zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +0 -1
  157. zenml/zen_server/dashboard/assets/update-server-settings-mutation-CR8e3Sir.js +0 -1
  158. zenml/zen_server/dashboard_legacy/static/js/main.382439a7.chunk.js +0 -2
  159. {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/LICENSE +0 -0
  160. {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/WHEEL +0 -0
  161. {zenml_nightly-0.61.0.dev20240712.dist-info → zenml_nightly-0.62.0.dev20240727.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,408 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Utility methods for Service Connectors."""
15
+
16
+ from typing import Dict, List, Union
17
+ from uuid import UUID
18
+
19
+ from zenml.client import Client
20
+ from zenml.enums import StackComponentType
21
+ from zenml.models.v2.core.service_connector import ServiceConnectorRequest
22
+ from zenml.models.v2.misc.full_stack import (
23
+ ResourcesInfo,
24
+ ServiceConnectorInfo,
25
+ ServiceConnectorResourcesInfo,
26
+ )
27
+ from zenml.utils.pagination_utils import depaginate
28
+
29
+
30
+ def _prepare_resource_info(
31
+ connector_details: Union[UUID, ServiceConnectorInfo],
32
+ resource_ids: List[str],
33
+ stack_component_type: StackComponentType,
34
+ flavor: str,
35
+ required_configuration: Dict[str, str],
36
+ flavor_display_name: str,
37
+ use_resource_value_as_fixed_config: bool = False,
38
+ ) -> ResourcesInfo:
39
+ existing_components = []
40
+ if isinstance(connector_details, UUID):
41
+ existing_components = depaginate(
42
+ Client().list_stack_components,
43
+ type=stack_component_type.value,
44
+ connector_id=connector_details,
45
+ flavor=flavor,
46
+ )
47
+ return ResourcesInfo(
48
+ flavor=flavor,
49
+ required_configuration=required_configuration,
50
+ flavor_display_name=flavor_display_name,
51
+ use_resource_value_as_fixed_config=use_resource_value_as_fixed_config,
52
+ accessible_by_service_connector=resource_ids,
53
+ connected_through_service_connector=existing_components,
54
+ )
55
+
56
+
57
+ def _raise_specific_cloud_exception_if_needed(
58
+ cloud_provider: str,
59
+ artifact_stores: List[ResourcesInfo],
60
+ orchestrators: List[ResourcesInfo],
61
+ container_registries: List[ResourcesInfo],
62
+ ) -> None:
63
+ AWS_DOCS = (
64
+ "https://docs.zenml.io/how-to/auth-management/aws-service-connector"
65
+ )
66
+ GCP_DOCS = (
67
+ "https://docs.zenml.io/how-to/auth-management/gcp-service-connector"
68
+ )
69
+ AZURE_DOCS = (
70
+ "https://docs.zenml.io/how-to/auth-management/azure-service-connector"
71
+ )
72
+
73
+ if not artifact_stores:
74
+ error_msg = (
75
+ "We were unable to find any {obj_name} available "
76
+ "to configured service connector. Please, verify "
77
+ "that needed permission are granted for the "
78
+ "service connector.\nDocumentation for the "
79
+ "{obj_name} configuration can be found at "
80
+ "{docs}"
81
+ )
82
+ if cloud_provider == "aws":
83
+ raise ValueError(
84
+ error_msg.format(
85
+ obj_name="S3 Bucket", docs=f"{AWS_DOCS}#s3-bucket"
86
+ )
87
+ )
88
+ elif cloud_provider == "gcp":
89
+ raise ValueError(
90
+ error_msg.format(
91
+ obj_name="GCS Bucket", docs=f"{GCP_DOCS}#gcs-bucket"
92
+ )
93
+ )
94
+ elif cloud_provider == "azure":
95
+ raise ValueError(
96
+ error_msg.format(
97
+ obj_name="Blob Container",
98
+ docs=f"{AZURE_DOCS}#azure-blob-storage-container",
99
+ )
100
+ )
101
+ if not orchestrators:
102
+ error_msg = (
103
+ "We were unable to find any orchestrator engines "
104
+ "available to the service connector. Please, verify "
105
+ "that needed permission are granted for the "
106
+ "service connector.\nDocumentation for the Generic "
107
+ "{cloud_name} resource configuration can be found at "
108
+ "{gen_docs}\n Documentation for the {k8s_name} resource "
109
+ "configuration can be found at {k8s_docs}"
110
+ )
111
+ if cloud_provider == "aws":
112
+ raise ValueError(
113
+ error_msg.format(
114
+ cloud_name="AWS",
115
+ gen_docs=f"{AWS_DOCS}#generic-aws-resource",
116
+ k8s_name="EKS",
117
+ k8s_docs=f"{AWS_DOCS}#eks-kubernetes-cluster",
118
+ )
119
+ )
120
+
121
+ elif cloud_provider == "gcp":
122
+ raise ValueError(
123
+ error_msg.format(
124
+ cloud_name="GCP",
125
+ gen_docs=f"{GCP_DOCS}#generic-gcp-resource",
126
+ k8s_name="GKE",
127
+ k8s_docs=f"{GCP_DOCS}#gke-kubernetes-cluster",
128
+ )
129
+ )
130
+ elif cloud_provider == "azure":
131
+ raise ValueError(
132
+ error_msg.format(
133
+ cloud_name="Azure",
134
+ gen_docs=f"{AZURE_DOCS}#generic-azure-resource",
135
+ k8s_name="AKS",
136
+ k8s_docs=f"{AZURE_DOCS}#aks-kubernetes-cluster",
137
+ )
138
+ )
139
+ if not container_registries:
140
+ error_msg = (
141
+ "We were unable to find any container registries "
142
+ "available to the service connector. Please, verify "
143
+ "that needed permission are granted for the "
144
+ "service connector.\nDocumentation for the {registry_name} "
145
+ "container registry resource configuration can "
146
+ "be found at {docs_link}"
147
+ )
148
+ if cloud_provider == "aws":
149
+ raise ValueError(
150
+ error_msg.format(
151
+ registry_name="ECR",
152
+ docs_link=f"{AWS_DOCS}#ecr-container-registry",
153
+ )
154
+ )
155
+ elif cloud_provider == "gcp":
156
+ raise ValueError(
157
+ error_msg.format(
158
+ registry_name="GCR",
159
+ docs_link=f"{GCP_DOCS}#gcr-container-registry",
160
+ )
161
+ )
162
+ elif cloud_provider == "azure":
163
+ raise ValueError(
164
+ error_msg.format(
165
+ registry_name="ACR",
166
+ docs_link=f"{AZURE_DOCS}#acr-container-registry",
167
+ )
168
+ )
169
+
170
+
171
+ def get_resources_options_from_resource_model_for_full_stack(
172
+ connector_details: Union[UUID, ServiceConnectorInfo],
173
+ ) -> ServiceConnectorResourcesInfo:
174
+ """Get the resource options from the resource model for the full stack.
175
+
176
+ Args:
177
+ connector_details: The service connector details (UUID or Info).
178
+
179
+ Returns:
180
+ All available service connector resource options.
181
+ """
182
+ client = Client()
183
+ zen_store = client.zen_store
184
+
185
+ if isinstance(connector_details, UUID):
186
+ resource_model = zen_store.verify_service_connector(
187
+ connector_details,
188
+ list_resources=True,
189
+ )
190
+ else:
191
+ resource_model = zen_store.verify_service_connector_config(
192
+ service_connector=ServiceConnectorRequest(
193
+ user=client.active_user.id,
194
+ workspace=client.active_workspace.id,
195
+ name="fake",
196
+ connector_type=connector_details.type,
197
+ auth_method=connector_details.auth_method,
198
+ configuration=connector_details.configuration,
199
+ secrets={},
200
+ labels={},
201
+ ),
202
+ list_resources=True,
203
+ )
204
+
205
+ resources = resource_model.resources
206
+
207
+ if isinstance(
208
+ resource_model.connector_type,
209
+ str,
210
+ ):
211
+ connector_type = resource_model.connector_type
212
+ else:
213
+ connector_type = resource_model.connector_type.connector_type
214
+
215
+ artifact_stores: List[ResourcesInfo] = []
216
+ orchestrators: List[ResourcesInfo] = []
217
+ container_registries: List[ResourcesInfo] = []
218
+
219
+ if connector_type == "aws":
220
+ for each in resources:
221
+ if each.resource_ids:
222
+ if each.resource_type == "s3-bucket":
223
+ artifact_stores.append(
224
+ _prepare_resource_info(
225
+ connector_details=connector_details,
226
+ resource_ids=each.resource_ids,
227
+ stack_component_type=StackComponentType.ARTIFACT_STORE,
228
+ flavor="s3",
229
+ required_configuration={"path": "Path"},
230
+ use_resource_value_as_fixed_config=True,
231
+ flavor_display_name="S3 Bucket",
232
+ )
233
+ )
234
+ if each.resource_type == "aws-generic":
235
+ orchestrators.append(
236
+ _prepare_resource_info(
237
+ connector_details=connector_details,
238
+ resource_ids=each.resource_ids,
239
+ stack_component_type=StackComponentType.ORCHESTRATOR,
240
+ flavor="sagemaker",
241
+ required_configuration={
242
+ "execution_role": "execution role ARN"
243
+ },
244
+ flavor_display_name="AWS Sagemaker",
245
+ )
246
+ )
247
+ orchestrators.append(
248
+ _prepare_resource_info(
249
+ connector_details=connector_details,
250
+ resource_ids=each.resource_ids,
251
+ stack_component_type=StackComponentType.ORCHESTRATOR,
252
+ flavor="vm_aws",
253
+ required_configuration={"region": "region"},
254
+ use_resource_value_as_fixed_config=True,
255
+ flavor_display_name="Skypilot (EC2)",
256
+ )
257
+ )
258
+
259
+ if each.resource_type == "kubernetes-cluster":
260
+ orchestrators.append(
261
+ _prepare_resource_info(
262
+ connector_details=connector_details,
263
+ resource_ids=each.resource_ids,
264
+ stack_component_type=StackComponentType.ORCHESTRATOR,
265
+ flavor="kubernetes",
266
+ required_configuration={},
267
+ flavor_display_name="Kubernetes",
268
+ )
269
+ )
270
+ if each.resource_type == "docker-registry":
271
+ container_registries.append(
272
+ _prepare_resource_info(
273
+ connector_details=connector_details,
274
+ resource_ids=each.resource_ids,
275
+ stack_component_type=StackComponentType.CONTAINER_REGISTRY,
276
+ flavor="aws",
277
+ required_configuration={"uri": "URI"},
278
+ use_resource_value_as_fixed_config=True,
279
+ flavor_display_name="ECR",
280
+ )
281
+ )
282
+
283
+ elif connector_type == "gcp":
284
+ for each in resources:
285
+ if each.resource_ids:
286
+ if each.resource_type == "gcs-bucket":
287
+ artifact_stores.append(
288
+ _prepare_resource_info(
289
+ connector_details=connector_details,
290
+ resource_ids=each.resource_ids,
291
+ stack_component_type=StackComponentType.ARTIFACT_STORE,
292
+ flavor="gcp",
293
+ required_configuration={},
294
+ flavor_display_name="GCS Bucket",
295
+ )
296
+ )
297
+ if each.resource_type == "gcp-generic":
298
+ orchestrators.append(
299
+ _prepare_resource_info(
300
+ connector_details=connector_details,
301
+ resource_ids=each.resource_ids,
302
+ stack_component_type=StackComponentType.ORCHESTRATOR,
303
+ flavor="vertex",
304
+ required_configuration={"location": "region name"},
305
+ flavor_display_name="Vertex AI",
306
+ )
307
+ )
308
+ orchestrators.append(
309
+ _prepare_resource_info(
310
+ connector_details=connector_details,
311
+ resource_ids=each.resource_ids,
312
+ stack_component_type=StackComponentType.ORCHESTRATOR,
313
+ flavor="vm_gcp",
314
+ required_configuration={"region": "region name"},
315
+ flavor_display_name="Skypilot (Compute)",
316
+ )
317
+ )
318
+
319
+ if each.resource_type == "kubernetes-cluster":
320
+ orchestrators.append(
321
+ _prepare_resource_info(
322
+ connector_details=connector_details,
323
+ resource_ids=each.resource_ids,
324
+ stack_component_type=StackComponentType.ORCHESTRATOR,
325
+ flavor="kubernetes",
326
+ required_configuration={},
327
+ flavor_display_name="Kubernetes",
328
+ )
329
+ )
330
+ if each.resource_type == "docker-registry":
331
+ container_registries.append(
332
+ _prepare_resource_info(
333
+ connector_details=connector_details,
334
+ resource_ids=each.resource_ids,
335
+ stack_component_type=StackComponentType.CONTAINER_REGISTRY,
336
+ flavor="gcp",
337
+ required_configuration={"uri": "URI"},
338
+ use_resource_value_as_fixed_config=True,
339
+ flavor_display_name="GCR",
340
+ )
341
+ )
342
+
343
+ elif connector_type == "azure":
344
+ for each in resources:
345
+ if each.resource_ids:
346
+ if each.resource_type == "blob-container":
347
+ artifact_stores.append(
348
+ _prepare_resource_info(
349
+ connector_details=connector_details,
350
+ resource_ids=each.resource_ids,
351
+ stack_component_type=StackComponentType.ARTIFACT_STORE,
352
+ flavor="azure",
353
+ required_configuration={},
354
+ flavor_display_name="Blob container",
355
+ )
356
+ )
357
+ if each.resource_type == "azure-generic":
358
+ # No native orchestrator ATM
359
+ orchestrators.append(
360
+ _prepare_resource_info(
361
+ connector_details=connector_details,
362
+ resource_ids=each.resource_ids,
363
+ stack_component_type=StackComponentType.ORCHESTRATOR,
364
+ flavor="vm_azure",
365
+ required_configuration={"region": "region name"},
366
+ flavor_display_name="Skypilot (VM)",
367
+ )
368
+ )
369
+
370
+ if each.resource_type == "kubernetes-cluster":
371
+ orchestrators.append(
372
+ _prepare_resource_info(
373
+ connector_details=connector_details,
374
+ resource_ids=each.resource_ids,
375
+ stack_component_type=StackComponentType.ORCHESTRATOR,
376
+ flavor="kubernetes",
377
+ required_configuration={},
378
+ flavor_display_name="Kubernetes",
379
+ )
380
+ )
381
+ if each.resource_type == "docker-registry":
382
+ container_registries.append(
383
+ _prepare_resource_info(
384
+ connector_details=connector_details,
385
+ resource_ids=each.resource_ids,
386
+ stack_component_type=StackComponentType.CONTAINER_REGISTRY,
387
+ flavor="azure",
388
+ required_configuration={"uri": "URI"},
389
+ use_resource_value_as_fixed_config=True,
390
+ flavor_display_name="ACR",
391
+ )
392
+ )
393
+
394
+ _raise_specific_cloud_exception_if_needed(
395
+ cloud_provider=connector_type,
396
+ artifact_stores=artifact_stores,
397
+ orchestrators=orchestrators,
398
+ container_registries=container_registries,
399
+ )
400
+
401
+ return ServiceConnectorResourcesInfo(
402
+ connector_type=connector_type,
403
+ components_resources_info={
404
+ StackComponentType.ARTIFACT_STORE: artifact_stores,
405
+ StackComponentType.ORCHESTRATOR: orchestrators,
406
+ StackComponentType.CONTAINER_REGISTRY: container_registries,
407
+ },
408
+ )
@@ -0,0 +1,179 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Functionality to deploy a ZenML stack to Azure."""
15
+
16
+ import re
17
+ from typing import ClassVar, Dict, List
18
+
19
+ from zenml.enums import StackDeploymentProvider
20
+ from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
21
+
22
+
23
+ # TODO: this class just implements the regions list, and is not suitable for other
24
+ # deployment tasks.
25
+ class AZUREZenMLCloudStackDeployment(ZenMLCloudStackDeployment):
26
+ """Azure ZenML Cloud Stack Deployment."""
27
+
28
+ provider: ClassVar[StackDeploymentProvider] = StackDeploymentProvider.AZURE
29
+
30
+ @classmethod
31
+ def description(cls) -> str:
32
+ """Return a description of the ZenML Cloud Stack Deployment.
33
+
34
+ This will be displayed when the user is prompted to deploy
35
+ the ZenML stack.
36
+
37
+ Returns:
38
+ A MarkDown description of the ZenML Cloud Stack Deployment.
39
+ """
40
+ # TODO: Implement this
41
+ return ""
42
+
43
+ @classmethod
44
+ def instructions(cls) -> str:
45
+ """Return instructions on how to deploy the ZenML stack to the specified cloud provider.
46
+
47
+ This will be displayed before the user is prompted to deploy the ZenML
48
+ stack.
49
+
50
+ Returns:
51
+ MarkDown instructions on how to deploy the ZenML stack to the
52
+ specified cloud provider.
53
+ """
54
+ # TODO: Implement this
55
+ return ""
56
+
57
+ @classmethod
58
+ def post_deploy_instructions(cls) -> str:
59
+ """Return instructions on what to do after the deployment is complete.
60
+
61
+ This will be displayed after the deployment is complete.
62
+
63
+ Returns:
64
+ MarkDown instructions on what to do after the deployment is
65
+ complete.
66
+ """
67
+ # TODO: Implement this
68
+ return ""
69
+
70
+ @classmethod
71
+ def integrations(cls) -> List[str]:
72
+ """Return the ZenML integrations required for the stack.
73
+
74
+ Returns:
75
+ The list of ZenML integrations that need to be installed for the
76
+ stack to be usable.
77
+ """
78
+ return ["azure"]
79
+
80
+ @classmethod
81
+ def permissions(cls) -> Dict[str, List[str]]:
82
+ """Return the permissions granted to ZenML to access the cloud resources.
83
+
84
+ Returns:
85
+ The permissions granted to ZenML to access the cloud resources, as
86
+ a dictionary grouping permissions by resource.
87
+ """
88
+ # TODO: Implement this
89
+ return {}
90
+
91
+ @classmethod
92
+ def locations(cls) -> Dict[str, str]:
93
+ """Return the locations where the ZenML stack can be deployed.
94
+
95
+ Returns:
96
+ The regions where the ZenML stack can be deployed as a map of region
97
+ names to region descriptions.
98
+ """
99
+ # Based on `az account list-locations -o table` on 16.07.2024
100
+ return {
101
+ "(US) East US": "eastus",
102
+ "(US) South Central US": "southcentralus",
103
+ "(US) West US 2": "westus2",
104
+ "(US) West US 3": "westus3",
105
+ "(Asia Pacific) Australia East": "australiaeast",
106
+ "(Asia Pacific) Southeast Asia": "southeastasia",
107
+ "(Europe) North Europe": "northeurope",
108
+ "(Europe) Sweden Central": "swedencentral",
109
+ "(Europe) UK South": "uksouth",
110
+ "(Europe) West Europe": "westeurope",
111
+ "(US) Central US": "centralus",
112
+ "(Africa) South Africa North": "southafricanorth",
113
+ "(Asia Pacific) Central India": "centralindia",
114
+ "(Asia Pacific) East Asia": "eastasia",
115
+ "(Asia Pacific) Japan East": "japaneast",
116
+ "(Asia Pacific) Korea Central": "koreacentral",
117
+ "(Canada) Canada Central": "canadacentral",
118
+ "(Europe) France Central": "francecentral",
119
+ "(Europe) Germany West Central": "germanywestcentral",
120
+ "(Europe) Italy North": "italynorth",
121
+ "(Europe) Norway East": "norwayeast",
122
+ "(Europe) Poland Central": "polandcentral",
123
+ "(Europe) Spain Central": "spaincentral",
124
+ "(Europe) Switzerland North": "switzerlandnorth",
125
+ "(Mexico) Mexico Central": "mexicocentral",
126
+ "(Middle East) UAE North": "uaenorth",
127
+ "(South America) Brazil South": "brazilsouth",
128
+ "(Middle East) Israel Central": "israelcentral",
129
+ "(Middle East) Qatar Central": "qatarcentral",
130
+ "(US) Central US (Stage)": "centralusstage",
131
+ "(US) East US (Stage)": "eastusstage",
132
+ "(US) East US 2 (Stage)": "eastus2stage",
133
+ "(US) North Central US (Stage)": "northcentralusstage",
134
+ "(US) South Central US (Stage)": "southcentralusstage",
135
+ "(US) West US (Stage)": "westusstage",
136
+ "(US) West US 2 (Stage)": "westus2stage",
137
+ "(Asia Pacific) East Asia (Stage)": "eastasiastage",
138
+ "(Asia Pacific) Southeast Asia (Stage)": "southeastasiastage",
139
+ "(South America) Brazil US": "brazilus",
140
+ "(US) East US 2": "eastus2",
141
+ "(US) East US STG": "eastusstg",
142
+ "(US) North Central US": "northcentralus",
143
+ "(US) West US": "westus",
144
+ "(Asia Pacific) Japan West": "japanwest",
145
+ "(Asia Pacific) Jio India West": "jioindiawest",
146
+ "(US) Central US EUAP": "centraluseuap",
147
+ "(US) East US 2 EUAP": "eastus2euap",
148
+ "(US) West Central US": "westcentralus",
149
+ "(Africa) South Africa West": "southafricawest",
150
+ "(Asia Pacific) Australia Central": "australiacentral",
151
+ "(Asia Pacific) Australia Central 2": "australiacentral2",
152
+ "(Asia Pacific) Australia Southeast": "australiasoutheast",
153
+ "(Asia Pacific) Jio India Central": "jioindiacentral",
154
+ "(Asia Pacific) Korea South": "koreasouth",
155
+ "(Asia Pacific) South India": "southindia",
156
+ "(Asia Pacific) West India": "westindia",
157
+ "(Canada) Canada East": "canadaeast",
158
+ "(Europe) France South": "francesouth",
159
+ "(Europe) Germany North": "germanynorth",
160
+ "(Europe) Norway West": "norwaywest",
161
+ "(Europe) Switzerland West": "switzerlandwest",
162
+ "(Europe) UK West": "ukwest",
163
+ "(Middle East) UAE Central": "uaecentral",
164
+ "(South America) Brazil Southeast": "brazilsoutheast",
165
+ }
166
+
167
+ @classmethod
168
+ def skypilot_default_regions(cls) -> Dict[str, str]:
169
+ """Returns the regions supported by default for the Skypilot.
170
+
171
+ Returns:
172
+ The regions supported by default for the Skypilot.
173
+ """
174
+ matcher = re.compile(r".*us\d*( |$)")
175
+ return {
176
+ k: v
177
+ for k, v in cls.locations().items()
178
+ if "(US)" in k and matcher.match(v)
179
+ }
@@ -13,6 +13,7 @@
13
13
  # permissions and limitations under the License.
14
14
  """Functionality to deploy a ZenML stack to GCP."""
15
15
 
16
+ import re
16
17
  from typing import ClassVar, Dict, List
17
18
 
18
19
  from zenml.enums import StackDeploymentProvider
@@ -208,6 +209,16 @@ GCP project and to clean up the resources created by the stack by using
208
209
  "US West (Las Vegas)": "us-west4",
209
210
  }
210
211
 
212
+ @classmethod
213
+ def skypilot_default_regions(cls) -> Dict[str, str]:
214
+ """Returns the regions supported by default for the Skypilot.
215
+
216
+ Returns:
217
+ The regions supported by default for the Skypilot.
218
+ """
219
+ matcher = re.compile(r"us-.*")
220
+ return {k: v for k, v in cls.locations().items() if matcher.match(v)}
221
+
211
222
  def get_deployment_config(
212
223
  self,
213
224
  ) -> StackDeploymentConfig:
@@ -236,8 +247,6 @@ GCP project and to clean up the resources created by the stack by using
236
247
  cloudshell_open_in_editor="gcp-gar-gcs-vertex.jinja,gcp-gar-gcs-vertex-deploy.sh",
237
248
  cloudshell_tutorial="gcp-gar-gcs-vertex.md",
238
249
  ephemeral="true",
239
- # TODO: remove this before the branch is merged
240
- cloudshell_git_branch="feature/prd-482-gcp-stack-deployment",
241
250
  )
242
251
  # Encode the parameters as URL query parameters
243
252
  query_params = "&".join([f"{k}={v}" for k, v in params.items()])
@@ -251,8 +260,8 @@ ZENML_STACK_NAME={self.stack_name}
251
260
  ZENML_STACK_REGION={self.location or "europe-west3"}
252
261
  ZENML_SERVER_URL={self.zenml_server_url}
253
262
  ZENML_SERVER_API_TOKEN={self.zenml_server_api_token}
254
- ### END CONFIGURATION ###
255
- """
263
+ ### END CONFIGURATION ###"""
264
+
256
265
  return StackDeploymentConfig(
257
266
  deployment_url=url,
258
267
  deployment_url_text="GCP Cloud Shell Console",
@@ -105,6 +105,15 @@ class ZenMLCloudStackDeployment(BaseModel):
105
105
  names to region descriptions.
106
106
  """
107
107
 
108
+ @classmethod
109
+ def skypilot_default_regions(cls) -> Dict[str, str]:
110
+ """Returns the regions supported by default for the Skypilot.
111
+
112
+ Returns:
113
+ The regions supported by default for the Skypilot.
114
+ """
115
+ return cls.locations()
116
+
108
117
  @classmethod
109
118
  def get_deployment_info(cls) -> StackDeploymentInfo:
110
119
  """Return information about the ZenML Cloud Stack Deployment.
@@ -120,6 +129,7 @@ class ZenMLCloudStackDeployment(BaseModel):
120
129
  integrations=cls.integrations(),
121
130
  permissions=cls.permissions(),
122
131
  locations=cls.locations(),
132
+ skypilot_default_regions=cls.skypilot_default_regions(),
123
133
  )
124
134
 
125
135
  @abstractmethod
@@ -19,6 +19,9 @@ from zenml.enums import StackDeploymentProvider
19
19
  from zenml.stack_deployments.aws_stack_deployment import (
20
20
  AWSZenMLCloudStackDeployment,
21
21
  )
22
+ from zenml.stack_deployments.azure_stack_deployment import (
23
+ AZUREZenMLCloudStackDeployment,
24
+ )
22
25
  from zenml.stack_deployments.gcp_stack_deployment import (
23
26
  GCPZenMLCloudStackDeployment,
24
27
  )
@@ -27,6 +30,7 @@ from zenml.stack_deployments.stack_deployment import ZenMLCloudStackDeployment
27
30
  STACK_DEPLOYMENT_PROVIDERS = {
28
31
  StackDeploymentProvider.AWS: AWSZenMLCloudStackDeployment,
29
32
  StackDeploymentProvider.GCP: GCPZenMLCloudStackDeployment,
33
+ StackDeploymentProvider.AZURE: AZUREZenMLCloudStackDeployment,
30
34
  }
31
35
 
32
36