zenml-nightly 0.60.0.dev20240627__py3-none-any.whl → 0.61.0.dev20240711__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. README.md +30 -9
  2. RELEASE_NOTES.md +34 -0
  3. zenml/VERSION +1 -1
  4. zenml/analytics/enums.py +3 -0
  5. zenml/cli/__init__.py +28 -0
  6. zenml/cli/artifact.py +1 -2
  7. zenml/cli/integration.py +9 -8
  8. zenml/cli/server.py +6 -0
  9. zenml/cli/stack.py +797 -39
  10. zenml/cli/stack_components.py +7 -0
  11. zenml/cli/text_utils.py +35 -1
  12. zenml/cli/utils.py +127 -10
  13. zenml/client.py +23 -14
  14. zenml/config/docker_settings.py +8 -5
  15. zenml/constants.py +5 -1
  16. zenml/container_registries/base_container_registry.py +1 -0
  17. zenml/enums.py +6 -0
  18. zenml/event_hub/event_hub.py +5 -8
  19. zenml/integrations/aws/__init__.py +1 -0
  20. zenml/integrations/azure/__init__.py +1 -0
  21. zenml/integrations/deepchecks/__init__.py +1 -0
  22. zenml/integrations/discord/__init__.py +1 -0
  23. zenml/integrations/evidently/__init__.py +1 -0
  24. zenml/integrations/facets/__init__.py +1 -0
  25. zenml/integrations/feast/__init__.py +1 -0
  26. zenml/integrations/gcp/__init__.py +3 -1
  27. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +203 -44
  28. zenml/integrations/huggingface/__init__.py +1 -0
  29. zenml/integrations/integration.py +24 -0
  30. zenml/integrations/kubeflow/__init__.py +3 -0
  31. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +1 -1
  32. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +0 -1
  33. zenml/integrations/kubernetes/__init__.py +3 -1
  34. zenml/integrations/kubernetes/orchestrators/kube_utils.py +4 -1
  35. zenml/integrations/label_studio/annotators/label_studio_annotator.py +1 -0
  36. zenml/integrations/langchain/__init__.py +1 -0
  37. zenml/integrations/mlflow/__init__.py +3 -1
  38. zenml/integrations/neural_prophet/__init__.py +1 -0
  39. zenml/integrations/polars/__init__.py +1 -0
  40. zenml/integrations/prodigy/__init__.py +1 -0
  41. zenml/integrations/pycaret/__init__.py +6 -0
  42. zenml/integrations/registry.py +37 -0
  43. zenml/integrations/s3/artifact_stores/s3_artifact_store.py +17 -6
  44. zenml/integrations/seldon/__init__.py +1 -0
  45. zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +1 -0
  46. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +2 -2
  47. zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +1 -1
  48. zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
  49. zenml/integrations/skypilot_aws/__init__.py +2 -1
  50. zenml/integrations/skypilot_azure/__init__.py +1 -1
  51. zenml/integrations/skypilot_gcp/__init__.py +1 -1
  52. zenml/integrations/skypilot_lambda/__init__.py +1 -1
  53. zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +1 -1
  54. zenml/integrations/slack/__init__.py +1 -0
  55. zenml/integrations/tekton/__init__.py +1 -0
  56. zenml/integrations/tensorboard/__init__.py +0 -1
  57. zenml/integrations/tensorflow/__init__.py +18 -6
  58. zenml/integrations/wandb/__init__.py +1 -0
  59. zenml/models/__init__.py +9 -0
  60. zenml/models/v2/core/component.py +18 -0
  61. zenml/models/v2/core/model.py +1 -2
  62. zenml/models/v2/core/service_connector.py +17 -0
  63. zenml/models/v2/core/stack.py +31 -0
  64. zenml/models/v2/misc/full_stack.py +97 -0
  65. zenml/models/v2/misc/stack_deployment.py +66 -0
  66. zenml/new/pipelines/pipeline.py +1 -1
  67. zenml/orchestrators/input_utils.py +3 -6
  68. zenml/stack/stack.py +3 -6
  69. zenml/stack_deployments/__init__.py +14 -0
  70. zenml/stack_deployments/aws_stack_deployment.py +289 -0
  71. zenml/stack_deployments/stack_deployment.py +130 -0
  72. zenml/stack_deployments/utils.py +40 -0
  73. zenml/utils/function_utils.py +1 -1
  74. zenml/utils/pagination_utils.py +7 -5
  75. zenml/utils/pipeline_docker_image_builder.py +97 -68
  76. zenml/utils/pydantic_utils.py +6 -5
  77. zenml/zen_server/cloud_utils.py +18 -3
  78. zenml/zen_server/dashboard/assets/{404-C1mcUujL.js → 404-DpJaNHKF.js} +1 -1
  79. zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +85 -0
  80. zenml/zen_server/dashboard/assets/{@react-router-DYovave8.js → @react-router-CO-OsFwI.js} +2 -2
  81. zenml/zen_server/dashboard/assets/{@reactflow-DYIyhCfd.js → @reactflow-DJfzkHO1.js} +2 -2
  82. zenml/zen_server/dashboard/assets/@tanstack-DYiOyJUL.js +22 -0
  83. zenml/zen_server/dashboard/assets/AwarenessChannel-BYDLT2xC.js +1 -0
  84. zenml/zen_server/dashboard/assets/{CodeSnippet-WEzpO0az.js → CodeSnippet-BkOuRmyq.js} +2 -2
  85. zenml/zen_server/dashboard/assets/Commands-ZvWR1BRs.js +1 -0
  86. zenml/zen_server/dashboard/assets/CopyButton-DVwLkafa.js +2 -0
  87. zenml/zen_server/dashboard/assets/{CsvVizualization-Bx931j4U.js → CsvVizualization-C2IiqX4I.js} +7 -7
  88. zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +1 -0
  89. zenml/zen_server/dashboard/assets/EmptyState-BMLnFVlB.js +1 -0
  90. zenml/zen_server/dashboard/assets/Error-CqX0VqW_.js +1 -0
  91. zenml/zen_server/dashboard/assets/ExecutionStatus-BoLUXR9t.js +1 -0
  92. zenml/zen_server/dashboard/assets/Helpbox-LFydyVwh.js +1 -0
  93. zenml/zen_server/dashboard/assets/Infobox-DnENC0sh.js +1 -0
  94. zenml/zen_server/dashboard/assets/InlineAvatar-CbJtYr0t.js +1 -0
  95. zenml/zen_server/dashboard/assets/{MarkdownVisualization-DsB2QZiK.js → MarkdownVisualization-xp3hhULl.js} +2 -2
  96. zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +1 -0
  97. zenml/zen_server/dashboard/assets/PasswordChecker-DUveqlva.js +1 -0
  98. zenml/zen_server/dashboard/assets/SetPassword-BYBdbQDo.js +1 -0
  99. zenml/zen_server/dashboard/assets/SuccessStep-Nx743hll.js +1 -0
  100. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CKrd3UZz.js → UpdatePasswordSchemas-DF9gSzE0.js} +1 -1
  101. zenml/zen_server/dashboard/assets/{aws-t0gKCj_R.js → aws-BgKTfTfx.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{check-circle-BVvhm5dy.js → check-circle-i56092KI.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{chevron-down-zcvCWmyP.js → chevron-down-D_ZlKMqH.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{chevron-right-double-CJ50E9Gr.js → chevron-right-double-BiEMg7rd.js} +1 -1
  105. zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +1 -0
  106. zenml/zen_server/dashboard/assets/{copy-BRhQz3j-.js → copy-BXNk6BjL.js} +1 -1
  107. zenml/zen_server/dashboard/assets/{database-CRRnyFWh.js → database-1xWSgZfO.js} +1 -1
  108. zenml/zen_server/dashboard/assets/{docker-BAonhm6G.js → docker-CQMVm_4d.js} +1 -1
  109. zenml/zen_server/dashboard/assets/{file-text-CbVERUON.js → file-text-CqD_iu6l.js} +1 -1
  110. zenml/zen_server/dashboard/assets/{help-B8rqCvqn.js → help-bu_DgLKI.js} +1 -1
  111. zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +1 -0
  112. zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +55 -0
  113. zenml/zen_server/dashboard/assets/index-inApY3KQ.css +1 -0
  114. zenml/zen_server/dashboard/assets/index-rK_Wuy2W.js +1 -0
  115. zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +1 -0
  116. zenml/zen_server/dashboard/assets/{login-mutation-Bk2tn324.js → login-mutation-BUnVASxp.js} +1 -1
  117. zenml/zen_server/dashboard/assets/not-found-B4VnX8gK.js +1 -0
  118. zenml/zen_server/dashboard/assets/package-CsUhPmou.js +1 -0
  119. zenml/zen_server/dashboard/assets/{page-D12Rvf0j.js → page-3efNCDeb.js} +2 -2
  120. zenml/zen_server/dashboard/assets/page-7zTHbhhI.js +1 -0
  121. zenml/zen_server/dashboard/assets/page-BEs6jK71.js +1 -0
  122. zenml/zen_server/dashboard/assets/page-BpSqIf4B.js +1 -0
  123. zenml/zen_server/dashboard/assets/{page-8vRWJ5b8.js → page-Bx6o0ARS.js} +1 -1
  124. zenml/zen_server/dashboard/assets/page-C43QGHTt.js +9 -0
  125. zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +1 -0
  126. zenml/zen_server/dashboard/assets/{page-CBuSUrE9.js → page-CRTJ0UuR.js} +1 -1
  127. zenml/zen_server/dashboard/assets/page-CUZIGO-3.js +1 -0
  128. zenml/zen_server/dashboard/assets/page-CaopxiU1.js +1 -0
  129. zenml/zen_server/dashboard/assets/{page-CCtCgG-x.js → page-Cx67M0QT.js} +1 -1
  130. zenml/zen_server/dashboard/assets/page-D7Z399xy.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-Dw9-aJV6.js → page-DKlIdAe5.js} +1 -1
  133. zenml/zen_server/dashboard/assets/{page-COafKNbw.js → page-DMOYZppS.js} +1 -1
  134. zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +2 -0
  135. zenml/zen_server/dashboard/assets/{page-C6v3o0Qj.js → page-Dc_7KMQE.js} +1 -1
  136. zenml/zen_server/dashboard/assets/page-DvCvroOM.js +1 -0
  137. zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +1 -0
  138. zenml/zen_server/dashboard/assets/page-JyfeDUfu.js +1 -0
  139. zenml/zen_server/dashboard/assets/{page-CH26py0a.js → page-Sxn82W-5.js} +1 -1
  140. zenml/zen_server/dashboard/assets/page-TKXERe16.js +1 -0
  141. zenml/zen_server/dashboard/assets/page-Xu8JEjSU.js +1 -0
  142. zenml/zen_server/dashboard/assets/{play-circle-DK5QMJyp.js → play-circle-CNtZKDnW.js} +1 -1
  143. zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +1 -0
  144. zenml/zen_server/dashboard/assets/{terminal-B2ovgWuz.js → terminal-By9cErXc.js} +1 -1
  145. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-bKxf7U9h.js → update-server-settings-mutation-CR8e3Sir.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{url-CgvM-IVM.js → url-DuQMeqYA.js} +1 -1
  147. zenml/zen_server/dashboard/assets/{zod-DrZvVLjd.js → zod-BhoGpZ63.js} +1 -1
  148. zenml/zen_server/dashboard/index.html +7 -7
  149. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  150. zenml/zen_server/dashboard_legacy/index.html +1 -1
  151. zenml/zen_server/dashboard_legacy/{precache-manifest.e7c29295aae591541ef59d1734d79387.js → precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js} +4 -4
  152. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  153. zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js → main.382439a7.chunk.js} +2 -2
  154. zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js.map → main.382439a7.chunk.js.map} +1 -1
  155. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  156. zenml/zen_server/deploy/helm/README.md +2 -2
  157. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +11 -5
  158. zenml/zen_server/pipeline_deployment/utils.py +57 -44
  159. zenml/zen_server/rbac/zenml_cloud_rbac.py +11 -5
  160. zenml/zen_server/routers/stack_deployment_endpoints.py +144 -0
  161. zenml/zen_server/routers/workspaces_endpoints.py +64 -0
  162. zenml/zen_server/zen_server_api.py +2 -0
  163. zenml/zen_stores/migrations/utils.py +1 -1
  164. zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
  165. zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
  166. zenml/zen_stores/rest_zen_store.py +117 -0
  167. zenml/zen_stores/schemas/stack_schemas.py +10 -0
  168. zenml/zen_stores/schemas/step_run_schemas.py +27 -11
  169. zenml/zen_stores/sql_zen_store.py +283 -0
  170. zenml/zen_stores/zen_store_interface.py +79 -0
  171. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/METADATA +32 -10
  172. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/RECORD +175 -161
  173. zenml/zen_server/dashboard/assets/@radix-C9DBgJhe.js +0 -77
  174. zenml/zen_server/dashboard/assets/@tanstack-CEbkxrhX.js +0 -30
  175. zenml/zen_server/dashboard/assets/AwarenessChannel-B2KR83Tr.js +0 -1
  176. zenml/zen_server/dashboard/assets/Cards-DSEdjsk8.js +0 -1
  177. zenml/zen_server/dashboard/assets/Commands-CTlhyic5.js +0 -1
  178. zenml/zen_server/dashboard/assets/CopyButton-CTrzKmUO.js +0 -2
  179. zenml/zen_server/dashboard/assets/DisplayDate-BdguISQF.js +0 -1
  180. zenml/zen_server/dashboard/assets/EmptyState-BkooiGtL.js +0 -1
  181. zenml/zen_server/dashboard/assets/Error-4sKxHad4.js +0 -1
  182. zenml/zen_server/dashboard/assets/Helpbox-DW21i5LD.js +0 -1
  183. zenml/zen_server/dashboard/assets/Infobox-C7bf70VS.js +0 -1
  184. zenml/zen_server/dashboard/assets/InlineAvatar-Dxrtafpg.js +0 -1
  185. zenml/zen_server/dashboard/assets/PageHeader-B0pUife2.js +0 -1
  186. zenml/zen_server/dashboard/assets/Pagination-B9WG_9cJ.js +0 -1
  187. zenml/zen_server/dashboard/assets/PasswordChecker-DSLBp7Vl.js +0 -1
  188. zenml/zen_server/dashboard/assets/SetPassword-CiNhT15a.js +0 -1
  189. zenml/zen_server/dashboard/assets/SuccessStep-CykrFndS.js +0 -1
  190. zenml/zen_server/dashboard/assets/cloud-only-Bkawp7CJ.js +0 -1
  191. zenml/zen_server/dashboard/assets/index-BawkpTlr.js +0 -55
  192. zenml/zen_server/dashboard/assets/index-CRmm7QhS.css +0 -1
  193. zenml/zen_server/dashboard/assets/index.esm-F7nqy9zY.js +0 -1
  194. zenml/zen_server/dashboard/assets/not-found-BAuhP4Jb.js +0 -1
  195. zenml/zen_server/dashboard/assets/page--5YvAHg3.js +0 -1
  196. zenml/zen_server/dashboard/assets/page-B0RAq4s_.js +0 -1
  197. zenml/zen_server/dashboard/assets/page-BePtEPHl.js +0 -1
  198. zenml/zen_server/dashboard/assets/page-C1pra1Bc.js +0 -9
  199. zenml/zen_server/dashboard/assets/page-CSs4C9jL.js +0 -1
  200. zenml/zen_server/dashboard/assets/page-Cf2XSej0.js +0 -1
  201. zenml/zen_server/dashboard/assets/page-ClPUAE_f.js +0 -1
  202. zenml/zen_server/dashboard/assets/page-D8pf2vis.js +0 -1
  203. zenml/zen_server/dashboard/assets/page-DHKMmIQH.js +0 -1
  204. zenml/zen_server/dashboard/assets/page-DMZ0VOda.js +0 -1
  205. zenml/zen_server/dashboard/assets/page-Dcg-yQv_.js +0 -1
  206. zenml/zen_server/dashboard/assets/page-DoAK5FSB.js +0 -1
  207. zenml/zen_server/dashboard/assets/page-iXiDqE0J.js +0 -2
  208. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/LICENSE +0 -0
  209. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/WHEEL +0 -0
  210. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/entry_points.txt +0 -0
zenml/cli/stack.py CHANGED
@@ -15,17 +15,34 @@
15
15
 
16
16
  import getpass
17
17
  import os
18
+ import re
19
+ import time
20
+ import webbrowser
21
+ from datetime import datetime
18
22
  from pathlib import Path
19
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
23
+ from typing import (
24
+ TYPE_CHECKING,
25
+ Any,
26
+ Dict,
27
+ List,
28
+ Optional,
29
+ Set,
30
+ Union,
31
+ )
20
32
  from uuid import UUID
21
33
 
22
34
  import click
35
+ from rich.console import Console
36
+ from rich.markdown import Markdown
37
+ from rich.prompt import Confirm
38
+ from rich.syntax import Syntax
23
39
 
24
40
  import zenml
25
41
  from zenml.analytics.enums import AnalyticsEvent
26
42
  from zenml.analytics.utils import track_handler
27
43
  from zenml.cli import utils as cli_utils
28
44
  from zenml.cli.cli import TagGroup, cli
45
+ from zenml.cli.text_utils import OldSchoolMarkdownHeading
29
46
  from zenml.cli.utils import (
30
47
  _component_display_name,
31
48
  confirmation,
@@ -45,7 +62,11 @@ from zenml.constants import (
45
62
  MLSTACKS_SUPPORTED_STACK_COMPONENTS,
46
63
  STACK_RECIPE_MODULAR_RECIPES,
47
64
  )
48
- from zenml.enums import CliCategories, StackComponentType
65
+ from zenml.enums import (
66
+ CliCategories,
67
+ StackComponentType,
68
+ StackDeploymentProvider,
69
+ )
49
70
  from zenml.exceptions import (
50
71
  IllegalOperationError,
51
72
  ProvisioningError,
@@ -53,7 +74,19 @@ from zenml.exceptions import (
53
74
  from zenml.io.fileio import rmtree
54
75
  from zenml.logger import get_logger
55
76
  from zenml.models import StackFilter
56
- from zenml.utils.dashboard_utils import get_stack_url
77
+ from zenml.models.v2.core.service_connector import (
78
+ ServiceConnectorRequest,
79
+ ServiceConnectorResponse,
80
+ )
81
+ from zenml.models.v2.misc.full_stack import (
82
+ ComponentInfo,
83
+ FullStackRequest,
84
+ ServiceConnectorInfo,
85
+ )
86
+ from zenml.models.v2.misc.service_connector_type import (
87
+ ServiceConnectorTypedResourcesModel,
88
+ )
89
+ from zenml.utils.dashboard_utils import get_component_url, get_stack_url
57
90
  from zenml.utils.io_utils import create_dir_recursive_if_not_exists
58
91
  from zenml.utils.mlstacks_utils import (
59
92
  convert_click_params_to_mlstacks_primitives,
@@ -93,7 +126,7 @@ def stack() -> None:
93
126
  "artifact_store",
94
127
  help="Name of the artifact store for this stack.",
95
128
  type=str,
96
- required=True,
129
+ required=False,
97
130
  )
98
131
  @click.option(
99
132
  "-o",
@@ -101,7 +134,7 @@ def stack() -> None:
101
134
  "orchestrator",
102
135
  help="Name of the orchestrator for this stack.",
103
136
  type=str,
104
- required=True,
137
+ required=False,
105
138
  )
106
139
  @click.option(
107
140
  "-c",
@@ -190,10 +223,24 @@ def stack() -> None:
190
223
  help="Immediately set this stack as active.",
191
224
  type=click.BOOL,
192
225
  )
226
+ @click.option(
227
+ "-p",
228
+ "--provider",
229
+ help="Name of the cloud provider for this stack.",
230
+ type=click.Choice(["aws", "azure", "gcp"]),
231
+ required=False,
232
+ )
233
+ @click.option(
234
+ "-sc",
235
+ "--connector",
236
+ help="Name of the service connector for this stack.",
237
+ type=str,
238
+ required=False,
239
+ )
193
240
  def register_stack(
194
241
  stack_name: str,
195
- artifact_store: str,
196
- orchestrator: str,
242
+ artifact_store: Optional[str] = None,
243
+ orchestrator: Optional[str] = None,
197
244
  container_registry: Optional[str] = None,
198
245
  model_registry: Optional[str] = None,
199
246
  step_operator: Optional[str] = None,
@@ -205,6 +252,8 @@ def register_stack(
205
252
  data_validator: Optional[str] = None,
206
253
  image_builder: Optional[str] = None,
207
254
  set_stack: bool = False,
255
+ provider: Optional[str] = None,
256
+ connector: Optional[str] = None,
208
257
  ) -> None:
209
258
  """Register a stack.
210
259
 
@@ -223,44 +272,240 @@ def register_stack(
223
272
  data_validator: Name of the data validator for this stack.
224
273
  image_builder: Name of the new image builder for this stack.
225
274
  set_stack: Immediately set this stack as active.
275
+ provider: Name of the cloud provider for this stack.
276
+ connector: Name of the service connector for this stack.
226
277
  """
227
- with console.status(f"Registering stack '{stack_name}'...\n"):
228
- client = Client()
278
+ if (provider is None and connector is None) and (
279
+ artifact_store is None or orchestrator is None
280
+ ):
281
+ cli_utils.error(
282
+ "Only stack using service connector can be registered "
283
+ "without specifying an artifact store and an orchestrator. "
284
+ "Please specify the artifact store and the orchestrator or "
285
+ "the service connector or cloud type settings."
286
+ )
229
287
 
230
- components: Dict[StackComponentType, Union[str, UUID]] = dict()
288
+ client = Client()
231
289
 
232
- components[StackComponentType.ARTIFACT_STORE] = artifact_store
233
- components[StackComponentType.ORCHESTRATOR] = orchestrator
290
+ try:
291
+ client.get_stack(
292
+ name_id_or_prefix=stack_name,
293
+ allow_name_prefix_match=False,
294
+ )
295
+ cli_utils.error(
296
+ f"A stack with name `{stack_name}` already exists, "
297
+ "please use a different name."
298
+ )
299
+ except KeyError:
300
+ pass
234
301
 
235
- if alerter:
236
- components[StackComponentType.ALERTER] = alerter
237
- if annotator:
238
- components[StackComponentType.ANNOTATOR] = annotator
239
- if data_validator:
240
- components[StackComponentType.DATA_VALIDATOR] = data_validator
241
- if feature_store:
242
- components[StackComponentType.FEATURE_STORE] = feature_store
243
- if image_builder:
244
- components[StackComponentType.IMAGE_BUILDER] = image_builder
245
- if model_deployer:
246
- components[StackComponentType.MODEL_DEPLOYER] = model_deployer
247
- if model_registry:
248
- components[StackComponentType.MODEL_REGISTRY] = model_registry
249
- if step_operator:
250
- components[StackComponentType.STEP_OPERATOR] = step_operator
251
- if experiment_tracker:
252
- components[StackComponentType.EXPERIMENT_TRACKER] = (
253
- experiment_tracker
302
+ labels: Dict[str, str] = {}
303
+ components: Dict[StackComponentType, Union[UUID, ComponentInfo]] = {}
304
+ # cloud flow
305
+ created_objects: Set[str] = set()
306
+ service_connector: Optional[Union[UUID, ServiceConnectorInfo]] = None
307
+ if provider is not None and connector is None:
308
+ service_connector_response = None
309
+ use_auto_configure = False
310
+ try:
311
+ service_connector_response, _ = client.create_service_connector(
312
+ name=stack_name,
313
+ connector_type=provider,
314
+ register=False,
315
+ auto_configure=True,
316
+ verify=False,
254
317
  )
255
- if container_registry:
256
- components[StackComponentType.CONTAINER_REGISTRY] = (
257
- container_registry
318
+ except Exception:
319
+ pass
320
+ if service_connector_response:
321
+ use_auto_configure = Confirm.ask(
322
+ f"[bold]{provider.upper()} cloud service connector[/bold] "
323
+ "has detected connection credentials in your environment.\n"
324
+ "Would you like to use these credentials or create a new "
325
+ "configuration by providing connection details?",
326
+ default=True,
327
+ show_choices=True,
328
+ show_default=True,
329
+ )
330
+
331
+ connector_selected: Optional[int] = None
332
+ if not use_auto_configure:
333
+ service_connector_response = None
334
+ existing_connectors = client.list_service_connectors(
335
+ connector_type=provider, size=100
336
+ )
337
+ if existing_connectors.total:
338
+ connector_selected = cli_utils.multi_choice_prompt(
339
+ object_type=f"{provider.upper()} service connectors",
340
+ choices=[
341
+ [connector.name]
342
+ for connector in existing_connectors.items
343
+ ],
344
+ headers=["Name"],
345
+ prompt_text=f"We found these {provider.upper()} service connectors. "
346
+ "Do you want to create a new one or use one of the existing ones?",
347
+ default_choice="0",
348
+ allow_zero_be_a_new_object=True,
349
+ )
350
+ if use_auto_configure or connector_selected is None:
351
+ service_connector = _get_service_connector_info(
352
+ cloud_provider=provider,
353
+ connector_details=service_connector_response,
258
354
  )
355
+ created_objects.add("service_connector")
356
+ else:
357
+ selected_connector = existing_connectors.items[connector_selected]
358
+ service_connector = selected_connector.id
359
+ connector = selected_connector.name
360
+ if isinstance(selected_connector.connector_type, str):
361
+ provider = selected_connector.connector_type
362
+ else:
363
+ provider = selected_connector.connector_type.connector_type
364
+ elif connector is not None:
365
+ service_connector_response = client.get_service_connector(connector)
366
+ service_connector = service_connector_response.id
367
+ if provider:
368
+ if service_connector_response.type != provider:
369
+ cli_utils.warning(
370
+ f"The service connector `{connector}` is not of type `{provider}`."
371
+ )
372
+ else:
373
+ provider = service_connector_response.type
374
+
375
+ if service_connector:
376
+ labels["zenml:wizard"] = "true"
377
+ if provider:
378
+ labels["zenml:provider"] = provider
379
+ service_connector_resource_model = None
380
+ # create components
381
+ needed_components = (
382
+ (StackComponentType.ARTIFACT_STORE, artifact_store),
383
+ (StackComponentType.ORCHESTRATOR, orchestrator),
384
+ (StackComponentType.CONTAINER_REGISTRY, container_registry),
385
+ )
386
+ for component_type, preset_name in needed_components:
387
+ component_info: Optional[Union[UUID, ComponentInfo]] = None
388
+ if preset_name is not None:
389
+ component_response = client.get_stack_component(
390
+ component_type, preset_name
391
+ )
392
+ component_info = component_response.id
393
+ else:
394
+ if isinstance(service_connector, UUID):
395
+ # find existing components under same connector
396
+ existing_components = client.list_stack_components(
397
+ type=component_type.value,
398
+ connector_id=service_connector,
399
+ size=100,
400
+ )
401
+ # if some existing components are found - prompt user what to do
402
+ component_selected: Optional[int] = None
403
+ if existing_components.total > 0:
404
+ component_selected = cli_utils.multi_choice_prompt(
405
+ object_type=component_type.value.replace("_", " "),
406
+ choices=[
407
+ [component.name]
408
+ for component in existing_components.items
409
+ ],
410
+ headers=["Name"],
411
+ prompt_text=f"We found these {component_type.value.replace('_', ' ')} "
412
+ "connected using the current service connector. Do you "
413
+ "want to create a new one or use existing one?",
414
+ default_choice="0",
415
+ allow_zero_be_a_new_object=True,
416
+ )
417
+ else:
418
+ component_selected = None
419
+
420
+ if component_selected is None:
421
+ if service_connector_resource_model is None:
422
+ if isinstance(service_connector, UUID):
423
+ service_connector_resource_model = (
424
+ client.verify_service_connector(
425
+ service_connector
426
+ )
427
+ )
428
+ else:
429
+ _, service_connector_resource_model = (
430
+ client.create_service_connector(
431
+ name=stack_name,
432
+ connector_type=service_connector.type,
433
+ auth_method=service_connector.auth_method,
434
+ configuration=service_connector.configuration,
435
+ register=False,
436
+ )
437
+ )
438
+ if service_connector_resource_model is None:
439
+ cli_utils.error(
440
+ f"Failed to validate service connector {service_connector}..."
441
+ )
442
+ if provider is None:
443
+ if isinstance(
444
+ service_connector_resource_model.connector_type,
445
+ str,
446
+ ):
447
+ provider = (
448
+ service_connector_resource_model.connector_type
449
+ )
450
+ else:
451
+ provider = service_connector_resource_model.connector_type.connector_type
452
+
453
+ component_info = _get_stack_component_info(
454
+ component_type=component_type.value,
455
+ cloud_provider=provider,
456
+ service_connector_resource_models=service_connector_resource_model.resources,
457
+ service_connector_index=0,
458
+ )
459
+ component_name = stack_name
460
+ created_objects.add(component_type.value)
461
+ else:
462
+ selected_component = existing_components.items[
463
+ component_selected
464
+ ]
465
+ component_info = selected_component.id
466
+ component_name = selected_component.name
467
+
468
+ components[component_type] = component_info
469
+ if component_type == StackComponentType.ARTIFACT_STORE:
470
+ artifact_store = component_name
471
+ if component_type == StackComponentType.ORCHESTRATOR:
472
+ orchestrator = component_name
473
+ if component_type == StackComponentType.CONTAINER_REGISTRY:
474
+ container_registry = component_name
475
+
476
+ # normal flow once all components are defined
477
+ with console.status(f"Registering stack '{stack_name}'...\n"):
478
+ for component_type_, component_name_ in [
479
+ (StackComponentType.ARTIFACT_STORE, artifact_store),
480
+ (StackComponentType.ORCHESTRATOR, orchestrator),
481
+ (StackComponentType.ALERTER, alerter),
482
+ (StackComponentType.ANNOTATOR, annotator),
483
+ (StackComponentType.DATA_VALIDATOR, data_validator),
484
+ (StackComponentType.FEATURE_STORE, feature_store),
485
+ (StackComponentType.IMAGE_BUILDER, image_builder),
486
+ (StackComponentType.MODEL_DEPLOYER, model_deployer),
487
+ (StackComponentType.MODEL_REGISTRY, model_registry),
488
+ (StackComponentType.STEP_OPERATOR, step_operator),
489
+ (StackComponentType.EXPERIMENT_TRACKER, experiment_tracker),
490
+ (StackComponentType.CONTAINER_REGISTRY, container_registry),
491
+ ]:
492
+ if component_name_ and component_type_ not in components:
493
+ components[component_type_] = client.get_stack_component(
494
+ component_type_, component_name_
495
+ ).id
259
496
 
260
497
  try:
261
- created_stack = client.create_stack(
262
- name=stack_name,
263
- components=components,
498
+ created_stack = client.zen_store.create_full_stack(
499
+ full_stack=FullStackRequest(
500
+ user=client.active_user.id,
501
+ workspace=client.active_workspace.id,
502
+ name=stack_name,
503
+ components=components,
504
+ service_connectors=[service_connector]
505
+ if service_connector
506
+ else [],
507
+ labels=labels,
508
+ )
264
509
  )
265
510
  except (KeyError, IllegalOperationError) as err:
266
511
  cli_utils.error(str(err))
@@ -268,6 +513,10 @@ def register_stack(
268
513
  cli_utils.declare(
269
514
  f"Stack '{created_stack.name}' successfully registered!"
270
515
  )
516
+ cli_utils.print_stack_configuration(
517
+ stack=created_stack,
518
+ active=created_stack.id == client.active_stack_model.id,
519
+ )
271
520
 
272
521
  if set_stack:
273
522
  client.activate_stack(created_stack.id)
@@ -277,6 +526,30 @@ def register_stack(
277
526
  f"Active {scope} stack set to:'{created_stack.name}'"
278
527
  )
279
528
 
529
+ delete_commands = []
530
+ if "service_connector" in created_objects:
531
+ created_objects.remove("service_connector")
532
+ connectors = set()
533
+ for each in created_objects:
534
+ if comps_ := created_stack.components[StackComponentType(each)]:
535
+ if conn_ := comps_[0].connector:
536
+ connectors.add(conn_.name)
537
+ for connector in connectors:
538
+ delete_commands.append(
539
+ "zenml service-connector delete " + connector
540
+ )
541
+ for each in created_objects:
542
+ if comps_ := created_stack.components[StackComponentType(each)]:
543
+ delete_commands.append(
544
+ f"zenml {each.replace('_', '-')} delete {comps_[0].name}"
545
+ )
546
+ delete_commands.append("zenml stack delete -y " + created_stack.name)
547
+
548
+ Console().print(
549
+ "To delete the objects created by this command run, please run in a sequence:\n"
550
+ )
551
+ Console().print(Syntax("\n".join(delete_commands[::-1]), "bash"))
552
+
280
553
  print_model_url(get_stack_url(created_stack))
281
554
 
282
555
 
@@ -1286,7 +1559,226 @@ def _get_deployment_params_interactively(
1286
1559
  return deployment_values
1287
1560
 
1288
1561
 
1289
- @stack.command(help="Deploy a stack using mlstacks.")
1562
+ def validate_name(ctx: click.Context, param: str, value: str) -> str:
1563
+ """Validate the name of the stack.
1564
+
1565
+ Args:
1566
+ ctx: The click context.
1567
+ param: The parameter name.
1568
+ value: The value of the parameter.
1569
+
1570
+ Returns:
1571
+ The validated value.
1572
+
1573
+ Raises:
1574
+ BadParameter: If the name is invalid.
1575
+ """
1576
+ if not value:
1577
+ return value
1578
+
1579
+ if not re.match(r"^[a-zA-Z0-9-]*$", value):
1580
+ raise click.BadParameter(
1581
+ "Stack name must contain only alphanumeric characters and hyphens."
1582
+ )
1583
+
1584
+ if len(value) > 16:
1585
+ raise click.BadParameter(
1586
+ "Stack name must have a maximum length of 16 characters."
1587
+ )
1588
+
1589
+ return value
1590
+
1591
+
1592
+ @stack.command(
1593
+ help="""Deploy a fully functional ZenML stack in one of the cloud providers.
1594
+
1595
+ Running this command will initiate an assisted process that will walk you
1596
+ through automatically provisioning all the cloud infrastructure resources
1597
+ necessary for a fully functional ZenML stack in the cloud provider of your
1598
+ choice. A corresponding ZenML stack will also be automatically registered along
1599
+ with all the necessary components and properly authenticated through service
1600
+ connectors.
1601
+ """
1602
+ )
1603
+ @click.option(
1604
+ "--provider",
1605
+ "-p",
1606
+ "provider",
1607
+ required=True,
1608
+ type=click.Choice(StackDeploymentProvider.values()),
1609
+ )
1610
+ @click.option(
1611
+ "--name",
1612
+ "-n",
1613
+ "stack_name",
1614
+ type=click.STRING,
1615
+ required=False,
1616
+ help="Custom string to use as a prefix to generate names for the ZenML "
1617
+ "stack, its components service connectors as well as provisioned cloud "
1618
+ "infrastructure resources. May only contain alphanumeric characters and "
1619
+ "hyphens and have a maximum length of 16 characters.",
1620
+ callback=validate_name,
1621
+ )
1622
+ @click.option(
1623
+ "--location",
1624
+ "-l",
1625
+ type=click.STRING,
1626
+ required=False,
1627
+ help="The location to deploy the stack to.",
1628
+ )
1629
+ @click.option(
1630
+ "--set",
1631
+ "set_stack",
1632
+ is_flag=True,
1633
+ help="Immediately set this stack as active.",
1634
+ type=click.BOOL,
1635
+ )
1636
+ @click.pass_context
1637
+ def deploy(
1638
+ ctx: click.Context,
1639
+ provider: str,
1640
+ stack_name: Optional[str] = None,
1641
+ location: Optional[str] = None,
1642
+ set_stack: bool = False,
1643
+ ) -> None:
1644
+ """Deploy and register a fully functional cloud ZenML stack.
1645
+
1646
+ Args:
1647
+ ctx: The click context.
1648
+ provider: The cloud provider to deploy the stack to.
1649
+ stack_name: A name for the ZenML stack that gets imported as a result
1650
+ of the recipe deployment.
1651
+ location: The location to deploy the stack to.
1652
+ set_stack: Immediately set the deployed stack as active.
1653
+
1654
+ Raises:
1655
+ Abort: If the user aborts the deployment.
1656
+ KeyboardInterrupt: If the user interrupts the deployment.
1657
+ """
1658
+ stack_name = stack_name or f"zenml-{provider}-stack"
1659
+
1660
+ # Set up the markdown renderer to use the old-school markdown heading
1661
+ Markdown.elements.update(
1662
+ {
1663
+ "heading_open": OldSchoolMarkdownHeading,
1664
+ }
1665
+ )
1666
+
1667
+ client = Client()
1668
+ if client.zen_store.is_local_store():
1669
+ cli_utils.error(
1670
+ "This feature cannot be used with a local ZenML deployment. "
1671
+ "ZenML needs to be accessible from the cloud provider to allow the "
1672
+ "stack and its components to be registered automatically. "
1673
+ "Please deploy ZenML in a remote environment as described in the "
1674
+ "documentation: https://docs.zenml.io/getting-started/deploying-zenml "
1675
+ "or use a managed ZenML Pro server instance for quick access to "
1676
+ "this feature and more: https://www.zenml.io/pro"
1677
+ )
1678
+
1679
+ with track_handler(
1680
+ event=AnalyticsEvent.DEPLOY_FULL_STACK,
1681
+ ) as analytics_handler:
1682
+ analytics_handler.metadata = {
1683
+ "provider": provider,
1684
+ }
1685
+
1686
+ deployment = client.zen_store.get_stack_deployment_info(
1687
+ provider=StackDeploymentProvider(provider),
1688
+ )
1689
+
1690
+ console.print(
1691
+ Markdown(
1692
+ f"# {provider.upper()} ZenML Cloud Stack Deployment\n"
1693
+ + deployment.description
1694
+ )
1695
+ )
1696
+ console.print(Markdown("## Instructions\n" + deployment.instructions))
1697
+
1698
+ if not cli_utils.confirmation(
1699
+ "\n\nProceed to continue with the deployment. You will be "
1700
+ f"automatically redirected to {provider.upper()} in your browser.",
1701
+ ):
1702
+ raise click.Abort()
1703
+
1704
+ deployment_url, deployment_url_title = (
1705
+ client.zen_store.get_stack_deployment_url(
1706
+ provider=StackDeploymentProvider(provider),
1707
+ stack_name=stack_name,
1708
+ location=location,
1709
+ )
1710
+ )
1711
+
1712
+ date_start = datetime.utcnow()
1713
+
1714
+ webbrowser.open(deployment_url)
1715
+ console.print(
1716
+ Markdown(
1717
+ f"If your browser did not open automatically, please open "
1718
+ f"the following URL into your browser to deploy the stack to "
1719
+ f"{provider.upper()}: "
1720
+ f"[{deployment_url_title}]({deployment_url}).\n\n"
1721
+ )
1722
+ )
1723
+
1724
+ try:
1725
+ with console.status(
1726
+ "Waiting for the deployment to complete and the stack to be "
1727
+ "registered. Press CTRL+C to abort...\n"
1728
+ ):
1729
+ while True:
1730
+ deployed_stack = (
1731
+ client.zen_store.get_stack_deployment_stack(
1732
+ provider=StackDeploymentProvider(provider),
1733
+ stack_name=stack_name,
1734
+ location=location,
1735
+ date_start=date_start,
1736
+ )
1737
+ )
1738
+ if deployed_stack:
1739
+ break
1740
+ time.sleep(10)
1741
+
1742
+ analytics_handler.metadata.update(
1743
+ {
1744
+ "stack_id": deployed_stack.stack.id,
1745
+ }
1746
+ )
1747
+
1748
+ except KeyboardInterrupt:
1749
+ cli_utils.declare("Stack deployment aborted.")
1750
+ raise
1751
+
1752
+ stack_desc = f"""## Stack successfully registered! 🚀
1753
+ Stack [{deployed_stack.stack.name}]({get_stack_url(deployed_stack.stack)}):\n"""
1754
+
1755
+ for component_type, components in deployed_stack.stack.components.items():
1756
+ if components:
1757
+ component = components[0]
1758
+ stack_desc += (
1759
+ f" * `{component.flavor}` {component_type.value}: "
1760
+ f"[{component.name}]({get_component_url(component)})\n"
1761
+ )
1762
+
1763
+ if deployed_stack.service_connector:
1764
+ stack_desc += (
1765
+ f" * Service Connector: {deployed_stack.service_connector.name}\n"
1766
+ )
1767
+
1768
+ console.print(Markdown(stack_desc))
1769
+
1770
+ console.print(
1771
+ Markdown("## Follow-up\n" + deployment.post_deploy_instructions)
1772
+ )
1773
+
1774
+ if set_stack:
1775
+ client.activate_stack(deployed_stack.stack.id)
1776
+ cli_utils.declare(
1777
+ f"\nStack `{deployed_stack.stack.name}` set as active"
1778
+ )
1779
+
1780
+
1781
+ @stack.command(help="[DEPRECATED] Deploy a stack using mlstacks.")
1290
1782
  @click.option(
1291
1783
  "--provider",
1292
1784
  "-p",
@@ -1426,7 +1918,7 @@ def _get_deployment_params_interactively(
1426
1918
  help="Deploy the stack interactively.",
1427
1919
  )
1428
1920
  @click.pass_context
1429
- def deploy(
1921
+ def deploy_mlstack(
1430
1922
  ctx: click.Context,
1431
1923
  provider: str,
1432
1924
  stack_name: str,
@@ -1476,6 +1968,13 @@ def deploy(
1476
1968
  region: The region to deploy the stack to.
1477
1969
  interactive: Deploy the stack interactively.
1478
1970
  """
1971
+ cli_utils.warning(
1972
+ "The `zenml stack deploy-mlstack` (former `zenml stack deploy`) CLI "
1973
+ "command has been deprecated and will be removed in a future release. "
1974
+ "Please use `zenml stack deploy` instead for a simplified "
1975
+ "experience."
1976
+ )
1977
+
1479
1978
  with track_handler(
1480
1979
  event=AnalyticsEvent.DEPLOY_STACK,
1481
1980
  ) as analytics_handler:
@@ -1727,3 +2226,262 @@ def connect_stack(
1727
2226
  interactive=interactive,
1728
2227
  no_verify=no_verify,
1729
2228
  )
2229
+
2230
+
2231
+ def _get_service_connector_info(
2232
+ cloud_provider: str,
2233
+ connector_details: Optional[
2234
+ Union[ServiceConnectorResponse, ServiceConnectorRequest]
2235
+ ],
2236
+ ) -> ServiceConnectorInfo:
2237
+ """Get a service connector info with given cloud provider.
2238
+
2239
+ Args:
2240
+ cloud_provider: The cloud provider to use.
2241
+ connector_details: Whether to use implicit credentials.
2242
+
2243
+ Returns:
2244
+ The info model of the created service connector.
2245
+
2246
+ Raises:
2247
+ ValueError: If the cloud provider is not supported.
2248
+ """
2249
+ from rich.prompt import Prompt
2250
+
2251
+ if cloud_provider not in {"aws"}:
2252
+ raise ValueError(f"Unknown cloud provider {cloud_provider}")
2253
+
2254
+ client = Client()
2255
+ auth_methods = client.get_service_connector_type(
2256
+ cloud_provider
2257
+ ).auth_method_dict
2258
+ if not connector_details:
2259
+ fixed_auth_methods = list(
2260
+ [
2261
+ (key, value)
2262
+ for key, value in auth_methods.items()
2263
+ if key != "implicit"
2264
+ ]
2265
+ )
2266
+ choices = []
2267
+ headers = ["Name", "Required"]
2268
+ for _, value in fixed_auth_methods:
2269
+ schema = value.config_schema
2270
+ required = ""
2271
+ for each_req in schema["required"]:
2272
+ field = schema["properties"][each_req]
2273
+ required += f"[bold]{each_req}[/bold] [italic]({field.get('title','no description')})[/italic]\n"
2274
+ choices.append([value.name, required])
2275
+
2276
+ selected_auth_idx = cli_utils.multi_choice_prompt(
2277
+ object_type=f"authentication methods for {cloud_provider}",
2278
+ choices=choices,
2279
+ headers=headers,
2280
+ prompt_text="Please choose one of the authentication option above.",
2281
+ )
2282
+ if selected_auth_idx is None:
2283
+ cli_utils.error("No authentication method selected.")
2284
+ auth_type = fixed_auth_methods[selected_auth_idx][0]
2285
+ else:
2286
+ auth_type = connector_details.auth_method
2287
+
2288
+ selected_auth_model = auth_methods[auth_type]
2289
+
2290
+ required_fields = selected_auth_model.config_schema["required"]
2291
+ properties = selected_auth_model.config_schema["properties"]
2292
+
2293
+ answers = {}
2294
+ for req_field in required_fields:
2295
+ if connector_details:
2296
+ if conf_value := connector_details.configuration.get(
2297
+ req_field, None
2298
+ ):
2299
+ answers[req_field] = conf_value
2300
+ elif secret_value := connector_details.secrets.get(
2301
+ req_field, None
2302
+ ):
2303
+ answers[req_field] = secret_value.get_secret_value()
2304
+ if req_field not in answers:
2305
+ answers[req_field] = Prompt.ask(
2306
+ f"Please enter value for `{req_field}`:",
2307
+ password="format" in properties[req_field]
2308
+ and properties[req_field]["format"] == "password",
2309
+ )
2310
+ Console().print("All mandatory configuration parameters received!")
2311
+
2312
+ return ServiceConnectorInfo(
2313
+ type=cloud_provider,
2314
+ auth_method=auth_type,
2315
+ configuration=answers,
2316
+ )
2317
+
2318
+
2319
+ def _get_stack_component_info(
2320
+ component_type: str,
2321
+ cloud_provider: str,
2322
+ service_connector_resource_models: List[
2323
+ ServiceConnectorTypedResourcesModel
2324
+ ],
2325
+ service_connector_index: Optional[int] = None,
2326
+ ) -> ComponentInfo:
2327
+ """Get a stack component info with given type and service connector.
2328
+
2329
+ Args:
2330
+ component_type: The type of component to create.
2331
+ cloud_provider: The cloud provider to use.
2332
+ service_connector_resource_models: The list of the available service connector resource models.
2333
+ service_connector_index: The index of the service connector to use.
2334
+
2335
+ Returns:
2336
+ The info model of the stack component.
2337
+
2338
+ Raises:
2339
+ ValueError: If the cloud provider is not supported.
2340
+ ValueError: If the component type is not supported.
2341
+ """
2342
+ from rich.prompt import Prompt
2343
+
2344
+ if cloud_provider not in {"aws", "azure", "gcp"}:
2345
+ raise ValueError(f"Unknown cloud provider {cloud_provider}")
2346
+
2347
+ AWS_DOCS = (
2348
+ "https://docs.zenml.io/how-to/auth-management/aws-service-connector"
2349
+ )
2350
+
2351
+ flavor = "undefined"
2352
+ service_connector_resource_id = None
2353
+ config = {}
2354
+ if component_type == "artifact_store":
2355
+ available_storages: List[str] = []
2356
+ if cloud_provider == "aws":
2357
+ for each in service_connector_resource_models:
2358
+ if each.resource_type == "s3-bucket":
2359
+ available_storages = each.resource_ids or []
2360
+ flavor = "s3"
2361
+ if not available_storages:
2362
+ cli_utils.error(
2363
+ "We were unable to find any S3 buckets available "
2364
+ "to configured service connector. Please, verify "
2365
+ "that needed permission are granted for the "
2366
+ "service connector.\nDocumentation for the S3 "
2367
+ "Buckets configuration can be found at "
2368
+ f"{AWS_DOCS}#s3-bucket"
2369
+ )
2370
+ elif cloud_provider == "azure":
2371
+ flavor = "azure"
2372
+ elif cloud_provider == "gcp":
2373
+ flavor = "gcs"
2374
+
2375
+ selected_storage_idx = cli_utils.multi_choice_prompt(
2376
+ object_type=f"{cloud_provider.upper()} storages",
2377
+ choices=[[st] for st in available_storages],
2378
+ headers=["Storage"],
2379
+ prompt_text="Please choose one of the storages for the new artifact store:",
2380
+ )
2381
+ if selected_storage_idx is None:
2382
+ cli_utils.error("No storage selected.")
2383
+
2384
+ selected_storage = available_storages[selected_storage_idx]
2385
+
2386
+ config = {"path": selected_storage}
2387
+ service_connector_resource_id = selected_storage
2388
+ elif component_type == "orchestrator":
2389
+ if cloud_provider == "aws":
2390
+ available_orchestrators = []
2391
+ for each in service_connector_resource_models:
2392
+ types = []
2393
+ if each.resource_type == "aws-generic":
2394
+ types = ["Sagemaker", "Skypilot (EC2)"]
2395
+ if each.resource_type == "kubernetes-cluster":
2396
+ types = ["Kubernetes"]
2397
+
2398
+ if each.resource_ids:
2399
+ for orchestrator in each.resource_ids:
2400
+ for t in types:
2401
+ available_orchestrators.append([t, orchestrator])
2402
+ if not available_orchestrators:
2403
+ cli_utils.error(
2404
+ "We were unable to find any orchestrator engines "
2405
+ "available to the service connector. Please, verify "
2406
+ "that needed permission are granted for the "
2407
+ "service connector.\nDocumentation for the Generic "
2408
+ "AWS resource configuration can be found at "
2409
+ f"{AWS_DOCS}#generic-aws-resource\n"
2410
+ "Documentation for the Kubernetes resource "
2411
+ "configuration can be found at "
2412
+ f"{AWS_DOCS}#eks-kubernetes-cluster"
2413
+ )
2414
+ elif cloud_provider == "gcp":
2415
+ pass
2416
+ elif cloud_provider == "azure":
2417
+ pass
2418
+
2419
+ selected_orchestrator_idx = cli_utils.multi_choice_prompt(
2420
+ object_type=f"orchestrators on {cloud_provider.upper()}",
2421
+ choices=available_orchestrators,
2422
+ headers=["Orchestrator Type", "Orchestrator details"],
2423
+ prompt_text="Please choose one of the orchestrators for the new orchestrator:",
2424
+ )
2425
+ if selected_orchestrator_idx is None:
2426
+ cli_utils.error("No orchestrator selected.")
2427
+
2428
+ selected_orchestrator = available_orchestrators[
2429
+ selected_orchestrator_idx
2430
+ ]
2431
+
2432
+ if selected_orchestrator[0] == "Sagemaker":
2433
+ flavor = "sagemaker"
2434
+ execution_role = Prompt.ask("Please enter an execution role ARN:")
2435
+ config = {"execution_role": execution_role}
2436
+ elif selected_orchestrator[0] == "Skypilot (EC2)":
2437
+ flavor = "vm_aws"
2438
+ config = {"region": selected_orchestrator[1]}
2439
+ elif selected_orchestrator[0] == "Kubernetes":
2440
+ flavor = "kubernetes"
2441
+ config = {}
2442
+ else:
2443
+ raise ValueError(
2444
+ f"Unknown orchestrator type {selected_orchestrator[0]}"
2445
+ )
2446
+ service_connector_resource_id = selected_orchestrator[1]
2447
+ elif component_type == "container_registry":
2448
+ available_registries: List[str] = []
2449
+ if cloud_provider == "aws":
2450
+ flavor = "aws"
2451
+ for each in service_connector_resource_models:
2452
+ if each.resource_type == "docker-registry":
2453
+ available_registries = each.resource_ids or []
2454
+ if not available_registries:
2455
+ cli_utils.error(
2456
+ "We were unable to find any container registries "
2457
+ "available to the service connector. Please, verify "
2458
+ "that needed permission are granted for the "
2459
+ "service connector.\nDocumentation for the ECR "
2460
+ "container registry resource configuration can "
2461
+ f"be found at {AWS_DOCS}#ecr-container-registry"
2462
+ )
2463
+ elif cloud_provider == "azure":
2464
+ flavor = "azure"
2465
+ elif cloud_provider == "gcp":
2466
+ flavor = "gcp"
2467
+
2468
+ selected_registry_idx = cli_utils.multi_choice_prompt(
2469
+ object_type=f"{cloud_provider.upper()} registries",
2470
+ choices=[[st] for st in available_registries],
2471
+ headers=["Container Registry"],
2472
+ prompt_text="Please choose one of the registries for the new container registry:",
2473
+ )
2474
+ if selected_registry_idx is None:
2475
+ cli_utils.error("No container registry selected.")
2476
+ selected_registry = available_registries[selected_registry_idx]
2477
+ config = {"uri": selected_registry}
2478
+ service_connector_resource_id = selected_registry
2479
+ else:
2480
+ raise ValueError(f"Unknown component type {component_type}")
2481
+
2482
+ return ComponentInfo(
2483
+ flavor=flavor,
2484
+ configuration=config,
2485
+ service_connector_index=service_connector_index,
2486
+ service_connector_resource_id=service_connector_resource_id,
2487
+ )