zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1240) hide show
  1. zenml/VERSION +1 -1
  2. zenml/__init__.py +9 -12
  3. zenml/actions/__init__.py +14 -0
  4. zenml/actions/base_action.py +682 -0
  5. zenml/actions/pipeline_run/__init__.py +0 -0
  6. zenml/actions/pipeline_run/pipeline_run_action.py +223 -0
  7. zenml/analytics/context.py +70 -15
  8. zenml/analytics/enums.py +21 -17
  9. zenml/analytics/models.py +1 -0
  10. zenml/analytics/utils.py +19 -7
  11. zenml/annotators/base_annotator.py +4 -3
  12. zenml/artifact_stores/base_artifact_store.py +155 -72
  13. zenml/artifact_stores/local_artifact_store.py +3 -2
  14. zenml/artifacts/artifact_config.py +67 -58
  15. zenml/artifacts/external_artifact.py +24 -39
  16. zenml/artifacts/external_artifact_config.py +27 -37
  17. zenml/artifacts/preexisting_data_materializer.py +90 -0
  18. zenml/artifacts/unmaterialized_artifact.py +2 -12
  19. zenml/artifacts/utils.py +521 -220
  20. zenml/cli/__init__.py +1511 -456
  21. zenml/cli/annotator.py +48 -14
  22. zenml/cli/artifact.py +22 -4
  23. zenml/cli/authorized_device.py +1 -0
  24. zenml/cli/base.py +185 -22
  25. zenml/cli/code_repository.py +1 -0
  26. zenml/cli/formatter.py +1 -1
  27. zenml/cli/integration.py +119 -24
  28. zenml/cli/login.py +1053 -0
  29. zenml/cli/model.py +42 -15
  30. zenml/cli/model_registry.py +1 -2
  31. zenml/cli/pipeline.py +168 -82
  32. zenml/cli/secret.py +1 -2
  33. zenml/cli/served_model.py +56 -26
  34. zenml/cli/server.py +486 -619
  35. zenml/cli/service_accounts.py +55 -14
  36. zenml/cli/service_connectors.py +97 -6
  37. zenml/cli/stack.py +821 -531
  38. zenml/cli/stack_components.py +15 -598
  39. zenml/cli/tag.py +1 -0
  40. zenml/cli/text_utils.py +36 -2
  41. zenml/cli/user_management.py +204 -6
  42. zenml/cli/utils.py +411 -290
  43. zenml/client.py +1742 -298
  44. zenml/client_lazy_loader.py +224 -0
  45. zenml/code_repositories/base_code_repository.py +5 -4
  46. zenml/code_repositories/git/local_git_repository_context.py +1 -0
  47. zenml/code_repositories/local_repository_context.py +1 -0
  48. zenml/config/__init__.py +2 -0
  49. zenml/config/base_settings.py +6 -6
  50. zenml/config/build_configuration.py +43 -17
  51. zenml/config/compiler.py +82 -49
  52. zenml/config/docker_settings.py +139 -83
  53. zenml/config/global_config.py +260 -234
  54. zenml/config/pipeline_configurations.py +32 -11
  55. zenml/config/pipeline_run_configuration.py +15 -3
  56. zenml/config/pipeline_spec.py +6 -6
  57. zenml/config/resource_settings.py +8 -9
  58. zenml/config/retry_config.py +27 -0
  59. zenml/config/schedule.py +27 -18
  60. zenml/config/secret_reference_mixin.py +8 -4
  61. zenml/config/secrets_store_config.py +16 -24
  62. zenml/config/server_config.py +434 -51
  63. zenml/config/settings_resolver.py +2 -1
  64. zenml/config/source.py +97 -31
  65. zenml/config/step_configurations.py +83 -39
  66. zenml/config/step_run_info.py +3 -0
  67. zenml/config/store_config.py +20 -54
  68. zenml/config/strict_base_model.py +2 -6
  69. zenml/console.py +0 -1
  70. zenml/constants.py +232 -74
  71. zenml/container_registries/azure_container_registry.py +1 -0
  72. zenml/container_registries/base_container_registry.py +7 -3
  73. zenml/container_registries/default_container_registry.py +4 -3
  74. zenml/container_registries/dockerhub_container_registry.py +1 -0
  75. zenml/container_registries/gcp_container_registry.py +1 -0
  76. zenml/container_registries/github_container_registry.py +2 -10
  77. zenml/data_validators/base_data_validator.py +2 -2
  78. zenml/entrypoints/base_entrypoint_configuration.py +76 -17
  79. zenml/entrypoints/pipeline_entrypoint_configuration.py +1 -0
  80. zenml/entrypoints/step_entrypoint_configuration.py +21 -2
  81. zenml/enums.py +91 -9
  82. zenml/environment.py +52 -319
  83. zenml/{steps/base_parameters.py → event_hub/__init__.py} +5 -7
  84. zenml/event_hub/base_event_hub.py +196 -0
  85. zenml/event_hub/event_hub.py +181 -0
  86. zenml/event_sources/__init__.py +14 -0
  87. zenml/{lineage_graph/edge.py → event_sources/base_event.py} +5 -7
  88. zenml/event_sources/base_event_source.py +695 -0
  89. zenml/event_sources/webhooks/__init__.py +14 -0
  90. zenml/event_sources/webhooks/base_webhook_event_source.py +231 -0
  91. zenml/exceptions.py +40 -41
  92. zenml/feature_stores/base_feature_store.py +4 -6
  93. zenml/hooks/hook_validators.py +3 -11
  94. zenml/image_builders/base_image_builder.py +5 -2
  95. zenml/image_builders/build_context.py +24 -80
  96. zenml/image_builders/local_image_builder.py +14 -6
  97. zenml/integrations/__init__.py +16 -4
  98. zenml/integrations/airflow/__init__.py +3 -5
  99. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +15 -7
  100. zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +15 -252
  101. zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
  102. zenml/integrations/argilla/__init__.py +46 -0
  103. zenml/integrations/argilla/annotators/__init__.py +20 -0
  104. zenml/integrations/argilla/annotators/argilla_annotator.py +443 -0
  105. zenml/{post_execution → integrations/argilla/flavors}/__init__.py +9 -13
  106. zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +150 -0
  107. zenml/integrations/aws/__init__.py +7 -3
  108. zenml/integrations/aws/container_registries/aws_container_registry.py +44 -8
  109. zenml/integrations/aws/flavors/__init__.py +6 -0
  110. zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
  111. zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
  112. zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +104 -16
  113. zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
  114. zenml/integrations/aws/image_builders/__init__.py +20 -0
  115. zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
  116. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +421 -133
  117. zenml/integrations/aws/service_connectors/aws_service_connector.py +175 -48
  118. zenml/integrations/aws/step_operators/sagemaker_step_operator.py +1 -1
  119. zenml/integrations/azure/__init__.py +14 -5
  120. zenml/integrations/azure/azureml_utils.py +201 -0
  121. zenml/integrations/azure/flavors/__init__.py +11 -0
  122. zenml/integrations/azure/flavors/azureml.py +139 -0
  123. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +166 -0
  124. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +71 -18
  125. zenml/integrations/azure/orchestrators/__init__.py +19 -0
  126. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +583 -0
  127. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +82 -0
  128. zenml/integrations/azure/service_connectors/azure_service_connector.py +15 -6
  129. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -173
  130. zenml/integrations/bentoml/__init__.py +1 -1
  131. zenml/integrations/bentoml/constants.py +1 -1
  132. zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
  133. zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +128 -239
  134. zenml/integrations/bentoml/services/__init__.py +15 -4
  135. zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
  136. zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +85 -43
  137. zenml/integrations/bentoml/services/deployment_type.py +23 -0
  138. zenml/integrations/bentoml/steps/bento_builder.py +2 -0
  139. zenml/integrations/bentoml/steps/bentoml_deployer.py +97 -47
  140. zenml/integrations/bitbucket/__init__.py +42 -0
  141. zenml/integrations/bitbucket/plugins/__init__.py +20 -0
  142. zenml/integrations/bitbucket/plugins/bitbucket_webhook_event_source_flavor.py +43 -0
  143. zenml/integrations/bitbucket/plugins/event_sources/__init__.py +0 -0
  144. zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +486 -0
  145. zenml/integrations/comet/__init__.py +49 -0
  146. zenml/integrations/{kserve/services → comet/experiment_trackers}/__init__.py +5 -6
  147. zenml/integrations/comet/experiment_trackers/comet_experiment_tracker.py +175 -0
  148. zenml/integrations/comet/flavors/__init__.py +24 -0
  149. zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +129 -0
  150. zenml/integrations/constants.py +14 -2
  151. zenml/integrations/databricks/__init__.py +70 -0
  152. zenml/{lineage_graph → integrations/databricks/flavors}/__init__.py +14 -18
  153. zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
  154. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +174 -0
  155. zenml/integrations/{kserve → databricks}/model_deployers/__init__.py +5 -5
  156. zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
  157. zenml/integrations/databricks/orchestrators/__init__.py +20 -0
  158. zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +501 -0
  159. zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
  160. zenml/integrations/databricks/services/__init__.py +19 -0
  161. zenml/integrations/databricks/services/databricks_deployment.py +407 -0
  162. zenml/integrations/{gcp/orchestrators/vertex_scheduler → databricks/utils}/__init__.py +2 -2
  163. zenml/integrations/databricks/utils/databricks_utils.py +87 -0
  164. zenml/integrations/deepchecks/__init__.py +30 -10
  165. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +59 -18
  166. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  167. zenml/integrations/deepchecks/materializers/deepchecks_results_materializer.py +3 -3
  168. zenml/integrations/deepchecks/validation_checks.py +62 -35
  169. zenml/integrations/discord/__init__.py +1 -0
  170. zenml/integrations/discord/steps/discord_alerter_ask_step.py +1 -0
  171. zenml/integrations/discord/steps/discord_alerter_post_step.py +1 -0
  172. zenml/integrations/evidently/__init__.py +22 -4
  173. zenml/integrations/evidently/column_mapping.py +11 -3
  174. zenml/integrations/evidently/data_validators/evidently_data_validator.py +24 -6
  175. zenml/integrations/evidently/metrics.py +7 -8
  176. zenml/integrations/evidently/tests.py +7 -8
  177. zenml/integrations/facets/__init__.py +22 -5
  178. zenml/integrations/facets/models.py +2 -7
  179. zenml/integrations/feast/__init__.py +19 -3
  180. zenml/integrations/feast/feature_stores/feast_feature_store.py +13 -32
  181. zenml/integrations/gcp/__init__.py +6 -6
  182. zenml/integrations/gcp/artifact_stores/gcp_artifact_store.py +6 -0
  183. zenml/integrations/gcp/flavors/gcp_artifact_store_flavor.py +1 -0
  184. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +24 -2
  185. zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +14 -1
  186. zenml/integrations/gcp/google_credentials_mixin.py +1 -1
  187. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +519 -296
  188. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +788 -113
  189. zenml/integrations/gcp/step_operators/vertex_step_operator.py +9 -1
  190. zenml/integrations/github/__init__.py +15 -0
  191. zenml/integrations/github/code_repositories/github_code_repository.py +3 -2
  192. zenml/integrations/github/plugins/__init__.py +19 -0
  193. zenml/integrations/github/plugins/event_sources/__init__.py +0 -0
  194. zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +510 -0
  195. zenml/integrations/github/plugins/github_webhook_event_source_flavor.py +43 -0
  196. zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +7 -1
  197. zenml/integrations/great_expectations/__init__.py +22 -10
  198. zenml/integrations/great_expectations/data_validators/ge_data_validator.py +61 -57
  199. zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
  200. zenml/integrations/great_expectations/ge_store_backend.py +24 -11
  201. zenml/integrations/great_expectations/materializers/ge_materializer.py +9 -9
  202. zenml/integrations/great_expectations/utils.py +5 -5
  203. zenml/integrations/huggingface/__init__.py +52 -1
  204. zenml/integrations/huggingface/flavors/__init__.py +26 -0
  205. zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +130 -0
  206. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  207. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +102 -19
  208. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +18 -19
  209. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +104 -0
  210. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +18 -19
  211. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +13 -15
  212. zenml/integrations/huggingface/model_deployers/__init__.py +20 -0
  213. zenml/integrations/huggingface/model_deployers/huggingface_model_deployer.py +247 -0
  214. zenml/integrations/huggingface/services/__init__.py +19 -0
  215. zenml/integrations/huggingface/services/huggingface_deployment.py +292 -0
  216. zenml/integrations/huggingface/steps/__init__.py +21 -0
  217. zenml/integrations/huggingface/steps/accelerate_runner.py +166 -0
  218. zenml/integrations/huggingface/steps/huggingface_deployer.py +110 -0
  219. zenml/integrations/hyperai/__init__.py +53 -0
  220. zenml/integrations/hyperai/flavors/__init__.py +20 -0
  221. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +170 -0
  222. zenml/integrations/hyperai/orchestrators/__init__.py +21 -0
  223. zenml/integrations/hyperai/orchestrators/hyperai_orchestrator.py +510 -0
  224. zenml/integrations/hyperai/service_connectors/__init__.py +20 -0
  225. zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +375 -0
  226. zenml/integrations/integration.py +101 -47
  227. zenml/integrations/kaniko/flavors/kaniko_image_builder_flavor.py +6 -44
  228. zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +9 -6
  229. zenml/integrations/kubeflow/__init__.py +4 -1
  230. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +67 -82
  231. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +306 -248
  232. zenml/integrations/kubernetes/__init__.py +6 -3
  233. zenml/integrations/kubernetes/flavors/__init__.py +8 -0
  234. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +43 -2
  235. zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
  236. zenml/integrations/kubernetes/orchestrators/kube_utils.py +67 -12
  237. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +89 -21
  238. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +39 -7
  239. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +42 -19
  240. zenml/integrations/kubernetes/pod_settings.py +21 -31
  241. zenml/integrations/kubernetes/serialization_utils.py +3 -3
  242. zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +14 -11
  243. zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
  244. zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +236 -0
  245. zenml/integrations/label_studio/__init__.py +1 -3
  246. zenml/integrations/label_studio/annotators/label_studio_annotator.py +46 -14
  247. zenml/integrations/label_studio/flavors/__init__.py +2 -0
  248. zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +20 -5
  249. zenml/integrations/langchain/__init__.py +7 -1
  250. zenml/integrations/langchain/materializers/document_materializer.py +44 -8
  251. zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
  252. zenml/integrations/lightgbm/__init__.py +1 -0
  253. zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
  254. zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
  255. zenml/integrations/lightning/__init__.py +48 -0
  256. zenml/integrations/lightning/flavors/__init__.py +23 -0
  257. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +168 -0
  258. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  259. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +617 -0
  260. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +303 -0
  261. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  262. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  263. zenml/integrations/mlflow/__init__.py +55 -8
  264. zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +31 -13
  265. zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +32 -37
  266. zenml/integrations/mlflow/model_deployers/mlflow_model_deployer.py +42 -233
  267. zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +42 -49
  268. zenml/integrations/mlflow/services/mlflow_deployment.py +30 -5
  269. zenml/integrations/mlflow/steps/mlflow_deployer.py +25 -27
  270. zenml/integrations/mlflow/steps/mlflow_registry.py +3 -1
  271. zenml/integrations/modal/__init__.py +46 -0
  272. zenml/integrations/modal/flavors/__init__.py +26 -0
  273. zenml/integrations/modal/flavors/modal_step_operator_flavor.py +125 -0
  274. zenml/integrations/modal/step_operators/__init__.py +22 -0
  275. zenml/integrations/modal/step_operators/modal_step_operator.py +242 -0
  276. zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
  277. zenml/integrations/neptune/experiment_trackers/run_state.py +71 -55
  278. zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
  279. zenml/integrations/neural_prophet/__init__.py +6 -1
  280. zenml/integrations/numpy/__init__.py +32 -0
  281. zenml/integrations/{kserve/constants.py → numpy/materializers/__init__.py} +5 -4
  282. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  283. zenml/integrations/openai/__init__.py +1 -1
  284. zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
  285. zenml/{steps/external_artifact.py → integrations/pandas/__init__.py} +17 -11
  286. zenml/integrations/{kserve/custom_deployer → pandas/materializers}/__init__.py +5 -5
  287. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  288. zenml/integrations/pigeon/__init__.py +44 -0
  289. zenml/integrations/pigeon/annotators/__init__.py +20 -0
  290. zenml/integrations/pigeon/annotators/pigeon_annotator.py +330 -0
  291. zenml/integrations/{kserve → pigeon}/flavors/__init__.py +7 -7
  292. zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +104 -0
  293. zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
  294. zenml/integrations/polars/__init__.py +1 -0
  295. zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
  296. zenml/integrations/prodigy/__init__.py +48 -0
  297. zenml/integrations/prodigy/annotators/__init__.py +20 -0
  298. zenml/integrations/prodigy/annotators/prodigy_annotator.py +275 -0
  299. zenml/integrations/prodigy/flavors/__init__.py +24 -0
  300. zenml/integrations/prodigy/flavors/prodigy_annotator_flavor.py +101 -0
  301. zenml/integrations/pycaret/__init__.py +6 -0
  302. zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
  303. zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
  304. zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
  305. zenml/integrations/registry.py +38 -1
  306. zenml/integrations/s3/__init__.py +2 -3
  307. zenml/integrations/s3/artifact_stores/s3_artifact_store.py +191 -9
  308. zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +22 -45
  309. zenml/integrations/s3/utils.py +39 -0
  310. zenml/integrations/seldon/__init__.py +18 -2
  311. zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +19 -141
  312. zenml/integrations/seldon/secret_schemas/secret_schemas.py +2 -2
  313. zenml/integrations/seldon/seldon_client.py +55 -70
  314. zenml/integrations/seldon/services/seldon_deployment.py +4 -5
  315. zenml/integrations/seldon/steps/seldon_deployer.py +21 -20
  316. zenml/integrations/sklearn/__init__.py +1 -1
  317. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +29 -7
  318. zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +47 -28
  319. zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
  320. zenml/integrations/skypilot_aws/__init__.py +3 -2
  321. zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
  322. zenml/integrations/skypilot_azure/__init__.py +2 -4
  323. zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
  324. zenml/integrations/skypilot_gcp/__init__.py +3 -2
  325. zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
  326. zenml/integrations/skypilot_kubernetes/__init__.py +52 -0
  327. zenml/integrations/skypilot_kubernetes/flavors/__init__.py +26 -0
  328. zenml/integrations/skypilot_kubernetes/flavors/skypilot_orchestrator_kubernetes_vm_flavor.py +125 -0
  329. zenml/integrations/skypilot_kubernetes/orchestrators/__init__.py +25 -0
  330. zenml/integrations/skypilot_kubernetes/orchestrators/skypilot_kubernetes_vm_orchestrator.py +74 -0
  331. zenml/integrations/skypilot_lambda/__init__.py +50 -0
  332. zenml/integrations/{kserve/secret_schemas → skypilot_lambda/flavors}/__init__.py +9 -12
  333. zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +130 -0
  334. zenml/{lineage_graph/node → integrations/skypilot_lambda/orchestrators}/__init__.py +8 -15
  335. zenml/integrations/skypilot_lambda/orchestrators/skypilot_lambda_vm_orchestrator.py +92 -0
  336. zenml/integrations/slack/__init__.py +1 -0
  337. zenml/integrations/slack/alerters/slack_alerter.py +22 -2
  338. zenml/integrations/slack/flavors/__init__.py +2 -0
  339. zenml/integrations/slack/flavors/slack_alerter_flavor.py +13 -4
  340. zenml/integrations/slack/steps/slack_alerter_ask_step.py +1 -0
  341. zenml/integrations/slack/steps/slack_alerter_post_step.py +1 -0
  342. zenml/integrations/spark/flavors/spark_step_operator_flavor.py +2 -39
  343. zenml/integrations/spark/step_operators/kubernetes_step_operator.py +1 -0
  344. zenml/integrations/spark/step_operators/spark_entrypoint_configuration.py +1 -0
  345. zenml/integrations/spark/step_operators/spark_step_operator.py +2 -0
  346. zenml/integrations/tekton/__init__.py +2 -1
  347. zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
  348. zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +549 -235
  349. zenml/integrations/tensorboard/__init__.py +1 -13
  350. zenml/integrations/tensorboard/services/tensorboard_service.py +3 -4
  351. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +66 -59
  352. zenml/integrations/tensorflow/__init__.py +10 -15
  353. zenml/integrations/tensorflow/materializers/keras_materializer.py +24 -27
  354. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +9 -16
  355. zenml/integrations/vllm/__init__.py +50 -0
  356. zenml/integrations/vllm/flavors/__init__.py +21 -0
  357. zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
  358. zenml/integrations/vllm/model_deployers/__init__.py +19 -0
  359. zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
  360. zenml/integrations/vllm/services/__init__.py +19 -0
  361. zenml/integrations/vllm/services/vllm_deployment.py +206 -0
  362. zenml/integrations/wandb/__init__.py +1 -0
  363. zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +21 -18
  364. zenml/integrations/whylogs/__init__.py +18 -2
  365. zenml/integrations/whylogs/data_validators/whylogs_data_validator.py +3 -3
  366. zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
  367. zenml/integrations/whylogs/materializers/whylogs_materializer.py +14 -21
  368. zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
  369. zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
  370. zenml/io/fileio.py +1 -0
  371. zenml/io/filesystem.py +2 -2
  372. zenml/io/local_filesystem.py +3 -3
  373. zenml/logger.py +41 -17
  374. zenml/logging/__init__.py +5 -0
  375. zenml/logging/step_logging.py +280 -32
  376. zenml/{new/steps → login}/__init__.py +4 -1
  377. zenml/login/credentials.py +370 -0
  378. zenml/login/credentials_store.py +638 -0
  379. zenml/{new/pipelines → login/pro}/__init__.py +4 -1
  380. zenml/login/pro/client.py +492 -0
  381. zenml/login/pro/constants.py +28 -0
  382. zenml/{zen_stores/schemas/identity_schemas.py → login/pro/models.py} +9 -11
  383. zenml/login/pro/organization/__init__.py +14 -0
  384. zenml/login/pro/organization/client.py +79 -0
  385. zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
  386. zenml/{_hub → login/pro/tenant}/__init__.py +2 -2
  387. zenml/login/pro/tenant/client.py +92 -0
  388. zenml/login/pro/tenant/models.py +176 -0
  389. zenml/login/pro/utils.py +107 -0
  390. zenml/login/server_info.py +52 -0
  391. zenml/{cli → login}/web_login.py +71 -21
  392. zenml/materializers/__init__.py +2 -4
  393. zenml/materializers/base_materializer.py +180 -51
  394. zenml/materializers/built_in_materializer.py +50 -23
  395. zenml/materializers/cloudpickle_materializer.py +4 -5
  396. zenml/materializers/numpy_materializer.py +23 -233
  397. zenml/materializers/pandas_materializer.py +22 -174
  398. zenml/materializers/pydantic_materializer.py +2 -2
  399. zenml/materializers/service_materializer.py +12 -10
  400. zenml/materializers/structured_string_materializer.py +12 -6
  401. zenml/materializers/uuid_materializer.py +79 -0
  402. zenml/metadata/lazy_load.py +33 -17
  403. zenml/metadata/metadata_types.py +112 -3
  404. zenml/model/lazy_load.py +85 -4
  405. zenml/model/model.py +236 -226
  406. zenml/model/utils.py +99 -141
  407. zenml/model_deployers/base_model_deployer.py +319 -47
  408. zenml/model_registries/base_model_registry.py +17 -15
  409. zenml/models/__init__.py +254 -175
  410. zenml/models/v2/base/base.py +254 -112
  411. zenml/models/v2/base/base_plugin_flavor.py +76 -0
  412. zenml/models/v2/base/filter.py +530 -199
  413. zenml/models/v2/base/page.py +2 -12
  414. zenml/models/v2/base/scoped.py +292 -22
  415. zenml/models/v2/core/action.py +276 -0
  416. zenml/models/v2/core/action_flavor.py +57 -0
  417. zenml/models/v2/core/api_key.py +35 -12
  418. zenml/models/v2/core/artifact.py +126 -6
  419. zenml/models/v2/core/artifact_version.py +289 -65
  420. zenml/models/v2/core/artifact_visualization.py +10 -4
  421. zenml/models/v2/core/code_reference.py +13 -4
  422. zenml/models/v2/core/code_repository.py +39 -16
  423. zenml/models/v2/core/component.py +113 -61
  424. zenml/models/v2/core/device.py +25 -5
  425. zenml/models/v2/core/event_source.py +244 -0
  426. zenml/models/v2/core/event_source_flavor.py +67 -0
  427. zenml/models/v2/core/flavor.py +90 -27
  428. zenml/models/v2/core/logs.py +81 -12
  429. zenml/models/v2/core/model.py +91 -42
  430. zenml/models/v2/core/model_version.py +100 -55
  431. zenml/models/v2/core/model_version_artifact.py +96 -89
  432. zenml/models/v2/core/model_version_pipeline_run.py +86 -53
  433. zenml/models/v2/core/pipeline.py +205 -80
  434. zenml/models/v2/core/pipeline_build.py +110 -21
  435. zenml/models/v2/core/pipeline_deployment.py +101 -36
  436. zenml/models/v2/core/pipeline_run.py +545 -33
  437. zenml/models/v2/core/run_metadata.py +23 -181
  438. zenml/models/v2/core/run_template.py +439 -0
  439. zenml/models/v2/core/schedule.py +66 -29
  440. zenml/models/v2/core/secret.py +33 -21
  441. zenml/models/v2/core/server_settings.py +224 -0
  442. zenml/models/v2/core/service.py +500 -0
  443. zenml/models/v2/core/service_account.py +40 -15
  444. zenml/models/v2/core/service_connector.py +247 -49
  445. zenml/models/v2/core/stack.py +163 -70
  446. zenml/models/v2/core/step_run.py +210 -48
  447. zenml/models/v2/core/tag.py +21 -8
  448. zenml/models/v2/core/tag_resource.py +13 -4
  449. zenml/models/v2/core/trigger.py +422 -0
  450. zenml/models/v2/core/trigger_execution.py +119 -0
  451. zenml/models/v2/core/user.py +136 -69
  452. zenml/models/v2/core/workspace.py +26 -7
  453. zenml/models/v2/misc/auth_models.py +11 -2
  454. zenml/models/v2/misc/build_item.py +3 -3
  455. zenml/models/v2/misc/external_user.py +3 -6
  456. zenml/models/v2/misc/info_models.py +78 -0
  457. zenml/models/v2/misc/loaded_visualization.py +2 -2
  458. zenml/models/v2/misc/run_metadata.py +38 -0
  459. zenml/models/v2/misc/server_models.py +100 -0
  460. zenml/models/v2/misc/service_connector_type.py +9 -17
  461. zenml/models/v2/misc/stack_deployment.py +96 -0
  462. zenml/models/v2/misc/user_auth.py +7 -9
  463. zenml/orchestrators/__init__.py +4 -0
  464. zenml/orchestrators/base_orchestrator.py +136 -25
  465. zenml/orchestrators/containerized_orchestrator.py +1 -0
  466. zenml/orchestrators/dag_runner.py +18 -3
  467. zenml/orchestrators/input_utils.py +109 -48
  468. zenml/orchestrators/local/local_orchestrator.py +10 -0
  469. zenml/orchestrators/local_docker/local_docker_orchestrator.py +14 -42
  470. zenml/orchestrators/output_utils.py +16 -6
  471. zenml/orchestrators/publish_utils.py +12 -5
  472. zenml/orchestrators/step_launcher.py +142 -194
  473. zenml/orchestrators/step_run_utils.py +386 -0
  474. zenml/orchestrators/step_runner.py +181 -270
  475. zenml/orchestrators/utils.py +219 -84
  476. zenml/orchestrators/wheeled_orchestrator.py +147 -0
  477. zenml/pipelines/__init__.py +3 -16
  478. zenml/{new/pipelines → pipelines}/build_utils.py +287 -47
  479. zenml/{new/pipelines → pipelines}/pipeline_context.py +6 -2
  480. zenml/pipelines/pipeline_decorator.py +40 -64
  481. zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +376 -440
  482. zenml/pipelines/run_utils.py +358 -0
  483. zenml/plugins/__init__.py +0 -0
  484. zenml/plugins/base_plugin_flavor.py +88 -0
  485. zenml/plugins/plugin_flavor_registry.py +342 -0
  486. zenml/secret/base_secret.py +7 -8
  487. zenml/secret/schemas/basic_auth_secret_schema.py +0 -1
  488. zenml/service_connectors/docker_service_connector.py +19 -4
  489. zenml/service_connectors/service_connector.py +12 -14
  490. zenml/service_connectors/service_connector_registry.py +71 -55
  491. zenml/service_connectors/service_connector_utils.py +418 -0
  492. zenml/services/__init__.py +0 -2
  493. zenml/services/container/container_service.py +9 -6
  494. zenml/services/container/container_service_endpoint.py +1 -1
  495. zenml/services/container/entrypoint.py +3 -2
  496. zenml/services/local/local_daemon_entrypoint.py +9 -6
  497. zenml/services/local/local_service.py +1 -1
  498. zenml/services/local/local_service_endpoint.py +1 -1
  499. zenml/services/service.py +222 -130
  500. zenml/services/service_status.py +2 -1
  501. zenml/services/service_type.py +6 -5
  502. zenml/stack/flavor.py +25 -18
  503. zenml/stack/flavor_registry.py +4 -4
  504. zenml/stack/stack.py +20 -131
  505. zenml/stack/stack_component.py +136 -110
  506. zenml/stack/utils.py +36 -15
  507. zenml/stack_deployments/__init__.py +14 -0
  508. zenml/stack_deployments/aws_stack_deployment.py +320 -0
  509. zenml/stack_deployments/azure_stack_deployment.py +315 -0
  510. zenml/stack_deployments/gcp_stack_deployment.py +315 -0
  511. zenml/stack_deployments/stack_deployment.py +232 -0
  512. zenml/stack_deployments/utils.py +48 -0
  513. zenml/step_operators/step_operator_entrypoint_configuration.py +2 -1
  514. zenml/steps/__init__.py +3 -9
  515. zenml/steps/base_step.py +172 -315
  516. zenml/{new/steps → steps}/decorated_step.py +1 -0
  517. zenml/steps/entrypoint_function_utils.py +33 -93
  518. zenml/{new/steps → steps}/step_context.py +70 -50
  519. zenml/steps/step_decorator.py +47 -93
  520. zenml/steps/step_invocation.py +22 -60
  521. zenml/steps/utils.py +161 -48
  522. zenml/types.py +14 -1
  523. zenml/utils/archivable.py +178 -0
  524. zenml/utils/callback_registry.py +71 -0
  525. zenml/utils/code_repository_utils.py +1 -0
  526. zenml/utils/code_utils.py +346 -0
  527. zenml/utils/cuda_utils.py +50 -0
  528. zenml/utils/dashboard_utils.py +67 -21
  529. zenml/utils/deprecation_utils.py +22 -24
  530. zenml/utils/dict_utils.py +22 -0
  531. zenml/utils/docker_utils.py +34 -5
  532. zenml/utils/downloaded_repository_context.py +1 -0
  533. zenml/utils/env_utils.py +55 -1
  534. zenml/utils/filesync_model.py +65 -28
  535. zenml/utils/function_utils.py +260 -0
  536. zenml/utils/integration_utils.py +1 -0
  537. zenml/utils/json_utils.py +131 -0
  538. zenml/utils/materializer_utils.py +1 -1
  539. zenml/utils/metadata_utils.py +368 -0
  540. zenml/utils/notebook_utils.py +136 -0
  541. zenml/utils/package_utils.py +89 -0
  542. zenml/utils/pagination_utils.py +9 -7
  543. zenml/utils/pipeline_docker_image_builder.py +152 -149
  544. zenml/utils/pydantic_utils.py +276 -66
  545. zenml/utils/requirements_utils.py +71 -0
  546. zenml/utils/secret_utils.py +66 -12
  547. zenml/utils/settings_utils.py +2 -1
  548. zenml/utils/singleton.py +15 -3
  549. zenml/utils/source_code_utils.py +1 -0
  550. zenml/utils/source_utils.py +236 -14
  551. zenml/utils/string_utils.py +140 -0
  552. zenml/utils/typed_model.py +5 -3
  553. zenml/utils/typing_utils.py +223 -0
  554. zenml/utils/visualization_utils.py +5 -3
  555. zenml/utils/yaml_utils.py +1 -1
  556. zenml/zen_server/auth.py +387 -55
  557. zenml/zen_server/cache.py +208 -0
  558. zenml/zen_server/cloud_utils.py +253 -0
  559. zenml/zen_server/csrf.py +91 -0
  560. zenml/zen_server/dashboard/assets/404-Dfq64Boz.js +1 -0
  561. zenml/zen_server/dashboard/assets/@radix-DeK6qiuw.js +85 -0
  562. zenml/zen_server/dashboard/assets/@react-router-B3Z5rLr2.js +29 -0
  563. zenml/zen_server/dashboard/assets/@reactflow-BUNIMFeC.js +17 -0
  564. zenml/zen_server/dashboard/assets/@reactflow-C26Olbza.css +1 -0
  565. zenml/zen_server/dashboard/assets/@tanstack-DT5WLu9C.js +22 -0
  566. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-B73Vs10T.js +1 -0
  567. zenml/zen_server/dashboard/assets/CodeSnippet-Bbx6fIb6.css +1 -0
  568. zenml/zen_server/dashboard/assets/CodeSnippet-DIJRT2NT.js +9 -0
  569. zenml/zen_server/dashboard/assets/CollapsibleCard-BzUHGZOU.js +1 -0
  570. zenml/zen_server/dashboard/assets/Commands-BEGyld4c.js +1 -0
  571. zenml/zen_server/dashboard/assets/ComponentBadge-xyKiek1s.js +1 -0
  572. zenml/zen_server/dashboard/assets/CopyButton-DhW-mapu.js +2 -0
  573. zenml/zen_server/dashboard/assets/CsvVizualization-D8oazBiE.js +15 -0
  574. zenml/zen_server/dashboard/assets/DeleteAlertDialog-WkSIIgfy.js +1 -0
  575. zenml/zen_server/dashboard/assets/DialogItem-Bgroeg29.js +1 -0
  576. zenml/zen_server/dashboard/assets/DisplayDate-CDMUcQHS.js +1 -0
  577. zenml/zen_server/dashboard/assets/EmptyState-BzdlCwp3.js +1 -0
  578. zenml/zen_server/dashboard/assets/Error-CY5tlu17.js +1 -0
  579. zenml/zen_server/dashboard/assets/ExecutionStatus-G8mjIaeA.js +1 -0
  580. zenml/zen_server/dashboard/assets/Helpbox-Bb1ed--O.js +1 -0
  581. zenml/zen_server/dashboard/assets/Infobox-Da6-76M2.js +1 -0
  582. zenml/zen_server/dashboard/assets/InlineAvatar-DqnZaBNq.js +1 -0
  583. zenml/zen_server/dashboard/assets/Lock-CYYy18Mm.js +1 -0
  584. zenml/zen_server/dashboard/assets/MarkdownVisualization-ylXaAxev.js +14 -0
  585. zenml/zen_server/dashboard/assets/NestedCollapsible-aK5ojKoF.js +1 -0
  586. zenml/zen_server/dashboard/assets/NumberBox-Dtp3J6g5.js +1 -0
  587. zenml/zen_server/dashboard/assets/Partials-CqZp5NMX.js +1 -0
  588. zenml/zen_server/dashboard/assets/PasswordChecker-B0nadgh6.js +1 -0
  589. zenml/zen_server/dashboard/assets/ProBadge-B4tRUYve.js +1 -0
  590. zenml/zen_server/dashboard/assets/ProCta-CZuP29Qz.js +1 -0
  591. zenml/zen_server/dashboard/assets/ProviderIcon-Bd7GUQ1_.js +1 -0
  592. zenml/zen_server/dashboard/assets/ProviderRadio-mstdqzsS.js +1 -0
  593. zenml/zen_server/dashboard/assets/RunSelector-CsruSB4i.js +1 -0
  594. zenml/zen_server/dashboard/assets/RunsBody-DxxtWVYz.js +1 -0
  595. zenml/zen_server/dashboard/assets/SearchField-D6tPxyqw.js +1 -0
  596. zenml/zen_server/dashboard/assets/SecretTooltip-CLzJIYW_.js +1 -0
  597. zenml/zen_server/dashboard/assets/SetPassword-Yn50ooBC.js +1 -0
  598. zenml/zen_server/dashboard/assets/StackList-U537qoYd.js +1 -0
  599. zenml/zen_server/dashboard/assets/Tabs-CNv-eTYM.js +1 -0
  600. zenml/zen_server/dashboard/assets/Tick-jEIevzVf.js +1 -0
  601. zenml/zen_server/dashboard/assets/UpdatePasswordSchemas-C16GW-kX.js +1 -0
  602. zenml/zen_server/dashboard/assets/UsageReason-Bf2tzhv1.js +1 -0
  603. zenml/zen_server/dashboard/assets/WizardFooter-D6i-AP1K.js +1 -0
  604. zenml/zen_server/dashboard/assets/acp-DOsXjFc7.webp +0 -0
  605. zenml/zen_server/dashboard/assets/adam-e-y0WnB_.webp +0 -0
  606. zenml/zen_server/dashboard/assets/alex-DcCuDHPg.webp +0 -0
  607. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-DUti43aF.js +1 -0
  608. zenml/zen_server/dashboard/assets/baris-C0ZrZ10g.webp +0 -0
  609. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  610. zenml/zen_server/dashboard/assets/check-circle-jNbX5-sR.js +1 -0
  611. zenml/zen_server/dashboard/assets/chevron-down-6JyMkfjR.js +1 -0
  612. zenml/zen_server/dashboard/assets/chevron-right-double-D7ojK9Co.js +1 -0
  613. zenml/zen_server/dashboard/assets/cloud-squares-DeRLMopf.svg +43 -0
  614. zenml/zen_server/dashboard/assets/code-browser-CUFUIHfp.js +1 -0
  615. zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
  616. zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
  617. zenml/zen_server/dashboard/assets/connectors-video-C9qY4syJ.svg +21 -0
  618. zenml/zen_server/dashboard/assets/copy-C8XQA2Ug.js +1 -0
  619. zenml/zen_server/dashboard/assets/create-stack-Ch2WPs9U.js +1 -0
  620. zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
  621. zenml/zen_server/dashboard/assets/delete-run-Byf9hTjA.js +1 -0
  622. zenml/zen_server/dashboard/assets/docker-BdA9vrnW.js +1 -0
  623. zenml/zen_server/dashboard/assets/dots-horizontal-otGBOSDJ.js +1 -0
  624. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  625. zenml/zen_server/dashboard/assets/form-schemas-BZqKBPBF.js +1 -0
  626. zenml/zen_server/dashboard/assets/gcp-CFtm4BA7.js +1 -0
  627. zenml/zen_server/dashboard/assets/hamza-NKKOZz1I.webp +0 -0
  628. zenml/zen_server/dashboard/assets/help-Cc9bBIJH.js +1 -0
  629. zenml/zen_server/dashboard/assets/index-CE0aQlv8.js +55 -0
  630. zenml/zen_server/dashboard/assets/index-CtdYkjUi.js +1 -0
  631. zenml/zen_server/dashboard/assets/index-CyBKZcpO.js +1 -0
  632. zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
  633. zenml/zen_server/dashboard/assets/index-Uu49AX48.js +1 -0
  634. zenml/zen_server/dashboard/assets/index-v6gQjDEo.js +1 -0
  635. zenml/zen_server/dashboard/assets/index.esm-Dy6Z9Ung.js +1 -0
  636. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
  637. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
  638. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
  639. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
  640. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
  641. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
  642. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
  643. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
  644. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
  645. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
  646. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
  647. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
  648. zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
  649. zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
  650. zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
  651. zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
  652. zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
  653. zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
  654. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
  655. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
  656. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
  657. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
  658. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
  659. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
  660. zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
  661. zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
  662. zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
  663. zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
  664. zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
  665. zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
  666. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
  667. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
  668. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
  669. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
  670. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
  671. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
  672. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
  673. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
  674. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
  675. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
  676. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
  677. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
  678. zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
  679. zenml/zen_server/dashboard/assets/kubernetes-B2wmAJ1d.js +1 -0
  680. zenml/zen_server/dashboard/assets/layout-BtHBmE4w.js +1 -0
  681. zenml/zen_server/dashboard/assets/link-external-b9AXw_sW.js +1 -0
  682. zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
  683. zenml/zen_server/dashboard/assets/login-mutation-DNDVp_2H.js +1 -0
  684. zenml/zen_server/dashboard/assets/logs-WMSM52RF.js +1 -0
  685. zenml/zen_server/dashboard/assets/mcp-Cb1aMeoq.webp +0 -0
  686. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  687. zenml/zen_server/dashboard/assets/not-found-Bmup4ctE.js +1 -0
  688. zenml/zen_server/dashboard/assets/package-C6uypY4h.js +1 -0
  689. zenml/zen_server/dashboard/assets/page--XLMzHrn.js +1 -0
  690. zenml/zen_server/dashboard/assets/page-ANYGfEUL.js +1 -0
  691. zenml/zen_server/dashboard/assets/page-B5Sr8pib.js +1 -0
  692. zenml/zen_server/dashboard/assets/page-BC27C_OI.js +2 -0
  693. zenml/zen_server/dashboard/assets/page-BNxYrN0q.js +1 -0
  694. zenml/zen_server/dashboard/assets/page-BYJfqgLN.js +1 -0
  695. zenml/zen_server/dashboard/assets/page-B_0XkV48.js +1 -0
  696. zenml/zen_server/dashboard/assets/page-BrmJp1Wt.js +1 -0
  697. zenml/zen_server/dashboard/assets/page-C2nU3Gxn.js +1 -0
  698. zenml/zen_server/dashboard/assets/page-C70wZtV2.js +1 -0
  699. zenml/zen_server/dashboard/assets/page-CHRn1fQm.js +1 -0
  700. zenml/zen_server/dashboard/assets/page-CWr96ZKN.js +1 -0
  701. zenml/zen_server/dashboard/assets/page-CXAbSyp9.js +1 -0
  702. zenml/zen_server/dashboard/assets/page-CaeI9ptC.js +1 -0
  703. zenml/zen_server/dashboard/assets/page-Cc8ZEuj4.js +1 -0
  704. zenml/zen_server/dashboard/assets/page-CltCNL0T.js +1 -0
  705. zenml/zen_server/dashboard/assets/page-CmlYj7Nl.js +1 -0
  706. zenml/zen_server/dashboard/assets/page-D6Ev5P8V.js +1 -0
  707. zenml/zen_server/dashboard/assets/page-D9Oh05fl.js +1 -0
  708. zenml/zen_server/dashboard/assets/page-DGlm1RVc.js +1 -0
  709. zenml/zen_server/dashboard/assets/page-DN4BVIOL.js +1 -0
  710. zenml/zen_server/dashboard/assets/page-Dif8CWyZ.js +1 -0
  711. zenml/zen_server/dashboard/assets/page-DlIi5ThM.js +1 -0
  712. zenml/zen_server/dashboard/assets/page-DoW7YxTu.js +1 -0
  713. zenml/zen_server/dashboard/assets/page-Dth9X1Ih.js +1 -0
  714. zenml/zen_server/dashboard/assets/page-DweqqCkF.js +1 -0
  715. zenml/zen_server/dashboard/assets/page-DyOJ_pq3.js +1 -0
  716. zenml/zen_server/dashboard/assets/page-Hn8q9iJZ.js +1 -0
  717. zenml/zen_server/dashboard/assets/page-IhckKFnD.js +6 -0
  718. zenml/zen_server/dashboard/assets/page-LyZ_l8vR.js +1 -0
  719. zenml/zen_server/dashboard/assets/page-PamGpk0j.js +1 -0
  720. zenml/zen_server/dashboard/assets/page-PxOWfKgF.js +2 -0
  721. zenml/zen_server/dashboard/assets/persist-DeXRG61d.js +1 -0
  722. zenml/zen_server/dashboard/assets/persist-vP0-Xl4f.js +1 -0
  723. zenml/zen_server/dashboard/assets/plus-tf1V2hTJ.js +1 -0
  724. zenml/zen_server/dashboard/assets/refresh-BjOeWlEq.js +1 -0
  725. zenml/zen_server/dashboard/assets/repos-video-D8kpu60k.svg +9 -0
  726. zenml/zen_server/dashboard/assets/rocket-DjT2cDvG.js +1 -0
  727. zenml/zen_server/dashboard/assets/service-DH_oUqQj.js +2 -0
  728. zenml/zen_server/dashboard/assets/settings_preview-0JLrRgHP.webp +0 -0
  729. zenml/zen_server/dashboard/assets/sharedSchema-Bw1_Wa7l.js +14 -0
  730. zenml/zen_server/dashboard/assets/stack-detail-query-B_0R_fd6.js +1 -0
  731. zenml/zen_server/dashboard/assets/stefan-B08Ftbba.webp +0 -0
  732. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  733. zenml/zen_server/dashboard/assets/tick-circle-BEX_Tp4v.js +1 -0
  734. zenml/zen_server/dashboard/assets/tour-cover-BYfeen6M.webp +0 -0
  735. zenml/zen_server/dashboard/assets/trash-arLUMWMS.js +1 -0
  736. zenml/zen_server/dashboard/assets/update-server-settings-mutation-D9qYhfaN.js +1 -0
  737. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  738. zenml/zen_server/dashboard/assets/url-Dh93fvh0.js +1 -0
  739. zenml/zen_server/dashboard/assets/zod-BwEbpOxH.js +1 -0
  740. zenml/zen_server/dashboard/index.html +19 -1
  741. zenml/zen_server/deploy/__init__.py +7 -16
  742. zenml/zen_server/deploy/base_provider.py +49 -78
  743. zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
  744. zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +48 -66
  745. zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +78 -62
  746. zenml/zen_server/deploy/deployer.py +94 -175
  747. zenml/zen_server/deploy/deployment.py +23 -17
  748. zenml/zen_server/deploy/docker/docker_provider.py +15 -31
  749. zenml/zen_server/deploy/docker/docker_zen_server.py +30 -35
  750. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  751. zenml/zen_server/deploy/helm/README.md +3 -13
  752. zenml/zen_server/deploy/helm/templates/NOTES.txt +23 -7
  753. zenml/zen_server/deploy/helm/templates/_environment.tpl +175 -23
  754. zenml/zen_server/deploy/helm/templates/server-db-job.yaml +45 -18
  755. zenml/zen_server/deploy/helm/templates/server-db-pvc.yaml +25 -0
  756. zenml/zen_server/deploy/helm/templates/server-deployment.yaml +22 -6
  757. zenml/zen_server/deploy/helm/templates/server-secret.yaml +11 -10
  758. zenml/zen_server/deploy/helm/values.yaml +210 -28
  759. zenml/zen_server/exceptions.py +20 -1
  760. zenml/zen_server/feature_gate/__init__.py +13 -0
  761. zenml/zen_server/feature_gate/endpoint_utils.py +61 -0
  762. zenml/zen_server/feature_gate/feature_gate_interface.py +49 -0
  763. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +125 -0
  764. zenml/zen_server/jwt.py +64 -32
  765. zenml/zen_server/rate_limit.py +200 -0
  766. zenml/zen_server/rbac/endpoint_utils.py +92 -9
  767. zenml/zen_server/rbac/models.py +21 -17
  768. zenml/zen_server/rbac/rbac_sql_zen_store.py +175 -0
  769. zenml/zen_server/rbac/utils.py +71 -30
  770. zenml/zen_server/rbac/zenml_cloud_rbac.py +13 -188
  771. zenml/zen_server/routers/actions_endpoints.py +324 -0
  772. zenml/zen_server/routers/artifact_version_endpoints.py +28 -2
  773. zenml/zen_server/routers/auth_endpoints.py +249 -131
  774. zenml/zen_server/routers/code_repositories_endpoints.py +1 -0
  775. zenml/zen_server/routers/devices_endpoints.py +56 -49
  776. zenml/zen_server/routers/event_source_endpoints.py +327 -0
  777. zenml/zen_server/routers/logs_endpoints.py +66 -0
  778. zenml/zen_server/routers/model_versions_endpoints.py +59 -0
  779. zenml/zen_server/routers/models_endpoints.py +7 -1
  780. zenml/zen_server/routers/pipeline_builds_endpoints.py +6 -1
  781. zenml/zen_server/routers/pipeline_deployments_endpoints.py +35 -0
  782. zenml/zen_server/routers/pipelines_endpoints.py +19 -32
  783. zenml/zen_server/routers/plugin_endpoints.py +107 -0
  784. zenml/zen_server/routers/run_templates_endpoints.py +212 -0
  785. zenml/zen_server/routers/runs_endpoints.py +91 -32
  786. zenml/zen_server/routers/schedule_endpoints.py +1 -0
  787. zenml/zen_server/routers/secrets_endpoints.py +4 -2
  788. zenml/zen_server/routers/server_endpoints.py +186 -4
  789. zenml/zen_server/routers/service_connectors_endpoints.py +56 -0
  790. zenml/zen_server/routers/service_endpoints.py +180 -0
  791. zenml/zen_server/routers/stack_components_endpoints.py +2 -1
  792. zenml/zen_server/routers/stack_deployment_endpoints.py +164 -0
  793. zenml/zen_server/routers/steps_endpoints.py +21 -12
  794. zenml/zen_server/routers/triggers_endpoints.py +336 -0
  795. zenml/zen_server/routers/users_endpoints.py +280 -45
  796. zenml/zen_server/routers/webhook_endpoints.py +127 -0
  797. zenml/zen_server/routers/workspaces_endpoints.py +220 -185
  798. zenml/zen_server/secure_headers.py +120 -0
  799. zenml/{new → zen_server/template_execution}/__init__.py +1 -1
  800. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +42 -0
  801. zenml/zen_server/template_execution/utils.py +474 -0
  802. zenml/zen_server/template_execution/workload_manager_interface.py +92 -0
  803. zenml/zen_server/utils.py +374 -74
  804. zenml/zen_server/zen_server_api.py +299 -52
  805. zenml/zen_stores/base_zen_store.py +90 -58
  806. zenml/zen_stores/migrations/alembic.py +22 -9
  807. zenml/zen_stores/migrations/env.py +2 -2
  808. zenml/zen_stores/migrations/utils.py +731 -0
  809. zenml/zen_stores/migrations/versions/0.21.0_release.py +0 -1
  810. zenml/zen_stores/migrations/versions/0.21.1_release.py +0 -1
  811. zenml/zen_stores/migrations/versions/0.22.0_release.py +0 -1
  812. zenml/zen_stores/migrations/versions/0.23.0_release.py +0 -1
  813. zenml/zen_stores/migrations/versions/0.30.0_release.py +0 -1
  814. zenml/zen_stores/migrations/versions/0.31.0_release.py +0 -1
  815. zenml/zen_stores/migrations/versions/0.31.1_release.py +0 -1
  816. zenml/zen_stores/migrations/versions/0.32.0_release.py +0 -1
  817. zenml/zen_stores/migrations/versions/0.32.1_release.py +0 -1
  818. zenml/zen_stores/migrations/versions/0.33.0_release.py +0 -1
  819. zenml/zen_stores/migrations/versions/0.34.0_release.py +0 -1
  820. zenml/zen_stores/migrations/versions/0.35.0_release.py +0 -1
  821. zenml/zen_stores/migrations/versions/0.35.1_release.py +0 -1
  822. zenml/zen_stores/migrations/versions/0.36.0_release.py +0 -1
  823. zenml/zen_stores/migrations/versions/0.36.1_release.py +0 -1
  824. zenml/zen_stores/migrations/versions/0.37.0_release.py +0 -1
  825. zenml/zen_stores/migrations/versions/0.38.0_release.py +0 -1
  826. zenml/zen_stores/migrations/versions/0.39.0_release.py +0 -1
  827. zenml/zen_stores/migrations/versions/0.39.1_release.py +0 -1
  828. zenml/zen_stores/migrations/versions/0.40.0_release.py +0 -1
  829. zenml/zen_stores/migrations/versions/0.40.1_release.py +0 -1
  830. zenml/zen_stores/migrations/versions/0.40.2_release.py +0 -1
  831. zenml/zen_stores/migrations/versions/0.40.3_release.py +0 -1
  832. zenml/zen_stores/migrations/versions/0.41.0_release.py +0 -1
  833. zenml/zen_stores/migrations/versions/0.42.0_release.py +0 -1
  834. zenml/zen_stores/migrations/versions/0.42.1_release.py +0 -1
  835. zenml/zen_stores/migrations/versions/0.43.0_release.py +0 -1
  836. zenml/zen_stores/migrations/versions/0.44.0_release.py +0 -1
  837. zenml/zen_stores/migrations/versions/0.44.1_release.py +0 -1
  838. zenml/zen_stores/migrations/versions/0.44.2_release.py +0 -1
  839. zenml/zen_stores/migrations/versions/0.44.3_release.py +0 -1
  840. zenml/zen_stores/migrations/versions/0.45.0_release.py +0 -1
  841. zenml/zen_stores/migrations/versions/0.45.1_release_0_45_1.py +0 -1
  842. zenml/zen_stores/migrations/versions/0.45.2_release.py +0 -1
  843. zenml/zen_stores/migrations/versions/0.45.3_release.py +0 -1
  844. zenml/zen_stores/migrations/versions/0.45.4_release.py +0 -1
  845. zenml/zen_stores/migrations/versions/0.45.5_release.py +0 -1
  846. zenml/zen_stores/migrations/versions/0.45.6_release.py +0 -1
  847. zenml/zen_stores/migrations/versions/0.46.0_release.py +0 -1
  848. zenml/zen_stores/migrations/versions/0.46.1_release.py +0 -1
  849. zenml/zen_stores/migrations/versions/0.47.0_release.py +0 -1
  850. zenml/zen_stores/migrations/versions/0.50.0_release.py +0 -1
  851. zenml/zen_stores/migrations/versions/0.51.0_release.py +0 -1
  852. zenml/zen_stores/migrations/versions/0.52.0_release.py +0 -1
  853. zenml/zen_stores/migrations/versions/0.53.0_release.py +0 -1
  854. zenml/zen_stores/migrations/versions/0.53.1_release.py +0 -1
  855. zenml/zen_stores/migrations/versions/0.54.0_release.py +0 -1
  856. zenml/zen_stores/migrations/versions/0.54.1_release.py +0 -1
  857. zenml/zen_stores/migrations/versions/0.55.0_release.py +0 -1
  858. zenml/zen_stores/migrations/versions/0.55.1_release.py +23 -0
  859. zenml/zen_stores/migrations/versions/0.55.2_release.py +23 -0
  860. zenml/zen_stores/migrations/versions/0.55.3_release.py +23 -0
  861. zenml/zen_stores/migrations/versions/0.55.4_release.py +23 -0
  862. zenml/zen_stores/migrations/versions/0.55.5_release.py +23 -0
  863. zenml/zen_stores/migrations/versions/0.56.0_release.py +23 -0
  864. zenml/zen_stores/migrations/versions/0.56.1_release.py +23 -0
  865. zenml/zen_stores/migrations/versions/0.56.2_release.py +23 -0
  866. zenml/zen_stores/migrations/versions/0.56.3_release.py +23 -0
  867. zenml/zen_stores/migrations/versions/0.56.4_release.py +23 -0
  868. zenml/zen_stores/migrations/versions/0.57.0.rc1_release.py +23 -0
  869. zenml/zen_stores/migrations/versions/0.57.0.rc2_release.py +23 -0
  870. zenml/zen_stores/migrations/versions/0.57.0_release.py +23 -0
  871. zenml/zen_stores/migrations/versions/0.57.1_release.py +23 -0
  872. zenml/zen_stores/migrations/versions/0.58.0_release.py +23 -0
  873. zenml/zen_stores/migrations/versions/0.58.1_release.py +23 -0
  874. zenml/zen_stores/migrations/versions/0.58.2_release.py +23 -0
  875. zenml/zen_stores/migrations/versions/0.60.0_release.py +23 -0
  876. zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
  877. zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
  878. zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
  879. zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
  880. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  881. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  882. zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
  883. zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
  884. zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
  885. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  886. zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
  887. zenml/zen_stores/migrations/versions/0.72.0_release.py +23 -0
  888. zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
  889. zenml/zen_stores/migrations/versions/03742aa7fdd7_add_secrets.py +1 -0
  890. zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +94 -0
  891. zenml/zen_stores/migrations/versions/0b06faa59c93_add_service_connectors.py +1 -0
  892. zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
  893. zenml/zen_stores/migrations/versions/0e4735b23577_increase_pipeline_spec_field_length.py +1 -0
  894. zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +6 -3
  895. zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +2 -1
  896. zenml/zen_stores/migrations/versions/14d687c8fa1c_rename_model_config_to_model_version.py +1 -0
  897. zenml/zen_stores/migrations/versions/19f27d5b234e_add_build_and_deployment_tables.py +1 -0
  898. zenml/zen_stores/migrations/versions/1a9a9d2a836d_admin_users.py +56 -0
  899. zenml/zen_stores/migrations/versions/1ac1b9c04da1_make_secrets_values_optional.py +1 -0
  900. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  901. zenml/zen_stores/migrations/versions/1d74e596abb8_add_run_once_start_time_to_schedule.py +36 -0
  902. zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
  903. zenml/zen_stores/migrations/versions/248dfd320b68_update_size_of_flavor_config_schema.py +1 -0
  904. zenml/zen_stores/migrations/versions/25155145c545_separate_actions_and_triggers.py +228 -0
  905. zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
  906. zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +9 -10
  907. zenml/zen_stores/migrations/versions/2d201872e23c_remove_db_dependency_loop.py +29 -0
  908. zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +4 -3
  909. zenml/zen_stores/migrations/versions/389046140cad_data_versioning.py +1 -0
  910. zenml/zen_stores/migrations/versions/3944116bbd56_rename_project_to_workspace.py +1 -0
  911. zenml/zen_stores/migrations/versions/3b68abe58f44_add_model_watchtower_entities.py +1 -0
  912. zenml/zen_stores/migrations/versions/3c5a367730c2_add_environment_info_to_runs.py +1 -0
  913. zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
  914. zenml/zen_stores/migrations/versions/43a86093b60e_add_labels_for_stack_components.py +1 -0
  915. zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +123 -0
  916. zenml/zen_stores/migrations/versions/479103df60b6_add_triggers.py +162 -0
  917. zenml/zen_stores/migrations/versions/4a3087070f4e_add_step_source_code.py +1 -0
  918. zenml/zen_stores/migrations/versions/4c41c0ca42db_add_code_repository_table.py +1 -0
  919. zenml/zen_stores/migrations/versions/4d688d8f7aff_rename_model_version_to_model.py +1 -0
  920. zenml/zen_stores/migrations/versions/4e1972485075_endpoint_artifact_deployment_artifact.py +1 -0
  921. zenml/zen_stores/migrations/versions/4f66af55fbb9_rename_model_config_model_to_model_.py +1 -0
  922. zenml/zen_stores/migrations/versions/5330ba58bf20_rename_tables_and_foreign_keys.py +8 -9
  923. zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +4 -2
  924. zenml/zen_stores/migrations/versions/5cc3f41cf048_add_save_models_to_registry.py +1 -0
  925. zenml/zen_stores/migrations/versions/6119cd9b93c2_tags_table.py +1 -0
  926. zenml/zen_stores/migrations/versions/623a234c11f5_add_sdk_docs_url_to_flavors.py +1 -0
  927. zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +5 -4
  928. zenml/zen_stores/migrations/versions/6a28c4fd0ef2_add_caching_info.py +1 -0
  929. zenml/zen_stores/migrations/versions/6f707b385dc1_fix_model_artifacts.py +1 -0
  930. zenml/zen_stores/migrations/versions/722392c91006_make_is_service_account_mandatory.py +1 -0
  931. zenml/zen_stores/migrations/versions/72675226b2de_unique_users.py +31 -0
  932. zenml/zen_stores/migrations/versions/72722dee4686_track_server_version.py +1 -0
  933. zenml/zen_stores/migrations/versions/7280c14811d6_use_text_type.py +1 -0
  934. zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +4 -2
  935. zenml/zen_stores/migrations/versions/729263e47b55_fix_external_input_artifacts.py +1 -0
  936. zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +3 -2
  937. zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +4 -2
  938. zenml/zen_stores/migrations/versions/76a7b9451ccd_add_build_template_deployment_id.py +52 -0
  939. zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +9 -7
  940. zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +7 -7
  941. zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
  942. zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +3 -2
  943. zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +2 -1
  944. zenml/zen_stores/migrations/versions/86fa52918b54_remove_teams_and_roles.py +1 -0
  945. zenml/zen_stores/migrations/versions/8a64fbfecda0_add_num_outputs_to_run_step.py +1 -0
  946. zenml/zen_stores/migrations/versions/8ed03137cacc_polymorthic_run_metadata.py +1 -0
  947. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  948. zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
  949. zenml/zen_stores/migrations/versions/93cbda80a732_add_service_accounts.py +1 -0
  950. zenml/zen_stores/migrations/versions/979eff8fc4b1_add_code_repo_description_and_logo_url.py +1 -0
  951. zenml/zen_stores/migrations/versions/9971237fa937_artifact_visualizations.py +1 -0
  952. zenml/zen_stores/migrations/versions/9d8020441014_increase_step_configuration_length.py +1 -0
  953. zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
  954. zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +3 -2
  955. zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +5 -4
  956. zenml/zen_stores/migrations/versions/ade72effebaf_added_logs_table.py +1 -0
  957. zenml/zen_stores/migrations/versions/alembic_start.py +2 -1
  958. zenml/zen_stores/migrations/versions/b4eccf34dfa3_add_hub_token_to_user_model.py +1 -0
  959. zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
  960. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  961. zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
  962. zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
  963. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  964. zenml/zen_stores/migrations/versions/c1b18cec3a48_increase_length_on_flavor_config_schema.py +1 -0
  965. zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
  966. zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
  967. zenml/zen_stores/migrations/versions/cc9894cb58aa_add_user_metadata.py +41 -0
  968. zenml/zen_stores/migrations/versions/ccd68b7825ae_add_status_to_pipeline_and_step_run.py +1 -0
  969. zenml/zen_stores/migrations/versions/d02b3d3464cf_add_orchestrator_run_id_column.py +1 -0
  970. zenml/zen_stores/migrations/versions/d26471b6fe8f_update_build_filtering.py +1 -0
  971. zenml/zen_stores/migrations/versions/d7b3acf9aa46_create_schedule_table.py +1 -0
  972. zenml/zen_stores/migrations/versions/e1d66d91a099_add_stack_and_component_spec_paths_to_.py +1 -0
  973. zenml/zen_stores/migrations/versions/e5225281b4d3_add_connector_skew_tolerance.py +1 -0
  974. zenml/zen_stores/migrations/versions/e65aa6708ff7_pipeline_versioning.py +1 -0
  975. zenml/zen_stores/migrations/versions/ec0d785ca296_create_run_metadata_table.py +1 -0
  976. zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +119 -0
  977. zenml/zen_stores/migrations/versions/f3b3964e3a0f_add_oauth_devices.py +1 -0
  978. zenml/zen_stores/migrations/versions/f49904a80aa7_increase_length_of_artifact_table_sources.py +1 -0
  979. zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +5 -4
  980. zenml/zen_stores/rest_zen_store.py +1326 -305
  981. zenml/zen_stores/schemas/__init__.py +22 -3
  982. zenml/zen_stores/schemas/action_schemas.py +192 -0
  983. zenml/zen_stores/schemas/api_key_schemas.py +23 -10
  984. zenml/zen_stores/schemas/artifact_schemas.py +112 -49
  985. zenml/zen_stores/schemas/artifact_visualization_schemas.py +17 -8
  986. zenml/zen_stores/schemas/base_schemas.py +27 -0
  987. zenml/zen_stores/schemas/code_repository_schemas.py +25 -10
  988. zenml/zen_stores/schemas/component_schemas.py +74 -11
  989. zenml/zen_stores/schemas/constants.py +16 -0
  990. zenml/zen_stores/schemas/device_schemas.py +29 -15
  991. zenml/zen_stores/schemas/event_source_schemas.py +188 -0
  992. zenml/zen_stores/schemas/flavor_schemas.py +19 -9
  993. zenml/zen_stores/schemas/logs_schemas.py +12 -6
  994. zenml/zen_stores/schemas/model_schemas.py +192 -139
  995. zenml/zen_stores/schemas/pipeline_build_schemas.py +16 -16
  996. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +58 -17
  997. zenml/zen_stores/schemas/pipeline_run_schemas.py +170 -35
  998. zenml/zen_stores/schemas/pipeline_schemas.py +51 -33
  999. zenml/zen_stores/schemas/run_metadata_schemas.py +28 -78
  1000. zenml/zen_stores/schemas/run_template_schemas.py +267 -0
  1001. zenml/zen_stores/schemas/schedule_schema.py +15 -5
  1002. zenml/zen_stores/schemas/secret_schemas.py +18 -10
  1003. zenml/zen_stores/schemas/server_settings_schemas.py +129 -0
  1004. zenml/zen_stores/schemas/service_connector_schemas.py +13 -6
  1005. zenml/zen_stores/schemas/service_schemas.py +258 -0
  1006. zenml/zen_stores/schemas/stack_schemas.py +23 -6
  1007. zenml/zen_stores/schemas/step_run_schemas.py +132 -41
  1008. zenml/zen_stores/schemas/tag_schemas.py +31 -50
  1009. zenml/zen_stores/schemas/trigger_schemas.py +316 -0
  1010. zenml/zen_stores/schemas/user_schemas.py +66 -23
  1011. zenml/zen_stores/schemas/utils.py +112 -0
  1012. zenml/zen_stores/schemas/workspace_schemas.py +36 -19
  1013. zenml/zen_stores/secrets_stores/aws_secrets_store.py +41 -32
  1014. zenml/zen_stores/secrets_stores/azure_secrets_store.py +20 -23
  1015. zenml/zen_stores/secrets_stores/base_secrets_store.py +80 -12
  1016. zenml/zen_stores/secrets_stores/gcp_secrets_store.py +42 -33
  1017. zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +7 -11
  1018. zenml/zen_stores/secrets_stores/secrets_store_interface.py +1 -0
  1019. zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +15 -8
  1020. zenml/zen_stores/secrets_stores/sql_secrets_store.py +8 -9
  1021. zenml/zen_stores/sql_zen_store.py +4062 -799
  1022. zenml/zen_stores/template_utils.py +263 -0
  1023. zenml/zen_stores/zen_store_interface.py +614 -44
  1024. zenml_nightly-0.72.0.dev20250116.dist-info/METADATA +486 -0
  1025. zenml_nightly-0.72.0.dev20250116.dist-info/RECORD +1294 -0
  1026. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/WHEEL +1 -1
  1027. CLA.md +0 -110
  1028. CODE-OF-CONDUCT.md +0 -132
  1029. CONTRIBUTING.md +0 -260
  1030. README.md +0 -304
  1031. RELEASE_NOTES.md +0 -3919
  1032. ROADMAP.md +0 -5
  1033. SECURITY.md +0 -15
  1034. zenml/_hub/client.py +0 -285
  1035. zenml/_hub/constants.py +0 -21
  1036. zenml/_hub/utils.py +0 -80
  1037. zenml/api.py +0 -61
  1038. zenml/cli/hub.py +0 -1115
  1039. zenml/cli/stack_recipes.py +0 -469
  1040. zenml/integrations/gcp/google_cloud_function.py +0 -187
  1041. zenml/integrations/gcp/google_cloud_scheduler.py +0 -83
  1042. zenml/integrations/gcp/orchestrators/vertex_scheduler/main.py +0 -91
  1043. zenml/integrations/gcp/orchestrators/vertex_scheduler/requirements.txt +0 -2
  1044. zenml/integrations/kserve/__init__.py +0 -57
  1045. zenml/integrations/kserve/custom_deployer/zenml_custom_model.py +0 -175
  1046. zenml/integrations/kserve/flavors/kserve_model_deployer_flavor.py +0 -137
  1047. zenml/integrations/kserve/model_deployers/kserve_model_deployer.py +0 -1003
  1048. zenml/integrations/kserve/secret_schemas/secret_schemas.py +0 -65
  1049. zenml/integrations/kserve/services/kserve_deployment.py +0 -596
  1050. zenml/integrations/kserve/steps/__init__.py +0 -22
  1051. zenml/integrations/kserve/steps/kserve_deployer.py +0 -472
  1052. zenml/integrations/kserve/steps/kserve_step_utils.py +0 -293
  1053. zenml/integrations/kubeflow/utils.py +0 -96
  1054. zenml/lineage_graph/lineage_graph.py +0 -244
  1055. zenml/lineage_graph/node/artifact_node.py +0 -52
  1056. zenml/lineage_graph/node/step_node.py +0 -41
  1057. zenml/models/v2/base/internal.py +0 -37
  1058. zenml/models/v2/base/update.py +0 -40
  1059. zenml/models/v2/misc/hub_plugin_models.py +0 -79
  1060. zenml/new/pipelines/deserialization_utils.py +0 -291
  1061. zenml/new/pipelines/model_utils.py +0 -72
  1062. zenml/new/pipelines/pipeline_decorator.py +0 -109
  1063. zenml/new/steps/step_decorator.py +0 -160
  1064. zenml/pipelines/base_pipeline.py +0 -274
  1065. zenml/post_execution/pipeline.py +0 -58
  1066. zenml/post_execution/pipeline_run.py +0 -55
  1067. zenml/services/service_registry.py +0 -214
  1068. zenml/services/terraform/__init__.py +0 -14
  1069. zenml/services/terraform/terraform_service.py +0 -441
  1070. zenml/steps/step_environment.py +0 -108
  1071. zenml/steps/step_output.py +0 -36
  1072. zenml/utils/mlstacks_utils.py +0 -635
  1073. zenml/utils/terraform_utils.py +0 -42
  1074. zenml/zen_server/dashboard/_redirects +0 -1
  1075. zenml/zen_server/dashboard/asset-manifest.json +0 -131
  1076. zenml/zen_server/dashboard/manifest.json +0 -25
  1077. zenml/zen_server/dashboard/precache-manifest.c139638dcc4d9d3425353266447a2fad.js +0 -462
  1078. zenml/zen_server/dashboard/robots.txt +0 -2
  1079. zenml/zen_server/dashboard/service-worker.js +0 -39
  1080. zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css +0 -16
  1081. zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css.map +0 -1
  1082. zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css +0 -2
  1083. zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css.map +0 -1
  1084. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js +0 -3
  1085. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.LICENSE.txt +0 -95
  1086. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.map +0 -1
  1087. zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js +0 -2
  1088. zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js.map +0 -1
  1089. zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js +0 -2
  1090. zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js.map +0 -1
  1091. zenml/zen_server/dashboard/static/media/AlertTriangle.28aee535.svg +0 -5
  1092. zenml/zen_server/dashboard/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
  1093. zenml/zen_server/dashboard/static/media/Back.86c23a22.svg +0 -4
  1094. zenml/zen_server/dashboard/static/media/BookOpen.5cb101ff.svg +0 -4
  1095. zenml/zen_server/dashboard/static/media/BoundingBox.1eb98717.svg +0 -10
  1096. zenml/zen_server/dashboard/static/media/Burger.9b1c67d7.svg +0 -3
  1097. zenml/zen_server/dashboard/static/media/Cached.2381fb8d.svg +0 -1
  1098. zenml/zen_server/dashboard/static/media/Calendar.356e11c7.svg +0 -3
  1099. zenml/zen_server/dashboard/static/media/ChartBarHorizontal.0247447b.svg +0 -6
  1100. zenml/zen_server/dashboard/static/media/ChartLine.0d79e18d.svg +0 -4
  1101. zenml/zen_server/dashboard/static/media/ChatDots.2e1c9211.svg +0 -6
  1102. zenml/zen_server/dashboard/static/media/Check.dad6beb2.svg +0 -3
  1103. zenml/zen_server/dashboard/static/media/CheckCircleFilled.c19566d0.svg +0 -3
  1104. zenml/zen_server/dashboard/static/media/Checkbox.af50e31e.svg +0 -3
  1105. zenml/zen_server/dashboard/static/media/ChevronDown.f860ce32.svg +0 -3
  1106. zenml/zen_server/dashboard/static/media/ChevronDownLight.6642d756.svg +0 -3
  1107. zenml/zen_server/dashboard/static/media/ChevronLeft.f6edfcdb.svg +0 -3
  1108. zenml/zen_server/dashboard/static/media/CircleCheck.f98fd6ca.svg +0 -1
  1109. zenml/zen_server/dashboard/static/media/Clock.ffc9de95.svg +0 -3
  1110. zenml/zen_server/dashboard/static/media/Close.74e9efbc.svg +0 -5
  1111. zenml/zen_server/dashboard/static/media/CloseWithBorder.6960930a.svg +0 -3
  1112. zenml/zen_server/dashboard/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
  1113. zenml/zen_server/dashboard/static/media/CloudArrowUp.0aecb235.svg +0 -6
  1114. zenml/zen_server/dashboard/static/media/Code.ef0f33b5.svg +0 -3
  1115. zenml/zen_server/dashboard/static/media/Config.0be63f8a.svg +0 -1
  1116. zenml/zen_server/dashboard/static/media/Connector.9fd46ef1.svg +0 -10
  1117. zenml/zen_server/dashboard/static/media/Copy.36e2112a.svg +0 -1
  1118. zenml/zen_server/dashboard/static/media/Dashboard.d05787e0.svg +0 -3
  1119. zenml/zen_server/dashboard/static/media/Data.b1c3b5f8.svg +0 -3
  1120. zenml/zen_server/dashboard/static/media/Delete.3c361b28.svg +0 -8
  1121. zenml/zen_server/dashboard/static/media/Docs.7541d478.svg +0 -7
  1122. zenml/zen_server/dashboard/static/media/Download.fba04d87.svg +0 -5
  1123. zenml/zen_server/dashboard/static/media/Edit.490eb294.svg +0 -6
  1124. zenml/zen_server/dashboard/static/media/EmptyRightArrow.23749d01.svg +0 -3
  1125. zenml/zen_server/dashboard/static/media/Example.6396cd37.svg +0 -5
  1126. zenml/zen_server/dashboard/static/media/Extension.1394cd4a.svg +0 -3
  1127. zenml/zen_server/dashboard/static/media/Eye.d9e4ee62.svg +0 -4
  1128. zenml/zen_server/dashboard/static/media/Failed.0213c1a0.svg +0 -1
  1129. zenml/zen_server/dashboard/static/media/FileText.1f15bacd.svg +0 -7
  1130. zenml/zen_server/dashboard/static/media/Filter.ab6b9c0d.svg +0 -3
  1131. zenml/zen_server/dashboard/static/media/Folders.12b29887.svg +0 -5
  1132. zenml/zen_server/dashboard/static/media/FunnelFill.6df4c143.svg +0 -3
  1133. zenml/zen_server/dashboard/static/media/GitCommit.7dd9c2aa.svg +0 -5
  1134. zenml/zen_server/dashboard/static/media/GitHub_Logo.cefc2023.png +0 -0
  1135. zenml/zen_server/dashboard/static/media/Graph.2c63a892.svg +0 -11
  1136. zenml/zen_server/dashboard/static/media/History.08329240.svg +0 -3
  1137. zenml/zen_server/dashboard/static/media/Home.0843b0d5.svg +0 -3
  1138. zenml/zen_server/dashboard/static/media/ImageBuilder.ea762d9c.svg +0 -6
  1139. zenml/zen_server/dashboard/static/media/InProgress.304a0edc.svg +0 -1
  1140. zenml/zen_server/dashboard/static/media/Info.9fe10c5c.svg +0 -3
  1141. zenml/zen_server/dashboard/static/media/KeyboardReturn.491afbe3.svg +0 -3
  1142. zenml/zen_server/dashboard/static/media/Link.72bbb55d.svg +0 -4
  1143. zenml/zen_server/dashboard/static/media/Lock.30f5e1fe.svg +0 -5
  1144. zenml/zen_server/dashboard/static/media/Lock2.a769ea52.svg +0 -3
  1145. zenml/zen_server/dashboard/static/media/LockKey.92f21621.svg +0 -6
  1146. zenml/zen_server/dashboard/static/media/Logs.8bf4d005.svg +0 -5
  1147. zenml/zen_server/dashboard/static/media/MinusCircle.4188f418.svg +0 -4
  1148. zenml/zen_server/dashboard/static/media/ModelRegistry.f0de050a.svg +0 -6
  1149. zenml/zen_server/dashboard/static/media/MultiUser.a2ba7c67.svg +0 -10
  1150. zenml/zen_server/dashboard/static/media/PaginationFirst.92628634.svg +0 -4
  1151. zenml/zen_server/dashboard/static/media/PaginationLast.00d3c732.svg +0 -4
  1152. zenml/zen_server/dashboard/static/media/PaginationNext.86158845.svg +0 -3
  1153. zenml/zen_server/dashboard/static/media/PaginationPrev.60c18a88.svg +0 -3
  1154. zenml/zen_server/dashboard/static/media/Pen.f2d831d4.svg +0 -6
  1155. zenml/zen_server/dashboard/static/media/PhotoCamera.179d6d4c.svg +0 -3
  1156. zenml/zen_server/dashboard/static/media/Pipeline.30d298b0.svg +0 -7
  1157. zenml/zen_server/dashboard/static/media/Plus.5aa1c16b.svg +0 -3
  1158. zenml/zen_server/dashboard/static/media/PlusCircle.92d860dd.svg +0 -5
  1159. zenml/zen_server/dashboard/static/media/Repositories.71a36b8c.svg +0 -3
  1160. zenml/zen_server/dashboard/static/media/RightArrow.f30d3871.svg +0 -29
  1161. zenml/zen_server/dashboard/static/media/Rocket.63bf7b9d.svg +0 -3
  1162. zenml/zen_server/dashboard/static/media/RocketLaunch.1bff2b59.svg +0 -6
  1163. zenml/zen_server/dashboard/static/media/Rubik-Medium.c87313aa.ttf +0 -0
  1164. zenml/zen_server/dashboard/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
  1165. zenml/zen_server/dashboard/static/media/Run.daec4fb2.svg +0 -6
  1166. zenml/zen_server/dashboard/static/media/Search.d1afcce5.svg +0 -4
  1167. zenml/zen_server/dashboard/static/media/Settings.59ca73ae.svg +0 -4
  1168. zenml/zen_server/dashboard/static/media/Share2.46c3ff66.svg +0 -3
  1169. zenml/zen_server/dashboard/static/media/SignOut.6aa718c5.svg +0 -3
  1170. zenml/zen_server/dashboard/static/media/SimplePlus.5cf7ec20.svg +0 -3
  1171. zenml/zen_server/dashboard/static/media/SingleUser.bef3a095.svg +0 -4
  1172. zenml/zen_server/dashboard/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
  1173. zenml/zen_server/dashboard/static/media/Stack.19b604ac.svg +0 -5
  1174. zenml/zen_server/dashboard/static/media/StackComponent.b1ba90b5.svg +0 -4
  1175. zenml/zen_server/dashboard/static/media/Star.f0c25022.svg +0 -9
  1176. zenml/zen_server/dashboard/static/media/StarOutline.94ca8cd9.svg +0 -3
  1177. zenml/zen_server/dashboard/static/media/Storefront.4b4796fe.svg +0 -3
  1178. zenml/zen_server/dashboard/static/media/Stream.543e3039.svg +0 -3
  1179. zenml/zen_server/dashboard/static/media/SupportAgent.510ddf1f.svg +0 -8
  1180. zenml/zen_server/dashboard/static/media/Table.77033750.svg +0 -6
  1181. zenml/zen_server/dashboard/static/media/Tool.d5785486.svg +0 -3
  1182. zenml/zen_server/dashboard/static/media/UserPlus.741a99d7.svg +0 -6
  1183. zenml/zen_server/dashboard/static/media/Verified.0625b2a0.svg +0 -3
  1184. zenml/zen_server/dashboard/static/media/addNew.4fb6c939.svg +0 -8
  1185. zenml/zen_server/dashboard/static/media/arrowClose.cbd53f3f.svg +0 -3
  1186. zenml/zen_server/dashboard/static/media/arrowOpen.6ceef0af.svg +0 -3
  1187. zenml/zen_server/dashboard/static/media/check_small.30bc0138.svg +0 -3
  1188. zenml/zen_server/dashboard/static/media/circleArrowSideClose.98d6013e.svg +0 -18
  1189. zenml/zen_server/dashboard/static/media/circleArrowSideOpen.63653df6.svg +0 -18
  1190. zenml/zen_server/dashboard/static/media/image.104fd14b.png +0 -0
  1191. zenml/zen_server/dashboard/static/media/imageAddIcon.e83004a9.svg +0 -7
  1192. zenml/zen_server/dashboard/static/media/logo.93333e5c.svg +0 -1
  1193. zenml/zen_server/dashboard/static/media/logo_small.4204397d.svg +0 -3
  1194. zenml/zen_server/dashboard/static/media/logo_white.d4b4414e.svg +0 -20
  1195. zenml/zen_server/dashboard/static/media/notConnected.5e2c8ea7.svg +0 -8
  1196. zenml/zen_server/dashboard/static/media/plugin-fallback.72c294e6.svg +0 -6
  1197. zenml/zen_server/dashboard/static/media/share.bcd998b0.svg +0 -5
  1198. zenml/zen_server/dashboard/static/media/stars.08a9b19a.svg +0 -8
  1199. zenml/zen_server/deploy/terraform/__init__.py +0 -41
  1200. zenml/zen_server/deploy/terraform/providers/__init__.py +0 -14
  1201. zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
  1202. zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
  1203. zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
  1204. zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -332
  1205. zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
  1206. zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
  1207. zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
  1208. zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -14
  1209. zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
  1210. zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
  1211. zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
  1212. zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -179
  1213. zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
  1214. zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -111
  1215. zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
  1216. zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
  1217. zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
  1218. zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
  1219. zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -14
  1220. zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
  1221. zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
  1222. zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
  1223. zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
  1224. zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -188
  1225. zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -111
  1226. zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
  1227. zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
  1228. zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
  1229. zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -14
  1230. zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
  1231. zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
  1232. zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
  1233. zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -183
  1234. zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -122
  1235. zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -255
  1236. zenml/zen_server/routers/run_metadata_endpoints.py +0 -97
  1237. zenml_nightly-0.55.0.dev20240124.dist-info/METADATA +0 -438
  1238. zenml_nightly-0.55.0.dev20240124.dist-info/RECORD +0 -1072
  1239. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/LICENSE +0 -0
  1240. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/entry_points.txt +0 -0
zenml/cli/stack.py CHANGED
@@ -12,57 +12,71 @@
12
12
  # or implied. See the License for the specific language governing
13
13
  # permissions and limitations under the License.
14
14
  """CLI for manipulating ZenML local and global config file."""
15
+
15
16
  import getpass
16
- import os
17
- from pathlib import Path
18
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
17
+ import re
18
+ import time
19
+ import webbrowser
20
+ from datetime import datetime
21
+ from typing import (
22
+ TYPE_CHECKING,
23
+ Any,
24
+ Dict,
25
+ List,
26
+ Optional,
27
+ Set,
28
+ Union,
29
+ )
19
30
  from uuid import UUID
20
31
 
21
32
  import click
33
+ from rich.console import Console
34
+ from rich.markdown import Markdown
35
+ from rich.prompt import Confirm
36
+ from rich.style import Style
37
+ from rich.syntax import Syntax
22
38
 
23
39
  import zenml
24
40
  from zenml.analytics.enums import AnalyticsEvent
25
41
  from zenml.analytics.utils import track_handler
26
42
  from zenml.cli import utils as cli_utils
27
43
  from zenml.cli.cli import TagGroup, cli
44
+ from zenml.cli.text_utils import OldSchoolMarkdownHeading
28
45
  from zenml.cli.utils import (
29
46
  _component_display_name,
30
- confirmation,
31
- declare,
32
- error,
33
47
  is_sorted_or_filtered,
34
48
  list_options,
35
49
  print_model_url,
36
50
  print_page_info,
37
51
  print_stacks_table,
38
- verify_mlstacks_prerequisites_installation,
39
52
  )
40
53
  from zenml.client import Client
41
54
  from zenml.console import console
42
- from zenml.constants import (
43
- ALPHA_MESSAGE,
44
- MLSTACKS_SUPPORTED_STACK_COMPONENTS,
45
- STACK_RECIPE_MODULAR_RECIPES,
55
+ from zenml.enums import (
56
+ CliCategories,
57
+ StackComponentType,
58
+ StackDeploymentProvider,
46
59
  )
47
- from zenml.enums import CliCategories, StackComponentType
48
60
  from zenml.exceptions import (
49
61
  IllegalOperationError,
50
- ProvisioningError,
51
62
  )
52
- from zenml.io.fileio import rmtree
53
63
  from zenml.logger import get_logger
54
- from zenml.models import StackFilter
55
- from zenml.utils.dashboard_utils import get_stack_url
56
- from zenml.utils.io_utils import create_dir_recursive_if_not_exists
57
- from zenml.utils.mlstacks_utils import (
58
- convert_click_params_to_mlstacks_primitives,
59
- convert_mlstacks_primitives_to_dicts,
60
- deploy_mlstacks_stack,
61
- get_stack_spec_file_path,
62
- stack_exists,
63
- stack_spec_exists,
64
- verify_spec_and_tf_files_exist,
65
- )
64
+ from zenml.models import (
65
+ ComponentInfo,
66
+ ServiceConnectorInfo,
67
+ ServiceConnectorResourcesInfo,
68
+ StackFilter,
69
+ StackRequest,
70
+ )
71
+ from zenml.models.v2.core.service_connector import (
72
+ ServiceConnectorRequest,
73
+ ServiceConnectorResponse,
74
+ )
75
+ from zenml.service_connectors.service_connector_utils import (
76
+ get_resources_options_from_resource_model_for_full_stack,
77
+ )
78
+ from zenml.utils import requirements_utils
79
+ from zenml.utils.dashboard_utils import get_component_url, get_stack_url
66
80
  from zenml.utils.yaml_utils import read_yaml, write_yaml
67
81
 
68
82
  if TYPE_CHECKING:
@@ -92,7 +106,7 @@ def stack() -> None:
92
106
  "artifact_store",
93
107
  help="Name of the artifact store for this stack.",
94
108
  type=str,
95
- required=True,
109
+ required=False,
96
110
  )
97
111
  @click.option(
98
112
  "-o",
@@ -100,7 +114,7 @@ def stack() -> None:
100
114
  "orchestrator",
101
115
  help="Name of the orchestrator for this stack.",
102
116
  type=str,
103
- required=True,
117
+ required=False,
104
118
  )
105
119
  @click.option(
106
120
  "-c",
@@ -189,10 +203,24 @@ def stack() -> None:
189
203
  help="Immediately set this stack as active.",
190
204
  type=click.BOOL,
191
205
  )
206
+ @click.option(
207
+ "-p",
208
+ "--provider",
209
+ help="Name of the cloud provider for this stack.",
210
+ type=click.Choice(["aws", "azure", "gcp"]),
211
+ required=False,
212
+ )
213
+ @click.option(
214
+ "-sc",
215
+ "--connector",
216
+ help="Name of the service connector for this stack.",
217
+ type=str,
218
+ required=False,
219
+ )
192
220
  def register_stack(
193
221
  stack_name: str,
194
- artifact_store: str,
195
- orchestrator: str,
222
+ artifact_store: Optional[str] = None,
223
+ orchestrator: Optional[str] = None,
196
224
  container_registry: Optional[str] = None,
197
225
  model_registry: Optional[str] = None,
198
226
  step_operator: Optional[str] = None,
@@ -204,6 +232,8 @@ def register_stack(
204
232
  data_validator: Optional[str] = None,
205
233
  image_builder: Optional[str] = None,
206
234
  set_stack: bool = False,
235
+ provider: Optional[str] = None,
236
+ connector: Optional[str] = None,
207
237
  ) -> None:
208
238
  """Register a stack.
209
239
 
@@ -222,44 +252,266 @@ def register_stack(
222
252
  data_validator: Name of the data validator for this stack.
223
253
  image_builder: Name of the new image builder for this stack.
224
254
  set_stack: Immediately set this stack as active.
255
+ provider: Name of the cloud provider for this stack.
256
+ connector: Name of the service connector for this stack.
225
257
  """
226
- with console.status(f"Registering stack '{stack_name}'...\n"):
227
- client = Client()
258
+ if (provider is None and connector is None) and (
259
+ artifact_store is None or orchestrator is None
260
+ ):
261
+ cli_utils.error(
262
+ "The only way to register a stack without specifying an "
263
+ "orchestrator and an artifact store is by using either a provider"
264
+ "(-p/--provider) or an existing service connector "
265
+ "(-sc/--connector). Please specify the artifact store and "
266
+ "the orchestrator or the service connector or cloud type settings."
267
+ )
228
268
 
229
- components: Dict[StackComponentType, Union[str, UUID]] = dict()
269
+ client = Client()
230
270
 
231
- components[StackComponentType.ARTIFACT_STORE] = artifact_store
232
- components[StackComponentType.ORCHESTRATOR] = orchestrator
271
+ if provider is not None or connector is not None:
272
+ if client.zen_store.is_local_store():
273
+ cli_utils.error(
274
+ "You are registering a stack using a service connector, but "
275
+ "this feature cannot be used with a local ZenML deployment. "
276
+ "ZenML needs to be accessible from the cloud provider to allow "
277
+ "the stack and its components to be registered automatically. "
278
+ "Please deploy ZenML in a remote environment as described in "
279
+ "the documentation: https://docs.zenml.io/getting-started/deploying-zenml "
280
+ "or use a managed ZenML Pro server instance for quick access "
281
+ "to this feature and more: https://www.zenml.io/pro"
282
+ )
233
283
 
234
- if alerter:
235
- components[StackComponentType.ALERTER] = alerter
236
- if annotator:
237
- components[StackComponentType.ANNOTATOR] = annotator
238
- if data_validator:
239
- components[StackComponentType.DATA_VALIDATOR] = data_validator
240
- if feature_store:
241
- components[StackComponentType.FEATURE_STORE] = feature_store
242
- if image_builder:
243
- components[StackComponentType.IMAGE_BUILDER] = image_builder
244
- if model_deployer:
245
- components[StackComponentType.MODEL_DEPLOYER] = model_deployer
246
- if model_registry:
247
- components[StackComponentType.MODEL_REGISTRY] = model_registry
248
- if step_operator:
249
- components[StackComponentType.STEP_OPERATOR] = step_operator
250
- if experiment_tracker:
251
- components[
252
- StackComponentType.EXPERIMENT_TRACKER
253
- ] = experiment_tracker
254
- if container_registry:
255
- components[
256
- StackComponentType.CONTAINER_REGISTRY
257
- ] = container_registry
284
+ try:
285
+ client.get_stack(
286
+ name_id_or_prefix=stack_name,
287
+ allow_name_prefix_match=False,
288
+ )
289
+ cli_utils.error(
290
+ f"A stack with name `{stack_name}` already exists, "
291
+ "please use a different name."
292
+ )
293
+ except KeyError:
294
+ pass
258
295
 
296
+ labels: Dict[str, str] = {}
297
+ components: Dict[StackComponentType, List[Union[UUID, ComponentInfo]]] = {}
298
+
299
+ # Cloud Flow
300
+ created_objects: Set[str] = set()
301
+ service_connector: Optional[Union[UUID, ServiceConnectorInfo]] = None
302
+ if provider is not None and connector is None:
303
+ service_connector_response = None
304
+ use_auto_configure = False
259
305
  try:
260
- created_stack = client.create_stack(
306
+ service_connector_response, _ = client.create_service_connector(
261
307
  name=stack_name,
262
- components=components,
308
+ connector_type=provider,
309
+ register=False,
310
+ auto_configure=True,
311
+ verify=False,
312
+ )
313
+ except NotImplementedError:
314
+ cli_utils.warning(
315
+ f"The {provider.upper()} service connector libraries are not "
316
+ "installed properly. Please run `zenml integration install "
317
+ f"{provider}` and try again to enable auto-discovery of the "
318
+ "connection configuration."
319
+ )
320
+ except Exception:
321
+ pass
322
+
323
+ if service_connector_response:
324
+ use_auto_configure = Confirm.ask(
325
+ f"[bold]{provider.upper()} cloud service connector[/bold] "
326
+ "has detected connection credentials in your environment.\n"
327
+ "Would you like to use these credentials or create a new "
328
+ "configuration by providing connection details?",
329
+ default=True,
330
+ show_choices=True,
331
+ show_default=True,
332
+ )
333
+
334
+ connector_selected: Optional[int] = None
335
+ if not use_auto_configure:
336
+ service_connector_response = None
337
+ existing_connectors = client.list_service_connectors(
338
+ connector_type=provider, size=100
339
+ )
340
+ if existing_connectors.total:
341
+ connector_selected = cli_utils.multi_choice_prompt(
342
+ object_type=f"{provider.upper()} service connectors",
343
+ choices=[
344
+ [connector.name]
345
+ for connector in existing_connectors.items
346
+ ],
347
+ headers=["Name"],
348
+ prompt_text=f"We found these {provider.upper()} service "
349
+ "connectors. Do you want to create a new one or use one "
350
+ "of the existing ones?",
351
+ default_choice="0",
352
+ allow_zero_be_a_new_object=True,
353
+ )
354
+ if use_auto_configure or connector_selected is None:
355
+ service_connector = _get_service_connector_info(
356
+ cloud_provider=provider,
357
+ connector_details=service_connector_response,
358
+ )
359
+ created_objects.add("service_connector")
360
+ else:
361
+ selected_connector = existing_connectors.items[connector_selected]
362
+ service_connector = selected_connector.id
363
+ connector = selected_connector.name
364
+ if isinstance(selected_connector.connector_type, str):
365
+ provider = selected_connector.connector_type
366
+ else:
367
+ provider = selected_connector.connector_type.connector_type
368
+ elif connector is not None:
369
+ service_connector_response = client.get_service_connector(connector)
370
+ service_connector = service_connector_response.id
371
+ if provider:
372
+ if service_connector_response.type != provider:
373
+ cli_utils.warning(
374
+ f"The service connector `{connector}` is not of type `{provider}`."
375
+ )
376
+ else:
377
+ provider = service_connector_response.type
378
+
379
+ if service_connector:
380
+ labels["zenml:wizard"] = "true"
381
+ if provider:
382
+ labels["zenml:provider"] = provider
383
+ resources_info = None
384
+ # explore the service connector
385
+ with console.status(
386
+ "Exploring resources available to the service connector...\n"
387
+ ):
388
+ resources_info = (
389
+ get_resources_options_from_resource_model_for_full_stack(
390
+ connector_details=service_connector
391
+ )
392
+ )
393
+ if resources_info is None:
394
+ cli_utils.error(
395
+ f"Failed to fetch service connector resources information for {service_connector}..."
396
+ )
397
+
398
+ # create components
399
+ needed_components = (
400
+ (StackComponentType.ARTIFACT_STORE, artifact_store),
401
+ (StackComponentType.ORCHESTRATOR, orchestrator),
402
+ (StackComponentType.CONTAINER_REGISTRY, container_registry),
403
+ )
404
+ for component_type, preset_name in needed_components:
405
+ component_info: Optional[Union[UUID, ComponentInfo]] = None
406
+ if preset_name is not None:
407
+ component_response = client.get_stack_component(
408
+ component_type, preset_name
409
+ )
410
+ component_info = component_response.id
411
+ component_name = component_response.name
412
+ else:
413
+ if isinstance(service_connector, UUID):
414
+ # find existing components under same connector
415
+ if (
416
+ component_type
417
+ in resources_info.components_resources_info
418
+ ):
419
+ existing_components = [
420
+ existing_response
421
+ for res_info in resources_info.components_resources_info[
422
+ component_type
423
+ ]
424
+ for existing_response in res_info.connected_through_service_connector
425
+ ]
426
+
427
+ # if some existing components are found - prompt user what to do
428
+ component_selected: Optional[int] = None
429
+ component_selected = cli_utils.multi_choice_prompt(
430
+ object_type=component_type.value.replace("_", " "),
431
+ choices=[
432
+ [
433
+ component.flavor_name,
434
+ component.name,
435
+ component.configuration or "",
436
+ component.connector_resource_id,
437
+ ]
438
+ for component in existing_components
439
+ ],
440
+ headers=[
441
+ "Type",
442
+ "Name",
443
+ "Configuration",
444
+ "Connected as",
445
+ ],
446
+ prompt_text=f"We found these {component_type.value.replace('_', ' ')} "
447
+ "connected using the current service connector. Do you "
448
+ "want to create a new one or use existing one?",
449
+ default_choice="0",
450
+ allow_zero_be_a_new_object=True,
451
+ )
452
+ else:
453
+ component_selected = None
454
+
455
+ if component_selected is None:
456
+ component_info = _get_stack_component_info(
457
+ component_type=component_type.value,
458
+ cloud_provider=provider
459
+ or resources_info.connector_type,
460
+ resources_info=resources_info,
461
+ service_connector_index=0,
462
+ )
463
+ component_name = stack_name
464
+ created_objects.add(component_type.value)
465
+ else:
466
+ selected_component = existing_components[
467
+ component_selected
468
+ ]
469
+ component_info = selected_component.id
470
+ component_name = selected_component.name
471
+
472
+ components[component_type] = [component_info]
473
+ if component_type == StackComponentType.ARTIFACT_STORE:
474
+ artifact_store = component_name
475
+ if component_type == StackComponentType.ORCHESTRATOR:
476
+ orchestrator = component_name
477
+ if component_type == StackComponentType.CONTAINER_REGISTRY:
478
+ container_registry = component_name
479
+
480
+ # normal flow once all components are defined
481
+ with console.status(f"Registering stack '{stack_name}'...\n"):
482
+ for component_type_, component_name_ in [
483
+ (StackComponentType.ARTIFACT_STORE, artifact_store),
484
+ (StackComponentType.ORCHESTRATOR, orchestrator),
485
+ (StackComponentType.ALERTER, alerter),
486
+ (StackComponentType.ANNOTATOR, annotator),
487
+ (StackComponentType.DATA_VALIDATOR, data_validator),
488
+ (StackComponentType.FEATURE_STORE, feature_store),
489
+ (StackComponentType.IMAGE_BUILDER, image_builder),
490
+ (StackComponentType.MODEL_DEPLOYER, model_deployer),
491
+ (StackComponentType.MODEL_REGISTRY, model_registry),
492
+ (StackComponentType.STEP_OPERATOR, step_operator),
493
+ (StackComponentType.EXPERIMENT_TRACKER, experiment_tracker),
494
+ (StackComponentType.CONTAINER_REGISTRY, container_registry),
495
+ ]:
496
+ if component_name_ and component_type_ not in components:
497
+ components[component_type_] = [
498
+ client.get_stack_component(
499
+ component_type_, component_name_
500
+ ).id
501
+ ]
502
+
503
+ try:
504
+ created_stack = client.zen_store.create_stack(
505
+ stack=StackRequest(
506
+ user=client.active_user.id,
507
+ workspace=client.active_workspace.id,
508
+ name=stack_name,
509
+ components=components,
510
+ service_connectors=[service_connector]
511
+ if service_connector
512
+ else [],
513
+ labels=labels,
514
+ )
263
515
  )
264
516
  except (KeyError, IllegalOperationError) as err:
265
517
  cli_utils.error(str(err))
@@ -267,6 +519,10 @@ def register_stack(
267
519
  cli_utils.declare(
268
520
  f"Stack '{created_stack.name}' successfully registered!"
269
521
  )
522
+ cli_utils.print_stack_configuration(
523
+ stack=created_stack,
524
+ active=created_stack.id == client.active_stack_model.id,
525
+ )
270
526
 
271
527
  if set_stack:
272
528
  client.activate_stack(created_stack.id)
@@ -276,6 +532,30 @@ def register_stack(
276
532
  f"Active {scope} stack set to:'{created_stack.name}'"
277
533
  )
278
534
 
535
+ delete_commands = []
536
+ if "service_connector" in created_objects:
537
+ created_objects.remove("service_connector")
538
+ connectors = set()
539
+ for each in created_objects:
540
+ if comps_ := created_stack.components[StackComponentType(each)]:
541
+ if conn_ := comps_[0].connector:
542
+ connectors.add(conn_.name)
543
+ for connector in connectors:
544
+ delete_commands.append(
545
+ "zenml service-connector delete " + connector
546
+ )
547
+ for each in created_objects:
548
+ if comps_ := created_stack.components[StackComponentType(each)]:
549
+ delete_commands.append(
550
+ f"zenml {each.replace('_', '-')} delete {comps_[0].name}"
551
+ )
552
+ delete_commands.append("zenml stack delete -y " + created_stack.name)
553
+
554
+ Console().print(
555
+ "To delete the objects created by this command run, please run in a sequence:\n"
556
+ )
557
+ Console().print(Syntax("\n".join(delete_commands[::-1]), "bash"))
558
+
279
559
  print_model_url(get_stack_url(created_stack))
280
560
 
281
561
 
@@ -488,7 +768,7 @@ def update_stack(
488
768
  "-r",
489
769
  "--model_registry",
490
770
  "model_registry_flag",
491
- help="Include this to remove the the model registry from this stack.",
771
+ help="Include this to remove the model registry from this stack.",
492
772
  is_flag=True,
493
773
  required=False,
494
774
  )
@@ -689,21 +969,11 @@ def list_stacks(ctx: click.Context, **kwargs: Any) -> None:
689
969
  type=click.STRING,
690
970
  required=False,
691
971
  )
692
- @click.option(
693
- "--outputs",
694
- "-o",
695
- is_flag=True,
696
- default=False,
697
- help="Include the outputs from mlstacks deployments.",
698
- )
699
- def describe_stack(
700
- stack_name_or_id: Optional[str] = None, outputs: bool = False
701
- ) -> None:
972
+ def describe_stack(stack_name_or_id: Optional[str] = None) -> None:
702
973
  """Show details about a named stack or the active stack.
703
974
 
704
975
  Args:
705
976
  stack_name_or_id: Name of the stack to describe.
706
- outputs: Include the outputs from mlstacks deployments.
707
977
  """
708
978
  client = Client()
709
979
 
@@ -719,8 +989,6 @@ def describe_stack(
719
989
  stack=stack_,
720
990
  active=stack_.id == client.active_stack_model.id,
721
991
  )
722
- if outputs:
723
- cli_utils.print_stack_outputs(stack_)
724
992
 
725
993
  print_model_url(get_stack_url(stack_))
726
994
 
@@ -805,8 +1073,7 @@ def set_active_stack_command(stack_name_or_id: str) -> None:
805
1073
  cli_utils.error(str(err))
806
1074
 
807
1075
  cli_utils.declare(
808
- f"Active {scope} stack set to: "
809
- f"'{client.active_stack_model.name}'"
1076
+ f"Active {scope} stack set to: '{client.active_stack_model.name}'"
810
1077
  )
811
1078
 
812
1079
 
@@ -825,51 +1092,6 @@ def get_active_stack() -> None:
825
1092
  cli_utils.error(str(err))
826
1093
 
827
1094
 
828
- @stack.command("up")
829
- def up_stack() -> None:
830
- """Provisions resources for the active stack."""
831
- stack_ = Client().active_stack
832
-
833
- cli_utils.declare(
834
- f"Provisioning resources for active stack '{stack_.name}'."
835
- )
836
- try:
837
- stack_.provision()
838
- stack_.resume()
839
- except ProvisioningError as e:
840
- cli_utils.error(str(e))
841
-
842
-
843
- @stack.command(
844
- "down", help="Suspends resources of the active stack deployment."
845
- )
846
- @click.option(
847
- "--force",
848
- "-f",
849
- "force",
850
- is_flag=True,
851
- help="Deprovisions local resources instead of suspending them.",
852
- )
853
- def down_stack(force: bool = False) -> None:
854
- """Suspends resources of the active stack deployment.
855
-
856
- Args:
857
- force: Deprovisions local resources instead of suspending them.
858
- """
859
- stack_ = Client().active_stack
860
-
861
- if force:
862
- cli_utils.declare(
863
- f"Deprovisioning resources for active stack '{stack_.name}'."
864
- )
865
- stack_.deprovision()
866
- else:
867
- cli_utils.declare(
868
- f"Suspending resources for active stack '{stack_.name}'."
869
- )
870
- stack_.suspend()
871
-
872
-
873
1095
  @stack.command("export", help="Exports a stack to a YAML file.")
874
1096
  @click.argument("stack_name_or_id", type=str, required=False)
875
1097
  @click.argument("filename", type=str, required=False)
@@ -906,14 +1128,12 @@ def export_stack(
906
1128
  def _import_stack_component(
907
1129
  component_type: StackComponentType,
908
1130
  component_dict: Dict[str, Any],
909
- component_spec_path: Optional[str] = None,
910
1131
  ) -> UUID:
911
1132
  """Import a single stack component with given type/config.
912
1133
 
913
1134
  Args:
914
1135
  component_type: The type of component to import.
915
1136
  component_dict: Dict representation of the component to import.
916
- component_spec_path: Path to the component spec file.
917
1137
 
918
1138
  Returns:
919
1139
  The ID of the imported component.
@@ -949,7 +1169,6 @@ def _import_stack_component(
949
1169
  component_type=component_type,
950
1170
  flavor=flavor,
951
1171
  configuration=config,
952
- component_spec_path=component_spec_path,
953
1172
  )
954
1173
  return component.id
955
1174
 
@@ -1186,483 +1405,256 @@ def register_secrets(
1186
1405
  client.update_secret(secret_name, add_or_update_values=secret_values)
1187
1406
 
1188
1407
 
1189
- def _get_deployment_params_interactively(
1190
- click_params: Dict[str, Any],
1191
- ) -> Dict[str, Any]:
1192
- """Get deployment values from command line arguments.
1408
+ def validate_name(ctx: click.Context, param: str, value: str) -> str:
1409
+ """Validate the name of the stack.
1193
1410
 
1194
1411
  Args:
1195
- click_params: Required and pre-existing values.
1412
+ ctx: The click context.
1413
+ param: The parameter name.
1414
+ value: The value of the parameter.
1196
1415
 
1197
1416
  Returns:
1198
- Full deployment arguments.
1417
+ The validated value.
1418
+
1419
+ Raises:
1420
+ BadParameter: If the name is invalid.
1199
1421
  """
1200
- deployment_values = {
1201
- "provider": click_params["provider"],
1202
- "stack_name": click_params["stack_name"],
1203
- "region": click_params["region"],
1204
- }
1205
- for component_type in MLSTACKS_SUPPORTED_STACK_COMPONENTS:
1206
- verify_mlstacks_prerequisites_installation()
1207
- from mlstacks.constants import ALLOWED_FLAVORS
1208
-
1209
- if (
1210
- click.prompt(
1211
- f"Enable {component_type}?",
1212
- type=click.Choice(["y", "n"]),
1213
- default="n",
1214
- )
1215
- == "y"
1216
- ):
1217
- component_flavor = click.prompt(
1218
- f" Enter {component_type} flavor",
1219
- type=click.Choice(ALLOWED_FLAVORS[component_type]),
1220
- )
1221
- deployment_values[component_type] = component_flavor
1422
+ if not value:
1423
+ return value
1222
1424
 
1223
- if (
1224
- click.prompt(
1225
- "Deploy using debug_mode?",
1226
- type=click.Choice(["y", "n"]),
1227
- default="n",
1228
- )
1229
- == "y"
1230
- ):
1231
- deployment_values["debug_mode"] = True
1232
-
1233
- extra_config = []
1234
- # use click.prompt to populate extra_config until someone just hits enter
1235
- while True:
1236
- declare(
1237
- "\nAdd to extra_config for stack deployment -->\n",
1238
- bold=True,
1239
- )
1240
- key = click.prompt(
1241
- "Enter `extra_config` key or hit enter to skip",
1242
- type=str,
1243
- default="",
1425
+ if not re.match(r"^[a-zA-Z0-9-]*$", value):
1426
+ raise click.BadParameter(
1427
+ "Stack name must contain only alphanumeric characters and hyphens."
1244
1428
  )
1245
- if key == "":
1246
- break
1247
- value = click.prompt(
1248
- f"Enter value for '{key}'",
1249
- type=str,
1250
- )
1251
- extra_config.append(f"{key}={value}")
1252
1429
 
1253
- # get mandatory GCP project_id if provider is GCP
1254
- # skip if project_id already specified in extra_config
1255
- if click_params["provider"] == "gcp" and not any(
1256
- s.startswith("project_id=") for s in extra_config
1257
- ):
1258
- project_id = click.prompt("What is your GCP project_id?", type=str)
1259
- extra_config.append(f"project_id={project_id}")
1260
- declare(f"Project ID '{project_id}' added to extra_config.")
1261
-
1262
- deployment_values["extra_config"] = extra_config
1263
-
1264
- tags = []
1265
- # use click.prompt to populate tags until someone just hits enter
1266
- while True:
1267
- declare(
1268
- "\nAdd to tags for stack deployment -->\n",
1269
- bold=True,
1270
- )
1271
- tag = click.prompt(
1272
- "Enter `tags` key or hit enter to skip",
1273
- type=str,
1274
- default="",
1430
+ if len(value) > 16:
1431
+ raise click.BadParameter(
1432
+ "Stack name must have a maximum length of 16 characters."
1275
1433
  )
1276
- if tag == "":
1277
- break
1278
- value = click.prompt(
1279
- f"Enter value for '{tag}'",
1280
- type=str,
1281
- )
1282
- tags.append(f"{tag}={value}")
1283
- deployment_values["tags"] = tags
1284
1434
 
1285
- return deployment_values
1435
+ return value
1436
+
1286
1437
 
1438
+ @stack.command(
1439
+ help="""Deploy a fully functional ZenML stack in one of the cloud providers.
1287
1440
 
1288
- @stack.command(help="Deploy a stack using mlstacks.")
1441
+ Running this command will initiate an assisted process that will walk you
1442
+ through automatically provisioning all the cloud infrastructure resources
1443
+ necessary for a fully functional ZenML stack in the cloud provider of your
1444
+ choice. A corresponding ZenML stack will also be automatically registered along
1445
+ with all the necessary components and properly authenticated through service
1446
+ connectors.
1447
+ """
1448
+ )
1289
1449
  @click.option(
1290
1450
  "--provider",
1291
1451
  "-p",
1292
1452
  "provider",
1293
1453
  required=True,
1294
- type=click.Choice(STACK_RECIPE_MODULAR_RECIPES),
1454
+ type=click.Choice(StackDeploymentProvider.values()),
1295
1455
  )
1296
1456
  @click.option(
1297
1457
  "--name",
1298
1458
  "-n",
1299
1459
  "stack_name",
1300
1460
  type=click.STRING,
1301
- required=True,
1302
- help="Set a name for the ZenML stack that will be imported from the YAML "
1303
- "configuration file which gets generated after deploying the stack recipe. "
1304
- "Defaults to the name of the stack recipe being deployed.",
1305
- )
1306
- @click.option(
1307
- "--region",
1308
- "-r",
1309
- "region",
1310
- type=click.STRING,
1311
- required=True,
1312
- help="The region to deploy the stack to.",
1313
- )
1314
- @click.option(
1315
- "--no-import",
1316
- "-ni",
1317
- "no_import_stack_flag",
1318
- is_flag=True,
1319
- help="If you don't want the stack to be imported automatically.",
1320
- )
1321
- @click.option(
1322
- "--artifact-store",
1323
- "-a",
1324
- "artifact_store",
1325
- required=False,
1326
- is_flag=True,
1327
- help="Whether to deploy an artifact store.",
1328
- )
1329
- @click.option(
1330
- "--container-registry",
1331
- "-c",
1332
- "container_registry",
1333
1461
  required=False,
1334
- is_flag=True,
1335
- help="Whether to deploy a container registry.",
1336
- )
1337
- @click.option(
1338
- "--mlops-platform",
1339
- "-m",
1340
- "mlops_platform",
1341
- type=click.Choice(["zenml"]),
1342
- required=False,
1343
- help="The flavor of MLOps platform to use."
1344
- "If not specified, the default MLOps platform will be used.",
1345
- )
1346
- @click.option(
1347
- "--orchestrator",
1348
- "-o",
1349
- required=False,
1350
- type=click.Choice(
1351
- [
1352
- "kubernetes",
1353
- "kubeflow",
1354
- "tekton",
1355
- "sagemaker",
1356
- "skypilot",
1357
- "vertex",
1358
- ]
1359
- ),
1360
- help="The flavor of orchestrator to use. "
1361
- "If not specified, the default orchestrator will be used.",
1362
- )
1363
- @click.option(
1364
- "--model-deployer",
1365
- "-md",
1366
- "model_deployer",
1367
- required=False,
1368
- type=click.Choice(["kserve", "seldon"]),
1369
- help="The flavor of model deployer to use. ",
1370
- )
1371
- @click.option(
1372
- "--experiment-tracker",
1373
- "-e",
1374
- "experiment_tracker",
1375
- required=False,
1376
- type=click.Choice(["mlflow"]),
1377
- help="The flavor of experiment tracker to use.",
1378
- )
1379
- @click.option(
1380
- "--step-operator",
1381
- "-s",
1382
- "step_operator",
1383
- required=False,
1384
- type=click.Choice(["sagemaker"]),
1385
- help="The flavor of step operator to use.",
1386
- )
1387
- @click.option(
1388
- "--file",
1389
- "-f",
1390
- "file",
1391
- required=False,
1392
- type=click.Path(exists=True, dir_okay=False, readable=True),
1393
- help="Use a YAML specification file as the basis of the stack deployment.",
1394
- )
1395
- @click.option(
1396
- "--debug-mode",
1397
- "-d",
1398
- "debug_mode",
1399
- is_flag=True,
1400
- default=False,
1401
- help="Whether to run the stack deployment in debug mode.",
1402
- )
1403
- @click.option(
1404
- "--extra-config",
1405
- "-x",
1406
- "extra_config",
1407
- multiple=True,
1408
- help="Extra configurations as key=value pairs. This option can be used multiple times.",
1462
+ help="Custom string to use as a prefix to generate names for the ZenML "
1463
+ "stack, its components service connectors as well as provisioned cloud "
1464
+ "infrastructure resources. May only contain alphanumeric characters and "
1465
+ "hyphens and have a maximum length of 16 characters.",
1466
+ callback=validate_name,
1409
1467
  )
1410
1468
  @click.option(
1411
- "--tags",
1412
- "-t",
1413
- "tags",
1414
- required=False,
1469
+ "--location",
1470
+ "-l",
1415
1471
  type=click.STRING,
1416
- help="Pass one or more tags.",
1417
- multiple=True,
1472
+ required=False,
1473
+ help="The location to deploy the stack to.",
1418
1474
  )
1419
1475
  @click.option(
1420
- "--interactive",
1421
- "-i",
1422
- "interactive",
1476
+ "--set",
1477
+ "set_stack",
1423
1478
  is_flag=True,
1424
- default=False,
1425
- help="Deploy the stack interactively.",
1479
+ help="Immediately set this stack as active.",
1480
+ type=click.BOOL,
1426
1481
  )
1427
1482
  @click.pass_context
1428
1483
  def deploy(
1429
1484
  ctx: click.Context,
1430
1485
  provider: str,
1431
- stack_name: str,
1432
- region: str,
1433
- mlops_platform: Optional[str] = None,
1434
- orchestrator: Optional[str] = None,
1435
- model_deployer: Optional[str] = None,
1436
- experiment_tracker: Optional[str] = None,
1437
- step_operator: Optional[str] = None,
1438
- no_import_stack_flag: bool = False,
1439
- artifact_store: Optional[bool] = None,
1440
- container_registry: Optional[bool] = None,
1441
- file: Optional[str] = None,
1442
- debug_mode: bool = False,
1443
- tags: Optional[List[str]] = None,
1444
- extra_config: Optional[List[str]] = None,
1445
- interactive: bool = False,
1486
+ stack_name: Optional[str] = None,
1487
+ location: Optional[str] = None,
1488
+ set_stack: bool = False,
1446
1489
  ) -> None:
1447
- """Deploy a stack with mlstacks.
1448
-
1449
- `zenml stack_recipe pull <STACK_RECIPE_NAME>` has to be called with the
1450
- same relative path before the `deploy` command.
1490
+ """Deploy and register a fully functional cloud ZenML stack.
1451
1491
 
1452
1492
  Args:
1453
1493
  ctx: The click context.
1454
1494
  provider: The cloud provider to deploy the stack to.
1455
1495
  stack_name: A name for the ZenML stack that gets imported as a result
1456
1496
  of the recipe deployment.
1457
- no_import_stack_flag: If you don't want the stack to be imported into
1458
- ZenML after deployment.
1459
- artifact_store: The flavor of artifact store to deploy. In the case of
1460
- the artifact store, it doesn't matter what you specify here, as
1461
- there's only one flavor per cloud provider and that will be deployed.
1462
- orchestrator: The flavor of orchestrator to use.
1463
- container_registry: The flavor of container registry to deploy. In the case of
1464
- the container registry, it doesn't matter what you specify here, as
1465
- there's only one flavor per cloud provider and that will be deployed.
1466
- model_deployer: The flavor of model deployer to deploy.
1467
- experiment_tracker: The flavor of experiment tracker to deploy.
1468
- step_operator: The flavor of step operator to deploy.
1469
- extra_config: Extra configurations as key=value pairs.
1470
- tags: Pass one or more tags.
1471
- debug_mode: Whether to run the stack deployment in debug mode.
1472
- file: Use a YAML specification file as the basis of the stack
1473
- deployment.
1474
- mlops_platform: The flavor of MLOps platform to use.
1475
- region: The region to deploy the stack to.
1476
- interactive: Deploy the stack interactively.
1497
+ location: The location to deploy the stack to.
1498
+ set_stack: Immediately set the deployed stack as active.
1499
+
1500
+ Raises:
1501
+ Abort: If the user aborts the deployment.
1502
+ KeyboardInterrupt: If the user interrupts the deployment.
1477
1503
  """
1504
+ stack_name = stack_name or f"zenml-{provider}-stack"
1505
+
1506
+ # Set up the markdown renderer to use the old-school markdown heading
1507
+ Markdown.elements.update(
1508
+ {
1509
+ "heading_open": OldSchoolMarkdownHeading,
1510
+ }
1511
+ )
1512
+
1513
+ client = Client()
1514
+ if client.zen_store.is_local_store():
1515
+ cli_utils.error(
1516
+ "This feature cannot be used with a local ZenML deployment. "
1517
+ "ZenML needs to be accessible from the cloud provider to allow the "
1518
+ "stack and its components to be registered automatically. "
1519
+ "Please deploy ZenML in a remote environment as described in the "
1520
+ "documentation: https://docs.zenml.io/getting-started/deploying-zenml "
1521
+ "or use a managed ZenML Pro server instance for quick access to "
1522
+ "this feature and more: https://www.zenml.io/pro"
1523
+ )
1524
+
1478
1525
  with track_handler(
1479
- event=AnalyticsEvent.DEPLOY_STACK,
1526
+ event=AnalyticsEvent.DEPLOY_FULL_STACK,
1480
1527
  ) as analytics_handler:
1481
- if stack_exists(stack_name):
1482
- cli_utils.error(
1483
- f"Stack with name '{stack_name}' already exists. Please choose a "
1484
- "different name."
1485
- )
1486
- elif stack_spec_exists(stack_name):
1487
- cli_utils.error(
1488
- f"Stack spec for stack named '{stack_name}' already exists. "
1489
- "Please choose a different name."
1490
- )
1528
+ analytics_handler.metadata = {
1529
+ "provider": provider,
1530
+ }
1491
1531
 
1492
- cli_utils.declare("Checking prerequisites are installed...")
1493
- cli_utils.verify_mlstacks_prerequisites_installation()
1494
- cli_utils.warning(ALPHA_MESSAGE)
1532
+ deployment = client.zen_store.get_stack_deployment_info(
1533
+ provider=StackDeploymentProvider(provider),
1534
+ )
1495
1535
 
1496
- if not file:
1497
- cli_params: Dict[str, Any] = ctx.params
1498
- if interactive:
1499
- cli_params = _get_deployment_params_interactively(cli_params)
1500
- stack, components = convert_click_params_to_mlstacks_primitives(
1501
- cli_params
1536
+ if location and location not in deployment.locations.values():
1537
+ cli_utils.error(
1538
+ f"Invalid location '{location}' for provider '{provider}'. "
1539
+ f"Valid locations are: {', '.join(deployment.locations.values())}"
1502
1540
  )
1503
1541
 
1504
- from mlstacks.utils import zenml_utils
1542
+ console.print(
1543
+ Markdown(
1544
+ f"# {provider.upper()} ZenML Cloud Stack Deployment\n"
1545
+ + deployment.description
1546
+ )
1547
+ )
1548
+ console.print(Markdown("## Details\n" + deployment.instructions))
1505
1549
 
1506
- cli_utils.declare("Checking flavor compatibility...")
1507
- if not zenml_utils.has_valid_flavor_combinations(
1508
- stack, components
1509
- ):
1510
- cli_utils.error(
1511
- "The specified stack and component flavors are not compatible "
1512
- "with the provider or with one another. Please try again."
1513
- )
1550
+ deployment_config = client.zen_store.get_stack_deployment_config(
1551
+ provider=StackDeploymentProvider(provider),
1552
+ stack_name=stack_name,
1553
+ location=location,
1554
+ )
1514
1555
 
1515
- stack_dict, component_dicts = convert_mlstacks_primitives_to_dicts(
1516
- stack, components
1556
+ if deployment_config.instructions:
1557
+ console.print(
1558
+ Markdown("## Instructions\n" + deployment_config.instructions),
1559
+ "\n",
1517
1560
  )
1518
- # write the stack and component yaml files
1519
- from mlstacks.constants import MLSTACKS_PACKAGE_NAME
1520
1561
 
1521
- spec_dir = os.path.join(
1522
- click.get_app_dir(MLSTACKS_PACKAGE_NAME),
1523
- "stack_specs",
1524
- stack.name,
1562
+ if deployment_config.configuration:
1563
+ console.print(
1564
+ deployment_config.configuration,
1565
+ no_wrap=True,
1566
+ overflow="ignore",
1567
+ crop=False,
1568
+ style=Style(bgcolor="grey15"),
1525
1569
  )
1526
- cli_utils.declare(f"Writing spec files to {spec_dir}...")
1527
- create_dir_recursive_if_not_exists(spec_dir)
1528
1570
 
1529
- stack_file_path = os.path.join(
1530
- spec_dir, f"stack-{stack.name}.yaml"
1571
+ if not cli_utils.confirmation(
1572
+ "\n\nProceed to continue with the deployment. You will be "
1573
+ f"automatically redirected to "
1574
+ f"{deployment_config.deployment_url_text} in your browser.",
1575
+ ):
1576
+ raise click.Abort()
1577
+
1578
+ date_start = datetime.utcnow()
1579
+
1580
+ webbrowser.open(deployment_config.deployment_url)
1581
+ console.print(
1582
+ Markdown(
1583
+ f"If your browser did not open automatically, please open "
1584
+ f"the following URL into your browser to deploy the stack to "
1585
+ f"{provider.upper()}: "
1586
+ f"[{deployment_config.deployment_url_text}]"
1587
+ f"({deployment_config.deployment_url}).\n\n"
1531
1588
  )
1532
- write_yaml(file_path=stack_file_path, contents=stack_dict)
1533
- for component in component_dicts:
1534
- write_yaml(
1535
- file_path=os.path.join(
1536
- spec_dir, f"{component['name']}.yaml"
1537
- ),
1538
- contents=component,
1539
- )
1540
- else:
1541
- declare("Importing from stack specification file...")
1542
- stack_file_path = file
1543
-
1544
- from mlstacks.utils.yaml_utils import load_stack_yaml
1545
-
1546
- stack = load_stack_yaml(stack_file_path)
1547
-
1548
- analytics_handler.metadata = {
1549
- "stack_provider": stack.provider,
1550
- "debug_mode": debug_mode,
1551
- "no_import_stack_flag": no_import_stack_flag,
1552
- "user_created_spec": bool(file),
1553
- "mlops_platform": mlops_platform,
1554
- "orchestrator": orchestrator,
1555
- "model_deployer": model_deployer,
1556
- "experiment_tracker": experiment_tracker,
1557
- "step_operator": step_operator,
1558
- "artifact_store": artifact_store,
1559
- "container_registry": container_registry,
1560
- }
1561
-
1562
- deploy_mlstacks_stack(
1563
- spec_file_path=stack_file_path,
1564
- stack_name=stack.name,
1565
- stack_provider=stack.provider,
1566
- debug_mode=debug_mode,
1567
- no_import_stack_flag=no_import_stack_flag,
1568
- user_created_spec=bool(file),
1569
1589
  )
1570
1590
 
1591
+ try:
1592
+ cli_utils.declare(
1593
+ "\n\nWaiting for the deployment to complete and the stack to be "
1594
+ "registered. Press CTRL+C to abort...\n"
1595
+ )
1571
1596
 
1572
- @stack.command(
1573
- help="Destroy stack components created previously with "
1574
- "`zenml stack deploy`"
1575
- )
1576
- @click.argument("stack_name", required=True)
1577
- @click.option(
1578
- "--debug",
1579
- "-d",
1580
- "debug_mode",
1581
- is_flag=True,
1582
- default=False,
1583
- help="Whether to run Terraform in debug mode.",
1584
- )
1585
- def destroy(
1586
- stack_name: str,
1587
- debug_mode: bool = False,
1588
- ) -> None:
1589
- """Destroy all resources previously created with `zenml stack deploy`.
1597
+ while True:
1598
+ deployed_stack = client.zen_store.get_stack_deployment_stack(
1599
+ provider=StackDeploymentProvider(provider),
1600
+ stack_name=stack_name,
1601
+ location=location,
1602
+ date_start=date_start,
1603
+ )
1604
+ if deployed_stack:
1605
+ break
1606
+ time.sleep(10)
1607
+
1608
+ analytics_handler.metadata.update(
1609
+ {
1610
+ "stack_id": deployed_stack.stack.id,
1611
+ }
1612
+ )
1590
1613
 
1591
- Args:
1592
- stack_name: Name of the stack
1593
- debug_mode: Whether to run Terraform in debug mode.
1594
- """
1595
- if not confirmation(
1596
- f"Are you sure you want to destroy stack '{stack_name}' and all "
1597
- "associated infrastructure?"
1598
- ):
1599
- error("Aborting stack destroy...")
1614
+ except KeyboardInterrupt:
1615
+ cli_utils.declare("Stack deployment aborted.")
1616
+ raise
1600
1617
 
1601
- with track_handler(
1602
- event=AnalyticsEvent.DESTROY_STACK,
1603
- ) as analytics_handler:
1604
- analytics_handler.metadata["debug_mode"] = debug_mode
1605
- cli_utils.verify_mlstacks_prerequisites_installation()
1606
- from mlstacks.constants import MLSTACKS_PACKAGE_NAME
1618
+ stack_desc = f"""## Stack successfully registered! 🚀
1619
+ Stack [{deployed_stack.stack.name}]({get_stack_url(deployed_stack.stack)}):\n"""
1607
1620
 
1608
- # check the stack actually exists
1609
- if not stack_exists(stack_name):
1610
- cli_utils.error(
1611
- f"Stack with name '{stack_name}' does not exist. Please check and "
1612
- "try again."
1621
+ for component_type, components in deployed_stack.stack.components.items():
1622
+ if components:
1623
+ component = components[0]
1624
+ stack_desc += (
1625
+ f" * `{component.flavor_name}` {component_type.value}: "
1626
+ f"[{component.name}]({get_component_url(component)})\n"
1613
1627
  )
1614
1628
 
1615
- spec_file_path = get_stack_spec_file_path(stack_name)
1616
- spec_files_dir: str = os.path.join(
1617
- click.get_app_dir(MLSTACKS_PACKAGE_NAME), "stack_specs", stack_name
1629
+ if deployed_stack.service_connector:
1630
+ stack_desc += (
1631
+ f" * Service Connector: {deployed_stack.service_connector.name}\n"
1618
1632
  )
1619
- user_created_spec = str(Path(spec_file_path).parent) != spec_files_dir
1620
1633
 
1621
- provider = read_yaml(file_path=spec_file_path).get("provider")
1622
- tf_definitions_path: str = os.path.join(
1623
- click.get_app_dir(MLSTACKS_PACKAGE_NAME),
1624
- "terraform",
1625
- f"{provider}-modular",
1626
- )
1627
-
1628
- cli_utils.declare(
1629
- "Checking Terraform definitions and spec files are present..."
1630
- )
1631
- verify_spec_and_tf_files_exist(spec_file_path, tf_definitions_path)
1634
+ console.print(Markdown(stack_desc))
1632
1635
 
1633
- from mlstacks.utils import terraform_utils
1636
+ follow_up = f"""
1637
+ ## Follow-up
1634
1638
 
1635
- cli_utils.declare(
1636
- f"Destroying stack '{stack_name}' using Terraform..."
1637
- )
1638
- terraform_utils.destroy_stack(
1639
- stack_path=spec_file_path, debug_mode=debug_mode
1640
- )
1641
- cli_utils.declare(f"Stack '{stack_name}' successfully destroyed.")
1639
+ {deployment.post_deploy_instructions}
1642
1640
 
1643
- if cli_utils.confirmation(
1644
- f"Would you like to recursively delete the associated ZenML "
1645
- f"stack '{stack_name}'?\nThis will delete the stack and any "
1646
- "underlying stack components."
1647
- ):
1648
- from zenml.client import Client
1641
+ To use the `{deployed_stack.stack.name}` stack to run pipelines:
1649
1642
 
1650
- client = Client()
1651
- client.delete_stack(name_id_or_prefix=stack_name, recursive=True)
1652
- cli_utils.declare(
1653
- f"Stack '{stack_name}' successfully deleted from ZenML."
1654
- )
1643
+ * install the required ZenML integrations by running: `zenml integration install {" ".join(deployment.integrations)}`
1644
+ """
1645
+ if set_stack:
1646
+ client.activate_stack(deployed_stack.stack.id)
1647
+ follow_up += f"""
1648
+ * the `{deployed_stack.stack.name}` stack has already been set as active
1649
+ """
1650
+ else:
1651
+ follow_up += f"""
1652
+ * set the `{deployed_stack.stack.name}` stack as active by running: `zenml stack set {deployed_stack.stack.name}`
1653
+ """
1655
1654
 
1656
- spec_dir = os.path.dirname(spec_file_path)
1657
- if not user_created_spec and cli_utils.confirmation(
1658
- f"Would you like to delete the `mlstacks` spec directory for "
1659
- f"this stack, located at {spec_dir}?"
1660
- ):
1661
- rmtree(spec_files_dir)
1662
- cli_utils.declare(
1663
- f"Spec directory for stack '{stack_name}' successfully deleted."
1664
- )
1665
- cli_utils.declare(f"Stack '{stack_name}' successfully destroyed.")
1655
+ console.print(
1656
+ Markdown(follow_up),
1657
+ )
1666
1658
 
1667
1659
 
1668
1660
  @stack.command(
@@ -1726,3 +1718,301 @@ def connect_stack(
1726
1718
  interactive=interactive,
1727
1719
  no_verify=no_verify,
1728
1720
  )
1721
+
1722
+
1723
+ def _get_service_connector_info(
1724
+ cloud_provider: str,
1725
+ connector_details: Optional[
1726
+ Union[ServiceConnectorResponse, ServiceConnectorRequest]
1727
+ ],
1728
+ ) -> ServiceConnectorInfo:
1729
+ """Get a service connector info with given cloud provider.
1730
+
1731
+ Args:
1732
+ cloud_provider: The cloud provider to use.
1733
+ connector_details: Whether to use implicit credentials.
1734
+
1735
+ Returns:
1736
+ The info model of the created service connector.
1737
+
1738
+ Raises:
1739
+ ValueError: If the cloud provider is not supported.
1740
+ """
1741
+ from rich.prompt import Prompt
1742
+
1743
+ if cloud_provider not in {"aws", "gcp", "azure"}:
1744
+ raise ValueError(f"Unknown cloud provider {cloud_provider}")
1745
+
1746
+ client = Client()
1747
+ auth_methods = client.get_service_connector_type(
1748
+ cloud_provider
1749
+ ).auth_method_dict
1750
+ if not connector_details:
1751
+ fixed_auth_methods = list(
1752
+ [
1753
+ (key, value)
1754
+ for key, value in auth_methods.items()
1755
+ if key != "implicit"
1756
+ ]
1757
+ )
1758
+ choices = []
1759
+ headers = ["Name", "Required"]
1760
+ for _, value in fixed_auth_methods:
1761
+ schema = value.config_schema
1762
+ required = ""
1763
+ for each_req in schema["required"]:
1764
+ field = schema["properties"][each_req]
1765
+ required += f"[bold]{each_req}[/bold] [italic]({field.get('title', 'no description')})[/italic]\n"
1766
+ choices.append([value.name, required])
1767
+
1768
+ selected_auth_idx = cli_utils.multi_choice_prompt(
1769
+ object_type=f"authentication methods for {cloud_provider.upper()}",
1770
+ choices=choices,
1771
+ headers=headers,
1772
+ prompt_text="Please choose one of the authentication option above",
1773
+ )
1774
+ if selected_auth_idx is None:
1775
+ cli_utils.error("No authentication method selected.")
1776
+ auth_type = fixed_auth_methods[selected_auth_idx][0]
1777
+ else:
1778
+ auth_type = connector_details.auth_method
1779
+
1780
+ selected_auth_model = auth_methods[auth_type]
1781
+
1782
+ required_fields = selected_auth_model.config_schema["required"]
1783
+ properties = selected_auth_model.config_schema["properties"]
1784
+
1785
+ answers = {}
1786
+ for req_field in required_fields:
1787
+ if connector_details:
1788
+ if conf_value := connector_details.configuration.get(
1789
+ req_field, None
1790
+ ):
1791
+ answers[req_field] = conf_value
1792
+ elif secret_value := connector_details.secrets.get(
1793
+ req_field, None
1794
+ ):
1795
+ answers[req_field] = secret_value.get_secret_value()
1796
+ if req_field not in answers:
1797
+ answers[req_field] = Prompt.ask(
1798
+ f"Please enter value for `{req_field}`:",
1799
+ password="format" in properties[req_field]
1800
+ and properties[req_field]["format"] == "password",
1801
+ )
1802
+
1803
+ return ServiceConnectorInfo(
1804
+ type=cloud_provider,
1805
+ auth_method=auth_type,
1806
+ configuration=answers,
1807
+ )
1808
+
1809
+
1810
+ def _get_stack_component_info(
1811
+ component_type: str,
1812
+ cloud_provider: str,
1813
+ resources_info: ServiceConnectorResourcesInfo,
1814
+ service_connector_index: Optional[int] = None,
1815
+ ) -> ComponentInfo:
1816
+ """Get a stack component info with given type and service connector.
1817
+
1818
+ Args:
1819
+ component_type: The type of component to create.
1820
+ cloud_provider: The cloud provider to use.
1821
+ resources_info: The resources info of the service connector.
1822
+ service_connector_index: The index of the service connector to use.
1823
+
1824
+ Returns:
1825
+ The info model of the stack component.
1826
+
1827
+ Raises:
1828
+ ValueError: If the cloud provider is not supported.
1829
+ ValueError: If the component type is not supported.
1830
+ """
1831
+ from rich.prompt import Prompt
1832
+
1833
+ if cloud_provider not in {"aws", "azure", "gcp"}:
1834
+ raise ValueError(f"Unknown cloud provider {cloud_provider}")
1835
+
1836
+ flavor = "undefined"
1837
+ service_connector_resource_id = None
1838
+ config = {}
1839
+ choices = [
1840
+ [cri.flavor, resource_id]
1841
+ for cri in resources_info.components_resources_info[
1842
+ StackComponentType(component_type)
1843
+ ]
1844
+ for resource_id in cri.accessible_by_service_connector
1845
+ ]
1846
+ if component_type == "artifact_store":
1847
+ selected_storage_idx = cli_utils.multi_choice_prompt(
1848
+ object_type=f"{cloud_provider.upper()} storages",
1849
+ choices=choices,
1850
+ headers=["Artifact Store Type", "Storage"],
1851
+ prompt_text="Please choose one of the storages for the new artifact store:",
1852
+ )
1853
+ if selected_storage_idx is None:
1854
+ cli_utils.error("No storage selected.")
1855
+
1856
+ selected_storage = choices[selected_storage_idx]
1857
+
1858
+ flavor = selected_storage[0]
1859
+ config = {"path": selected_storage[1]}
1860
+ service_connector_resource_id = selected_storage[1]
1861
+ elif component_type == "orchestrator":
1862
+
1863
+ def query_region(
1864
+ provider: StackDeploymentProvider,
1865
+ compute_type: str,
1866
+ is_skypilot: bool = False,
1867
+ ) -> str:
1868
+ deployment_info = Client().zen_store.get_stack_deployment_info(
1869
+ provider
1870
+ )
1871
+ region = Prompt.ask(
1872
+ f"Select the location for your {compute_type}:",
1873
+ choices=sorted(
1874
+ deployment_info.skypilot_default_regions.values()
1875
+ if is_skypilot
1876
+ else deployment_info.locations.values()
1877
+ ),
1878
+ show_choices=True,
1879
+ )
1880
+ return region
1881
+
1882
+ selected_orchestrator_idx = cli_utils.multi_choice_prompt(
1883
+ object_type=f"orchestrators on {cloud_provider.upper()}",
1884
+ choices=choices,
1885
+ headers=["Orchestrator Type", "Details"],
1886
+ prompt_text="Please choose one of the orchestrators for the new orchestrator:",
1887
+ )
1888
+ if selected_orchestrator_idx is None:
1889
+ cli_utils.error("No orchestrator selected.")
1890
+
1891
+ selected_orchestrator = choices[selected_orchestrator_idx]
1892
+
1893
+ config = {}
1894
+ flavor = selected_orchestrator[0]
1895
+ if flavor == "sagemaker":
1896
+ execution_role = Prompt.ask("Enter an execution role ARN:")
1897
+ config["execution_role"] = execution_role
1898
+ elif flavor == "vm_aws":
1899
+ config["region"] = selected_orchestrator[1]
1900
+ elif flavor == "vm_gcp":
1901
+ config["region"] = query_region(
1902
+ StackDeploymentProvider.GCP,
1903
+ "Skypilot cluster",
1904
+ is_skypilot=True,
1905
+ )
1906
+ elif flavor == "vm_azure":
1907
+ config["region"] = query_region(
1908
+ StackDeploymentProvider.AZURE,
1909
+ "Skypilot cluster",
1910
+ is_skypilot=True,
1911
+ )
1912
+ elif flavor == "azureml":
1913
+ config["subscription_id"] = Prompt.ask(
1914
+ "Enter the subscription ID:"
1915
+ )
1916
+ config["resource_group"] = Prompt.ask("Enter the resource group:")
1917
+ config["workspace"] = Prompt.ask("Enter the workspace name:")
1918
+ elif flavor == "vertex":
1919
+ config["location"] = query_region(
1920
+ StackDeploymentProvider.GCP, "Vertex AI job"
1921
+ )
1922
+ service_connector_resource_id = selected_orchestrator[1]
1923
+ elif component_type == "container_registry":
1924
+ selected_registry_idx = cli_utils.multi_choice_prompt(
1925
+ object_type=f"{cloud_provider.upper()} registries",
1926
+ choices=choices,
1927
+ headers=["Container Registry Type", "Container Registry"],
1928
+ prompt_text="Please choose one of the registries for the new container registry:",
1929
+ )
1930
+ if selected_registry_idx is None:
1931
+ cli_utils.error("No container registry selected.")
1932
+ selected_registry = choices[selected_registry_idx]
1933
+ flavor = selected_registry[0]
1934
+ config = {"uri": selected_registry[1]}
1935
+ service_connector_resource_id = selected_registry[1]
1936
+ else:
1937
+ raise ValueError(f"Unknown component type {component_type}")
1938
+
1939
+ return ComponentInfo(
1940
+ flavor=flavor,
1941
+ configuration=config,
1942
+ service_connector_index=service_connector_index,
1943
+ service_connector_resource_id=service_connector_resource_id,
1944
+ )
1945
+
1946
+
1947
+ @stack.command(
1948
+ name="export-requirements", help="Export the stack requirements."
1949
+ )
1950
+ @click.argument(
1951
+ "stack_name_or_id",
1952
+ type=click.STRING,
1953
+ required=False,
1954
+ )
1955
+ @click.option(
1956
+ "--output-file",
1957
+ "-o",
1958
+ "output_file",
1959
+ type=str,
1960
+ required=False,
1961
+ help="File to which to export the stack requirements. If not "
1962
+ "provided, the requirements will be printed to stdout instead.",
1963
+ )
1964
+ @click.option(
1965
+ "--overwrite",
1966
+ "-ov",
1967
+ "overwrite",
1968
+ type=bool,
1969
+ required=False,
1970
+ is_flag=True,
1971
+ help="Overwrite the output file if it already exists. This option is "
1972
+ "only valid if the output file is provided.",
1973
+ )
1974
+ def export_requirements(
1975
+ stack_name_or_id: Optional[str] = None,
1976
+ output_file: Optional[str] = None,
1977
+ overwrite: bool = False,
1978
+ ) -> None:
1979
+ """Exports stack requirements so they can be installed using pip.
1980
+
1981
+ Args:
1982
+ stack_name_or_id: Stack name or ID. If not given, the active stack will
1983
+ be used.
1984
+ output_file: Optional path to the requirements output file.
1985
+ overwrite: Overwrite the output file if it already exists. This option
1986
+ is only valid if the output file is provided.
1987
+ """
1988
+ try:
1989
+ stack_model: "StackResponse" = Client().get_stack(
1990
+ name_id_or_prefix=stack_name_or_id
1991
+ )
1992
+ except KeyError as err:
1993
+ cli_utils.error(str(err))
1994
+
1995
+ requirements, _ = requirements_utils.get_requirements_for_stack(
1996
+ stack_model
1997
+ )
1998
+
1999
+ if not requirements:
2000
+ cli_utils.declare(f"Stack `{stack_model.name}` has no requirements.")
2001
+ return
2002
+
2003
+ if output_file:
2004
+ try:
2005
+ with open(output_file, "x") as f:
2006
+ f.write("\n".join(requirements))
2007
+ except FileExistsError:
2008
+ if overwrite or cli_utils.confirmation(
2009
+ "A file already exists at the specified path. "
2010
+ "Would you like to overwrite it?"
2011
+ ):
2012
+ with open(output_file, "w") as f:
2013
+ f.write("\n".join(requirements))
2014
+ cli_utils.declare(
2015
+ f"Requirements for stack `{stack_model.name}` exported to {output_file}."
2016
+ )
2017
+ else:
2018
+ click.echo(" ".join(requirements), nl=False)