zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/__init__.py +9 -12
- zenml/actions/__init__.py +14 -0
- zenml/actions/base_action.py +682 -0
- zenml/actions/pipeline_run/__init__.py +0 -0
- zenml/actions/pipeline_run/pipeline_run_action.py +223 -0
- zenml/analytics/context.py +70 -15
- zenml/analytics/enums.py +21 -17
- zenml/analytics/models.py +1 -0
- zenml/analytics/utils.py +19 -7
- zenml/annotators/base_annotator.py +4 -3
- zenml/artifact_stores/base_artifact_store.py +155 -72
- zenml/artifact_stores/local_artifact_store.py +3 -2
- zenml/artifacts/artifact_config.py +67 -58
- zenml/artifacts/external_artifact.py +24 -39
- zenml/artifacts/external_artifact_config.py +27 -37
- zenml/artifacts/preexisting_data_materializer.py +90 -0
- zenml/artifacts/unmaterialized_artifact.py +2 -12
- zenml/artifacts/utils.py +521 -220
- zenml/cli/__init__.py +1511 -456
- zenml/cli/annotator.py +48 -14
- zenml/cli/artifact.py +22 -4
- zenml/cli/authorized_device.py +1 -0
- zenml/cli/base.py +185 -22
- zenml/cli/code_repository.py +1 -0
- zenml/cli/formatter.py +1 -1
- zenml/cli/integration.py +119 -24
- zenml/cli/login.py +1053 -0
- zenml/cli/model.py +42 -15
- zenml/cli/model_registry.py +1 -2
- zenml/cli/pipeline.py +168 -82
- zenml/cli/secret.py +1 -2
- zenml/cli/served_model.py +56 -26
- zenml/cli/server.py +486 -619
- zenml/cli/service_accounts.py +55 -14
- zenml/cli/service_connectors.py +97 -6
- zenml/cli/stack.py +821 -531
- zenml/cli/stack_components.py +15 -598
- zenml/cli/tag.py +1 -0
- zenml/cli/text_utils.py +36 -2
- zenml/cli/user_management.py +204 -6
- zenml/cli/utils.py +411 -290
- zenml/client.py +1742 -298
- zenml/client_lazy_loader.py +224 -0
- zenml/code_repositories/base_code_repository.py +5 -4
- zenml/code_repositories/git/local_git_repository_context.py +1 -0
- zenml/code_repositories/local_repository_context.py +1 -0
- zenml/config/__init__.py +2 -0
- zenml/config/base_settings.py +6 -6
- zenml/config/build_configuration.py +43 -17
- zenml/config/compiler.py +82 -49
- zenml/config/docker_settings.py +139 -83
- zenml/config/global_config.py +260 -234
- zenml/config/pipeline_configurations.py +32 -11
- zenml/config/pipeline_run_configuration.py +15 -3
- zenml/config/pipeline_spec.py +6 -6
- zenml/config/resource_settings.py +8 -9
- zenml/config/retry_config.py +27 -0
- zenml/config/schedule.py +27 -18
- zenml/config/secret_reference_mixin.py +8 -4
- zenml/config/secrets_store_config.py +16 -24
- zenml/config/server_config.py +434 -51
- zenml/config/settings_resolver.py +2 -1
- zenml/config/source.py +97 -31
- zenml/config/step_configurations.py +83 -39
- zenml/config/step_run_info.py +3 -0
- zenml/config/store_config.py +20 -54
- zenml/config/strict_base_model.py +2 -6
- zenml/console.py +0 -1
- zenml/constants.py +232 -74
- zenml/container_registries/azure_container_registry.py +1 -0
- zenml/container_registries/base_container_registry.py +7 -3
- zenml/container_registries/default_container_registry.py +4 -3
- zenml/container_registries/dockerhub_container_registry.py +1 -0
- zenml/container_registries/gcp_container_registry.py +1 -0
- zenml/container_registries/github_container_registry.py +2 -10
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/entrypoints/base_entrypoint_configuration.py +76 -17
- zenml/entrypoints/pipeline_entrypoint_configuration.py +1 -0
- zenml/entrypoints/step_entrypoint_configuration.py +21 -2
- zenml/enums.py +91 -9
- zenml/environment.py +52 -319
- zenml/{steps/base_parameters.py → event_hub/__init__.py} +5 -7
- zenml/event_hub/base_event_hub.py +196 -0
- zenml/event_hub/event_hub.py +181 -0
- zenml/event_sources/__init__.py +14 -0
- zenml/{lineage_graph/edge.py → event_sources/base_event.py} +5 -7
- zenml/event_sources/base_event_source.py +695 -0
- zenml/event_sources/webhooks/__init__.py +14 -0
- zenml/event_sources/webhooks/base_webhook_event_source.py +231 -0
- zenml/exceptions.py +40 -41
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/hooks/hook_validators.py +3 -11
- zenml/image_builders/base_image_builder.py +5 -2
- zenml/image_builders/build_context.py +24 -80
- zenml/image_builders/local_image_builder.py +14 -6
- zenml/integrations/__init__.py +16 -4
- zenml/integrations/airflow/__init__.py +3 -5
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +15 -7
- zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +15 -252
- zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
- zenml/integrations/argilla/__init__.py +46 -0
- zenml/integrations/argilla/annotators/__init__.py +20 -0
- zenml/integrations/argilla/annotators/argilla_annotator.py +443 -0
- zenml/{post_execution → integrations/argilla/flavors}/__init__.py +9 -13
- zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +150 -0
- zenml/integrations/aws/__init__.py +7 -3
- zenml/integrations/aws/container_registries/aws_container_registry.py +44 -8
- zenml/integrations/aws/flavors/__init__.py +6 -0
- zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
- zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
- zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +104 -16
- zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
- zenml/integrations/aws/image_builders/__init__.py +20 -0
- zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +421 -133
- zenml/integrations/aws/service_connectors/aws_service_connector.py +175 -48
- zenml/integrations/aws/step_operators/sagemaker_step_operator.py +1 -1
- zenml/integrations/azure/__init__.py +14 -5
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/__init__.py +11 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +166 -0
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +71 -18
- zenml/integrations/azure/orchestrators/__init__.py +19 -0
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +583 -0
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +82 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +15 -6
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -173
- zenml/integrations/bentoml/__init__.py +1 -1
- zenml/integrations/bentoml/constants.py +1 -1
- zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
- zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +128 -239
- zenml/integrations/bentoml/services/__init__.py +15 -4
- zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
- zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +85 -43
- zenml/integrations/bentoml/services/deployment_type.py +23 -0
- zenml/integrations/bentoml/steps/bento_builder.py +2 -0
- zenml/integrations/bentoml/steps/bentoml_deployer.py +97 -47
- zenml/integrations/bitbucket/__init__.py +42 -0
- zenml/integrations/bitbucket/plugins/__init__.py +20 -0
- zenml/integrations/bitbucket/plugins/bitbucket_webhook_event_source_flavor.py +43 -0
- zenml/integrations/bitbucket/plugins/event_sources/__init__.py +0 -0
- zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +486 -0
- zenml/integrations/comet/__init__.py +49 -0
- zenml/integrations/{kserve/services → comet/experiment_trackers}/__init__.py +5 -6
- zenml/integrations/comet/experiment_trackers/comet_experiment_tracker.py +175 -0
- zenml/integrations/comet/flavors/__init__.py +24 -0
- zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +129 -0
- zenml/integrations/constants.py +14 -2
- zenml/integrations/databricks/__init__.py +70 -0
- zenml/{lineage_graph → integrations/databricks/flavors}/__init__.py +14 -18
- zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +174 -0
- zenml/integrations/{kserve → databricks}/model_deployers/__init__.py +5 -5
- zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
- zenml/integrations/databricks/orchestrators/__init__.py +20 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +501 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
- zenml/integrations/databricks/services/__init__.py +19 -0
- zenml/integrations/databricks/services/databricks_deployment.py +407 -0
- zenml/integrations/{gcp/orchestrators/vertex_scheduler → databricks/utils}/__init__.py +2 -2
- zenml/integrations/databricks/utils/databricks_utils.py +87 -0
- zenml/integrations/deepchecks/__init__.py +30 -10
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +59 -18
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/materializers/deepchecks_results_materializer.py +3 -3
- zenml/integrations/deepchecks/validation_checks.py +62 -35
- zenml/integrations/discord/__init__.py +1 -0
- zenml/integrations/discord/steps/discord_alerter_ask_step.py +1 -0
- zenml/integrations/discord/steps/discord_alerter_post_step.py +1 -0
- zenml/integrations/evidently/__init__.py +22 -4
- zenml/integrations/evidently/column_mapping.py +11 -3
- zenml/integrations/evidently/data_validators/evidently_data_validator.py +24 -6
- zenml/integrations/evidently/metrics.py +7 -8
- zenml/integrations/evidently/tests.py +7 -8
- zenml/integrations/facets/__init__.py +22 -5
- zenml/integrations/facets/models.py +2 -7
- zenml/integrations/feast/__init__.py +19 -3
- zenml/integrations/feast/feature_stores/feast_feature_store.py +13 -32
- zenml/integrations/gcp/__init__.py +6 -6
- zenml/integrations/gcp/artifact_stores/gcp_artifact_store.py +6 -0
- zenml/integrations/gcp/flavors/gcp_artifact_store_flavor.py +1 -0
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +24 -2
- zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +14 -1
- zenml/integrations/gcp/google_credentials_mixin.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +519 -296
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +788 -113
- zenml/integrations/gcp/step_operators/vertex_step_operator.py +9 -1
- zenml/integrations/github/__init__.py +15 -0
- zenml/integrations/github/code_repositories/github_code_repository.py +3 -2
- zenml/integrations/github/plugins/__init__.py +19 -0
- zenml/integrations/github/plugins/event_sources/__init__.py +0 -0
- zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +510 -0
- zenml/integrations/github/plugins/github_webhook_event_source_flavor.py +43 -0
- zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +7 -1
- zenml/integrations/great_expectations/__init__.py +22 -10
- zenml/integrations/great_expectations/data_validators/ge_data_validator.py +61 -57
- zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
- zenml/integrations/great_expectations/ge_store_backend.py +24 -11
- zenml/integrations/great_expectations/materializers/ge_materializer.py +9 -9
- zenml/integrations/great_expectations/utils.py +5 -5
- zenml/integrations/huggingface/__init__.py +52 -1
- zenml/integrations/huggingface/flavors/__init__.py +26 -0
- zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +130 -0
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +102 -19
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +18 -19
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +104 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +18 -19
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +13 -15
- zenml/integrations/huggingface/model_deployers/__init__.py +20 -0
- zenml/integrations/huggingface/model_deployers/huggingface_model_deployer.py +247 -0
- zenml/integrations/huggingface/services/__init__.py +19 -0
- zenml/integrations/huggingface/services/huggingface_deployment.py +292 -0
- zenml/integrations/huggingface/steps/__init__.py +21 -0
- zenml/integrations/huggingface/steps/accelerate_runner.py +166 -0
- zenml/integrations/huggingface/steps/huggingface_deployer.py +110 -0
- zenml/integrations/hyperai/__init__.py +53 -0
- zenml/integrations/hyperai/flavors/__init__.py +20 -0
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +170 -0
- zenml/integrations/hyperai/orchestrators/__init__.py +21 -0
- zenml/integrations/hyperai/orchestrators/hyperai_orchestrator.py +510 -0
- zenml/integrations/hyperai/service_connectors/__init__.py +20 -0
- zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +375 -0
- zenml/integrations/integration.py +101 -47
- zenml/integrations/kaniko/flavors/kaniko_image_builder_flavor.py +6 -44
- zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +9 -6
- zenml/integrations/kubeflow/__init__.py +4 -1
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +67 -82
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +306 -248
- zenml/integrations/kubernetes/__init__.py +6 -3
- zenml/integrations/kubernetes/flavors/__init__.py +8 -0
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +43 -2
- zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +67 -12
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +89 -21
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +39 -7
- zenml/integrations/kubernetes/orchestrators/manifest_utils.py +42 -19
- zenml/integrations/kubernetes/pod_settings.py +21 -31
- zenml/integrations/kubernetes/serialization_utils.py +3 -3
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +14 -11
- zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
- zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +236 -0
- zenml/integrations/label_studio/__init__.py +1 -3
- zenml/integrations/label_studio/annotators/label_studio_annotator.py +46 -14
- zenml/integrations/label_studio/flavors/__init__.py +2 -0
- zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +20 -5
- zenml/integrations/langchain/__init__.py +7 -1
- zenml/integrations/langchain/materializers/document_materializer.py +44 -8
- zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
- zenml/integrations/lightgbm/__init__.py +1 -0
- zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
- zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +168 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +617 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +303 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +55 -8
- zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +31 -13
- zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +32 -37
- zenml/integrations/mlflow/model_deployers/mlflow_model_deployer.py +42 -233
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +42 -49
- zenml/integrations/mlflow/services/mlflow_deployment.py +30 -5
- zenml/integrations/mlflow/steps/mlflow_deployer.py +25 -27
- zenml/integrations/mlflow/steps/mlflow_registry.py +3 -1
- zenml/integrations/modal/__init__.py +46 -0
- zenml/integrations/modal/flavors/__init__.py +26 -0
- zenml/integrations/modal/flavors/modal_step_operator_flavor.py +125 -0
- zenml/integrations/modal/step_operators/__init__.py +22 -0
- zenml/integrations/modal/step_operators/modal_step_operator.py +242 -0
- zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
- zenml/integrations/neptune/experiment_trackers/run_state.py +71 -55
- zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
- zenml/integrations/neural_prophet/__init__.py +6 -1
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/{kserve/constants.py → numpy/materializers/__init__.py} +5 -4
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/openai/__init__.py +1 -1
- zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
- zenml/{steps/external_artifact.py → integrations/pandas/__init__.py} +17 -11
- zenml/integrations/{kserve/custom_deployer → pandas/materializers}/__init__.py +5 -5
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/pigeon/__init__.py +44 -0
- zenml/integrations/pigeon/annotators/__init__.py +20 -0
- zenml/integrations/pigeon/annotators/pigeon_annotator.py +330 -0
- zenml/integrations/{kserve → pigeon}/flavors/__init__.py +7 -7
- zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +104 -0
- zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
- zenml/integrations/polars/__init__.py +1 -0
- zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
- zenml/integrations/prodigy/__init__.py +48 -0
- zenml/integrations/prodigy/annotators/__init__.py +20 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +275 -0
- zenml/integrations/prodigy/flavors/__init__.py +24 -0
- zenml/integrations/prodigy/flavors/prodigy_annotator_flavor.py +101 -0
- zenml/integrations/pycaret/__init__.py +6 -0
- zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
- zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
- zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
- zenml/integrations/registry.py +38 -1
- zenml/integrations/s3/__init__.py +2 -3
- zenml/integrations/s3/artifact_stores/s3_artifact_store.py +191 -9
- zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +22 -45
- zenml/integrations/s3/utils.py +39 -0
- zenml/integrations/seldon/__init__.py +18 -2
- zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +19 -141
- zenml/integrations/seldon/secret_schemas/secret_schemas.py +2 -2
- zenml/integrations/seldon/seldon_client.py +55 -70
- zenml/integrations/seldon/services/seldon_deployment.py +4 -5
- zenml/integrations/seldon/steps/seldon_deployer.py +21 -20
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +29 -7
- zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +47 -28
- zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
- zenml/integrations/skypilot_aws/__init__.py +3 -2
- zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +2 -4
- zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
- zenml/integrations/skypilot_gcp/__init__.py +3 -2
- zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
- zenml/integrations/skypilot_kubernetes/__init__.py +52 -0
- zenml/integrations/skypilot_kubernetes/flavors/__init__.py +26 -0
- zenml/integrations/skypilot_kubernetes/flavors/skypilot_orchestrator_kubernetes_vm_flavor.py +125 -0
- zenml/integrations/skypilot_kubernetes/orchestrators/__init__.py +25 -0
- zenml/integrations/skypilot_kubernetes/orchestrators/skypilot_kubernetes_vm_orchestrator.py +74 -0
- zenml/integrations/skypilot_lambda/__init__.py +50 -0
- zenml/integrations/{kserve/secret_schemas → skypilot_lambda/flavors}/__init__.py +9 -12
- zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +130 -0
- zenml/{lineage_graph/node → integrations/skypilot_lambda/orchestrators}/__init__.py +8 -15
- zenml/integrations/skypilot_lambda/orchestrators/skypilot_lambda_vm_orchestrator.py +92 -0
- zenml/integrations/slack/__init__.py +1 -0
- zenml/integrations/slack/alerters/slack_alerter.py +22 -2
- zenml/integrations/slack/flavors/__init__.py +2 -0
- zenml/integrations/slack/flavors/slack_alerter_flavor.py +13 -4
- zenml/integrations/slack/steps/slack_alerter_ask_step.py +1 -0
- zenml/integrations/slack/steps/slack_alerter_post_step.py +1 -0
- zenml/integrations/spark/flavors/spark_step_operator_flavor.py +2 -39
- zenml/integrations/spark/step_operators/kubernetes_step_operator.py +1 -0
- zenml/integrations/spark/step_operators/spark_entrypoint_configuration.py +1 -0
- zenml/integrations/spark/step_operators/spark_step_operator.py +2 -0
- zenml/integrations/tekton/__init__.py +2 -1
- zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
- zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +549 -235
- zenml/integrations/tensorboard/__init__.py +1 -13
- zenml/integrations/tensorboard/services/tensorboard_service.py +3 -4
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +66 -59
- zenml/integrations/tensorflow/__init__.py +10 -15
- zenml/integrations/tensorflow/materializers/keras_materializer.py +24 -27
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +9 -16
- zenml/integrations/vllm/__init__.py +50 -0
- zenml/integrations/vllm/flavors/__init__.py +21 -0
- zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
- zenml/integrations/vllm/model_deployers/__init__.py +19 -0
- zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
- zenml/integrations/vllm/services/__init__.py +19 -0
- zenml/integrations/vllm/services/vllm_deployment.py +206 -0
- zenml/integrations/wandb/__init__.py +1 -0
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +21 -18
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/integrations/whylogs/data_validators/whylogs_data_validator.py +3 -3
- zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
- zenml/integrations/whylogs/materializers/whylogs_materializer.py +14 -21
- zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
- zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
- zenml/io/fileio.py +1 -0
- zenml/io/filesystem.py +2 -2
- zenml/io/local_filesystem.py +3 -3
- zenml/logger.py +41 -17
- zenml/logging/__init__.py +5 -0
- zenml/logging/step_logging.py +280 -32
- zenml/{new/steps → login}/__init__.py +4 -1
- zenml/login/credentials.py +370 -0
- zenml/login/credentials_store.py +638 -0
- zenml/{new/pipelines → login/pro}/__init__.py +4 -1
- zenml/login/pro/client.py +492 -0
- zenml/login/pro/constants.py +28 -0
- zenml/{zen_stores/schemas/identity_schemas.py → login/pro/models.py} +9 -11
- zenml/login/pro/organization/__init__.py +14 -0
- zenml/login/pro/organization/client.py +79 -0
- zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
- zenml/{_hub → login/pro/tenant}/__init__.py +2 -2
- zenml/login/pro/tenant/client.py +92 -0
- zenml/login/pro/tenant/models.py +176 -0
- zenml/login/pro/utils.py +107 -0
- zenml/login/server_info.py +52 -0
- zenml/{cli → login}/web_login.py +71 -21
- zenml/materializers/__init__.py +2 -4
- zenml/materializers/base_materializer.py +180 -51
- zenml/materializers/built_in_materializer.py +50 -23
- zenml/materializers/cloudpickle_materializer.py +4 -5
- zenml/materializers/numpy_materializer.py +23 -233
- zenml/materializers/pandas_materializer.py +22 -174
- zenml/materializers/pydantic_materializer.py +2 -2
- zenml/materializers/service_materializer.py +12 -10
- zenml/materializers/structured_string_materializer.py +12 -6
- zenml/materializers/uuid_materializer.py +79 -0
- zenml/metadata/lazy_load.py +33 -17
- zenml/metadata/metadata_types.py +112 -3
- zenml/model/lazy_load.py +85 -4
- zenml/model/model.py +236 -226
- zenml/model/utils.py +99 -141
- zenml/model_deployers/base_model_deployer.py +319 -47
- zenml/model_registries/base_model_registry.py +17 -15
- zenml/models/__init__.py +254 -175
- zenml/models/v2/base/base.py +254 -112
- zenml/models/v2/base/base_plugin_flavor.py +76 -0
- zenml/models/v2/base/filter.py +530 -199
- zenml/models/v2/base/page.py +2 -12
- zenml/models/v2/base/scoped.py +292 -22
- zenml/models/v2/core/action.py +276 -0
- zenml/models/v2/core/action_flavor.py +57 -0
- zenml/models/v2/core/api_key.py +35 -12
- zenml/models/v2/core/artifact.py +126 -6
- zenml/models/v2/core/artifact_version.py +289 -65
- zenml/models/v2/core/artifact_visualization.py +10 -4
- zenml/models/v2/core/code_reference.py +13 -4
- zenml/models/v2/core/code_repository.py +39 -16
- zenml/models/v2/core/component.py +113 -61
- zenml/models/v2/core/device.py +25 -5
- zenml/models/v2/core/event_source.py +244 -0
- zenml/models/v2/core/event_source_flavor.py +67 -0
- zenml/models/v2/core/flavor.py +90 -27
- zenml/models/v2/core/logs.py +81 -12
- zenml/models/v2/core/model.py +91 -42
- zenml/models/v2/core/model_version.py +100 -55
- zenml/models/v2/core/model_version_artifact.py +96 -89
- zenml/models/v2/core/model_version_pipeline_run.py +86 -53
- zenml/models/v2/core/pipeline.py +205 -80
- zenml/models/v2/core/pipeline_build.py +110 -21
- zenml/models/v2/core/pipeline_deployment.py +101 -36
- zenml/models/v2/core/pipeline_run.py +545 -33
- zenml/models/v2/core/run_metadata.py +23 -181
- zenml/models/v2/core/run_template.py +439 -0
- zenml/models/v2/core/schedule.py +66 -29
- zenml/models/v2/core/secret.py +33 -21
- zenml/models/v2/core/server_settings.py +224 -0
- zenml/models/v2/core/service.py +500 -0
- zenml/models/v2/core/service_account.py +40 -15
- zenml/models/v2/core/service_connector.py +247 -49
- zenml/models/v2/core/stack.py +163 -70
- zenml/models/v2/core/step_run.py +210 -48
- zenml/models/v2/core/tag.py +21 -8
- zenml/models/v2/core/tag_resource.py +13 -4
- zenml/models/v2/core/trigger.py +422 -0
- zenml/models/v2/core/trigger_execution.py +119 -0
- zenml/models/v2/core/user.py +136 -69
- zenml/models/v2/core/workspace.py +26 -7
- zenml/models/v2/misc/auth_models.py +11 -2
- zenml/models/v2/misc/build_item.py +3 -3
- zenml/models/v2/misc/external_user.py +3 -6
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/models/v2/misc/loaded_visualization.py +2 -2
- zenml/models/v2/misc/run_metadata.py +38 -0
- zenml/models/v2/misc/server_models.py +100 -0
- zenml/models/v2/misc/service_connector_type.py +9 -17
- zenml/models/v2/misc/stack_deployment.py +96 -0
- zenml/models/v2/misc/user_auth.py +7 -9
- zenml/orchestrators/__init__.py +4 -0
- zenml/orchestrators/base_orchestrator.py +136 -25
- zenml/orchestrators/containerized_orchestrator.py +1 -0
- zenml/orchestrators/dag_runner.py +18 -3
- zenml/orchestrators/input_utils.py +109 -48
- zenml/orchestrators/local/local_orchestrator.py +10 -0
- zenml/orchestrators/local_docker/local_docker_orchestrator.py +14 -42
- zenml/orchestrators/output_utils.py +16 -6
- zenml/orchestrators/publish_utils.py +12 -5
- zenml/orchestrators/step_launcher.py +142 -194
- zenml/orchestrators/step_run_utils.py +386 -0
- zenml/orchestrators/step_runner.py +181 -270
- zenml/orchestrators/utils.py +219 -84
- zenml/orchestrators/wheeled_orchestrator.py +147 -0
- zenml/pipelines/__init__.py +3 -16
- zenml/{new/pipelines → pipelines}/build_utils.py +287 -47
- zenml/{new/pipelines → pipelines}/pipeline_context.py +6 -2
- zenml/pipelines/pipeline_decorator.py +40 -64
- zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +376 -440
- zenml/pipelines/run_utils.py +358 -0
- zenml/plugins/__init__.py +0 -0
- zenml/plugins/base_plugin_flavor.py +88 -0
- zenml/plugins/plugin_flavor_registry.py +342 -0
- zenml/secret/base_secret.py +7 -8
- zenml/secret/schemas/basic_auth_secret_schema.py +0 -1
- zenml/service_connectors/docker_service_connector.py +19 -4
- zenml/service_connectors/service_connector.py +12 -14
- zenml/service_connectors/service_connector_registry.py +71 -55
- zenml/service_connectors/service_connector_utils.py +418 -0
- zenml/services/__init__.py +0 -2
- zenml/services/container/container_service.py +9 -6
- zenml/services/container/container_service_endpoint.py +1 -1
- zenml/services/container/entrypoint.py +3 -2
- zenml/services/local/local_daemon_entrypoint.py +9 -6
- zenml/services/local/local_service.py +1 -1
- zenml/services/local/local_service_endpoint.py +1 -1
- zenml/services/service.py +222 -130
- zenml/services/service_status.py +2 -1
- zenml/services/service_type.py +6 -5
- zenml/stack/flavor.py +25 -18
- zenml/stack/flavor_registry.py +4 -4
- zenml/stack/stack.py +20 -131
- zenml/stack/stack_component.py +136 -110
- zenml/stack/utils.py +36 -15
- zenml/stack_deployments/__init__.py +14 -0
- zenml/stack_deployments/aws_stack_deployment.py +320 -0
- zenml/stack_deployments/azure_stack_deployment.py +315 -0
- zenml/stack_deployments/gcp_stack_deployment.py +315 -0
- zenml/stack_deployments/stack_deployment.py +232 -0
- zenml/stack_deployments/utils.py +48 -0
- zenml/step_operators/step_operator_entrypoint_configuration.py +2 -1
- zenml/steps/__init__.py +3 -9
- zenml/steps/base_step.py +172 -315
- zenml/{new/steps → steps}/decorated_step.py +1 -0
- zenml/steps/entrypoint_function_utils.py +33 -93
- zenml/{new/steps → steps}/step_context.py +70 -50
- zenml/steps/step_decorator.py +47 -93
- zenml/steps/step_invocation.py +22 -60
- zenml/steps/utils.py +161 -48
- zenml/types.py +14 -1
- zenml/utils/archivable.py +178 -0
- zenml/utils/callback_registry.py +71 -0
- zenml/utils/code_repository_utils.py +1 -0
- zenml/utils/code_utils.py +346 -0
- zenml/utils/cuda_utils.py +50 -0
- zenml/utils/dashboard_utils.py +67 -21
- zenml/utils/deprecation_utils.py +22 -24
- zenml/utils/dict_utils.py +22 -0
- zenml/utils/docker_utils.py +34 -5
- zenml/utils/downloaded_repository_context.py +1 -0
- zenml/utils/env_utils.py +55 -1
- zenml/utils/filesync_model.py +65 -28
- zenml/utils/function_utils.py +260 -0
- zenml/utils/integration_utils.py +1 -0
- zenml/utils/json_utils.py +131 -0
- zenml/utils/materializer_utils.py +1 -1
- zenml/utils/metadata_utils.py +368 -0
- zenml/utils/notebook_utils.py +136 -0
- zenml/utils/package_utils.py +89 -0
- zenml/utils/pagination_utils.py +9 -7
- zenml/utils/pipeline_docker_image_builder.py +152 -149
- zenml/utils/pydantic_utils.py +276 -66
- zenml/utils/requirements_utils.py +71 -0
- zenml/utils/secret_utils.py +66 -12
- zenml/utils/settings_utils.py +2 -1
- zenml/utils/singleton.py +15 -3
- zenml/utils/source_code_utils.py +1 -0
- zenml/utils/source_utils.py +236 -14
- zenml/utils/string_utils.py +140 -0
- zenml/utils/typed_model.py +5 -3
- zenml/utils/typing_utils.py +223 -0
- zenml/utils/visualization_utils.py +5 -3
- zenml/utils/yaml_utils.py +1 -1
- zenml/zen_server/auth.py +387 -55
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/cloud_utils.py +253 -0
- zenml/zen_server/csrf.py +91 -0
- zenml/zen_server/dashboard/assets/404-Dfq64Boz.js +1 -0
- zenml/zen_server/dashboard/assets/@radix-DeK6qiuw.js +85 -0
- zenml/zen_server/dashboard/assets/@react-router-B3Z5rLr2.js +29 -0
- zenml/zen_server/dashboard/assets/@reactflow-BUNIMFeC.js +17 -0
- zenml/zen_server/dashboard/assets/@reactflow-C26Olbza.css +1 -0
- zenml/zen_server/dashboard/assets/@tanstack-DT5WLu9C.js +22 -0
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-B73Vs10T.js +1 -0
- zenml/zen_server/dashboard/assets/CodeSnippet-Bbx6fIb6.css +1 -0
- zenml/zen_server/dashboard/assets/CodeSnippet-DIJRT2NT.js +9 -0
- zenml/zen_server/dashboard/assets/CollapsibleCard-BzUHGZOU.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-BEGyld4c.js +1 -0
- zenml/zen_server/dashboard/assets/ComponentBadge-xyKiek1s.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-DhW-mapu.js +2 -0
- zenml/zen_server/dashboard/assets/CsvVizualization-D8oazBiE.js +15 -0
- zenml/zen_server/dashboard/assets/DeleteAlertDialog-WkSIIgfy.js +1 -0
- zenml/zen_server/dashboard/assets/DialogItem-Bgroeg29.js +1 -0
- zenml/zen_server/dashboard/assets/DisplayDate-CDMUcQHS.js +1 -0
- zenml/zen_server/dashboard/assets/EmptyState-BzdlCwp3.js +1 -0
- zenml/zen_server/dashboard/assets/Error-CY5tlu17.js +1 -0
- zenml/zen_server/dashboard/assets/ExecutionStatus-G8mjIaeA.js +1 -0
- zenml/zen_server/dashboard/assets/Helpbox-Bb1ed--O.js +1 -0
- zenml/zen_server/dashboard/assets/Infobox-Da6-76M2.js +1 -0
- zenml/zen_server/dashboard/assets/InlineAvatar-DqnZaBNq.js +1 -0
- zenml/zen_server/dashboard/assets/Lock-CYYy18Mm.js +1 -0
- zenml/zen_server/dashboard/assets/MarkdownVisualization-ylXaAxev.js +14 -0
- zenml/zen_server/dashboard/assets/NestedCollapsible-aK5ojKoF.js +1 -0
- zenml/zen_server/dashboard/assets/NumberBox-Dtp3J6g5.js +1 -0
- zenml/zen_server/dashboard/assets/Partials-CqZp5NMX.js +1 -0
- zenml/zen_server/dashboard/assets/PasswordChecker-B0nadgh6.js +1 -0
- zenml/zen_server/dashboard/assets/ProBadge-B4tRUYve.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-CZuP29Qz.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bd7GUQ1_.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-mstdqzsS.js +1 -0
- zenml/zen_server/dashboard/assets/RunSelector-CsruSB4i.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-DxxtWVYz.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-D6tPxyqw.js +1 -0
- zenml/zen_server/dashboard/assets/SecretTooltip-CLzJIYW_.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-Yn50ooBC.js +1 -0
- zenml/zen_server/dashboard/assets/StackList-U537qoYd.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-CNv-eTYM.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-jEIevzVf.js +1 -0
- zenml/zen_server/dashboard/assets/UpdatePasswordSchemas-C16GW-kX.js +1 -0
- zenml/zen_server/dashboard/assets/UsageReason-Bf2tzhv1.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-D6i-AP1K.js +1 -0
- zenml/zen_server/dashboard/assets/acp-DOsXjFc7.webp +0 -0
- zenml/zen_server/dashboard/assets/adam-e-y0WnB_.webp +0 -0
- zenml/zen_server/dashboard/assets/alex-DcCuDHPg.webp +0 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-DUti43aF.js +1 -0
- zenml/zen_server/dashboard/assets/baris-C0ZrZ10g.webp +0 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/check-circle-jNbX5-sR.js +1 -0
- zenml/zen_server/dashboard/assets/chevron-down-6JyMkfjR.js +1 -0
- zenml/zen_server/dashboard/assets/chevron-right-double-D7ojK9Co.js +1 -0
- zenml/zen_server/dashboard/assets/cloud-squares-DeRLMopf.svg +43 -0
- zenml/zen_server/dashboard/assets/code-browser-CUFUIHfp.js +1 -0
- zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
- zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
- zenml/zen_server/dashboard/assets/connectors-video-C9qY4syJ.svg +21 -0
- zenml/zen_server/dashboard/assets/copy-C8XQA2Ug.js +1 -0
- zenml/zen_server/dashboard/assets/create-stack-Ch2WPs9U.js +1 -0
- zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-Byf9hTjA.js +1 -0
- zenml/zen_server/dashboard/assets/docker-BdA9vrnW.js +1 -0
- zenml/zen_server/dashboard/assets/dots-horizontal-otGBOSDJ.js +1 -0
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-BZqKBPBF.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-CFtm4BA7.js +1 -0
- zenml/zen_server/dashboard/assets/hamza-NKKOZz1I.webp +0 -0
- zenml/zen_server/dashboard/assets/help-Cc9bBIJH.js +1 -0
- zenml/zen_server/dashboard/assets/index-CE0aQlv8.js +55 -0
- zenml/zen_server/dashboard/assets/index-CtdYkjUi.js +1 -0
- zenml/zen_server/dashboard/assets/index-CyBKZcpO.js +1 -0
- zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
- zenml/zen_server/dashboard/assets/index-Uu49AX48.js +1 -0
- zenml/zen_server/dashboard/assets/index-v6gQjDEo.js +1 -0
- zenml/zen_server/dashboard/assets/index.esm-Dy6Z9Ung.js +1 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
- zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
- zenml/zen_server/dashboard/assets/kubernetes-B2wmAJ1d.js +1 -0
- zenml/zen_server/dashboard/assets/layout-BtHBmE4w.js +1 -0
- zenml/zen_server/dashboard/assets/link-external-b9AXw_sW.js +1 -0
- zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
- zenml/zen_server/dashboard/assets/login-mutation-DNDVp_2H.js +1 -0
- zenml/zen_server/dashboard/assets/logs-WMSM52RF.js +1 -0
- zenml/zen_server/dashboard/assets/mcp-Cb1aMeoq.webp +0 -0
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/not-found-Bmup4ctE.js +1 -0
- zenml/zen_server/dashboard/assets/package-C6uypY4h.js +1 -0
- zenml/zen_server/dashboard/assets/page--XLMzHrn.js +1 -0
- zenml/zen_server/dashboard/assets/page-ANYGfEUL.js +1 -0
- zenml/zen_server/dashboard/assets/page-B5Sr8pib.js +1 -0
- zenml/zen_server/dashboard/assets/page-BC27C_OI.js +2 -0
- zenml/zen_server/dashboard/assets/page-BNxYrN0q.js +1 -0
- zenml/zen_server/dashboard/assets/page-BYJfqgLN.js +1 -0
- zenml/zen_server/dashboard/assets/page-B_0XkV48.js +1 -0
- zenml/zen_server/dashboard/assets/page-BrmJp1Wt.js +1 -0
- zenml/zen_server/dashboard/assets/page-C2nU3Gxn.js +1 -0
- zenml/zen_server/dashboard/assets/page-C70wZtV2.js +1 -0
- zenml/zen_server/dashboard/assets/page-CHRn1fQm.js +1 -0
- zenml/zen_server/dashboard/assets/page-CWr96ZKN.js +1 -0
- zenml/zen_server/dashboard/assets/page-CXAbSyp9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CaeI9ptC.js +1 -0
- zenml/zen_server/dashboard/assets/page-Cc8ZEuj4.js +1 -0
- zenml/zen_server/dashboard/assets/page-CltCNL0T.js +1 -0
- zenml/zen_server/dashboard/assets/page-CmlYj7Nl.js +1 -0
- zenml/zen_server/dashboard/assets/page-D6Ev5P8V.js +1 -0
- zenml/zen_server/dashboard/assets/page-D9Oh05fl.js +1 -0
- zenml/zen_server/dashboard/assets/page-DGlm1RVc.js +1 -0
- zenml/zen_server/dashboard/assets/page-DN4BVIOL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dif8CWyZ.js +1 -0
- zenml/zen_server/dashboard/assets/page-DlIi5ThM.js +1 -0
- zenml/zen_server/dashboard/assets/page-DoW7YxTu.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dth9X1Ih.js +1 -0
- zenml/zen_server/dashboard/assets/page-DweqqCkF.js +1 -0
- zenml/zen_server/dashboard/assets/page-DyOJ_pq3.js +1 -0
- zenml/zen_server/dashboard/assets/page-Hn8q9iJZ.js +1 -0
- zenml/zen_server/dashboard/assets/page-IhckKFnD.js +6 -0
- zenml/zen_server/dashboard/assets/page-LyZ_l8vR.js +1 -0
- zenml/zen_server/dashboard/assets/page-PamGpk0j.js +1 -0
- zenml/zen_server/dashboard/assets/page-PxOWfKgF.js +2 -0
- zenml/zen_server/dashboard/assets/persist-DeXRG61d.js +1 -0
- zenml/zen_server/dashboard/assets/persist-vP0-Xl4f.js +1 -0
- zenml/zen_server/dashboard/assets/plus-tf1V2hTJ.js +1 -0
- zenml/zen_server/dashboard/assets/refresh-BjOeWlEq.js +1 -0
- zenml/zen_server/dashboard/assets/repos-video-D8kpu60k.svg +9 -0
- zenml/zen_server/dashboard/assets/rocket-DjT2cDvG.js +1 -0
- zenml/zen_server/dashboard/assets/service-DH_oUqQj.js +2 -0
- zenml/zen_server/dashboard/assets/settings_preview-0JLrRgHP.webp +0 -0
- zenml/zen_server/dashboard/assets/sharedSchema-Bw1_Wa7l.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-B_0R_fd6.js +1 -0
- zenml/zen_server/dashboard/assets/stefan-B08Ftbba.webp +0 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/tick-circle-BEX_Tp4v.js +1 -0
- zenml/zen_server/dashboard/assets/tour-cover-BYfeen6M.webp +0 -0
- zenml/zen_server/dashboard/assets/trash-arLUMWMS.js +1 -0
- zenml/zen_server/dashboard/assets/update-server-settings-mutation-D9qYhfaN.js +1 -0
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-Dh93fvh0.js +1 -0
- zenml/zen_server/dashboard/assets/zod-BwEbpOxH.js +1 -0
- zenml/zen_server/dashboard/index.html +19 -1
- zenml/zen_server/deploy/__init__.py +7 -16
- zenml/zen_server/deploy/base_provider.py +49 -78
- zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
- zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +48 -66
- zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +78 -62
- zenml/zen_server/deploy/deployer.py +94 -175
- zenml/zen_server/deploy/deployment.py +23 -17
- zenml/zen_server/deploy/docker/docker_provider.py +15 -31
- zenml/zen_server/deploy/docker/docker_zen_server.py +30 -35
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +3 -13
- zenml/zen_server/deploy/helm/templates/NOTES.txt +23 -7
- zenml/zen_server/deploy/helm/templates/_environment.tpl +175 -23
- zenml/zen_server/deploy/helm/templates/server-db-job.yaml +45 -18
- zenml/zen_server/deploy/helm/templates/server-db-pvc.yaml +25 -0
- zenml/zen_server/deploy/helm/templates/server-deployment.yaml +22 -6
- zenml/zen_server/deploy/helm/templates/server-secret.yaml +11 -10
- zenml/zen_server/deploy/helm/values.yaml +210 -28
- zenml/zen_server/exceptions.py +20 -1
- zenml/zen_server/feature_gate/__init__.py +13 -0
- zenml/zen_server/feature_gate/endpoint_utils.py +61 -0
- zenml/zen_server/feature_gate/feature_gate_interface.py +49 -0
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +125 -0
- zenml/zen_server/jwt.py +64 -32
- zenml/zen_server/rate_limit.py +200 -0
- zenml/zen_server/rbac/endpoint_utils.py +92 -9
- zenml/zen_server/rbac/models.py +21 -17
- zenml/zen_server/rbac/rbac_sql_zen_store.py +175 -0
- zenml/zen_server/rbac/utils.py +71 -30
- zenml/zen_server/rbac/zenml_cloud_rbac.py +13 -188
- zenml/zen_server/routers/actions_endpoints.py +324 -0
- zenml/zen_server/routers/artifact_version_endpoints.py +28 -2
- zenml/zen_server/routers/auth_endpoints.py +249 -131
- zenml/zen_server/routers/code_repositories_endpoints.py +1 -0
- zenml/zen_server/routers/devices_endpoints.py +56 -49
- zenml/zen_server/routers/event_source_endpoints.py +327 -0
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/model_versions_endpoints.py +59 -0
- zenml/zen_server/routers/models_endpoints.py +7 -1
- zenml/zen_server/routers/pipeline_builds_endpoints.py +6 -1
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +35 -0
- zenml/zen_server/routers/pipelines_endpoints.py +19 -32
- zenml/zen_server/routers/plugin_endpoints.py +107 -0
- zenml/zen_server/routers/run_templates_endpoints.py +212 -0
- zenml/zen_server/routers/runs_endpoints.py +91 -32
- zenml/zen_server/routers/schedule_endpoints.py +1 -0
- zenml/zen_server/routers/secrets_endpoints.py +4 -2
- zenml/zen_server/routers/server_endpoints.py +186 -4
- zenml/zen_server/routers/service_connectors_endpoints.py +56 -0
- zenml/zen_server/routers/service_endpoints.py +180 -0
- zenml/zen_server/routers/stack_components_endpoints.py +2 -1
- zenml/zen_server/routers/stack_deployment_endpoints.py +164 -0
- zenml/zen_server/routers/steps_endpoints.py +21 -12
- zenml/zen_server/routers/triggers_endpoints.py +336 -0
- zenml/zen_server/routers/users_endpoints.py +280 -45
- zenml/zen_server/routers/webhook_endpoints.py +127 -0
- zenml/zen_server/routers/workspaces_endpoints.py +220 -185
- zenml/zen_server/secure_headers.py +120 -0
- zenml/{new → zen_server/template_execution}/__init__.py +1 -1
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +42 -0
- zenml/zen_server/template_execution/utils.py +474 -0
- zenml/zen_server/template_execution/workload_manager_interface.py +92 -0
- zenml/zen_server/utils.py +374 -74
- zenml/zen_server/zen_server_api.py +299 -52
- zenml/zen_stores/base_zen_store.py +90 -58
- zenml/zen_stores/migrations/alembic.py +22 -9
- zenml/zen_stores/migrations/env.py +2 -2
- zenml/zen_stores/migrations/utils.py +731 -0
- zenml/zen_stores/migrations/versions/0.21.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.21.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.22.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.23.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.30.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.31.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.31.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.32.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.32.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.33.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.34.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.35.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.35.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.36.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.36.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.37.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.38.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.39.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.39.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.41.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.42.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.42.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.43.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.1_release_0_45_1.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.4_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.5_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.6_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.46.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.46.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.47.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.50.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.51.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.52.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.53.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.53.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.54.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.54.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.55.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.55.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.3_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.4_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.5_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.3_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.4_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0.rc1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0.rc2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.60.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.72.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
- zenml/zen_stores/migrations/versions/03742aa7fdd7_add_secrets.py +1 -0
- zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +94 -0
- zenml/zen_stores/migrations/versions/0b06faa59c93_add_service_connectors.py +1 -0
- zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
- zenml/zen_stores/migrations/versions/0e4735b23577_increase_pipeline_spec_field_length.py +1 -0
- zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +6 -3
- zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +2 -1
- zenml/zen_stores/migrations/versions/14d687c8fa1c_rename_model_config_to_model_version.py +1 -0
- zenml/zen_stores/migrations/versions/19f27d5b234e_add_build_and_deployment_tables.py +1 -0
- zenml/zen_stores/migrations/versions/1a9a9d2a836d_admin_users.py +56 -0
- zenml/zen_stores/migrations/versions/1ac1b9c04da1_make_secrets_values_optional.py +1 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/1d74e596abb8_add_run_once_start_time_to_schedule.py +36 -0
- zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
- zenml/zen_stores/migrations/versions/248dfd320b68_update_size_of_flavor_config_schema.py +1 -0
- zenml/zen_stores/migrations/versions/25155145c545_separate_actions_and_triggers.py +228 -0
- zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
- zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +9 -10
- zenml/zen_stores/migrations/versions/2d201872e23c_remove_db_dependency_loop.py +29 -0
- zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +4 -3
- zenml/zen_stores/migrations/versions/389046140cad_data_versioning.py +1 -0
- zenml/zen_stores/migrations/versions/3944116bbd56_rename_project_to_workspace.py +1 -0
- zenml/zen_stores/migrations/versions/3b68abe58f44_add_model_watchtower_entities.py +1 -0
- zenml/zen_stores/migrations/versions/3c5a367730c2_add_environment_info_to_runs.py +1 -0
- zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
- zenml/zen_stores/migrations/versions/43a86093b60e_add_labels_for_stack_components.py +1 -0
- zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +123 -0
- zenml/zen_stores/migrations/versions/479103df60b6_add_triggers.py +162 -0
- zenml/zen_stores/migrations/versions/4a3087070f4e_add_step_source_code.py +1 -0
- zenml/zen_stores/migrations/versions/4c41c0ca42db_add_code_repository_table.py +1 -0
- zenml/zen_stores/migrations/versions/4d688d8f7aff_rename_model_version_to_model.py +1 -0
- zenml/zen_stores/migrations/versions/4e1972485075_endpoint_artifact_deployment_artifact.py +1 -0
- zenml/zen_stores/migrations/versions/4f66af55fbb9_rename_model_config_model_to_model_.py +1 -0
- zenml/zen_stores/migrations/versions/5330ba58bf20_rename_tables_and_foreign_keys.py +8 -9
- zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +4 -2
- zenml/zen_stores/migrations/versions/5cc3f41cf048_add_save_models_to_registry.py +1 -0
- zenml/zen_stores/migrations/versions/6119cd9b93c2_tags_table.py +1 -0
- zenml/zen_stores/migrations/versions/623a234c11f5_add_sdk_docs_url_to_flavors.py +1 -0
- zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +5 -4
- zenml/zen_stores/migrations/versions/6a28c4fd0ef2_add_caching_info.py +1 -0
- zenml/zen_stores/migrations/versions/6f707b385dc1_fix_model_artifacts.py +1 -0
- zenml/zen_stores/migrations/versions/722392c91006_make_is_service_account_mandatory.py +1 -0
- zenml/zen_stores/migrations/versions/72675226b2de_unique_users.py +31 -0
- zenml/zen_stores/migrations/versions/72722dee4686_track_server_version.py +1 -0
- zenml/zen_stores/migrations/versions/7280c14811d6_use_text_type.py +1 -0
- zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +4 -2
- zenml/zen_stores/migrations/versions/729263e47b55_fix_external_input_artifacts.py +1 -0
- zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +3 -2
- zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +4 -2
- zenml/zen_stores/migrations/versions/76a7b9451ccd_add_build_template_deployment_id.py +52 -0
- zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +9 -7
- zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +7 -7
- zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
- zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +3 -2
- zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +2 -1
- zenml/zen_stores/migrations/versions/86fa52918b54_remove_teams_and_roles.py +1 -0
- zenml/zen_stores/migrations/versions/8a64fbfecda0_add_num_outputs_to_run_step.py +1 -0
- zenml/zen_stores/migrations/versions/8ed03137cacc_polymorthic_run_metadata.py +1 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
- zenml/zen_stores/migrations/versions/93cbda80a732_add_service_accounts.py +1 -0
- zenml/zen_stores/migrations/versions/979eff8fc4b1_add_code_repo_description_and_logo_url.py +1 -0
- zenml/zen_stores/migrations/versions/9971237fa937_artifact_visualizations.py +1 -0
- zenml/zen_stores/migrations/versions/9d8020441014_increase_step_configuration_length.py +1 -0
- zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
- zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +3 -2
- zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +5 -4
- zenml/zen_stores/migrations/versions/ade72effebaf_added_logs_table.py +1 -0
- zenml/zen_stores/migrations/versions/alembic_start.py +2 -1
- zenml/zen_stores/migrations/versions/b4eccf34dfa3_add_hub_token_to_user_model.py +1 -0
- zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
- zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/migrations/versions/c1b18cec3a48_increase_length_on_flavor_config_schema.py +1 -0
- zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
- zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
- zenml/zen_stores/migrations/versions/cc9894cb58aa_add_user_metadata.py +41 -0
- zenml/zen_stores/migrations/versions/ccd68b7825ae_add_status_to_pipeline_and_step_run.py +1 -0
- zenml/zen_stores/migrations/versions/d02b3d3464cf_add_orchestrator_run_id_column.py +1 -0
- zenml/zen_stores/migrations/versions/d26471b6fe8f_update_build_filtering.py +1 -0
- zenml/zen_stores/migrations/versions/d7b3acf9aa46_create_schedule_table.py +1 -0
- zenml/zen_stores/migrations/versions/e1d66d91a099_add_stack_and_component_spec_paths_to_.py +1 -0
- zenml/zen_stores/migrations/versions/e5225281b4d3_add_connector_skew_tolerance.py +1 -0
- zenml/zen_stores/migrations/versions/e65aa6708ff7_pipeline_versioning.py +1 -0
- zenml/zen_stores/migrations/versions/ec0d785ca296_create_run_metadata_table.py +1 -0
- zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +119 -0
- zenml/zen_stores/migrations/versions/f3b3964e3a0f_add_oauth_devices.py +1 -0
- zenml/zen_stores/migrations/versions/f49904a80aa7_increase_length_of_artifact_table_sources.py +1 -0
- zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +5 -4
- zenml/zen_stores/rest_zen_store.py +1326 -305
- zenml/zen_stores/schemas/__init__.py +22 -3
- zenml/zen_stores/schemas/action_schemas.py +192 -0
- zenml/zen_stores/schemas/api_key_schemas.py +23 -10
- zenml/zen_stores/schemas/artifact_schemas.py +112 -49
- zenml/zen_stores/schemas/artifact_visualization_schemas.py +17 -8
- zenml/zen_stores/schemas/base_schemas.py +27 -0
- zenml/zen_stores/schemas/code_repository_schemas.py +25 -10
- zenml/zen_stores/schemas/component_schemas.py +74 -11
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/device_schemas.py +29 -15
- zenml/zen_stores/schemas/event_source_schemas.py +188 -0
- zenml/zen_stores/schemas/flavor_schemas.py +19 -9
- zenml/zen_stores/schemas/logs_schemas.py +12 -6
- zenml/zen_stores/schemas/model_schemas.py +192 -139
- zenml/zen_stores/schemas/pipeline_build_schemas.py +16 -16
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +58 -17
- zenml/zen_stores/schemas/pipeline_run_schemas.py +170 -35
- zenml/zen_stores/schemas/pipeline_schemas.py +51 -33
- zenml/zen_stores/schemas/run_metadata_schemas.py +28 -78
- zenml/zen_stores/schemas/run_template_schemas.py +267 -0
- zenml/zen_stores/schemas/schedule_schema.py +15 -5
- zenml/zen_stores/schemas/secret_schemas.py +18 -10
- zenml/zen_stores/schemas/server_settings_schemas.py +129 -0
- zenml/zen_stores/schemas/service_connector_schemas.py +13 -6
- zenml/zen_stores/schemas/service_schemas.py +258 -0
- zenml/zen_stores/schemas/stack_schemas.py +23 -6
- zenml/zen_stores/schemas/step_run_schemas.py +132 -41
- zenml/zen_stores/schemas/tag_schemas.py +31 -50
- zenml/zen_stores/schemas/trigger_schemas.py +316 -0
- zenml/zen_stores/schemas/user_schemas.py +66 -23
- zenml/zen_stores/schemas/utils.py +112 -0
- zenml/zen_stores/schemas/workspace_schemas.py +36 -19
- zenml/zen_stores/secrets_stores/aws_secrets_store.py +41 -32
- zenml/zen_stores/secrets_stores/azure_secrets_store.py +20 -23
- zenml/zen_stores/secrets_stores/base_secrets_store.py +80 -12
- zenml/zen_stores/secrets_stores/gcp_secrets_store.py +42 -33
- zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +7 -11
- zenml/zen_stores/secrets_stores/secrets_store_interface.py +1 -0
- zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +15 -8
- zenml/zen_stores/secrets_stores/sql_secrets_store.py +8 -9
- zenml/zen_stores/sql_zen_store.py +4062 -799
- zenml/zen_stores/template_utils.py +263 -0
- zenml/zen_stores/zen_store_interface.py +614 -44
- zenml_nightly-0.72.0.dev20250116.dist-info/METADATA +486 -0
- zenml_nightly-0.72.0.dev20250116.dist-info/RECORD +1294 -0
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/WHEEL +1 -1
- CLA.md +0 -110
- CODE-OF-CONDUCT.md +0 -132
- CONTRIBUTING.md +0 -260
- README.md +0 -304
- RELEASE_NOTES.md +0 -3919
- ROADMAP.md +0 -5
- SECURITY.md +0 -15
- zenml/_hub/client.py +0 -285
- zenml/_hub/constants.py +0 -21
- zenml/_hub/utils.py +0 -80
- zenml/api.py +0 -61
- zenml/cli/hub.py +0 -1115
- zenml/cli/stack_recipes.py +0 -469
- zenml/integrations/gcp/google_cloud_function.py +0 -187
- zenml/integrations/gcp/google_cloud_scheduler.py +0 -83
- zenml/integrations/gcp/orchestrators/vertex_scheduler/main.py +0 -91
- zenml/integrations/gcp/orchestrators/vertex_scheduler/requirements.txt +0 -2
- zenml/integrations/kserve/__init__.py +0 -57
- zenml/integrations/kserve/custom_deployer/zenml_custom_model.py +0 -175
- zenml/integrations/kserve/flavors/kserve_model_deployer_flavor.py +0 -137
- zenml/integrations/kserve/model_deployers/kserve_model_deployer.py +0 -1003
- zenml/integrations/kserve/secret_schemas/secret_schemas.py +0 -65
- zenml/integrations/kserve/services/kserve_deployment.py +0 -596
- zenml/integrations/kserve/steps/__init__.py +0 -22
- zenml/integrations/kserve/steps/kserve_deployer.py +0 -472
- zenml/integrations/kserve/steps/kserve_step_utils.py +0 -293
- zenml/integrations/kubeflow/utils.py +0 -96
- zenml/lineage_graph/lineage_graph.py +0 -244
- zenml/lineage_graph/node/artifact_node.py +0 -52
- zenml/lineage_graph/node/step_node.py +0 -41
- zenml/models/v2/base/internal.py +0 -37
- zenml/models/v2/base/update.py +0 -40
- zenml/models/v2/misc/hub_plugin_models.py +0 -79
- zenml/new/pipelines/deserialization_utils.py +0 -291
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/new/pipelines/pipeline_decorator.py +0 -109
- zenml/new/steps/step_decorator.py +0 -160
- zenml/pipelines/base_pipeline.py +0 -274
- zenml/post_execution/pipeline.py +0 -58
- zenml/post_execution/pipeline_run.py +0 -55
- zenml/services/service_registry.py +0 -214
- zenml/services/terraform/__init__.py +0 -14
- zenml/services/terraform/terraform_service.py +0 -441
- zenml/steps/step_environment.py +0 -108
- zenml/steps/step_output.py +0 -36
- zenml/utils/mlstacks_utils.py +0 -635
- zenml/utils/terraform_utils.py +0 -42
- zenml/zen_server/dashboard/_redirects +0 -1
- zenml/zen_server/dashboard/asset-manifest.json +0 -131
- zenml/zen_server/dashboard/manifest.json +0 -25
- zenml/zen_server/dashboard/precache-manifest.c139638dcc4d9d3425353266447a2fad.js +0 -462
- zenml/zen_server/dashboard/robots.txt +0 -2
- zenml/zen_server/dashboard/service-worker.js +0 -39
- zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css +0 -16
- zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css.map +0 -1
- zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css +0 -2
- zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css.map +0 -1
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js +0 -3
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.LICENSE.txt +0 -95
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.map +0 -1
- zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js +0 -2
- zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js.map +0 -1
- zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js +0 -2
- zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js.map +0 -1
- zenml/zen_server/dashboard/static/media/AlertTriangle.28aee535.svg +0 -5
- zenml/zen_server/dashboard/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
- zenml/zen_server/dashboard/static/media/Back.86c23a22.svg +0 -4
- zenml/zen_server/dashboard/static/media/BookOpen.5cb101ff.svg +0 -4
- zenml/zen_server/dashboard/static/media/BoundingBox.1eb98717.svg +0 -10
- zenml/zen_server/dashboard/static/media/Burger.9b1c67d7.svg +0 -3
- zenml/zen_server/dashboard/static/media/Cached.2381fb8d.svg +0 -1
- zenml/zen_server/dashboard/static/media/Calendar.356e11c7.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChartBarHorizontal.0247447b.svg +0 -6
- zenml/zen_server/dashboard/static/media/ChartLine.0d79e18d.svg +0 -4
- zenml/zen_server/dashboard/static/media/ChatDots.2e1c9211.svg +0 -6
- zenml/zen_server/dashboard/static/media/Check.dad6beb2.svg +0 -3
- zenml/zen_server/dashboard/static/media/CheckCircleFilled.c19566d0.svg +0 -3
- zenml/zen_server/dashboard/static/media/Checkbox.af50e31e.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronDown.f860ce32.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronDownLight.6642d756.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronLeft.f6edfcdb.svg +0 -3
- zenml/zen_server/dashboard/static/media/CircleCheck.f98fd6ca.svg +0 -1
- zenml/zen_server/dashboard/static/media/Clock.ffc9de95.svg +0 -3
- zenml/zen_server/dashboard/static/media/Close.74e9efbc.svg +0 -5
- zenml/zen_server/dashboard/static/media/CloseWithBorder.6960930a.svg +0 -3
- zenml/zen_server/dashboard/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
- zenml/zen_server/dashboard/static/media/CloudArrowUp.0aecb235.svg +0 -6
- zenml/zen_server/dashboard/static/media/Code.ef0f33b5.svg +0 -3
- zenml/zen_server/dashboard/static/media/Config.0be63f8a.svg +0 -1
- zenml/zen_server/dashboard/static/media/Connector.9fd46ef1.svg +0 -10
- zenml/zen_server/dashboard/static/media/Copy.36e2112a.svg +0 -1
- zenml/zen_server/dashboard/static/media/Dashboard.d05787e0.svg +0 -3
- zenml/zen_server/dashboard/static/media/Data.b1c3b5f8.svg +0 -3
- zenml/zen_server/dashboard/static/media/Delete.3c361b28.svg +0 -8
- zenml/zen_server/dashboard/static/media/Docs.7541d478.svg +0 -7
- zenml/zen_server/dashboard/static/media/Download.fba04d87.svg +0 -5
- zenml/zen_server/dashboard/static/media/Edit.490eb294.svg +0 -6
- zenml/zen_server/dashboard/static/media/EmptyRightArrow.23749d01.svg +0 -3
- zenml/zen_server/dashboard/static/media/Example.6396cd37.svg +0 -5
- zenml/zen_server/dashboard/static/media/Extension.1394cd4a.svg +0 -3
- zenml/zen_server/dashboard/static/media/Eye.d9e4ee62.svg +0 -4
- zenml/zen_server/dashboard/static/media/Failed.0213c1a0.svg +0 -1
- zenml/zen_server/dashboard/static/media/FileText.1f15bacd.svg +0 -7
- zenml/zen_server/dashboard/static/media/Filter.ab6b9c0d.svg +0 -3
- zenml/zen_server/dashboard/static/media/Folders.12b29887.svg +0 -5
- zenml/zen_server/dashboard/static/media/FunnelFill.6df4c143.svg +0 -3
- zenml/zen_server/dashboard/static/media/GitCommit.7dd9c2aa.svg +0 -5
- zenml/zen_server/dashboard/static/media/GitHub_Logo.cefc2023.png +0 -0
- zenml/zen_server/dashboard/static/media/Graph.2c63a892.svg +0 -11
- zenml/zen_server/dashboard/static/media/History.08329240.svg +0 -3
- zenml/zen_server/dashboard/static/media/Home.0843b0d5.svg +0 -3
- zenml/zen_server/dashboard/static/media/ImageBuilder.ea762d9c.svg +0 -6
- zenml/zen_server/dashboard/static/media/InProgress.304a0edc.svg +0 -1
- zenml/zen_server/dashboard/static/media/Info.9fe10c5c.svg +0 -3
- zenml/zen_server/dashboard/static/media/KeyboardReturn.491afbe3.svg +0 -3
- zenml/zen_server/dashboard/static/media/Link.72bbb55d.svg +0 -4
- zenml/zen_server/dashboard/static/media/Lock.30f5e1fe.svg +0 -5
- zenml/zen_server/dashboard/static/media/Lock2.a769ea52.svg +0 -3
- zenml/zen_server/dashboard/static/media/LockKey.92f21621.svg +0 -6
- zenml/zen_server/dashboard/static/media/Logs.8bf4d005.svg +0 -5
- zenml/zen_server/dashboard/static/media/MinusCircle.4188f418.svg +0 -4
- zenml/zen_server/dashboard/static/media/ModelRegistry.f0de050a.svg +0 -6
- zenml/zen_server/dashboard/static/media/MultiUser.a2ba7c67.svg +0 -10
- zenml/zen_server/dashboard/static/media/PaginationFirst.92628634.svg +0 -4
- zenml/zen_server/dashboard/static/media/PaginationLast.00d3c732.svg +0 -4
- zenml/zen_server/dashboard/static/media/PaginationNext.86158845.svg +0 -3
- zenml/zen_server/dashboard/static/media/PaginationPrev.60c18a88.svg +0 -3
- zenml/zen_server/dashboard/static/media/Pen.f2d831d4.svg +0 -6
- zenml/zen_server/dashboard/static/media/PhotoCamera.179d6d4c.svg +0 -3
- zenml/zen_server/dashboard/static/media/Pipeline.30d298b0.svg +0 -7
- zenml/zen_server/dashboard/static/media/Plus.5aa1c16b.svg +0 -3
- zenml/zen_server/dashboard/static/media/PlusCircle.92d860dd.svg +0 -5
- zenml/zen_server/dashboard/static/media/Repositories.71a36b8c.svg +0 -3
- zenml/zen_server/dashboard/static/media/RightArrow.f30d3871.svg +0 -29
- zenml/zen_server/dashboard/static/media/Rocket.63bf7b9d.svg +0 -3
- zenml/zen_server/dashboard/static/media/RocketLaunch.1bff2b59.svg +0 -6
- zenml/zen_server/dashboard/static/media/Rubik-Medium.c87313aa.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Run.daec4fb2.svg +0 -6
- zenml/zen_server/dashboard/static/media/Search.d1afcce5.svg +0 -4
- zenml/zen_server/dashboard/static/media/Settings.59ca73ae.svg +0 -4
- zenml/zen_server/dashboard/static/media/Share2.46c3ff66.svg +0 -3
- zenml/zen_server/dashboard/static/media/SignOut.6aa718c5.svg +0 -3
- zenml/zen_server/dashboard/static/media/SimplePlus.5cf7ec20.svg +0 -3
- zenml/zen_server/dashboard/static/media/SingleUser.bef3a095.svg +0 -4
- zenml/zen_server/dashboard/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Stack.19b604ac.svg +0 -5
- zenml/zen_server/dashboard/static/media/StackComponent.b1ba90b5.svg +0 -4
- zenml/zen_server/dashboard/static/media/Star.f0c25022.svg +0 -9
- zenml/zen_server/dashboard/static/media/StarOutline.94ca8cd9.svg +0 -3
- zenml/zen_server/dashboard/static/media/Storefront.4b4796fe.svg +0 -3
- zenml/zen_server/dashboard/static/media/Stream.543e3039.svg +0 -3
- zenml/zen_server/dashboard/static/media/SupportAgent.510ddf1f.svg +0 -8
- zenml/zen_server/dashboard/static/media/Table.77033750.svg +0 -6
- zenml/zen_server/dashboard/static/media/Tool.d5785486.svg +0 -3
- zenml/zen_server/dashboard/static/media/UserPlus.741a99d7.svg +0 -6
- zenml/zen_server/dashboard/static/media/Verified.0625b2a0.svg +0 -3
- zenml/zen_server/dashboard/static/media/addNew.4fb6c939.svg +0 -8
- zenml/zen_server/dashboard/static/media/arrowClose.cbd53f3f.svg +0 -3
- zenml/zen_server/dashboard/static/media/arrowOpen.6ceef0af.svg +0 -3
- zenml/zen_server/dashboard/static/media/check_small.30bc0138.svg +0 -3
- zenml/zen_server/dashboard/static/media/circleArrowSideClose.98d6013e.svg +0 -18
- zenml/zen_server/dashboard/static/media/circleArrowSideOpen.63653df6.svg +0 -18
- zenml/zen_server/dashboard/static/media/image.104fd14b.png +0 -0
- zenml/zen_server/dashboard/static/media/imageAddIcon.e83004a9.svg +0 -7
- zenml/zen_server/dashboard/static/media/logo.93333e5c.svg +0 -1
- zenml/zen_server/dashboard/static/media/logo_small.4204397d.svg +0 -3
- zenml/zen_server/dashboard/static/media/logo_white.d4b4414e.svg +0 -20
- zenml/zen_server/dashboard/static/media/notConnected.5e2c8ea7.svg +0 -8
- zenml/zen_server/dashboard/static/media/plugin-fallback.72c294e6.svg +0 -6
- zenml/zen_server/dashboard/static/media/share.bcd998b0.svg +0 -5
- zenml/zen_server/dashboard/static/media/stars.08a9b19a.svg +0 -8
- zenml/zen_server/deploy/terraform/__init__.py +0 -41
- zenml/zen_server/deploy/terraform/providers/__init__.py +0 -14
- zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
- zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -332
- zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
- zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -179
- zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
- zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -111
- zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
- zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
- zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
- zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
- zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -188
- zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -111
- zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
- zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -183
- zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -122
- zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -255
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -97
- zenml_nightly-0.55.0.dev20240124.dist-info/METADATA +0 -438
- zenml_nightly-0.55.0.dev20240124.dist-info/RECORD +0 -1072
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/entry_points.txt +0 -0
zenml/cli/stack.py
CHANGED
@@ -12,57 +12,71 @@
|
|
12
12
|
# or implied. See the License for the specific language governing
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""CLI for manipulating ZenML local and global config file."""
|
15
|
+
|
15
16
|
import getpass
|
16
|
-
import
|
17
|
-
|
18
|
-
|
17
|
+
import re
|
18
|
+
import time
|
19
|
+
import webbrowser
|
20
|
+
from datetime import datetime
|
21
|
+
from typing import (
|
22
|
+
TYPE_CHECKING,
|
23
|
+
Any,
|
24
|
+
Dict,
|
25
|
+
List,
|
26
|
+
Optional,
|
27
|
+
Set,
|
28
|
+
Union,
|
29
|
+
)
|
19
30
|
from uuid import UUID
|
20
31
|
|
21
32
|
import click
|
33
|
+
from rich.console import Console
|
34
|
+
from rich.markdown import Markdown
|
35
|
+
from rich.prompt import Confirm
|
36
|
+
from rich.style import Style
|
37
|
+
from rich.syntax import Syntax
|
22
38
|
|
23
39
|
import zenml
|
24
40
|
from zenml.analytics.enums import AnalyticsEvent
|
25
41
|
from zenml.analytics.utils import track_handler
|
26
42
|
from zenml.cli import utils as cli_utils
|
27
43
|
from zenml.cli.cli import TagGroup, cli
|
44
|
+
from zenml.cli.text_utils import OldSchoolMarkdownHeading
|
28
45
|
from zenml.cli.utils import (
|
29
46
|
_component_display_name,
|
30
|
-
confirmation,
|
31
|
-
declare,
|
32
|
-
error,
|
33
47
|
is_sorted_or_filtered,
|
34
48
|
list_options,
|
35
49
|
print_model_url,
|
36
50
|
print_page_info,
|
37
51
|
print_stacks_table,
|
38
|
-
verify_mlstacks_prerequisites_installation,
|
39
52
|
)
|
40
53
|
from zenml.client import Client
|
41
54
|
from zenml.console import console
|
42
|
-
from zenml.
|
43
|
-
|
44
|
-
|
45
|
-
|
55
|
+
from zenml.enums import (
|
56
|
+
CliCategories,
|
57
|
+
StackComponentType,
|
58
|
+
StackDeploymentProvider,
|
46
59
|
)
|
47
|
-
from zenml.enums import CliCategories, StackComponentType
|
48
60
|
from zenml.exceptions import (
|
49
61
|
IllegalOperationError,
|
50
|
-
ProvisioningError,
|
51
62
|
)
|
52
|
-
from zenml.io.fileio import rmtree
|
53
63
|
from zenml.logger import get_logger
|
54
|
-
from zenml.models import
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
64
|
+
from zenml.models import (
|
65
|
+
ComponentInfo,
|
66
|
+
ServiceConnectorInfo,
|
67
|
+
ServiceConnectorResourcesInfo,
|
68
|
+
StackFilter,
|
69
|
+
StackRequest,
|
70
|
+
)
|
71
|
+
from zenml.models.v2.core.service_connector import (
|
72
|
+
ServiceConnectorRequest,
|
73
|
+
ServiceConnectorResponse,
|
74
|
+
)
|
75
|
+
from zenml.service_connectors.service_connector_utils import (
|
76
|
+
get_resources_options_from_resource_model_for_full_stack,
|
77
|
+
)
|
78
|
+
from zenml.utils import requirements_utils
|
79
|
+
from zenml.utils.dashboard_utils import get_component_url, get_stack_url
|
66
80
|
from zenml.utils.yaml_utils import read_yaml, write_yaml
|
67
81
|
|
68
82
|
if TYPE_CHECKING:
|
@@ -92,7 +106,7 @@ def stack() -> None:
|
|
92
106
|
"artifact_store",
|
93
107
|
help="Name of the artifact store for this stack.",
|
94
108
|
type=str,
|
95
|
-
required=
|
109
|
+
required=False,
|
96
110
|
)
|
97
111
|
@click.option(
|
98
112
|
"-o",
|
@@ -100,7 +114,7 @@ def stack() -> None:
|
|
100
114
|
"orchestrator",
|
101
115
|
help="Name of the orchestrator for this stack.",
|
102
116
|
type=str,
|
103
|
-
required=
|
117
|
+
required=False,
|
104
118
|
)
|
105
119
|
@click.option(
|
106
120
|
"-c",
|
@@ -189,10 +203,24 @@ def stack() -> None:
|
|
189
203
|
help="Immediately set this stack as active.",
|
190
204
|
type=click.BOOL,
|
191
205
|
)
|
206
|
+
@click.option(
|
207
|
+
"-p",
|
208
|
+
"--provider",
|
209
|
+
help="Name of the cloud provider for this stack.",
|
210
|
+
type=click.Choice(["aws", "azure", "gcp"]),
|
211
|
+
required=False,
|
212
|
+
)
|
213
|
+
@click.option(
|
214
|
+
"-sc",
|
215
|
+
"--connector",
|
216
|
+
help="Name of the service connector for this stack.",
|
217
|
+
type=str,
|
218
|
+
required=False,
|
219
|
+
)
|
192
220
|
def register_stack(
|
193
221
|
stack_name: str,
|
194
|
-
artifact_store: str,
|
195
|
-
orchestrator: str,
|
222
|
+
artifact_store: Optional[str] = None,
|
223
|
+
orchestrator: Optional[str] = None,
|
196
224
|
container_registry: Optional[str] = None,
|
197
225
|
model_registry: Optional[str] = None,
|
198
226
|
step_operator: Optional[str] = None,
|
@@ -204,6 +232,8 @@ def register_stack(
|
|
204
232
|
data_validator: Optional[str] = None,
|
205
233
|
image_builder: Optional[str] = None,
|
206
234
|
set_stack: bool = False,
|
235
|
+
provider: Optional[str] = None,
|
236
|
+
connector: Optional[str] = None,
|
207
237
|
) -> None:
|
208
238
|
"""Register a stack.
|
209
239
|
|
@@ -222,44 +252,266 @@ def register_stack(
|
|
222
252
|
data_validator: Name of the data validator for this stack.
|
223
253
|
image_builder: Name of the new image builder for this stack.
|
224
254
|
set_stack: Immediately set this stack as active.
|
255
|
+
provider: Name of the cloud provider for this stack.
|
256
|
+
connector: Name of the service connector for this stack.
|
225
257
|
"""
|
226
|
-
|
227
|
-
|
258
|
+
if (provider is None and connector is None) and (
|
259
|
+
artifact_store is None or orchestrator is None
|
260
|
+
):
|
261
|
+
cli_utils.error(
|
262
|
+
"The only way to register a stack without specifying an "
|
263
|
+
"orchestrator and an artifact store is by using either a provider"
|
264
|
+
"(-p/--provider) or an existing service connector "
|
265
|
+
"(-sc/--connector). Please specify the artifact store and "
|
266
|
+
"the orchestrator or the service connector or cloud type settings."
|
267
|
+
)
|
228
268
|
|
229
|
-
|
269
|
+
client = Client()
|
230
270
|
|
231
|
-
|
232
|
-
|
271
|
+
if provider is not None or connector is not None:
|
272
|
+
if client.zen_store.is_local_store():
|
273
|
+
cli_utils.error(
|
274
|
+
"You are registering a stack using a service connector, but "
|
275
|
+
"this feature cannot be used with a local ZenML deployment. "
|
276
|
+
"ZenML needs to be accessible from the cloud provider to allow "
|
277
|
+
"the stack and its components to be registered automatically. "
|
278
|
+
"Please deploy ZenML in a remote environment as described in "
|
279
|
+
"the documentation: https://docs.zenml.io/getting-started/deploying-zenml "
|
280
|
+
"or use a managed ZenML Pro server instance for quick access "
|
281
|
+
"to this feature and more: https://www.zenml.io/pro"
|
282
|
+
)
|
233
283
|
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
components[StackComponentType.MODEL_DEPLOYER] = model_deployer
|
246
|
-
if model_registry:
|
247
|
-
components[StackComponentType.MODEL_REGISTRY] = model_registry
|
248
|
-
if step_operator:
|
249
|
-
components[StackComponentType.STEP_OPERATOR] = step_operator
|
250
|
-
if experiment_tracker:
|
251
|
-
components[
|
252
|
-
StackComponentType.EXPERIMENT_TRACKER
|
253
|
-
] = experiment_tracker
|
254
|
-
if container_registry:
|
255
|
-
components[
|
256
|
-
StackComponentType.CONTAINER_REGISTRY
|
257
|
-
] = container_registry
|
284
|
+
try:
|
285
|
+
client.get_stack(
|
286
|
+
name_id_or_prefix=stack_name,
|
287
|
+
allow_name_prefix_match=False,
|
288
|
+
)
|
289
|
+
cli_utils.error(
|
290
|
+
f"A stack with name `{stack_name}` already exists, "
|
291
|
+
"please use a different name."
|
292
|
+
)
|
293
|
+
except KeyError:
|
294
|
+
pass
|
258
295
|
|
296
|
+
labels: Dict[str, str] = {}
|
297
|
+
components: Dict[StackComponentType, List[Union[UUID, ComponentInfo]]] = {}
|
298
|
+
|
299
|
+
# Cloud Flow
|
300
|
+
created_objects: Set[str] = set()
|
301
|
+
service_connector: Optional[Union[UUID, ServiceConnectorInfo]] = None
|
302
|
+
if provider is not None and connector is None:
|
303
|
+
service_connector_response = None
|
304
|
+
use_auto_configure = False
|
259
305
|
try:
|
260
|
-
|
306
|
+
service_connector_response, _ = client.create_service_connector(
|
261
307
|
name=stack_name,
|
262
|
-
|
308
|
+
connector_type=provider,
|
309
|
+
register=False,
|
310
|
+
auto_configure=True,
|
311
|
+
verify=False,
|
312
|
+
)
|
313
|
+
except NotImplementedError:
|
314
|
+
cli_utils.warning(
|
315
|
+
f"The {provider.upper()} service connector libraries are not "
|
316
|
+
"installed properly. Please run `zenml integration install "
|
317
|
+
f"{provider}` and try again to enable auto-discovery of the "
|
318
|
+
"connection configuration."
|
319
|
+
)
|
320
|
+
except Exception:
|
321
|
+
pass
|
322
|
+
|
323
|
+
if service_connector_response:
|
324
|
+
use_auto_configure = Confirm.ask(
|
325
|
+
f"[bold]{provider.upper()} cloud service connector[/bold] "
|
326
|
+
"has detected connection credentials in your environment.\n"
|
327
|
+
"Would you like to use these credentials or create a new "
|
328
|
+
"configuration by providing connection details?",
|
329
|
+
default=True,
|
330
|
+
show_choices=True,
|
331
|
+
show_default=True,
|
332
|
+
)
|
333
|
+
|
334
|
+
connector_selected: Optional[int] = None
|
335
|
+
if not use_auto_configure:
|
336
|
+
service_connector_response = None
|
337
|
+
existing_connectors = client.list_service_connectors(
|
338
|
+
connector_type=provider, size=100
|
339
|
+
)
|
340
|
+
if existing_connectors.total:
|
341
|
+
connector_selected = cli_utils.multi_choice_prompt(
|
342
|
+
object_type=f"{provider.upper()} service connectors",
|
343
|
+
choices=[
|
344
|
+
[connector.name]
|
345
|
+
for connector in existing_connectors.items
|
346
|
+
],
|
347
|
+
headers=["Name"],
|
348
|
+
prompt_text=f"We found these {provider.upper()} service "
|
349
|
+
"connectors. Do you want to create a new one or use one "
|
350
|
+
"of the existing ones?",
|
351
|
+
default_choice="0",
|
352
|
+
allow_zero_be_a_new_object=True,
|
353
|
+
)
|
354
|
+
if use_auto_configure or connector_selected is None:
|
355
|
+
service_connector = _get_service_connector_info(
|
356
|
+
cloud_provider=provider,
|
357
|
+
connector_details=service_connector_response,
|
358
|
+
)
|
359
|
+
created_objects.add("service_connector")
|
360
|
+
else:
|
361
|
+
selected_connector = existing_connectors.items[connector_selected]
|
362
|
+
service_connector = selected_connector.id
|
363
|
+
connector = selected_connector.name
|
364
|
+
if isinstance(selected_connector.connector_type, str):
|
365
|
+
provider = selected_connector.connector_type
|
366
|
+
else:
|
367
|
+
provider = selected_connector.connector_type.connector_type
|
368
|
+
elif connector is not None:
|
369
|
+
service_connector_response = client.get_service_connector(connector)
|
370
|
+
service_connector = service_connector_response.id
|
371
|
+
if provider:
|
372
|
+
if service_connector_response.type != provider:
|
373
|
+
cli_utils.warning(
|
374
|
+
f"The service connector `{connector}` is not of type `{provider}`."
|
375
|
+
)
|
376
|
+
else:
|
377
|
+
provider = service_connector_response.type
|
378
|
+
|
379
|
+
if service_connector:
|
380
|
+
labels["zenml:wizard"] = "true"
|
381
|
+
if provider:
|
382
|
+
labels["zenml:provider"] = provider
|
383
|
+
resources_info = None
|
384
|
+
# explore the service connector
|
385
|
+
with console.status(
|
386
|
+
"Exploring resources available to the service connector...\n"
|
387
|
+
):
|
388
|
+
resources_info = (
|
389
|
+
get_resources_options_from_resource_model_for_full_stack(
|
390
|
+
connector_details=service_connector
|
391
|
+
)
|
392
|
+
)
|
393
|
+
if resources_info is None:
|
394
|
+
cli_utils.error(
|
395
|
+
f"Failed to fetch service connector resources information for {service_connector}..."
|
396
|
+
)
|
397
|
+
|
398
|
+
# create components
|
399
|
+
needed_components = (
|
400
|
+
(StackComponentType.ARTIFACT_STORE, artifact_store),
|
401
|
+
(StackComponentType.ORCHESTRATOR, orchestrator),
|
402
|
+
(StackComponentType.CONTAINER_REGISTRY, container_registry),
|
403
|
+
)
|
404
|
+
for component_type, preset_name in needed_components:
|
405
|
+
component_info: Optional[Union[UUID, ComponentInfo]] = None
|
406
|
+
if preset_name is not None:
|
407
|
+
component_response = client.get_stack_component(
|
408
|
+
component_type, preset_name
|
409
|
+
)
|
410
|
+
component_info = component_response.id
|
411
|
+
component_name = component_response.name
|
412
|
+
else:
|
413
|
+
if isinstance(service_connector, UUID):
|
414
|
+
# find existing components under same connector
|
415
|
+
if (
|
416
|
+
component_type
|
417
|
+
in resources_info.components_resources_info
|
418
|
+
):
|
419
|
+
existing_components = [
|
420
|
+
existing_response
|
421
|
+
for res_info in resources_info.components_resources_info[
|
422
|
+
component_type
|
423
|
+
]
|
424
|
+
for existing_response in res_info.connected_through_service_connector
|
425
|
+
]
|
426
|
+
|
427
|
+
# if some existing components are found - prompt user what to do
|
428
|
+
component_selected: Optional[int] = None
|
429
|
+
component_selected = cli_utils.multi_choice_prompt(
|
430
|
+
object_type=component_type.value.replace("_", " "),
|
431
|
+
choices=[
|
432
|
+
[
|
433
|
+
component.flavor_name,
|
434
|
+
component.name,
|
435
|
+
component.configuration or "",
|
436
|
+
component.connector_resource_id,
|
437
|
+
]
|
438
|
+
for component in existing_components
|
439
|
+
],
|
440
|
+
headers=[
|
441
|
+
"Type",
|
442
|
+
"Name",
|
443
|
+
"Configuration",
|
444
|
+
"Connected as",
|
445
|
+
],
|
446
|
+
prompt_text=f"We found these {component_type.value.replace('_', ' ')} "
|
447
|
+
"connected using the current service connector. Do you "
|
448
|
+
"want to create a new one or use existing one?",
|
449
|
+
default_choice="0",
|
450
|
+
allow_zero_be_a_new_object=True,
|
451
|
+
)
|
452
|
+
else:
|
453
|
+
component_selected = None
|
454
|
+
|
455
|
+
if component_selected is None:
|
456
|
+
component_info = _get_stack_component_info(
|
457
|
+
component_type=component_type.value,
|
458
|
+
cloud_provider=provider
|
459
|
+
or resources_info.connector_type,
|
460
|
+
resources_info=resources_info,
|
461
|
+
service_connector_index=0,
|
462
|
+
)
|
463
|
+
component_name = stack_name
|
464
|
+
created_objects.add(component_type.value)
|
465
|
+
else:
|
466
|
+
selected_component = existing_components[
|
467
|
+
component_selected
|
468
|
+
]
|
469
|
+
component_info = selected_component.id
|
470
|
+
component_name = selected_component.name
|
471
|
+
|
472
|
+
components[component_type] = [component_info]
|
473
|
+
if component_type == StackComponentType.ARTIFACT_STORE:
|
474
|
+
artifact_store = component_name
|
475
|
+
if component_type == StackComponentType.ORCHESTRATOR:
|
476
|
+
orchestrator = component_name
|
477
|
+
if component_type == StackComponentType.CONTAINER_REGISTRY:
|
478
|
+
container_registry = component_name
|
479
|
+
|
480
|
+
# normal flow once all components are defined
|
481
|
+
with console.status(f"Registering stack '{stack_name}'...\n"):
|
482
|
+
for component_type_, component_name_ in [
|
483
|
+
(StackComponentType.ARTIFACT_STORE, artifact_store),
|
484
|
+
(StackComponentType.ORCHESTRATOR, orchestrator),
|
485
|
+
(StackComponentType.ALERTER, alerter),
|
486
|
+
(StackComponentType.ANNOTATOR, annotator),
|
487
|
+
(StackComponentType.DATA_VALIDATOR, data_validator),
|
488
|
+
(StackComponentType.FEATURE_STORE, feature_store),
|
489
|
+
(StackComponentType.IMAGE_BUILDER, image_builder),
|
490
|
+
(StackComponentType.MODEL_DEPLOYER, model_deployer),
|
491
|
+
(StackComponentType.MODEL_REGISTRY, model_registry),
|
492
|
+
(StackComponentType.STEP_OPERATOR, step_operator),
|
493
|
+
(StackComponentType.EXPERIMENT_TRACKER, experiment_tracker),
|
494
|
+
(StackComponentType.CONTAINER_REGISTRY, container_registry),
|
495
|
+
]:
|
496
|
+
if component_name_ and component_type_ not in components:
|
497
|
+
components[component_type_] = [
|
498
|
+
client.get_stack_component(
|
499
|
+
component_type_, component_name_
|
500
|
+
).id
|
501
|
+
]
|
502
|
+
|
503
|
+
try:
|
504
|
+
created_stack = client.zen_store.create_stack(
|
505
|
+
stack=StackRequest(
|
506
|
+
user=client.active_user.id,
|
507
|
+
workspace=client.active_workspace.id,
|
508
|
+
name=stack_name,
|
509
|
+
components=components,
|
510
|
+
service_connectors=[service_connector]
|
511
|
+
if service_connector
|
512
|
+
else [],
|
513
|
+
labels=labels,
|
514
|
+
)
|
263
515
|
)
|
264
516
|
except (KeyError, IllegalOperationError) as err:
|
265
517
|
cli_utils.error(str(err))
|
@@ -267,6 +519,10 @@ def register_stack(
|
|
267
519
|
cli_utils.declare(
|
268
520
|
f"Stack '{created_stack.name}' successfully registered!"
|
269
521
|
)
|
522
|
+
cli_utils.print_stack_configuration(
|
523
|
+
stack=created_stack,
|
524
|
+
active=created_stack.id == client.active_stack_model.id,
|
525
|
+
)
|
270
526
|
|
271
527
|
if set_stack:
|
272
528
|
client.activate_stack(created_stack.id)
|
@@ -276,6 +532,30 @@ def register_stack(
|
|
276
532
|
f"Active {scope} stack set to:'{created_stack.name}'"
|
277
533
|
)
|
278
534
|
|
535
|
+
delete_commands = []
|
536
|
+
if "service_connector" in created_objects:
|
537
|
+
created_objects.remove("service_connector")
|
538
|
+
connectors = set()
|
539
|
+
for each in created_objects:
|
540
|
+
if comps_ := created_stack.components[StackComponentType(each)]:
|
541
|
+
if conn_ := comps_[0].connector:
|
542
|
+
connectors.add(conn_.name)
|
543
|
+
for connector in connectors:
|
544
|
+
delete_commands.append(
|
545
|
+
"zenml service-connector delete " + connector
|
546
|
+
)
|
547
|
+
for each in created_objects:
|
548
|
+
if comps_ := created_stack.components[StackComponentType(each)]:
|
549
|
+
delete_commands.append(
|
550
|
+
f"zenml {each.replace('_', '-')} delete {comps_[0].name}"
|
551
|
+
)
|
552
|
+
delete_commands.append("zenml stack delete -y " + created_stack.name)
|
553
|
+
|
554
|
+
Console().print(
|
555
|
+
"To delete the objects created by this command run, please run in a sequence:\n"
|
556
|
+
)
|
557
|
+
Console().print(Syntax("\n".join(delete_commands[::-1]), "bash"))
|
558
|
+
|
279
559
|
print_model_url(get_stack_url(created_stack))
|
280
560
|
|
281
561
|
|
@@ -488,7 +768,7 @@ def update_stack(
|
|
488
768
|
"-r",
|
489
769
|
"--model_registry",
|
490
770
|
"model_registry_flag",
|
491
|
-
help="Include this to remove the
|
771
|
+
help="Include this to remove the model registry from this stack.",
|
492
772
|
is_flag=True,
|
493
773
|
required=False,
|
494
774
|
)
|
@@ -689,21 +969,11 @@ def list_stacks(ctx: click.Context, **kwargs: Any) -> None:
|
|
689
969
|
type=click.STRING,
|
690
970
|
required=False,
|
691
971
|
)
|
692
|
-
|
693
|
-
"--outputs",
|
694
|
-
"-o",
|
695
|
-
is_flag=True,
|
696
|
-
default=False,
|
697
|
-
help="Include the outputs from mlstacks deployments.",
|
698
|
-
)
|
699
|
-
def describe_stack(
|
700
|
-
stack_name_or_id: Optional[str] = None, outputs: bool = False
|
701
|
-
) -> None:
|
972
|
+
def describe_stack(stack_name_or_id: Optional[str] = None) -> None:
|
702
973
|
"""Show details about a named stack or the active stack.
|
703
974
|
|
704
975
|
Args:
|
705
976
|
stack_name_or_id: Name of the stack to describe.
|
706
|
-
outputs: Include the outputs from mlstacks deployments.
|
707
977
|
"""
|
708
978
|
client = Client()
|
709
979
|
|
@@ -719,8 +989,6 @@ def describe_stack(
|
|
719
989
|
stack=stack_,
|
720
990
|
active=stack_.id == client.active_stack_model.id,
|
721
991
|
)
|
722
|
-
if outputs:
|
723
|
-
cli_utils.print_stack_outputs(stack_)
|
724
992
|
|
725
993
|
print_model_url(get_stack_url(stack_))
|
726
994
|
|
@@ -805,8 +1073,7 @@ def set_active_stack_command(stack_name_or_id: str) -> None:
|
|
805
1073
|
cli_utils.error(str(err))
|
806
1074
|
|
807
1075
|
cli_utils.declare(
|
808
|
-
f"Active {scope} stack set to: "
|
809
|
-
f"'{client.active_stack_model.name}'"
|
1076
|
+
f"Active {scope} stack set to: '{client.active_stack_model.name}'"
|
810
1077
|
)
|
811
1078
|
|
812
1079
|
|
@@ -825,51 +1092,6 @@ def get_active_stack() -> None:
|
|
825
1092
|
cli_utils.error(str(err))
|
826
1093
|
|
827
1094
|
|
828
|
-
@stack.command("up")
|
829
|
-
def up_stack() -> None:
|
830
|
-
"""Provisions resources for the active stack."""
|
831
|
-
stack_ = Client().active_stack
|
832
|
-
|
833
|
-
cli_utils.declare(
|
834
|
-
f"Provisioning resources for active stack '{stack_.name}'."
|
835
|
-
)
|
836
|
-
try:
|
837
|
-
stack_.provision()
|
838
|
-
stack_.resume()
|
839
|
-
except ProvisioningError as e:
|
840
|
-
cli_utils.error(str(e))
|
841
|
-
|
842
|
-
|
843
|
-
@stack.command(
|
844
|
-
"down", help="Suspends resources of the active stack deployment."
|
845
|
-
)
|
846
|
-
@click.option(
|
847
|
-
"--force",
|
848
|
-
"-f",
|
849
|
-
"force",
|
850
|
-
is_flag=True,
|
851
|
-
help="Deprovisions local resources instead of suspending them.",
|
852
|
-
)
|
853
|
-
def down_stack(force: bool = False) -> None:
|
854
|
-
"""Suspends resources of the active stack deployment.
|
855
|
-
|
856
|
-
Args:
|
857
|
-
force: Deprovisions local resources instead of suspending them.
|
858
|
-
"""
|
859
|
-
stack_ = Client().active_stack
|
860
|
-
|
861
|
-
if force:
|
862
|
-
cli_utils.declare(
|
863
|
-
f"Deprovisioning resources for active stack '{stack_.name}'."
|
864
|
-
)
|
865
|
-
stack_.deprovision()
|
866
|
-
else:
|
867
|
-
cli_utils.declare(
|
868
|
-
f"Suspending resources for active stack '{stack_.name}'."
|
869
|
-
)
|
870
|
-
stack_.suspend()
|
871
|
-
|
872
|
-
|
873
1095
|
@stack.command("export", help="Exports a stack to a YAML file.")
|
874
1096
|
@click.argument("stack_name_or_id", type=str, required=False)
|
875
1097
|
@click.argument("filename", type=str, required=False)
|
@@ -906,14 +1128,12 @@ def export_stack(
|
|
906
1128
|
def _import_stack_component(
|
907
1129
|
component_type: StackComponentType,
|
908
1130
|
component_dict: Dict[str, Any],
|
909
|
-
component_spec_path: Optional[str] = None,
|
910
1131
|
) -> UUID:
|
911
1132
|
"""Import a single stack component with given type/config.
|
912
1133
|
|
913
1134
|
Args:
|
914
1135
|
component_type: The type of component to import.
|
915
1136
|
component_dict: Dict representation of the component to import.
|
916
|
-
component_spec_path: Path to the component spec file.
|
917
1137
|
|
918
1138
|
Returns:
|
919
1139
|
The ID of the imported component.
|
@@ -949,7 +1169,6 @@ def _import_stack_component(
|
|
949
1169
|
component_type=component_type,
|
950
1170
|
flavor=flavor,
|
951
1171
|
configuration=config,
|
952
|
-
component_spec_path=component_spec_path,
|
953
1172
|
)
|
954
1173
|
return component.id
|
955
1174
|
|
@@ -1186,483 +1405,256 @@ def register_secrets(
|
|
1186
1405
|
client.update_secret(secret_name, add_or_update_values=secret_values)
|
1187
1406
|
|
1188
1407
|
|
1189
|
-
def
|
1190
|
-
|
1191
|
-
) -> Dict[str, Any]:
|
1192
|
-
"""Get deployment values from command line arguments.
|
1408
|
+
def validate_name(ctx: click.Context, param: str, value: str) -> str:
|
1409
|
+
"""Validate the name of the stack.
|
1193
1410
|
|
1194
1411
|
Args:
|
1195
|
-
|
1412
|
+
ctx: The click context.
|
1413
|
+
param: The parameter name.
|
1414
|
+
value: The value of the parameter.
|
1196
1415
|
|
1197
1416
|
Returns:
|
1198
|
-
|
1417
|
+
The validated value.
|
1418
|
+
|
1419
|
+
Raises:
|
1420
|
+
BadParameter: If the name is invalid.
|
1199
1421
|
"""
|
1200
|
-
|
1201
|
-
|
1202
|
-
"stack_name": click_params["stack_name"],
|
1203
|
-
"region": click_params["region"],
|
1204
|
-
}
|
1205
|
-
for component_type in MLSTACKS_SUPPORTED_STACK_COMPONENTS:
|
1206
|
-
verify_mlstacks_prerequisites_installation()
|
1207
|
-
from mlstacks.constants import ALLOWED_FLAVORS
|
1208
|
-
|
1209
|
-
if (
|
1210
|
-
click.prompt(
|
1211
|
-
f"Enable {component_type}?",
|
1212
|
-
type=click.Choice(["y", "n"]),
|
1213
|
-
default="n",
|
1214
|
-
)
|
1215
|
-
== "y"
|
1216
|
-
):
|
1217
|
-
component_flavor = click.prompt(
|
1218
|
-
f" Enter {component_type} flavor",
|
1219
|
-
type=click.Choice(ALLOWED_FLAVORS[component_type]),
|
1220
|
-
)
|
1221
|
-
deployment_values[component_type] = component_flavor
|
1422
|
+
if not value:
|
1423
|
+
return value
|
1222
1424
|
|
1223
|
-
if (
|
1224
|
-
click.
|
1225
|
-
"
|
1226
|
-
type=click.Choice(["y", "n"]),
|
1227
|
-
default="n",
|
1228
|
-
)
|
1229
|
-
== "y"
|
1230
|
-
):
|
1231
|
-
deployment_values["debug_mode"] = True
|
1232
|
-
|
1233
|
-
extra_config = []
|
1234
|
-
# use click.prompt to populate extra_config until someone just hits enter
|
1235
|
-
while True:
|
1236
|
-
declare(
|
1237
|
-
"\nAdd to extra_config for stack deployment -->\n",
|
1238
|
-
bold=True,
|
1239
|
-
)
|
1240
|
-
key = click.prompt(
|
1241
|
-
"Enter `extra_config` key or hit enter to skip",
|
1242
|
-
type=str,
|
1243
|
-
default="",
|
1425
|
+
if not re.match(r"^[a-zA-Z0-9-]*$", value):
|
1426
|
+
raise click.BadParameter(
|
1427
|
+
"Stack name must contain only alphanumeric characters and hyphens."
|
1244
1428
|
)
|
1245
|
-
if key == "":
|
1246
|
-
break
|
1247
|
-
value = click.prompt(
|
1248
|
-
f"Enter value for '{key}'",
|
1249
|
-
type=str,
|
1250
|
-
)
|
1251
|
-
extra_config.append(f"{key}={value}")
|
1252
1429
|
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
s.startswith("project_id=") for s in extra_config
|
1257
|
-
):
|
1258
|
-
project_id = click.prompt("What is your GCP project_id?", type=str)
|
1259
|
-
extra_config.append(f"project_id={project_id}")
|
1260
|
-
declare(f"Project ID '{project_id}' added to extra_config.")
|
1261
|
-
|
1262
|
-
deployment_values["extra_config"] = extra_config
|
1263
|
-
|
1264
|
-
tags = []
|
1265
|
-
# use click.prompt to populate tags until someone just hits enter
|
1266
|
-
while True:
|
1267
|
-
declare(
|
1268
|
-
"\nAdd to tags for stack deployment -->\n",
|
1269
|
-
bold=True,
|
1270
|
-
)
|
1271
|
-
tag = click.prompt(
|
1272
|
-
"Enter `tags` key or hit enter to skip",
|
1273
|
-
type=str,
|
1274
|
-
default="",
|
1430
|
+
if len(value) > 16:
|
1431
|
+
raise click.BadParameter(
|
1432
|
+
"Stack name must have a maximum length of 16 characters."
|
1275
1433
|
)
|
1276
|
-
if tag == "":
|
1277
|
-
break
|
1278
|
-
value = click.prompt(
|
1279
|
-
f"Enter value for '{tag}'",
|
1280
|
-
type=str,
|
1281
|
-
)
|
1282
|
-
tags.append(f"{tag}={value}")
|
1283
|
-
deployment_values["tags"] = tags
|
1284
1434
|
|
1285
|
-
return
|
1435
|
+
return value
|
1436
|
+
|
1286
1437
|
|
1438
|
+
@stack.command(
|
1439
|
+
help="""Deploy a fully functional ZenML stack in one of the cloud providers.
|
1287
1440
|
|
1288
|
-
|
1441
|
+
Running this command will initiate an assisted process that will walk you
|
1442
|
+
through automatically provisioning all the cloud infrastructure resources
|
1443
|
+
necessary for a fully functional ZenML stack in the cloud provider of your
|
1444
|
+
choice. A corresponding ZenML stack will also be automatically registered along
|
1445
|
+
with all the necessary components and properly authenticated through service
|
1446
|
+
connectors.
|
1447
|
+
"""
|
1448
|
+
)
|
1289
1449
|
@click.option(
|
1290
1450
|
"--provider",
|
1291
1451
|
"-p",
|
1292
1452
|
"provider",
|
1293
1453
|
required=True,
|
1294
|
-
type=click.Choice(
|
1454
|
+
type=click.Choice(StackDeploymentProvider.values()),
|
1295
1455
|
)
|
1296
1456
|
@click.option(
|
1297
1457
|
"--name",
|
1298
1458
|
"-n",
|
1299
1459
|
"stack_name",
|
1300
1460
|
type=click.STRING,
|
1301
|
-
required=True,
|
1302
|
-
help="Set a name for the ZenML stack that will be imported from the YAML "
|
1303
|
-
"configuration file which gets generated after deploying the stack recipe. "
|
1304
|
-
"Defaults to the name of the stack recipe being deployed.",
|
1305
|
-
)
|
1306
|
-
@click.option(
|
1307
|
-
"--region",
|
1308
|
-
"-r",
|
1309
|
-
"region",
|
1310
|
-
type=click.STRING,
|
1311
|
-
required=True,
|
1312
|
-
help="The region to deploy the stack to.",
|
1313
|
-
)
|
1314
|
-
@click.option(
|
1315
|
-
"--no-import",
|
1316
|
-
"-ni",
|
1317
|
-
"no_import_stack_flag",
|
1318
|
-
is_flag=True,
|
1319
|
-
help="If you don't want the stack to be imported automatically.",
|
1320
|
-
)
|
1321
|
-
@click.option(
|
1322
|
-
"--artifact-store",
|
1323
|
-
"-a",
|
1324
|
-
"artifact_store",
|
1325
|
-
required=False,
|
1326
|
-
is_flag=True,
|
1327
|
-
help="Whether to deploy an artifact store.",
|
1328
|
-
)
|
1329
|
-
@click.option(
|
1330
|
-
"--container-registry",
|
1331
|
-
"-c",
|
1332
|
-
"container_registry",
|
1333
1461
|
required=False,
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
"-m",
|
1340
|
-
"mlops_platform",
|
1341
|
-
type=click.Choice(["zenml"]),
|
1342
|
-
required=False,
|
1343
|
-
help="The flavor of MLOps platform to use."
|
1344
|
-
"If not specified, the default MLOps platform will be used.",
|
1345
|
-
)
|
1346
|
-
@click.option(
|
1347
|
-
"--orchestrator",
|
1348
|
-
"-o",
|
1349
|
-
required=False,
|
1350
|
-
type=click.Choice(
|
1351
|
-
[
|
1352
|
-
"kubernetes",
|
1353
|
-
"kubeflow",
|
1354
|
-
"tekton",
|
1355
|
-
"sagemaker",
|
1356
|
-
"skypilot",
|
1357
|
-
"vertex",
|
1358
|
-
]
|
1359
|
-
),
|
1360
|
-
help="The flavor of orchestrator to use. "
|
1361
|
-
"If not specified, the default orchestrator will be used.",
|
1362
|
-
)
|
1363
|
-
@click.option(
|
1364
|
-
"--model-deployer",
|
1365
|
-
"-md",
|
1366
|
-
"model_deployer",
|
1367
|
-
required=False,
|
1368
|
-
type=click.Choice(["kserve", "seldon"]),
|
1369
|
-
help="The flavor of model deployer to use. ",
|
1370
|
-
)
|
1371
|
-
@click.option(
|
1372
|
-
"--experiment-tracker",
|
1373
|
-
"-e",
|
1374
|
-
"experiment_tracker",
|
1375
|
-
required=False,
|
1376
|
-
type=click.Choice(["mlflow"]),
|
1377
|
-
help="The flavor of experiment tracker to use.",
|
1378
|
-
)
|
1379
|
-
@click.option(
|
1380
|
-
"--step-operator",
|
1381
|
-
"-s",
|
1382
|
-
"step_operator",
|
1383
|
-
required=False,
|
1384
|
-
type=click.Choice(["sagemaker"]),
|
1385
|
-
help="The flavor of step operator to use.",
|
1386
|
-
)
|
1387
|
-
@click.option(
|
1388
|
-
"--file",
|
1389
|
-
"-f",
|
1390
|
-
"file",
|
1391
|
-
required=False,
|
1392
|
-
type=click.Path(exists=True, dir_okay=False, readable=True),
|
1393
|
-
help="Use a YAML specification file as the basis of the stack deployment.",
|
1394
|
-
)
|
1395
|
-
@click.option(
|
1396
|
-
"--debug-mode",
|
1397
|
-
"-d",
|
1398
|
-
"debug_mode",
|
1399
|
-
is_flag=True,
|
1400
|
-
default=False,
|
1401
|
-
help="Whether to run the stack deployment in debug mode.",
|
1402
|
-
)
|
1403
|
-
@click.option(
|
1404
|
-
"--extra-config",
|
1405
|
-
"-x",
|
1406
|
-
"extra_config",
|
1407
|
-
multiple=True,
|
1408
|
-
help="Extra configurations as key=value pairs. This option can be used multiple times.",
|
1462
|
+
help="Custom string to use as a prefix to generate names for the ZenML "
|
1463
|
+
"stack, its components service connectors as well as provisioned cloud "
|
1464
|
+
"infrastructure resources. May only contain alphanumeric characters and "
|
1465
|
+
"hyphens and have a maximum length of 16 characters.",
|
1466
|
+
callback=validate_name,
|
1409
1467
|
)
|
1410
1468
|
@click.option(
|
1411
|
-
"--
|
1412
|
-
"-
|
1413
|
-
"tags",
|
1414
|
-
required=False,
|
1469
|
+
"--location",
|
1470
|
+
"-l",
|
1415
1471
|
type=click.STRING,
|
1416
|
-
|
1417
|
-
|
1472
|
+
required=False,
|
1473
|
+
help="The location to deploy the stack to.",
|
1418
1474
|
)
|
1419
1475
|
@click.option(
|
1420
|
-
"--
|
1421
|
-
"
|
1422
|
-
"interactive",
|
1476
|
+
"--set",
|
1477
|
+
"set_stack",
|
1423
1478
|
is_flag=True,
|
1424
|
-
|
1425
|
-
|
1479
|
+
help="Immediately set this stack as active.",
|
1480
|
+
type=click.BOOL,
|
1426
1481
|
)
|
1427
1482
|
@click.pass_context
|
1428
1483
|
def deploy(
|
1429
1484
|
ctx: click.Context,
|
1430
1485
|
provider: str,
|
1431
|
-
stack_name: str,
|
1432
|
-
|
1433
|
-
|
1434
|
-
orchestrator: Optional[str] = None,
|
1435
|
-
model_deployer: Optional[str] = None,
|
1436
|
-
experiment_tracker: Optional[str] = None,
|
1437
|
-
step_operator: Optional[str] = None,
|
1438
|
-
no_import_stack_flag: bool = False,
|
1439
|
-
artifact_store: Optional[bool] = None,
|
1440
|
-
container_registry: Optional[bool] = None,
|
1441
|
-
file: Optional[str] = None,
|
1442
|
-
debug_mode: bool = False,
|
1443
|
-
tags: Optional[List[str]] = None,
|
1444
|
-
extra_config: Optional[List[str]] = None,
|
1445
|
-
interactive: bool = False,
|
1486
|
+
stack_name: Optional[str] = None,
|
1487
|
+
location: Optional[str] = None,
|
1488
|
+
set_stack: bool = False,
|
1446
1489
|
) -> None:
|
1447
|
-
"""Deploy a
|
1448
|
-
|
1449
|
-
`zenml stack_recipe pull <STACK_RECIPE_NAME>` has to be called with the
|
1450
|
-
same relative path before the `deploy` command.
|
1490
|
+
"""Deploy and register a fully functional cloud ZenML stack.
|
1451
1491
|
|
1452
1492
|
Args:
|
1453
1493
|
ctx: The click context.
|
1454
1494
|
provider: The cloud provider to deploy the stack to.
|
1455
1495
|
stack_name: A name for the ZenML stack that gets imported as a result
|
1456
1496
|
of the recipe deployment.
|
1457
|
-
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
container_registry: The flavor of container registry to deploy. In the case of
|
1464
|
-
the container registry, it doesn't matter what you specify here, as
|
1465
|
-
there's only one flavor per cloud provider and that will be deployed.
|
1466
|
-
model_deployer: The flavor of model deployer to deploy.
|
1467
|
-
experiment_tracker: The flavor of experiment tracker to deploy.
|
1468
|
-
step_operator: The flavor of step operator to deploy.
|
1469
|
-
extra_config: Extra configurations as key=value pairs.
|
1470
|
-
tags: Pass one or more tags.
|
1471
|
-
debug_mode: Whether to run the stack deployment in debug mode.
|
1472
|
-
file: Use a YAML specification file as the basis of the stack
|
1473
|
-
deployment.
|
1474
|
-
mlops_platform: The flavor of MLOps platform to use.
|
1475
|
-
region: The region to deploy the stack to.
|
1476
|
-
interactive: Deploy the stack interactively.
|
1497
|
+
location: The location to deploy the stack to.
|
1498
|
+
set_stack: Immediately set the deployed stack as active.
|
1499
|
+
|
1500
|
+
Raises:
|
1501
|
+
Abort: If the user aborts the deployment.
|
1502
|
+
KeyboardInterrupt: If the user interrupts the deployment.
|
1477
1503
|
"""
|
1504
|
+
stack_name = stack_name or f"zenml-{provider}-stack"
|
1505
|
+
|
1506
|
+
# Set up the markdown renderer to use the old-school markdown heading
|
1507
|
+
Markdown.elements.update(
|
1508
|
+
{
|
1509
|
+
"heading_open": OldSchoolMarkdownHeading,
|
1510
|
+
}
|
1511
|
+
)
|
1512
|
+
|
1513
|
+
client = Client()
|
1514
|
+
if client.zen_store.is_local_store():
|
1515
|
+
cli_utils.error(
|
1516
|
+
"This feature cannot be used with a local ZenML deployment. "
|
1517
|
+
"ZenML needs to be accessible from the cloud provider to allow the "
|
1518
|
+
"stack and its components to be registered automatically. "
|
1519
|
+
"Please deploy ZenML in a remote environment as described in the "
|
1520
|
+
"documentation: https://docs.zenml.io/getting-started/deploying-zenml "
|
1521
|
+
"or use a managed ZenML Pro server instance for quick access to "
|
1522
|
+
"this feature and more: https://www.zenml.io/pro"
|
1523
|
+
)
|
1524
|
+
|
1478
1525
|
with track_handler(
|
1479
|
-
event=AnalyticsEvent.
|
1526
|
+
event=AnalyticsEvent.DEPLOY_FULL_STACK,
|
1480
1527
|
) as analytics_handler:
|
1481
|
-
|
1482
|
-
|
1483
|
-
|
1484
|
-
"different name."
|
1485
|
-
)
|
1486
|
-
elif stack_spec_exists(stack_name):
|
1487
|
-
cli_utils.error(
|
1488
|
-
f"Stack spec for stack named '{stack_name}' already exists. "
|
1489
|
-
"Please choose a different name."
|
1490
|
-
)
|
1528
|
+
analytics_handler.metadata = {
|
1529
|
+
"provider": provider,
|
1530
|
+
}
|
1491
1531
|
|
1492
|
-
|
1493
|
-
|
1494
|
-
|
1532
|
+
deployment = client.zen_store.get_stack_deployment_info(
|
1533
|
+
provider=StackDeploymentProvider(provider),
|
1534
|
+
)
|
1495
1535
|
|
1496
|
-
if not
|
1497
|
-
|
1498
|
-
|
1499
|
-
|
1500
|
-
stack, components = convert_click_params_to_mlstacks_primitives(
|
1501
|
-
cli_params
|
1536
|
+
if location and location not in deployment.locations.values():
|
1537
|
+
cli_utils.error(
|
1538
|
+
f"Invalid location '{location}' for provider '{provider}'. "
|
1539
|
+
f"Valid locations are: {', '.join(deployment.locations.values())}"
|
1502
1540
|
)
|
1503
1541
|
|
1504
|
-
|
1542
|
+
console.print(
|
1543
|
+
Markdown(
|
1544
|
+
f"# {provider.upper()} ZenML Cloud Stack Deployment\n"
|
1545
|
+
+ deployment.description
|
1546
|
+
)
|
1547
|
+
)
|
1548
|
+
console.print(Markdown("## Details\n" + deployment.instructions))
|
1505
1549
|
|
1506
|
-
|
1507
|
-
|
1508
|
-
|
1509
|
-
|
1510
|
-
|
1511
|
-
"The specified stack and component flavors are not compatible "
|
1512
|
-
"with the provider or with one another. Please try again."
|
1513
|
-
)
|
1550
|
+
deployment_config = client.zen_store.get_stack_deployment_config(
|
1551
|
+
provider=StackDeploymentProvider(provider),
|
1552
|
+
stack_name=stack_name,
|
1553
|
+
location=location,
|
1554
|
+
)
|
1514
1555
|
|
1515
|
-
|
1516
|
-
|
1556
|
+
if deployment_config.instructions:
|
1557
|
+
console.print(
|
1558
|
+
Markdown("## Instructions\n" + deployment_config.instructions),
|
1559
|
+
"\n",
|
1517
1560
|
)
|
1518
|
-
# write the stack and component yaml files
|
1519
|
-
from mlstacks.constants import MLSTACKS_PACKAGE_NAME
|
1520
1561
|
|
1521
|
-
|
1522
|
-
|
1523
|
-
|
1524
|
-
|
1562
|
+
if deployment_config.configuration:
|
1563
|
+
console.print(
|
1564
|
+
deployment_config.configuration,
|
1565
|
+
no_wrap=True,
|
1566
|
+
overflow="ignore",
|
1567
|
+
crop=False,
|
1568
|
+
style=Style(bgcolor="grey15"),
|
1525
1569
|
)
|
1526
|
-
cli_utils.declare(f"Writing spec files to {spec_dir}...")
|
1527
|
-
create_dir_recursive_if_not_exists(spec_dir)
|
1528
1570
|
|
1529
|
-
|
1530
|
-
|
1571
|
+
if not cli_utils.confirmation(
|
1572
|
+
"\n\nProceed to continue with the deployment. You will be "
|
1573
|
+
f"automatically redirected to "
|
1574
|
+
f"{deployment_config.deployment_url_text} in your browser.",
|
1575
|
+
):
|
1576
|
+
raise click.Abort()
|
1577
|
+
|
1578
|
+
date_start = datetime.utcnow()
|
1579
|
+
|
1580
|
+
webbrowser.open(deployment_config.deployment_url)
|
1581
|
+
console.print(
|
1582
|
+
Markdown(
|
1583
|
+
f"If your browser did not open automatically, please open "
|
1584
|
+
f"the following URL into your browser to deploy the stack to "
|
1585
|
+
f"{provider.upper()}: "
|
1586
|
+
f"[{deployment_config.deployment_url_text}]"
|
1587
|
+
f"({deployment_config.deployment_url}).\n\n"
|
1531
1588
|
)
|
1532
|
-
write_yaml(file_path=stack_file_path, contents=stack_dict)
|
1533
|
-
for component in component_dicts:
|
1534
|
-
write_yaml(
|
1535
|
-
file_path=os.path.join(
|
1536
|
-
spec_dir, f"{component['name']}.yaml"
|
1537
|
-
),
|
1538
|
-
contents=component,
|
1539
|
-
)
|
1540
|
-
else:
|
1541
|
-
declare("Importing from stack specification file...")
|
1542
|
-
stack_file_path = file
|
1543
|
-
|
1544
|
-
from mlstacks.utils.yaml_utils import load_stack_yaml
|
1545
|
-
|
1546
|
-
stack = load_stack_yaml(stack_file_path)
|
1547
|
-
|
1548
|
-
analytics_handler.metadata = {
|
1549
|
-
"stack_provider": stack.provider,
|
1550
|
-
"debug_mode": debug_mode,
|
1551
|
-
"no_import_stack_flag": no_import_stack_flag,
|
1552
|
-
"user_created_spec": bool(file),
|
1553
|
-
"mlops_platform": mlops_platform,
|
1554
|
-
"orchestrator": orchestrator,
|
1555
|
-
"model_deployer": model_deployer,
|
1556
|
-
"experiment_tracker": experiment_tracker,
|
1557
|
-
"step_operator": step_operator,
|
1558
|
-
"artifact_store": artifact_store,
|
1559
|
-
"container_registry": container_registry,
|
1560
|
-
}
|
1561
|
-
|
1562
|
-
deploy_mlstacks_stack(
|
1563
|
-
spec_file_path=stack_file_path,
|
1564
|
-
stack_name=stack.name,
|
1565
|
-
stack_provider=stack.provider,
|
1566
|
-
debug_mode=debug_mode,
|
1567
|
-
no_import_stack_flag=no_import_stack_flag,
|
1568
|
-
user_created_spec=bool(file),
|
1569
1589
|
)
|
1570
1590
|
|
1591
|
+
try:
|
1592
|
+
cli_utils.declare(
|
1593
|
+
"\n\nWaiting for the deployment to complete and the stack to be "
|
1594
|
+
"registered. Press CTRL+C to abort...\n"
|
1595
|
+
)
|
1571
1596
|
|
1572
|
-
|
1573
|
-
|
1574
|
-
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1583
|
-
|
1584
|
-
|
1585
|
-
|
1586
|
-
|
1587
|
-
|
1588
|
-
) -> None:
|
1589
|
-
"""Destroy all resources previously created with `zenml stack deploy`.
|
1597
|
+
while True:
|
1598
|
+
deployed_stack = client.zen_store.get_stack_deployment_stack(
|
1599
|
+
provider=StackDeploymentProvider(provider),
|
1600
|
+
stack_name=stack_name,
|
1601
|
+
location=location,
|
1602
|
+
date_start=date_start,
|
1603
|
+
)
|
1604
|
+
if deployed_stack:
|
1605
|
+
break
|
1606
|
+
time.sleep(10)
|
1607
|
+
|
1608
|
+
analytics_handler.metadata.update(
|
1609
|
+
{
|
1610
|
+
"stack_id": deployed_stack.stack.id,
|
1611
|
+
}
|
1612
|
+
)
|
1590
1613
|
|
1591
|
-
|
1592
|
-
|
1593
|
-
|
1594
|
-
"""
|
1595
|
-
if not confirmation(
|
1596
|
-
f"Are you sure you want to destroy stack '{stack_name}' and all "
|
1597
|
-
"associated infrastructure?"
|
1598
|
-
):
|
1599
|
-
error("Aborting stack destroy...")
|
1614
|
+
except KeyboardInterrupt:
|
1615
|
+
cli_utils.declare("Stack deployment aborted.")
|
1616
|
+
raise
|
1600
1617
|
|
1601
|
-
|
1602
|
-
|
1603
|
-
) as analytics_handler:
|
1604
|
-
analytics_handler.metadata["debug_mode"] = debug_mode
|
1605
|
-
cli_utils.verify_mlstacks_prerequisites_installation()
|
1606
|
-
from mlstacks.constants import MLSTACKS_PACKAGE_NAME
|
1618
|
+
stack_desc = f"""## Stack successfully registered! 🚀
|
1619
|
+
Stack [{deployed_stack.stack.name}]({get_stack_url(deployed_stack.stack)}):\n"""
|
1607
1620
|
|
1608
|
-
|
1609
|
-
if
|
1610
|
-
|
1611
|
-
|
1612
|
-
"
|
1621
|
+
for component_type, components in deployed_stack.stack.components.items():
|
1622
|
+
if components:
|
1623
|
+
component = components[0]
|
1624
|
+
stack_desc += (
|
1625
|
+
f" * `{component.flavor_name}` {component_type.value}: "
|
1626
|
+
f"[{component.name}]({get_component_url(component)})\n"
|
1613
1627
|
)
|
1614
1628
|
|
1615
|
-
|
1616
|
-
|
1617
|
-
|
1629
|
+
if deployed_stack.service_connector:
|
1630
|
+
stack_desc += (
|
1631
|
+
f" * Service Connector: {deployed_stack.service_connector.name}\n"
|
1618
1632
|
)
|
1619
|
-
user_created_spec = str(Path(spec_file_path).parent) != spec_files_dir
|
1620
1633
|
|
1621
|
-
|
1622
|
-
tf_definitions_path: str = os.path.join(
|
1623
|
-
click.get_app_dir(MLSTACKS_PACKAGE_NAME),
|
1624
|
-
"terraform",
|
1625
|
-
f"{provider}-modular",
|
1626
|
-
)
|
1627
|
-
|
1628
|
-
cli_utils.declare(
|
1629
|
-
"Checking Terraform definitions and spec files are present..."
|
1630
|
-
)
|
1631
|
-
verify_spec_and_tf_files_exist(spec_file_path, tf_definitions_path)
|
1634
|
+
console.print(Markdown(stack_desc))
|
1632
1635
|
|
1633
|
-
|
1636
|
+
follow_up = f"""
|
1637
|
+
## Follow-up
|
1634
1638
|
|
1635
|
-
|
1636
|
-
f"Destroying stack '{stack_name}' using Terraform..."
|
1637
|
-
)
|
1638
|
-
terraform_utils.destroy_stack(
|
1639
|
-
stack_path=spec_file_path, debug_mode=debug_mode
|
1640
|
-
)
|
1641
|
-
cli_utils.declare(f"Stack '{stack_name}' successfully destroyed.")
|
1639
|
+
{deployment.post_deploy_instructions}
|
1642
1640
|
|
1643
|
-
|
1644
|
-
f"Would you like to recursively delete the associated ZenML "
|
1645
|
-
f"stack '{stack_name}'?\nThis will delete the stack and any "
|
1646
|
-
"underlying stack components."
|
1647
|
-
):
|
1648
|
-
from zenml.client import Client
|
1641
|
+
To use the `{deployed_stack.stack.name}` stack to run pipelines:
|
1649
1642
|
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1653
|
-
|
1654
|
-
|
1643
|
+
* install the required ZenML integrations by running: `zenml integration install {" ".join(deployment.integrations)}`
|
1644
|
+
"""
|
1645
|
+
if set_stack:
|
1646
|
+
client.activate_stack(deployed_stack.stack.id)
|
1647
|
+
follow_up += f"""
|
1648
|
+
* the `{deployed_stack.stack.name}` stack has already been set as active
|
1649
|
+
"""
|
1650
|
+
else:
|
1651
|
+
follow_up += f"""
|
1652
|
+
* set the `{deployed_stack.stack.name}` stack as active by running: `zenml stack set {deployed_stack.stack.name}`
|
1653
|
+
"""
|
1655
1654
|
|
1656
|
-
|
1657
|
-
|
1658
|
-
|
1659
|
-
f"this stack, located at {spec_dir}?"
|
1660
|
-
):
|
1661
|
-
rmtree(spec_files_dir)
|
1662
|
-
cli_utils.declare(
|
1663
|
-
f"Spec directory for stack '{stack_name}' successfully deleted."
|
1664
|
-
)
|
1665
|
-
cli_utils.declare(f"Stack '{stack_name}' successfully destroyed.")
|
1655
|
+
console.print(
|
1656
|
+
Markdown(follow_up),
|
1657
|
+
)
|
1666
1658
|
|
1667
1659
|
|
1668
1660
|
@stack.command(
|
@@ -1726,3 +1718,301 @@ def connect_stack(
|
|
1726
1718
|
interactive=interactive,
|
1727
1719
|
no_verify=no_verify,
|
1728
1720
|
)
|
1721
|
+
|
1722
|
+
|
1723
|
+
def _get_service_connector_info(
|
1724
|
+
cloud_provider: str,
|
1725
|
+
connector_details: Optional[
|
1726
|
+
Union[ServiceConnectorResponse, ServiceConnectorRequest]
|
1727
|
+
],
|
1728
|
+
) -> ServiceConnectorInfo:
|
1729
|
+
"""Get a service connector info with given cloud provider.
|
1730
|
+
|
1731
|
+
Args:
|
1732
|
+
cloud_provider: The cloud provider to use.
|
1733
|
+
connector_details: Whether to use implicit credentials.
|
1734
|
+
|
1735
|
+
Returns:
|
1736
|
+
The info model of the created service connector.
|
1737
|
+
|
1738
|
+
Raises:
|
1739
|
+
ValueError: If the cloud provider is not supported.
|
1740
|
+
"""
|
1741
|
+
from rich.prompt import Prompt
|
1742
|
+
|
1743
|
+
if cloud_provider not in {"aws", "gcp", "azure"}:
|
1744
|
+
raise ValueError(f"Unknown cloud provider {cloud_provider}")
|
1745
|
+
|
1746
|
+
client = Client()
|
1747
|
+
auth_methods = client.get_service_connector_type(
|
1748
|
+
cloud_provider
|
1749
|
+
).auth_method_dict
|
1750
|
+
if not connector_details:
|
1751
|
+
fixed_auth_methods = list(
|
1752
|
+
[
|
1753
|
+
(key, value)
|
1754
|
+
for key, value in auth_methods.items()
|
1755
|
+
if key != "implicit"
|
1756
|
+
]
|
1757
|
+
)
|
1758
|
+
choices = []
|
1759
|
+
headers = ["Name", "Required"]
|
1760
|
+
for _, value in fixed_auth_methods:
|
1761
|
+
schema = value.config_schema
|
1762
|
+
required = ""
|
1763
|
+
for each_req in schema["required"]:
|
1764
|
+
field = schema["properties"][each_req]
|
1765
|
+
required += f"[bold]{each_req}[/bold] [italic]({field.get('title', 'no description')})[/italic]\n"
|
1766
|
+
choices.append([value.name, required])
|
1767
|
+
|
1768
|
+
selected_auth_idx = cli_utils.multi_choice_prompt(
|
1769
|
+
object_type=f"authentication methods for {cloud_provider.upper()}",
|
1770
|
+
choices=choices,
|
1771
|
+
headers=headers,
|
1772
|
+
prompt_text="Please choose one of the authentication option above",
|
1773
|
+
)
|
1774
|
+
if selected_auth_idx is None:
|
1775
|
+
cli_utils.error("No authentication method selected.")
|
1776
|
+
auth_type = fixed_auth_methods[selected_auth_idx][0]
|
1777
|
+
else:
|
1778
|
+
auth_type = connector_details.auth_method
|
1779
|
+
|
1780
|
+
selected_auth_model = auth_methods[auth_type]
|
1781
|
+
|
1782
|
+
required_fields = selected_auth_model.config_schema["required"]
|
1783
|
+
properties = selected_auth_model.config_schema["properties"]
|
1784
|
+
|
1785
|
+
answers = {}
|
1786
|
+
for req_field in required_fields:
|
1787
|
+
if connector_details:
|
1788
|
+
if conf_value := connector_details.configuration.get(
|
1789
|
+
req_field, None
|
1790
|
+
):
|
1791
|
+
answers[req_field] = conf_value
|
1792
|
+
elif secret_value := connector_details.secrets.get(
|
1793
|
+
req_field, None
|
1794
|
+
):
|
1795
|
+
answers[req_field] = secret_value.get_secret_value()
|
1796
|
+
if req_field not in answers:
|
1797
|
+
answers[req_field] = Prompt.ask(
|
1798
|
+
f"Please enter value for `{req_field}`:",
|
1799
|
+
password="format" in properties[req_field]
|
1800
|
+
and properties[req_field]["format"] == "password",
|
1801
|
+
)
|
1802
|
+
|
1803
|
+
return ServiceConnectorInfo(
|
1804
|
+
type=cloud_provider,
|
1805
|
+
auth_method=auth_type,
|
1806
|
+
configuration=answers,
|
1807
|
+
)
|
1808
|
+
|
1809
|
+
|
1810
|
+
def _get_stack_component_info(
|
1811
|
+
component_type: str,
|
1812
|
+
cloud_provider: str,
|
1813
|
+
resources_info: ServiceConnectorResourcesInfo,
|
1814
|
+
service_connector_index: Optional[int] = None,
|
1815
|
+
) -> ComponentInfo:
|
1816
|
+
"""Get a stack component info with given type and service connector.
|
1817
|
+
|
1818
|
+
Args:
|
1819
|
+
component_type: The type of component to create.
|
1820
|
+
cloud_provider: The cloud provider to use.
|
1821
|
+
resources_info: The resources info of the service connector.
|
1822
|
+
service_connector_index: The index of the service connector to use.
|
1823
|
+
|
1824
|
+
Returns:
|
1825
|
+
The info model of the stack component.
|
1826
|
+
|
1827
|
+
Raises:
|
1828
|
+
ValueError: If the cloud provider is not supported.
|
1829
|
+
ValueError: If the component type is not supported.
|
1830
|
+
"""
|
1831
|
+
from rich.prompt import Prompt
|
1832
|
+
|
1833
|
+
if cloud_provider not in {"aws", "azure", "gcp"}:
|
1834
|
+
raise ValueError(f"Unknown cloud provider {cloud_provider}")
|
1835
|
+
|
1836
|
+
flavor = "undefined"
|
1837
|
+
service_connector_resource_id = None
|
1838
|
+
config = {}
|
1839
|
+
choices = [
|
1840
|
+
[cri.flavor, resource_id]
|
1841
|
+
for cri in resources_info.components_resources_info[
|
1842
|
+
StackComponentType(component_type)
|
1843
|
+
]
|
1844
|
+
for resource_id in cri.accessible_by_service_connector
|
1845
|
+
]
|
1846
|
+
if component_type == "artifact_store":
|
1847
|
+
selected_storage_idx = cli_utils.multi_choice_prompt(
|
1848
|
+
object_type=f"{cloud_provider.upper()} storages",
|
1849
|
+
choices=choices,
|
1850
|
+
headers=["Artifact Store Type", "Storage"],
|
1851
|
+
prompt_text="Please choose one of the storages for the new artifact store:",
|
1852
|
+
)
|
1853
|
+
if selected_storage_idx is None:
|
1854
|
+
cli_utils.error("No storage selected.")
|
1855
|
+
|
1856
|
+
selected_storage = choices[selected_storage_idx]
|
1857
|
+
|
1858
|
+
flavor = selected_storage[0]
|
1859
|
+
config = {"path": selected_storage[1]}
|
1860
|
+
service_connector_resource_id = selected_storage[1]
|
1861
|
+
elif component_type == "orchestrator":
|
1862
|
+
|
1863
|
+
def query_region(
|
1864
|
+
provider: StackDeploymentProvider,
|
1865
|
+
compute_type: str,
|
1866
|
+
is_skypilot: bool = False,
|
1867
|
+
) -> str:
|
1868
|
+
deployment_info = Client().zen_store.get_stack_deployment_info(
|
1869
|
+
provider
|
1870
|
+
)
|
1871
|
+
region = Prompt.ask(
|
1872
|
+
f"Select the location for your {compute_type}:",
|
1873
|
+
choices=sorted(
|
1874
|
+
deployment_info.skypilot_default_regions.values()
|
1875
|
+
if is_skypilot
|
1876
|
+
else deployment_info.locations.values()
|
1877
|
+
),
|
1878
|
+
show_choices=True,
|
1879
|
+
)
|
1880
|
+
return region
|
1881
|
+
|
1882
|
+
selected_orchestrator_idx = cli_utils.multi_choice_prompt(
|
1883
|
+
object_type=f"orchestrators on {cloud_provider.upper()}",
|
1884
|
+
choices=choices,
|
1885
|
+
headers=["Orchestrator Type", "Details"],
|
1886
|
+
prompt_text="Please choose one of the orchestrators for the new orchestrator:",
|
1887
|
+
)
|
1888
|
+
if selected_orchestrator_idx is None:
|
1889
|
+
cli_utils.error("No orchestrator selected.")
|
1890
|
+
|
1891
|
+
selected_orchestrator = choices[selected_orchestrator_idx]
|
1892
|
+
|
1893
|
+
config = {}
|
1894
|
+
flavor = selected_orchestrator[0]
|
1895
|
+
if flavor == "sagemaker":
|
1896
|
+
execution_role = Prompt.ask("Enter an execution role ARN:")
|
1897
|
+
config["execution_role"] = execution_role
|
1898
|
+
elif flavor == "vm_aws":
|
1899
|
+
config["region"] = selected_orchestrator[1]
|
1900
|
+
elif flavor == "vm_gcp":
|
1901
|
+
config["region"] = query_region(
|
1902
|
+
StackDeploymentProvider.GCP,
|
1903
|
+
"Skypilot cluster",
|
1904
|
+
is_skypilot=True,
|
1905
|
+
)
|
1906
|
+
elif flavor == "vm_azure":
|
1907
|
+
config["region"] = query_region(
|
1908
|
+
StackDeploymentProvider.AZURE,
|
1909
|
+
"Skypilot cluster",
|
1910
|
+
is_skypilot=True,
|
1911
|
+
)
|
1912
|
+
elif flavor == "azureml":
|
1913
|
+
config["subscription_id"] = Prompt.ask(
|
1914
|
+
"Enter the subscription ID:"
|
1915
|
+
)
|
1916
|
+
config["resource_group"] = Prompt.ask("Enter the resource group:")
|
1917
|
+
config["workspace"] = Prompt.ask("Enter the workspace name:")
|
1918
|
+
elif flavor == "vertex":
|
1919
|
+
config["location"] = query_region(
|
1920
|
+
StackDeploymentProvider.GCP, "Vertex AI job"
|
1921
|
+
)
|
1922
|
+
service_connector_resource_id = selected_orchestrator[1]
|
1923
|
+
elif component_type == "container_registry":
|
1924
|
+
selected_registry_idx = cli_utils.multi_choice_prompt(
|
1925
|
+
object_type=f"{cloud_provider.upper()} registries",
|
1926
|
+
choices=choices,
|
1927
|
+
headers=["Container Registry Type", "Container Registry"],
|
1928
|
+
prompt_text="Please choose one of the registries for the new container registry:",
|
1929
|
+
)
|
1930
|
+
if selected_registry_idx is None:
|
1931
|
+
cli_utils.error("No container registry selected.")
|
1932
|
+
selected_registry = choices[selected_registry_idx]
|
1933
|
+
flavor = selected_registry[0]
|
1934
|
+
config = {"uri": selected_registry[1]}
|
1935
|
+
service_connector_resource_id = selected_registry[1]
|
1936
|
+
else:
|
1937
|
+
raise ValueError(f"Unknown component type {component_type}")
|
1938
|
+
|
1939
|
+
return ComponentInfo(
|
1940
|
+
flavor=flavor,
|
1941
|
+
configuration=config,
|
1942
|
+
service_connector_index=service_connector_index,
|
1943
|
+
service_connector_resource_id=service_connector_resource_id,
|
1944
|
+
)
|
1945
|
+
|
1946
|
+
|
1947
|
+
@stack.command(
|
1948
|
+
name="export-requirements", help="Export the stack requirements."
|
1949
|
+
)
|
1950
|
+
@click.argument(
|
1951
|
+
"stack_name_or_id",
|
1952
|
+
type=click.STRING,
|
1953
|
+
required=False,
|
1954
|
+
)
|
1955
|
+
@click.option(
|
1956
|
+
"--output-file",
|
1957
|
+
"-o",
|
1958
|
+
"output_file",
|
1959
|
+
type=str,
|
1960
|
+
required=False,
|
1961
|
+
help="File to which to export the stack requirements. If not "
|
1962
|
+
"provided, the requirements will be printed to stdout instead.",
|
1963
|
+
)
|
1964
|
+
@click.option(
|
1965
|
+
"--overwrite",
|
1966
|
+
"-ov",
|
1967
|
+
"overwrite",
|
1968
|
+
type=bool,
|
1969
|
+
required=False,
|
1970
|
+
is_flag=True,
|
1971
|
+
help="Overwrite the output file if it already exists. This option is "
|
1972
|
+
"only valid if the output file is provided.",
|
1973
|
+
)
|
1974
|
+
def export_requirements(
|
1975
|
+
stack_name_or_id: Optional[str] = None,
|
1976
|
+
output_file: Optional[str] = None,
|
1977
|
+
overwrite: bool = False,
|
1978
|
+
) -> None:
|
1979
|
+
"""Exports stack requirements so they can be installed using pip.
|
1980
|
+
|
1981
|
+
Args:
|
1982
|
+
stack_name_or_id: Stack name or ID. If not given, the active stack will
|
1983
|
+
be used.
|
1984
|
+
output_file: Optional path to the requirements output file.
|
1985
|
+
overwrite: Overwrite the output file if it already exists. This option
|
1986
|
+
is only valid if the output file is provided.
|
1987
|
+
"""
|
1988
|
+
try:
|
1989
|
+
stack_model: "StackResponse" = Client().get_stack(
|
1990
|
+
name_id_or_prefix=stack_name_or_id
|
1991
|
+
)
|
1992
|
+
except KeyError as err:
|
1993
|
+
cli_utils.error(str(err))
|
1994
|
+
|
1995
|
+
requirements, _ = requirements_utils.get_requirements_for_stack(
|
1996
|
+
stack_model
|
1997
|
+
)
|
1998
|
+
|
1999
|
+
if not requirements:
|
2000
|
+
cli_utils.declare(f"Stack `{stack_model.name}` has no requirements.")
|
2001
|
+
return
|
2002
|
+
|
2003
|
+
if output_file:
|
2004
|
+
try:
|
2005
|
+
with open(output_file, "x") as f:
|
2006
|
+
f.write("\n".join(requirements))
|
2007
|
+
except FileExistsError:
|
2008
|
+
if overwrite or cli_utils.confirmation(
|
2009
|
+
"A file already exists at the specified path. "
|
2010
|
+
"Would you like to overwrite it?"
|
2011
|
+
):
|
2012
|
+
with open(output_file, "w") as f:
|
2013
|
+
f.write("\n".join(requirements))
|
2014
|
+
cli_utils.declare(
|
2015
|
+
f"Requirements for stack `{stack_model.name}` exported to {output_file}."
|
2016
|
+
)
|
2017
|
+
else:
|
2018
|
+
click.echo(" ".join(requirements), nl=False)
|