zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/__init__.py +9 -12
- zenml/actions/__init__.py +14 -0
- zenml/actions/base_action.py +682 -0
- zenml/actions/pipeline_run/__init__.py +0 -0
- zenml/actions/pipeline_run/pipeline_run_action.py +223 -0
- zenml/analytics/context.py +70 -15
- zenml/analytics/enums.py +21 -17
- zenml/analytics/models.py +1 -0
- zenml/analytics/utils.py +19 -7
- zenml/annotators/base_annotator.py +4 -3
- zenml/artifact_stores/base_artifact_store.py +155 -72
- zenml/artifact_stores/local_artifact_store.py +3 -2
- zenml/artifacts/artifact_config.py +67 -58
- zenml/artifacts/external_artifact.py +24 -39
- zenml/artifacts/external_artifact_config.py +27 -37
- zenml/artifacts/preexisting_data_materializer.py +90 -0
- zenml/artifacts/unmaterialized_artifact.py +2 -12
- zenml/artifacts/utils.py +521 -220
- zenml/cli/__init__.py +1511 -456
- zenml/cli/annotator.py +48 -14
- zenml/cli/artifact.py +22 -4
- zenml/cli/authorized_device.py +1 -0
- zenml/cli/base.py +185 -22
- zenml/cli/code_repository.py +1 -0
- zenml/cli/formatter.py +1 -1
- zenml/cli/integration.py +119 -24
- zenml/cli/login.py +1053 -0
- zenml/cli/model.py +42 -15
- zenml/cli/model_registry.py +1 -2
- zenml/cli/pipeline.py +168 -82
- zenml/cli/secret.py +1 -2
- zenml/cli/served_model.py +56 -26
- zenml/cli/server.py +486 -619
- zenml/cli/service_accounts.py +55 -14
- zenml/cli/service_connectors.py +97 -6
- zenml/cli/stack.py +821 -531
- zenml/cli/stack_components.py +15 -598
- zenml/cli/tag.py +1 -0
- zenml/cli/text_utils.py +36 -2
- zenml/cli/user_management.py +204 -6
- zenml/cli/utils.py +411 -290
- zenml/client.py +1742 -298
- zenml/client_lazy_loader.py +224 -0
- zenml/code_repositories/base_code_repository.py +5 -4
- zenml/code_repositories/git/local_git_repository_context.py +1 -0
- zenml/code_repositories/local_repository_context.py +1 -0
- zenml/config/__init__.py +2 -0
- zenml/config/base_settings.py +6 -6
- zenml/config/build_configuration.py +43 -17
- zenml/config/compiler.py +82 -49
- zenml/config/docker_settings.py +139 -83
- zenml/config/global_config.py +260 -234
- zenml/config/pipeline_configurations.py +32 -11
- zenml/config/pipeline_run_configuration.py +15 -3
- zenml/config/pipeline_spec.py +6 -6
- zenml/config/resource_settings.py +8 -9
- zenml/config/retry_config.py +27 -0
- zenml/config/schedule.py +27 -18
- zenml/config/secret_reference_mixin.py +8 -4
- zenml/config/secrets_store_config.py +16 -24
- zenml/config/server_config.py +434 -51
- zenml/config/settings_resolver.py +2 -1
- zenml/config/source.py +97 -31
- zenml/config/step_configurations.py +83 -39
- zenml/config/step_run_info.py +3 -0
- zenml/config/store_config.py +20 -54
- zenml/config/strict_base_model.py +2 -6
- zenml/console.py +0 -1
- zenml/constants.py +232 -74
- zenml/container_registries/azure_container_registry.py +1 -0
- zenml/container_registries/base_container_registry.py +7 -3
- zenml/container_registries/default_container_registry.py +4 -3
- zenml/container_registries/dockerhub_container_registry.py +1 -0
- zenml/container_registries/gcp_container_registry.py +1 -0
- zenml/container_registries/github_container_registry.py +2 -10
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/entrypoints/base_entrypoint_configuration.py +76 -17
- zenml/entrypoints/pipeline_entrypoint_configuration.py +1 -0
- zenml/entrypoints/step_entrypoint_configuration.py +21 -2
- zenml/enums.py +91 -9
- zenml/environment.py +52 -319
- zenml/{steps/base_parameters.py → event_hub/__init__.py} +5 -7
- zenml/event_hub/base_event_hub.py +196 -0
- zenml/event_hub/event_hub.py +181 -0
- zenml/event_sources/__init__.py +14 -0
- zenml/{lineage_graph/edge.py → event_sources/base_event.py} +5 -7
- zenml/event_sources/base_event_source.py +695 -0
- zenml/event_sources/webhooks/__init__.py +14 -0
- zenml/event_sources/webhooks/base_webhook_event_source.py +231 -0
- zenml/exceptions.py +40 -41
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/hooks/hook_validators.py +3 -11
- zenml/image_builders/base_image_builder.py +5 -2
- zenml/image_builders/build_context.py +24 -80
- zenml/image_builders/local_image_builder.py +14 -6
- zenml/integrations/__init__.py +16 -4
- zenml/integrations/airflow/__init__.py +3 -5
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +15 -7
- zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +15 -252
- zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
- zenml/integrations/argilla/__init__.py +46 -0
- zenml/integrations/argilla/annotators/__init__.py +20 -0
- zenml/integrations/argilla/annotators/argilla_annotator.py +443 -0
- zenml/{post_execution → integrations/argilla/flavors}/__init__.py +9 -13
- zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +150 -0
- zenml/integrations/aws/__init__.py +7 -3
- zenml/integrations/aws/container_registries/aws_container_registry.py +44 -8
- zenml/integrations/aws/flavors/__init__.py +6 -0
- zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
- zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
- zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +104 -16
- zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
- zenml/integrations/aws/image_builders/__init__.py +20 -0
- zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +421 -133
- zenml/integrations/aws/service_connectors/aws_service_connector.py +175 -48
- zenml/integrations/aws/step_operators/sagemaker_step_operator.py +1 -1
- zenml/integrations/azure/__init__.py +14 -5
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/__init__.py +11 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +166 -0
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +71 -18
- zenml/integrations/azure/orchestrators/__init__.py +19 -0
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +583 -0
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +82 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +15 -6
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -173
- zenml/integrations/bentoml/__init__.py +1 -1
- zenml/integrations/bentoml/constants.py +1 -1
- zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
- zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +128 -239
- zenml/integrations/bentoml/services/__init__.py +15 -4
- zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
- zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +85 -43
- zenml/integrations/bentoml/services/deployment_type.py +23 -0
- zenml/integrations/bentoml/steps/bento_builder.py +2 -0
- zenml/integrations/bentoml/steps/bentoml_deployer.py +97 -47
- zenml/integrations/bitbucket/__init__.py +42 -0
- zenml/integrations/bitbucket/plugins/__init__.py +20 -0
- zenml/integrations/bitbucket/plugins/bitbucket_webhook_event_source_flavor.py +43 -0
- zenml/integrations/bitbucket/plugins/event_sources/__init__.py +0 -0
- zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +486 -0
- zenml/integrations/comet/__init__.py +49 -0
- zenml/integrations/{kserve/services → comet/experiment_trackers}/__init__.py +5 -6
- zenml/integrations/comet/experiment_trackers/comet_experiment_tracker.py +175 -0
- zenml/integrations/comet/flavors/__init__.py +24 -0
- zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +129 -0
- zenml/integrations/constants.py +14 -2
- zenml/integrations/databricks/__init__.py +70 -0
- zenml/{lineage_graph → integrations/databricks/flavors}/__init__.py +14 -18
- zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +174 -0
- zenml/integrations/{kserve → databricks}/model_deployers/__init__.py +5 -5
- zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
- zenml/integrations/databricks/orchestrators/__init__.py +20 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +501 -0
- zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
- zenml/integrations/databricks/services/__init__.py +19 -0
- zenml/integrations/databricks/services/databricks_deployment.py +407 -0
- zenml/integrations/{gcp/orchestrators/vertex_scheduler → databricks/utils}/__init__.py +2 -2
- zenml/integrations/databricks/utils/databricks_utils.py +87 -0
- zenml/integrations/deepchecks/__init__.py +30 -10
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +59 -18
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/materializers/deepchecks_results_materializer.py +3 -3
- zenml/integrations/deepchecks/validation_checks.py +62 -35
- zenml/integrations/discord/__init__.py +1 -0
- zenml/integrations/discord/steps/discord_alerter_ask_step.py +1 -0
- zenml/integrations/discord/steps/discord_alerter_post_step.py +1 -0
- zenml/integrations/evidently/__init__.py +22 -4
- zenml/integrations/evidently/column_mapping.py +11 -3
- zenml/integrations/evidently/data_validators/evidently_data_validator.py +24 -6
- zenml/integrations/evidently/metrics.py +7 -8
- zenml/integrations/evidently/tests.py +7 -8
- zenml/integrations/facets/__init__.py +22 -5
- zenml/integrations/facets/models.py +2 -7
- zenml/integrations/feast/__init__.py +19 -3
- zenml/integrations/feast/feature_stores/feast_feature_store.py +13 -32
- zenml/integrations/gcp/__init__.py +6 -6
- zenml/integrations/gcp/artifact_stores/gcp_artifact_store.py +6 -0
- zenml/integrations/gcp/flavors/gcp_artifact_store_flavor.py +1 -0
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +24 -2
- zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +14 -1
- zenml/integrations/gcp/google_credentials_mixin.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +519 -296
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +788 -113
- zenml/integrations/gcp/step_operators/vertex_step_operator.py +9 -1
- zenml/integrations/github/__init__.py +15 -0
- zenml/integrations/github/code_repositories/github_code_repository.py +3 -2
- zenml/integrations/github/plugins/__init__.py +19 -0
- zenml/integrations/github/plugins/event_sources/__init__.py +0 -0
- zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +510 -0
- zenml/integrations/github/plugins/github_webhook_event_source_flavor.py +43 -0
- zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +7 -1
- zenml/integrations/great_expectations/__init__.py +22 -10
- zenml/integrations/great_expectations/data_validators/ge_data_validator.py +61 -57
- zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
- zenml/integrations/great_expectations/ge_store_backend.py +24 -11
- zenml/integrations/great_expectations/materializers/ge_materializer.py +9 -9
- zenml/integrations/great_expectations/utils.py +5 -5
- zenml/integrations/huggingface/__init__.py +52 -1
- zenml/integrations/huggingface/flavors/__init__.py +26 -0
- zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +130 -0
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +102 -19
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +18 -19
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +104 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +18 -19
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +13 -15
- zenml/integrations/huggingface/model_deployers/__init__.py +20 -0
- zenml/integrations/huggingface/model_deployers/huggingface_model_deployer.py +247 -0
- zenml/integrations/huggingface/services/__init__.py +19 -0
- zenml/integrations/huggingface/services/huggingface_deployment.py +292 -0
- zenml/integrations/huggingface/steps/__init__.py +21 -0
- zenml/integrations/huggingface/steps/accelerate_runner.py +166 -0
- zenml/integrations/huggingface/steps/huggingface_deployer.py +110 -0
- zenml/integrations/hyperai/__init__.py +53 -0
- zenml/integrations/hyperai/flavors/__init__.py +20 -0
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +170 -0
- zenml/integrations/hyperai/orchestrators/__init__.py +21 -0
- zenml/integrations/hyperai/orchestrators/hyperai_orchestrator.py +510 -0
- zenml/integrations/hyperai/service_connectors/__init__.py +20 -0
- zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +375 -0
- zenml/integrations/integration.py +101 -47
- zenml/integrations/kaniko/flavors/kaniko_image_builder_flavor.py +6 -44
- zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +9 -6
- zenml/integrations/kubeflow/__init__.py +4 -1
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +67 -82
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +306 -248
- zenml/integrations/kubernetes/__init__.py +6 -3
- zenml/integrations/kubernetes/flavors/__init__.py +8 -0
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +43 -2
- zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +67 -12
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +89 -21
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +39 -7
- zenml/integrations/kubernetes/orchestrators/manifest_utils.py +42 -19
- zenml/integrations/kubernetes/pod_settings.py +21 -31
- zenml/integrations/kubernetes/serialization_utils.py +3 -3
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +14 -11
- zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
- zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +236 -0
- zenml/integrations/label_studio/__init__.py +1 -3
- zenml/integrations/label_studio/annotators/label_studio_annotator.py +46 -14
- zenml/integrations/label_studio/flavors/__init__.py +2 -0
- zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +20 -5
- zenml/integrations/langchain/__init__.py +7 -1
- zenml/integrations/langchain/materializers/document_materializer.py +44 -8
- zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
- zenml/integrations/lightgbm/__init__.py +1 -0
- zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
- zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +168 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +617 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +303 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +55 -8
- zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +31 -13
- zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +32 -37
- zenml/integrations/mlflow/model_deployers/mlflow_model_deployer.py +42 -233
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +42 -49
- zenml/integrations/mlflow/services/mlflow_deployment.py +30 -5
- zenml/integrations/mlflow/steps/mlflow_deployer.py +25 -27
- zenml/integrations/mlflow/steps/mlflow_registry.py +3 -1
- zenml/integrations/modal/__init__.py +46 -0
- zenml/integrations/modal/flavors/__init__.py +26 -0
- zenml/integrations/modal/flavors/modal_step_operator_flavor.py +125 -0
- zenml/integrations/modal/step_operators/__init__.py +22 -0
- zenml/integrations/modal/step_operators/modal_step_operator.py +242 -0
- zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
- zenml/integrations/neptune/experiment_trackers/run_state.py +71 -55
- zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
- zenml/integrations/neural_prophet/__init__.py +6 -1
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/{kserve/constants.py → numpy/materializers/__init__.py} +5 -4
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/openai/__init__.py +1 -1
- zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
- zenml/{steps/external_artifact.py → integrations/pandas/__init__.py} +17 -11
- zenml/integrations/{kserve/custom_deployer → pandas/materializers}/__init__.py +5 -5
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/pigeon/__init__.py +44 -0
- zenml/integrations/pigeon/annotators/__init__.py +20 -0
- zenml/integrations/pigeon/annotators/pigeon_annotator.py +330 -0
- zenml/integrations/{kserve → pigeon}/flavors/__init__.py +7 -7
- zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +104 -0
- zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
- zenml/integrations/polars/__init__.py +1 -0
- zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
- zenml/integrations/prodigy/__init__.py +48 -0
- zenml/integrations/prodigy/annotators/__init__.py +20 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +275 -0
- zenml/integrations/prodigy/flavors/__init__.py +24 -0
- zenml/integrations/prodigy/flavors/prodigy_annotator_flavor.py +101 -0
- zenml/integrations/pycaret/__init__.py +6 -0
- zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
- zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
- zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
- zenml/integrations/registry.py +38 -1
- zenml/integrations/s3/__init__.py +2 -3
- zenml/integrations/s3/artifact_stores/s3_artifact_store.py +191 -9
- zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +22 -45
- zenml/integrations/s3/utils.py +39 -0
- zenml/integrations/seldon/__init__.py +18 -2
- zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +19 -141
- zenml/integrations/seldon/secret_schemas/secret_schemas.py +2 -2
- zenml/integrations/seldon/seldon_client.py +55 -70
- zenml/integrations/seldon/services/seldon_deployment.py +4 -5
- zenml/integrations/seldon/steps/seldon_deployer.py +21 -20
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +29 -7
- zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +47 -28
- zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
- zenml/integrations/skypilot_aws/__init__.py +3 -2
- zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +2 -4
- zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
- zenml/integrations/skypilot_gcp/__init__.py +3 -2
- zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
- zenml/integrations/skypilot_kubernetes/__init__.py +52 -0
- zenml/integrations/skypilot_kubernetes/flavors/__init__.py +26 -0
- zenml/integrations/skypilot_kubernetes/flavors/skypilot_orchestrator_kubernetes_vm_flavor.py +125 -0
- zenml/integrations/skypilot_kubernetes/orchestrators/__init__.py +25 -0
- zenml/integrations/skypilot_kubernetes/orchestrators/skypilot_kubernetes_vm_orchestrator.py +74 -0
- zenml/integrations/skypilot_lambda/__init__.py +50 -0
- zenml/integrations/{kserve/secret_schemas → skypilot_lambda/flavors}/__init__.py +9 -12
- zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +130 -0
- zenml/{lineage_graph/node → integrations/skypilot_lambda/orchestrators}/__init__.py +8 -15
- zenml/integrations/skypilot_lambda/orchestrators/skypilot_lambda_vm_orchestrator.py +92 -0
- zenml/integrations/slack/__init__.py +1 -0
- zenml/integrations/slack/alerters/slack_alerter.py +22 -2
- zenml/integrations/slack/flavors/__init__.py +2 -0
- zenml/integrations/slack/flavors/slack_alerter_flavor.py +13 -4
- zenml/integrations/slack/steps/slack_alerter_ask_step.py +1 -0
- zenml/integrations/slack/steps/slack_alerter_post_step.py +1 -0
- zenml/integrations/spark/flavors/spark_step_operator_flavor.py +2 -39
- zenml/integrations/spark/step_operators/kubernetes_step_operator.py +1 -0
- zenml/integrations/spark/step_operators/spark_entrypoint_configuration.py +1 -0
- zenml/integrations/spark/step_operators/spark_step_operator.py +2 -0
- zenml/integrations/tekton/__init__.py +2 -1
- zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
- zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +549 -235
- zenml/integrations/tensorboard/__init__.py +1 -13
- zenml/integrations/tensorboard/services/tensorboard_service.py +3 -4
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +66 -59
- zenml/integrations/tensorflow/__init__.py +10 -15
- zenml/integrations/tensorflow/materializers/keras_materializer.py +24 -27
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +9 -16
- zenml/integrations/vllm/__init__.py +50 -0
- zenml/integrations/vllm/flavors/__init__.py +21 -0
- zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
- zenml/integrations/vllm/model_deployers/__init__.py +19 -0
- zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
- zenml/integrations/vllm/services/__init__.py +19 -0
- zenml/integrations/vllm/services/vllm_deployment.py +206 -0
- zenml/integrations/wandb/__init__.py +1 -0
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +21 -18
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/integrations/whylogs/data_validators/whylogs_data_validator.py +3 -3
- zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
- zenml/integrations/whylogs/materializers/whylogs_materializer.py +14 -21
- zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
- zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
- zenml/io/fileio.py +1 -0
- zenml/io/filesystem.py +2 -2
- zenml/io/local_filesystem.py +3 -3
- zenml/logger.py +41 -17
- zenml/logging/__init__.py +5 -0
- zenml/logging/step_logging.py +280 -32
- zenml/{new/steps → login}/__init__.py +4 -1
- zenml/login/credentials.py +370 -0
- zenml/login/credentials_store.py +638 -0
- zenml/{new/pipelines → login/pro}/__init__.py +4 -1
- zenml/login/pro/client.py +492 -0
- zenml/login/pro/constants.py +28 -0
- zenml/{zen_stores/schemas/identity_schemas.py → login/pro/models.py} +9 -11
- zenml/login/pro/organization/__init__.py +14 -0
- zenml/login/pro/organization/client.py +79 -0
- zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
- zenml/{_hub → login/pro/tenant}/__init__.py +2 -2
- zenml/login/pro/tenant/client.py +92 -0
- zenml/login/pro/tenant/models.py +176 -0
- zenml/login/pro/utils.py +107 -0
- zenml/login/server_info.py +52 -0
- zenml/{cli → login}/web_login.py +71 -21
- zenml/materializers/__init__.py +2 -4
- zenml/materializers/base_materializer.py +180 -51
- zenml/materializers/built_in_materializer.py +50 -23
- zenml/materializers/cloudpickle_materializer.py +4 -5
- zenml/materializers/numpy_materializer.py +23 -233
- zenml/materializers/pandas_materializer.py +22 -174
- zenml/materializers/pydantic_materializer.py +2 -2
- zenml/materializers/service_materializer.py +12 -10
- zenml/materializers/structured_string_materializer.py +12 -6
- zenml/materializers/uuid_materializer.py +79 -0
- zenml/metadata/lazy_load.py +33 -17
- zenml/metadata/metadata_types.py +112 -3
- zenml/model/lazy_load.py +85 -4
- zenml/model/model.py +236 -226
- zenml/model/utils.py +99 -141
- zenml/model_deployers/base_model_deployer.py +319 -47
- zenml/model_registries/base_model_registry.py +17 -15
- zenml/models/__init__.py +254 -175
- zenml/models/v2/base/base.py +254 -112
- zenml/models/v2/base/base_plugin_flavor.py +76 -0
- zenml/models/v2/base/filter.py +530 -199
- zenml/models/v2/base/page.py +2 -12
- zenml/models/v2/base/scoped.py +292 -22
- zenml/models/v2/core/action.py +276 -0
- zenml/models/v2/core/action_flavor.py +57 -0
- zenml/models/v2/core/api_key.py +35 -12
- zenml/models/v2/core/artifact.py +126 -6
- zenml/models/v2/core/artifact_version.py +289 -65
- zenml/models/v2/core/artifact_visualization.py +10 -4
- zenml/models/v2/core/code_reference.py +13 -4
- zenml/models/v2/core/code_repository.py +39 -16
- zenml/models/v2/core/component.py +113 -61
- zenml/models/v2/core/device.py +25 -5
- zenml/models/v2/core/event_source.py +244 -0
- zenml/models/v2/core/event_source_flavor.py +67 -0
- zenml/models/v2/core/flavor.py +90 -27
- zenml/models/v2/core/logs.py +81 -12
- zenml/models/v2/core/model.py +91 -42
- zenml/models/v2/core/model_version.py +100 -55
- zenml/models/v2/core/model_version_artifact.py +96 -89
- zenml/models/v2/core/model_version_pipeline_run.py +86 -53
- zenml/models/v2/core/pipeline.py +205 -80
- zenml/models/v2/core/pipeline_build.py +110 -21
- zenml/models/v2/core/pipeline_deployment.py +101 -36
- zenml/models/v2/core/pipeline_run.py +545 -33
- zenml/models/v2/core/run_metadata.py +23 -181
- zenml/models/v2/core/run_template.py +439 -0
- zenml/models/v2/core/schedule.py +66 -29
- zenml/models/v2/core/secret.py +33 -21
- zenml/models/v2/core/server_settings.py +224 -0
- zenml/models/v2/core/service.py +500 -0
- zenml/models/v2/core/service_account.py +40 -15
- zenml/models/v2/core/service_connector.py +247 -49
- zenml/models/v2/core/stack.py +163 -70
- zenml/models/v2/core/step_run.py +210 -48
- zenml/models/v2/core/tag.py +21 -8
- zenml/models/v2/core/tag_resource.py +13 -4
- zenml/models/v2/core/trigger.py +422 -0
- zenml/models/v2/core/trigger_execution.py +119 -0
- zenml/models/v2/core/user.py +136 -69
- zenml/models/v2/core/workspace.py +26 -7
- zenml/models/v2/misc/auth_models.py +11 -2
- zenml/models/v2/misc/build_item.py +3 -3
- zenml/models/v2/misc/external_user.py +3 -6
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/models/v2/misc/loaded_visualization.py +2 -2
- zenml/models/v2/misc/run_metadata.py +38 -0
- zenml/models/v2/misc/server_models.py +100 -0
- zenml/models/v2/misc/service_connector_type.py +9 -17
- zenml/models/v2/misc/stack_deployment.py +96 -0
- zenml/models/v2/misc/user_auth.py +7 -9
- zenml/orchestrators/__init__.py +4 -0
- zenml/orchestrators/base_orchestrator.py +136 -25
- zenml/orchestrators/containerized_orchestrator.py +1 -0
- zenml/orchestrators/dag_runner.py +18 -3
- zenml/orchestrators/input_utils.py +109 -48
- zenml/orchestrators/local/local_orchestrator.py +10 -0
- zenml/orchestrators/local_docker/local_docker_orchestrator.py +14 -42
- zenml/orchestrators/output_utils.py +16 -6
- zenml/orchestrators/publish_utils.py +12 -5
- zenml/orchestrators/step_launcher.py +142 -194
- zenml/orchestrators/step_run_utils.py +386 -0
- zenml/orchestrators/step_runner.py +181 -270
- zenml/orchestrators/utils.py +219 -84
- zenml/orchestrators/wheeled_orchestrator.py +147 -0
- zenml/pipelines/__init__.py +3 -16
- zenml/{new/pipelines → pipelines}/build_utils.py +287 -47
- zenml/{new/pipelines → pipelines}/pipeline_context.py +6 -2
- zenml/pipelines/pipeline_decorator.py +40 -64
- zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +376 -440
- zenml/pipelines/run_utils.py +358 -0
- zenml/plugins/__init__.py +0 -0
- zenml/plugins/base_plugin_flavor.py +88 -0
- zenml/plugins/plugin_flavor_registry.py +342 -0
- zenml/secret/base_secret.py +7 -8
- zenml/secret/schemas/basic_auth_secret_schema.py +0 -1
- zenml/service_connectors/docker_service_connector.py +19 -4
- zenml/service_connectors/service_connector.py +12 -14
- zenml/service_connectors/service_connector_registry.py +71 -55
- zenml/service_connectors/service_connector_utils.py +418 -0
- zenml/services/__init__.py +0 -2
- zenml/services/container/container_service.py +9 -6
- zenml/services/container/container_service_endpoint.py +1 -1
- zenml/services/container/entrypoint.py +3 -2
- zenml/services/local/local_daemon_entrypoint.py +9 -6
- zenml/services/local/local_service.py +1 -1
- zenml/services/local/local_service_endpoint.py +1 -1
- zenml/services/service.py +222 -130
- zenml/services/service_status.py +2 -1
- zenml/services/service_type.py +6 -5
- zenml/stack/flavor.py +25 -18
- zenml/stack/flavor_registry.py +4 -4
- zenml/stack/stack.py +20 -131
- zenml/stack/stack_component.py +136 -110
- zenml/stack/utils.py +36 -15
- zenml/stack_deployments/__init__.py +14 -0
- zenml/stack_deployments/aws_stack_deployment.py +320 -0
- zenml/stack_deployments/azure_stack_deployment.py +315 -0
- zenml/stack_deployments/gcp_stack_deployment.py +315 -0
- zenml/stack_deployments/stack_deployment.py +232 -0
- zenml/stack_deployments/utils.py +48 -0
- zenml/step_operators/step_operator_entrypoint_configuration.py +2 -1
- zenml/steps/__init__.py +3 -9
- zenml/steps/base_step.py +172 -315
- zenml/{new/steps → steps}/decorated_step.py +1 -0
- zenml/steps/entrypoint_function_utils.py +33 -93
- zenml/{new/steps → steps}/step_context.py +70 -50
- zenml/steps/step_decorator.py +47 -93
- zenml/steps/step_invocation.py +22 -60
- zenml/steps/utils.py +161 -48
- zenml/types.py +14 -1
- zenml/utils/archivable.py +178 -0
- zenml/utils/callback_registry.py +71 -0
- zenml/utils/code_repository_utils.py +1 -0
- zenml/utils/code_utils.py +346 -0
- zenml/utils/cuda_utils.py +50 -0
- zenml/utils/dashboard_utils.py +67 -21
- zenml/utils/deprecation_utils.py +22 -24
- zenml/utils/dict_utils.py +22 -0
- zenml/utils/docker_utils.py +34 -5
- zenml/utils/downloaded_repository_context.py +1 -0
- zenml/utils/env_utils.py +55 -1
- zenml/utils/filesync_model.py +65 -28
- zenml/utils/function_utils.py +260 -0
- zenml/utils/integration_utils.py +1 -0
- zenml/utils/json_utils.py +131 -0
- zenml/utils/materializer_utils.py +1 -1
- zenml/utils/metadata_utils.py +368 -0
- zenml/utils/notebook_utils.py +136 -0
- zenml/utils/package_utils.py +89 -0
- zenml/utils/pagination_utils.py +9 -7
- zenml/utils/pipeline_docker_image_builder.py +152 -149
- zenml/utils/pydantic_utils.py +276 -66
- zenml/utils/requirements_utils.py +71 -0
- zenml/utils/secret_utils.py +66 -12
- zenml/utils/settings_utils.py +2 -1
- zenml/utils/singleton.py +15 -3
- zenml/utils/source_code_utils.py +1 -0
- zenml/utils/source_utils.py +236 -14
- zenml/utils/string_utils.py +140 -0
- zenml/utils/typed_model.py +5 -3
- zenml/utils/typing_utils.py +223 -0
- zenml/utils/visualization_utils.py +5 -3
- zenml/utils/yaml_utils.py +1 -1
- zenml/zen_server/auth.py +387 -55
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/cloud_utils.py +253 -0
- zenml/zen_server/csrf.py +91 -0
- zenml/zen_server/dashboard/assets/404-Dfq64Boz.js +1 -0
- zenml/zen_server/dashboard/assets/@radix-DeK6qiuw.js +85 -0
- zenml/zen_server/dashboard/assets/@react-router-B3Z5rLr2.js +29 -0
- zenml/zen_server/dashboard/assets/@reactflow-BUNIMFeC.js +17 -0
- zenml/zen_server/dashboard/assets/@reactflow-C26Olbza.css +1 -0
- zenml/zen_server/dashboard/assets/@tanstack-DT5WLu9C.js +22 -0
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-B73Vs10T.js +1 -0
- zenml/zen_server/dashboard/assets/CodeSnippet-Bbx6fIb6.css +1 -0
- zenml/zen_server/dashboard/assets/CodeSnippet-DIJRT2NT.js +9 -0
- zenml/zen_server/dashboard/assets/CollapsibleCard-BzUHGZOU.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-BEGyld4c.js +1 -0
- zenml/zen_server/dashboard/assets/ComponentBadge-xyKiek1s.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-DhW-mapu.js +2 -0
- zenml/zen_server/dashboard/assets/CsvVizualization-D8oazBiE.js +15 -0
- zenml/zen_server/dashboard/assets/DeleteAlertDialog-WkSIIgfy.js +1 -0
- zenml/zen_server/dashboard/assets/DialogItem-Bgroeg29.js +1 -0
- zenml/zen_server/dashboard/assets/DisplayDate-CDMUcQHS.js +1 -0
- zenml/zen_server/dashboard/assets/EmptyState-BzdlCwp3.js +1 -0
- zenml/zen_server/dashboard/assets/Error-CY5tlu17.js +1 -0
- zenml/zen_server/dashboard/assets/ExecutionStatus-G8mjIaeA.js +1 -0
- zenml/zen_server/dashboard/assets/Helpbox-Bb1ed--O.js +1 -0
- zenml/zen_server/dashboard/assets/Infobox-Da6-76M2.js +1 -0
- zenml/zen_server/dashboard/assets/InlineAvatar-DqnZaBNq.js +1 -0
- zenml/zen_server/dashboard/assets/Lock-CYYy18Mm.js +1 -0
- zenml/zen_server/dashboard/assets/MarkdownVisualization-ylXaAxev.js +14 -0
- zenml/zen_server/dashboard/assets/NestedCollapsible-aK5ojKoF.js +1 -0
- zenml/zen_server/dashboard/assets/NumberBox-Dtp3J6g5.js +1 -0
- zenml/zen_server/dashboard/assets/Partials-CqZp5NMX.js +1 -0
- zenml/zen_server/dashboard/assets/PasswordChecker-B0nadgh6.js +1 -0
- zenml/zen_server/dashboard/assets/ProBadge-B4tRUYve.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-CZuP29Qz.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bd7GUQ1_.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-mstdqzsS.js +1 -0
- zenml/zen_server/dashboard/assets/RunSelector-CsruSB4i.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-DxxtWVYz.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-D6tPxyqw.js +1 -0
- zenml/zen_server/dashboard/assets/SecretTooltip-CLzJIYW_.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-Yn50ooBC.js +1 -0
- zenml/zen_server/dashboard/assets/StackList-U537qoYd.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-CNv-eTYM.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-jEIevzVf.js +1 -0
- zenml/zen_server/dashboard/assets/UpdatePasswordSchemas-C16GW-kX.js +1 -0
- zenml/zen_server/dashboard/assets/UsageReason-Bf2tzhv1.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-D6i-AP1K.js +1 -0
- zenml/zen_server/dashboard/assets/acp-DOsXjFc7.webp +0 -0
- zenml/zen_server/dashboard/assets/adam-e-y0WnB_.webp +0 -0
- zenml/zen_server/dashboard/assets/alex-DcCuDHPg.webp +0 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-DUti43aF.js +1 -0
- zenml/zen_server/dashboard/assets/baris-C0ZrZ10g.webp +0 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/check-circle-jNbX5-sR.js +1 -0
- zenml/zen_server/dashboard/assets/chevron-down-6JyMkfjR.js +1 -0
- zenml/zen_server/dashboard/assets/chevron-right-double-D7ojK9Co.js +1 -0
- zenml/zen_server/dashboard/assets/cloud-squares-DeRLMopf.svg +43 -0
- zenml/zen_server/dashboard/assets/code-browser-CUFUIHfp.js +1 -0
- zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
- zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
- zenml/zen_server/dashboard/assets/connectors-video-C9qY4syJ.svg +21 -0
- zenml/zen_server/dashboard/assets/copy-C8XQA2Ug.js +1 -0
- zenml/zen_server/dashboard/assets/create-stack-Ch2WPs9U.js +1 -0
- zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-Byf9hTjA.js +1 -0
- zenml/zen_server/dashboard/assets/docker-BdA9vrnW.js +1 -0
- zenml/zen_server/dashboard/assets/dots-horizontal-otGBOSDJ.js +1 -0
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-BZqKBPBF.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-CFtm4BA7.js +1 -0
- zenml/zen_server/dashboard/assets/hamza-NKKOZz1I.webp +0 -0
- zenml/zen_server/dashboard/assets/help-Cc9bBIJH.js +1 -0
- zenml/zen_server/dashboard/assets/index-CE0aQlv8.js +55 -0
- zenml/zen_server/dashboard/assets/index-CtdYkjUi.js +1 -0
- zenml/zen_server/dashboard/assets/index-CyBKZcpO.js +1 -0
- zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
- zenml/zen_server/dashboard/assets/index-Uu49AX48.js +1 -0
- zenml/zen_server/dashboard/assets/index-v6gQjDEo.js +1 -0
- zenml/zen_server/dashboard/assets/index.esm-Dy6Z9Ung.js +1 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
- zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
- zenml/zen_server/dashboard/assets/kubernetes-B2wmAJ1d.js +1 -0
- zenml/zen_server/dashboard/assets/layout-BtHBmE4w.js +1 -0
- zenml/zen_server/dashboard/assets/link-external-b9AXw_sW.js +1 -0
- zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
- zenml/zen_server/dashboard/assets/login-mutation-DNDVp_2H.js +1 -0
- zenml/zen_server/dashboard/assets/logs-WMSM52RF.js +1 -0
- zenml/zen_server/dashboard/assets/mcp-Cb1aMeoq.webp +0 -0
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/not-found-Bmup4ctE.js +1 -0
- zenml/zen_server/dashboard/assets/package-C6uypY4h.js +1 -0
- zenml/zen_server/dashboard/assets/page--XLMzHrn.js +1 -0
- zenml/zen_server/dashboard/assets/page-ANYGfEUL.js +1 -0
- zenml/zen_server/dashboard/assets/page-B5Sr8pib.js +1 -0
- zenml/zen_server/dashboard/assets/page-BC27C_OI.js +2 -0
- zenml/zen_server/dashboard/assets/page-BNxYrN0q.js +1 -0
- zenml/zen_server/dashboard/assets/page-BYJfqgLN.js +1 -0
- zenml/zen_server/dashboard/assets/page-B_0XkV48.js +1 -0
- zenml/zen_server/dashboard/assets/page-BrmJp1Wt.js +1 -0
- zenml/zen_server/dashboard/assets/page-C2nU3Gxn.js +1 -0
- zenml/zen_server/dashboard/assets/page-C70wZtV2.js +1 -0
- zenml/zen_server/dashboard/assets/page-CHRn1fQm.js +1 -0
- zenml/zen_server/dashboard/assets/page-CWr96ZKN.js +1 -0
- zenml/zen_server/dashboard/assets/page-CXAbSyp9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CaeI9ptC.js +1 -0
- zenml/zen_server/dashboard/assets/page-Cc8ZEuj4.js +1 -0
- zenml/zen_server/dashboard/assets/page-CltCNL0T.js +1 -0
- zenml/zen_server/dashboard/assets/page-CmlYj7Nl.js +1 -0
- zenml/zen_server/dashboard/assets/page-D6Ev5P8V.js +1 -0
- zenml/zen_server/dashboard/assets/page-D9Oh05fl.js +1 -0
- zenml/zen_server/dashboard/assets/page-DGlm1RVc.js +1 -0
- zenml/zen_server/dashboard/assets/page-DN4BVIOL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dif8CWyZ.js +1 -0
- zenml/zen_server/dashboard/assets/page-DlIi5ThM.js +1 -0
- zenml/zen_server/dashboard/assets/page-DoW7YxTu.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dth9X1Ih.js +1 -0
- zenml/zen_server/dashboard/assets/page-DweqqCkF.js +1 -0
- zenml/zen_server/dashboard/assets/page-DyOJ_pq3.js +1 -0
- zenml/zen_server/dashboard/assets/page-Hn8q9iJZ.js +1 -0
- zenml/zen_server/dashboard/assets/page-IhckKFnD.js +6 -0
- zenml/zen_server/dashboard/assets/page-LyZ_l8vR.js +1 -0
- zenml/zen_server/dashboard/assets/page-PamGpk0j.js +1 -0
- zenml/zen_server/dashboard/assets/page-PxOWfKgF.js +2 -0
- zenml/zen_server/dashboard/assets/persist-DeXRG61d.js +1 -0
- zenml/zen_server/dashboard/assets/persist-vP0-Xl4f.js +1 -0
- zenml/zen_server/dashboard/assets/plus-tf1V2hTJ.js +1 -0
- zenml/zen_server/dashboard/assets/refresh-BjOeWlEq.js +1 -0
- zenml/zen_server/dashboard/assets/repos-video-D8kpu60k.svg +9 -0
- zenml/zen_server/dashboard/assets/rocket-DjT2cDvG.js +1 -0
- zenml/zen_server/dashboard/assets/service-DH_oUqQj.js +2 -0
- zenml/zen_server/dashboard/assets/settings_preview-0JLrRgHP.webp +0 -0
- zenml/zen_server/dashboard/assets/sharedSchema-Bw1_Wa7l.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-B_0R_fd6.js +1 -0
- zenml/zen_server/dashboard/assets/stefan-B08Ftbba.webp +0 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/tick-circle-BEX_Tp4v.js +1 -0
- zenml/zen_server/dashboard/assets/tour-cover-BYfeen6M.webp +0 -0
- zenml/zen_server/dashboard/assets/trash-arLUMWMS.js +1 -0
- zenml/zen_server/dashboard/assets/update-server-settings-mutation-D9qYhfaN.js +1 -0
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-Dh93fvh0.js +1 -0
- zenml/zen_server/dashboard/assets/zod-BwEbpOxH.js +1 -0
- zenml/zen_server/dashboard/index.html +19 -1
- zenml/zen_server/deploy/__init__.py +7 -16
- zenml/zen_server/deploy/base_provider.py +49 -78
- zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
- zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +48 -66
- zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +78 -62
- zenml/zen_server/deploy/deployer.py +94 -175
- zenml/zen_server/deploy/deployment.py +23 -17
- zenml/zen_server/deploy/docker/docker_provider.py +15 -31
- zenml/zen_server/deploy/docker/docker_zen_server.py +30 -35
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +3 -13
- zenml/zen_server/deploy/helm/templates/NOTES.txt +23 -7
- zenml/zen_server/deploy/helm/templates/_environment.tpl +175 -23
- zenml/zen_server/deploy/helm/templates/server-db-job.yaml +45 -18
- zenml/zen_server/deploy/helm/templates/server-db-pvc.yaml +25 -0
- zenml/zen_server/deploy/helm/templates/server-deployment.yaml +22 -6
- zenml/zen_server/deploy/helm/templates/server-secret.yaml +11 -10
- zenml/zen_server/deploy/helm/values.yaml +210 -28
- zenml/zen_server/exceptions.py +20 -1
- zenml/zen_server/feature_gate/__init__.py +13 -0
- zenml/zen_server/feature_gate/endpoint_utils.py +61 -0
- zenml/zen_server/feature_gate/feature_gate_interface.py +49 -0
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +125 -0
- zenml/zen_server/jwt.py +64 -32
- zenml/zen_server/rate_limit.py +200 -0
- zenml/zen_server/rbac/endpoint_utils.py +92 -9
- zenml/zen_server/rbac/models.py +21 -17
- zenml/zen_server/rbac/rbac_sql_zen_store.py +175 -0
- zenml/zen_server/rbac/utils.py +71 -30
- zenml/zen_server/rbac/zenml_cloud_rbac.py +13 -188
- zenml/zen_server/routers/actions_endpoints.py +324 -0
- zenml/zen_server/routers/artifact_version_endpoints.py +28 -2
- zenml/zen_server/routers/auth_endpoints.py +249 -131
- zenml/zen_server/routers/code_repositories_endpoints.py +1 -0
- zenml/zen_server/routers/devices_endpoints.py +56 -49
- zenml/zen_server/routers/event_source_endpoints.py +327 -0
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/model_versions_endpoints.py +59 -0
- zenml/zen_server/routers/models_endpoints.py +7 -1
- zenml/zen_server/routers/pipeline_builds_endpoints.py +6 -1
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +35 -0
- zenml/zen_server/routers/pipelines_endpoints.py +19 -32
- zenml/zen_server/routers/plugin_endpoints.py +107 -0
- zenml/zen_server/routers/run_templates_endpoints.py +212 -0
- zenml/zen_server/routers/runs_endpoints.py +91 -32
- zenml/zen_server/routers/schedule_endpoints.py +1 -0
- zenml/zen_server/routers/secrets_endpoints.py +4 -2
- zenml/zen_server/routers/server_endpoints.py +186 -4
- zenml/zen_server/routers/service_connectors_endpoints.py +56 -0
- zenml/zen_server/routers/service_endpoints.py +180 -0
- zenml/zen_server/routers/stack_components_endpoints.py +2 -1
- zenml/zen_server/routers/stack_deployment_endpoints.py +164 -0
- zenml/zen_server/routers/steps_endpoints.py +21 -12
- zenml/zen_server/routers/triggers_endpoints.py +336 -0
- zenml/zen_server/routers/users_endpoints.py +280 -45
- zenml/zen_server/routers/webhook_endpoints.py +127 -0
- zenml/zen_server/routers/workspaces_endpoints.py +220 -185
- zenml/zen_server/secure_headers.py +120 -0
- zenml/{new → zen_server/template_execution}/__init__.py +1 -1
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +42 -0
- zenml/zen_server/template_execution/utils.py +474 -0
- zenml/zen_server/template_execution/workload_manager_interface.py +92 -0
- zenml/zen_server/utils.py +374 -74
- zenml/zen_server/zen_server_api.py +299 -52
- zenml/zen_stores/base_zen_store.py +90 -58
- zenml/zen_stores/migrations/alembic.py +22 -9
- zenml/zen_stores/migrations/env.py +2 -2
- zenml/zen_stores/migrations/utils.py +731 -0
- zenml/zen_stores/migrations/versions/0.21.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.21.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.22.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.23.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.30.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.31.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.31.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.32.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.32.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.33.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.34.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.35.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.35.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.36.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.36.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.37.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.38.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.39.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.39.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.40.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.41.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.42.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.42.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.43.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.44.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.1_release_0_45_1.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.2_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.3_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.4_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.5_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.45.6_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.46.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.46.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.47.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.50.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.51.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.52.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.53.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.53.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.54.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.54.1_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.55.0_release.py +0 -1
- zenml/zen_stores/migrations/versions/0.55.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.3_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.4_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.55.5_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.3_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.56.4_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0.rc1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0.rc2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.57.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.58.2_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.60.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.72.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
- zenml/zen_stores/migrations/versions/03742aa7fdd7_add_secrets.py +1 -0
- zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +94 -0
- zenml/zen_stores/migrations/versions/0b06faa59c93_add_service_connectors.py +1 -0
- zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
- zenml/zen_stores/migrations/versions/0e4735b23577_increase_pipeline_spec_field_length.py +1 -0
- zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +6 -3
- zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +2 -1
- zenml/zen_stores/migrations/versions/14d687c8fa1c_rename_model_config_to_model_version.py +1 -0
- zenml/zen_stores/migrations/versions/19f27d5b234e_add_build_and_deployment_tables.py +1 -0
- zenml/zen_stores/migrations/versions/1a9a9d2a836d_admin_users.py +56 -0
- zenml/zen_stores/migrations/versions/1ac1b9c04da1_make_secrets_values_optional.py +1 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/1d74e596abb8_add_run_once_start_time_to_schedule.py +36 -0
- zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
- zenml/zen_stores/migrations/versions/248dfd320b68_update_size_of_flavor_config_schema.py +1 -0
- zenml/zen_stores/migrations/versions/25155145c545_separate_actions_and_triggers.py +228 -0
- zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
- zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +9 -10
- zenml/zen_stores/migrations/versions/2d201872e23c_remove_db_dependency_loop.py +29 -0
- zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +4 -3
- zenml/zen_stores/migrations/versions/389046140cad_data_versioning.py +1 -0
- zenml/zen_stores/migrations/versions/3944116bbd56_rename_project_to_workspace.py +1 -0
- zenml/zen_stores/migrations/versions/3b68abe58f44_add_model_watchtower_entities.py +1 -0
- zenml/zen_stores/migrations/versions/3c5a367730c2_add_environment_info_to_runs.py +1 -0
- zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
- zenml/zen_stores/migrations/versions/43a86093b60e_add_labels_for_stack_components.py +1 -0
- zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +123 -0
- zenml/zen_stores/migrations/versions/479103df60b6_add_triggers.py +162 -0
- zenml/zen_stores/migrations/versions/4a3087070f4e_add_step_source_code.py +1 -0
- zenml/zen_stores/migrations/versions/4c41c0ca42db_add_code_repository_table.py +1 -0
- zenml/zen_stores/migrations/versions/4d688d8f7aff_rename_model_version_to_model.py +1 -0
- zenml/zen_stores/migrations/versions/4e1972485075_endpoint_artifact_deployment_artifact.py +1 -0
- zenml/zen_stores/migrations/versions/4f66af55fbb9_rename_model_config_model_to_model_.py +1 -0
- zenml/zen_stores/migrations/versions/5330ba58bf20_rename_tables_and_foreign_keys.py +8 -9
- zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +4 -2
- zenml/zen_stores/migrations/versions/5cc3f41cf048_add_save_models_to_registry.py +1 -0
- zenml/zen_stores/migrations/versions/6119cd9b93c2_tags_table.py +1 -0
- zenml/zen_stores/migrations/versions/623a234c11f5_add_sdk_docs_url_to_flavors.py +1 -0
- zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +5 -4
- zenml/zen_stores/migrations/versions/6a28c4fd0ef2_add_caching_info.py +1 -0
- zenml/zen_stores/migrations/versions/6f707b385dc1_fix_model_artifacts.py +1 -0
- zenml/zen_stores/migrations/versions/722392c91006_make_is_service_account_mandatory.py +1 -0
- zenml/zen_stores/migrations/versions/72675226b2de_unique_users.py +31 -0
- zenml/zen_stores/migrations/versions/72722dee4686_track_server_version.py +1 -0
- zenml/zen_stores/migrations/versions/7280c14811d6_use_text_type.py +1 -0
- zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +4 -2
- zenml/zen_stores/migrations/versions/729263e47b55_fix_external_input_artifacts.py +1 -0
- zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +3 -2
- zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +4 -2
- zenml/zen_stores/migrations/versions/76a7b9451ccd_add_build_template_deployment_id.py +52 -0
- zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +9 -7
- zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +7 -7
- zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
- zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +3 -2
- zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +2 -1
- zenml/zen_stores/migrations/versions/86fa52918b54_remove_teams_and_roles.py +1 -0
- zenml/zen_stores/migrations/versions/8a64fbfecda0_add_num_outputs_to_run_step.py +1 -0
- zenml/zen_stores/migrations/versions/8ed03137cacc_polymorthic_run_metadata.py +1 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
- zenml/zen_stores/migrations/versions/93cbda80a732_add_service_accounts.py +1 -0
- zenml/zen_stores/migrations/versions/979eff8fc4b1_add_code_repo_description_and_logo_url.py +1 -0
- zenml/zen_stores/migrations/versions/9971237fa937_artifact_visualizations.py +1 -0
- zenml/zen_stores/migrations/versions/9d8020441014_increase_step_configuration_length.py +1 -0
- zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
- zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +3 -2
- zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +5 -4
- zenml/zen_stores/migrations/versions/ade72effebaf_added_logs_table.py +1 -0
- zenml/zen_stores/migrations/versions/alembic_start.py +2 -1
- zenml/zen_stores/migrations/versions/b4eccf34dfa3_add_hub_token_to_user_model.py +1 -0
- zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
- zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/migrations/versions/c1b18cec3a48_increase_length_on_flavor_config_schema.py +1 -0
- zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
- zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
- zenml/zen_stores/migrations/versions/cc9894cb58aa_add_user_metadata.py +41 -0
- zenml/zen_stores/migrations/versions/ccd68b7825ae_add_status_to_pipeline_and_step_run.py +1 -0
- zenml/zen_stores/migrations/versions/d02b3d3464cf_add_orchestrator_run_id_column.py +1 -0
- zenml/zen_stores/migrations/versions/d26471b6fe8f_update_build_filtering.py +1 -0
- zenml/zen_stores/migrations/versions/d7b3acf9aa46_create_schedule_table.py +1 -0
- zenml/zen_stores/migrations/versions/e1d66d91a099_add_stack_and_component_spec_paths_to_.py +1 -0
- zenml/zen_stores/migrations/versions/e5225281b4d3_add_connector_skew_tolerance.py +1 -0
- zenml/zen_stores/migrations/versions/e65aa6708ff7_pipeline_versioning.py +1 -0
- zenml/zen_stores/migrations/versions/ec0d785ca296_create_run_metadata_table.py +1 -0
- zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +119 -0
- zenml/zen_stores/migrations/versions/f3b3964e3a0f_add_oauth_devices.py +1 -0
- zenml/zen_stores/migrations/versions/f49904a80aa7_increase_length_of_artifact_table_sources.py +1 -0
- zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +5 -4
- zenml/zen_stores/rest_zen_store.py +1326 -305
- zenml/zen_stores/schemas/__init__.py +22 -3
- zenml/zen_stores/schemas/action_schemas.py +192 -0
- zenml/zen_stores/schemas/api_key_schemas.py +23 -10
- zenml/zen_stores/schemas/artifact_schemas.py +112 -49
- zenml/zen_stores/schemas/artifact_visualization_schemas.py +17 -8
- zenml/zen_stores/schemas/base_schemas.py +27 -0
- zenml/zen_stores/schemas/code_repository_schemas.py +25 -10
- zenml/zen_stores/schemas/component_schemas.py +74 -11
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/device_schemas.py +29 -15
- zenml/zen_stores/schemas/event_source_schemas.py +188 -0
- zenml/zen_stores/schemas/flavor_schemas.py +19 -9
- zenml/zen_stores/schemas/logs_schemas.py +12 -6
- zenml/zen_stores/schemas/model_schemas.py +192 -139
- zenml/zen_stores/schemas/pipeline_build_schemas.py +16 -16
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +58 -17
- zenml/zen_stores/schemas/pipeline_run_schemas.py +170 -35
- zenml/zen_stores/schemas/pipeline_schemas.py +51 -33
- zenml/zen_stores/schemas/run_metadata_schemas.py +28 -78
- zenml/zen_stores/schemas/run_template_schemas.py +267 -0
- zenml/zen_stores/schemas/schedule_schema.py +15 -5
- zenml/zen_stores/schemas/secret_schemas.py +18 -10
- zenml/zen_stores/schemas/server_settings_schemas.py +129 -0
- zenml/zen_stores/schemas/service_connector_schemas.py +13 -6
- zenml/zen_stores/schemas/service_schemas.py +258 -0
- zenml/zen_stores/schemas/stack_schemas.py +23 -6
- zenml/zen_stores/schemas/step_run_schemas.py +132 -41
- zenml/zen_stores/schemas/tag_schemas.py +31 -50
- zenml/zen_stores/schemas/trigger_schemas.py +316 -0
- zenml/zen_stores/schemas/user_schemas.py +66 -23
- zenml/zen_stores/schemas/utils.py +112 -0
- zenml/zen_stores/schemas/workspace_schemas.py +36 -19
- zenml/zen_stores/secrets_stores/aws_secrets_store.py +41 -32
- zenml/zen_stores/secrets_stores/azure_secrets_store.py +20 -23
- zenml/zen_stores/secrets_stores/base_secrets_store.py +80 -12
- zenml/zen_stores/secrets_stores/gcp_secrets_store.py +42 -33
- zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +7 -11
- zenml/zen_stores/secrets_stores/secrets_store_interface.py +1 -0
- zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +15 -8
- zenml/zen_stores/secrets_stores/sql_secrets_store.py +8 -9
- zenml/zen_stores/sql_zen_store.py +4062 -799
- zenml/zen_stores/template_utils.py +263 -0
- zenml/zen_stores/zen_store_interface.py +614 -44
- zenml_nightly-0.72.0.dev20250116.dist-info/METADATA +486 -0
- zenml_nightly-0.72.0.dev20250116.dist-info/RECORD +1294 -0
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/WHEEL +1 -1
- CLA.md +0 -110
- CODE-OF-CONDUCT.md +0 -132
- CONTRIBUTING.md +0 -260
- README.md +0 -304
- RELEASE_NOTES.md +0 -3919
- ROADMAP.md +0 -5
- SECURITY.md +0 -15
- zenml/_hub/client.py +0 -285
- zenml/_hub/constants.py +0 -21
- zenml/_hub/utils.py +0 -80
- zenml/api.py +0 -61
- zenml/cli/hub.py +0 -1115
- zenml/cli/stack_recipes.py +0 -469
- zenml/integrations/gcp/google_cloud_function.py +0 -187
- zenml/integrations/gcp/google_cloud_scheduler.py +0 -83
- zenml/integrations/gcp/orchestrators/vertex_scheduler/main.py +0 -91
- zenml/integrations/gcp/orchestrators/vertex_scheduler/requirements.txt +0 -2
- zenml/integrations/kserve/__init__.py +0 -57
- zenml/integrations/kserve/custom_deployer/zenml_custom_model.py +0 -175
- zenml/integrations/kserve/flavors/kserve_model_deployer_flavor.py +0 -137
- zenml/integrations/kserve/model_deployers/kserve_model_deployer.py +0 -1003
- zenml/integrations/kserve/secret_schemas/secret_schemas.py +0 -65
- zenml/integrations/kserve/services/kserve_deployment.py +0 -596
- zenml/integrations/kserve/steps/__init__.py +0 -22
- zenml/integrations/kserve/steps/kserve_deployer.py +0 -472
- zenml/integrations/kserve/steps/kserve_step_utils.py +0 -293
- zenml/integrations/kubeflow/utils.py +0 -96
- zenml/lineage_graph/lineage_graph.py +0 -244
- zenml/lineage_graph/node/artifact_node.py +0 -52
- zenml/lineage_graph/node/step_node.py +0 -41
- zenml/models/v2/base/internal.py +0 -37
- zenml/models/v2/base/update.py +0 -40
- zenml/models/v2/misc/hub_plugin_models.py +0 -79
- zenml/new/pipelines/deserialization_utils.py +0 -291
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/new/pipelines/pipeline_decorator.py +0 -109
- zenml/new/steps/step_decorator.py +0 -160
- zenml/pipelines/base_pipeline.py +0 -274
- zenml/post_execution/pipeline.py +0 -58
- zenml/post_execution/pipeline_run.py +0 -55
- zenml/services/service_registry.py +0 -214
- zenml/services/terraform/__init__.py +0 -14
- zenml/services/terraform/terraform_service.py +0 -441
- zenml/steps/step_environment.py +0 -108
- zenml/steps/step_output.py +0 -36
- zenml/utils/mlstacks_utils.py +0 -635
- zenml/utils/terraform_utils.py +0 -42
- zenml/zen_server/dashboard/_redirects +0 -1
- zenml/zen_server/dashboard/asset-manifest.json +0 -131
- zenml/zen_server/dashboard/manifest.json +0 -25
- zenml/zen_server/dashboard/precache-manifest.c139638dcc4d9d3425353266447a2fad.js +0 -462
- zenml/zen_server/dashboard/robots.txt +0 -2
- zenml/zen_server/dashboard/service-worker.js +0 -39
- zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css +0 -16
- zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css.map +0 -1
- zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css +0 -2
- zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css.map +0 -1
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js +0 -3
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.LICENSE.txt +0 -95
- zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.map +0 -1
- zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js +0 -2
- zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js.map +0 -1
- zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js +0 -2
- zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js.map +0 -1
- zenml/zen_server/dashboard/static/media/AlertTriangle.28aee535.svg +0 -5
- zenml/zen_server/dashboard/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
- zenml/zen_server/dashboard/static/media/Back.86c23a22.svg +0 -4
- zenml/zen_server/dashboard/static/media/BookOpen.5cb101ff.svg +0 -4
- zenml/zen_server/dashboard/static/media/BoundingBox.1eb98717.svg +0 -10
- zenml/zen_server/dashboard/static/media/Burger.9b1c67d7.svg +0 -3
- zenml/zen_server/dashboard/static/media/Cached.2381fb8d.svg +0 -1
- zenml/zen_server/dashboard/static/media/Calendar.356e11c7.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChartBarHorizontal.0247447b.svg +0 -6
- zenml/zen_server/dashboard/static/media/ChartLine.0d79e18d.svg +0 -4
- zenml/zen_server/dashboard/static/media/ChatDots.2e1c9211.svg +0 -6
- zenml/zen_server/dashboard/static/media/Check.dad6beb2.svg +0 -3
- zenml/zen_server/dashboard/static/media/CheckCircleFilled.c19566d0.svg +0 -3
- zenml/zen_server/dashboard/static/media/Checkbox.af50e31e.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronDown.f860ce32.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronDownLight.6642d756.svg +0 -3
- zenml/zen_server/dashboard/static/media/ChevronLeft.f6edfcdb.svg +0 -3
- zenml/zen_server/dashboard/static/media/CircleCheck.f98fd6ca.svg +0 -1
- zenml/zen_server/dashboard/static/media/Clock.ffc9de95.svg +0 -3
- zenml/zen_server/dashboard/static/media/Close.74e9efbc.svg +0 -5
- zenml/zen_server/dashboard/static/media/CloseWithBorder.6960930a.svg +0 -3
- zenml/zen_server/dashboard/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
- zenml/zen_server/dashboard/static/media/CloudArrowUp.0aecb235.svg +0 -6
- zenml/zen_server/dashboard/static/media/Code.ef0f33b5.svg +0 -3
- zenml/zen_server/dashboard/static/media/Config.0be63f8a.svg +0 -1
- zenml/zen_server/dashboard/static/media/Connector.9fd46ef1.svg +0 -10
- zenml/zen_server/dashboard/static/media/Copy.36e2112a.svg +0 -1
- zenml/zen_server/dashboard/static/media/Dashboard.d05787e0.svg +0 -3
- zenml/zen_server/dashboard/static/media/Data.b1c3b5f8.svg +0 -3
- zenml/zen_server/dashboard/static/media/Delete.3c361b28.svg +0 -8
- zenml/zen_server/dashboard/static/media/Docs.7541d478.svg +0 -7
- zenml/zen_server/dashboard/static/media/Download.fba04d87.svg +0 -5
- zenml/zen_server/dashboard/static/media/Edit.490eb294.svg +0 -6
- zenml/zen_server/dashboard/static/media/EmptyRightArrow.23749d01.svg +0 -3
- zenml/zen_server/dashboard/static/media/Example.6396cd37.svg +0 -5
- zenml/zen_server/dashboard/static/media/Extension.1394cd4a.svg +0 -3
- zenml/zen_server/dashboard/static/media/Eye.d9e4ee62.svg +0 -4
- zenml/zen_server/dashboard/static/media/Failed.0213c1a0.svg +0 -1
- zenml/zen_server/dashboard/static/media/FileText.1f15bacd.svg +0 -7
- zenml/zen_server/dashboard/static/media/Filter.ab6b9c0d.svg +0 -3
- zenml/zen_server/dashboard/static/media/Folders.12b29887.svg +0 -5
- zenml/zen_server/dashboard/static/media/FunnelFill.6df4c143.svg +0 -3
- zenml/zen_server/dashboard/static/media/GitCommit.7dd9c2aa.svg +0 -5
- zenml/zen_server/dashboard/static/media/GitHub_Logo.cefc2023.png +0 -0
- zenml/zen_server/dashboard/static/media/Graph.2c63a892.svg +0 -11
- zenml/zen_server/dashboard/static/media/History.08329240.svg +0 -3
- zenml/zen_server/dashboard/static/media/Home.0843b0d5.svg +0 -3
- zenml/zen_server/dashboard/static/media/ImageBuilder.ea762d9c.svg +0 -6
- zenml/zen_server/dashboard/static/media/InProgress.304a0edc.svg +0 -1
- zenml/zen_server/dashboard/static/media/Info.9fe10c5c.svg +0 -3
- zenml/zen_server/dashboard/static/media/KeyboardReturn.491afbe3.svg +0 -3
- zenml/zen_server/dashboard/static/media/Link.72bbb55d.svg +0 -4
- zenml/zen_server/dashboard/static/media/Lock.30f5e1fe.svg +0 -5
- zenml/zen_server/dashboard/static/media/Lock2.a769ea52.svg +0 -3
- zenml/zen_server/dashboard/static/media/LockKey.92f21621.svg +0 -6
- zenml/zen_server/dashboard/static/media/Logs.8bf4d005.svg +0 -5
- zenml/zen_server/dashboard/static/media/MinusCircle.4188f418.svg +0 -4
- zenml/zen_server/dashboard/static/media/ModelRegistry.f0de050a.svg +0 -6
- zenml/zen_server/dashboard/static/media/MultiUser.a2ba7c67.svg +0 -10
- zenml/zen_server/dashboard/static/media/PaginationFirst.92628634.svg +0 -4
- zenml/zen_server/dashboard/static/media/PaginationLast.00d3c732.svg +0 -4
- zenml/zen_server/dashboard/static/media/PaginationNext.86158845.svg +0 -3
- zenml/zen_server/dashboard/static/media/PaginationPrev.60c18a88.svg +0 -3
- zenml/zen_server/dashboard/static/media/Pen.f2d831d4.svg +0 -6
- zenml/zen_server/dashboard/static/media/PhotoCamera.179d6d4c.svg +0 -3
- zenml/zen_server/dashboard/static/media/Pipeline.30d298b0.svg +0 -7
- zenml/zen_server/dashboard/static/media/Plus.5aa1c16b.svg +0 -3
- zenml/zen_server/dashboard/static/media/PlusCircle.92d860dd.svg +0 -5
- zenml/zen_server/dashboard/static/media/Repositories.71a36b8c.svg +0 -3
- zenml/zen_server/dashboard/static/media/RightArrow.f30d3871.svg +0 -29
- zenml/zen_server/dashboard/static/media/Rocket.63bf7b9d.svg +0 -3
- zenml/zen_server/dashboard/static/media/RocketLaunch.1bff2b59.svg +0 -6
- zenml/zen_server/dashboard/static/media/Rubik-Medium.c87313aa.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Run.daec4fb2.svg +0 -6
- zenml/zen_server/dashboard/static/media/Search.d1afcce5.svg +0 -4
- zenml/zen_server/dashboard/static/media/Settings.59ca73ae.svg +0 -4
- zenml/zen_server/dashboard/static/media/Share2.46c3ff66.svg +0 -3
- zenml/zen_server/dashboard/static/media/SignOut.6aa718c5.svg +0 -3
- zenml/zen_server/dashboard/static/media/SimplePlus.5cf7ec20.svg +0 -3
- zenml/zen_server/dashboard/static/media/SingleUser.bef3a095.svg +0 -4
- zenml/zen_server/dashboard/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
- zenml/zen_server/dashboard/static/media/Stack.19b604ac.svg +0 -5
- zenml/zen_server/dashboard/static/media/StackComponent.b1ba90b5.svg +0 -4
- zenml/zen_server/dashboard/static/media/Star.f0c25022.svg +0 -9
- zenml/zen_server/dashboard/static/media/StarOutline.94ca8cd9.svg +0 -3
- zenml/zen_server/dashboard/static/media/Storefront.4b4796fe.svg +0 -3
- zenml/zen_server/dashboard/static/media/Stream.543e3039.svg +0 -3
- zenml/zen_server/dashboard/static/media/SupportAgent.510ddf1f.svg +0 -8
- zenml/zen_server/dashboard/static/media/Table.77033750.svg +0 -6
- zenml/zen_server/dashboard/static/media/Tool.d5785486.svg +0 -3
- zenml/zen_server/dashboard/static/media/UserPlus.741a99d7.svg +0 -6
- zenml/zen_server/dashboard/static/media/Verified.0625b2a0.svg +0 -3
- zenml/zen_server/dashboard/static/media/addNew.4fb6c939.svg +0 -8
- zenml/zen_server/dashboard/static/media/arrowClose.cbd53f3f.svg +0 -3
- zenml/zen_server/dashboard/static/media/arrowOpen.6ceef0af.svg +0 -3
- zenml/zen_server/dashboard/static/media/check_small.30bc0138.svg +0 -3
- zenml/zen_server/dashboard/static/media/circleArrowSideClose.98d6013e.svg +0 -18
- zenml/zen_server/dashboard/static/media/circleArrowSideOpen.63653df6.svg +0 -18
- zenml/zen_server/dashboard/static/media/image.104fd14b.png +0 -0
- zenml/zen_server/dashboard/static/media/imageAddIcon.e83004a9.svg +0 -7
- zenml/zen_server/dashboard/static/media/logo.93333e5c.svg +0 -1
- zenml/zen_server/dashboard/static/media/logo_small.4204397d.svg +0 -3
- zenml/zen_server/dashboard/static/media/logo_white.d4b4414e.svg +0 -20
- zenml/zen_server/dashboard/static/media/notConnected.5e2c8ea7.svg +0 -8
- zenml/zen_server/dashboard/static/media/plugin-fallback.72c294e6.svg +0 -6
- zenml/zen_server/dashboard/static/media/share.bcd998b0.svg +0 -5
- zenml/zen_server/dashboard/static/media/stars.08a9b19a.svg +0 -8
- zenml/zen_server/deploy/terraform/__init__.py +0 -41
- zenml/zen_server/deploy/terraform/providers/__init__.py +0 -14
- zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
- zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -332
- zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
- zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -179
- zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
- zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -111
- zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
- zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
- zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
- zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
- zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -188
- zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -111
- zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -14
- zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
- zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -183
- zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -122
- zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -255
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -97
- zenml_nightly-0.55.0.dev20240124.dist-info/METADATA +0 -438
- zenml_nightly-0.55.0.dev20240124.dist-info/RECORD +0 -1072
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/entry_points.txt +0 -0
@@ -31,21 +31,35 @@
|
|
31
31
|
|
32
32
|
import os
|
33
33
|
import re
|
34
|
-
|
34
|
+
import types
|
35
|
+
import urllib
|
36
|
+
from typing import (
|
37
|
+
TYPE_CHECKING,
|
38
|
+
Any,
|
39
|
+
Dict,
|
40
|
+
Iterator,
|
41
|
+
List,
|
42
|
+
Optional,
|
43
|
+
Tuple,
|
44
|
+
Type,
|
45
|
+
cast,
|
46
|
+
)
|
35
47
|
from uuid import UUID
|
36
48
|
|
37
|
-
import kfp
|
38
49
|
from google.api_core import exceptions as google_exceptions
|
39
50
|
from google.cloud import aiplatform
|
51
|
+
from google.cloud.aiplatform_v1.types import PipelineState
|
40
52
|
from kfp import dsl
|
41
|
-
from kfp.
|
42
|
-
from kfp.v2.compiler import Compiler as KFPV2Compiler
|
53
|
+
from kfp.compiler import Compiler
|
43
54
|
|
55
|
+
from zenml.config.resource_settings import ResourceSettings
|
44
56
|
from zenml.constants import (
|
57
|
+
METADATA_ORCHESTRATOR_LOGS_URL,
|
58
|
+
METADATA_ORCHESTRATOR_RUN_ID,
|
45
59
|
METADATA_ORCHESTRATOR_URL,
|
46
60
|
)
|
47
61
|
from zenml.entrypoints import StepEntrypointConfiguration
|
48
|
-
from zenml.enums import StackComponentType
|
62
|
+
from zenml.enums import ExecutionStatus, StackComponentType
|
49
63
|
from zenml.integrations.gcp import GCP_ARTIFACT_STORE_FLAVOR
|
50
64
|
from zenml.integrations.gcp.constants import (
|
51
65
|
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL,
|
@@ -54,39 +68,26 @@ from zenml.integrations.gcp.flavors.vertex_orchestrator_flavor import (
|
|
54
68
|
VertexOrchestratorConfig,
|
55
69
|
VertexOrchestratorSettings,
|
56
70
|
)
|
57
|
-
from zenml.integrations.gcp.google_cloud_function import create_cloud_function
|
58
|
-
from zenml.integrations.gcp.google_cloud_scheduler import create_scheduler_job
|
59
71
|
from zenml.integrations.gcp.google_credentials_mixin import (
|
60
72
|
GoogleCredentialsMixin,
|
61
73
|
)
|
62
|
-
from zenml.integrations.gcp.orchestrators import vertex_scheduler
|
63
|
-
from zenml.integrations.gcp.orchestrators.vertex_scheduler.main import (
|
64
|
-
ENABLE_CACHING,
|
65
|
-
ENCRYPTION_SPEC_KEY_NAME,
|
66
|
-
JOB_ID,
|
67
|
-
LABELS,
|
68
|
-
LOCATION,
|
69
|
-
NETWORK,
|
70
|
-
PARAMETER_VALUES,
|
71
|
-
PIPELINE_ROOT,
|
72
|
-
PROJECT,
|
73
|
-
TEMPLATE_PATH,
|
74
|
-
WORKLOAD_SERVICE_ACCOUNT,
|
75
|
-
)
|
76
|
-
from zenml.integrations.kubeflow.utils import apply_pod_settings
|
77
74
|
from zenml.io import fileio
|
78
75
|
from zenml.logger import get_logger
|
79
76
|
from zenml.metadata.metadata_types import MetadataType, Uri
|
80
77
|
from zenml.orchestrators import ContainerizedOrchestrator
|
81
78
|
from zenml.orchestrators.utils import get_orchestrator_run_name
|
82
79
|
from zenml.stack.stack_validator import StackValidator
|
80
|
+
from zenml.utils import yaml_utils
|
83
81
|
from zenml.utils.io_utils import get_global_config_directory
|
84
82
|
|
85
83
|
if TYPE_CHECKING:
|
86
84
|
from zenml.config.base_settings import BaseSettings
|
87
|
-
from zenml.models import
|
85
|
+
from zenml.models import (
|
86
|
+
PipelineDeploymentResponse,
|
87
|
+
PipelineRunResponse,
|
88
|
+
ScheduleResponse,
|
89
|
+
)
|
88
90
|
from zenml.stack import Stack
|
89
|
-
from zenml.steps import ResourceSettings
|
90
91
|
|
91
92
|
logger = get_logger(__name__)
|
92
93
|
ENV_ZENML_VERTEX_RUN_ID = "ZENML_VERTEX_RUN_ID"
|
@@ -249,14 +250,12 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
249
250
|
if deployment.schedule:
|
250
251
|
if (
|
251
252
|
deployment.schedule.catchup
|
252
|
-
or deployment.schedule.start_time
|
253
|
-
or deployment.schedule.end_time
|
254
253
|
or deployment.schedule.interval_second
|
255
254
|
):
|
256
255
|
logger.warning(
|
257
256
|
"Vertex orchestrator only uses schedules with the "
|
258
|
-
"`cron_expression` property
|
259
|
-
"are ignored."
|
257
|
+
"`cron_expression` property, with optional `start_time` "
|
258
|
+
"and/or `end_time`. All other properties are ignored."
|
260
259
|
)
|
261
260
|
if deployment.schedule.cron_expression is None:
|
262
261
|
raise ValueError(
|
@@ -264,61 +263,55 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
264
263
|
"schedule to a Vertex orchestrator."
|
265
264
|
)
|
266
265
|
|
267
|
-
def
|
266
|
+
def _create_dynamic_component(
|
268
267
|
self,
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
268
|
+
image: str,
|
269
|
+
command: List[str],
|
270
|
+
arguments: List[str],
|
271
|
+
component_name: str,
|
272
|
+
) -> dsl.PipelineTask:
|
273
|
+
"""Creates a dynamic container component for a Vertex pipeline.
|
274
274
|
|
275
275
|
Args:
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
276
|
+
image: The image to use for the component.
|
277
|
+
command: The command to use for the component.
|
278
|
+
arguments: The arguments to use for the component.
|
279
|
+
component_name: The name of the component.
|
280
|
+
|
281
|
+
Returns:
|
282
|
+
The dynamic container component.
|
281
283
|
"""
|
282
|
-
# Set optional CPU, RAM and GPU constraints for the pipeline
|
283
284
|
|
284
|
-
|
285
|
+
def dynamic_container_component() -> dsl.ContainerSpec:
|
286
|
+
"""Dynamic container component.
|
285
287
|
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
if memory_limit is not None:
|
295
|
-
container_op = container_op.set_memory_limit(memory_limit)
|
288
|
+
Returns:
|
289
|
+
The dynamic container component.
|
290
|
+
"""
|
291
|
+
return dsl.ContainerSpec(
|
292
|
+
image=image,
|
293
|
+
command=command,
|
294
|
+
args=arguments,
|
295
|
+
)
|
296
296
|
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
297
|
+
# Change the name of the function
|
298
|
+
new_container_spec_func = types.FunctionType(
|
299
|
+
dynamic_container_component.__code__,
|
300
|
+
dynamic_container_component.__globals__,
|
301
|
+
name=component_name,
|
302
|
+
argdefs=dynamic_container_component.__defaults__,
|
303
|
+
closure=dynamic_container_component.__closure__,
|
301
304
|
)
|
302
|
-
|
303
|
-
container_op = container_op.set_gpu_limit(gpu_limit)
|
305
|
+
pipeline_task = dsl.container_component(new_container_spec_func)
|
304
306
|
|
305
|
-
|
306
|
-
constraint_label, value = node_selector_constraint
|
307
|
-
if not (
|
308
|
-
constraint_label
|
309
|
-
== GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL
|
310
|
-
and gpu_limit == 0
|
311
|
-
):
|
312
|
-
container_op.add_node_selector_constraint(
|
313
|
-
constraint_label, value
|
314
|
-
)
|
307
|
+
return pipeline_task
|
315
308
|
|
316
309
|
def prepare_or_run_pipeline(
|
317
310
|
self,
|
318
311
|
deployment: "PipelineDeploymentResponse",
|
319
312
|
stack: "Stack",
|
320
313
|
environment: Dict[str, str],
|
321
|
-
) ->
|
314
|
+
) -> Iterator[Dict[str, MetadataType]]:
|
322
315
|
"""Creates a KFP JSON pipeline.
|
323
316
|
|
324
317
|
# noqa: DAR402
|
@@ -353,12 +346,15 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
353
346
|
environment.
|
354
347
|
|
355
348
|
Raises:
|
356
|
-
ValueError: If the attribute `pipeline_root` is not set and it
|
349
|
+
ValueError: If the attribute `pipeline_root` is not set, and it
|
357
350
|
can be not generated using the path of the artifact store in the
|
358
351
|
stack because it is not a
|
359
352
|
`zenml.integrations.gcp.artifact_store.GCPArtifactStore`. Also gets
|
360
353
|
raised if attempting to schedule pipeline run without using the
|
361
354
|
`zenml.integrations.gcp.artifact_store.GCPArtifactStore`.
|
355
|
+
|
356
|
+
Yields:
|
357
|
+
A dictionary of metadata related to the pipeline run.
|
362
358
|
"""
|
363
359
|
orchestrator_run_name = get_orchestrator_run_name(
|
364
360
|
pipeline_name=deployment.pipeline_configuration.name
|
@@ -381,77 +377,166 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
381
377
|
else:
|
382
378
|
self._pipeline_root = self.config.pipeline_root
|
383
379
|
|
384
|
-
def
|
385
|
-
"""Create a
|
386
|
-
|
387
|
-
This should contain the name of the Docker image and configures the
|
388
|
-
entrypoint of the Docker image to run the step.
|
380
|
+
def _create_dynamic_pipeline() -> Any:
|
381
|
+
"""Create a dynamic pipeline including each step.
|
389
382
|
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
If this callable is passed to the `compile()` method of
|
394
|
-
`KFPV2Compiler` all `dsl.ContainerOp` instances will be
|
395
|
-
automatically added to a singular `dsl.Pipeline` instance.
|
383
|
+
Returns:
|
384
|
+
pipeline_func
|
396
385
|
"""
|
397
|
-
|
398
|
-
step_name_to_container_op: Dict[str, dsl.ContainerOp] = {}
|
386
|
+
step_name_to_dynamic_component: Dict[str, Any] = {}
|
399
387
|
|
400
388
|
for step_name, step in deployment.step_configurations.items():
|
401
389
|
image = self.get_image(
|
402
|
-
deployment=deployment,
|
390
|
+
deployment=deployment,
|
391
|
+
step_name=step_name,
|
403
392
|
)
|
393
|
+
command = StepEntrypointConfiguration.get_entrypoint_command()
|
404
394
|
arguments = (
|
405
395
|
StepEntrypointConfiguration.get_entrypoint_arguments(
|
406
|
-
step_name=step_name,
|
396
|
+
step_name=step_name,
|
397
|
+
deployment_id=deployment.id,
|
407
398
|
)
|
408
399
|
)
|
409
|
-
|
410
|
-
|
411
|
-
# `dsl.ContainerOp`
|
412
|
-
# class directly is deprecated when using the Kubeflow SDK v2.
|
413
|
-
container_op = kfp.components.load_component_from_text(
|
414
|
-
f"""
|
415
|
-
name: {step_name}
|
416
|
-
implementation:
|
417
|
-
container:
|
418
|
-
image: {image}
|
419
|
-
command: {command + arguments}"""
|
420
|
-
)()
|
421
|
-
|
422
|
-
container_op.set_env_variable(
|
423
|
-
name=ENV_ZENML_VERTEX_RUN_ID,
|
424
|
-
value=dslv2.PIPELINE_JOB_NAME_PLACEHOLDER,
|
400
|
+
dynamic_component = self._create_dynamic_component(
|
401
|
+
image, command, arguments, step_name
|
425
402
|
)
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
403
|
+
step_settings = cast(
|
404
|
+
VertexOrchestratorSettings, self.get_settings(step)
|
405
|
+
)
|
406
|
+
pod_settings = step_settings.pod_settings
|
407
|
+
if pod_settings:
|
408
|
+
if pod_settings.host_ipc:
|
409
|
+
logger.warning(
|
410
|
+
"Host IPC is set to `True` but not supported in "
|
411
|
+
"this orchestrator. Ignoring..."
|
412
|
+
)
|
413
|
+
if pod_settings.affinity:
|
414
|
+
logger.warning(
|
415
|
+
"Affinity is set but not supported in Vertex with "
|
416
|
+
"Kubeflow Pipelines 2.x. Ignoring..."
|
417
|
+
)
|
418
|
+
if pod_settings.tolerations:
|
419
|
+
logger.warning(
|
420
|
+
"Tolerations are set but not supported in "
|
421
|
+
"Vertex with Kubeflow Pipelines 2.x. Ignoring..."
|
422
|
+
)
|
423
|
+
if pod_settings.volumes:
|
424
|
+
logger.warning(
|
425
|
+
"Volumes are set but not supported in Vertex with "
|
426
|
+
"Kubeflow Pipelines 2.x. Ignoring..."
|
427
|
+
)
|
428
|
+
if pod_settings.volume_mounts:
|
429
|
+
logger.warning(
|
430
|
+
"Volume mounts are set but not supported in "
|
431
|
+
"Vertex with Kubeflow Pipelines 2.x. Ignoring..."
|
432
|
+
)
|
433
|
+
for key in pod_settings.node_selectors:
|
434
|
+
if (
|
435
|
+
key
|
436
|
+
!= GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL
|
437
|
+
):
|
438
|
+
logger.warning(
|
439
|
+
"Vertex only allows the %s node selector, "
|
440
|
+
"ignoring the node selector %s.",
|
441
|
+
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL,
|
442
|
+
key,
|
443
|
+
)
|
444
|
+
|
445
|
+
step_name_to_dynamic_component[step_name] = dynamic_component
|
446
|
+
|
447
|
+
@dsl.pipeline( # type: ignore[misc]
|
448
|
+
display_name=orchestrator_run_name,
|
449
|
+
)
|
450
|
+
def dynamic_pipeline() -> None:
|
451
|
+
"""Dynamic pipeline."""
|
452
|
+
# iterate through the components one by one
|
453
|
+
# (from step_name_to_dynamic_component)
|
454
|
+
for (
|
455
|
+
component_name,
|
456
|
+
component,
|
457
|
+
) in step_name_to_dynamic_component.items():
|
458
|
+
# for each component, check to see what other steps are
|
459
|
+
# upstream of it
|
460
|
+
step = deployment.step_configurations[component_name]
|
461
|
+
upstream_step_components = [
|
462
|
+
step_name_to_dynamic_component[upstream_step_name]
|
463
|
+
for upstream_step_name in step.spec.upstream_steps
|
434
464
|
]
|
435
|
-
|
465
|
+
task = (
|
466
|
+
component()
|
467
|
+
.set_display_name(
|
468
|
+
name=component_name,
|
469
|
+
)
|
470
|
+
.set_caching_options(enable_caching=False)
|
471
|
+
.set_env_variable(
|
472
|
+
name=ENV_ZENML_VERTEX_RUN_ID,
|
473
|
+
value=dsl.PIPELINE_JOB_NAME_PLACEHOLDER,
|
474
|
+
)
|
475
|
+
.after(*upstream_step_components)
|
476
|
+
)
|
436
477
|
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
478
|
+
step_settings = cast(
|
479
|
+
VertexOrchestratorSettings, self.get_settings(step)
|
480
|
+
)
|
481
|
+
pod_settings = step_settings.pod_settings
|
482
|
+
|
483
|
+
node_selector_constraint: Optional[Tuple[str, str]] = None
|
484
|
+
if pod_settings and (
|
485
|
+
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL
|
486
|
+
in pod_settings.node_selectors.keys()
|
487
|
+
):
|
488
|
+
node_selector_constraint = (
|
489
|
+
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL,
|
490
|
+
pod_settings.node_selectors[
|
491
|
+
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL
|
492
|
+
],
|
493
|
+
)
|
494
|
+
elif step_settings.node_selector_constraint:
|
495
|
+
node_selector_constraint = (
|
496
|
+
GKE_ACCELERATOR_NODE_SELECTOR_CONSTRAINT_LABEL,
|
497
|
+
step_settings.node_selector_constraint[1],
|
498
|
+
)
|
499
|
+
|
500
|
+
self._configure_container_resources(
|
501
|
+
dynamic_component=task,
|
502
|
+
resource_settings=step.config.resource_settings,
|
503
|
+
node_selector_constraint=node_selector_constraint,
|
445
504
|
)
|
446
505
|
|
447
|
-
|
448
|
-
container_op=container_op,
|
449
|
-
resource_settings=step.config.resource_settings,
|
450
|
-
node_selector_constraint=settings.node_selector_constraint,
|
451
|
-
)
|
452
|
-
container_op.set_caching_options(enable_caching=False)
|
506
|
+
return dynamic_pipeline
|
453
507
|
|
454
|
-
|
508
|
+
def _update_json_with_environment(
|
509
|
+
yaml_file_path: str, environment: Dict[str, str]
|
510
|
+
) -> None:
|
511
|
+
"""Updates the env section of the steps in the YAML file with the given environment variables.
|
512
|
+
|
513
|
+
Args:
|
514
|
+
yaml_file_path: The path to the YAML file to update.
|
515
|
+
environment: A dictionary of environment variables to add.
|
516
|
+
"""
|
517
|
+
pipeline_definition = yaml_utils.read_json(pipeline_file_path)
|
518
|
+
|
519
|
+
# Iterate through each component and add the environment variables
|
520
|
+
for executor in pipeline_definition["deploymentSpec"]["executors"]:
|
521
|
+
if (
|
522
|
+
"container"
|
523
|
+
in pipeline_definition["deploymentSpec"]["executors"][
|
524
|
+
executor
|
525
|
+
]
|
526
|
+
):
|
527
|
+
container = pipeline_definition["deploymentSpec"][
|
528
|
+
"executors"
|
529
|
+
][executor]["container"]
|
530
|
+
if "env" not in container:
|
531
|
+
container["env"] = []
|
532
|
+
for key, value in environment.items():
|
533
|
+
container["env"].append({"name": key, "value": value})
|
534
|
+
|
535
|
+
yaml_utils.write_json(pipeline_file_path, pipeline_definition)
|
536
|
+
|
537
|
+
print(
|
538
|
+
f"Updated YAML file with environment variables at {yaml_file_path}"
|
539
|
+
)
|
455
540
|
|
456
541
|
# Save the generated pipeline to a file.
|
457
542
|
fileio.makedirs(self.pipeline_directory)
|
@@ -463,13 +548,17 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
463
548
|
# Compile the pipeline using the Kubeflow SDK V2 compiler that allows
|
464
549
|
# to generate a JSON representation of the pipeline that can be later
|
465
550
|
# upload to Vertex AI Pipelines service.
|
466
|
-
|
467
|
-
pipeline_func=
|
551
|
+
Compiler().compile(
|
552
|
+
pipeline_func=_create_dynamic_pipeline(),
|
468
553
|
package_path=pipeline_file_path,
|
469
554
|
pipeline_name=_clean_pipeline_name(
|
470
555
|
deployment.pipeline_configuration.name
|
471
556
|
),
|
472
557
|
)
|
558
|
+
|
559
|
+
# Let's update the YAML file with the environment variables
|
560
|
+
_update_json_with_environment(pipeline_file_path, environment)
|
561
|
+
|
473
562
|
logger.info(
|
474
563
|
"Writing Vertex workflow definition to `%s`.", pipeline_file_path
|
475
564
|
)
|
@@ -478,144 +567,16 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
478
567
|
VertexOrchestratorSettings, self.get_settings(deployment)
|
479
568
|
)
|
480
569
|
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
pipeline_file_path=pipeline_file_path,
|
492
|
-
settings=settings,
|
493
|
-
)
|
494
|
-
|
495
|
-
else:
|
496
|
-
logger.info("No schedule detected. Creating one-off vertex job...")
|
497
|
-
# Using the Google Cloud AIPlatform client, upload and execute the
|
498
|
-
# pipeline
|
499
|
-
# on the Vertex AI Pipelines service.
|
500
|
-
self._upload_and_run_pipeline(
|
501
|
-
pipeline_name=deployment.pipeline_configuration.name,
|
502
|
-
pipeline_file_path=pipeline_file_path,
|
503
|
-
run_name=orchestrator_run_name,
|
504
|
-
settings=settings,
|
505
|
-
)
|
506
|
-
|
507
|
-
def _upload_and_schedule_pipeline(
|
508
|
-
self,
|
509
|
-
pipeline_name: str,
|
510
|
-
run_name: str,
|
511
|
-
stack: "Stack",
|
512
|
-
schedule: "ScheduleResponse",
|
513
|
-
pipeline_file_path: str,
|
514
|
-
settings: VertexOrchestratorSettings,
|
515
|
-
) -> None:
|
516
|
-
"""Uploads and schedules pipeline on GCP.
|
517
|
-
|
518
|
-
Args:
|
519
|
-
pipeline_name: Name of the pipeline.
|
520
|
-
run_name: Orchestrator run name.
|
521
|
-
stack: The stack the pipeline will run on.
|
522
|
-
schedule: The schedule the pipeline will run on.
|
523
|
-
pipeline_file_path: Path of the JSON file containing the compiled
|
524
|
-
Kubeflow pipeline (compiled with Kubeflow SDK v2).
|
525
|
-
settings: Pipeline level settings for this orchestrator.
|
526
|
-
|
527
|
-
Raises:
|
528
|
-
ValueError: If the attribute `pipeline_root` is not set, and it
|
529
|
-
can be not generated using the path of the artifact store in the
|
530
|
-
stack because it is not a
|
531
|
-
`zenml.integrations.gcp.artifact_store.GCPArtifactStore`. Also
|
532
|
-
gets raised if attempting to schedule pipeline run without using
|
533
|
-
the `zenml.integrations.gcp.artifact_store.GCPArtifactStore`.
|
534
|
-
"""
|
535
|
-
# First, do some validation
|
536
|
-
artifact_store = stack.artifact_store
|
537
|
-
if artifact_store.flavor != GCP_ARTIFACT_STORE_FLAVOR:
|
538
|
-
raise ValueError(
|
539
|
-
"Currently, the Vertex AI orchestrator only supports "
|
540
|
-
"scheduled runs in combination with an artifact store of "
|
541
|
-
f"flavor: {GCP_ARTIFACT_STORE_FLAVOR}. The current stacks "
|
542
|
-
f"artifact store is of flavor: {artifact_store.flavor}. "
|
543
|
-
"Please update your stack accordingly."
|
544
|
-
)
|
545
|
-
|
546
|
-
# Get the credentials that would be used to create resources.
|
547
|
-
credentials, project_id = self._get_authentication()
|
548
|
-
|
549
|
-
scheduler_service_account_email: Optional[str] = None
|
550
|
-
if self.config.scheduler_service_account:
|
551
|
-
scheduler_service_account_email = (
|
552
|
-
self.config.scheduler_service_account
|
553
|
-
)
|
554
|
-
elif hasattr(credentials, "signer_email"):
|
555
|
-
scheduler_service_account_email = credentials.signer_email
|
556
|
-
else:
|
557
|
-
scheduler_service_account_email = (
|
558
|
-
self.config.function_service_account
|
559
|
-
or self.config.workload_service_account
|
560
|
-
)
|
561
|
-
|
562
|
-
if not scheduler_service_account_email:
|
563
|
-
raise ValueError(
|
564
|
-
"A GCP service account is required to schedule a pipeline run. "
|
565
|
-
"The credentials used to authenticate with GCP do not have a "
|
566
|
-
"service account associated with them and a service account "
|
567
|
-
"was not configured in the `scheduler_service_account` field "
|
568
|
-
"of the orchestrator config. Please update your orchestrator "
|
569
|
-
"configuration or credentials accordingly."
|
570
|
-
)
|
571
|
-
|
572
|
-
# Copy over the scheduled pipeline to the artifact store
|
573
|
-
artifact_store_base_uri = f"{artifact_store.path.rstrip('/')}/vertex_scheduled_pipelines/{pipeline_name}/{run_name}"
|
574
|
-
artifact_store_pipeline_uri = (
|
575
|
-
f"{artifact_store_base_uri}/vertex_pipeline.json"
|
576
|
-
)
|
577
|
-
fileio.copy(pipeline_file_path, artifact_store_pipeline_uri)
|
578
|
-
logger.info(
|
579
|
-
"The scheduled pipeline representation has been "
|
580
|
-
"automatically copied to this path of the `GCPArtifactStore`: "
|
581
|
-
f"{artifact_store_pipeline_uri}",
|
582
|
-
)
|
583
|
-
|
584
|
-
# Create cloud function
|
585
|
-
function_uri = create_cloud_function(
|
586
|
-
directory_path=vertex_scheduler.__path__[0], # fixed path
|
587
|
-
upload_path=f"{artifact_store_base_uri}/code.zip",
|
588
|
-
project=project_id,
|
589
|
-
location=self.config.location,
|
590
|
-
function_name=run_name,
|
591
|
-
credentials=credentials,
|
592
|
-
function_service_account_email=self.config.function_service_account,
|
593
|
-
)
|
594
|
-
|
595
|
-
# Create the scheduler job
|
596
|
-
body = {
|
597
|
-
TEMPLATE_PATH: artifact_store_pipeline_uri,
|
598
|
-
JOB_ID: _clean_pipeline_name(pipeline_name),
|
599
|
-
PIPELINE_ROOT: self._pipeline_root,
|
600
|
-
PARAMETER_VALUES: None,
|
601
|
-
ENABLE_CACHING: False,
|
602
|
-
ENCRYPTION_SPEC_KEY_NAME: self.config.encryption_spec_key_name,
|
603
|
-
LABELS: settings.labels,
|
604
|
-
PROJECT: project_id,
|
605
|
-
LOCATION: self.config.location,
|
606
|
-
WORKLOAD_SERVICE_ACCOUNT: self.config.workload_service_account,
|
607
|
-
NETWORK: self.config.network,
|
608
|
-
}
|
609
|
-
|
610
|
-
create_scheduler_job(
|
611
|
-
project=project_id,
|
612
|
-
region=self.config.location,
|
613
|
-
http_uri=function_uri,
|
614
|
-
body=body,
|
615
|
-
schedule=str(schedule.cron_expression),
|
616
|
-
credentials=credentials,
|
617
|
-
service_account_email=scheduler_service_account_email,
|
618
|
-
)
|
570
|
+
# Using the Google Cloud AIPlatform client, upload and execute the
|
571
|
+
# pipeline on the Vertex AI Pipelines service.
|
572
|
+
if metadata := self._upload_and_run_pipeline(
|
573
|
+
pipeline_name=deployment.pipeline_configuration.name,
|
574
|
+
pipeline_file_path=pipeline_file_path,
|
575
|
+
run_name=orchestrator_run_name,
|
576
|
+
settings=settings,
|
577
|
+
schedule=deployment.schedule,
|
578
|
+
):
|
579
|
+
yield from metadata
|
619
580
|
|
620
581
|
def _upload_and_run_pipeline(
|
621
582
|
self,
|
@@ -623,7 +584,8 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
623
584
|
pipeline_file_path: str,
|
624
585
|
run_name: str,
|
625
586
|
settings: VertexOrchestratorSettings,
|
626
|
-
|
587
|
+
schedule: Optional["ScheduleResponse"] = None,
|
588
|
+
) -> Iterator[Dict[str, MetadataType]]:
|
627
589
|
"""Uploads and run the pipeline on the Vertex AI Pipelines service.
|
628
590
|
|
629
591
|
Args:
|
@@ -632,6 +594,14 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
632
594
|
Kubeflow pipeline (compiled with Kubeflow SDK v2).
|
633
595
|
run_name: Orchestrator run name.
|
634
596
|
settings: Pipeline level settings for this orchestrator.
|
597
|
+
schedule: The schedule the pipeline will run on.
|
598
|
+
|
599
|
+
Raises:
|
600
|
+
RuntimeError: If the Vertex Orchestrator fails to provision or any
|
601
|
+
other Runtime errors.
|
602
|
+
|
603
|
+
Yields:
|
604
|
+
A dictionary of metadata related to the pipeline run.
|
635
605
|
"""
|
636
606
|
# We have to replace the hyphens in the run name with underscores
|
637
607
|
# and lower case the string, because the Vertex AI Pipelines service
|
@@ -639,8 +609,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
639
609
|
job_id = _clean_pipeline_name(run_name)
|
640
610
|
|
641
611
|
# Get the credentials that would be used to create the Vertex AI
|
642
|
-
# Pipelines
|
643
|
-
# job.
|
612
|
+
# Pipelines job.
|
644
613
|
credentials, project_id = self._get_authentication()
|
645
614
|
|
646
615
|
# Instantiate the Vertex AI Pipelines job
|
@@ -658,52 +627,74 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
658
627
|
location=self.config.location,
|
659
628
|
)
|
660
629
|
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
630
|
+
if self.config.workload_service_account:
|
631
|
+
logger.info(
|
632
|
+
"The Vertex AI Pipelines job workload will be executed "
|
633
|
+
"using the `%s` "
|
634
|
+
"service account.",
|
635
|
+
self.config.workload_service_account,
|
636
|
+
)
|
637
|
+
if self.config.network:
|
638
|
+
logger.info(
|
639
|
+
"The Vertex AI Pipelines job will be peered with the `%s` "
|
640
|
+
"network.",
|
641
|
+
self.config.network,
|
642
|
+
)
|
666
643
|
|
667
|
-
# Submit the job to Vertex AI Pipelines service.
|
668
644
|
try:
|
669
|
-
if
|
645
|
+
if schedule:
|
670
646
|
logger.info(
|
671
|
-
"
|
672
|
-
"
|
673
|
-
|
674
|
-
|
647
|
+
"Scheduling job using native Vertex AI Pipelines "
|
648
|
+
"scheduling..."
|
649
|
+
)
|
650
|
+
run.create_schedule(
|
651
|
+
display_name=schedule.name,
|
652
|
+
cron=schedule.cron_expression,
|
653
|
+
start_time=schedule.utc_start_time,
|
654
|
+
end_time=schedule.utc_end_time,
|
655
|
+
service_account=self.config.workload_service_account,
|
656
|
+
network=self.config.network,
|
675
657
|
)
|
676
658
|
|
677
|
-
|
659
|
+
else:
|
678
660
|
logger.info(
|
679
|
-
"
|
680
|
-
|
681
|
-
|
661
|
+
"No schedule detected. Creating one-off Vertex job..."
|
662
|
+
)
|
663
|
+
logger.info(
|
664
|
+
"Submitting pipeline job with job_id `%s` to Vertex AI "
|
665
|
+
"Pipelines service.",
|
666
|
+
job_id,
|
682
667
|
)
|
683
668
|
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
"View the Vertex AI Pipelines job at %s", run._dashboard_uri()
|
690
|
-
)
|
691
|
-
|
692
|
-
if settings.synchronous:
|
669
|
+
# Submit the job to Vertex AI Pipelines service.
|
670
|
+
run.submit(
|
671
|
+
service_account=self.config.workload_service_account,
|
672
|
+
network=self.config.network,
|
673
|
+
)
|
693
674
|
logger.info(
|
694
|
-
"
|
675
|
+
"View the Vertex AI Pipelines job at %s",
|
676
|
+
run._dashboard_uri(),
|
695
677
|
)
|
696
|
-
|
678
|
+
|
679
|
+
# Yield metadata based on the generated job object
|
680
|
+
yield from self.compute_metadata(run)
|
681
|
+
|
682
|
+
if settings.synchronous:
|
683
|
+
logger.info(
|
684
|
+
"Waiting for the Vertex AI Pipelines job to finish..."
|
685
|
+
)
|
686
|
+
run.wait()
|
697
687
|
|
698
688
|
except google_exceptions.ClientError as e:
|
699
|
-
logger.
|
700
|
-
|
689
|
+
logger.error("Failed to create the Vertex AI Pipelines job: %s", e)
|
690
|
+
raise RuntimeError(
|
691
|
+
f"Failed to create the Vertex AI Pipelines job: {e}"
|
701
692
|
)
|
702
|
-
|
703
693
|
except RuntimeError as e:
|
704
694
|
logger.error(
|
705
695
|
"The Vertex AI Pipelines job execution has failed: %s", e
|
706
696
|
)
|
697
|
+
raise
|
707
698
|
|
708
699
|
def get_orchestrator_run_id(self) -> str:
|
709
700
|
"""Returns the active orchestrator run id.
|
@@ -744,3 +735,235 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
744
735
|
return {
|
745
736
|
METADATA_ORCHESTRATOR_URL: Uri(run_url),
|
746
737
|
}
|
738
|
+
|
739
|
+
def _configure_container_resources(
|
740
|
+
self,
|
741
|
+
dynamic_component: dsl.PipelineTask,
|
742
|
+
resource_settings: "ResourceSettings",
|
743
|
+
node_selector_constraint: Optional[Tuple[str, str]] = None,
|
744
|
+
) -> dsl.PipelineTask:
|
745
|
+
"""Adds resource requirements to the container.
|
746
|
+
|
747
|
+
Args:
|
748
|
+
dynamic_component: The dynamic component to add the resource
|
749
|
+
settings to.
|
750
|
+
resource_settings: The resource settings to use for this
|
751
|
+
container.
|
752
|
+
node_selector_constraint: Node selector constraint to apply to
|
753
|
+
the container.
|
754
|
+
|
755
|
+
Returns:
|
756
|
+
The dynamic component with the resource settings applied.
|
757
|
+
"""
|
758
|
+
# Set optional CPU, RAM and GPU constraints for the pipeline
|
759
|
+
cpu_limit = None
|
760
|
+
if resource_settings:
|
761
|
+
cpu_limit = resource_settings.cpu_count or self.config.cpu_limit
|
762
|
+
|
763
|
+
if cpu_limit is not None:
|
764
|
+
dynamic_component = dynamic_component.set_cpu_limit(str(cpu_limit))
|
765
|
+
|
766
|
+
memory_limit = (
|
767
|
+
resource_settings.memory[:-1]
|
768
|
+
if resource_settings.memory
|
769
|
+
else self.config.memory_limit
|
770
|
+
)
|
771
|
+
if memory_limit is not None:
|
772
|
+
dynamic_component = dynamic_component.set_memory_limit(
|
773
|
+
memory_limit
|
774
|
+
)
|
775
|
+
|
776
|
+
gpu_limit = (
|
777
|
+
resource_settings.gpu_count
|
778
|
+
if resource_settings.gpu_count is not None
|
779
|
+
else self.config.gpu_limit
|
780
|
+
)
|
781
|
+
|
782
|
+
if node_selector_constraint:
|
783
|
+
_, value = node_selector_constraint
|
784
|
+
if gpu_limit is not None and gpu_limit > 0:
|
785
|
+
dynamic_component = (
|
786
|
+
dynamic_component.set_accelerator_type(value)
|
787
|
+
.set_accelerator_limit(gpu_limit)
|
788
|
+
.set_gpu_limit(gpu_limit)
|
789
|
+
)
|
790
|
+
else:
|
791
|
+
logger.warning(
|
792
|
+
"Accelerator type %s specified, but the GPU limit is not "
|
793
|
+
"set or set to 0. The accelerator type will be ignored. "
|
794
|
+
"To fix this warning, either remove the specified "
|
795
|
+
"accelerator type or set the `gpu_count` using the "
|
796
|
+
"ResourceSettings (https://docs.zenml.io/how-to/advanced-topics/training-with-gpus)."
|
797
|
+
)
|
798
|
+
|
799
|
+
return dynamic_component
|
800
|
+
|
801
|
+
def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
|
802
|
+
"""Refreshes the status of a specific pipeline run.
|
803
|
+
|
804
|
+
Args:
|
805
|
+
run: The run that was executed by this orchestrator.
|
806
|
+
|
807
|
+
Returns:
|
808
|
+
the actual status of the pipeline job.
|
809
|
+
|
810
|
+
Raises:
|
811
|
+
AssertionError: If the run was not executed by to this orchestrator.
|
812
|
+
ValueError: If it fetches an unknown state or if we can not fetch
|
813
|
+
the orchestrator run ID.
|
814
|
+
"""
|
815
|
+
# Make sure that the stack exists and is accessible
|
816
|
+
if run.stack is None:
|
817
|
+
raise ValueError(
|
818
|
+
"The stack that the run was executed on is not available "
|
819
|
+
"anymore."
|
820
|
+
)
|
821
|
+
|
822
|
+
# Make sure that the run belongs to this orchestrator
|
823
|
+
assert (
|
824
|
+
self.id
|
825
|
+
== run.stack.components[StackComponentType.ORCHESTRATOR][0].id
|
826
|
+
)
|
827
|
+
|
828
|
+
# Initialize the Vertex client
|
829
|
+
credentials, project_id = self._get_authentication()
|
830
|
+
aiplatform.init(
|
831
|
+
project=project_id,
|
832
|
+
location=self.config.location,
|
833
|
+
credentials=credentials,
|
834
|
+
)
|
835
|
+
|
836
|
+
# Fetch the status of the PipelineJob
|
837
|
+
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
838
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
839
|
+
elif run.orchestrator_run_id is not None:
|
840
|
+
run_id = run.orchestrator_run_id
|
841
|
+
else:
|
842
|
+
raise ValueError(
|
843
|
+
"Can not find the orchestrator run ID, thus can not fetch "
|
844
|
+
"the status."
|
845
|
+
)
|
846
|
+
status = aiplatform.PipelineJob.get(run_id).state
|
847
|
+
|
848
|
+
# Map the potential outputs to ZenML ExecutionStatus. Potential values:
|
849
|
+
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker/client/describe_pipeline_execution.html#
|
850
|
+
if status in [PipelineState.PIPELINE_STATE_UNSPECIFIED]:
|
851
|
+
return run.status
|
852
|
+
elif status in [
|
853
|
+
PipelineState.PIPELINE_STATE_QUEUED,
|
854
|
+
PipelineState.PIPELINE_STATE_PENDING,
|
855
|
+
]:
|
856
|
+
return ExecutionStatus.INITIALIZING
|
857
|
+
elif status in [
|
858
|
+
PipelineState.PIPELINE_STATE_RUNNING,
|
859
|
+
PipelineState.PIPELINE_STATE_PAUSED,
|
860
|
+
]:
|
861
|
+
return ExecutionStatus.RUNNING
|
862
|
+
elif status in [PipelineState.PIPELINE_STATE_SUCCEEDED]:
|
863
|
+
return ExecutionStatus.COMPLETED
|
864
|
+
|
865
|
+
elif status in [
|
866
|
+
PipelineState.PIPELINE_STATE_FAILED,
|
867
|
+
PipelineState.PIPELINE_STATE_CANCELLING,
|
868
|
+
PipelineState.PIPELINE_STATE_CANCELLED,
|
869
|
+
]:
|
870
|
+
return ExecutionStatus.FAILED
|
871
|
+
else:
|
872
|
+
raise ValueError("Unknown status for the pipeline job.")
|
873
|
+
|
874
|
+
def compute_metadata(
|
875
|
+
self, job: aiplatform.PipelineJob
|
876
|
+
) -> Iterator[Dict[str, MetadataType]]:
|
877
|
+
"""Generate run metadata based on the corresponding Vertex PipelineJob.
|
878
|
+
|
879
|
+
Args:
|
880
|
+
job: The corresponding PipelineJob object.
|
881
|
+
|
882
|
+
Yields:
|
883
|
+
A dictionary of metadata related to the pipeline run.
|
884
|
+
"""
|
885
|
+
metadata: Dict[str, MetadataType] = {}
|
886
|
+
|
887
|
+
# Orchestrator Run ID
|
888
|
+
if run_id := self._compute_orchestrator_run_id(job):
|
889
|
+
metadata[METADATA_ORCHESTRATOR_RUN_ID] = run_id
|
890
|
+
|
891
|
+
# URL to the Vertex's pipeline view
|
892
|
+
if orchestrator_url := self._compute_orchestrator_url(job):
|
893
|
+
metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_url)
|
894
|
+
|
895
|
+
# URL to the corresponding Logs Explorer page
|
896
|
+
if logs_url := self._compute_orchestrator_logs_url(job):
|
897
|
+
metadata[METADATA_ORCHESTRATOR_LOGS_URL] = Uri(logs_url)
|
898
|
+
|
899
|
+
yield metadata
|
900
|
+
|
901
|
+
@staticmethod
|
902
|
+
def _compute_orchestrator_url(
|
903
|
+
job: aiplatform.PipelineJob,
|
904
|
+
) -> Optional[str]:
|
905
|
+
"""Generate the Orchestrator Dashboard URL upon pipeline execution.
|
906
|
+
|
907
|
+
Args:
|
908
|
+
job: The corresponding PipelineJob object.
|
909
|
+
|
910
|
+
Returns:
|
911
|
+
the URL to the dashboard view in Vertex.
|
912
|
+
"""
|
913
|
+
try:
|
914
|
+
return str(job._dashboard_uri())
|
915
|
+
except Exception as e:
|
916
|
+
logger.warning(
|
917
|
+
f"There was an issue while extracting the pipeline url: {e}"
|
918
|
+
)
|
919
|
+
return None
|
920
|
+
|
921
|
+
@staticmethod
|
922
|
+
def _compute_orchestrator_logs_url(
|
923
|
+
job: aiplatform.PipelineJob,
|
924
|
+
) -> Optional[str]:
|
925
|
+
"""Generate the Logs Explorer URL upon pipeline execution.
|
926
|
+
|
927
|
+
Args:
|
928
|
+
job: The corresponding PipelineJob object.
|
929
|
+
|
930
|
+
Returns:
|
931
|
+
the URL querying the pipeline logs in Logs Explorer on GCP.
|
932
|
+
"""
|
933
|
+
try:
|
934
|
+
base_url = "https://console.cloud.google.com/logs/query"
|
935
|
+
query = f"""
|
936
|
+
resource.type="aiplatform.googleapis.com/PipelineJob"
|
937
|
+
resource.labels.pipeline_job_id="{job.job_id}"
|
938
|
+
"""
|
939
|
+
encoded_query = urllib.parse.quote(query)
|
940
|
+
return f"{base_url}?project={job.project}&query={encoded_query}"
|
941
|
+
|
942
|
+
except Exception as e:
|
943
|
+
logger.warning(
|
944
|
+
f"There was an issue while extracting the logs url: {e}"
|
945
|
+
)
|
946
|
+
return None
|
947
|
+
|
948
|
+
@staticmethod
|
949
|
+
def _compute_orchestrator_run_id(
|
950
|
+
job: aiplatform.PipelineJob,
|
951
|
+
) -> Optional[str]:
|
952
|
+
"""Fetch the Orchestrator Run ID upon pipeline execution.
|
953
|
+
|
954
|
+
Args:
|
955
|
+
job: The corresponding PipelineJob object.
|
956
|
+
|
957
|
+
Returns:
|
958
|
+
the Execution ID of the run in Vertex.
|
959
|
+
"""
|
960
|
+
try:
|
961
|
+
if job.job_id:
|
962
|
+
return str(job.job_id)
|
963
|
+
|
964
|
+
return None
|
965
|
+
except Exception as e:
|
966
|
+
logger.warning(
|
967
|
+
f"There was an issue while extracting the pipeline run ID: {e}"
|
968
|
+
)
|
969
|
+
return None
|