zenml-nightly 0.58.2.dev20240615__py3-none-any.whl → 0.58.2.dev20240622__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/_hub/client.py +8 -5
- zenml/actions/base_action.py +8 -10
- zenml/artifact_stores/base_artifact_store.py +20 -15
- zenml/artifact_stores/local_artifact_store.py +3 -2
- zenml/artifacts/artifact_config.py +34 -19
- zenml/artifacts/external_artifact.py +18 -8
- zenml/artifacts/external_artifact_config.py +14 -6
- zenml/artifacts/unmaterialized_artifact.py +2 -11
- zenml/cli/__init__.py +6 -0
- zenml/cli/artifact.py +20 -2
- zenml/cli/base.py +1 -1
- zenml/cli/served_model.py +0 -1
- zenml/cli/server.py +3 -3
- zenml/cli/utils.py +36 -40
- zenml/cli/web_login.py +2 -2
- zenml/client.py +198 -24
- zenml/client_lazy_loader.py +20 -14
- zenml/config/base_settings.py +5 -6
- zenml/config/build_configuration.py +1 -1
- zenml/config/compiler.py +3 -3
- zenml/config/docker_settings.py +27 -28
- zenml/config/global_config.py +33 -37
- zenml/config/pipeline_configurations.py +8 -11
- zenml/config/pipeline_run_configuration.py +6 -2
- zenml/config/pipeline_spec.py +3 -4
- zenml/config/resource_settings.py +8 -9
- zenml/config/schedule.py +16 -20
- zenml/config/secret_reference_mixin.py +6 -3
- zenml/config/secrets_store_config.py +16 -23
- zenml/config/server_config.py +50 -46
- zenml/config/settings_resolver.py +1 -1
- zenml/config/source.py +45 -35
- zenml/config/step_configurations.py +53 -31
- zenml/config/step_run_info.py +3 -0
- zenml/config/store_config.py +20 -19
- zenml/config/strict_base_model.py +2 -6
- zenml/constants.py +26 -2
- zenml/container_registries/base_container_registry.py +3 -2
- zenml/container_registries/default_container_registry.py +3 -3
- zenml/event_hub/base_event_hub.py +1 -1
- zenml/event_sources/base_event_source.py +11 -16
- zenml/exceptions.py +4 -0
- zenml/integrations/airflow/__init__.py +2 -6
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +6 -7
- zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +13 -249
- zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
- zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +5 -4
- zenml/integrations/aws/__init__.py +1 -1
- zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
- zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +11 -5
- zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
- zenml/integrations/aws/service_connectors/aws_service_connector.py +5 -4
- zenml/integrations/aws/step_operators/sagemaker_step_operator.py +1 -1
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +4 -4
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -3
- zenml/integrations/azure/step_operators/azureml_step_operator.py +2 -1
- zenml/integrations/bentoml/steps/bentoml_deployer.py +1 -1
- zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +8 -12
- zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +1 -1
- zenml/integrations/constants.py +0 -1
- zenml/integrations/deepchecks/__init__.py +1 -0
- zenml/integrations/evidently/__init__.py +5 -3
- zenml/integrations/evidently/column_mapping.py +11 -3
- zenml/integrations/evidently/data_validators/evidently_data_validator.py +21 -3
- zenml/integrations/evidently/metrics.py +5 -6
- zenml/integrations/evidently/tests.py +5 -6
- zenml/integrations/facets/models.py +2 -6
- zenml/integrations/feast/__init__.py +3 -1
- zenml/integrations/feast/feature_stores/feast_feature_store.py +0 -23
- zenml/integrations/gcp/__init__.py +1 -1
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +1 -1
- zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +234 -103
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +57 -42
- zenml/integrations/gcp/step_operators/vertex_step_operator.py +1 -0
- zenml/integrations/github/code_repositories/github_code_repository.py +1 -1
- zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +9 -13
- zenml/integrations/great_expectations/__init__.py +1 -1
- zenml/integrations/great_expectations/data_validators/ge_data_validator.py +44 -44
- zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
- zenml/integrations/great_expectations/ge_store_backend.py +24 -11
- zenml/integrations/great_expectations/materializers/ge_materializer.py +3 -3
- zenml/integrations/great_expectations/utils.py +5 -5
- zenml/integrations/huggingface/__init__.py +3 -0
- zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +1 -1
- zenml/integrations/huggingface/steps/__init__.py +3 -0
- zenml/integrations/huggingface/steps/accelerate_runner.py +149 -0
- zenml/integrations/huggingface/steps/huggingface_deployer.py +2 -2
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +1 -1
- zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +4 -3
- zenml/integrations/kubeflow/__init__.py +1 -1
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +48 -81
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +295 -245
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +1 -1
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -2
- zenml/integrations/kubernetes/pod_settings.py +17 -31
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +8 -7
- zenml/integrations/label_studio/__init__.py +1 -3
- zenml/integrations/label_studio/annotators/label_studio_annotator.py +3 -4
- zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +2 -2
- zenml/integrations/langchain/__init__.py +5 -1
- zenml/integrations/langchain/materializers/document_materializer.py +44 -8
- zenml/integrations/mlflow/__init__.py +9 -3
- zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +1 -1
- zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +29 -37
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +4 -4
- zenml/integrations/mlflow/steps/mlflow_deployer.py +1 -1
- zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
- zenml/integrations/neural_prophet/__init__.py +5 -1
- zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +1 -1
- zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +9 -8
- zenml/integrations/seldon/seldon_client.py +52 -67
- zenml/integrations/seldon/services/seldon_deployment.py +3 -3
- zenml/integrations/seldon/steps/seldon_deployer.py +4 -4
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +15 -5
- zenml/integrations/skypilot_aws/__init__.py +1 -1
- zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
- zenml/integrations/skypilot_gcp/__init__.py +2 -1
- zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
- zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +2 -2
- zenml/integrations/spark/flavors/spark_step_operator_flavor.py +1 -1
- zenml/integrations/spark/step_operators/spark_step_operator.py +2 -0
- zenml/integrations/tekton/__init__.py +1 -1
- zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
- zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +547 -233
- zenml/integrations/tensorboard/__init__.py +1 -12
- zenml/integrations/tensorboard/services/tensorboard_service.py +3 -5
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +6 -6
- zenml/integrations/tensorflow/__init__.py +2 -10
- zenml/integrations/tensorflow/materializers/keras_materializer.py +17 -9
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +9 -14
- zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
- zenml/lineage_graph/lineage_graph.py +1 -1
- zenml/logging/step_logging.py +15 -7
- zenml/materializers/built_in_materializer.py +3 -3
- zenml/materializers/pydantic_materializer.py +2 -2
- zenml/metadata/lazy_load.py +4 -4
- zenml/metadata/metadata_types.py +64 -4
- zenml/model/model.py +79 -54
- zenml/model_deployers/base_model_deployer.py +14 -12
- zenml/model_registries/base_model_registry.py +17 -15
- zenml/models/__init__.py +79 -206
- zenml/models/v2/base/base.py +54 -41
- zenml/models/v2/base/base_plugin_flavor.py +2 -6
- zenml/models/v2/base/filter.py +91 -76
- zenml/models/v2/base/page.py +2 -12
- zenml/models/v2/base/scoped.py +4 -7
- zenml/models/v2/core/api_key.py +22 -8
- zenml/models/v2/core/artifact.py +2 -2
- zenml/models/v2/core/artifact_version.py +74 -40
- zenml/models/v2/core/code_repository.py +37 -10
- zenml/models/v2/core/component.py +65 -16
- zenml/models/v2/core/device.py +14 -4
- zenml/models/v2/core/event_source.py +1 -2
- zenml/models/v2/core/flavor.py +74 -8
- zenml/models/v2/core/logs.py +68 -8
- zenml/models/v2/core/model.py +8 -4
- zenml/models/v2/core/model_version.py +25 -6
- zenml/models/v2/core/model_version_artifact.py +51 -21
- zenml/models/v2/core/model_version_pipeline_run.py +45 -13
- zenml/models/v2/core/pipeline.py +37 -72
- zenml/models/v2/core/pipeline_build.py +29 -17
- zenml/models/v2/core/pipeline_deployment.py +18 -6
- zenml/models/v2/core/pipeline_namespace.py +113 -0
- zenml/models/v2/core/pipeline_run.py +50 -22
- zenml/models/v2/core/run_metadata.py +59 -36
- zenml/models/v2/core/schedule.py +37 -24
- zenml/models/v2/core/secret.py +31 -12
- zenml/models/v2/core/service.py +64 -36
- zenml/models/v2/core/service_account.py +24 -11
- zenml/models/v2/core/service_connector.py +219 -44
- zenml/models/v2/core/stack.py +45 -17
- zenml/models/v2/core/step_run.py +28 -8
- zenml/models/v2/core/tag.py +8 -4
- zenml/models/v2/core/trigger.py +2 -2
- zenml/models/v2/core/trigger_execution.py +1 -0
- zenml/models/v2/core/user.py +18 -21
- zenml/models/v2/core/workspace.py +13 -3
- zenml/models/v2/misc/build_item.py +3 -3
- zenml/models/v2/misc/external_user.py +2 -6
- zenml/models/v2/misc/hub_plugin_models.py +9 -9
- zenml/models/v2/misc/loaded_visualization.py +2 -2
- zenml/models/v2/misc/service_connector_type.py +8 -17
- zenml/models/v2/misc/user_auth.py +7 -2
- zenml/new/pipelines/build_utils.py +3 -3
- zenml/new/pipelines/pipeline.py +17 -13
- zenml/new/pipelines/run_utils.py +103 -1
- zenml/orchestrators/base_orchestrator.py +10 -7
- zenml/orchestrators/local_docker/local_docker_orchestrator.py +1 -1
- zenml/orchestrators/step_launcher.py +28 -4
- zenml/orchestrators/step_runner.py +3 -6
- zenml/orchestrators/utils.py +1 -1
- zenml/plugins/base_plugin_flavor.py +6 -10
- zenml/plugins/plugin_flavor_registry.py +3 -7
- zenml/secret/base_secret.py +7 -8
- zenml/service_connectors/docker_service_connector.py +4 -3
- zenml/service_connectors/service_connector.py +5 -12
- zenml/service_connectors/service_connector_registry.py +2 -4
- zenml/services/container/container_service.py +1 -1
- zenml/services/container/container_service_endpoint.py +1 -1
- zenml/services/local/local_service.py +1 -1
- zenml/services/local/local_service_endpoint.py +1 -1
- zenml/services/service.py +16 -10
- zenml/services/service_type.py +4 -5
- zenml/services/terraform/terraform_service.py +1 -1
- zenml/stack/flavor.py +1 -5
- zenml/stack/flavor_registry.py +4 -4
- zenml/stack/stack.py +4 -1
- zenml/stack/stack_component.py +55 -31
- zenml/step_operators/step_operator_entrypoint_configuration.py +1 -0
- zenml/steps/base_step.py +34 -28
- zenml/steps/entrypoint_function_utils.py +3 -5
- zenml/steps/utils.py +12 -14
- zenml/utils/cuda_utils.py +50 -0
- zenml/utils/deprecation_utils.py +18 -20
- zenml/utils/dict_utils.py +1 -1
- zenml/utils/filesync_model.py +65 -28
- zenml/utils/function_utils.py +260 -0
- zenml/utils/json_utils.py +131 -0
- zenml/utils/mlstacks_utils.py +2 -2
- zenml/utils/pipeline_docker_image_builder.py +9 -0
- zenml/utils/pydantic_utils.py +270 -62
- zenml/utils/secret_utils.py +65 -12
- zenml/utils/source_utils.py +2 -2
- zenml/utils/typed_model.py +5 -3
- zenml/utils/typing_utils.py +243 -0
- zenml/utils/yaml_utils.py +1 -1
- zenml/zen_server/auth.py +2 -2
- zenml/zen_server/cloud_utils.py +6 -6
- zenml/zen_server/deploy/base_provider.py +1 -1
- zenml/zen_server/deploy/deployment.py +6 -8
- zenml/zen_server/deploy/docker/docker_zen_server.py +3 -4
- zenml/zen_server/deploy/local/local_provider.py +0 -1
- zenml/zen_server/deploy/local/local_zen_server.py +6 -6
- zenml/zen_server/deploy/terraform/terraform_zen_server.py +4 -6
- zenml/zen_server/exceptions.py +4 -1
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +1 -1
- zenml/zen_server/pipeline_deployment/utils.py +48 -68
- zenml/zen_server/rbac/models.py +2 -5
- zenml/zen_server/rbac/utils.py +11 -14
- zenml/zen_server/routers/auth_endpoints.py +2 -2
- zenml/zen_server/routers/pipeline_builds_endpoints.py +1 -1
- zenml/zen_server/routers/runs_endpoints.py +1 -1
- zenml/zen_server/routers/secrets_endpoints.py +3 -2
- zenml/zen_server/routers/server_endpoints.py +1 -1
- zenml/zen_server/routers/steps_endpoints.py +1 -1
- zenml/zen_server/routers/workspaces_endpoints.py +1 -1
- zenml/zen_stores/base_zen_store.py +46 -9
- zenml/zen_stores/migrations/utils.py +42 -46
- zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +1 -1
- zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +5 -3
- zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +1 -1
- zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +8 -10
- zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +3 -3
- zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +10 -12
- zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +3 -2
- zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +4 -4
- zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +3 -2
- zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +2 -2
- zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +3 -2
- zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +8 -7
- zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +6 -4
- zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +2 -2
- zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +1 -1
- zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +2 -2
- zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +4 -4
- zenml/zen_stores/migrations/versions/alembic_start.py +1 -1
- zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +4 -4
- zenml/zen_stores/rest_zen_store.py +109 -49
- zenml/zen_stores/schemas/api_key_schemas.py +1 -1
- zenml/zen_stores/schemas/artifact_schemas.py +8 -8
- zenml/zen_stores/schemas/artifact_visualization_schemas.py +3 -3
- zenml/zen_stores/schemas/code_repository_schemas.py +1 -1
- zenml/zen_stores/schemas/component_schemas.py +8 -3
- zenml/zen_stores/schemas/device_schemas.py +8 -6
- zenml/zen_stores/schemas/event_source_schemas.py +3 -4
- zenml/zen_stores/schemas/flavor_schemas.py +5 -3
- zenml/zen_stores/schemas/model_schemas.py +26 -1
- zenml/zen_stores/schemas/pipeline_build_schemas.py +1 -1
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +4 -4
- zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -6
- zenml/zen_stores/schemas/pipeline_schemas.py +5 -2
- zenml/zen_stores/schemas/run_metadata_schemas.py +2 -2
- zenml/zen_stores/schemas/secret_schemas.py +8 -5
- zenml/zen_stores/schemas/server_settings_schemas.py +3 -1
- zenml/zen_stores/schemas/service_connector_schemas.py +1 -1
- zenml/zen_stores/schemas/service_schemas.py +11 -2
- zenml/zen_stores/schemas/stack_schemas.py +1 -1
- zenml/zen_stores/schemas/step_run_schemas.py +11 -11
- zenml/zen_stores/schemas/tag_schemas.py +6 -2
- zenml/zen_stores/schemas/trigger_schemas.py +2 -2
- zenml/zen_stores/schemas/user_schemas.py +2 -2
- zenml/zen_stores/schemas/workspace_schemas.py +3 -1
- zenml/zen_stores/secrets_stores/aws_secrets_store.py +19 -20
- zenml/zen_stores/secrets_stores/azure_secrets_store.py +17 -20
- zenml/zen_stores/secrets_stores/base_secrets_store.py +79 -12
- zenml/zen_stores/secrets_stores/gcp_secrets_store.py +17 -20
- zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +4 -8
- zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +10 -7
- zenml/zen_stores/secrets_stores/sql_secrets_store.py +5 -6
- zenml/zen_stores/sql_zen_store.py +196 -120
- zenml/zen_stores/zen_store_interface.py +33 -0
- {zenml_nightly-0.58.2.dev20240615.dist-info → zenml_nightly-0.58.2.dev20240622.dist-info}/METADATA +9 -7
- {zenml_nightly-0.58.2.dev20240615.dist-info → zenml_nightly-0.58.2.dev20240622.dist-info}/RECORD +310 -307
- zenml/integrations/kubeflow/utils.py +0 -95
- zenml/models/v2/base/internal.py +0 -37
- zenml/models/v2/base/update.py +0 -44
- {zenml_nightly-0.58.2.dev20240615.dist-info → zenml_nightly-0.58.2.dev20240622.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.58.2.dev20240615.dist-info → zenml_nightly-0.58.2.dev20240622.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.58.2.dev20240615.dist-info → zenml_nightly-0.58.2.dev20240622.dist-info}/entry_points.txt +0 -0
@@ -30,18 +30,27 @@ from typing import (
|
|
30
30
|
Dict,
|
31
31
|
ForwardRef,
|
32
32
|
List,
|
33
|
+
NoReturn,
|
33
34
|
Optional,
|
35
|
+
Sequence,
|
34
36
|
Tuple,
|
35
37
|
Type,
|
36
38
|
TypeVar,
|
37
39
|
Union,
|
38
40
|
cast,
|
41
|
+
get_origin,
|
39
42
|
)
|
40
43
|
from uuid import UUID
|
41
44
|
|
42
45
|
from packaging import version
|
43
|
-
from pydantic import
|
44
|
-
|
46
|
+
from pydantic import (
|
47
|
+
ConfigDict,
|
48
|
+
Field,
|
49
|
+
SecretStr,
|
50
|
+
SerializeAsAny,
|
51
|
+
field_validator,
|
52
|
+
model_validator,
|
53
|
+
)
|
45
54
|
from sqlalchemy import asc, desc, func
|
46
55
|
from sqlalchemy.engine import URL, Engine, make_url
|
47
56
|
from sqlalchemy.exc import (
|
@@ -49,7 +58,8 @@ from sqlalchemy.exc import (
|
|
49
58
|
IntegrityError,
|
50
59
|
NoResultFound,
|
51
60
|
)
|
52
|
-
from sqlalchemy.orm import noload
|
61
|
+
from sqlalchemy.orm import Mapped, noload
|
62
|
+
from sqlalchemy.util import immutabledict
|
53
63
|
from sqlmodel import (
|
54
64
|
Session,
|
55
65
|
SQLModel,
|
@@ -71,6 +81,7 @@ from zenml.analytics.utils import (
|
|
71
81
|
track_handler,
|
72
82
|
)
|
73
83
|
from zenml.config.global_config import GlobalConfiguration
|
84
|
+
from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
|
74
85
|
from zenml.config.secrets_store_config import SecretsStoreConfiguration
|
75
86
|
from zenml.config.server_config import ServerConfiguration
|
76
87
|
from zenml.config.store_config import StoreConfiguration
|
@@ -87,6 +98,8 @@ from zenml.constants import (
|
|
87
98
|
SQL_STORE_BACKUP_DIRECTORY_NAME,
|
88
99
|
TEXT_FIELD_MAX_LENGTH,
|
89
100
|
handle_bool_env_var,
|
101
|
+
is_false_string_value,
|
102
|
+
is_true_string_value,
|
90
103
|
)
|
91
104
|
from zenml.enums import (
|
92
105
|
AuthScheme,
|
@@ -266,6 +279,7 @@ from zenml.utils.enum_utils import StrEnum
|
|
266
279
|
from zenml.utils.networking_utils import (
|
267
280
|
replace_localhost_with_internal_hostname,
|
268
281
|
)
|
282
|
+
from zenml.utils.pydantic_utils import before_validator_handler
|
269
283
|
from zenml.utils.string_utils import random_str, validate_name
|
270
284
|
from zenml.zen_stores.base_zen_store import (
|
271
285
|
BaseZenStore,
|
@@ -384,8 +398,10 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
384
398
|
|
385
399
|
type: StoreType = StoreType.SQL
|
386
400
|
|
387
|
-
secrets_store: Optional[SecretsStoreConfiguration] = None
|
388
|
-
backup_secrets_store: Optional[
|
401
|
+
secrets_store: Optional[SerializeAsAny[SecretsStoreConfiguration]] = None
|
402
|
+
backup_secrets_store: Optional[
|
403
|
+
SerializeAsAny[SecretsStoreConfiguration]
|
404
|
+
] = None
|
389
405
|
|
390
406
|
driver: Optional[SQLDatabaseDriver] = None
|
391
407
|
database: Optional[str] = None
|
@@ -409,7 +425,8 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
409
425
|
)
|
410
426
|
backup_database: Optional[str] = None
|
411
427
|
|
412
|
-
@
|
428
|
+
@field_validator("secrets_store")
|
429
|
+
@classmethod
|
413
430
|
def validate_secrets_store(
|
414
431
|
cls, secrets_store: Optional[SecretsStoreConfiguration]
|
415
432
|
) -> SecretsStoreConfiguration:
|
@@ -426,12 +443,14 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
426
443
|
|
427
444
|
return secrets_store
|
428
445
|
|
429
|
-
@
|
430
|
-
|
446
|
+
@model_validator(mode="before")
|
447
|
+
@classmethod
|
448
|
+
@before_validator_handler
|
449
|
+
def _remove_grpc_attributes(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
431
450
|
"""Removes old GRPC attributes.
|
432
451
|
|
433
452
|
Args:
|
434
|
-
|
453
|
+
data: All model attribute values.
|
435
454
|
|
436
455
|
Returns:
|
437
456
|
The model attribute values
|
@@ -443,7 +462,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
443
462
|
"grpc_metadata_ssl_key",
|
444
463
|
"grpc_metadata_ssl_cert",
|
445
464
|
]
|
446
|
-
grpc_values = [
|
465
|
+
grpc_values = [data.pop(key, None) for key in grpc_attribute_keys]
|
447
466
|
if any(grpc_values):
|
448
467
|
logger.warning(
|
449
468
|
"The GRPC attributes %s are unused and will be removed soon. "
|
@@ -451,17 +470,12 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
451
470
|
"become an error in future versions of ZenML."
|
452
471
|
)
|
453
472
|
|
454
|
-
return
|
473
|
+
return data
|
455
474
|
|
456
|
-
@
|
457
|
-
def _validate_backup_strategy(
|
458
|
-
cls, values: Dict[str, Any]
|
459
|
-
) -> Dict[str, Any]:
|
475
|
+
@model_validator(mode="after")
|
476
|
+
def _validate_backup_strategy(self) -> "SqlZenStoreConfiguration":
|
460
477
|
"""Validate the backup strategy.
|
461
478
|
|
462
|
-
Args:
|
463
|
-
values: All model attribute values.
|
464
|
-
|
465
479
|
Returns:
|
466
480
|
The model attribute values.
|
467
481
|
|
@@ -469,28 +483,25 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
469
483
|
ValueError: If the backup database name is not set when the backup
|
470
484
|
database is requested.
|
471
485
|
"""
|
472
|
-
|
473
|
-
|
474
|
-
not
|
486
|
+
if (
|
487
|
+
self.backup_strategy == DatabaseBackupStrategy.DATABASE
|
488
|
+
and not self.backup_database
|
475
489
|
):
|
476
490
|
raise ValueError(
|
477
491
|
"The `backup_database` attribute must also be set if the "
|
478
492
|
"backup strategy is set to use a backup database."
|
479
493
|
)
|
480
494
|
|
481
|
-
return
|
495
|
+
return self
|
482
496
|
|
483
|
-
@
|
484
|
-
def _validate_url(
|
497
|
+
@model_validator(mode="after")
|
498
|
+
def _validate_url(self) -> "SqlZenStoreConfiguration":
|
485
499
|
"""Validate the SQL URL.
|
486
500
|
|
487
501
|
The validator also moves the MySQL username, password and database
|
488
502
|
parameters from the URL into the other configuration arguments, if they
|
489
503
|
are present in the URL.
|
490
504
|
|
491
|
-
Args:
|
492
|
-
values: The values to validate.
|
493
|
-
|
494
505
|
Returns:
|
495
506
|
The validated values.
|
496
507
|
|
@@ -498,14 +509,13 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
498
509
|
ValueError: If the URL is invalid or the SQL driver is not
|
499
510
|
supported.
|
500
511
|
"""
|
501
|
-
url
|
502
|
-
|
503
|
-
return values
|
512
|
+
if self.url is None:
|
513
|
+
return self
|
504
514
|
|
505
515
|
# When running inside a container, if the URL uses localhost, the
|
506
516
|
# target service will not be available. We try to replace localhost
|
507
517
|
# with one of the special Docker or K3D internal hostnames.
|
508
|
-
url = replace_localhost_with_internal_hostname(url)
|
518
|
+
url = replace_localhost_with_internal_hostname(self.url)
|
509
519
|
|
510
520
|
try:
|
511
521
|
sql_url = make_url(url)
|
@@ -524,7 +534,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
524
534
|
url,
|
525
535
|
", ".join(SQLDatabaseDriver.values()),
|
526
536
|
)
|
527
|
-
|
537
|
+
self.driver = SQLDatabaseDriver(sql_url.drivername)
|
528
538
|
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
|
529
539
|
if (
|
530
540
|
sql_url.username
|
@@ -537,33 +547,59 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
537
547
|
"format `sqlite:///path/to/database.db`.",
|
538
548
|
url,
|
539
549
|
)
|
540
|
-
if
|
550
|
+
if self.username or self.password:
|
541
551
|
raise ValueError(
|
542
552
|
"Invalid SQLite configuration: The username and password "
|
543
553
|
"must not be set",
|
544
554
|
url,
|
545
555
|
)
|
546
|
-
|
556
|
+
self.database = sql_url.database
|
547
557
|
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
|
548
558
|
if sql_url.username:
|
549
|
-
|
559
|
+
self.username = sql_url.username
|
550
560
|
sql_url = sql_url._replace(username=None)
|
551
561
|
if sql_url.password:
|
552
|
-
|
562
|
+
self.password = sql_url.password
|
553
563
|
sql_url = sql_url._replace(password=None)
|
554
564
|
if sql_url.database:
|
555
|
-
|
565
|
+
self.database = sql_url.database
|
556
566
|
sql_url = sql_url._replace(database=None)
|
557
567
|
if sql_url.query:
|
568
|
+
|
569
|
+
def _get_query_result(
|
570
|
+
result: Union[str, Tuple[str, ...]],
|
571
|
+
) -> Optional[str]:
|
572
|
+
"""Returns the only or the first result of a query.
|
573
|
+
|
574
|
+
Args:
|
575
|
+
result: The result of the query.
|
576
|
+
|
577
|
+
Returns:
|
578
|
+
The only or the first result, None otherwise.
|
579
|
+
"""
|
580
|
+
if isinstance(result, str):
|
581
|
+
return result
|
582
|
+
elif isinstance(result, tuple) and len(result) > 0:
|
583
|
+
return result[0]
|
584
|
+
else:
|
585
|
+
return None
|
586
|
+
|
558
587
|
for k, v in sql_url.query.items():
|
559
588
|
if k == "ssl_ca":
|
560
|
-
|
589
|
+
if r := _get_query_result(v):
|
590
|
+
self.ssl_ca = r
|
561
591
|
elif k == "ssl_cert":
|
562
|
-
|
592
|
+
if r := _get_query_result(v):
|
593
|
+
self.ssl_cert = r
|
563
594
|
elif k == "ssl_key":
|
564
|
-
|
595
|
+
if r := _get_query_result(v):
|
596
|
+
self.ssl_key = r
|
565
597
|
elif k == "ssl_verify_server_cert":
|
566
|
-
|
598
|
+
if r := _get_query_result(v):
|
599
|
+
if is_true_string_value(r):
|
600
|
+
self.ssl_verify_server_cert = True
|
601
|
+
elif is_false_string_value(r):
|
602
|
+
self.ssl_verify_server_cert = False
|
567
603
|
else:
|
568
604
|
raise ValueError(
|
569
605
|
"Invalid MySQL URL query parameter `%s`: The "
|
@@ -571,14 +607,10 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
571
607
|
"ssl_key, or ssl_verify_server_cert.",
|
572
608
|
k,
|
573
609
|
)
|
574
|
-
sql_url = sql_url._replace(query=
|
610
|
+
sql_url = sql_url._replace(query=immutabledict())
|
575
611
|
|
576
|
-
database =
|
577
|
-
if
|
578
|
-
not values.get("username")
|
579
|
-
or not values.get("password")
|
580
|
-
or not database
|
581
|
-
):
|
612
|
+
database = self.database
|
613
|
+
if not self.username or not self.password or not database:
|
582
614
|
raise ValueError(
|
583
615
|
"Invalid MySQL configuration: The username, password and "
|
584
616
|
"database must be set in the URL or as configuration "
|
@@ -600,7 +632,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
600
632
|
"certificates",
|
601
633
|
)
|
602
634
|
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
|
603
|
-
content =
|
635
|
+
content = getattr(self, key)
|
604
636
|
if content and not os.path.isfile(content):
|
605
637
|
fileio.makedirs(str(secret_folder))
|
606
638
|
file_path = Path(secret_folder, f"{key}.pem")
|
@@ -611,10 +643,10 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
611
643
|
"w",
|
612
644
|
) as f:
|
613
645
|
f.write(content)
|
614
|
-
|
646
|
+
setattr(self, key, str(file_path))
|
615
647
|
|
616
|
-
|
617
|
-
return
|
648
|
+
self.url = str(sql_url)
|
649
|
+
return self
|
618
650
|
|
619
651
|
@staticmethod
|
620
652
|
def get_local_url(path: str) -> str:
|
@@ -722,15 +754,14 @@ class SqlZenStoreConfiguration(StoreConfiguration):
|
|
722
754
|
|
723
755
|
return sql_url, sqlalchemy_connect_args, engine_args
|
724
756
|
|
725
|
-
|
726
|
-
"""Pydantic configuration class."""
|
727
|
-
|
757
|
+
model_config = ConfigDict(
|
728
758
|
# Don't validate attributes when assigning them. This is necessary
|
729
759
|
# because the certificate attributes can be expanded to the contents
|
730
760
|
# of the certificate files.
|
731
|
-
validate_assignment
|
761
|
+
validate_assignment=False,
|
732
762
|
# Forbid extra attributes set in the class.
|
733
|
-
extra
|
763
|
+
extra="forbid",
|
764
|
+
)
|
734
765
|
|
735
766
|
|
736
767
|
class SqlZenStore(BaseZenStore):
|
@@ -893,7 +924,7 @@ class SqlZenStore(BaseZenStore):
|
|
893
924
|
Union[Select[Any], SelectOfScalar[Any]],
|
894
925
|
BaseFilter,
|
895
926
|
],
|
896
|
-
|
927
|
+
Sequence[Any],
|
897
928
|
]
|
898
929
|
] = None,
|
899
930
|
hydrate: bool = False,
|
@@ -928,28 +959,33 @@ class SqlZenStore(BaseZenStore):
|
|
928
959
|
query = filter_model.apply_filter(query=query, table=table)
|
929
960
|
|
930
961
|
# Get the total amount of items in the database for a given query
|
931
|
-
custom_fetch_result: Optional[
|
962
|
+
custom_fetch_result: Optional[Sequence[Any]] = None
|
932
963
|
if custom_fetch:
|
933
964
|
custom_fetch_result = custom_fetch(session, query, filter_model)
|
934
965
|
total = len(custom_fetch_result)
|
935
966
|
else:
|
936
|
-
|
937
|
-
select(
|
967
|
+
result = session.scalar(
|
968
|
+
select(func.count()).select_from(
|
938
969
|
query.options(noload("*")).subquery()
|
939
970
|
)
|
940
971
|
)
|
941
972
|
|
973
|
+
if result:
|
974
|
+
total = result
|
975
|
+
else:
|
976
|
+
total = 0
|
977
|
+
|
942
978
|
# Sorting
|
943
979
|
column, operand = filter_model.sorting_params
|
944
980
|
if operand == SorterOps.DESCENDING:
|
945
|
-
sort_clause = desc(getattr(table, column))
|
981
|
+
sort_clause = desc(getattr(table, column)) # type: ignore[var-annotated]
|
946
982
|
else:
|
947
983
|
sort_clause = asc(getattr(table, column))
|
948
984
|
|
949
985
|
# We always add the `id` column as a tiebreaker to ensure a stable,
|
950
986
|
# repeatable order of items, otherwise subsequent pages might contain
|
951
987
|
# the same items.
|
952
|
-
query = query.order_by(sort_clause, asc(table.id))
|
988
|
+
query = query.order_by(sort_clause, asc(table.id)) # type: ignore[arg-type]
|
953
989
|
|
954
990
|
# Get the total amount of pages in the database for a given query
|
955
991
|
if total == 0:
|
@@ -966,7 +1002,7 @@ class SqlZenStore(BaseZenStore):
|
|
966
1002
|
)
|
967
1003
|
|
968
1004
|
# Get a page of the actual data
|
969
|
-
item_schemas:
|
1005
|
+
item_schemas: Sequence[AnySchema]
|
970
1006
|
if custom_fetch:
|
971
1007
|
assert custom_fetch_result is not None
|
972
1008
|
item_schemas = custom_fetch_result
|
@@ -975,13 +1011,9 @@ class SqlZenStore(BaseZenStore):
|
|
975
1011
|
filter_model.offset : filter_model.offset + filter_model.size
|
976
1012
|
]
|
977
1013
|
else:
|
978
|
-
item_schemas = (
|
979
|
-
|
980
|
-
|
981
|
-
)
|
982
|
-
.unique()
|
983
|
-
.all()
|
984
|
-
)
|
1014
|
+
item_schemas = session.exec(
|
1015
|
+
query.limit(filter_model.size).offset(filter_model.offset)
|
1016
|
+
).all()
|
985
1017
|
|
986
1018
|
# Convert this page of items from schemas to models.
|
987
1019
|
items: List[AnyResponse] = []
|
@@ -1465,9 +1497,7 @@ class SqlZenStore(BaseZenStore):
|
|
1465
1497
|
# the settings table with needed info.
|
1466
1498
|
logger.info("Creating database tables")
|
1467
1499
|
with self.engine.begin() as conn:
|
1468
|
-
|
1469
|
-
SQLModel.metadata.create_all # type: ignore[arg-type]
|
1470
|
-
)
|
1500
|
+
SQLModel.metadata.create_all(conn)
|
1471
1501
|
with Session(self.engine) as session:
|
1472
1502
|
server_config = ServerConfiguration.get_server_config()
|
1473
1503
|
|
@@ -1614,7 +1644,7 @@ class SqlZenStore(BaseZenStore):
|
|
1614
1644
|
with Session(self.engine) as session:
|
1615
1645
|
settings = self._get_server_settings(session=session)
|
1616
1646
|
|
1617
|
-
analytics_metadata = settings_update.
|
1647
|
+
analytics_metadata = settings_update.model_dump(
|
1618
1648
|
include={
|
1619
1649
|
"enable_analytics",
|
1620
1650
|
"display_announcements",
|
@@ -2052,7 +2082,6 @@ class SqlZenStore(BaseZenStore):
|
|
2052
2082
|
"""
|
2053
2083
|
# Check if service with the same domain key (name, config, workspace)
|
2054
2084
|
# already exists
|
2055
|
-
|
2056
2085
|
existing_domain_service = session.exec(
|
2057
2086
|
select(ServiceSchema).where(
|
2058
2087
|
ServiceSchema.config
|
@@ -2060,7 +2089,6 @@ class SqlZenStore(BaseZenStore):
|
|
2060
2089
|
json.dumps(
|
2061
2090
|
service_request.config,
|
2062
2091
|
sort_keys=False,
|
2063
|
-
default=pydantic_encoder,
|
2064
2092
|
).encode("utf-8")
|
2065
2093
|
)
|
2066
2094
|
)
|
@@ -2068,8 +2096,9 @@ class SqlZenStore(BaseZenStore):
|
|
2068
2096
|
|
2069
2097
|
if existing_domain_service:
|
2070
2098
|
raise EntityExistsError(
|
2071
|
-
f"Unable to create service '{service_request.name}' with the
|
2072
|
-
"A service with the same configuration
|
2099
|
+
f"Unable to create service '{service_request.name}' with the "
|
2100
|
+
"given configuration: A service with the same configuration "
|
2101
|
+
"already exists."
|
2073
2102
|
)
|
2074
2103
|
|
2075
2104
|
def create_service(self, service: ServiceRequest) -> ServiceResponse:
|
@@ -3016,7 +3045,9 @@ class SqlZenStore(BaseZenStore):
|
|
3016
3045
|
if existing_component.name != component_update.name:
|
3017
3046
|
self._fail_if_component_with_name_type_exists(
|
3018
3047
|
name=component_update.name,
|
3019
|
-
component_type=
|
3048
|
+
component_type=StackComponentType(
|
3049
|
+
existing_component.type
|
3050
|
+
),
|
3020
3051
|
workspace_id=existing_component.workspace_id,
|
3021
3052
|
session=session,
|
3022
3053
|
)
|
@@ -3736,18 +3767,18 @@ class SqlZenStore(BaseZenStore):
|
|
3736
3767
|
session: Session,
|
3737
3768
|
query: Union[Select[Any], SelectOfScalar[Any]],
|
3738
3769
|
filter: BaseFilter,
|
3739
|
-
) ->
|
3740
|
-
return session.exec(query).
|
3770
|
+
) -> Sequence[Any]:
|
3771
|
+
return session.exec(query).all()
|
3741
3772
|
|
3742
3773
|
with Session(self.engine) as session:
|
3743
3774
|
max_date_subquery = (
|
3744
|
-
select(
|
3775
|
+
select(
|
3745
3776
|
PipelineSchema.name,
|
3746
3777
|
func.max(PipelineRunSchema.created).label("max_created"),
|
3747
3778
|
)
|
3748
3779
|
.outerjoin(
|
3749
3780
|
PipelineRunSchema,
|
3750
|
-
PipelineSchema.id == PipelineRunSchema.pipeline_id,
|
3781
|
+
PipelineSchema.id == PipelineRunSchema.pipeline_id, # type: ignore[arg-type]
|
3751
3782
|
)
|
3752
3783
|
.group_by(PipelineSchema.name)
|
3753
3784
|
.subquery()
|
@@ -3759,16 +3790,12 @@ class SqlZenStore(BaseZenStore):
|
|
3759
3790
|
PipelineRunSchema.id,
|
3760
3791
|
PipelineRunSchema.status,
|
3761
3792
|
)
|
3762
|
-
.outerjoin(
|
3763
|
-
PipelineSchema,
|
3764
|
-
PipelineSchema.name == max_date_subquery.c.name,
|
3765
|
-
)
|
3766
3793
|
.outerjoin(
|
3767
3794
|
PipelineRunSchema,
|
3768
|
-
PipelineRunSchema.created
|
3795
|
+
PipelineRunSchema.created # type: ignore[arg-type]
|
3769
3796
|
== max_date_subquery.c.max_created,
|
3770
3797
|
)
|
3771
|
-
.order_by(desc(PipelineRunSchema.updated))
|
3798
|
+
.order_by(desc(PipelineRunSchema.updated)) # type: ignore[arg-type]
|
3772
3799
|
)
|
3773
3800
|
|
3774
3801
|
return self.filter_and_paginate(
|
@@ -3984,6 +4011,24 @@ class SqlZenStore(BaseZenStore):
|
|
3984
4011
|
session.delete(build)
|
3985
4012
|
session.commit()
|
3986
4013
|
|
4014
|
+
def run_build(
|
4015
|
+
self,
|
4016
|
+
build_id: UUID,
|
4017
|
+
run_configuration: Optional[PipelineRunConfiguration] = None,
|
4018
|
+
) -> NoReturn:
|
4019
|
+
"""Run a pipeline from a build.
|
4020
|
+
|
4021
|
+
Args:
|
4022
|
+
build_id: The ID of the build to run.
|
4023
|
+
run_configuration: Configuration for the run.
|
4024
|
+
|
4025
|
+
Raises:
|
4026
|
+
NotImplementedError: Always.
|
4027
|
+
"""
|
4028
|
+
raise NotImplementedError(
|
4029
|
+
"Running a build is not possible with a local store."
|
4030
|
+
)
|
4031
|
+
|
3987
4032
|
# -------------------------- Pipeline Deployments --------------------------
|
3988
4033
|
|
3989
4034
|
def create_deployment(
|
@@ -4109,6 +4154,24 @@ class SqlZenStore(BaseZenStore):
|
|
4109
4154
|
|
4110
4155
|
session.commit()
|
4111
4156
|
|
4157
|
+
def run_deployment(
|
4158
|
+
self,
|
4159
|
+
deployment_id: UUID,
|
4160
|
+
run_configuration: Optional[PipelineRunConfiguration] = None,
|
4161
|
+
) -> NoReturn:
|
4162
|
+
"""Run a pipeline from a deployment.
|
4163
|
+
|
4164
|
+
Args:
|
4165
|
+
deployment_id: The ID of the deployment to run.
|
4166
|
+
run_configuration: Configuration for the run.
|
4167
|
+
|
4168
|
+
Raises:
|
4169
|
+
NotImplementedError: Always.
|
4170
|
+
"""
|
4171
|
+
raise NotImplementedError(
|
4172
|
+
"Running a deployment is not possible with a local store."
|
4173
|
+
)
|
4174
|
+
|
4112
4175
|
# -------------------- Event Sources --------------------
|
4113
4176
|
|
4114
4177
|
def _fail_if_event_source_with_name_exists(
|
@@ -5932,7 +5995,7 @@ class SqlZenStore(BaseZenStore):
|
|
5932
5995
|
SelectOfScalar[ServiceConnectorSchema],
|
5933
5996
|
],
|
5934
5997
|
filter_model: BaseFilter,
|
5935
|
-
) ->
|
5998
|
+
) -> Sequence[ServiceConnectorSchema]:
|
5936
5999
|
"""Custom fetch function for connector filtering and pagination.
|
5937
6000
|
|
5938
6001
|
Applies resource type and label filters to the query.
|
@@ -6286,7 +6349,7 @@ class SqlZenStore(BaseZenStore):
|
|
6286
6349
|
SelectOfScalar[ServiceConnectorSchema],
|
6287
6350
|
],
|
6288
6351
|
filter_model: ServiceConnectorFilter,
|
6289
|
-
) ->
|
6352
|
+
) -> Sequence[ServiceConnectorSchema]:
|
6290
6353
|
"""Refine a service connector query.
|
6291
6354
|
|
6292
6355
|
Applies resource type and label filters to the query.
|
@@ -6299,9 +6362,7 @@ class SqlZenStore(BaseZenStore):
|
|
6299
6362
|
Returns:
|
6300
6363
|
The filtered list of service connectors.
|
6301
6364
|
"""
|
6302
|
-
items:
|
6303
|
-
session.exec(query).unique().all()
|
6304
|
-
)
|
6365
|
+
items: Sequence[ServiceConnectorSchema] = session.exec(query).all()
|
6305
6366
|
|
6306
6367
|
# filter out items that don't match the resource type
|
6307
6368
|
if filter_model.resource_type:
|
@@ -6609,7 +6670,11 @@ class SqlZenStore(BaseZenStore):
|
|
6609
6670
|
"""
|
6610
6671
|
validate_name(stack)
|
6611
6672
|
with Session(self.engine) as session:
|
6612
|
-
self._fail_if_stack_with_name_exists(
|
6673
|
+
self._fail_if_stack_with_name_exists(
|
6674
|
+
stack_name=stack.name,
|
6675
|
+
workspace_id=stack.workspace,
|
6676
|
+
session=session,
|
6677
|
+
)
|
6613
6678
|
|
6614
6679
|
# Get the Schemas of all components mentioned
|
6615
6680
|
component_ids = (
|
@@ -6731,20 +6796,23 @@ class SqlZenStore(BaseZenStore):
|
|
6731
6796
|
if stack_update.name:
|
6732
6797
|
if existing_stack.name != stack_update.name:
|
6733
6798
|
self._fail_if_stack_with_name_exists(
|
6734
|
-
|
6799
|
+
stack_name=stack_update.name,
|
6800
|
+
workspace_id=existing_stack.workspace.id,
|
6735
6801
|
session=session,
|
6736
6802
|
)
|
6737
6803
|
|
6738
|
-
components = []
|
6804
|
+
components: List["StackComponentSchema"] = []
|
6739
6805
|
if stack_update.components:
|
6740
6806
|
filters = [
|
6741
6807
|
(StackComponentSchema.id == component_id)
|
6742
6808
|
for list_of_component_ids in stack_update.components.values()
|
6743
6809
|
for component_id in list_of_component_ids
|
6744
6810
|
]
|
6745
|
-
components =
|
6746
|
-
|
6747
|
-
|
6811
|
+
components = list(
|
6812
|
+
session.exec(
|
6813
|
+
select(StackComponentSchema).where(or_(*filters))
|
6814
|
+
).all()
|
6815
|
+
)
|
6748
6816
|
|
6749
6817
|
existing_stack.update(
|
6750
6818
|
stack_update=stack_update,
|
@@ -6800,14 +6868,16 @@ class SqlZenStore(BaseZenStore):
|
|
6800
6868
|
|
6801
6869
|
def _fail_if_stack_with_name_exists(
|
6802
6870
|
self,
|
6803
|
-
|
6871
|
+
stack_name: str,
|
6872
|
+
workspace_id: UUID,
|
6804
6873
|
session: Session,
|
6805
6874
|
) -> None:
|
6806
6875
|
"""Raise an exception if a stack with same name exists.
|
6807
6876
|
|
6808
6877
|
Args:
|
6809
|
-
|
6810
|
-
|
6878
|
+
stack_name: The name of the stack
|
6879
|
+
workspace_id: The ID of the workspace
|
6880
|
+
session: The session
|
6811
6881
|
|
6812
6882
|
Returns:
|
6813
6883
|
None
|
@@ -6817,16 +6887,16 @@ class SqlZenStore(BaseZenStore):
|
|
6817
6887
|
"""
|
6818
6888
|
existing_domain_stack = session.exec(
|
6819
6889
|
select(StackSchema)
|
6820
|
-
.where(StackSchema.name ==
|
6821
|
-
.where(StackSchema.workspace_id ==
|
6890
|
+
.where(StackSchema.name == stack_name)
|
6891
|
+
.where(StackSchema.workspace_id == workspace_id)
|
6822
6892
|
).first()
|
6823
6893
|
if existing_domain_stack is not None:
|
6824
6894
|
workspace = self._get_workspace_schema(
|
6825
|
-
workspace_name_or_id=
|
6895
|
+
workspace_name_or_id=workspace_id, session=session
|
6826
6896
|
)
|
6827
6897
|
raise StackExistsError(
|
6828
6898
|
f"Unable to register stack with name "
|
6829
|
-
f"'{
|
6899
|
+
f"'{stack_name}': Found an existing stack with the same "
|
6830
6900
|
f"name in the active workspace, '{workspace.name}'."
|
6831
6901
|
)
|
6832
6902
|
return None
|
@@ -7251,7 +7321,7 @@ class SqlZenStore(BaseZenStore):
|
|
7251
7321
|
step_id=run_step_id,
|
7252
7322
|
artifact_id=artifact_version_id,
|
7253
7323
|
name=name,
|
7254
|
-
type=input_type,
|
7324
|
+
type=input_type.value,
|
7255
7325
|
)
|
7256
7326
|
session.add(assignment)
|
7257
7327
|
|
@@ -7313,7 +7383,7 @@ class SqlZenStore(BaseZenStore):
|
|
7313
7383
|
step_id=step_run_id,
|
7314
7384
|
artifact_id=artifact_version_id,
|
7315
7385
|
name=name,
|
7316
|
-
type=output_type,
|
7386
|
+
type=output_type.value,
|
7317
7387
|
)
|
7318
7388
|
session.add(assignment)
|
7319
7389
|
|
@@ -7345,7 +7415,9 @@ class SqlZenStore(BaseZenStore):
|
|
7345
7415
|
assert pipeline_run.deployment
|
7346
7416
|
num_steps = len(pipeline_run.deployment.to_model().step_configurations)
|
7347
7417
|
new_status = get_pipeline_run_status(
|
7348
|
-
step_statuses=[
|
7418
|
+
step_statuses=[
|
7419
|
+
ExecutionStatus(step_run.status) for step_run in step_runs
|
7420
|
+
],
|
7349
7421
|
num_steps=num_steps,
|
7350
7422
|
)
|
7351
7423
|
|
@@ -7755,6 +7827,8 @@ class SqlZenStore(BaseZenStore):
|
|
7755
7827
|
for resource_attr in resource_attrs:
|
7756
7828
|
# Extract the target schema from the annotation
|
7757
7829
|
annotation = UserSchema.__annotations__[resource_attr]
|
7830
|
+
if get_origin(annotation) == Mapped:
|
7831
|
+
annotation = annotation.__args__[0]
|
7758
7832
|
|
7759
7833
|
# The annotation must be of the form
|
7760
7834
|
# `typing.List[ForwardRef('<schema-class>')]`
|
@@ -7812,11 +7886,13 @@ class SqlZenStore(BaseZenStore):
|
|
7812
7886
|
resource_attrs = self._get_resource_references()
|
7813
7887
|
for schema, resource_attr in resource_attrs:
|
7814
7888
|
# Check if the user owns any resources of this type
|
7815
|
-
count =
|
7816
|
-
|
7889
|
+
count = (
|
7890
|
+
session.query(func.count())
|
7817
7891
|
.select_from(schema)
|
7818
7892
|
.where(getattr(schema, resource_attr) == account.id)
|
7893
|
+
.scalar()
|
7819
7894
|
)
|
7895
|
+
|
7820
7896
|
if count > 0:
|
7821
7897
|
logger.debug(
|
7822
7898
|
f"User {account.name} owns {count} resources of type "
|
@@ -8052,7 +8128,7 @@ class SqlZenStore(BaseZenStore):
|
|
8052
8128
|
):
|
8053
8129
|
# There must be at least one admin account configured
|
8054
8130
|
admin_accounts_count = session.scalar(
|
8055
|
-
select(
|
8131
|
+
select(func.count(UserSchema.id)).where( # type: ignore[arg-type]
|
8056
8132
|
UserSchema.is_admin == True # noqa: E712
|
8057
8133
|
)
|
8058
8134
|
)
|
@@ -8141,7 +8217,7 @@ class SqlZenStore(BaseZenStore):
|
|
8141
8217
|
if user.is_admin:
|
8142
8218
|
# Don't allow the last admin to be deleted
|
8143
8219
|
admin_accounts_count = session.scalar(
|
8144
|
-
select(
|
8220
|
+
select(func.count(UserSchema.id)).where( # type: ignore[arg-type]
|
8145
8221
|
UserSchema.is_admin == True # noqa: E712
|
8146
8222
|
)
|
8147
8223
|
)
|
@@ -8351,7 +8427,7 @@ class SqlZenStore(BaseZenStore):
|
|
8351
8427
|
)
|
8352
8428
|
if (
|
8353
8429
|
existing_workspace.name == self._default_workspace_name
|
8354
|
-
and "name" in workspace_update.
|
8430
|
+
and "name" in workspace_update.model_fields_set
|
8355
8431
|
and workspace_update.name != existing_workspace.name
|
8356
8432
|
):
|
8357
8433
|
raise IllegalOperationError(
|
@@ -8426,14 +8502,14 @@ class SqlZenStore(BaseZenStore):
|
|
8426
8502
|
Count of the entity as integer.
|
8427
8503
|
"""
|
8428
8504
|
with Session(self.engine) as session:
|
8429
|
-
query = select(
|
8505
|
+
query = select(func.count(schema.id)) # type: ignore[arg-type]
|
8430
8506
|
|
8431
8507
|
if filter_model:
|
8432
8508
|
query = filter_model.apply_filter(query=query, table=schema)
|
8433
8509
|
|
8434
8510
|
entity_count = session.scalar(query)
|
8435
8511
|
|
8436
|
-
return int(entity_count)
|
8512
|
+
return int(entity_count) if entity_count else 0
|
8437
8513
|
|
8438
8514
|
def entity_exists(
|
8439
8515
|
self, entity_id: UUID, schema_class: Type[AnySchema]
|
@@ -8956,7 +9032,7 @@ class SqlZenStore(BaseZenStore):
|
|
8956
9032
|
"`number` field must be None during model version creation."
|
8957
9033
|
)
|
8958
9034
|
with Session(self.engine) as session:
|
8959
|
-
model_version_ = model_version.
|
9035
|
+
model_version_ = model_version.model_copy()
|
8960
9036
|
model = self.get_model(model_version_.model)
|
8961
9037
|
|
8962
9038
|
def _check(tolerance: int = 0) -> None:
|
@@ -9345,7 +9421,7 @@ class SqlZenStore(BaseZenStore):
|
|
9345
9421
|
)
|
9346
9422
|
session.execute(
|
9347
9423
|
delete(ModelVersionArtifactSchema).where(
|
9348
|
-
ModelVersionArtifactSchema.model_version_id
|
9424
|
+
ModelVersionArtifactSchema.model_version_id # type: ignore[arg-type]
|
9349
9425
|
== model_version_id
|
9350
9426
|
)
|
9351
9427
|
)
|