zenml-nightly 0.75.0.dev20250312__py3-none-any.whl → 0.75.0.dev20250314__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/analytics/context.py +7 -0
- zenml/analytics/enums.py +2 -2
- zenml/artifacts/utils.py +2 -4
- zenml/cli/__init__.py +8 -9
- zenml/cli/base.py +2 -2
- zenml/cli/code_repository.py +1 -1
- zenml/cli/login.py +6 -0
- zenml/cli/model.py +7 -15
- zenml/cli/pipeline.py +3 -3
- zenml/cli/project.py +172 -0
- zenml/cli/secret.py +47 -44
- zenml/cli/service_accounts.py +0 -1
- zenml/cli/service_connectors.py +15 -17
- zenml/cli/stack.py +0 -3
- zenml/cli/stack_components.py +2 -2
- zenml/cli/tag.py +3 -5
- zenml/cli/utils.py +25 -23
- zenml/client.py +749 -475
- zenml/config/global_config.py +48 -37
- zenml/config/pipeline_configurations.py +3 -2
- zenml/config/pipeline_run_configuration.py +2 -1
- zenml/config/secret_reference_mixin.py +1 -1
- zenml/constants.py +6 -6
- zenml/enums.py +0 -7
- zenml/event_hub/event_hub.py +3 -1
- zenml/exceptions.py +0 -24
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +5 -3
- zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +1 -4
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +7 -6
- zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +1 -4
- zenml/integrations/mlflow/steps/mlflow_registry.py +3 -3
- zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +1 -1
- zenml/integrations/wandb/__init__.py +1 -1
- zenml/integrations/wandb/experiment_trackers/wandb_experiment_tracker.py +29 -9
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +5 -3
- zenml/model/model.py +10 -10
- zenml/model_registries/base_model_registry.py +1 -1
- zenml/models/__init__.py +45 -28
- zenml/models/v2/base/base.py +0 -5
- zenml/models/v2/base/filter.py +2 -2
- zenml/models/v2/base/scoped.py +135 -156
- zenml/models/v2/core/action.py +12 -12
- zenml/models/v2/core/api_key.py +1 -1
- zenml/models/v2/core/artifact.py +31 -18
- zenml/models/v2/core/artifact_version.py +57 -40
- zenml/models/v2/core/code_repository.py +12 -12
- zenml/models/v2/core/component.py +22 -33
- zenml/models/v2/core/device.py +3 -2
- zenml/models/v2/core/event_source.py +14 -14
- zenml/models/v2/core/flavor.py +19 -47
- zenml/models/v2/core/logs.py +1 -2
- zenml/models/v2/core/model.py +23 -20
- zenml/models/v2/core/model_version.py +51 -42
- zenml/models/v2/core/pipeline.py +16 -16
- zenml/models/v2/core/pipeline_build.py +14 -14
- zenml/models/v2/core/pipeline_deployment.py +12 -14
- zenml/models/v2/core/pipeline_run.py +21 -29
- zenml/models/v2/core/project.py +203 -0
- zenml/models/v2/core/run_metadata.py +2 -2
- zenml/models/v2/core/run_template.py +16 -17
- zenml/models/v2/core/schedule.py +12 -21
- zenml/models/v2/core/secret.py +94 -128
- zenml/models/v2/core/server_settings.py +2 -2
- zenml/models/v2/core/service.py +57 -26
- zenml/models/v2/core/service_connector.py +14 -16
- zenml/models/v2/core/stack.py +24 -26
- zenml/models/v2/core/step_run.py +16 -28
- zenml/models/v2/core/tag.py +41 -15
- zenml/models/v2/core/trigger.py +13 -13
- zenml/models/v2/core/trigger_execution.py +2 -2
- zenml/models/v2/core/user.py +2 -2
- zenml/models/v2/misc/statistics.py +45 -0
- zenml/models/v2/misc/tag.py +27 -0
- zenml/orchestrators/cache_utils.py +7 -7
- zenml/orchestrators/input_utils.py +1 -0
- zenml/orchestrators/step_launcher.py +1 -2
- zenml/orchestrators/step_run_utils.py +2 -4
- zenml/orchestrators/step_runner.py +10 -1
- zenml/orchestrators/utils.py +4 -4
- zenml/pipelines/build_utils.py +2 -4
- zenml/pipelines/pipeline_decorator.py +3 -2
- zenml/pipelines/pipeline_definition.py +8 -9
- zenml/pipelines/run_utils.py +4 -4
- zenml/service_connectors/service_connector.py +0 -10
- zenml/service_connectors/service_connector_utils.py +0 -2
- zenml/stack/authentication_mixin.py +1 -1
- zenml/stack/flavor.py +3 -14
- zenml/stack/stack.py +0 -1
- zenml/stack/stack_component.py +1 -5
- zenml/steps/base_step.py +10 -2
- zenml/steps/step_context.py +19 -0
- zenml/utils/string_utils.py +1 -1
- zenml/utils/tag_utils.py +642 -0
- zenml/zen_server/cloud_utils.py +21 -0
- zenml/zen_server/exceptions.py +0 -6
- zenml/zen_server/rbac/endpoint_utils.py +134 -46
- zenml/zen_server/rbac/models.py +65 -3
- zenml/zen_server/rbac/rbac_interface.py +9 -0
- zenml/zen_server/rbac/rbac_sql_zen_store.py +15 -7
- zenml/zen_server/rbac/utils.py +155 -30
- zenml/zen_server/rbac/zenml_cloud_rbac.py +39 -11
- zenml/zen_server/routers/actions_endpoints.py +3 -5
- zenml/zen_server/routers/artifact_endpoint.py +0 -5
- zenml/zen_server/routers/artifact_version_endpoints.py +15 -9
- zenml/zen_server/routers/auth_endpoints.py +22 -7
- zenml/zen_server/routers/code_repositories_endpoints.py +54 -3
- zenml/zen_server/routers/devices_endpoints.py +0 -4
- zenml/zen_server/routers/event_source_endpoints.py +0 -5
- zenml/zen_server/routers/flavors_endpoints.py +0 -5
- zenml/zen_server/routers/logs_endpoints.py +0 -1
- zenml/zen_server/routers/model_versions_endpoints.py +100 -23
- zenml/zen_server/routers/models_endpoints.py +50 -69
- zenml/zen_server/routers/pipeline_builds_endpoints.py +55 -3
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +56 -4
- zenml/zen_server/routers/pipelines_endpoints.py +70 -3
- zenml/zen_server/routers/plugin_endpoints.py +0 -1
- zenml/zen_server/routers/projects_endpoints.py +283 -0
- zenml/zen_server/routers/run_metadata_endpoints.py +97 -0
- zenml/zen_server/routers/run_templates_endpoints.py +64 -3
- zenml/zen_server/routers/runs_endpoints.py +58 -8
- zenml/zen_server/routers/schedule_endpoints.py +67 -6
- zenml/zen_server/routers/secrets_endpoints.py +38 -4
- zenml/zen_server/routers/server_endpoints.py +53 -1
- zenml/zen_server/routers/service_accounts_endpoints.py +14 -15
- zenml/zen_server/routers/service_connectors_endpoints.py +94 -14
- zenml/zen_server/routers/service_endpoints.py +18 -7
- zenml/zen_server/routers/stack_components_endpoints.py +66 -7
- zenml/zen_server/routers/stacks_endpoints.py +95 -6
- zenml/zen_server/routers/steps_endpoints.py +17 -11
- zenml/zen_server/routers/tag_resource_endpoints.py +115 -0
- zenml/zen_server/routers/tags_endpoints.py +6 -17
- zenml/zen_server/routers/triggers_endpoints.py +5 -8
- zenml/zen_server/routers/users_endpoints.py +9 -12
- zenml/zen_server/template_execution/utils.py +8 -7
- zenml/zen_server/utils.py +21 -0
- zenml/zen_server/zen_server_api.py +7 -2
- zenml/zen_stores/base_zen_store.py +50 -69
- zenml/zen_stores/migrations/versions/12eff0206201_rename_workspace_to_project.py +768 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +20 -10
- zenml/zen_stores/migrations/versions/1f9d1cd00b90_add_unique_name_constraints.py +231 -0
- zenml/zen_stores/migrations/versions/288f4fb6e112_make_tags_user_scoped.py +74 -0
- zenml/zen_stores/migrations/versions/2e695a26fe7a_add_user_default_workspace.py +45 -0
- zenml/zen_stores/migrations/versions/3b1776345020_remove_workspace_from_globals.py +81 -0
- zenml/zen_stores/migrations/versions/41b28cae31ce_make_artifacts_workspace_scoped.py +136 -0
- zenml/zen_stores/migrations/versions/9e7bf0970266_adding_exclusive_attribute_to_tags.py +47 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +8 -4
- zenml/zen_stores/migrations/versions/cbc6acd71f92_add_workspace_display_name.py +58 -0
- zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +12 -6
- zenml/zen_stores/migrations/versions/f1d723fd723b_add_secret_private_attr.py +61 -0
- zenml/zen_stores/migrations/versions/f76a368a25a5_add_stack_description.py +35 -0
- zenml/zen_stores/rest_zen_store.py +223 -230
- zenml/zen_stores/schemas/__init__.py +2 -2
- zenml/zen_stores/schemas/action_schemas.py +15 -8
- zenml/zen_stores/schemas/api_key_schemas.py +8 -1
- zenml/zen_stores/schemas/artifact_schemas.py +35 -10
- zenml/zen_stores/schemas/code_repository_schemas.py +22 -17
- zenml/zen_stores/schemas/component_schemas.py +9 -14
- zenml/zen_stores/schemas/event_source_schemas.py +15 -8
- zenml/zen_stores/schemas/flavor_schemas.py +14 -20
- zenml/zen_stores/schemas/model_schemas.py +18 -17
- zenml/zen_stores/schemas/pipeline_build_schemas.py +7 -7
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +10 -8
- zenml/zen_stores/schemas/pipeline_run_schemas.py +9 -12
- zenml/zen_stores/schemas/pipeline_schemas.py +9 -9
- zenml/zen_stores/schemas/{workspace_schemas.py → project_schemas.py} +53 -65
- zenml/zen_stores/schemas/run_metadata_schemas.py +5 -5
- zenml/zen_stores/schemas/run_template_schemas.py +17 -13
- zenml/zen_stores/schemas/schedule_schema.py +16 -21
- zenml/zen_stores/schemas/secret_schemas.py +15 -25
- zenml/zen_stores/schemas/service_connector_schemas.py +8 -17
- zenml/zen_stores/schemas/service_schemas.py +7 -8
- zenml/zen_stores/schemas/stack_schemas.py +12 -15
- zenml/zen_stores/schemas/step_run_schemas.py +14 -15
- zenml/zen_stores/schemas/tag_schemas.py +30 -2
- zenml/zen_stores/schemas/trigger_schemas.py +15 -8
- zenml/zen_stores/schemas/user_schemas.py +12 -2
- zenml/zen_stores/schemas/utils.py +16 -0
- zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +0 -3
- zenml/zen_stores/sql_zen_store.py +2984 -2369
- zenml/zen_stores/template_utils.py +1 -1
- zenml/zen_stores/zen_store_interface.py +136 -126
- {zenml_nightly-0.75.0.dev20250312.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/METADATA +1 -1
- {zenml_nightly-0.75.0.dev20250312.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/RECORD +188 -173
- zenml/cli/workspace.py +0 -86
- zenml/models/v2/core/workspace.py +0 -131
- zenml/zen_server/routers/workspaces_endpoints.py +0 -1469
- {zenml_nightly-0.75.0.dev20250312.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.75.0.dev20250312.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.75.0.dev20250312.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/entry_points.txt +0 -0
zenml/config/global_config.py
CHANGED
@@ -48,7 +48,7 @@ from zenml.logger import get_logger
|
|
48
48
|
from zenml.utils import io_utils, yaml_utils
|
49
49
|
|
50
50
|
if TYPE_CHECKING:
|
51
|
-
from zenml.models import
|
51
|
+
from zenml.models import ProjectResponse, StackResponse
|
52
52
|
from zenml.zen_stores.base_zen_store import BaseZenStore
|
53
53
|
|
54
54
|
logger = get_logger(__name__)
|
@@ -113,7 +113,7 @@ class GlobalConfiguration(BaseModel, metaclass=GlobalConfigMetaClass):
|
|
113
113
|
global config.
|
114
114
|
store: Store configuration.
|
115
115
|
active_stack_id: The ID of the active stack.
|
116
|
-
|
116
|
+
active_project_name: The name of the active project.
|
117
117
|
"""
|
118
118
|
|
119
119
|
user_id: uuid.UUID = Field(default_factory=uuid.uuid4)
|
@@ -123,10 +123,10 @@ class GlobalConfiguration(BaseModel, metaclass=GlobalConfigMetaClass):
|
|
123
123
|
version: Optional[str] = None
|
124
124
|
store: Optional[SerializeAsAny[StoreConfiguration]] = None
|
125
125
|
active_stack_id: Optional[uuid.UUID] = None
|
126
|
-
|
126
|
+
active_project_name: Optional[str] = None
|
127
127
|
|
128
128
|
_zen_store: Optional["BaseZenStore"] = None
|
129
|
-
|
129
|
+
_active_project: Optional["ProjectResponse"] = None
|
130
130
|
_active_stack: Optional["StackResponse"] = None
|
131
131
|
|
132
132
|
def __init__(self, **data: Any) -> None:
|
@@ -385,20 +385,25 @@ class GlobalConfiguration(BaseModel, metaclass=GlobalConfigMetaClass):
|
|
385
385
|
def _sanitize_config(self) -> None:
|
386
386
|
"""Sanitize and save the global configuration.
|
387
387
|
|
388
|
-
This method is called to ensure that the active stack and
|
388
|
+
This method is called to ensure that the active stack and project
|
389
389
|
are set to their default values, if possible.
|
390
390
|
"""
|
391
391
|
# If running in a ZenML server environment, the active stack and
|
392
|
-
#
|
392
|
+
# project are not relevant
|
393
393
|
if ENV_ZENML_SERVER in os.environ:
|
394
394
|
return
|
395
|
-
|
396
|
-
self.
|
395
|
+
active_project, active_stack = self.zen_store.validate_active_config(
|
396
|
+
self.active_project_name,
|
397
397
|
self.active_stack_id,
|
398
398
|
config_name="global",
|
399
399
|
)
|
400
|
-
|
401
|
-
|
400
|
+
if active_project:
|
401
|
+
self.active_project_name = active_project.name
|
402
|
+
self._active_project = active_project
|
403
|
+
else:
|
404
|
+
self.active_project_name = None
|
405
|
+
self._active_project = None
|
406
|
+
|
402
407
|
self.set_active_stack(active_stack)
|
403
408
|
|
404
409
|
@property
|
@@ -714,22 +719,22 @@ class GlobalConfiguration(BaseModel, metaclass=GlobalConfigMetaClass):
|
|
714
719
|
|
715
720
|
return self._zen_store
|
716
721
|
|
717
|
-
def
|
718
|
-
self,
|
719
|
-
) -> "
|
720
|
-
"""Set the
|
722
|
+
def set_active_project(
|
723
|
+
self, project: "ProjectResponse"
|
724
|
+
) -> "ProjectResponse":
|
725
|
+
"""Set the project for the local client.
|
721
726
|
|
722
727
|
Args:
|
723
|
-
|
728
|
+
project: The project to set active.
|
724
729
|
|
725
730
|
Returns:
|
726
|
-
The
|
731
|
+
The project that was set active.
|
727
732
|
"""
|
728
|
-
self.
|
729
|
-
self.
|
730
|
-
# Sanitize the global configuration to reflect the new
|
733
|
+
self.active_project_name = project.name
|
734
|
+
self._active_project = project
|
735
|
+
# Sanitize the global configuration to reflect the new project
|
731
736
|
self._sanitize_config()
|
732
|
-
return
|
737
|
+
return project
|
733
738
|
|
734
739
|
def set_active_stack(self, stack: "StackResponse") -> None:
|
735
740
|
"""Set the active stack for the local client.
|
@@ -740,35 +745,41 @@ class GlobalConfiguration(BaseModel, metaclass=GlobalConfigMetaClass):
|
|
740
745
|
self.active_stack_id = stack.id
|
741
746
|
self._active_stack = stack
|
742
747
|
|
743
|
-
def
|
744
|
-
"""Get a model of the active
|
748
|
+
def get_active_project(self) -> "ProjectResponse":
|
749
|
+
"""Get a model of the active project for the local client.
|
745
750
|
|
746
751
|
Returns:
|
747
|
-
The model of the active
|
752
|
+
The model of the active project.
|
748
753
|
"""
|
749
|
-
|
754
|
+
project_name = self.get_active_project_name()
|
750
755
|
|
751
|
-
if self.
|
752
|
-
return self.
|
756
|
+
if self._active_project is not None:
|
757
|
+
return self._active_project
|
753
758
|
|
754
|
-
|
755
|
-
|
759
|
+
project = self.zen_store.get_project(
|
760
|
+
project_name_or_id=project_name,
|
756
761
|
)
|
757
|
-
return self.
|
758
|
-
|
759
|
-
def get_active_workspace_name(self) -> str:
|
760
|
-
"""Get the name of the active workspace.
|
762
|
+
return self.set_active_project(project)
|
761
763
|
|
762
|
-
|
764
|
+
def get_active_project_name(self) -> str:
|
765
|
+
"""Get the name of the active project.
|
763
766
|
|
764
767
|
Returns:
|
765
|
-
The name of the active
|
768
|
+
The name of the active project.
|
769
|
+
|
770
|
+
Raises:
|
771
|
+
RuntimeError: If the active project is not set.
|
766
772
|
"""
|
767
|
-
if self.
|
773
|
+
if self.active_project_name is None:
|
768
774
|
_ = self.zen_store
|
769
|
-
|
775
|
+
if self.active_project_name is None:
|
776
|
+
raise RuntimeError(
|
777
|
+
"No project is currently set as active. Please set the "
|
778
|
+
"active project using the `zenml project set` CLI "
|
779
|
+
"command."
|
780
|
+
)
|
770
781
|
|
771
|
-
return self.
|
782
|
+
return self.active_project_name
|
772
783
|
|
773
784
|
def get_active_stack_id(self) -> UUID:
|
774
785
|
"""Get the ID of the active stack.
|
@@ -14,7 +14,7 @@
|
|
14
14
|
"""Pipeline configuration classes."""
|
15
15
|
|
16
16
|
from datetime import datetime
|
17
|
-
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
17
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
18
18
|
|
19
19
|
from pydantic import SerializeAsAny, field_validator
|
20
20
|
|
@@ -23,6 +23,7 @@ from zenml.config.retry_config import StepRetryConfig
|
|
23
23
|
from zenml.config.source import SourceWithValidator
|
24
24
|
from zenml.config.strict_base_model import StrictBaseModel
|
25
25
|
from zenml.model.model import Model
|
26
|
+
from zenml.utils.tag_utils import Tag
|
26
27
|
from zenml.utils.time_utils import utc_now
|
27
28
|
|
28
29
|
if TYPE_CHECKING:
|
@@ -41,7 +42,7 @@ class PipelineConfigurationUpdate(StrictBaseModel):
|
|
41
42
|
enable_artifact_visualization: Optional[bool] = None
|
42
43
|
enable_step_logs: Optional[bool] = None
|
43
44
|
settings: Dict[str, SerializeAsAny[BaseSettings]] = {}
|
44
|
-
tags: Optional[List[str]] = None
|
45
|
+
tags: Optional[List[Union[str, "Tag"]]] = None
|
45
46
|
extra: Dict[str, Any] = {}
|
46
47
|
failure_hook_source: Optional[SourceWithValidator] = None
|
47
48
|
success_hook_source: Optional[SourceWithValidator] = None
|
@@ -27,6 +27,7 @@ from zenml.config.strict_base_model import StrictBaseModel
|
|
27
27
|
from zenml.model.model import Model
|
28
28
|
from zenml.models import PipelineBuildBase
|
29
29
|
from zenml.utils import pydantic_utils
|
30
|
+
from zenml.utils.tag_utils import Tag
|
30
31
|
|
31
32
|
|
32
33
|
class PipelineRunConfiguration(
|
@@ -45,7 +46,7 @@ class PipelineRunConfiguration(
|
|
45
46
|
)
|
46
47
|
steps: Dict[str, StepConfigurationUpdate] = {}
|
47
48
|
settings: Dict[str, SerializeAsAny[BaseSettings]] = {}
|
48
|
-
tags: Optional[List[str]] = None
|
49
|
+
tags: Optional[List[Union[str, Tag]]] = None
|
49
50
|
extra: Dict[str, Any] = {}
|
50
51
|
model: Optional[Model] = None
|
51
52
|
parameters: Optional[Dict[str, Any]] = None
|
@@ -115,7 +115,7 @@ class SecretReferenceMixin(BaseModel):
|
|
115
115
|
|
116
116
|
# Try to resolve the secret using the secret store
|
117
117
|
try:
|
118
|
-
secret = Client().
|
118
|
+
secret = Client().get_secret_by_name_and_private_status(
|
119
119
|
name=secret_ref.name,
|
120
120
|
)
|
121
121
|
except (KeyError, NotImplementedError):
|
zenml/constants.py
CHANGED
@@ -143,13 +143,13 @@ ENV_ZENML_REPOSITORY_PATH = "ZENML_REPOSITORY_PATH"
|
|
143
143
|
ENV_ZENML_PREVENT_PIPELINE_EXECUTION = "ZENML_PREVENT_PIPELINE_EXECUTION"
|
144
144
|
ENV_ZENML_ENABLE_RICH_TRACEBACK = "ZENML_ENABLE_RICH_TRACEBACK"
|
145
145
|
ENV_ZENML_ACTIVE_STACK_ID = "ZENML_ACTIVE_STACK_ID"
|
146
|
-
|
146
|
+
ENV_ZENML_ACTIVE_PROJECT_ID = "ZENML_ACTIVE_PROJECT_ID"
|
147
147
|
ENV_ZENML_SUPPRESS_LOGS = "ZENML_SUPPRESS_LOGS"
|
148
148
|
ENV_ZENML_ENABLE_REPO_INIT_WARNINGS = "ZENML_ENABLE_REPO_INIT_WARNINGS"
|
149
149
|
ENV_ZENML_SECRET_VALIDATION_LEVEL = "ZENML_SECRET_VALIDATION_LEVEL"
|
150
150
|
ENV_ZENML_DEFAULT_USER_NAME = "ZENML_DEFAULT_USER_NAME"
|
151
151
|
ENV_ZENML_DEFAULT_USER_PASSWORD = "ZENML_DEFAULT_USER_PASSWORD"
|
152
|
-
|
152
|
+
ENV_ZENML_DEFAULT_PROJECT_NAME = "ZENML_DEFAULT_PROJECT_NAME"
|
153
153
|
ENV_ZENML_STORE_PREFIX = "ZENML_STORE_"
|
154
154
|
ENV_ZENML_SECRETS_STORE_PREFIX = "ZENML_SECRETS_STORE_"
|
155
155
|
ENV_ZENML_BACKUP_SECRETS_STORE_PREFIX = "ZENML_BACKUP_SECRETS_STORE_"
|
@@ -163,7 +163,6 @@ ENV_ZENML_PAGINATION_DEFAULT_LIMIT = "ZENML_PAGINATION_DEFAULT_LIMIT"
|
|
163
163
|
ENV_ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING = (
|
164
164
|
"ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING"
|
165
165
|
)
|
166
|
-
ENV_ZENML_DISABLE_WORKSPACE_WARNINGS = "ZENML_DISABLE_WORKSPACE_WARNINGS"
|
167
166
|
ENV_ZENML_SKIP_IMAGE_BUILDER_DEFAULT = "ZENML_SKIP_IMAGE_BUILDER_DEFAULT"
|
168
167
|
ENV_ZENML_SKIP_STACK_VALIDATION = "ZENML_SKIP_STACK_VALIDATION"
|
169
168
|
ENV_ZENML_SERVER = "ZENML_SERVER"
|
@@ -240,7 +239,7 @@ SQL_STORE_BACKUP_DIRECTORY_NAME = "database_backup"
|
|
240
239
|
|
241
240
|
DEFAULT_USERNAME = "default"
|
242
241
|
DEFAULT_PASSWORD = ""
|
243
|
-
|
242
|
+
DEFAULT_PROJECT_NAME = "default"
|
244
243
|
DEFAULT_STACK_AND_COMPONENT_NAME = "default"
|
245
244
|
|
246
245
|
# Rich config
|
@@ -331,7 +330,6 @@ DEFAULT_ZENML_SERVER_REPORT_USER_ACTIVITY_TO_DB_SECONDS = 30
|
|
331
330
|
DEFAULT_ZENML_SERVER_MAX_REQUEST_BODY_SIZE_IN_BYTES = 256 * 1024 * 1024
|
332
331
|
|
333
332
|
DEFAULT_REPORTABLE_RESOURCES = ["pipeline", "pipeline_run", "model"]
|
334
|
-
REQUIRES_CUSTOM_RESOURCE_REPORTING = ["pipeline", "pipeline_run"]
|
335
333
|
|
336
334
|
# API Endpoint paths:
|
337
335
|
ACTIVATE = "/activate"
|
@@ -358,7 +356,6 @@ EMAIL_ANALYTICS = "/email-opt-in"
|
|
358
356
|
EVENT_FLAVORS = "/event-flavors"
|
359
357
|
EVENT_SOURCES = "/event-sources"
|
360
358
|
FLAVORS = "/flavors"
|
361
|
-
GET_OR_CREATE = "/get-or-create"
|
362
359
|
HEALTH = "/health"
|
363
360
|
INFO = "/info"
|
364
361
|
LOAD_INFO = "/load-info"
|
@@ -402,6 +399,7 @@ STATUS = "/status"
|
|
402
399
|
STEP_CONFIGURATION = "/step-configuration"
|
403
400
|
STEPS = "/steps"
|
404
401
|
TAGS = "/tags"
|
402
|
+
TAG_RESOURCES = "/tag_resources"
|
405
403
|
TRIGGERS = "/triggers"
|
406
404
|
TRIGGER_EXECUTIONS = "/trigger_executions"
|
407
405
|
ONBOARDING_STATE = "/onboarding_state"
|
@@ -411,6 +409,7 @@ VERSION_1 = "/v1"
|
|
411
409
|
VISUALIZE = "/visualize"
|
412
410
|
WEBHOOKS = "/webhooks"
|
413
411
|
WORKSPACES = "/workspaces"
|
412
|
+
PROJECTS = "/projects"
|
414
413
|
|
415
414
|
# model metadata yaml file name
|
416
415
|
MODEL_METADATA_YAML_FILE_NAME = "model_metadata.yaml"
|
@@ -452,6 +451,7 @@ STACK_RECIPES_GITHUB_REPO = "https://github.com/zenml-io/mlops-stacks.git"
|
|
452
451
|
|
453
452
|
# Parameters for internal ZenML Models
|
454
453
|
TEXT_FIELD_MAX_LENGTH = 65535
|
454
|
+
STR_ID_FIELD_MAX_LENGTH = 50
|
455
455
|
STR_FIELD_MAX_LENGTH = 255
|
456
456
|
MEDIUMTEXT_MAX_LENGTH = 2**24 - 1
|
457
457
|
|
zenml/enums.py
CHANGED
@@ -135,13 +135,6 @@ class StackComponentType(StrEnum):
|
|
135
135
|
return f"{self.value}s"
|
136
136
|
|
137
137
|
|
138
|
-
class SecretScope(StrEnum):
|
139
|
-
"""Enum for the scope of a secret."""
|
140
|
-
|
141
|
-
WORKSPACE = "workspace"
|
142
|
-
USER = "user"
|
143
|
-
|
144
|
-
|
145
138
|
class StoreType(StrEnum):
|
146
139
|
"""Zen Store Backend Types."""
|
147
140
|
|
zenml/event_hub/event_hub.py
CHANGED
@@ -126,7 +126,9 @@ class InternalEventHub(BaseEventHub):
|
|
126
126
|
triggers: List[TriggerResponse] = depaginate(
|
127
127
|
self.zen_store.list_triggers,
|
128
128
|
trigger_filter_model=TriggerFilter(
|
129
|
-
|
129
|
+
project=event_source.project.id,
|
130
|
+
event_source_id=event_source.id,
|
131
|
+
is_active=True,
|
130
132
|
),
|
131
133
|
hydrate=True,
|
132
134
|
)
|
zenml/exceptions.py
CHANGED
@@ -172,34 +172,10 @@ class EntityCreationError(ZenMLBaseException, RuntimeError):
|
|
172
172
|
"""Raised when failing to create an entity."""
|
173
173
|
|
174
174
|
|
175
|
-
class ActionExistsError(EntityExistsError):
|
176
|
-
"""Raised when registering an action with a name that already exists."""
|
177
|
-
|
178
|
-
|
179
|
-
class TriggerExistsError(EntityExistsError):
|
180
|
-
"""Raised when registering a trigger with name that already exists."""
|
181
|
-
|
182
|
-
|
183
175
|
class WebhookInactiveError(ZenMLBaseException):
|
184
176
|
"""Raised when source is inactive."""
|
185
177
|
|
186
178
|
|
187
|
-
class StackExistsError(EntityExistsError):
|
188
|
-
"""Raised when trying to register a stack with name that already exists."""
|
189
|
-
|
190
|
-
|
191
|
-
class StackComponentExistsError(EntityExistsError):
|
192
|
-
"""Raised when trying to register a stack component with existing name."""
|
193
|
-
|
194
|
-
|
195
|
-
class EventSourceExistsError(EntityExistsError):
|
196
|
-
"""Raised when trying to register an event source with existing name."""
|
197
|
-
|
198
|
-
|
199
|
-
class SecretExistsError(EntityExistsError):
|
200
|
-
"""Raised when trying to register a secret with existing name."""
|
201
|
-
|
202
|
-
|
203
179
|
class StackValidationError(ZenMLBaseException):
|
204
180
|
"""Raised when a stack configuration is not valid."""
|
205
181
|
|
@@ -870,9 +870,11 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
870
870
|
the URL to the dashboard view in SageMaker.
|
871
871
|
"""
|
872
872
|
try:
|
873
|
-
|
874
|
-
|
875
|
-
|
873
|
+
(
|
874
|
+
region_name,
|
875
|
+
pipeline_name,
|
876
|
+
execution_id,
|
877
|
+
) = dissect_pipeline_execution_arn(execution_arn)
|
876
878
|
|
877
879
|
# Get the Sagemaker session
|
878
880
|
session = self._get_sagemaker_session()
|
@@ -18,7 +18,6 @@ from uuid import UUID
|
|
18
18
|
|
19
19
|
from pydantic import BaseModel, ConfigDict, Field
|
20
20
|
|
21
|
-
from zenml.enums import SecretScope
|
22
21
|
from zenml.event_sources.base_event import (
|
23
22
|
BaseEvent,
|
24
23
|
)
|
@@ -335,9 +334,7 @@ class BitbucketWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
|
|
335
334
|
webhook_secret = SecretRequest(
|
336
335
|
name=f"event_source-{str(event_source.id)}-{random_str(4)}".lower(),
|
337
336
|
values={"webhook_secret": secret_key_value},
|
338
|
-
|
339
|
-
user=event_source.user.id,
|
340
|
-
scope=SecretScope.WORKSPACE,
|
337
|
+
private=False,
|
341
338
|
)
|
342
339
|
secret = self.zen_store.create_secret(webhook_secret)
|
343
340
|
|
@@ -438,7 +438,7 @@ class GCPUserAccountConfig(GCPBaseProjectIDConfig, GCPUserAccountCredentials):
|
|
438
438
|
class GCPServiceAccountConfig(GCPBaseConfig, GCPServiceAccountCredentials):
|
439
439
|
"""GCP service account configuration."""
|
440
440
|
|
441
|
-
|
441
|
+
project_id: Optional[str] = None
|
442
442
|
|
443
443
|
@property
|
444
444
|
def gcp_project_id(self) -> str:
|
@@ -450,14 +450,14 @@ class GCPServiceAccountConfig(GCPBaseConfig, GCPServiceAccountCredentials):
|
|
450
450
|
Returns:
|
451
451
|
The GCP project ID.
|
452
452
|
"""
|
453
|
-
if self.
|
454
|
-
self.
|
453
|
+
if self.project_id is None:
|
454
|
+
self.project_id = json.loads(
|
455
455
|
self.service_account_json.get_secret_value()
|
456
456
|
)["project_id"]
|
457
457
|
# Guaranteed by the field validator
|
458
|
-
assert self.
|
458
|
+
assert self.project_id is not None
|
459
459
|
|
460
|
-
return self.
|
460
|
+
return self.project_id
|
461
461
|
|
462
462
|
|
463
463
|
class GCPExternalAccountConfig(
|
@@ -798,7 +798,8 @@ connector will distribute the service account credentials JSON to clients
|
|
798
798
|
instead (not recommended).
|
799
799
|
|
800
800
|
A GCP project is required and the connector may only be used to access GCP
|
801
|
-
resources in the specified project.
|
801
|
+
resources in the specified project. If the `project_id` is not provided, the
|
802
|
+
connector will use the one extracted from the service account key JSON.
|
802
803
|
|
803
804
|
If you already have the GOOGLE_APPLICATION_CREDENTIALS environment variable
|
804
805
|
configured to point to a service account key JSON file, it will be automatically
|
@@ -19,7 +19,6 @@ from uuid import UUID
|
|
19
19
|
|
20
20
|
from pydantic import BaseModel, ConfigDict, Field
|
21
21
|
|
22
|
-
from zenml.enums import SecretScope
|
23
22
|
from zenml.event_sources.base_event import (
|
24
23
|
BaseEvent,
|
25
24
|
)
|
@@ -361,9 +360,7 @@ class GithubWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
|
|
361
360
|
webhook_secret = SecretRequest(
|
362
361
|
name=f"event_source-{str(event_source.id)}-{random_str(4)}".lower(),
|
363
362
|
values={"webhook_secret": secret_key_value},
|
364
|
-
|
365
|
-
user=event_source.user.id,
|
366
|
-
scope=SecretScope.WORKSPACE,
|
363
|
+
private=False,
|
367
364
|
)
|
368
365
|
secret = self.zen_store.create_secret(webhook_secret)
|
369
366
|
|
@@ -92,7 +92,7 @@ def mlflow_register_model_step(
|
|
92
92
|
pipeline_name = step_context.pipeline.name
|
93
93
|
current_run_name = step_context.pipeline_run.name
|
94
94
|
pipeline_run_uuid = str(step_context.pipeline_run.id)
|
95
|
-
|
95
|
+
zenml_project = str(step_context.pipeline.project.name)
|
96
96
|
|
97
97
|
# Get MLflow run ID either from params or from experiment tracker using
|
98
98
|
# pipeline name and run name
|
@@ -144,8 +144,8 @@ def mlflow_register_model_step(
|
|
144
144
|
metadata.zenml_run_name = run_name
|
145
145
|
if metadata.zenml_pipeline_run_uuid is None:
|
146
146
|
metadata.zenml_pipeline_run_uuid = pipeline_run_uuid
|
147
|
-
if metadata.
|
148
|
-
metadata.
|
147
|
+
if metadata.zenml_project is None:
|
148
|
+
metadata.zenml_project = zenml_project
|
149
149
|
if getattr(metadata, "mlflow_run_id", None) is None:
|
150
150
|
setattr(metadata, "mlflow_run_id", mlflow_run_id)
|
151
151
|
|
@@ -30,7 +30,7 @@ class WandbIntegration(Integration):
|
|
30
30
|
"""Definition of Plotly integration for ZenML."""
|
31
31
|
|
32
32
|
NAME = WANDB
|
33
|
-
REQUIREMENTS = ["wandb>=0.12.12", "Pillow>=9.1.0"]
|
33
|
+
REQUIREMENTS = ["wandb>=0.12.12,<1.0.0", "Pillow>=9.1.0", "weave>=0.51.33,<1.0.0"]
|
34
34
|
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["Pillow"]
|
35
35
|
|
36
36
|
@classmethod
|
@@ -14,7 +14,7 @@
|
|
14
14
|
"""Implementation for the wandb experiment tracker."""
|
15
15
|
|
16
16
|
import os
|
17
|
-
from typing import TYPE_CHECKING,
|
17
|
+
from typing import TYPE_CHECKING, Dict, List, Optional, Type, cast
|
18
18
|
|
19
19
|
import wandb
|
20
20
|
|
@@ -30,8 +30,6 @@ from zenml.logger import get_logger
|
|
30
30
|
from zenml.metadata.metadata_types import Uri
|
31
31
|
|
32
32
|
if TYPE_CHECKING:
|
33
|
-
from wandb import Settings
|
34
|
-
|
35
33
|
from zenml.config.step_run_info import StepRunInfo
|
36
34
|
from zenml.metadata.metadata_types import MetadataType
|
37
35
|
|
@@ -76,9 +74,7 @@ class WandbExperimentTracker(BaseExperimentTracker):
|
|
76
74
|
wandb_run_name = (
|
77
75
|
settings.run_name or f"{info.run_name}_{info.pipeline_step_name}"
|
78
76
|
)
|
79
|
-
self._initialize_wandb(
|
80
|
-
run_name=wandb_run_name, tags=tags, settings=settings.settings
|
81
|
-
)
|
77
|
+
self._initialize_wandb(run_name=wandb_run_name, tags=tags, info=info)
|
82
78
|
|
83
79
|
def get_step_run_metadata(
|
84
80
|
self, info: "StepRunInfo"
|
@@ -131,25 +127,49 @@ class WandbExperimentTracker(BaseExperimentTracker):
|
|
131
127
|
|
132
128
|
def _initialize_wandb(
|
133
129
|
self,
|
130
|
+
info: "StepRunInfo",
|
134
131
|
run_name: str,
|
135
132
|
tags: List[str],
|
136
|
-
settings: Union["Settings", Dict[str, Any], None] = None,
|
137
133
|
) -> None:
|
138
134
|
"""Initializes a wandb run.
|
139
135
|
|
140
136
|
Args:
|
137
|
+
info: Step run information.
|
141
138
|
run_name: Name of the wandb run to create.
|
142
139
|
tags: Tags to attach to the wandb run.
|
143
|
-
settings: Additional settings for the wandb run.
|
144
140
|
"""
|
145
141
|
logger.info(
|
146
142
|
f"Initializing wandb with entity {self.config.entity}, project "
|
147
143
|
f"name: {self.config.project_name}, run_name: {run_name}."
|
148
144
|
)
|
145
|
+
settings = cast(
|
146
|
+
WandbExperimentTrackerSettings, self.get_settings(info)
|
147
|
+
)
|
149
148
|
wandb.init(
|
150
149
|
entity=self.config.entity,
|
151
150
|
project=self.config.project_name,
|
152
151
|
name=run_name,
|
153
152
|
tags=tags,
|
154
|
-
settings=settings,
|
153
|
+
settings=settings.settings,
|
155
154
|
)
|
155
|
+
|
156
|
+
if settings.enable_weave:
|
157
|
+
import weave
|
158
|
+
|
159
|
+
if self.config.project_name:
|
160
|
+
logger.info("Initializing weave")
|
161
|
+
weave.init(project_name=self.config.project_name)
|
162
|
+
else:
|
163
|
+
logger.info(
|
164
|
+
"Weave enabled but no project_name specified. "
|
165
|
+
"Skipping weave initialization."
|
166
|
+
)
|
167
|
+
else:
|
168
|
+
import weave
|
169
|
+
|
170
|
+
if self.config.project_name:
|
171
|
+
logger.info("Disabling weave")
|
172
|
+
weave.init(
|
173
|
+
project_name=self.config.project_name,
|
174
|
+
settings={"disabled": True},
|
175
|
+
)
|
@@ -23,7 +23,7 @@ from typing import (
|
|
23
23
|
cast,
|
24
24
|
)
|
25
25
|
|
26
|
-
from pydantic import
|
26
|
+
from pydantic import BaseModel, field_validator
|
27
27
|
|
28
28
|
from zenml.config.base_settings import BaseSettings
|
29
29
|
from zenml.experiment_trackers.base_experiment_tracker import (
|
@@ -46,11 +46,13 @@ class WandbExperimentTrackerSettings(BaseSettings):
|
|
46
46
|
run_name: The Wandb run name.
|
47
47
|
tags: Tags for the Wandb run.
|
48
48
|
settings: Settings for the Wandb run.
|
49
|
+
enable_weave: Whether to enable Weave integration.
|
49
50
|
"""
|
50
51
|
|
51
52
|
run_name: Optional[str] = None
|
52
53
|
tags: List[str] = []
|
53
54
|
settings: Dict[str, Any] = {}
|
55
|
+
enable_weave: bool = False
|
54
56
|
|
55
57
|
@field_validator("settings", mode="before")
|
56
58
|
@classmethod
|
@@ -69,8 +71,8 @@ class WandbExperimentTrackerSettings(BaseSettings):
|
|
69
71
|
import wandb
|
70
72
|
|
71
73
|
if isinstance(value, wandb.Settings):
|
72
|
-
# Depending on the wandb version, either `model_dump`,
|
73
|
-
# `make_static` or `to_dict` is available to convert the settings
|
74
|
+
# Depending on the wandb version, either `model_dump`,
|
75
|
+
# `make_static` or `to_dict` is available to convert the settings
|
74
76
|
# to a dictionary
|
75
77
|
if isinstance(value, BaseModel):
|
76
78
|
return value.model_dump()
|
zenml/model/model.py
CHANGED
@@ -532,9 +532,11 @@ class Model(BaseModel):
|
|
532
532
|
)
|
533
533
|
model = mv.model
|
534
534
|
else:
|
535
|
+
# TODO: This whole thing needs to be refactored as a get_or_create
|
536
|
+
# REST API call.
|
535
537
|
try:
|
536
|
-
model = zenml_client.
|
537
|
-
model_name_or_id=self.name
|
538
|
+
model = zenml_client.get_model(
|
539
|
+
model_name_or_id=self.name, bypass_lazy_loader=True
|
538
540
|
)
|
539
541
|
except KeyError:
|
540
542
|
model_request = ModelRequest(
|
@@ -546,8 +548,7 @@ class Model(BaseModel):
|
|
546
548
|
limitations=self.limitations,
|
547
549
|
trade_offs=self.trade_offs,
|
548
550
|
ethics=self.ethics,
|
549
|
-
|
550
|
-
workspace=zenml_client.active_workspace.id,
|
551
|
+
project=zenml_client.active_project.id,
|
551
552
|
save_models_to_registry=self.save_models_to_registry,
|
552
553
|
)
|
553
554
|
model_request = ModelRequest.model_validate(model_request)
|
@@ -559,8 +560,8 @@ class Model(BaseModel):
|
|
559
560
|
f"New model `{self.name}` was created implicitly."
|
560
561
|
)
|
561
562
|
except EntityExistsError:
|
562
|
-
model = zenml_client.
|
563
|
-
model_name_or_id=self.name
|
563
|
+
model = zenml_client.get_model(
|
564
|
+
model_name_or_id=self.name, bypass_lazy_loader=True
|
564
565
|
)
|
565
566
|
|
566
567
|
self._model_id = model.id
|
@@ -686,7 +687,7 @@ class Model(BaseModel):
|
|
686
687
|
"model version in given stage exists. It might be missing, if "
|
687
688
|
"the pipeline promoting model version to this stage failed,"
|
688
689
|
" as an example. You can explore model versions using "
|
689
|
-
f"`zenml model version list
|
690
|
+
f"`zenml model version list {self.name}` CLI command."
|
690
691
|
)
|
691
692
|
if str(self.version).isnumeric():
|
692
693
|
raise RuntimeError(
|
@@ -696,13 +697,12 @@ class Model(BaseModel):
|
|
696
697
|
"model version with given number exists. It might be missing, if "
|
697
698
|
"the pipeline creating model version failed,"
|
698
699
|
" as an example. You can explore model versions using "
|
699
|
-
f"`zenml model version list
|
700
|
+
f"`zenml model version list {self.name}` CLI command."
|
700
701
|
)
|
701
702
|
|
702
703
|
client = Client()
|
703
704
|
model_version_request = ModelVersionRequest(
|
704
|
-
|
705
|
-
workspace=client.active_workspace.id,
|
705
|
+
project=client.active_project.id,
|
706
706
|
name=str(self.version) if self.version else None,
|
707
707
|
description=self.description,
|
708
708
|
model=model.id,
|
@@ -68,7 +68,7 @@ class ModelRegistryModelMetadata(BaseModel):
|
|
68
68
|
zenml_pipeline_uuid: Optional[str] = None
|
69
69
|
zenml_pipeline_run_uuid: Optional[str] = None
|
70
70
|
zenml_step_name: Optional[str] = None
|
71
|
-
|
71
|
+
zenml_project: Optional[str] = None
|
72
72
|
|
73
73
|
@property
|
74
74
|
def custom_attributes(self) -> Dict[str, str]:
|