zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

zenml/models/__init__.py

@@ -332,7 +332,6 @@ from zenml.models.v2.misc.service_connector_type import (
     ResourceTypeModel,
 )
 from zenml.models.v2.misc.server_models import ServerDatabaseType, ServerModel
-from zenml.models.v2.misc.full_stack import FullStackRequest
 from zenml.models.v2.core.trigger import (
     TriggerRequest,
     TriggerFilter,
@@ -390,6 +389,12 @@ from zenml.models.v2.misc.stack_deployment import (
     StackDeploymentConfig,
     StackDeploymentInfo,
 )
+from zenml.models.v2.misc.info_models import (
+    ComponentInfo,
+    ServiceConnectorInfo,
+    ServiceConnectorResourcesInfo,
+    ResourcesInfo,
+)
 
 # ----------------------------- Forward References -----------------------------
 
@@ -409,7 +414,6 @@ EventSourceResponseMetadata.model_rebuild()
 EventSourceResponseResources.model_rebuild()
 FlavorResponseBody.model_rebuild()
 FlavorResponseMetadata.model_rebuild()
-FullStackRequest.model_rebuild()
 LazyArtifactVersionResponse.model_rebuild()
 LazyRunMetadataResponse.model_rebuild()
 ModelResponseBody.model_rebuild()
@@ -448,8 +452,10 @@ ServiceResponseMetadata.model_rebuild()
 ServiceResponseResources.model_rebuild()
 ServiceConnectorResponseBody.model_rebuild()
 ServiceConnectorResponseMetadata.model_rebuild()
+StackRequest.model_rebuild()
 StackResponseBody.model_rebuild()
 StackResponseMetadata.model_rebuild()
+StackResponse.model_rebuild()
 StepRunRequest.model_rebuild()
 StepRunResponseBody.model_rebuild()
 StepRunResponseMetadata.model_rebuild()
@@ -458,6 +464,10 @@ TriggerResponseBody.model_rebuild()
 TriggerResponseMetadata.model_rebuild()
 TriggerResponseResources.model_rebuild()
 TriggerResponseResources.model_rebuild()
+ComponentInfo.model_rebuild()
+ServiceConnectorInfo.model_rebuild()
+ServiceConnectorResourcesInfo.model_rebuild()
+ResourcesInfo.model_rebuild()
 
 
 __all__ = [
@@ -723,7 +733,6 @@ __all__ = [
     "ServiceConnectorTypedResourcesModel",
     "ServiceConnectorRequirements",
     "ResourceTypeModel",
-    "FullStackRequest",
     "UserAuthModel",
     "ExternalUserModel",
     "BuildItem",
@@ -740,4 +749,8 @@ __all__ = [
     "OAuthDeviceVerificationRequest",
     "OAuthRedirectResponse",
     "OAuthTokenResponse",
+    "ComponentInfo",
+    "ServiceConnectorInfo",
+    "ServiceConnectorResourcesInfo",
+    "ResourcesInfo",
 ]

zenml/models/v2/core/model_version.py

@@ -343,8 +343,8 @@ class ModelVersionResponse(
             version=self.name,
             was_created_in_this_run=was_created_in_this_run,
             suppress_class_validation_warnings=suppress_class_validation_warnings,
+            model_version_id=self.id,
         )
-        mv.model_version_id = self.id
 
         return mv
 

zenml/models/v2/core/pipeline_run.py

@@ -115,6 +115,13 @@ class PipelineRunRequest(WorkspaceScopedRequest):
         default=None,
         title="Tags of the pipeline run.",
     )
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this pipeline run explicitly.",
+        default=None,
+    )
+
+    model_config = ConfigDict(protected_namespaces=())
 
 
 # ------------------ Update Model ------------------
@@ -125,7 +132,12 @@ class PipelineRunUpdate(BaseModel):
 
     status: Optional[ExecutionStatus] = None
     end_time: Optional[datetime] = None
-    # TODO: we should maybe have a different update model here, the upper two attributes should only be for internal use
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this pipeline run explicitly.",
+        default=None,
+    )
+    # TODO: we should maybe have a different update model here, the upper three attributes should only be for internal use
     add_tags: Optional[List[str]] = Field(
         default=None, title="New tags to add to the pipeline run."
     )
@@ -133,6 +145,8 @@ class PipelineRunUpdate(BaseModel):
         default=None, title="Tags to remove from the pipeline run."
     )
 
+    model_config = ConfigDict(protected_namespaces=())
+
 
 # ------------------ Response Model ------------------
 
@@ -164,6 +178,13 @@ class PipelineRunResponseBody(WorkspaceScopedResponseBody):
     trigger_execution: Optional["TriggerExecutionResponse"] = Field(
         default=None, title="The trigger execution that triggered this run."
     )
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this pipeline run explicitly.",
+        default=None,
+    )
+
+    model_config = ConfigDict(protected_namespaces=())
 
 
 class PipelineRunResponseMetadata(WorkspaceScopedResponseMetadata):
@@ -357,6 +378,15 @@ class PipelineRunResponse(
         """
         return self.get_body().deployment_id
 
+    @property
+    def model_version_id(self) -> Optional[UUID]:
+        """The `model_version_id` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_body().model_version_id
+
     @property
     def run_metadata(self) -> Dict[str, "RunMetadataResponse"]:
         """The `run_metadata` property.

zenml/models/v2/core/stack.py

@@ -17,31 +17,38 @@ import json
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union
 from uuid import UUID
 
-from pydantic import Field
+from pydantic import Field, model_validator
 from sqlmodel import and_
 
 from zenml.constants import STR_FIELD_MAX_LENGTH
 from zenml.enums import StackComponentType
-from zenml.models.v2.base.base import BaseUpdate
+from zenml.models.v2.base.base import BaseRequest, BaseUpdate
 from zenml.models.v2.base.scoped import (
     WorkspaceScopedFilter,
-    WorkspaceScopedRequest,
     WorkspaceScopedResponse,
     WorkspaceScopedResponseBody,
     WorkspaceScopedResponseMetadata,
     WorkspaceScopedResponseResources,
 )
-from zenml.models.v2.core.component import ComponentResponse
+from zenml.models.v2.misc.info_models import (
+    ComponentInfo,
+    ServiceConnectorInfo,
+)
 
 if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
 
+    from zenml.models.v2.core.component import ComponentResponse
+
 
 # ------------------ Request Model ------------------
 
 
-class StackRequest(WorkspaceScopedRequest):
-    """Request model for stacks."""
+class StackRequest(BaseRequest):
+    """Request model for a stack."""
+
+    user: Optional[UUID] = None
+    workspace: Optional[UUID] = None
 
     name: str = Field(
         title="The name of the stack.", max_length=STR_FIELD_MAX_LENGTH
@@ -55,15 +62,26 @@ class StackRequest(WorkspaceScopedRequest):
         default=None,
         title="The path to the stack spec used for mlstacks deployments.",
     )
-    components: Optional[Dict[StackComponentType, List[UUID]]] = Field(
-        default=None,
-        title="A mapping of stack component types to the actual"
-        "instances of components of this type.",
+    components: Dict[StackComponentType, List[Union[UUID, ComponentInfo]]] = (
+        Field(
+            title="The mapping for the components of the full stack registration.",
+            description="The mapping from component types to either UUIDs of "
+            "existing components or request information for brand new "
+            "components.",
+        )
     )
     labels: Optional[Dict[str, Any]] = Field(
         default=None,
         title="The stack labels.",
     )
+    service_connectors: List[Union[UUID, ServiceConnectorInfo]] = Field(
+        default=[],
+        title="The service connectors dictionary for the full stack "
+        "registration.",
+        description="The UUID of an already existing service connector or "
+        "request information to create a service connector from "
+        "scratch.",
+    )
 
     @property
     def is_valid(self) -> bool:
@@ -79,14 +97,25 @@ class StackRequest(WorkspaceScopedRequest):
             and StackComponentType.ORCHESTRATOR in self.components
         )
 
-
-class InternalStackRequest(StackRequest):
-    """Internal stack request model."""
-
-    user: Optional[UUID] = Field(  # type: ignore[assignment]
-        title="The id of the user that created this resource.",
-        default=None,
-    )
+    @model_validator(mode="after")
+    def _validate_indexes_in_components(self) -> "StackRequest":
+        for components in self.components.values():
+            for component in components:
+                if isinstance(component, ComponentInfo):
+                    if component.service_connector_index is not None:
+                        if (
+                            component.service_connector_index < 0
+                            or component.service_connector_index
+                            >= len(self.service_connectors)
+                        ):
+                            raise ValueError(
+                                f"Service connector index "
+                                f"{component.service_connector_index} "
+                                "is out of range. Please provide a valid index "
+                                "referring to the position in the list of service "
+                                "connectors."
+                            )
+        return self
 
 
 # ------------------ Update Model ------------------
@@ -130,7 +159,7 @@ class StackResponseBody(WorkspaceScopedResponseBody):
 class StackResponseMetadata(WorkspaceScopedResponseMetadata):
     """Response metadata for stacks."""
 
-    components: Dict[StackComponentType, List[ComponentResponse]] = Field(
+    components: Dict[StackComponentType, List["ComponentResponse"]] = Field(
         title="A mapping of stack component types to the actual"
         "instances of components of this type."
     )
@@ -257,7 +286,9 @@ class StackResponse(
         return self.get_metadata().stack_spec_path
 
     @property
-    def components(self) -> Dict[StackComponentType, List[ComponentResponse]]:
+    def components(
+        self,
+    ) -> Dict[StackComponentType, List["ComponentResponse"]]:
         """The `components` property.
 
         Returns:

zenml/models/v2/core/step_run.py

@@ -108,6 +108,13 @@ class StepRunRequest(WorkspaceScopedRequest):
     deployment: UUID = Field(
         title="The deployment associated with the step run."
     )
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this step run explicitly.",
+        default=None,
+    )
+
+    model_config = ConfigDict(protected_namespaces=())
 
 
 # ------------------ Update Model ------------------
@@ -136,6 +143,12 @@ class StepRunUpdate(BaseModel):
         title="The end time of the step run.",
         default=None,
     )
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this step run explicitly.",
+        default=None,
+    )
+    model_config = ConfigDict(protected_namespaces=())
 
 
 # ------------------ Response Model ------------------
@@ -151,6 +164,12 @@ class StepRunResponseBody(WorkspaceScopedResponseBody):
         title="The output artifact versions of the step run.",
         default={},
     )
+    model_version_id: Optional[UUID] = Field(
+        title="The ID of the model version that was "
+        "configured by this step run explicitly.",
+        default=None,
+    )
+    model_config = ConfigDict(protected_namespaces=())
 
 
 class StepRunResponseMetadata(WorkspaceScopedResponseMetadata):
@@ -320,6 +339,15 @@ class StepRunResponse(
         """
         return self.get_body().outputs
 
+    @property
+    def model_version_id(self) -> Optional[UUID]:
+        """The `model_version_id` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_body().model_version_id
+
     @property
     def config(self) -> "StepConfiguration":
         """The `config` property.

zenml/models/v2/misc/info_models.py

@@ -0,0 +1,78 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Models representing full stack requests."""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, model_validator
+
+from zenml.enums import StackComponentType
+
+if TYPE_CHECKING:
+    from zenml.models.v2.core.component import ComponentResponse
+
+
+class ServiceConnectorInfo(BaseModel):
+    """Information about the service connector when creating a full stack."""
+
+    type: str
+    auth_method: str
+    configuration: Dict[str, Any] = {}
+
+
+class ComponentInfo(BaseModel):
+    """Information about each stack components when creating a full stack."""
+
+    flavor: str
+    service_connector_index: Optional[int] = Field(
+        default=None,
+        title="The id of the service connector from the list "
+        "`service_connectors`.",
+        description="The id of the service connector from the list "
+        "`service_connectors` from `FullStackRequest`.",
+    )
+    service_connector_resource_id: Optional[str] = None
+    configuration: Dict[str, Any] = {}
+
+
+class ResourcesInfo(BaseModel):
+    """Information about the resources needed for CLI and UI."""
+
+    flavor: str
+    flavor_display_name: str
+    required_configuration: Dict[str, str] = {}
+    use_resource_value_as_fixed_config: bool = False
+
+    accessible_by_service_connector: List[str]
+    connected_through_service_connector: List["ComponentResponse"]
+
+    @model_validator(mode="after")
+    def _validate_resource_info(self) -> "ResourcesInfo":
+        if (
+            self.use_resource_value_as_fixed_config
+            and len(self.required_configuration) > 1
+        ):
+            raise ValueError(
+                "Cannot use resource value as fixed config if more than "
+                "one required configuration key is provided."
+            )
+        return self
+
+
+class ServiceConnectorResourcesInfo(BaseModel):
+    """Information about the service connector resources needed for CLI and UI."""
+
+    connector_type: str
+
+    components_resources_info: Dict[StackComponentType, List[ResourcesInfo]]

zenml/new/pipelines/pipeline.py

@@ -17,7 +17,6 @@ import copy
 import hashlib
 import inspect
 from contextlib import contextmanager
-from datetime import datetime
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
@@ -72,7 +71,6 @@ from zenml.new.pipelines import build_utils
 from zenml.new.pipelines.run_utils import (
     create_placeholder_run,
     deploy_pipeline,
-    prepare_model_versions,
     upload_notebook_cell_code_if_necessary,
 )
 from zenml.stack import Stack
@@ -91,6 +89,7 @@ from zenml.utils import (
     source_utils,
     yaml_utils,
 )
+from zenml.utils.string_utils import format_name_template
 
 if TYPE_CHECKING:
     from zenml.artifacts.external_artifact import ExternalArtifact
@@ -587,6 +586,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
         Returns:
             Model of the pipeline run if running without a schedule, `None` if
             running with a schedule.
+
+        Raises:
+            ValueError: if the orchestrator doesn't support scheduling, but schedule was given
         """
         if constants.SHOULD_PREVENT_PIPELINE_EXECUTION:
             # An environment variable was set to stop the execution of
@@ -632,17 +634,24 @@ To avoid this consider setting pipeline parameters only in one place (config or
             else:
                 logger.debug(f"Pipeline {self.name} is unlisted.")
 
-            # TODO: check whether orchestrator even support scheduling before
-            #   registering the schedule
+            stack = Client().active_stack
+            stack.validate()
+
             schedule_id = None
             if schedule:
+                if not stack.orchestrator.config.is_schedulable:
+                    raise ValueError(
+                        f"Stack {stack.name} does not support scheduling. "
+                        "Not all orchestrator types support scheduling, "
+                        "kindly consult with "
+                        "https://docs.zenml.io/how-to/build-pipelines/schedule-a-pipeline "
+                        "for details."
+                    )
                 if schedule.name:
                     schedule_name = schedule.name
                 else:
-                    date = datetime.utcnow().strftime("%Y_%m_%d")
-                    time = datetime.utcnow().strftime("%H_%M_%S_%f")
-                    schedule_name = deployment.run_name_template.format(
-                        date=date, time=time
+                    schedule_name = format_name_template(
+                        deployment.run_name_template
                     )
                 components = Client().active_stack_model.components
                 orchestrator = components[StackComponentType.ORCHESTRATOR][0]
@@ -674,8 +683,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 deployment=deployment, stack=stack
             )
 
-            prepare_model_versions(deployment)
-
             local_repo_context = (
                 code_repository_utils.find_active_code_repository()
             )
@@ -753,7 +760,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
             if run:
                 run_url = dashboard_utils.get_run_url(run)
                 if run_url:
-                    logger.info(f"Dashboard URL: {run_url}")
+                    logger.info(f"Dashboard URL for Pipeline Run: {run_url}")
                 else:
                     logger.info(
                         "You can visualize your pipeline runs in the `ZenML "
@@ -1009,12 +1016,14 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         integration_registry.activate_integrations()
 
-        self._parse_config_file(
+        _from_config_file = self._parse_config_file(
             config_path=config_path,
             matcher=list(PipelineRunConfiguration.model_fields.keys()),
         )
 
-        run_config = PipelineRunConfiguration(**self._from_config_file)
+        self._reconfigure_from_file_with_overrides(config_path=config_path)
+
+        run_config = PipelineRunConfiguration(**_from_config_file)
 
         new_values = dict_utils.remove_none_values(run_configuration_args)
         update = PipelineRunConfiguration.model_validate(new_values)
@@ -1229,12 +1238,15 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
     def _parse_config_file(
         self, config_path: Optional[str], matcher: List[str]
-    ) -> None:
+    ) -> Dict[str, Any]:
         """Parses the given configuration file and sets `self._from_config_file`.
 
         Args:
             config_path: Path to a yaml configuration file.
             matcher: List of keys to match in the configuration file.
+
+        Returns:
+            Parsed config file according to matcher settings.
         """
         _from_config_file: Dict[str, Any] = {}
         if config_path:
@@ -1263,7 +1275,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
                     _from_config_file["model"] = Model.model_validate(
                         _from_config_file["model"]
                     )
-        self._from_config_file = _from_config_file
+        return _from_config_file
 
     def with_options(
         self,
@@ -1314,17 +1326,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
 
         pipeline_copy = self.copy()
 
-        pipeline_copy._parse_config_file(
-            config_path=config_path,
-            matcher=inspect.getfullargspec(self.configure)[0]
-            + ["model_version"],  # TODO: deprecate `model_version` later on
+        pipeline_copy._reconfigure_from_file_with_overrides(
+            config_path=config_path, **kwargs
         )
-        pipeline_copy._from_config_file = dict_utils.recursive_update(
-            pipeline_copy._from_config_file, kwargs
-        )
-
-        with pipeline_copy.__suppress_configure_warnings__():
-            pipeline_copy.configure(**pipeline_copy._from_config_file)
 
         run_args = dict_utils.remove_none_values(
             {
@@ -1422,3 +1426,39 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 )
             else:
                 self.prepare()
+
+    def _reconfigure_from_file_with_overrides(
+        self,
+        config_path: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Update the pipeline configuration from config file.
+
+        Accepts overrides as kwargs.
+
+        Args:
+            config_path: Path to a yaml configuration file. This file will
+                be parsed as a
+                `zenml.config.pipeline_configurations.PipelineRunConfiguration`
+                object. Options provided in this file will be overwritten by
+                options provided in code using the other arguments of this
+                method.
+            **kwargs: Pipeline configuration options. These will be passed
+                to the `pipeline.configure(...)` method.
+        """
+        self._from_config_file = {}
+        if config_path:
+            self._from_config_file = self._parse_config_file(
+                config_path=config_path,
+                matcher=inspect.getfullargspec(self.configure)[0]
+                + [
+                    "model_version"
+                ],  # TODO: deprecate `model_version` later on
+            )
+
+        _from_config_file = dict_utils.recursive_update(
+            self._from_config_file, kwargs
+        )
+
+        with self.__suppress_configure_warnings__():
+            self.configure(**_from_config_file)