zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl

zenml/models/v2/core/pipeline.py

@@ -13,27 +13,35 @@
 #  permissions and limitations under the License.
 """Models representing pipelines."""
 
-from typing import TYPE_CHECKING, Any, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Type, TypeVar, Union
 from uuid import UUID
 
 from pydantic import Field
 
-from zenml.config.pipeline_spec import PipelineSpec
-from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
+from zenml.constants import (
+    SORT_PIPELINES_BY_LATEST_RUN_KEY,
+    STR_FIELD_MAX_LENGTH,
+    TEXT_FIELD_MAX_LENGTH,
+)
 from zenml.enums import ExecutionStatus
 from zenml.models.v2.base.base import BaseUpdate
 from zenml.models.v2.base.scoped import (
-    WorkspaceScopedFilter,
     WorkspaceScopedRequest,
     WorkspaceScopedResponse,
     WorkspaceScopedResponseBody,
     WorkspaceScopedResponseMetadata,
     WorkspaceScopedResponseResources,
+    WorkspaceScopedTaggableFilter,
 )
+from zenml.models.v2.core.tag import TagResponse
 
 if TYPE_CHECKING:
     from zenml.models.v2.core.pipeline_run import PipelineRunResponse
+    from zenml.zen_stores.schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
 
+AnyQuery = TypeVar("AnyQuery", bound=Any)
 
 # ------------------ Request Model ------------------
 
@@ -45,20 +53,15 @@ class PipelineRequest(WorkspaceScopedRequest):
         title="The name of the pipeline.",
         max_length=STR_FIELD_MAX_LENGTH,
     )
-    version: str = Field(
-        title="The version of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
-    )
-    version_hash: str = Field(
-        title="The version hash of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
-    )
-    docstring: Optional[str] = Field(
-        title="The docstring of the pipeline.",
+    description: Optional[str] = Field(
+        default=None,
+        title="The description of the pipeline.",
         max_length=TEXT_FIELD_MAX_LENGTH,
+    )
+    tags: Optional[List[str]] = Field(
         default=None,
+        title="Tags of the pipeline.",
     )
-    spec: PipelineSpec = Field(title="The spec of the pipeline.")
 
 
 # ------------------ Update Model ------------------
@@ -67,29 +70,16 @@ class PipelineRequest(WorkspaceScopedRequest):
 class PipelineUpdate(BaseUpdate):
     """Update model for pipelines."""
 
-    name: Optional[str] = Field(
-        title="The name of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
-        default=None,
-    )
-    version: Optional[str] = Field(
-        title="The version of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
+    description: Optional[str] = Field(
         default=None,
-    )
-    version_hash: Optional[str] = Field(
-        title="The version hash of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
-        default=None,
-    )
-    docstring: Optional[str] = Field(
-        title="The docstring of the pipeline.",
+        title="The description of the pipeline.",
         max_length=TEXT_FIELD_MAX_LENGTH,
-        default=None,
     )
-    spec: Optional[PipelineSpec] = Field(
-        title="The spec of the pipeline.",
-        default=None,
+    add_tags: Optional[List[str]] = Field(
+        default=None, title="New tags to add to the pipeline."
+    )
+    remove_tags: Optional[List[str]] = Field(
+        default=None, title="Tags to remove from the pipeline."
     )
 
 
@@ -99,33 +89,32 @@ class PipelineUpdate(BaseUpdate):
 class PipelineResponseBody(WorkspaceScopedResponseBody):
     """Response body for pipelines."""
 
-    status: Optional[List[ExecutionStatus]] = Field(
-        default=None, title="The status of the last 3 Pipeline Runs."
+    latest_run_id: Optional[UUID] = Field(
+        default=None,
+        title="The ID of the latest run of the pipeline.",
     )
-    version: str = Field(
-        title="The version of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
+    latest_run_status: Optional[ExecutionStatus] = Field(
+        default=None,
+        title="The status of the latest run of the pipeline.",
     )
 
 
 class PipelineResponseMetadata(WorkspaceScopedResponseMetadata):
     """Response metadata for pipelines."""
 
-    version_hash: str = Field(
-        title="The version hash of the pipeline.",
-        max_length=STR_FIELD_MAX_LENGTH,
-    )
-    spec: PipelineSpec = Field(title="The spec of the pipeline.")
-    docstring: Optional[str] = Field(
-        title="The docstring of the pipeline.",
-        max_length=TEXT_FIELD_MAX_LENGTH,
+    description: Optional[str] = Field(
         default=None,
+        title="The description of the pipeline.",
     )
 
 
 class PipelineResponseResources(WorkspaceScopedResponseResources):
     """Class for all resource models associated with the pipeline entity."""
 
+    tags: List[TagResponse] = Field(
+        title="Tags associated with the pipeline.",
+    )
+
 
 class PipelineResponse(
     WorkspaceScopedResponse[
@@ -224,75 +213,46 @@ class PipelineResponse(
             )
         return runs[0]
 
-    # Body and metadata properties
-    @property
-    def status(self) -> Optional[List[ExecutionStatus]]:
-        """The `status` property.
-
-        Returns:
-            the value of the property.
-        """
-        return self.get_body().status
-
-    @property
-    def version(self) -> str:
-        """The `version` property.
-
-        Returns:
-            the value of the property.
-        """
-        return self.get_body().version
-
     @property
-    def spec(self) -> PipelineSpec:
-        """The `spec` property.
+    def latest_run_id(self) -> Optional[UUID]:
+        """The `latest_run_id` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().spec
+        return self.get_body().latest_run_id
 
     @property
-    def version_hash(self) -> str:
-        """The `version_hash` property.
+    def latest_run_status(self) -> Optional[ExecutionStatus]:
+        """The `latest_run_status` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().version_hash
+        return self.get_body().latest_run_status
 
     @property
-    def docstring(self) -> Optional[str]:
-        """The `docstring` property.
+    def tags(self) -> List[TagResponse]:
+        """The `tags` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().docstring
+        return self.get_resources().tags
 
 
 # ------------------ Filter Model ------------------
 
 
-class PipelineFilter(WorkspaceScopedFilter):
+class PipelineFilter(WorkspaceScopedTaggableFilter):
     """Pipeline filter model."""
 
+    CUSTOM_SORTING_OPTIONS = [SORT_PIPELINES_BY_LATEST_RUN_KEY]
+
     name: Optional[str] = Field(
         default=None,
         description="Name of the Pipeline",
     )
-    version: Optional[str] = Field(
-        default=None,
-        description="Version of the Pipeline",
-    )
-    version_hash: Optional[str] = Field(
-        default=None,
-        description="Version hash of the Pipeline",
-    )
-    docstring: Optional[str] = Field(
-        default=None,
-        description="Docstring of the Pipeline",
-    )
     workspace_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Workspace of the Pipeline",
@@ -303,3 +263,27 @@ class PipelineFilter(WorkspaceScopedFilter):
         description="User of the Pipeline",
         union_mode="left_to_right",
     )
+
+    def apply_sorting(
+        self,
+        query: AnyQuery,
+        table: Type["AnySchema"],
+    ) -> AnyQuery:
+        """Apply sorting to the query.
+
+        Args:
+            query: The query to which to apply the sorting.
+            table: The query table.
+
+        Returns:
+            The query with sorting applied.
+        """
+        column, _ = self.sorting_params
+
+        if column == SORT_PIPELINES_BY_LATEST_RUN_KEY:
+            # If sorting by the latest run, the sorting is already done in the
+            # base query in `SqlZenStore.list_pipelines(...)` and we don't need
+            # to to anything here
+            return query
+        else:
+            return super().apply_sorting(query=query, table=table)

zenml/models/v2/core/pipeline_build.py

@@ -157,6 +157,9 @@ class PipelineBuildRequest(PipelineBuildBase, WorkspaceScopedRequest):
     """Request model for pipelines builds."""
 
     checksum: Optional[str] = Field(title="The build checksum.", default=None)
+    stack_checksum: Optional[str] = Field(
+        title="The stack checksum.", default=None
+    )
 
     stack: Optional[UUID] = Field(
         title="The stack that was used for this build.", default=None
@@ -164,7 +167,6 @@ class PipelineBuildRequest(PipelineBuildBase, WorkspaceScopedRequest):
     pipeline: Optional[UUID] = Field(
         title="The pipeline that was used for this build.", default=None
     )
-    template_deployment_id: Optional[UUID] = None
 
 
 # ------------------ Update Model ------------------
@@ -196,6 +198,9 @@ class PipelineBuildResponseMetadata(WorkspaceScopedResponseMetadata):
         default=None, title="The Python version used for this build."
     )
     checksum: Optional[str] = Field(default=None, title="The build checksum.")
+    stack_checksum: Optional[str] = Field(
+        default=None, title="The stack checksum."
+    )
     is_local: bool = Field(
         title="Whether the build images are stored in a container "
         "registry or locally.",
@@ -203,7 +208,6 @@ class PipelineBuildResponseMetadata(WorkspaceScopedResponseMetadata):
     contains_code: bool = Field(
         title="Whether any image of the build contains user code.",
     )
-    template_deployment_id: Optional[UUID] = None
 
 
 class PipelineBuildResponseResources(WorkspaceScopedResponseResources):
@@ -409,31 +413,31 @@ class PipelineBuildResponse(
         return self.get_metadata().checksum
 
     @property
-    def is_local(self) -> bool:
-        """The `is_local` property.
+    def stack_checksum(self) -> Optional[str]:
+        """The `stack_checksum` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().is_local
+        return self.get_metadata().stack_checksum
 
     @property
-    def contains_code(self) -> bool:
-        """The `contains_code` property.
+    def is_local(self) -> bool:
+        """The `is_local` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().contains_code
+        return self.get_metadata().is_local
 
     @property
-    def template_deployment_id(self) -> Optional[UUID]:
-        """The `template_deployment_id` property.
+    def contains_code(self) -> bool:
+        """The `contains_code` property.
 
         Returns:
             the value of the property.
         """
-        return self.get_metadata().template_deployment_id
+        return self.get_metadata().contains_code
 
 
 # ------------------ Filter Model ------------------

zenml/models/v2/core/pipeline_deployment.py

@@ -18,8 +18,8 @@ from uuid import UUID
 
 from pydantic import Field
 
-from zenml.config.docker_settings import SourceFileMode
 from zenml.config.pipeline_configurations import PipelineConfiguration
+from zenml.config.pipeline_spec import PipelineSpec
 from zenml.config.step_configurations import Step
 from zenml.models.v2.base.base import BaseZenModel
 from zenml.models.v2.base.page import Page
@@ -71,28 +71,24 @@ class PipelineDeploymentBase(BaseZenModel):
         default=None,
         title="The version of the ZenML installation on the server side.",
     )
+    pipeline_version_hash: Optional[str] = Field(
+        default=None,
+        title="The pipeline version hash of the deployment.",
+    )
+    pipeline_spec: Optional[PipelineSpec] = Field(
+        default=None,
+        title="The pipeline spec of the deployment.",
+    )
 
     @property
-    def requires_included_files(self) -> bool:
-        """Whether the deployment requires included files.
-
-        Returns:
-            Whether the deployment requires included files.
-        """
-        return any(
-            step.config.docker_settings.source_files == SourceFileMode.INCLUDE
-            for step in self.step_configurations.values()
-        )
-
-    @property
-    def requires_code_download(self) -> bool:
-        """Whether the deployment requires downloading some code files.
+    def should_prevent_build_reuse(self) -> bool:
+        """Whether the deployment prevents a build reuse.
 
         Returns:
-            Whether the deployment requires downloading some code files.
+            Whether the deployment prevents a build reuse.
         """
         return any(
-            step.config.docker_settings.source_files == SourceFileMode.DOWNLOAD
+            step.config.docker_settings.prevent_build_reuse
             for step in self.step_configurations.values()
         )
 
@@ -116,6 +112,14 @@ class PipelineDeploymentRequest(
         default=None,
         title="The code reference associated with the deployment.",
     )
+    code_path: Optional[str] = Field(
+        default=None,
+        title="Optional path where the code is stored in the artifact store.",
+    )
+    template: Optional[UUID] = Field(
+        default=None,
+        description="Template used for the deployment.",
+    )
 
 
 # ------------------ Update Model ------------------
@@ -150,6 +154,17 @@ class PipelineDeploymentResponseMetadata(WorkspaceScopedResponseMetadata):
     server_version: Optional[str] = Field(
         title="The version of the ZenML installation on the server side."
     )
+    pipeline_version_hash: Optional[str] = Field(
+        default=None, title="The pipeline version hash of the deployment."
+    )
+    pipeline_spec: Optional[PipelineSpec] = Field(
+        default=None, title="The pipeline spec of the deployment."
+    )
+    code_path: Optional[str] = Field(
+        default=None,
+        title="Optional path where the code is stored in the artifact store.",
+    )
+
     pipeline: Optional[PipelineResponse] = Field(
         default=None, title="The pipeline associated with the deployment."
     )
@@ -167,6 +182,10 @@ class PipelineDeploymentResponseMetadata(WorkspaceScopedResponseMetadata):
         default=None,
         title="The code reference associated with the deployment.",
     )
+    template_id: Optional[UUID] = Field(
+        default=None,
+        description="Template used for the pipeline run.",
+    )
 
 
 class PipelineDeploymentResponseResources(WorkspaceScopedResponseResources):
@@ -251,6 +270,33 @@ class PipelineDeploymentResponse(
         """
         return self.get_metadata().server_version
 
+    @property
+    def pipeline_version_hash(self) -> Optional[str]:
+        """The `pipeline_version_hash` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().pipeline_version_hash
+
+    @property
+    def pipeline_spec(self) -> Optional[PipelineSpec]:
+        """The `pipeline_spec` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().pipeline_spec
+
+    @property
+    def code_path(self) -> Optional[str]:
+        """The `code_path` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().code_path
+
     @property
     def pipeline(self) -> Optional[PipelineResponse]:
         """The `pipeline` property.
@@ -297,16 +343,13 @@ class PipelineDeploymentResponse(
         return self.get_metadata().code_reference
 
     @property
-    def requires_code_download(self) -> bool:
-        """Whether the deployment requires downloading some code files.
+    def template_id(self) -> Optional[UUID]:
+        """The `template_id` property.
 
         Returns:
-            Whether the deployment requires downloading some code files.
+            the value of the property.
         """
-        return any(
-            step.config.docker_settings.source_files == SourceFileMode.DOWNLOAD
-            for step in self.step_configurations.values()
-        )
+        return self.get_metadata().template_id
 
 
 # ------------------ Filter Model ------------------
@@ -345,3 +388,8 @@ class PipelineDeploymentFilter(WorkspaceScopedFilter):
         description="Schedule associated with the deployment.",
         union_mode="left_to_right",
     )
+    template_id: Optional[Union[UUID, str]] = Field(
+        default=None,
+        description="Template used as base for the deployment.",
+        union_mode="left_to_right",
+    )

zenml/models/v2/core/pipeline_run.py

@@ -37,8 +37,10 @@ from zenml.models.v2.base.scoped import (
     WorkspaceScopedResponseBody,
     WorkspaceScopedResponseMetadata,
     WorkspaceScopedResponseResources,
+    WorkspaceScopedTaggableFilter,
 )
 from zenml.models.v2.core.model_version import ModelVersionResponse
+from zenml.models.v2.core.tag import TagResponse
 
 if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
@@ -109,6 +111,10 @@ class PipelineRunRequest(WorkspaceScopedRequest):
         default=None,
         title="ID of the trigger execution that triggered this run.",
     )
+    tags: Optional[List[str]] = Field(
+        default=None,
+        title="Tags of the pipeline run.",
+    )
 
 
 # ------------------ Update Model ------------------
@@ -119,6 +125,13 @@ class PipelineRunUpdate(BaseModel):
 
     status: Optional[ExecutionStatus] = None
     end_time: Optional[datetime] = None
+    # TODO: we should maybe have a different update model here, the upper two attributes should only be for internal use
+    add_tags: Optional[List[str]] = Field(
+        default=None, title="New tags to add to the pipeline run."
+    )
+    remove_tags: Optional[List[str]] = Field(
+        default=None, title="Tags to remove from the pipeline run."
+    )
 
 
 # ------------------ Response Model ------------------
@@ -193,12 +206,23 @@ class PipelineRunResponseMetadata(WorkspaceScopedResponseMetadata):
         max_length=STR_FIELD_MAX_LENGTH,
         default=None,
     )
+    code_path: Optional[str] = Field(
+        default=None,
+        title="Optional path where the code is stored in the artifact store.",
+    )
+    template_id: Optional[UUID] = Field(
+        default=None,
+        description="Template used for the pipeline run.",
+    )
 
 
 class PipelineRunResponseResources(WorkspaceScopedResponseResources):
     """Class for all resource models associated with the pipeline run entity."""
 
     model_version: Optional[ModelVersionResponse] = None
+    tags: List[TagResponse] = Field(
+        title="Tags associated with the pipeline run.",
+    )
 
     # TODO: In Pydantic v2, the `model_` is a protected namespaces for all
     #  fields defined under base models. If not handled, this raises a warning.
@@ -405,6 +429,24 @@ class PipelineRunResponse(
         """
         return self.get_metadata().orchestrator_run_id
 
+    @property
+    def code_path(self) -> Optional[str]:
+        """The `code_path` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().code_path
+
+    @property
+    def template_id(self) -> Optional[UUID]:
+        """The `template_id` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().template_id
+
     @property
     def model_version(self) -> Optional[ModelVersionResponse]:
         """The `model_version` property.
@@ -414,11 +456,20 @@ class PipelineRunResponse(
         """
         return self.get_resources().model_version
 
+    @property
+    def tags(self) -> List[TagResponse]:
+        """The `tags` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_resources().tags
+
 
 # ------------------ Filter Model ------------------
 
 
-class PipelineRunFilter(WorkspaceScopedFilter):
+class PipelineRunFilter(WorkspaceScopedTaggableFilter):
     """Model to enable advanced filtering of all Workspaces."""
 
     FILTER_EXCLUDE_FIELDS: ClassVar[List[str]] = [
@@ -428,6 +479,7 @@ class PipelineRunFilter(WorkspaceScopedFilter):
         "build_id",
         "schedule_id",
         "stack_id",
+        "template_id",
         "pipeline_name",
     ]
     name: Optional[str] = Field(
@@ -482,6 +534,11 @@ class PipelineRunFilter(WorkspaceScopedFilter):
         description="Code repository used for the Pipeline Run",
         union_mode="left_to_right",
     )
+    template_id: Optional[Union[UUID, str]] = Field(
+        default=None,
+        description="Template used for the pipeline run.",
+        union_mode="left_to_right",
+    )
     status: Optional[str] = Field(
         default=None,
         description="Name of the Pipeline Run",
@@ -574,4 +631,11 @@ class PipelineRunFilter(WorkspaceScopedFilter):
             )
             custom_filters.append(pipeline_build_filter)
 
+        if self.template_id:
+            run_template_filter = and_(
+                PipelineRunSchema.deployment_id == PipelineDeploymentSchema.id,
+                PipelineDeploymentSchema.template_id == self.template_id,
+            )
+            custom_filters.append(run_template_filter)
+
         return custom_filters