zenml-nightly 0.70.0.dev20241125__py3-none-any.whl → 0.71.0.dev20241220__py3-none-any.whl

zenml/models/v2/core/pipeline_run.py

@@ -16,10 +16,13 @@
 from datetime import datetime
 from typing import (
     TYPE_CHECKING,
+    Any,
     ClassVar,
     Dict,
     List,
     Optional,
+    Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -55,6 +58,11 @@ if TYPE_CHECKING:
     from zenml.models.v2.core.schedule import ScheduleResponse
     from zenml.models.v2.core.stack import StackResponse
     from zenml.models.v2.core.step_run import StepRunResponse
+    from zenml.zen_stores.schemas.base_schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
+
+AnyQuery = TypeVar("AnyQuery", bound=Any)
 
 
 # ------------------ Request Model ------------------
@@ -237,6 +245,10 @@ class PipelineRunResponseMetadata(WorkspaceScopedResponseMetadata):
         default=False,
         description="Whether a template can be created from this run.",
     )
+    step_substitutions: Dict[str, Dict[str, str]] = Field(
+        title="Substitutions used in the step runs of this pipeline run.",
+        default_factory=dict,
+    )
 
 
 class PipelineRunResponseResources(WorkspaceScopedResponseResources):
@@ -546,6 +558,15 @@ class PipelineRunResponse(
         """
         return self.get_metadata().is_templatable
 
+    @property
+    def step_substitutions(self) -> Dict[str, Dict[str, str]]:
+        """The `step_substitutions` property.
+
+        Returns:
+            the value of the property.
+        """
+        return self.get_metadata().step_substitutions
+
     @property
     def model_version(self) -> Optional[ModelVersionResponse]:
         """The `model_version` property.
@@ -571,6 +592,15 @@ class PipelineRunResponse(
 class PipelineRunFilter(WorkspaceScopedTaggableFilter):
     """Model to enable advanced filtering of all Workspaces."""
 
+    CUSTOM_SORTING_OPTIONS: ClassVar[List[str]] = [
+        *WorkspaceScopedTaggableFilter.CUSTOM_SORTING_OPTIONS,
+        "tag",
+        "stack",
+        "pipeline",
+        "model",
+        "model_version",
+    ]
+
     FILTER_EXCLUDE_FIELDS: ClassVar[List[str]] = [
         *WorkspaceScopedTaggableFilter.FILTER_EXCLUDE_FIELDS,
         "unlisted",
@@ -579,7 +609,6 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
         "schedule_id",
         "stack_id",
         "template_id",
-        "user",
         "pipeline",
         "stack",
         "code_repository",
@@ -602,16 +631,6 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
         description="Pipeline associated with the Pipeline Run",
         union_mode="left_to_right",
     )
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace of the Pipeline Run",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User that created the Pipeline Run",
-        union_mode="left_to_right",
-    )
     stack_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Stack used for the Pipeline Run",
@@ -662,16 +681,12 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
         union_mode="left_to_right",
     )
     unlisted: Optional[bool] = None
-    user: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Name/ID of the user that created the run.",
-    )
     run_metadata: Optional[Dict[str, str]] = Field(
         default=None,
         description="The run_metadata to filter the pipeline runs by.",
     )
     # TODO: Remove once frontend is ready for it. This is replaced by the more
-    # generic `pipeline` filter below.
+    #   generic `pipeline` filter below.
     pipeline_name: Optional[str] = Field(
         default=None,
         description="Name of the pipeline associated with the run",
@@ -703,13 +718,17 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
 
     def get_custom_filters(
         self,
+        table: Type["AnySchema"],
     ) -> List["ColumnElement[bool]"]:
         """Get custom filters.
 
+        Args:
+            table: The query table.
+
         Returns:
             A list of custom filters.
         """
-        custom_filters = super().get_custom_filters()
+        custom_filters = super().get_custom_filters(table)
 
         from sqlmodel import and_, col, or_
 
@@ -722,12 +741,12 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
             PipelineDeploymentSchema,
             PipelineRunSchema,
             PipelineSchema,
+            RunMetadataResourceSchema,
             RunMetadataSchema,
             ScheduleSchema,
             StackComponentSchema,
             StackCompositionSchema,
             StackSchema,
-            UserSchema,
         )
 
         if self.unlisted is not None:
@@ -778,17 +797,6 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
             )
             custom_filters.append(run_template_filter)
 
-        if self.user:
-            user_filter = and_(
-                PipelineRunSchema.user_id == UserSchema.id,
-                self.generate_name_or_id_query_conditions(
-                    value=self.user,
-                    table=UserSchema,
-                    additional_columns=["full_name"],
-                ),
-            )
-            custom_filters.append(user_filter)
-
         if self.pipeline:
             pipeline_filter = and_(
                 PipelineRunSchema.pipeline_id == PipelineSchema.id,
@@ -897,10 +905,12 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
 
             for key, value in self.run_metadata.items():
                 additional_filter = and_(
-                    RunMetadataSchema.resource_id == PipelineRunSchema.id,
-                    RunMetadataSchema.resource_type
+                    RunMetadataResourceSchema.resource_id
+                    == PipelineRunSchema.id,
+                    RunMetadataResourceSchema.resource_type
                     == MetadataResourceTypes.PIPELINE_RUN,
-                    RunMetadataSchema.key == key,
+                    RunMetadataResourceSchema.run_metadata_id
+                    == RunMetadataSchema.id,
                     self.generate_custom_query_conditions_for_column(
                         value=value,
                         table=RunMetadataSchema,
@@ -910,3 +920,71 @@ class PipelineRunFilter(WorkspaceScopedTaggableFilter):
                 custom_filters.append(additional_filter)
 
         return custom_filters
+
+    def apply_sorting(
+        self,
+        query: AnyQuery,
+        table: Type["AnySchema"],
+    ) -> AnyQuery:
+        """Apply sorting to the query.
+
+        Args:
+            query: The query to which to apply the sorting.
+            table: The query table.
+
+        Returns:
+            The query with sorting applied.
+        """
+        from sqlmodel import asc, desc
+
+        from zenml.enums import SorterOps
+        from zenml.zen_stores.schemas import (
+            ModelSchema,
+            ModelVersionSchema,
+            PipelineDeploymentSchema,
+            PipelineRunSchema,
+            PipelineSchema,
+            StackSchema,
+        )
+
+        sort_by, operand = self.sorting_params
+
+        if sort_by == "pipeline":
+            query = query.join(
+                PipelineSchema,
+                PipelineRunSchema.pipeline_id == PipelineSchema.id,
+            )
+            column = PipelineSchema.name
+        elif sort_by == "stack":
+            query = query.join(
+                PipelineDeploymentSchema,
+                PipelineRunSchema.deployment_id == PipelineDeploymentSchema.id,
+            ).join(
+                StackSchema,
+                PipelineDeploymentSchema.stack_id == StackSchema.id,
+            )
+            column = StackSchema.name
+        elif sort_by == "model":
+            query = query.join(
+                ModelVersionSchema,
+                PipelineRunSchema.model_version_id == ModelVersionSchema.id,
+            ).join(
+                ModelSchema,
+                ModelVersionSchema.model_id == ModelSchema.id,
+            )
+            column = ModelSchema.name
+        elif sort_by == "model_version":
+            query = query.join(
+                ModelVersionSchema,
+                PipelineRunSchema.model_version_id == ModelVersionSchema.id,
+            )
+            column = ModelVersionSchema.name
+        else:
+            return super().apply_sorting(query=query, table=table)
+
+        if operand == SorterOps.ASCENDING:
+            query = query.order_by(asc(column))
+        else:
+            query = query.order_by(desc(column))
+
+        return query

zenml/models/v2/core/run_metadata.py

@@ -13,16 +13,16 @@
 #  permissions and limitations under the License.
 """Models representing run metadata."""
 
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 from uuid import UUID
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
-from zenml.enums import MetadataResourceTypes
 from zenml.metadata.metadata_types import MetadataType, MetadataTypeEnum
 from zenml.models.v2.base.scoped import (
     WorkspaceScopedRequest,
 )
+from zenml.models.v2.misc.run_metadata import RunMetadataResource
 
 # ------------------ Request Model ------------------
 
@@ -30,14 +30,12 @@ from zenml.models.v2.base.scoped import (
 class RunMetadataRequest(WorkspaceScopedRequest):
     """Request model for run metadata."""
 
-    resource_id: UUID = Field(
-        title="The ID of the resource that this metadata belongs to.",
-    )
-    resource_type: MetadataResourceTypes = Field(
-        title="The type of the resource that this metadata belongs to.",
+    resources: List[RunMetadataResource] = Field(
+        title="The list of resources that this metadata belongs to."
     )
     stack_component_id: Optional[UUID] = Field(
-        title="The ID of the stack component that this metadata belongs to."
+        title="The ID of the stack component that this metadata belongs to.",
+        default=None,
     )
     values: Dict[str, "MetadataType"] = Field(
         title="The metadata to be created.",
@@ -45,3 +43,26 @@ class RunMetadataRequest(WorkspaceScopedRequest):
     types: Dict[str, "MetadataTypeEnum"] = Field(
         title="The types of the metadata to be created.",
     )
+    publisher_step_id: Optional[UUID] = Field(
+        title="The ID of the step execution that published this metadata.",
+        default=None,
+    )
+
+    @model_validator(mode="after")
+    def validate_values_keys(self) -> "RunMetadataRequest":
+        """Validates if the keys in the metadata are properly defined.
+
+        Returns:
+            self
+
+        Raises:
+            ValueError: if one of the key in the metadata contains `:`
+        """
+        invalid_keys = [key for key in self.values.keys() if ":" in key]
+        if invalid_keys:
+            raise ValueError(
+                "You can not use colons (`:`) in the key names when you "
+                "are creating metadata for your ZenML objects. Please change "
+                f"the following keys: {invalid_keys}"
+            )
+        return self

zenml/models/v2/core/run_template.py

@@ -13,7 +13,17 @@
 #  permissions and limitations under the License.
 """Models representing pipeline templates."""
 
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+)
 from uuid import UUID
 
 from pydantic import Field
@@ -45,6 +55,11 @@ from zenml.models.v2.core.tag import TagResponse
 if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
 
+    from zenml.zen_stores.schemas.base_schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
+
+
 # ------------------ Request Model ------------------
 
 
@@ -310,16 +325,6 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
         default=None,
         description="Name of the run template.",
     )
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace associated with the template.",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User that created the template.",
-        union_mode="left_to_right",
-    )
     pipeline_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Pipeline associated with the template.",
@@ -340,10 +345,6 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
         description="Code repository associated with the template.",
         union_mode="left_to_right",
     )
-    user: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Name/ID of the user that created the template.",
-    )
     pipeline: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Name/ID of the pipeline associated with the template.",
@@ -354,14 +355,17 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
     )
 
     def get_custom_filters(
-        self,
+        self, table: Type["AnySchema"]
     ) -> List["ColumnElement[bool]"]:
         """Get custom filters.
 
+        Args:
+            table: The query table.
+
         Returns:
             A list of custom filters.
         """
-        custom_filters = super().get_custom_filters()
+        custom_filters = super().get_custom_filters(table)
 
         from sqlmodel import and_
 
@@ -371,7 +375,6 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
             PipelineSchema,
             RunTemplateSchema,
             StackSchema,
-            UserSchema,
         )
 
         if self.code_repository_id:
@@ -409,17 +412,6 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
             )
             custom_filters.append(pipeline_filter)
 
-        if self.user:
-            user_filter = and_(
-                RunTemplateSchema.user_id == UserSchema.id,
-                self.generate_name_or_id_query_conditions(
-                    value=self.user,
-                    table=UserSchema,
-                    additional_columns=["full_name"],
-                ),
-            )
-            custom_filters.append(user_filter)
-
         if self.pipeline:
             pipeline_filter = and_(
                 RunTemplateSchema.source_deployment_id

zenml/models/v2/core/schedule.py

@@ -279,16 +279,6 @@ class ScheduleResponse(
 class ScheduleFilter(WorkspaceScopedFilter):
     """Model to enable advanced filtering of all Users."""
 
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace scope of the schedule.",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User that created the schedule",
-        union_mode="left_to_right",
-    )
     pipeline_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Pipeline that the schedule is attached to.",

zenml/models/v2/core/secret.py

@@ -15,7 +15,6 @@
 
 from datetime import datetime
 from typing import Any, ClassVar, Dict, List, Optional, Union
-from uuid import UUID
 
 from pydantic import Field, SecretStr
 
@@ -253,25 +252,12 @@ class SecretFilter(WorkspaceScopedFilter):
         default=None,
         description="Name of the secret",
     )
-
     scope: Optional[Union[SecretScope, str]] = Field(
         default=None,
         description="Scope in which to filter secrets",
         union_mode="left_to_right",
     )
 
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace of the Secret",
-        union_mode="left_to_right",
-    )
-
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User that created the Secret",
-        union_mode="left_to_right",
-    )
-
     @staticmethod
     def _get_filtering_value(value: Optional[Any]) -> str:
         """Convert the value to a string that can be used for lexicographical filtering and sorting.

zenml/models/v2/core/service.py

@@ -15,19 +15,20 @@
 
 from datetime import datetime
 from typing import (
+    TYPE_CHECKING,
     Any,
     ClassVar,
     Dict,
     List,
     Optional,
     Type,
+    TypeVar,
     Union,
 )
 from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field
 from sqlalchemy.sql.elements import ColumnElement
-from sqlmodel import SQLModel
 
 from zenml.constants import STR_FIELD_MAX_LENGTH
 from zenml.models.v2.base.scoped import (
@@ -37,11 +38,15 @@ from zenml.models.v2.base.scoped import (
     WorkspaceScopedResponseBody,
     WorkspaceScopedResponseMetadata,
     WorkspaceScopedResponseResources,
-    WorkspaceScopedTaggableFilter,
 )
 from zenml.services.service_status import ServiceState
 from zenml.services.service_type import ServiceType
 
+if TYPE_CHECKING:
+    from zenml.zen_stores.schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
+
 # ------------------ Request Model ------------------
 
 
@@ -376,16 +381,6 @@ class ServiceFilter(WorkspaceScopedFilter):
         description="Name of the service. Use this to filter services by "
         "their name.",
     )
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace of the service",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User of the service",
-        union_mode="left_to_right",
-    )
     type: Optional[str] = Field(
         default=None,
         description="Type of the service. Filter services by their type.",
@@ -457,9 +452,7 @@ class ServiceFilter(WorkspaceScopedFilter):
         "config",
     ]
     CLI_EXCLUDE_FIELDS: ClassVar[List[str]] = [
-        *WorkspaceScopedTaggableFilter.CLI_EXCLUDE_FIELDS,
-        "workspace_id",
-        "user_id",
+        *WorkspaceScopedFilter.CLI_EXCLUDE_FIELDS,
         "flavor",
         "type",
         "pipeline_step_name",
@@ -468,7 +461,7 @@ class ServiceFilter(WorkspaceScopedFilter):
     ]
 
     def generate_filter(
-        self, table: Type["SQLModel"]
+        self, table: Type["AnySchema"]
     ) -> Union["ColumnElement[bool]"]:
         """Generate the filter for the query.
 

zenml/models/v2/core/service_connector.py

@@ -801,7 +801,6 @@ class ServiceConnectorFilter(WorkspaceScopedFilter):
         default=None,
         description="The type to scope this query to.",
     )
-
     name: Optional[str] = Field(
         default=None,
         description="The name to filter by",
@@ -810,16 +809,6 @@ class ServiceConnectorFilter(WorkspaceScopedFilter):
         default=None,
         description="The type of service connector to filter by",
     )
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace to filter by",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User to filter by",
-        union_mode="left_to_right",
-    )
     auth_method: Optional[str] = Field(
         default=None,
         title="Filter by the authentication method configured for the "

zenml/models/v2/core/stack.py

@@ -14,7 +14,17 @@
 """Models representing stacks."""
 
 import json
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+)
 from uuid import UUID
 
 from pydantic import Field, model_validator
@@ -39,6 +49,9 @@ if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
 
     from zenml.models.v2.core.component import ComponentResponse
+    from zenml.zen_stores.schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
 
 
 # ------------------ Request Model ------------------
@@ -323,7 +336,6 @@ class StackFilter(WorkspaceScopedFilter):
     FILTER_EXCLUDE_FIELDS: ClassVar[List[str]] = [
         *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
         "component_id",
-        "user",
         "component",
     ]
 
@@ -334,42 +346,32 @@ class StackFilter(WorkspaceScopedFilter):
     description: Optional[str] = Field(
         default=None, description="Description of the stack"
     )
-    workspace_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Workspace of the stack",
-        union_mode="left_to_right",
-    )
-    user_id: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="User of the stack",
-        union_mode="left_to_right",
-    )
     component_id: Optional[Union[UUID, str]] = Field(
         default=None,
         description="Component in the stack",
         union_mode="left_to_right",
     )
-    user: Optional[Union[UUID, str]] = Field(
-        default=None,
-        description="Name/ID of the user that created the stack.",
-    )
     component: Optional[Union[UUID, str]] = Field(
         default=None, description="Name/ID of a component in the stack."
     )
 
-    def get_custom_filters(self) -> List["ColumnElement[bool]"]:
+    def get_custom_filters(
+        self, table: Type["AnySchema"]
+    ) -> List["ColumnElement[bool]"]:
         """Get custom filters.
 
+        Args:
+            table: The query table.
+
         Returns:
             A list of custom filters.
         """
-        custom_filters = super().get_custom_filters()
+        custom_filters = super().get_custom_filters(table)
 
         from zenml.zen_stores.schemas import (
             StackComponentSchema,
             StackCompositionSchema,
             StackSchema,
-            UserSchema,
         )
 
         if self.component_id:
@@ -379,17 +381,6 @@ class StackFilter(WorkspaceScopedFilter):
             )
             custom_filters.append(component_id_filter)
 
-        if self.user:
-            user_filter = and_(
-                StackSchema.user_id == UserSchema.id,
-                self.generate_name_or_id_query_conditions(
-                    value=self.user,
-                    table=UserSchema,
-                    additional_columns=["full_name"],
-                ),
-            )
-            custom_filters.append(user_filter)
-
         if self.component:
             component_filter = and_(
                 StackCompositionSchema.stack_id == StackSchema.id,