zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl

zenml/zen_stores/schemas/pipeline_deployment_schemas.py

@@ -14,15 +14,15 @@
 """SQLModel implementation of pipeline deployment tables."""
 
 import json
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional
 from uuid import UUID
 
-from pydantic.json import pydantic_encoder
 from sqlalchemy import TEXT, Column, String
 from sqlalchemy.dialects.mysql import MEDIUMTEXT
 from sqlmodel import Field, Relationship
 
 from zenml.config.pipeline_configurations import PipelineConfiguration
+from zenml.config.pipeline_spec import PipelineSpec
 from zenml.config.step_configurations import Step
 from zenml.constants import MEDIUMTEXT_MAX_LENGTH
 from zenml.models import (
@@ -31,6 +31,7 @@ from zenml.models import (
     PipelineDeploymentResponseBody,
     PipelineDeploymentResponseMetadata,
 )
+from zenml.utils.json_utils import pydantic_encoder
 from zenml.zen_stores.schemas.base_schemas import BaseSchema
 from zenml.zen_stores.schemas.code_repository_schemas import (
     CodeReferenceSchema,
@@ -74,6 +75,16 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
     run_name_template: str = Field(nullable=False)
     client_version: str = Field(nullable=True)
     server_version: str = Field(nullable=True)
+    pipeline_version_hash: Optional[str] = Field(nullable=True, default=None)
+    pipeline_spec: Optional[str] = Field(
+        sa_column=Column(
+            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
+                MEDIUMTEXT, "mysql"
+            ),
+            nullable=True,
+        )
+    )
+    code_path: Optional[str] = Field(nullable=True)
 
     # Foreign keys
     user_id: Optional[UUID] = build_foreign_key_field(
@@ -132,14 +143,21 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
         ondelete="SET NULL",
         nullable=True,
     )
+    # This is not a foreign key to remove a cycle which messes with our DB
+    # backup process
+    template_id: Optional[UUID] = None
 
     # SQLModel Relationships
     user: Optional["UserSchema"] = Relationship()
     workspace: "WorkspaceSchema" = Relationship()
-    stack: "StackSchema" = Relationship()
-    pipeline: "PipelineSchema" = Relationship()
+    stack: Optional["StackSchema"] = Relationship()
+    pipeline: Optional["PipelineSchema"] = Relationship()
     schedule: Optional["ScheduleSchema"] = Relationship()
-    build: Optional["PipelineBuildSchema"] = Relationship()
+    build: Optional["PipelineBuildSchema"] = Relationship(
+        sa_relationship_kwargs={
+            "foreign_keys": "[PipelineDeploymentSchema.build_id]"
+        }
+    )
     code_reference: Optional["CodeReferenceSchema"] = Relationship()
 
     pipeline_runs: List["PipelineRunSchema"] = Relationship(
@@ -172,9 +190,10 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
             build_id=request.build,
             user_id=request.user,
             schedule_id=request.schedule,
+            template_id=request.template,
             code_reference_id=code_reference_id,
             run_name_template=request.run_name_template,
-            pipeline_configuration=request.pipeline_configuration.json(),
+            pipeline_configuration=request.pipeline_configuration.model_dump_json(),
             step_configurations=json.dumps(
                 request.step_configurations,
                 sort_keys=False,
@@ -183,32 +202,46 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
             client_environment=json.dumps(request.client_environment),
             client_version=request.client_version,
             server_version=request.server_version,
+            pipeline_version_hash=request.pipeline_version_hash,
+            pipeline_spec=json.dumps(
+                request.pipeline_spec.model_dump(mode="json"), sort_keys=True
+            )
+            if request.pipeline_spec
+            else None,
+            code_path=request.code_path,
         )
 
-    def to_model(self, hydrate: bool = False) -> PipelineDeploymentResponse:
+    def to_model(
+        self,
+        include_metadata: bool = False,
+        include_resources: bool = False,
+        **kwargs: Any,
+    ) -> PipelineDeploymentResponse:
         """Convert a `PipelineDeploymentSchema` to a `PipelineDeploymentResponse`.
 
         Args:
-            hydrate: bool to decide whether to return a hydrated version of the
-                model.
+            include_metadata: Whether the metadata will be filled.
+            include_resources: Whether the resources will be filled.
+            **kwargs: Keyword arguments to allow schema specific logic
+
 
         Returns:
             The created `PipelineDeploymentResponse`.
         """
-        pipeline_configuration = PipelineConfiguration.parse_raw(
-            self.pipeline_configuration
-        )
-        step_configurations = json.loads(self.step_configurations)
-        for s, c in step_configurations.items():
-            step_configurations[s] = Step.parse_obj(c)
-
         body = PipelineDeploymentResponseBody(
             user=self.user.to_model() if self.user else None,
             created=self.created,
             updated=self.updated,
         )
         metadata = None
-        if hydrate:
+        if include_metadata:
+            pipeline_configuration = PipelineConfiguration.model_validate_json(
+                self.pipeline_configuration
+            )
+            step_configurations = json.loads(self.step_configurations)
+            for s, c in step_configurations.items():
+                step_configurations[s] = Step.model_validate(c)
+
             metadata = PipelineDeploymentResponseMetadata(
                 workspace=self.workspace.to_model(),
                 run_name_template=self.run_name_template,
@@ -224,6 +257,14 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
                 code_reference=self.code_reference.to_model()
                 if self.code_reference
                 else None,
+                pipeline_version_hash=self.pipeline_version_hash,
+                pipeline_spec=PipelineSpec.model_validate_json(
+                    self.pipeline_spec
+                )
+                if self.pipeline_spec
+                else None,
+                code_path=self.code_path,
+                template_id=self.template_id,
             )
         return PipelineDeploymentResponse(
             id=self.id,

zenml/zen_stores/schemas/pipeline_run_schemas.py

@@ -15,22 +15,30 @@
 
 import json
 from datetime import datetime
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 from uuid import UUID
 
+from pydantic import ConfigDict
 from sqlalchemy import UniqueConstraint
 from sqlmodel import TEXT, Column, Field, Relationship
 
 from zenml.config.pipeline_configurations import PipelineConfiguration
-from zenml.enums import ExecutionStatus, MetadataResourceTypes
+from zenml.enums import (
+    ExecutionStatus,
+    MetadataResourceTypes,
+    TaggableResourceTypes,
+)
 from zenml.models import (
     PipelineRunRequest,
     PipelineRunResponse,
     PipelineRunResponseBody,
     PipelineRunResponseMetadata,
     PipelineRunUpdate,
+    RunMetadataEntry,
 )
+from zenml.models.v2.core.pipeline_run import PipelineRunResponseResources
 from zenml.zen_stores.schemas.base_schemas import NamedSchema
+from zenml.zen_stores.schemas.constants import MODEL_VERSION_TABLENAME
 from zenml.zen_stores.schemas.pipeline_build_schemas import PipelineBuildSchema
 from zenml.zen_stores.schemas.pipeline_deployment_schemas import (
     PipelineDeploymentSchema,
@@ -39,19 +47,24 @@ from zenml.zen_stores.schemas.pipeline_schemas import PipelineSchema
 from zenml.zen_stores.schemas.schedule_schema import ScheduleSchema
 from zenml.zen_stores.schemas.schema_utils import build_foreign_key_field
 from zenml.zen_stores.schemas.stack_schemas import StackSchema
+from zenml.zen_stores.schemas.trigger_schemas import TriggerExecutionSchema
 from zenml.zen_stores.schemas.user_schemas import UserSchema
+from zenml.zen_stores.schemas.utils import RunMetadataInterface
 from zenml.zen_stores.schemas.workspace_schemas import WorkspaceSchema
 
 if TYPE_CHECKING:
     from zenml.zen_stores.schemas.logs_schemas import LogsSchema
     from zenml.zen_stores.schemas.model_schemas import (
         ModelVersionPipelineRunSchema,
+        ModelVersionSchema,
     )
     from zenml.zen_stores.schemas.run_metadata_schemas import RunMetadataSchema
+    from zenml.zen_stores.schemas.service_schemas import ServiceSchema
     from zenml.zen_stores.schemas.step_run_schemas import StepRunSchema
+    from zenml.zen_stores.schemas.tag_schemas import TagSchema
 
 
-class PipelineRunSchema(NamedSchema, table=True):
+class PipelineRunSchema(NamedSchema, RunMetadataInterface, table=True):
     """SQL Model for pipeline runs."""
 
     __tablename__ = "pipeline_run"
@@ -61,13 +74,18 @@ class PipelineRunSchema(NamedSchema, table=True):
             "orchestrator_run_id",
             name="unique_orchestrator_run_id_for_deployment_id",
         ),
+        UniqueConstraint(
+            "name",
+            "workspace_id",
+            name="unique_run_name_in_workspace",
+        ),
     )
 
     # Fields
     orchestrator_run_id: Optional[str] = Field(nullable=True)
     start_time: Optional[datetime] = Field(nullable=True)
     end_time: Optional[datetime] = Field(nullable=True, default=None)
-    status: ExecutionStatus = Field(nullable=False)
+    status: str = Field(nullable=False)
     orchestrator_environment: Optional[str] = Field(
         sa_column=Column(TEXT, nullable=True)
     )
@@ -105,6 +123,14 @@ class PipelineRunSchema(NamedSchema, table=True):
         ondelete="SET NULL",
         nullable=True,
     )
+    model_version_id: UUID = build_foreign_key_field(
+        source=__tablename__,
+        target=MODEL_VERSION_TABLENAME,
+        source_column="model_version_id",
+        target_column="id",
+        ondelete="SET NULL",
+        nullable=True,
+    )
 
     # Relationships
     deployment: Optional["PipelineDeploymentSchema"] = Relationship(
@@ -113,10 +139,10 @@ class PipelineRunSchema(NamedSchema, table=True):
     workspace: "WorkspaceSchema" = Relationship(back_populates="runs")
     user: Optional["UserSchema"] = Relationship(back_populates="runs")
     run_metadata: List["RunMetadataSchema"] = Relationship(
-        back_populates="pipeline_run",
         sa_relationship_kwargs=dict(
-            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.PIPELINE_RUN.value}', foreign(RunMetadataSchema.resource_id)==PipelineRunSchema.id)",
-            cascade="delete",
+            secondary="run_metadata_resource",
+            primaryjoin=f"and_(foreign(RunMetadataResourceSchema.resource_type)=='{MetadataResourceTypes.PIPELINE_RUN.value}', foreign(RunMetadataResourceSchema.resource_id)==PipelineRunSchema.id)",
+            secondaryjoin="RunMetadataSchema.id==foreign(RunMetadataResourceSchema.run_metadata_id)",
             overlaps="run_metadata",
         ),
     )
@@ -133,6 +159,9 @@ class PipelineRunSchema(NamedSchema, table=True):
     step_runs: List["StepRunSchema"] = Relationship(
         sa_relationship_kwargs={"cascade": "delete"},
     )
+    model_version: "ModelVersionSchema" = Relationship(
+        back_populates="pipeline_runs",
+    )
 
     # Temporary fields and foreign keys to be deprecated
     pipeline_configuration: Optional[str] = Field(
@@ -166,11 +195,35 @@ class PipelineRunSchema(NamedSchema, table=True):
         ondelete="SET NULL",
         nullable=True,
     )
+    trigger_execution_id: Optional[UUID] = build_foreign_key_field(
+        source=__tablename__,
+        target=TriggerExecutionSchema.__tablename__,
+        source_column="trigger_execution_id",
+        target_column="id",
+        ondelete="SET NULL",
+        nullable=True,
+    )
 
     stack: Optional["StackSchema"] = Relationship()
     build: Optional["PipelineBuildSchema"] = Relationship()
     schedule: Optional["ScheduleSchema"] = Relationship()
     pipeline: Optional["PipelineSchema"] = Relationship(back_populates="runs")
+    trigger_execution: Optional["TriggerExecutionSchema"] = Relationship()
+
+    services: List["ServiceSchema"] = Relationship(
+        back_populates="pipeline_run",
+    )
+    tags: List["TagSchema"] = Relationship(
+        sa_relationship_kwargs=dict(
+            primaryjoin=f"and_(foreign(TagResourceSchema.resource_type)=='{TaggableResourceTypes.PIPELINE_RUN.value}', foreign(TagResourceSchema.resource_id)==PipelineRunSchema.id)",
+            secondary="tag_resource",
+            secondaryjoin="TagSchema.id == foreign(TagResourceSchema.tag_id)",
+            order_by="TagSchema.name",
+            overlaps="tags",
+        ),
+    )
+
+    model_config = ConfigDict(protected_namespaces=())  # type: ignore[assignment]
 
     @classmethod
     def from_request(
@@ -193,17 +246,44 @@ class PipelineRunSchema(NamedSchema, table=True):
             orchestrator_run_id=request.orchestrator_run_id,
             orchestrator_environment=orchestrator_environment,
             start_time=request.start_time,
-            status=request.status,
+            status=request.status.value,
             pipeline_id=request.pipeline,
             deployment_id=request.deployment,
+            trigger_execution_id=request.trigger_execution_id,
+            model_version_id=request.model_version_id,
         )
 
-    def to_model(self, hydrate: bool = False) -> "PipelineRunResponse":
+    def fetch_metadata_collection(self) -> Dict[str, List[RunMetadataEntry]]:
+        """Fetches all the metadata entries related to the pipeline run.
+
+        Returns:
+            a dictionary, where the key is the key of the metadata entry
+                and the values represent the list of entries with this key.
+        """
+        # Fetch the metadata related to this run
+        metadata_collection = super().fetch_metadata_collection()
+
+        # Fetch the metadata related to the steps of this run
+        for s in self.step_runs:
+            step_metadata = s.fetch_metadata_collection()
+            for k, v in step_metadata.items():
+                metadata_collection[f"{s.name}::{k}"] = v
+
+        return metadata_collection
+
+    def to_model(
+        self,
+        include_metadata: bool = False,
+        include_resources: bool = False,
+        **kwargs: Any,
+    ) -> "PipelineRunResponse":
         """Convert a `PipelineRunSchema` to a `PipelineRunResponse`.
 
         Args:
-            hydrate: bool to decide whether to return a hydrated version of the
-                model.
+            include_metadata: Whether the metadata will be filled.
+            include_resources: Whether the resources will be filled.
+            **kwargs: Keyword arguments to allow schema specific logic
+
 
         Returns:
             The created `PipelineRunResponse`.
@@ -211,21 +291,14 @@ class PipelineRunSchema(NamedSchema, table=True):
         Raises:
             RuntimeError: if the model creation fails.
         """
-        orchestrator_environment = (
-            json.loads(self.orchestrator_environment)
-            if self.orchestrator_environment
-            else {}
-        )
-
-        run_metadata = {
-            metadata_schema.key: metadata_schema.to_model()
-            for metadata_schema in self.run_metadata
-        }
-
         if self.deployment is not None:
-            deployment = self.deployment.to_model()
+            deployment = self.deployment.to_model(include_metadata=True)
 
             config = deployment.pipeline_configuration
+            new_substitutions = config._get_full_substitutions(self.start_time)
+            config = config.model_copy(
+                update={"substitutions": new_substitutions}
+            )
             client_environment = deployment.client_environment
 
             stack = deployment.stack
@@ -235,7 +308,7 @@ class PipelineRunSchema(NamedSchema, table=True):
             code_reference = deployment.code_reference
 
         elif self.pipeline_configuration is not None:
-            config = PipelineConfiguration.parse_raw(
+            config = PipelineConfiguration.model_validate_json(
                 self.pipeline_configuration
             )
             client_environment = (
@@ -259,22 +332,53 @@ class PipelineRunSchema(NamedSchema, table=True):
 
         body = PipelineRunResponseBody(
             user=self.user.to_model() if self.user else None,
-            status=self.status,
+            status=ExecutionStatus(self.status),
             stack=stack,
             pipeline=pipeline,
             build=build,
             schedule=schedule,
             code_reference=code_reference,
+            trigger_execution=(
+                self.trigger_execution.to_model()
+                if self.trigger_execution
+                else None
+            ),
             created=self.created,
             updated=self.updated,
+            deployment_id=self.deployment_id,
+            model_version_id=self.model_version_id,
         )
         metadata = None
-        if hydrate:
-            steps = {step.name: step.to_model() for step in self.step_runs}
-
+        if include_metadata:
+            is_templatable = False
+            if (
+                self.deployment
+                and self.deployment.build
+                and not self.deployment.build.is_local
+                and self.deployment.build.stack
+            ):
+                is_templatable = True
+
+            steps = {
+                step.name: step.to_model(include_metadata=True)
+                for step in self.step_runs
+            }
+
+            step_substitutions = {}
+            for step_name, step in steps.items():
+                step_substitutions[step_name] = step.config.substitutions
+                # We fetch the steps hydrated before, but want them unhydrated
+                # in the response -> We need to reset the metadata here
+                step.metadata = None
+
+            orchestrator_environment = (
+                json.loads(self.orchestrator_environment)
+                if self.orchestrator_environment
+                else {}
+            )
             metadata = PipelineRunResponseMetadata(
                 workspace=self.workspace.to_model(),
-                run_metadata=run_metadata,
+                run_metadata=self.fetch_metadata(),
                 config=config,
                 steps=steps,
                 start_time=self.start_time,
@@ -282,12 +386,33 @@ class PipelineRunSchema(NamedSchema, table=True):
                 client_environment=client_environment,
                 orchestrator_environment=orchestrator_environment,
                 orchestrator_run_id=self.orchestrator_run_id,
+                code_path=self.deployment.code_path
+                if self.deployment
+                else None,
+                template_id=self.deployment.template_id
+                if self.deployment
+                else None,
+                is_templatable=is_templatable,
+                step_substitutions=step_substitutions,
             )
+
+        resources = None
+        if include_resources:
+            model_version = None
+            if self.model_version:
+                model_version = self.model_version.to_model()
+
+            resources = PipelineRunResponseResources(
+                model_version=model_version,
+                tags=[tag.to_model() for tag in self.tags],
+            )
+
         return PipelineRunResponse(
             id=self.id,
             name=self.name,
             body=body,
             metadata=metadata,
+            resources=resources,
         )
 
     def update(self, run_update: "PipelineRunUpdate") -> "PipelineRunSchema":
@@ -300,8 +425,10 @@ class PipelineRunSchema(NamedSchema, table=True):
             The updated `PipelineRunSchema`.
         """
         if run_update.status:
-            self.status = run_update.status
+            self.status = run_update.status.value
             self.end_time = run_update.end_time
+        if run_update.model_version_id and self.model_version_id is None:
+            self.model_version_id = run_update.model_version_id
 
         self.updated = datetime.utcnow()
         return self
@@ -323,10 +450,7 @@ class PipelineRunSchema(NamedSchema, table=True):
         Returns:
             The updated `PipelineRunSchema`.
         """
-        if (
-            self.orchestrator_run_id
-            or self.status != ExecutionStatus.INITIALIZING
-        ):
+        if not self.is_placeholder_run():
             raise RuntimeError(
                 f"Unable to replace pipeline run {self.id} which is not a "
                 "placeholder run."
@@ -345,8 +469,19 @@ class PipelineRunSchema(NamedSchema, table=True):
 
         self.orchestrator_run_id = request.orchestrator_run_id
         self.orchestrator_environment = orchestrator_environment
-        self.status = request.status
+        self.status = request.status.value
 
         self.updated = datetime.utcnow()
 
         return self
+
+    def is_placeholder_run(self) -> bool:
+        """Whether the pipeline run is a placeholder run.
+
+        Returns:
+            Whether the pipeline run is a placeholder run.
+        """
+        return (
+            self.orchestrator_run_id is None
+            and self.status == ExecutionStatus.INITIALIZING
+        )

zenml/zen_stores/schemas/pipeline_schemas.py

@@ -14,20 +14,19 @@
 """SQL Model Implementations for Pipelines and Pipeline Runs."""
 
 from datetime import datetime
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional
 from uuid import UUID
 
-from sqlalchemy import TEXT, Column, String
-from sqlalchemy.dialects.mysql import MEDIUMTEXT
+from sqlalchemy import TEXT, Column, UniqueConstraint
 from sqlmodel import Field, Relationship
 
-from zenml.config.pipeline_spec import PipelineSpec
-from zenml.constants import MEDIUMTEXT_MAX_LENGTH
+from zenml.enums import TaggableResourceTypes
 from zenml.models import (
     PipelineRequest,
     PipelineResponse,
     PipelineResponseBody,
     PipelineResponseMetadata,
+    PipelineResponseResources,
     PipelineUpdate,
 )
 from zenml.zen_stores.schemas.base_schemas import NamedSchema
@@ -44,25 +43,22 @@ if TYPE_CHECKING:
     )
     from zenml.zen_stores.schemas.pipeline_run_schemas import PipelineRunSchema
     from zenml.zen_stores.schemas.schedule_schema import ScheduleSchema
+    from zenml.zen_stores.schemas.tag_schemas import TagSchema
 
 
 class PipelineSchema(NamedSchema, table=True):
     """SQL Model for pipelines."""
 
     __tablename__ = "pipeline"
-
-    # Fields
-    version: str
-    version_hash: str
-    docstring: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
-    spec: str = Field(
-        sa_column=Column(
-            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
-                MEDIUMTEXT, "mysql"
-            ),
-            nullable=False,
-        )
+    __table_args__ = (
+        UniqueConstraint(
+            "name",
+            "workspace_id",
+            name="unique_pipeline_name_in_workspace",
+        ),
     )
+    # Fields
+    description: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
 
     # Foreign keys
     workspace_id: UUID = build_foreign_key_field(
@@ -89,13 +85,25 @@ class PipelineSchema(NamedSchema, table=True):
     schedules: List["ScheduleSchema"] = Relationship(
         back_populates="pipeline",
     )
-    runs: List["PipelineRunSchema"] = Relationship(back_populates="pipeline")
+    runs: List["PipelineRunSchema"] = Relationship(
+        back_populates="pipeline",
+        sa_relationship_kwargs={"order_by": "PipelineRunSchema.created"},
+    )
     builds: List["PipelineBuildSchema"] = Relationship(
         back_populates="pipeline"
     )
     deployments: List["PipelineDeploymentSchema"] = Relationship(
         back_populates="pipeline",
     )
+    tags: List["TagSchema"] = Relationship(
+        sa_relationship_kwargs=dict(
+            primaryjoin=f"and_(foreign(TagResourceSchema.resource_type)=='{TaggableResourceTypes.PIPELINE.value}', foreign(TagResourceSchema.resource_id)==PipelineSchema.id)",
+            secondary="tag_resource",
+            secondaryjoin="TagSchema.id == foreign(TagResourceSchema.tag_id)",
+            order_by="TagSchema.name",
+            overlaps="tags",
+        ),
+    )
 
     @classmethod
     def from_request(
@@ -112,43 +120,51 @@ class PipelineSchema(NamedSchema, table=True):
         """
         return cls(
             name=pipeline_request.name,
-            version=pipeline_request.version,
-            version_hash=pipeline_request.version_hash,
+            description=pipeline_request.description,
             workspace_id=pipeline_request.workspace,
             user_id=pipeline_request.user,
-            docstring=pipeline_request.docstring,
-            spec=pipeline_request.spec.json(sort_keys=True),
         )
 
     def to_model(
         self,
-        last_x_runs: int = 3,
-        hydrate: bool = False,
+        include_metadata: bool = False,
+        include_resources: bool = False,
+        **kwargs: Any,
     ) -> "PipelineResponse":
         """Convert a `PipelineSchema` to a `PipelineResponse`.
 
         Args:
-            hydrate: bool to decide whether to return a hydrated version of the
-                model.
-            last_x_runs: How many runs to use for the execution status
+            include_metadata: Whether the metadata will be filled.
+            include_resources: Whether the resources will be filled.
+            **kwargs: Keyword arguments to allow schema specific logic
 
         Returns:
             The created PipelineResponse.
         """
         body = PipelineResponseBody(
             user=self.user.to_model() if self.user else None,
-            status=[run.status for run in self.runs[:last_x_runs]],
+            latest_run_id=self.runs[-1].id if self.runs else None,
+            latest_run_status=self.runs[-1].status if self.runs else None,
             created=self.created,
             updated=self.updated,
-            version=self.version,
         )
+
         metadata = None
-        if hydrate:
+        if include_metadata:
             metadata = PipelineResponseMetadata(
                 workspace=self.workspace.to_model(),
-                version_hash=self.version_hash,
-                spec=PipelineSpec.parse_raw(self.spec),
-                docstring=self.docstring,
+                description=self.description,
+            )
+
+        resources = None
+        if include_resources:
+            latest_run_user = self.runs[-1].user if self.runs else None
+
+            resources = PipelineResponseResources(
+                latest_run_user=latest_run_user.to_model()
+                if latest_run_user
+                else None,
+                tags=[tag.to_model() for tag in self.tags],
             )
 
         return PipelineResponse(
@@ -156,6 +172,7 @@ class PipelineSchema(NamedSchema, table=True):
             name=self.name,
             body=body,
             metadata=metadata,
+            resources=resources,
         )
 
     def update(self, pipeline_update: "PipelineUpdate") -> "PipelineSchema":
@@ -167,5 +184,6 @@ class PipelineSchema(NamedSchema, table=True):
         Returns:
             The updated `PipelineSchema`.
         """
+        self.description = pipeline_update.description
         self.updated = datetime.utcnow()
         return self