zenml-nightly 0.73.0.dev20250204__py3-none-any.whl → 0.73.0.dev20250206__py3-none-any.whl

zenml/models/v2/core/run_template.py

@@ -33,12 +33,13 @@ from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
 from zenml.enums import ExecutionStatus
 from zenml.models.v2.base.base import BaseUpdate
 from zenml.models.v2.base.scoped import (
+    TaggableFilter,
+    WorkspaceScopedFilter,
     WorkspaceScopedRequest,
     WorkspaceScopedResponse,
     WorkspaceScopedResponseBody,
     WorkspaceScopedResponseMetadata,
     WorkspaceScopedResponseResources,
-    WorkspaceScopedTaggableFilter,
 )
 from zenml.models.v2.core.code_reference import (
     CodeReferenceResponse,
@@ -307,11 +308,12 @@ class RunTemplateResponse(
 # ------------------ Filter Model ------------------
 
 
-class RunTemplateFilter(WorkspaceScopedTaggableFilter):
+class RunTemplateFilter(WorkspaceScopedFilter, TaggableFilter):
     """Model for filtering of run templates."""
 
     FILTER_EXCLUDE_FIELDS: ClassVar[List[str]] = [
-        *WorkspaceScopedTaggableFilter.FILTER_EXCLUDE_FIELDS,
+        *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
+        *TaggableFilter.FILTER_EXCLUDE_FIELDS,
         "code_repository_id",
         "stack_id",
         "build_id",
@@ -320,6 +322,14 @@ class RunTemplateFilter(WorkspaceScopedTaggableFilter):
         "pipeline",
         "stack",
     ]
+    CUSTOM_SORTING_OPTIONS = [
+        *WorkspaceScopedFilter.CUSTOM_SORTING_OPTIONS,
+        *TaggableFilter.CUSTOM_SORTING_OPTIONS,
+    ]
+    CLI_EXCLUDE_FIELDS = [
+        *WorkspaceScopedFilter.CLI_EXCLUDE_FIELDS,
+        *TaggableFilter.CLI_EXCLUDE_FIELDS,
+    ]
 
     name: Optional[str] = Field(
         default=None,

zenml/models/v2/core/step_run.py

@@ -16,6 +16,7 @@
 from datetime import datetime
 from typing import (
     TYPE_CHECKING,
+    Any,
     ClassVar,
     Dict,
     List,
@@ -574,7 +575,7 @@ class StepRunFilter(WorkspaceScopedFilter):
         default=None,
         description="Name/ID of the model associated with the step run.",
     )
-    run_metadata: Optional[Dict[str, str]] = Field(
+    run_metadata: Optional[Dict[str, Any]] = Field(
         default=None,
         description="The run_metadata to filter the step runs by.",
     )
@@ -619,13 +620,19 @@ class StepRunFilter(WorkspaceScopedFilter):
                 additional_filter = and_(
                     RunMetadataResourceSchema.resource_id == StepRunSchema.id,
                     RunMetadataResourceSchema.resource_type
-                    == MetadataResourceTypes.STEP_RUN,
+                    == MetadataResourceTypes.STEP_RUN.value,
                     RunMetadataResourceSchema.run_metadata_id
                     == RunMetadataSchema.id,
+                    self.generate_custom_query_conditions_for_column(
+                        value=key,
+                        table=RunMetadataSchema,
+                        column="key",
+                    ),
                     self.generate_custom_query_conditions_for_column(
                         value=value,
                         table=RunMetadataSchema,
                         column="value",
+                        json_encode_value=True,
                     ),
                 )
                 custom_filters.append(additional_filter)

zenml/pipelines/pipeline_definition.py

@@ -407,17 +407,31 @@ class Pipeline:
         return self
 
     @property
-    def requires_parameters(self) -> bool:
-        """If the pipeline entrypoint requires parameters.
+    def required_parameters(self) -> List[str]:
+        """List of required parameters for the pipeline entrypoint.
 
         Returns:
-            If the pipeline entrypoint requires parameters.
+            List of required parameters for the pipeline entrypoint.
         """
         signature = inspect.signature(self.entrypoint, follow_wrapped=True)
-        return any(
-            parameter.default is inspect.Parameter.empty
+        return [
+            parameter.name
             for parameter in signature.parameters.values()
-        )
+            if parameter.default is inspect.Parameter.empty
+        ]
+
+    @property
+    def missing_parameters(self) -> List[str]:
+        """List of missing parameters for the pipeline entrypoint.
+
+        Returns:
+            List of missing parameters for the pipeline entrypoint.
+        """
+        available_parameters = set(self.configuration.parameters or {})
+        if params_from_file := self._from_config_file.get("parameters", None):
+            available_parameters.update(params_from_file)
+
+        return list(set(self.required_parameters) - available_parameters)
 
     @property
     def is_prepared(self) -> bool:
@@ -1412,7 +1426,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
         except ValidationError as e:
             raise ValueError(
                 "Invalid or missing pipeline function entrypoint arguments. "
-                "Only JSON serializable inputs are allowed as pipeline inputs."
+                "Only JSON serializable inputs are allowed as pipeline inputs. "
                 "Check out the pydantic error above for more details."
             ) from e
 
@@ -1427,15 +1441,17 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 requires parameters.
         """
         if not self.is_prepared:
-            if self.requires_parameters:
+            if missing_parameters := self.missing_parameters:
                 raise RuntimeError(
                     f"Failed while trying to prepare pipeline {self.name}. "
                     "The entrypoint function of the pipeline requires "
-                    "arguments. Please prepare the pipeline by calling "
-                    "`pipeline_instance.prepare(...)` and try again."
+                    "arguments which have not been configured yet: "
+                    f"{missing_parameters}. Please provide those parameters by "
+                    "calling `pipeline_instance.configure(parameters=...)` or "
+                    "by calling `pipeline_instance.prepare(...)` and try again."
                 )
-            else:
-                self.prepare()
+
+            self.prepare()
 
     def create_run_template(
         self, name: str, **kwargs: Any

zenml/stack/stack.py

@@ -37,6 +37,7 @@ from zenml.config.global_config import GlobalConfiguration
 from zenml.constants import (
     ENV_ZENML_SECRET_VALIDATION_LEVEL,
     ENV_ZENML_SKIP_IMAGE_BUILDER_DEFAULT,
+    ENV_ZENML_SKIP_STACK_VALIDATION,
     handle_bool_env_var,
 )
 from zenml.enums import SecretValidationLevel, StackComponentType
@@ -706,6 +707,10 @@ class Stack:
                 if a secret for a component is missing. Otherwise, only a
                 warning will be logged.
         """
+        if handle_bool_env_var(ENV_ZENML_SKIP_STACK_VALIDATION, default=False):
+            logger.debug("Skipping stack validation.")
+            return
+
         self.validate_image_builder()
         for component in self.components.values():
             if component.validator:

zenml/zen_stores/schemas/artifact_schemas.py

@@ -18,7 +18,8 @@ from uuid import UUID
 
 from pydantic import ValidationError
 from sqlalchemy import TEXT, Column, UniqueConstraint
-from sqlmodel import Field, Relationship
+from sqlalchemy.orm import object_session
+from sqlmodel import Field, Relationship, desc, select
 
 from zenml.config.source import Source
 from zenml.enums import (
@@ -90,6 +91,32 @@ class ArtifactSchema(NamedSchema, table=True):
         ),
     )
 
+    @property
+    def latest_version(self) -> Optional["ArtifactVersionSchema"]:
+        """Fetch the latest version for this artifact.
+
+        Raises:
+            RuntimeError: If no session for the schema exists.
+
+        Returns:
+            The latest version for this artifact.
+        """
+        if session := object_session(self):
+            return (
+                session.execute(
+                    select(ArtifactVersionSchema)
+                    .where(ArtifactVersionSchema.artifact_id == self.id)
+                    .order_by(desc(ArtifactVersionSchema.created))
+                    .limit(1)
+                )
+                .scalars()
+                .one_or_none()
+            )
+        else:
+            raise RuntimeError(
+                "Missing DB session to fetch latest version for artifact."
+            )
+
     @classmethod
     def from_request(
         cls,
@@ -127,9 +154,9 @@ class ArtifactSchema(NamedSchema, table=True):
             The created `ArtifactResponse`.
         """
         latest_id, latest_name = None, None
-        if self.versions:
-            latest_version = max(self.versions, key=lambda x: x.created)
-            latest_id, latest_name = latest_version.id, latest_version.version
+        if latest_version := self.latest_version:
+            latest_id = latest_version.id
+            latest_name = latest_version.version
 
         # Create the body of the model
         body = ArtifactResponseBody(

zenml/zen_stores/schemas/model_schemas.py

@@ -24,7 +24,8 @@ from sqlalchemy import (
     Column,
     UniqueConstraint,
 )
-from sqlmodel import Field, Relationship
+from sqlalchemy.orm import object_session
+from sqlmodel import Field, Relationship, desc, select
 
 from zenml.enums import (
     ArtifactType,
@@ -126,6 +127,32 @@ class ModelSchema(NamedSchema, table=True):
         sa_relationship_kwargs={"cascade": "delete"},
     )
 
+    @property
+    def latest_version(self) -> Optional["ModelVersionSchema"]:
+        """Fetch the latest version for this model.
+
+        Raises:
+            RuntimeError: If no session for the schema exists.
+
+        Returns:
+            The latest version for this model.
+        """
+        if session := object_session(self):
+            return (
+                session.execute(
+                    select(ModelVersionSchema)
+                    .where(ModelVersionSchema.model_id == self.id)
+                    .order_by(desc(ModelVersionSchema.number))
+                    .limit(1)
+                )
+                .scalars()
+                .one_or_none()
+            )
+        else:
+            raise RuntimeError(
+                "Missing DB session to fetch latest version for model."
+            )
+
     @classmethod
     def from_request(cls, model_request: ModelRequest) -> "ModelSchema":
         """Convert an `ModelRequest` to an `ModelSchema`.
@@ -169,11 +196,9 @@ class ModelSchema(NamedSchema, table=True):
         """
         tags = [tag.to_model() for tag in self.tags]
 
-        if self.model_versions:
-            version_numbers = [mv.number for mv in self.model_versions]
-            latest_version_idx = version_numbers.index(max(version_numbers))
-            latest_version_name = self.model_versions[latest_version_idx].name
-            latest_version_id = self.model_versions[latest_version_idx].id
+        if latest_version := self.latest_version:
+            latest_version_name = latest_version.name
+            latest_version_id = latest_version.id
         else:
             latest_version_name = None
             latest_version_id = None

zenml/zen_stores/schemas/pipeline_run_schemas.py

@@ -208,7 +208,7 @@ class PipelineRunSchema(NamedSchema, RunMetadataInterface, table=True):
     stack: Optional["StackSchema"] = Relationship()
     build: Optional["PipelineBuildSchema"] = Relationship()
     schedule: Optional["ScheduleSchema"] = Relationship()
-    pipeline: Optional["PipelineSchema"] = Relationship(back_populates="runs")
+    pipeline: Optional["PipelineSchema"] = Relationship()
     trigger_execution: Optional["TriggerExecutionSchema"] = Relationship()
 
     services: List["ServiceSchema"] = Relationship(

zenml/zen_stores/schemas/pipeline_schemas.py

@@ -17,7 +17,8 @@ from typing import TYPE_CHECKING, Any, List, Optional
 from uuid import UUID
 
 from sqlalchemy import TEXT, Column, UniqueConstraint
-from sqlmodel import Field, Relationship
+from sqlalchemy.orm import object_session
+from sqlmodel import Field, Relationship, desc, select
 
 from zenml.enums import TaggableResourceTypes
 from zenml.models import (
@@ -85,10 +86,6 @@ class PipelineSchema(NamedSchema, table=True):
     schedules: List["ScheduleSchema"] = Relationship(
         back_populates="pipeline",
     )
-    runs: List["PipelineRunSchema"] = Relationship(
-        back_populates="pipeline",
-        sa_relationship_kwargs={"order_by": "PipelineRunSchema.created"},
-    )
     builds: List["PipelineBuildSchema"] = Relationship(
         back_populates="pipeline"
     )
@@ -105,6 +102,34 @@ class PipelineSchema(NamedSchema, table=True):
         ),
     )
 
+    @property
+    def latest_run(self) -> Optional["PipelineRunSchema"]:
+        """Fetch the latest run for this pipeline.
+
+        Raises:
+            RuntimeError: If no session for the schema exists.
+
+        Returns:
+            The latest run for this pipeline.
+        """
+        from zenml.zen_stores.schemas import PipelineRunSchema
+
+        if session := object_session(self):
+            return (
+                session.execute(
+                    select(PipelineRunSchema)
+                    .where(PipelineRunSchema.pipeline_id == self.id)
+                    .order_by(desc(PipelineRunSchema.created))
+                    .limit(1)
+                )
+                .scalars()
+                .one_or_none()
+            )
+        else:
+            raise RuntimeError(
+                "Missing DB session to fetch latest run for pipeline."
+            )
+
     @classmethod
     def from_request(
         cls,
@@ -141,10 +166,12 @@ class PipelineSchema(NamedSchema, table=True):
         Returns:
             The created PipelineResponse.
         """
+        latest_run = self.latest_run
+
         body = PipelineResponseBody(
             user=self.user.to_model() if self.user else None,
-            latest_run_id=self.runs[-1].id if self.runs else None,
-            latest_run_status=self.runs[-1].status if self.runs else None,
+            latest_run_id=latest_run.id if latest_run else None,
+            latest_run_status=latest_run.status if latest_run else None,
             created=self.created,
             updated=self.updated,
         )
@@ -158,7 +185,7 @@ class PipelineSchema(NamedSchema, table=True):
 
         resources = None
         if include_resources:
-            latest_run_user = self.runs[-1].user if self.runs else None
+            latest_run_user = latest_run.user if latest_run else None
 
             resources = PipelineResponseResources(
                 latest_run_user=latest_run_user.to_model()

zenml/zen_stores/schemas/run_template_schemas.py

@@ -18,7 +18,8 @@ from uuid import UUID
 
 from sqlalchemy import Column, String, UniqueConstraint
 from sqlalchemy.dialects.mysql import MEDIUMTEXT
-from sqlmodel import Field, Relationship
+from sqlalchemy.orm import object_session
+from sqlmodel import Field, Relationship, col, desc, select
 
 from zenml.constants import MEDIUMTEXT_MAX_LENGTH
 from zenml.enums import TaggableResourceTypes
@@ -99,17 +100,6 @@ class RunTemplateSchema(BaseSchema, table=True):
         }
     )
 
-    runs: List["PipelineRunSchema"] = Relationship(
-        sa_relationship_kwargs={
-            "primaryjoin": "RunTemplateSchema.id==PipelineDeploymentSchema.template_id",
-            "secondaryjoin": "PipelineDeploymentSchema.id==PipelineRunSchema.deployment_id",
-            "secondary": "pipeline_deployment",
-            "cascade": "delete",
-            "viewonly": True,
-            "order_by": "PipelineRunSchema.created",
-        }
-    )
-
     tags: List["TagSchema"] = Relationship(
         sa_relationship_kwargs=dict(
             primaryjoin=f"and_(foreign(TagResourceSchema.resource_type)=='{TaggableResourceTypes.RUN_TEMPLATE.value}', foreign(TagResourceSchema.resource_id)==RunTemplateSchema.id)",
@@ -120,6 +110,42 @@ class RunTemplateSchema(BaseSchema, table=True):
         ),
     )
 
+    @property
+    def latest_run(self) -> Optional["PipelineRunSchema"]:
+        """Fetch the latest run for this template.
+
+        Raises:
+            RuntimeError: If no session for the schema exists.
+
+        Returns:
+            The latest run for this template.
+        """
+        from zenml.zen_stores.schemas import (
+            PipelineDeploymentSchema,
+            PipelineRunSchema,
+        )
+
+        if session := object_session(self):
+            return (
+                session.execute(
+                    select(PipelineRunSchema)
+                    .join(
+                        PipelineDeploymentSchema,
+                        col(PipelineDeploymentSchema.id)
+                        == col(PipelineRunSchema.deployment_id),
+                    )
+                    .where(PipelineDeploymentSchema.template_id == self.id)
+                    .order_by(desc(PipelineRunSchema.created))
+                    .limit(1)
+                )
+                .scalars()
+                .one_or_none()
+            )
+        else:
+            raise RuntimeError(
+                "Missing DB session to fetch latest run for template."
+            )
+
     @classmethod
     def from_request(
         cls,
@@ -184,13 +210,15 @@ class RunTemplateSchema(BaseSchema, table=True):
         ):
             runnable = True
 
+        latest_run = self.latest_run
+
         body = RunTemplateResponseBody(
             user=self.user.to_model() if self.user else None,
             created=self.created,
             updated=self.updated,
             runnable=runnable,
-            latest_run_id=self.runs[-1].id if self.runs else None,
-            latest_run_status=self.runs[-1].status if self.runs else None,
+            latest_run_id=latest_run.id if latest_run else None,
+            latest_run_status=latest_run.status if latest_run else None,
         )
 
         metadata = None

zenml/zen_stores/sql_zen_store.py

@@ -303,7 +303,6 @@ from zenml.utils.enum_utils import StrEnum
 from zenml.utils.networking_utils import (
     replace_localhost_with_internal_hostname,
 )
-from zenml.utils.pydantic_utils import before_validator_handler
 from zenml.utils.secret_utils import PlainSerializedSecretStr
 from zenml.utils.string_utils import (
     format_name_template,
@@ -434,6 +433,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
             be created automatically on first access.
         username: The database username.
         password: The database password.
+        ssl: Whether to use SSL.
         ssl_ca: certificate authority certificate. Required for SSL
             enabled authentication if the CA certificate is not part of the
             certificates shipped by the operating system.
@@ -463,6 +463,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
     database: Optional[str] = None
     username: Optional[PlainSerializedSecretStr] = None
     password: Optional[PlainSerializedSecretStr] = None
+    ssl: bool = False
     ssl_ca: Optional[PlainSerializedSecretStr] = None
     ssl_cert: Optional[PlainSerializedSecretStr] = None
     ssl_key: Optional[PlainSerializedSecretStr] = None
@@ -499,35 +500,6 @@ class SqlZenStoreConfiguration(StoreConfiguration):
 
         return secrets_store
 
-    @model_validator(mode="before")
-    @classmethod
-    @before_validator_handler
-    def _remove_grpc_attributes(cls, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Removes old GRPC attributes.
-
-        Args:
-            data: All model attribute values.
-
-        Returns:
-            The model attribute values
-        """
-        grpc_attribute_keys = [
-            "grpc_metadata_host",
-            "grpc_metadata_port",
-            "grpc_metadata_ssl_ca",
-            "grpc_metadata_ssl_key",
-            "grpc_metadata_ssl_cert",
-        ]
-        grpc_values = [data.pop(key, None) for key in grpc_attribute_keys]
-        if any(grpc_values):
-            logger.warning(
-                "The GRPC attributes %s are unused and will be removed soon. "
-                "Please remove them from SQLZenStore configuration. This will "
-                "become an error in future versions of ZenML."
-            )
-
-        return data
-
     @model_validator(mode="after")
     def _validate_backup_strategy(self) -> "SqlZenStoreConfiguration":
         """Validate the backup strategy.
@@ -641,15 +613,21 @@ class SqlZenStoreConfiguration(StoreConfiguration):
                         return None
 
                 for k, v in sql_url.query.items():
-                    if k == "ssl_ca":
+                    if k == "ssl":
+                        if r := _get_query_result(v):
+                            self.ssl = is_true_string_value(r)
+                    elif k == "ssl_ca":
                         if r := _get_query_result(v):
                             self.ssl_ca = PlainSerializedSecretStr(r)
+                            self.ssl = True
                     elif k == "ssl_cert":
                         if r := _get_query_result(v):
                             self.ssl_cert = PlainSerializedSecretStr(r)
+                            self.ssl = True
                     elif k == "ssl_key":
                         if r := _get_query_result(v):
                             self.ssl_key = PlainSerializedSecretStr(r)
+                            self.ssl = True
                     elif k == "ssl_verify_server_cert":
                         if r := _get_query_result(v):
                             if is_true_string_value(r):
@@ -659,7 +637,7 @@ class SqlZenStoreConfiguration(StoreConfiguration):
                     else:
                         raise ValueError(
                             "Invalid MySQL URL query parameter `%s`: The "
-                            "parameter must be one of: ssl_ca, ssl_cert, "
+                            "parameter must be one of: ssl, ssl_ca, ssl_cert, "
                             "ssl_key, or ssl_verify_server_cert.",
                             k,
                         )
@@ -728,15 +706,6 @@ class SqlZenStoreConfiguration(StoreConfiguration):
         """
         return make_url(url).drivername in SQLDatabaseDriver.values()
 
-    def expand_certificates(self) -> None:
-        """Expands the certificates in the verify_ssl field."""
-        # Load the certificate values back into the configuration
-        for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
-            file_path = getattr(self, key, None)
-            if file_path and os.path.isfile(file_path.get_secret_value()):
-                with open(file_path, "r") as f:
-                    setattr(self, key, f.read())
-
     def get_sqlalchemy_config(
         self,
         database: Optional[str] = None,
@@ -789,22 +758,19 @@ class SqlZenStoreConfiguration(StoreConfiguration):
             sqlalchemy_ssl_args: Dict[str, Any] = {}
 
             # Handle SSL params
-            for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
-                ssl_setting = getattr(self, key)
-                if not ssl_setting:
-                    continue
-                if not os.path.isfile(ssl_setting.get_secret_value()):
-                    logger.warning(
-                        f"Database SSL setting `{key}` is not a file. "
+            if self.ssl:
+                sqlalchemy_ssl_args["ssl"] = True
+                for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
+                    ssl_setting = getattr(self, key)
+                    if not ssl_setting:
+                        continue
+                    if not os.path.isfile(ssl_setting.get_secret_value()):
+                        logger.warning(
+                            f"Database SSL setting `{key}` is not a file. "
+                        )
+                    sqlalchemy_ssl_args[key.removeprefix("ssl_")] = (
+                        ssl_setting.get_secret_value()
                     )
-                sqlalchemy_ssl_args[key.lstrip("ssl_")] = (
-                    ssl_setting.get_secret_value()
-                )
-                sqlalchemy_ssl_args[key.removeprefix("ssl_")] = (
-                    ssl_setting.get_secret_value()
-                )
-
-            if len(sqlalchemy_ssl_args) > 0:
                 sqlalchemy_ssl_args["check_hostname"] = (
                     self.ssl_verify_server_cert
                 )

{zenml_nightly-0.73.0.dev20250204.dist-info → zenml_nightly-0.73.0.dev20250206.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.73.0.dev20250204
+Version: 0.73.0.dev20250206
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops