zenml-nightly 0.84.3.dev20250908__tar.gz → 0.84.3.dev20250909__tar.gz

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.84.3.dev20250908
+Version: 0.84.3.dev20250909
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zenml-nightly"
-version = "0.84.3.dev20250908"
+version = "0.84.3.dev20250909"
 packages = [{ include = "zenml", from = "src" }]
 description = "ZenML: Write production-ready ML code."
 authors = ["ZenML GmbH <info@zenml.io>"]

zenml_nightly-0.84.3.dev20250909/src/zenml/VERSION

@@ -0,0 +1 @@
+0.84.3.dev20250909

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/cli/utils.py

@@ -2251,7 +2251,7 @@ def get_execution_status_emoji(status: "ExecutionStatus") -> str:
     """
     from zenml.enums import ExecutionStatus
 
-    if status == ExecutionStatus.INITIALIZING:
+    if status in {ExecutionStatus.INITIALIZING, ExecutionStatus.PROVISIONING}:
         return ":hourglass_flowing_sand:"
     if status == ExecutionStatus.FAILED:
         return ":x:"

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/config/compiler.py

@@ -116,7 +116,7 @@ class Compiler:
             invocation_id: self._compile_step_invocation(
                 invocation=invocation,
                 stack=stack,
-                step_config=run_configuration.steps.get(invocation_id),
+                step_config=(run_configuration.steps or {}).get(invocation_id),
                 pipeline_configuration=pipeline.configuration,
             )
             for invocation_id, invocation in self._get_sorted_invocations(
@@ -211,7 +211,10 @@ class Compiler:
                 cache_policy=config.cache_policy,
             )
 
-        invalid_step_configs = set(config.steps) - set(pipeline.invocations)
+        configured_step_configs = config.steps or {}
+        invalid_step_configs = set(configured_step_configs) - set(
+            pipeline.invocations
+        )
         if invalid_step_configs:
             logger.warning(
                 f"Configuration for step invocations {invalid_step_configs} "
@@ -220,7 +223,7 @@ class Compiler:
             )
 
         for key in invalid_step_configs:
-            config.steps.pop(key)
+            configured_step_configs.pop(key)
 
         # Override `enable_cache` of all steps if set at run level
         if config.enable_cache is not None:
@@ -480,7 +483,7 @@ class Compiler:
         )
 
         parameters_to_ignore = (
-            set(step_config.parameters) if step_config else set()
+            set(step_config.parameters or {}) if step_config else set()
         )
         step_configuration_overrides = invocation.finalize(
             parameters_to_ignore=parameters_to_ignore

zenml_nightly-0.84.3.dev20250909/src/zenml/config/pipeline_run_configuration.py

@@ -0,0 +1,110 @@
+#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Pipeline run configuration class."""
+
+from typing import Any, Dict, List, Optional, Union
+from uuid import UUID
+
+from pydantic import Field, SerializeAsAny
+
+from zenml.config.base_settings import BaseSettings
+from zenml.config.cache_policy import CachePolicyWithValidator
+from zenml.config.retry_config import StepRetryConfig
+from zenml.config.schedule import Schedule
+from zenml.config.source import SourceWithValidator
+from zenml.config.step_configurations import StepConfigurationUpdate
+from zenml.config.strict_base_model import StrictBaseModel
+from zenml.model.model import Model
+from zenml.models import PipelineBuildBase
+from zenml.utils import pydantic_utils
+from zenml.utils.tag_utils import Tag
+
+
+class PipelineRunConfiguration(
+    StrictBaseModel, pydantic_utils.YAMLSerializationMixin
+):
+    """Class for pipeline run configurations."""
+
+    run_name: Optional[str] = Field(
+        default=None, description="The name of the pipeline run."
+    )
+    enable_cache: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable cache for all steps of the pipeline "
+        "run.",
+    )
+    enable_artifact_metadata: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable metadata for the output artifacts of "
+        "all steps of the pipeline run.",
+    )
+    enable_artifact_visualization: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable visualizations for the output "
+        "artifacts of all steps of the pipeline run.",
+    )
+    enable_step_logs: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable logs for all steps of the pipeline run.",
+    )
+    enable_pipeline_logs: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable pipeline logs for the pipeline run.",
+    )
+    schedule: Optional[Schedule] = Field(
+        default=None, description="The schedule on which to run the pipeline."
+    )
+    build: Union[PipelineBuildBase, UUID, None] = Field(
+        default=None,
+        union_mode="left_to_right",
+        description="The build to use for the pipeline run.",
+    )
+    steps: Optional[Dict[str, StepConfigurationUpdate]] = Field(
+        default=None,
+        description="Configurations for the steps of the pipeline run.",
+    )
+    settings: Optional[Dict[str, SerializeAsAny[BaseSettings]]] = Field(
+        default=None, description="Settings for the pipeline run."
+    )
+    tags: Optional[List[Union[str, Tag]]] = Field(
+        default=None, description="Tags to apply to the pipeline run."
+    )
+    extra: Optional[Dict[str, Any]] = Field(
+        default=None, description="Extra configurations for the pipeline run."
+    )
+    model: Optional[Model] = Field(
+        default=None, description="The model to use for the pipeline run."
+    )
+    parameters: Optional[Dict[str, Any]] = Field(
+        default=None, description="Parameters for the pipeline function."
+    )
+    retry: Optional[StepRetryConfig] = Field(
+        default=None,
+        description="The retry configuration for all steps of the pipeline run.",
+    )
+    failure_hook_source: Optional[SourceWithValidator] = Field(
+        default=None,
+        description="The failure hook source for all steps of the pipeline run.",
+    )
+    success_hook_source: Optional[SourceWithValidator] = Field(
+        default=None,
+        description="The success hook source for all steps of the pipeline run.",
+    )
+    substitutions: Optional[Dict[str, str]] = Field(
+        default=None, description="The substitutions for the pipeline run."
+    )
+    cache_policy: Optional[CachePolicyWithValidator] = Field(
+        default=None,
+        description="The cache policy for all steps of the pipeline run.",
+    )

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/config/step_configurations.py

@@ -26,6 +26,7 @@ from typing import (
 
 from pydantic import (
     ConfigDict,
+    Field,
     SerializeAsAny,
     field_validator,
     model_validator,
@@ -142,21 +143,68 @@ class ArtifactConfiguration(PartialArtifactConfiguration):
 class StepConfigurationUpdate(StrictBaseModel):
     """Class for step configuration updates."""
 
-    enable_cache: Optional[bool] = None
-    enable_artifact_metadata: Optional[bool] = None
-    enable_artifact_visualization: Optional[bool] = None
-    enable_step_logs: Optional[bool] = None
-    step_operator: Optional[Union[bool, str]] = None
-    experiment_tracker: Optional[Union[bool, str]] = None
-    parameters: Dict[str, Any] = {}
-    settings: Dict[str, SerializeAsAny[BaseSettings]] = {}
-    extra: Dict[str, Any] = {}
-    failure_hook_source: Optional[SourceWithValidator] = None
-    success_hook_source: Optional[SourceWithValidator] = None
-    model: Optional[Model] = None
-    retry: Optional[StepRetryConfig] = None
-    substitutions: Dict[str, str] = {}
-    cache_policy: Optional[CachePolicyWithValidator] = None
+    enable_cache: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable cache for the step.",
+    )
+    enable_artifact_metadata: Optional[bool] = Field(
+        default=None,
+        description="Whether to store metadata for the output artifacts of "
+        "the step.",
+    )
+    enable_artifact_visualization: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable visualizations for the output "
+        "artifacts of the step.",
+    )
+    enable_step_logs: Optional[bool] = Field(
+        default=None,
+        description="Whether to enable logs for the step.",
+    )
+    step_operator: Optional[Union[bool, str]] = Field(
+        default=None,
+        description="The step operator to use for the step.",
+    )
+    experiment_tracker: Optional[Union[bool, str]] = Field(
+        default=None,
+        description="The experiment tracker to use for the step.",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Parameters for the step function.",
+    )
+    settings: Optional[Dict[str, SerializeAsAny[BaseSettings]]] = Field(
+        default=None,
+        description="Settings for the step.",
+    )
+    extra: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Extra configurations for the step.",
+    )
+    failure_hook_source: Optional[SourceWithValidator] = Field(
+        default=None,
+        description="The failure hook source for the step.",
+    )
+    success_hook_source: Optional[SourceWithValidator] = Field(
+        default=None,
+        description="The success hook source for the step.",
+    )
+    model: Optional[Model] = Field(
+        default=None,
+        description="The model to use for the step.",
+    )
+    retry: Optional[StepRetryConfig] = Field(
+        default=None,
+        description="The retry configuration for the step.",
+    )
+    substitutions: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="The substitutions for the step.",
+    )
+    cache_policy: Optional[CachePolicyWithValidator] = Field(
+        default=None,
+        description="The cache policy for the step.",
+    )
 
     outputs: Mapping[str, PartialArtifactConfiguration] = {}
 
@@ -197,6 +245,10 @@ class PartialStepConfiguration(StepConfigurationUpdate):
     """Class representing a partial step configuration."""
 
     name: str
+    parameters: Dict[str, Any] = {}
+    settings: Dict[str, SerializeAsAny[BaseSettings]] = {}
+    extra: Dict[str, Any] = {}
+    substitutions: Dict[str, str] = {}
     caching_parameters: Mapping[str, Any] = {}
     external_input_artifacts: Mapping[str, ExternalArtifactConfiguration] = {}
     model_artifacts_or_metadata: Mapping[str, ModelVersionDataLazyLoader] = {}

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/enums.py

@@ -74,6 +74,7 @@ class ExecutionStatus(StrEnum):
     """Enum that represents the execution status of a step or pipeline run."""
 
     INITIALIZING = "initializing"
+    PROVISIONING = "provisioning"
     FAILED = "failed"
     COMPLETED = "completed"
     RUNNING = "running"

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/models/v2/core/pipeline_run.py

@@ -104,6 +104,11 @@ class PipelineRunRequest(ProjectScopedRequest):
     status: ExecutionStatus = Field(
         title="The status of the pipeline run.",
     )
+    status_reason: Optional[str] = Field(
+        title="The reason for the status of the pipeline run.",
+        default=None,
+        max_length=STR_FIELD_MAX_LENGTH,
+    )
     orchestrator_environment: Dict[str, Any] = Field(
         default={},
         title=(
@@ -131,7 +136,10 @@ class PipelineRunRequest(ProjectScopedRequest):
         Returns:
             Whether the request is a placeholder request.
         """
-        return self.status == ExecutionStatus.INITIALIZING
+        return self.status in {
+            ExecutionStatus.INITIALIZING,
+            ExecutionStatus.PROVISIONING,
+        }
 
     model_config = ConfigDict(protected_namespaces=())
 
@@ -143,6 +151,11 @@ class PipelineRunUpdate(BaseUpdate):
     """Pipeline run update model."""
 
     status: Optional[ExecutionStatus] = None
+    status_reason: Optional[str] = Field(
+        default=None,
+        title="The reason for the status of the pipeline run.",
+        max_length=STR_FIELD_MAX_LENGTH,
+    )
     end_time: Optional[datetime] = None
     orchestrator_run_id: Optional[str] = None
     # TODO: we should maybe have a different update model here, the upper
@@ -169,6 +182,10 @@ class PipelineRunResponseBody(ProjectScopedResponseBody):
     status: ExecutionStatus = Field(
         title="The status of the pipeline run.",
     )
+    status_reason: Optional[str] = Field(
+        default=None,
+        title="The reason for the status of the pipeline run.",
+    )
     stack: Optional["StackResponse"] = Field(
         default=None, title="The stack that was used for this run."
     )

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/orchestrators/base_orchestrator.py

@@ -316,6 +316,11 @@ class BaseOrchestrator(StackComponent, ABC):
                     environment=environment,
                     placeholder_run=placeholder_run,
                 )
+                if placeholder_run:
+                    publish_pipeline_run_status_update(
+                        pipeline_run_id=placeholder_run.id,
+                        status=ExecutionStatus.PROVISIONING,
+                    )
 
                 if submission_result:
                     if submission_result.metadata:
@@ -511,6 +516,7 @@ class BaseOrchestrator(StackComponent, ABC):
         publish_pipeline_run_status_update(
             pipeline_run_id=run.id,
             status=ExecutionStatus.STOPPING,
+            status_reason="Manual stop requested.",
         )
 
         # Now call the concrete implementation

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/orchestrators/publish_utils.py

@@ -138,6 +138,7 @@ def publish_failed_pipeline_run(
 def publish_pipeline_run_status_update(
     pipeline_run_id: "UUID",
     status: ExecutionStatus,
+    status_reason: Optional[str] = None,
     end_time: Optional[datetime] = None,
 ) -> "PipelineRunResponse":
     """Publishes a pipeline run status update.
@@ -145,6 +146,7 @@ def publish_pipeline_run_status_update(
     Args:
         pipeline_run_id: The ID of the pipeline run to update.
         status: The new status for the pipeline run.
+        status_reason: The reason for the status of the pipeline run.
         end_time: The end time for the pipeline run. If None, will be set to current time
             for finished statuses.
 
@@ -158,6 +160,7 @@ def publish_pipeline_run_status_update(
         run_id=pipeline_run_id,
         run_update=PipelineRunUpdate(
             status=status,
+            status_reason=status_reason,
             end_time=end_time,
         ),
     )

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/pipelines/run_utils.py

@@ -91,36 +91,6 @@ def create_placeholder_run(
     return run
 
 
-def get_placeholder_run(
-    deployment_id: UUID,
-) -> Optional["PipelineRunResponse"]:
-    """Get the placeholder run for a deployment.
-
-    Args:
-        deployment_id: ID of the deployment for which to get the placeholder
-            run.
-
-    Returns:
-        The placeholder run or `None` if there exists no placeholder run for the
-        deployment.
-    """
-    runs = Client().list_pipeline_runs(
-        sort_by="asc:created",
-        size=1,
-        deployment_id=deployment_id,
-        status=ExecutionStatus.INITIALIZING,
-        hydrate=True,
-    )
-    if len(runs.items) == 0:
-        return None
-
-    run = runs.items[0]
-    if run.orchestrator_run_id is None:
-        return run
-
-    return None
-
-
 def deploy_pipeline(
     deployment: "PipelineDeploymentResponse",
     stack: "Stack",
@@ -268,16 +238,18 @@ def validate_run_config_is_runnable_from_server(
             "Can't set schedule when running pipeline via Rest API."
         )
 
-    if run_configuration.settings.get("docker"):
+    if run_configuration.settings and run_configuration.settings.get("docker"):
         raise ValueError(
             "Can't set DockerSettings when running pipeline via Rest API."
         )
 
-    for step_update in run_configuration.steps.values():
-        if step_update.settings.get("docker"):
-            raise ValueError(
-                "Can't set DockerSettings when running pipeline via Rest API."
-            )
+    if run_configuration.steps:
+        for step_update in run_configuration.steps.values():
+            if step_update.settings and step_update.settings.get("docker"):
+                raise ValueError(
+                    "Can't set DockerSettings when running pipeline via "
+                    "Rest API."
+                )
 
 
 def upload_notebook_cell_code_if_necessary(

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/steps/base_step.py

@@ -862,7 +862,8 @@ class BaseStep:
             config: The configuration update to validate.
             runtime_parameters: Dictionary of parameters passed to a step from runtime
         """
-        settings_utils.validate_setting_keys(list(config.settings))
+        if config.settings:
+            settings_utils.validate_setting_keys(list(config.settings))
         self._validate_function_parameters(
             parameters=config.parameters, runtime_parameters=runtime_parameters
         )
@@ -870,7 +871,7 @@ class BaseStep:
 
     def _validate_function_parameters(
         self,
-        parameters: Dict[str, Any],
+        parameters: Optional[Dict[str, Any]],
         runtime_parameters: Dict[str, Any],
     ) -> None:
         """Validates step function parameters.

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/zen_server/template_execution/runner_entrypoint_configuration.py

@@ -13,16 +13,50 @@
 #  permissions and limitations under the License.
 """Runner entrypoint configuration."""
 
+from typing import Any, List, Set
+from uuid import UUID
+
 from zenml.client import Client
 from zenml.entrypoints.base_entrypoint_configuration import (
     BaseEntrypointConfiguration,
 )
-from zenml.pipelines.run_utils import deploy_pipeline, get_placeholder_run
+from zenml.pipelines.run_utils import deploy_pipeline
+
+PLACEHOLDER_RUN_ID_OPTION = "placeholder_run_id"
 
 
 class RunnerEntrypointConfiguration(BaseEntrypointConfiguration):
     """Runner entrypoint configuration."""
 
+    @classmethod
+    def get_entrypoint_options(cls) -> Set[str]:
+        """Gets all options required for running with this configuration.
+
+        Returns:
+            The superclass options as well as an option for the name of the
+            step to run.
+        """
+        return super().get_entrypoint_options() | {PLACEHOLDER_RUN_ID_OPTION}
+
+    @classmethod
+    def get_entrypoint_arguments(
+        cls,
+        **kwargs: Any,
+    ) -> List[str]:
+        """Gets all arguments that the entrypoint command should be called with.
+
+        Args:
+            **kwargs: Kwargs, must include the placeholder run id.
+
+        Returns:
+            The superclass arguments as well as arguments for the placeholder
+            run id.
+        """
+        return super().get_entrypoint_arguments(**kwargs) + [
+            f"--{PLACEHOLDER_RUN_ID_OPTION}",
+            str(kwargs[PLACEHOLDER_RUN_ID_OPTION]),
+        ]
+
     def run(self) -> None:
         """Run the entrypoint configuration.
 
@@ -30,11 +64,14 @@ class RunnerEntrypointConfiguration(BaseEntrypointConfiguration):
         to the entrypoint configuration.
         """
         deployment = self.load_deployment()
+        placeholder_run_id = UUID(
+            self.entrypoint_args[PLACEHOLDER_RUN_ID_OPTION]
+        )
+        placeholder_run = Client().get_pipeline_run(placeholder_run_id)
 
         stack = Client().active_stack
         assert deployment.stack and stack.id == deployment.stack.id
 
-        placeholder_run = get_placeholder_run(deployment_id=deployment.id)
         deploy_pipeline(
             deployment=deployment,
             stack=stack,

{zenml_nightly-0.84.3.dev20250908 → zenml_nightly-0.84.3.dev20250909}/src/zenml/zen_server/template_execution/utils.py

@@ -222,7 +222,8 @@ def run_template(
 
     command = RunnerEntrypointConfiguration.get_entrypoint_command()
     args = RunnerEntrypointConfiguration.get_entrypoint_arguments(
-        deployment_id=new_deployment.id
+        deployment_id=new_deployment.id,
+        placeholder_run_id=placeholder_run.id,
     )
 
     if build.python_version:
@@ -306,14 +307,20 @@ def run_template(
                     str(template.id),
                     str(placeholder_run.id),
                 )
-                zen_store().update_run(
-                    run_id=placeholder_run.id,
-                    run_update=PipelineRunUpdate(
-                        status=ExecutionStatus.FAILED,
-                        end_time=utc_now(),
-                    ),
-                )
-                raise
+                run_status, _ = zen_store().get_run_status(placeholder_run.id)
+                if run_status == ExecutionStatus.INITIALIZING:
+                    # The run isn't in the provisioning status yet, which means
+                    # the orchestrator wasn't able to submit the run. In this
+                    # case, we can update the run to failed.
+                    zen_store().update_run(
+                        run_id=placeholder_run.id,
+                        run_update=PipelineRunUpdate(
+                            status=ExecutionStatus.FAILED,
+                            status_reason="Failed to start run.",
+                            end_time=utc_now(),
+                        ),
+                    )
+                    raise
 
     if sync:
         _task_with_analytics_and_error_handling()
@@ -461,8 +468,9 @@ def deployment_request_from_template(
     )
 
     steps = {}
+    step_config_updates = config.steps or {}
     for invocation_id, step in deployment.step_configurations.items():
-        step_update = config.steps.get(
+        step_update = step_config_updates.get(
             invocation_id, StepConfigurationUpdate()
         ).model_dump(
             # Get rid of deprecated name to prevent overriding the step name

zenml_nightly-0.84.3.dev20250909/src/zenml/zen_stores/migrations/versions/83ef3cb746a5_add_status_reason.py

@@ -0,0 +1,41 @@
+"""Add status reason [83ef3cb746a5].
+
+Revision ID: 83ef3cb746a5
+Revises: aae4eed923b5
+Create Date: 2025-09-04 15:41:30.829628
+
+"""
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "83ef3cb746a5"
+down_revision = "aae4eed923b5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("pipeline_run", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "status_reason",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("pipeline_run", schema=None) as batch_op:
+        batch_op.drop_column("status_reason")
+
+    # ### end Alembic commands ###