zenml-nightly 0.83.1.dev20250627__py3-none-any.whl → 0.83.1.dev20250628__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.83.1.dev20250627
+0.83.1.dev20250628

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py

@@ -46,6 +46,9 @@ from kubernetes import client as k8s_client
 from kubernetes import config as k8s_config
 
 from zenml.config.base_settings import BaseSettings
+from zenml.constants import (
+    METADATA_ORCHESTRATOR_RUN_ID,
+)
 from zenml.enums import StackComponentType
 from zenml.integrations.kubernetes.flavors.kubernetes_orchestrator_flavor import (
     KubernetesOrchestratorConfig,
@@ -61,6 +64,7 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
 )
 from zenml.integrations.kubernetes.pod_settings import KubernetesPodSettings
 from zenml.logger import get_logger
+from zenml.metadata.metadata_types import MetadataType
 from zenml.orchestrators import ContainerizedOrchestrator, SubmissionResult
 from zenml.orchestrators.utils import get_orchestrator_run_name
 from zenml.stack import StackValidator
@@ -467,9 +471,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
         # This will internally also build the command/args for all step pods.
         command = KubernetesOrchestratorEntrypointConfiguration.get_entrypoint_command()
         args = KubernetesOrchestratorEntrypointConfiguration.get_entrypoint_arguments(
-            run_name=orchestrator_run_name,
             deployment_id=deployment.id,
-            kubernetes_namespace=self.config.kubernetes_namespace,
             run_id=placeholder_run.id if placeholder_run else None,
         )
 
@@ -508,6 +510,18 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 }
             )
 
+        orchestrator_pod_labels = {
+            "pipeline": kube_utils.sanitize_label(pipeline_name),
+        }
+
+        if placeholder_run:
+            orchestrator_pod_labels["run_id"] = kube_utils.sanitize_label(
+                str(placeholder_run.id)
+            )
+            orchestrator_pod_labels["run_name"] = kube_utils.sanitize_label(
+                str(placeholder_run.name)
+            )
+
         # Schedule as CRON job if CRON schedule is given.
         if deployment.schedule:
             if not deployment.schedule.cron_expression:
@@ -519,9 +533,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
             cron_expression = deployment.schedule.cron_expression
             cron_job_manifest = build_cron_job_manifest(
                 cron_expression=cron_expression,
-                run_name=orchestrator_run_name,
                 pod_name=pod_name,
-                pipeline_name=pipeline_name,
                 image_name=image,
                 command=command,
                 args=args,
@@ -533,6 +545,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 successful_jobs_history_limit=settings.successful_jobs_history_limit,
                 failed_jobs_history_limit=settings.failed_jobs_history_limit,
                 ttl_seconds_after_finished=settings.ttl_seconds_after_finished,
+                labels=orchestrator_pod_labels,
             )
 
             self._k8s_batch_api.create_namespaced_cron_job(
@@ -547,9 +560,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
         else:
             # Create and run the orchestrator pod.
             pod_manifest = build_pod_manifest(
-                run_name=orchestrator_run_name,
                 pod_name=pod_name,
-                pipeline_name=pipeline_name,
                 image_name=image,
                 command=command,
                 args=args,
@@ -557,6 +568,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 pod_settings=orchestrator_pod_settings,
                 service_account_name=service_account_name,
                 env=environment,
+                labels=orchestrator_pod_labels,
                 mount_local_stores=self.config.is_local,
             )
 
@@ -572,6 +584,11 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 startup_timeout=settings.pod_startup_timeout,
             )
 
+            metadata: Dict[str, MetadataType] = {
+                METADATA_ORCHESTRATOR_RUN_ID: pod_name,
+            }
+
+            # Wait for the orchestrator pod to finish and stream logs.
             if settings.synchronous:
 
                 def _wait_for_run_to_finish() -> None:
@@ -588,7 +605,8 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                     )
 
                 return SubmissionResult(
-                    wait_for_completion=_wait_for_run_to_finish
+                    metadata=metadata,
+                    wait_for_completion=_wait_for_run_to_finish,
                 )
             else:
                 logger.info(
@@ -597,7 +615,9 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                     f"Run the following command to inspect the logs: "
                     f"`kubectl logs {pod_name} -n {self.config.kubernetes_namespace}`."
                 )
-                return None
+                return SubmissionResult(
+                    metadata=metadata,
+                )
 
     def _get_service_account_name(
         self, settings: KubernetesOrchestratorSettings
@@ -642,3 +662,18 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
                 "Unable to read run id from environment variable "
                 f"{ENV_ZENML_KUBERNETES_RUN_ID}."
             )
+
+    def get_pipeline_run_metadata(
+        self, run_id: UUID
+    ) -> Dict[str, "MetadataType"]:
+        """Get general component-specific metadata for a pipeline run.
+
+        Args:
+            run_id: The ID of the pipeline run.
+
+        Returns:
+            A dictionary of metadata.
+        """
+        return {
+            METADATA_ORCHESTRATOR_RUN_ID: self.get_orchestrator_run_id(),
+        }

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py

@@ -15,8 +15,7 @@
 
 import argparse
 import socket
-from typing import Any, Dict, cast
-from uuid import UUID
+from typing import Callable, Dict, Optional, cast
 
 from kubernetes import client as k8s_client
 
@@ -41,10 +40,16 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
 from zenml.logger import get_logger
 from zenml.orchestrators import publish_utils
 from zenml.orchestrators.dag_runner import NodeStatus, ThreadedDagRunner
+from zenml.orchestrators.step_run_utils import (
+    StepRunRequestFactory,
+    fetch_step_runs_by_names,
+    publish_cached_step_run,
+)
 from zenml.orchestrators.utils import (
     get_config_environment_vars,
     get_orchestrator_run_name,
 )
+from zenml.pipelines.run_utils import create_placeholder_run
 
 logger = get_logger(__name__)
 
@@ -56,9 +61,7 @@ def parse_args() -> argparse.Namespace:
         Parsed args.
     """
     parser = argparse.ArgumentParser()
-    parser.add_argument("--run_name", type=str, required=True)
     parser.add_argument("--deployment_id", type=str, required=True)
-    parser.add_argument("--kubernetes_namespace", type=str, required=True)
     parser.add_argument("--run_id", type=str, required=False)
     return parser.parse_args()
 
@@ -68,7 +71,6 @@ def main() -> None:
     # Log to the container's stdout so it can be streamed by the client.
     logger.info("Kubernetes orchestrator pod started.")
 
-    # Parse / extract args.
     args = parse_args()
 
     orchestrator_pod_name = socket.gethostname()
@@ -77,6 +79,7 @@ def main() -> None:
     active_stack = client.active_stack
     orchestrator = active_stack.orchestrator
     assert isinstance(orchestrator, KubernetesOrchestrator)
+    namespace = orchestrator.config.kubernetes_namespace
 
     deployment = client.get_deployment(args.deployment_id)
     pipeline_settings = cast(
@@ -86,20 +89,6 @@ def main() -> None:
 
     step_command = StepEntrypointConfiguration.get_entrypoint_command()
 
-    if args.run_id and not pipeline_settings.prevent_orchestrator_pod_caching:
-        from zenml.orchestrators import cache_utils
-
-        run_required = (
-            cache_utils.create_cached_step_runs_and_prune_deployment(
-                deployment=deployment,
-                pipeline_run=client.get_pipeline_run(args.run_id),
-                stack=active_stack,
-            )
-        )
-
-        if not run_required:
-            return
-
     mount_local_stores = active_stack.orchestrator.config.is_local
 
     # Get a Kubernetes client from the active Kubernetes orchestrator, but
@@ -115,7 +104,7 @@ def main() -> None:
         owner_references = kube_utils.get_pod_owner_references(
             core_api=core_api,
             pod_name=orchestrator_pod_name,
-            namespace=args.kubernetes_namespace,
+            namespace=namespace,
         )
     except Exception as e:
         logger.warning(f"Failed to get pod owner references: {str(e)}")
@@ -126,6 +115,62 @@ def main() -> None:
         for owner_reference in owner_references:
             owner_reference.controller = False
 
+    if args.run_id:
+        pipeline_run = client.get_pipeline_run(args.run_id)
+    else:
+        pipeline_run = create_placeholder_run(
+            deployment=deployment,
+            orchestrator_run_id=orchestrator_pod_name,
+        )
+
+    pre_step_run: Optional[Callable[[str], bool]] = None
+
+    if not pipeline_settings.prevent_orchestrator_pod_caching:
+        step_run_request_factory = StepRunRequestFactory(
+            deployment=deployment,
+            pipeline_run=pipeline_run,
+            stack=active_stack,
+        )
+        step_runs = {}
+
+        def pre_step_run(step_name: str) -> bool:
+            """Pre-step run.
+
+            Args:
+                step_name: Name of the step.
+
+            Returns:
+                Whether the step node needs to be run.
+            """
+            step_run_request = step_run_request_factory.create_request(
+                step_name
+            )
+            try:
+                step_run_request_factory.populate_request(step_run_request)
+            except Exception as e:
+                logger.error(
+                    f"Failed to populate step run request for step {step_name}: {e}"
+                )
+                return True
+
+            if step_run_request.status == ExecutionStatus.CACHED:
+                step_run = publish_cached_step_run(
+                    step_run_request, pipeline_run
+                )
+                step_runs[step_name] = step_run
+                logger.info("Using cached version of step `%s`.", step_name)
+                return False
+
+            return True
+
+    step_pod_labels = {
+        "run_id": kube_utils.sanitize_label(str(pipeline_run.id)),
+        "run_name": kube_utils.sanitize_label(str(pipeline_run.name)),
+        "pipeline": kube_utils.sanitize_label(
+            deployment.pipeline_configuration.name
+        ),
+    }
+
     def run_step_on_kubernetes(step_name: str) -> None:
         """Run a pipeline step in a separate Kubernetes pod.
 
@@ -146,7 +191,7 @@ def main() -> None:
         ):
             max_length = (
                 kube_utils.calculate_max_pod_name_length_for_namespace(
-                    namespace=args.kubernetes_namespace
+                    namespace=namespace
                 )
             )
             pod_name_prefix = get_orchestrator_run_name(
@@ -156,9 +201,7 @@ def main() -> None:
         else:
             pod_name = f"{orchestrator_pod_name}-{step_name}"
 
-        pod_name = kube_utils.sanitize_pod_name(
-            pod_name, namespace=args.kubernetes_namespace
-        )
+        pod_name = kube_utils.sanitize_pod_name(pod_name, namespace=namespace)
 
         image = KubernetesOrchestrator.get_image(
             deployment=deployment, step_name=step_name
@@ -195,8 +238,6 @@ def main() -> None:
         # Define Kubernetes pod manifest.
         pod_manifest = build_pod_manifest(
             pod_name=pod_name,
-            run_name=args.run_name,
-            pipeline_name=deployment.pipeline_configuration.name,
             image_name=image,
             command=step_command,
             args=step_args,
@@ -207,6 +248,7 @@ def main() -> None:
             or settings.service_account_name,
             mount_local_stores=mount_local_stores,
             owner_references=owner_references,
+            labels=step_pod_labels,
         )
 
         kube_utils.create_and_wait_for_pod_to_start(
@@ -214,7 +256,7 @@ def main() -> None:
             pod_display_name=f"pod for step `{step_name}`",
             pod_name=pod_name,
             pod_manifest=pod_manifest,
-            namespace=args.kubernetes_namespace,
+            namespace=namespace,
             startup_max_retries=settings.pod_failure_max_retries,
             startup_failure_delay=settings.pod_failure_retry_delay,
             startup_failure_backoff=settings.pod_failure_backoff,
@@ -229,7 +271,7 @@ def main() -> None:
                     incluster=True
                 ),
                 pod_name=pod_name,
-                namespace=args.kubernetes_namespace,
+                namespace=namespace,
                 exit_condition_lambda=kube_utils.pod_is_done,
                 stream_logs=True,
             )
@@ -249,30 +291,15 @@ def main() -> None:
         try:
             # Some steps may have failed because the pods could not be created.
             # We need to check for this and mark the step run as failed if so.
-
-            # Fetch the pipeline run using any means possible.
-            list_args: Dict[str, Any] = {}
-            if args.run_id:
-                # For a run triggered outside of a schedule, we can use the
-                # placeholder run ID to find the pipeline run.
-                list_args = dict(id=UUID(args.run_id))
-            else:
-                # For a run triggered by a schedule, we can only use the
-                # orchestrator run ID to find the pipeline run.
-                list_args = dict(orchestrator_run_id=orchestrator_pod_name)
-
-            pipeline_runs = client.list_pipeline_runs(
-                hydrate=True,
-                project=deployment.project_id,
-                deployment_id=deployment.id,
-                **list_args,
-            )
-            if not len(pipeline_runs):
-                # No pipeline run found, so we can't mark any step runs as failed.
-                return
-
-            pipeline_run = pipeline_runs[0]
             pipeline_failed = False
+            failed_step_names = [
+                step_name
+                for step_name, node_state in node_states.items()
+                if node_state == NodeStatus.FAILED
+            ]
+            step_runs = fetch_step_runs_by_names(
+                step_run_names=failed_step_names, pipeline_run=pipeline_run
+            )
 
             for step_name, node_state in node_states.items():
                 if node_state != NodeStatus.FAILED:
@@ -280,18 +307,14 @@ def main() -> None:
 
                 pipeline_failed = True
 
-                # If steps failed for any reason, we need to mark the step run as
-                # failed, if it exists and it wasn't already in a final state.
-
-                step_run = pipeline_run.steps.get(step_name)
-
-                # Try to update the step run status, if it exists and is in
-                # a transient state.
-                if step_run and step_run.status in {
-                    ExecutionStatus.INITIALIZING,
-                    ExecutionStatus.RUNNING,
-                }:
-                    publish_utils.publish_failed_step_run(step_run.id)
+                if step_run := step_runs.get(step_name, None):
+                    # Try to update the step run status, if it exists and is in
+                    # a transient state.
+                    if step_run and step_run.status in {
+                        ExecutionStatus.INITIALIZING,
+                        ExecutionStatus.RUNNING,
+                    }:
+                        publish_utils.publish_failed_step_run(step_run.id)
 
             # If any steps failed and the pipeline run is still in a transient
             # state, we need to mark it as failed.
@@ -319,6 +342,7 @@ def main() -> None:
         ThreadedDagRunner(
             dag=pipeline_dag,
             run_fn=run_step_on_kubernetes,
+            preparation_fn=pre_step_run,
             finalize_fn=finalize_run,
             parallel_node_startup_waiting_period=parallel_node_startup_waiting_period,
             max_parallelism=pipeline_settings.max_parallelism,
@@ -333,7 +357,7 @@ def main() -> None:
             try:
                 kube_utils.delete_secret(
                     core_api=core_api,
-                    namespace=args.kubernetes_namespace,
+                    namespace=namespace,
                     secret_name=secret_name,
                 )
             except k8s_client.rest.ApiException as e:

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint_configuration.py

@@ -18,9 +18,7 @@ from typing import TYPE_CHECKING, List, Optional, Set
 if TYPE_CHECKING:
     from uuid import UUID
 
-RUN_NAME_OPTION = "run_name"
 DEPLOYMENT_ID_OPTION = "deployment_id"
-NAMESPACE_OPTION = "kubernetes_namespace"
 RUN_ID_OPTION = "run_id"
 
 
@@ -35,9 +33,7 @@ class KubernetesOrchestratorEntrypointConfiguration:
             Entrypoint options.
         """
         options = {
-            RUN_NAME_OPTION,
             DEPLOYMENT_ID_OPTION,
-            NAMESPACE_OPTION,
         }
         return options
 
@@ -58,29 +54,21 @@ class KubernetesOrchestratorEntrypointConfiguration:
     @classmethod
     def get_entrypoint_arguments(
         cls,
-        run_name: str,
         deployment_id: "UUID",
-        kubernetes_namespace: str,
         run_id: Optional["UUID"] = None,
     ) -> List[str]:
         """Gets all arguments that the entrypoint command should be called with.
 
         Args:
-            run_name: Name of the ZenML run.
             deployment_id: ID of the deployment.
-            kubernetes_namespace: Name of the Kubernetes namespace.
             run_id: Optional ID of the pipeline run. Not set for scheduled runs.
 
         Returns:
             List of entrypoint arguments.
         """
         args = [
-            f"--{RUN_NAME_OPTION}",
-            run_name,
             f"--{DEPLOYMENT_ID_OPTION}",
             str(deployment_id),
-            f"--{NAMESPACE_OPTION}",
-            kubernetes_namespace,
         ]
 
         if run_id:

zenml/integrations/kubernetes/orchestrators/manifest_utils.py

@@ -26,7 +26,6 @@ from zenml.constants import ENV_ZENML_ENABLE_REPO_INIT_WARNINGS
 from zenml.integrations.airflow.orchestrators.dag_generator import (
     ENV_ZENML_LOCAL_STORES_PATH,
 )
-from zenml.integrations.kubernetes.orchestrators import kube_utils
 from zenml.integrations.kubernetes.pod_settings import KubernetesPodSettings
 from zenml.logger import get_logger
 
@@ -97,8 +96,6 @@ def add_local_stores_mount(
 
 def build_pod_manifest(
     pod_name: str,
-    run_name: str,
-    pipeline_name: str,
     image_name: str,
     command: List[str],
     args: List[str],
@@ -106,6 +103,7 @@ def build_pod_manifest(
     pod_settings: Optional[KubernetesPodSettings] = None,
     service_account_name: Optional[str] = None,
     env: Optional[Dict[str, str]] = None,
+    labels: Optional[Dict[str, str]] = None,
     mount_local_stores: bool = False,
     owner_references: Optional[List[k8s_client.V1OwnerReference]] = None,
 ) -> k8s_client.V1Pod:
@@ -113,8 +111,6 @@ def build_pod_manifest(
 
     Args:
         pod_name: Name of the pod.
-        run_name: Name of the ZenML run.
-        pipeline_name: Name of the ZenML pipeline.
         image_name: Name of the Docker image.
         command: Command to execute the entrypoint in the pod.
         args: Arguments provided to the entrypoint command.
@@ -124,6 +120,7 @@ def build_pod_manifest(
             Can be used to assign certain roles to a pod, e.g., to allow it to
             run Kubernetes commands from within the cluster.
         env: Environment variables to set.
+        labels: Labels to add to the pod.
         mount_local_stores: Whether to mount the local stores path inside the
             pod.
         owner_references: List of owner references for the pod.
@@ -162,7 +159,7 @@ def build_pod_manifest(
     if service_account_name is not None:
         pod_spec.service_account_name = service_account_name
 
-    labels = {}
+    labels = labels or {}
 
     if pod_settings:
         add_pod_settings(pod_spec, pod_settings)
@@ -171,14 +168,6 @@ def build_pod_manifest(
         if pod_settings.labels:
             labels.update(pod_settings.labels)
 
-    # Add run_name and pipeline_name to the labels
-    labels.update(
-        {
-            "run": kube_utils.sanitize_label(run_name),
-            "pipeline": kube_utils.sanitize_label(pipeline_name),
-        }
-    )
-
     pod_metadata = k8s_client.V1ObjectMeta(
         name=pod_name,
         labels=labels,
@@ -272,8 +261,6 @@ def add_pod_settings(
 def build_cron_job_manifest(
     cron_expression: str,
     pod_name: str,
-    run_name: str,
-    pipeline_name: str,
     image_name: str,
     command: List[str],
     args: List[str],
@@ -281,6 +268,7 @@ def build_cron_job_manifest(
     pod_settings: Optional[KubernetesPodSettings] = None,
     service_account_name: Optional[str] = None,
     env: Optional[Dict[str, str]] = None,
+    labels: Optional[Dict[str, str]] = None,
     mount_local_stores: bool = False,
     successful_jobs_history_limit: Optional[int] = None,
     failed_jobs_history_limit: Optional[int] = None,
@@ -291,8 +279,6 @@ def build_cron_job_manifest(
     Args:
         cron_expression: CRON job schedule expression, e.g. "* * * * *".
         pod_name: Name of the pod.
-        run_name: Name of the ZenML run.
-        pipeline_name: Name of the ZenML pipeline.
         image_name: Name of the Docker image.
         command: Command to execute the entrypoint in the pod.
         args: Arguments provided to the entrypoint command.
@@ -302,6 +288,7 @@ def build_cron_job_manifest(
             Can be used to assign certain roles to a pod, e.g., to allow it to
             run Kubernetes commands from within the cluster.
         env: Environment variables to set.
+        labels: Labels to add to the pod.
         mount_local_stores: Whether to mount the local stores path inside the
             pod.
         successful_jobs_history_limit: The number of successful jobs to retain.
@@ -314,8 +301,6 @@ def build_cron_job_manifest(
     """
     pod_manifest = build_pod_manifest(
         pod_name=pod_name,
-        run_name=run_name,
-        pipeline_name=pipeline_name,
         image_name=image_name,
         command=command,
         args=args,
@@ -323,6 +308,7 @@ def build_cron_job_manifest(
         pod_settings=pod_settings,
         service_account_name=service_account_name,
         env=env,
+        labels=labels,
         mount_local_stores=mount_local_stores,
     )
 

zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py

@@ -205,9 +205,7 @@ class KubernetesStepOperator(BaseStepOperator):
 
         # Create and run the orchestrator pod.
         pod_manifest = build_pod_manifest(
-            run_name=info.run_name,
             pod_name=pod_name,
-            pipeline_name=info.pipeline.name,
             image_name=image_name,
             command=command,
             args=args,
@@ -216,6 +214,10 @@ class KubernetesStepOperator(BaseStepOperator):
             pod_settings=settings.pod_settings,
             env=environment,
             mount_local_stores=False,
+            labels={
+                "run_id": kube_utils.sanitize_label(str(info.run_id)),
+                "pipeline": kube_utils.sanitize_label(info.pipeline.name),
+            },
         )
 
         kube_utils.create_and_wait_for_pod_to_start(

zenml/integrations/vllm/services/vllm_deployment.py

@@ -150,7 +150,7 @@ class VLLMDeploymentService(LocalDaemonService, BaseDeploymentService):
 
         try:
             parser: argparse.ArgumentParser = make_arg_parser(
-                FlexibleArgumentParser()  # type: ignore[no-untyped-call]
+                FlexibleArgumentParser()
             )
             # pass in empty list to get default args
             # otherwise it will try to get the args from sys.argv

zenml/models/v2/core/pipeline_run.py

@@ -125,6 +125,15 @@ class PipelineRunRequest(ProjectScopedRequest):
         title="Logs of the pipeline run.",
     )
 
+    @property
+    def is_placeholder_request(self) -> bool:
+        """Whether the request is a placeholder request.
+
+        Returns:
+            Whether the request is a placeholder request.
+        """
+        return self.status == ExecutionStatus.INITIALIZING
+
     model_config = ConfigDict(protected_namespaces=())
 
 

zenml/orchestrators/dag_runner.py

@@ -72,6 +72,7 @@ class ThreadedDagRunner:
         self,
         dag: Dict[str, List[str]],
         run_fn: Callable[[str], Any],
+        preparation_fn: Optional[Callable[[str], bool]] = None,
         finalize_fn: Optional[Callable[[Dict[str, NodeStatus]], None]] = None,
         parallel_node_startup_waiting_period: float = 0.0,
         max_parallelism: Optional[int] = None,
@@ -83,6 +84,9 @@ class ThreadedDagRunner:
                 E.g.: [(1->2), (1->3), (2->4), (3->4)] should be represented as
                 `dag={2: [1], 3: [1], 4: [2, 3]}`
             run_fn: A function `run_fn(node)` that runs a single node
+            preparation_fn: A function that is called before the node is run.
+                If provided, the function return value determines whether the
+                node should be run or can be skipped.
             finalize_fn: A function `finalize_fn(node_states)` that is called
                 when all nodes have completed.
             parallel_node_startup_waiting_period: Delay in seconds to wait in
@@ -102,6 +106,7 @@ class ThreadedDagRunner:
         self.dag = dag
         self.reversed_dag = reverse_dag(dag)
         self.run_fn = run_fn
+        self.preparation_fn = preparation_fn
         self.finalize_fn = finalize_fn
         self.nodes = dag.keys()
         self.node_states = {
@@ -156,7 +161,7 @@ class ThreadedDagRunner:
                         break
 
                 logger.debug(f"Waiting for {running_nodes} nodes to finish.")
-                time.sleep(10)
+                time.sleep(1)
 
     def _run_node(self, node: str) -> None:
         """Run a single node.
@@ -168,6 +173,12 @@ class ThreadedDagRunner:
         """
         self._prepare_node_run(node)
 
+        if self.preparation_fn:
+            run_required = self.preparation_fn(node)
+            if not run_required:
+                self._finish_node(node)
+                return
+
         try:
             self.run_fn(node)
             self._finish_node(node)
@@ -203,8 +214,6 @@ class ThreadedDagRunner:
             node: The node.
             failed: Whether the node failed.
         """
-        # Update node status to completed.
-        assert self.node_states[node] == NodeStatus.RUNNING
         with self._lock:
             if failed:
                 self.node_states[node] = NodeStatus.FAILED

zenml/orchestrators/input_utils.py

@@ -13,14 +13,13 @@
 #  permissions and limitations under the License.
 """Utilities for inputs."""
 
-import json
 from typing import TYPE_CHECKING, Dict, Optional
 
 from zenml.client import Client
 from zenml.config.step_configurations import Step
 from zenml.enums import StepRunInputArtifactType
 from zenml.exceptions import InputResolutionError
-from zenml.utils import pagination_utils, string_utils
+from zenml.utils import string_utils
 
 if TYPE_CHECKING:
     from zenml.models import PipelineRunResponse, StepRunResponse
@@ -52,6 +51,7 @@ def resolve_step_inputs(
     """
     from zenml.models import ArtifactVersionResponse
     from zenml.models.v2.core.step_run import StepRunInputResponse
+    from zenml.orchestrators.step_run_utils import fetch_step_runs_by_names
 
     step_runs = step_runs or {}
 
@@ -62,40 +62,11 @@ def resolve_step_inputs(
     steps_to_fetch.difference_update(step_runs.keys())
 
     if steps_to_fetch:
-        # The list of steps might be too big to fit in the default max URL
-        # length of 8KB supported by most servers. So we need to split it into
-        # smaller chunks.
-        steps_list = list(steps_to_fetch)
-        chunks = []
-        current_chunk = []
-        current_length = 0
-        # stay under 6KB for good measure.
-        max_chunk_length = 6000
-
-        for step_name in steps_list:
-            current_chunk.append(step_name)
-            current_length += len(step_name) + 5  # 5 is for the JSON encoding
-
-            if current_length > max_chunk_length:
-                chunks.append(current_chunk)
-                current_chunk = []
-                current_length = 0
-
-        if current_chunk:
-            chunks.append(current_chunk)
-
-        for chunk in chunks:
-            step_runs.update(
-                {
-                    run_step.name: run_step
-                    for run_step in pagination_utils.depaginate(
-                        Client().list_run_steps,
-                        pipeline_run_id=pipeline_run.id,
-                        project=pipeline_run.project_id,
-                        name="oneof:" + json.dumps(chunk),
-                    )
-                }
+        step_runs.update(
+            fetch_step_runs_by_names(
+                step_run_names=list(steps_to_fetch), pipeline_run=pipeline_run
             )
+        )
 
     input_artifacts: Dict[str, StepRunInputResponse] = {}
     for name, input_ in step.spec.inputs.items():

zenml/orchestrators/step_run_utils.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Utilities for creating step runs."""
 
+import json
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 from zenml import Tag, add_tags
@@ -32,6 +33,7 @@ from zenml.models import (
 )
 from zenml.orchestrators import cache_utils, input_utils, utils
 from zenml.stack import Stack
+from zenml.utils import pagination_utils
 from zenml.utils.time_utils import utc_now
 
 logger = get_logger(__name__)
@@ -151,6 +153,15 @@ class StepRunRequestFactory:
                 request.status = ExecutionStatus.CACHED
                 request.end_time = request.start_time
 
+                # As a last resort, we try to reuse the docstring/source code
+                # from the cached step run. This is part of the cache key
+                # computation, so it must be identical to the one we would have
+                # computed ourselves.
+                if request.source_code is None:
+                    request.source_code = cached_step_run.source_code
+                if request.docstring is None:
+                    request.docstring = cached_step_run.docstring
+
     def _get_docstring_and_source_code(
         self, invocation_id: str
     ) -> Tuple[Optional[str], Optional[str]]:
@@ -333,27 +344,15 @@ def create_cached_step_runs(
                 # -> We don't need to do anything here
                 continue
 
-            step_run = Client().zen_store.create_run_step(step_run_request)
+            step_run = publish_cached_step_run(
+                step_run_request, pipeline_run=pipeline_run
+            )
 
             # Include the newly created step run in the step runs dictionary to
             # avoid fetching it again later when downstream steps need it for
             # input resolution.
             step_runs[invocation_id] = step_run
 
-            if (
-                model_version := step_run.model_version
-                or pipeline_run.model_version
-            ):
-                link_output_artifacts_to_model_version(
-                    artifacts=step_run.outputs,
-                    model_version=model_version,
-                )
-
-            cascade_tags_for_output_artifacts(
-                artifacts=step_run.outputs,
-                tags=pipeline_run.config.tags,
-            )
-
             logger.info("Using cached version of step `%s`.", invocation_id)
             cached_invocations.add(invocation_id)
 
@@ -426,3 +425,78 @@ def cascade_tags_for_output_artifacts(
                 tags=[t.name for t in cascade_tags],
                 artifact_version_id=output_artifact.id,
             )
+
+
+def publish_cached_step_run(
+    request: "StepRunRequest", pipeline_run: "PipelineRunResponse"
+) -> "StepRunResponse":
+    """Create a cached step run and link to model version and tags.
+
+    Args:
+        request: The request for the step run.
+        pipeline_run: The pipeline run of the step.
+
+    Returns:
+        The createdstep run.
+    """
+    step_run = Client().zen_store.create_run_step(request)
+
+    if model_version := step_run.model_version or pipeline_run.model_version:
+        link_output_artifacts_to_model_version(
+            artifacts=step_run.outputs,
+            model_version=model_version,
+        )
+
+    cascade_tags_for_output_artifacts(
+        artifacts=step_run.outputs,
+        tags=pipeline_run.config.tags,
+    )
+
+    return step_run
+
+
+def fetch_step_runs_by_names(
+    step_run_names: List[str], pipeline_run: "PipelineRunResponse"
+) -> Dict[str, "StepRunResponse"]:
+    """Fetch step runs by names.
+
+    Args:
+        step_run_names: The names of the step runs to fetch.
+        pipeline_run: The pipeline run of the step runs.
+
+    Returns:
+        A dictionary of step runs by name.
+    """
+    step_runs = {}
+
+    chunks = []
+    current_chunk = []
+    current_length = 0
+    # stay under 6KB for good measure.
+    max_chunk_length = 6000
+
+    for step_name in step_run_names:
+        current_chunk.append(step_name)
+        current_length += len(step_name) + 5  # 5 is for the JSON encoding
+
+        if current_length > max_chunk_length:
+            chunks.append(current_chunk)
+            current_chunk = []
+            current_length = 0
+
+    if current_chunk:
+        chunks.append(current_chunk)
+
+    for chunk in chunks:
+        step_runs.update(
+            {
+                run_step.name: run_step
+                for run_step in pagination_utils.depaginate(
+                    Client().list_run_steps,
+                    pipeline_run_id=pipeline_run.id,
+                    project=pipeline_run.project_id,
+                    name="oneof:" + json.dumps(chunk),
+                )
+            }
+        )
+    return step_runs

zenml/pipelines/pipeline_definition.py

@@ -863,8 +863,12 @@ To avoid this consider setting pipeline parameters only in one place (config or
                 deployment = self._create_deployment(**self._run_args)
 
                 self.log_pipeline_deployment_metadata(deployment)
-                run = create_placeholder_run(
-                    deployment=deployment, logs=logs_model
+                run = (
+                    create_placeholder_run(
+                        deployment=deployment, logs=logs_model
+                    )
+                    if not deployment.schedule
+                    else None
                 )
 
                 analytics_handler.metadata = (

zenml/pipelines/run_utils.py

@@ -51,23 +51,19 @@ def get_default_run_name(pipeline_name: str) -> str:
 
 def create_placeholder_run(
     deployment: "PipelineDeploymentResponse",
+    orchestrator_run_id: Optional[str] = None,
     logs: Optional["LogsRequest"] = None,
-) -> Optional["PipelineRunResponse"]:
+) -> "PipelineRunResponse":
     """Create a placeholder run for the deployment.
 
-    If the deployment contains a schedule, no placeholder run will be
-    created.
-
     Args:
         deployment: The deployment for which to create the placeholder run.
+        orchestrator_run_id: The orchestrator run ID for the run.
         logs: The logs for the run.
 
     Returns:
-        The placeholder run or `None` if no run was created.
+        The placeholder run.
     """
-    if deployment.schedule:
-        return None
-
     start_time = utc_now()
     run_request = PipelineRunRequest(
         name=string_utils.format_name_template(
@@ -83,7 +79,7 @@ def create_placeholder_run(
         # the start_time is only set once the first step starts
         # running.
         start_time=start_time,
-        orchestrator_run_id=None,
+        orchestrator_run_id=orchestrator_run_id,
         project=deployment.project_id,
         deployment=deployment.id,
         pipeline=deployment.pipeline.id if deployment.pipeline else None,

zenml/zen_server/template_execution/utils.py

@@ -193,7 +193,6 @@ def run_template(
     zenml_version = build.zenml_version
 
     placeholder_run = create_placeholder_run(deployment=new_deployment)
-    assert placeholder_run
 
     report_usage(
         feature=RUN_TEMPLATE_TRIGGERS_FEATURE_NAME,

zenml/zen_stores/schemas/pipeline_run_schemas.py

@@ -51,7 +51,9 @@ from zenml.zen_stores.schemas.pipeline_deployment_schemas import (
 from zenml.zen_stores.schemas.pipeline_schemas import PipelineSchema
 from zenml.zen_stores.schemas.project_schemas import ProjectSchema
 from zenml.zen_stores.schemas.schedule_schema import ScheduleSchema
-from zenml.zen_stores.schemas.schema_utils import build_foreign_key_field
+from zenml.zen_stores.schemas.schema_utils import (
+    build_foreign_key_field,
+)
 from zenml.zen_stores.schemas.stack_schemas import StackSchema
 from zenml.zen_stores.schemas.trigger_schemas import TriggerExecutionSchema
 from zenml.zen_stores.schemas.user_schemas import UserSchema
@@ -550,8 +552,8 @@ class PipelineRunSchema(NamedSchema, RunMetadataInterface, table=True):
 
         Raises:
             RuntimeError: If the DB entry does not represent a placeholder run.
-            ValueError: If the run request does not match the deployment or
-                pipeline ID of the placeholder run.
+            ValueError: If the run request is not a valid request to replace the
+                placeholder run.
 
         Returns:
             The updated `PipelineRunSchema`.
@@ -562,13 +564,33 @@ class PipelineRunSchema(NamedSchema, RunMetadataInterface, table=True):
                 "placeholder run."
             )
 
+        if request.is_placeholder_request:
+            raise ValueError(
+                "Cannot replace a placeholder run with another placeholder run."
+            )
+
         if (
             self.deployment_id != request.deployment
             or self.pipeline_id != request.pipeline
+            or self.project_id != request.project
         ):
             raise ValueError(
-                "Deployment or orchestrator run ID of placeholder run do not "
-                "match the IDs of the run request."
+                "Deployment, project or pipeline ID of placeholder run "
+                "do not match the IDs of the run request."
+            )
+
+        if not request.orchestrator_run_id:
+            raise ValueError(
+                "Orchestrator run ID is required to replace a placeholder run."
+            )
+
+        if (
+            self.orchestrator_run_id
+            and self.orchestrator_run_id != request.orchestrator_run_id
+        ):
+            raise ValueError(
+                "Orchestrator run ID of placeholder run does not match the "
+                "ID of the run request."
             )
 
         orchestrator_environment = json.dumps(request.orchestrator_environment)
@@ -587,7 +609,4 @@ class PipelineRunSchema(NamedSchema, RunMetadataInterface, table=True):
         Returns:
             Whether the pipeline run is a placeholder run.
         """
-        return (
-            self.orchestrator_run_id is None
-            and self.status == ExecutionStatus.INITIALIZING
-        )
+        return self.status == ExecutionStatus.INITIALIZING.value

zenml/zen_stores/sql_zen_store.py

@@ -5844,18 +5844,33 @@ class SqlZenStore(BaseZenStore):
             # transaction to do so finishes. After the first transaction
             # finishes, the subsequent queries will not be able to find a
             # placeholder run anymore, as we already updated the
-            # orchestrator_run_id.
-            # Note: This only locks a single row if the where clause of
-            # the query is indexed (we have a unique index due to the
-            # unique constraint on those columns). Otherwise, this will lock
-            # multiple rows or even the complete table which we want to
-            # avoid.
+            # status.
+            # Note: Due to our unique index on deployment_id and
+            # orchestrator_run_id, this only locks a single row. If you're
+            # modifying this WHERE clause, make sure to test/adjust so this
+            # does not lock multiple rows or even the complete table.
             .with_for_update()
             .where(PipelineRunSchema.deployment_id == pipeline_run.deployment)
             .where(
-                PipelineRunSchema.orchestrator_run_id.is_(None)  # type: ignore[union-attr]
+                or_(
+                    PipelineRunSchema.orchestrator_run_id
+                    == pipeline_run.orchestrator_run_id,
+                    col(PipelineRunSchema.orchestrator_run_id).is_(None),
+                )
             )
-            .where(PipelineRunSchema.project_id == pipeline_run.project)
+            .where(
+                PipelineRunSchema.status == ExecutionStatus.INITIALIZING.value
+            )
+            # In very rare cases, there can be multiple placeholder runs for
+            # the same deployment. By ordering by the orchestrator_run_id, we
+            # make sure that we use the placeholder run with the matching
+            # orchestrator_run_id if it exists, before falling back to the
+            # placeholder run without any orchestrator_run_id provided.
+            # Note: This works because both SQLite and MySQL consider NULLs
+            # to be lower than any other value. If we add support for other
+            # databases (e.g. PostgreSQL, which considers NULLs to be greater
+            # than any other value), we need to potentially adjust this.
+            .order_by(desc(PipelineRunSchema.orchestrator_run_id))
         ).first()
 
         if not run_schema:
@@ -5903,6 +5918,9 @@ class SqlZenStore(BaseZenStore):
             .where(
                 PipelineRunSchema.orchestrator_run_id == orchestrator_run_id
             )
+            .where(
+                PipelineRunSchema.status != ExecutionStatus.INITIALIZING.value
+            )
         ).first()
 
         if not run_schema:
@@ -5955,27 +5973,31 @@ class SqlZenStore(BaseZenStore):
                 except KeyError:
                     pass
 
-            try:
-                return (
-                    self._replace_placeholder_run(
-                        pipeline_run=pipeline_run,
-                        pre_replacement_hook=pre_creation_hook,
-                        session=session,
-                    ),
-                    True,
-                )
-            except KeyError:
-                # We were not able to find/replace a placeholder run. This could
-                # be due to one of the following three reasons:
-                # (1) There never was a placeholder run for the deployment. This
-                #     is the case if the user ran the pipeline on a schedule.
-                # (2) There was a placeholder run, but a previous pipeline run
-                #     already used it. This is the case if users rerun a
-                #     pipeline run e.g. from the orchestrator UI, as they will
-                #     use the same deployment_id with a new orchestrator_run_id.
-                # (3) A step of the same pipeline run already replaced the
-                #     placeholder run.
-                pass
+            if not pipeline_run.is_placeholder_request:
+                # Only run this if the request is not a placeholder run itself,
+                # as we don't want to replace a placeholder run with another
+                # placeholder run.
+                try:
+                    return (
+                        self._replace_placeholder_run(
+                            pipeline_run=pipeline_run,
+                            pre_replacement_hook=pre_creation_hook,
+                            session=session,
+                        ),
+                        True,
+                    )
+                except KeyError:
+                    # We were not able to find/replace a placeholder run. This could
+                    # be due to one of the following three reasons:
+                    # (1) There never was a placeholder run for the deployment. This
+                    #     is the case if the user ran the pipeline on a schedule.
+                    # (2) There was a placeholder run, but a previous pipeline run
+                    #     already used it. This is the case if users rerun a
+                    #     pipeline run e.g. from the orchestrator UI, as they will
+                    #     use the same deployment_id with a new orchestrator_run_id.
+                    # (3) A step of the same pipeline run already replaced the
+                    #     placeholder run.
+                    pass
 
             try:
                 # We now try to create a new run. The following will happen in

{zenml_nightly-0.83.1.dev20250627.dist-info → zenml_nightly-0.83.1.dev20250628.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.83.1.dev20250627
+Version: 0.83.1.dev20250628
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops

{zenml_nightly-0.83.1.dev20250627.dist-info → zenml_nightly-0.83.1.dev20250628.dist-info}/RECORD

@@ -1,5 +1,5 @@
 zenml/README.md,sha256=827dekbOWAs1BpW7VF1a4d7EbwPbjwccX-2zdXBENZo,1777
-zenml/VERSION,sha256=fwg9S_GFBEaPtDBc6LSBWzttPLbJ-KFy1HhTDpt6TXs,19
+zenml/VERSION,sha256=x5dBMBD6SExu8DR6dvH97yI7GYYyLeKD8pBqRadIu00,19
 zenml/__init__.py,sha256=r7JUg2SVDf_dPhS7iU6vudKusEqK4ics7_jFMZhq0o4,2731
 zenml/actions/__init__.py,sha256=mrt6wPo73iKRxK754_NqsGyJ3buW7RnVeIGXr1xEw8Y,681
 zenml/actions/base_action.py,sha256=UcaHev6BTuLDwuswnyaPjdA8AgUqB5xPZ-lRtuvf2FU,25553
@@ -338,16 +338,16 @@ zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py,sha256=8
 zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py,sha256=xFO7cSusji-mgbRrt4mU29gdyC9iEjEHKtomdFLp9mM,6265
 zenml/integrations/kubernetes/orchestrators/__init__.py,sha256=TJID3OTieZBox36WpQpzD0jdVRA_aZVcs_bNtfXS8ik,811
 zenml/integrations/kubernetes/orchestrators/kube_utils.py,sha256=N66GH5ac22Xm_A3nr162kbFBhMeypSFaQjOQRHlGXIQ,18942
-zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py,sha256=qmF3i9Zvo2mMtF9WW80SWIPnbths9L_HEUGYZ_OZSus,26575
-zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py,sha256=WAeZH5ro1k_edCoE4y2nhX-1lGS_0EEFwzSbp1BItRw,13010
-zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint_configuration.py,sha256=KjHfQK9VQEQkkkM2i9w51AzqolgIU01M5dgb2YGamvY,2754
-zenml/integrations/kubernetes/orchestrators/manifest_utils.py,sha256=QmK0HrAMalalcQMyG0rqBysoB3srQ4ywnKARdf-LtFc,14083
+zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py,sha256=iuUllVHlXTvTftGk0Gs3EIITG0XXKwCT68UOfLHoJMs,27603
+zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py,sha256=LTqbww_bQFnRS18VAPrxB356UoX8YJI6nlDR74tzqoo,13572
+zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint_configuration.py,sha256=QOwQnWCfB-t_BQ2eOZN0SakurGUd0GTMCSdUlREhk6I,2324
+zenml/integrations/kubernetes/orchestrators/manifest_utils.py,sha256=8_OU6cQxLRzkd1r40YNRoAnz5qHSFQOvVxHUhh88T04,13670
 zenml/integrations/kubernetes/pod_settings.py,sha256=QPV8Fq8SHZ7pwKavplcADJep-Y9olIbC3RlG10HI0pw,6683
 zenml/integrations/kubernetes/serialization_utils.py,sha256=cPSe4szdBLzDnUZT9nQc2CCA8h84aj5oTA8vsUE36ig,7000
 zenml/integrations/kubernetes/service_connectors/__init__.py,sha256=Uf6zlHIapYrRDl3xOPWQ2jA7jt85SXx1U7DmSxzxTvQ,818
 zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py,sha256=Cv4tiVxoQOz9ex0lf3JdJrooEkgMwfDfwt5GOeNRpQU,19669
 zenml/integrations/kubernetes/step_operators/__init__.py,sha256=40utDPYAezxHsFgO0UUIT_6XpCDzDapje6OH951XsTs,806
-zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py,sha256=v_Itgr31-I9EeMngSHk4eSO48taurMEiQNE7SV2L0no,8748
+zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py,sha256=lpkrA_barndyNLKqEUTaOIYJ6V8kHX9SfK5XFKK1md4,8848
 zenml/integrations/label_studio/__init__.py,sha256=sF2c9FxTDRlbcu95OxaUNKNtIhC1LgfmBRKY4jBME38,1475
 zenml/integrations/label_studio/annotators/__init__.py,sha256=YtOtSfS1_NBoLoXIygEerElBP1-B98UU0HOAEfzdRY0,821
 zenml/integrations/label_studio/annotators/label_studio_annotator.py,sha256=VkuW4zsZhHz8__P9WTTLRTF-FOmoYB-_cFqBdu-PUyA,30498
@@ -549,7 +549,7 @@ zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py,sha256=_3P0-qyjdsV
 zenml/integrations/vllm/model_deployers/__init__.py,sha256=Z38oWIfkArNsxCm3rQkTdYK4dbtx2BpTUw1gw_kl6Do,803
 zenml/integrations/vllm/model_deployers/vllm_model_deployer.py,sha256=OYPNSkB-I5r4eQ_7kr4F7GDwNj6efcsio8WRteQ5cYI,9665
 zenml/integrations/vllm/services/__init__.py,sha256=Id28GEfHECI0RnGAGGNioD9eZ6aJxdNebe112VgC59g,788
-zenml/integrations/vllm/services/vllm_deployment.py,sha256=lU1icw10uMJ8fehc94eqyAQ53WQDkzRwW0G3hiy0hNs,7140
+zenml/integrations/vllm/services/vllm_deployment.py,sha256=wRFo8ebN6BEBLs1ZdEdNsFXSk38MJatKEyu_euJJbLc,7107
 zenml/integrations/wandb/__init__.py,sha256=5aTIc27MeYzODDGeJHZmtRhTSNkg7Vt2tHgrtGTCD_c,1705
 zenml/integrations/wandb/experiment_trackers/__init__.py,sha256=8nFyyvh-PTF5d9ZfjS7xFSWTWSpreRB1azePv-Ex2sc,771
 zenml/integrations/wandb/experiment_trackers/wandb_experiment_tracker.py,sha256=GV5zDPgj6Dh3ho2MMUC1Da1ezPrNtr4RE9tisWGde00,5749
@@ -649,7 +649,7 @@ zenml/models/v2/core/model_version_pipeline_run.py,sha256=JbPZZEQvOK9I32htkWdAON
 zenml/models/v2/core/pipeline.py,sha256=OXArih3YodMAZBS_GzuLL6VPpFv3c59EthwDfDZiNGk,12044
 zenml/models/v2/core/pipeline_build.py,sha256=z0LCc8aR8AdNFLLtzaAP0U0ffv7WpmSx9nNAyI__e14,17008
 zenml/models/v2/core/pipeline_deployment.py,sha256=nrJHrweNlwFsIdExINMF7jwhjOwBpOd8iwtEyAnvlz4,11971
-zenml/models/v2/core/pipeline_run.py,sha256=8IzcMYE3Lmx6PCuCwIsnhD-f_E8tvNgF7fnx3rlHuhM,32257
+zenml/models/v2/core/pipeline_run.py,sha256=hWnM6Tf72uE9nxuSV6xacCMogOozuExvdtILgO6il98,32522
 zenml/models/v2/core/project.py,sha256=fNNO8Tg5OhSzmFf2t6g4SpUzGWC96oHhUccVyWytvIE,5627
 zenml/models/v2/core/run_metadata.py,sha256=hRGQa_sk99uDSab3EyyOQhefypVpiQDCH3oAtblREDk,2432
 zenml/models/v2/core/run_template.py,sha256=6jdH1son7kpvFv-vtaOL2nXMATtVpqk_7a2xOVv_7cc,14097
@@ -685,8 +685,8 @@ zenml/orchestrators/__init__.py,sha256=Nhmc6U-q6c8TEH1Jb5l8XaKnc4KmLNspDpvvV5Tcv
 zenml/orchestrators/base_orchestrator.py,sha256=1L2oJTNpVWUu5ndqyQPdcf2S5ebM7r2IHHrFatV6kPQ,14612
 zenml/orchestrators/cache_utils.py,sha256=QkmTs-ANfXve9_QzTqgGlyulZDEWOngoTcsiSjG5aA8,5906
 zenml/orchestrators/containerized_orchestrator.py,sha256=rdebgBW0Bk--JcHcT0NpLkAbyhY0VS5xO1uwWEgkLpA,3230
-zenml/orchestrators/dag_runner.py,sha256=4oAGc52nHv8HeI_Vj7GH1jzjJom1uwUJ_la9usQoOFY,9404
-zenml/orchestrators/input_utils.py,sha256=dSnL4KDLARY4Ldn3CI61sfCOcgsc2RfLue83WkgRrio,7950
+zenml/orchestrators/dag_runner.py,sha256=Ol5P24i8Oa0mmZeNZnUdd_HrQDawx8kP1epWa-b6f3g,9801
+zenml/orchestrators/input_utils.py,sha256=IwDYEwhFVcvZcdi9Myg__UB1bkZfpa8xZ0XRSwc4-go,6934
 zenml/orchestrators/local/__init__.py,sha256=qlU91hgqGKeycw-Qntrn-iMuoMTaNrq-RgfOFeqwlM4,662
 zenml/orchestrators/local/local_orchestrator.py,sha256=KCzc901_wrb1DPTDu_IY6HFxTghe2oiLYPAdxEpj7K4,6033
 zenml/orchestrators/local_docker/__init__.py,sha256=k8J68ydy6HmmvE9tWo32g761H8P_Dw4AxWNf4UMpsbs,669
@@ -694,7 +694,7 @@ zenml/orchestrators/local_docker/local_docker_orchestrator.py,sha256=RA88Yq8K9-z
 zenml/orchestrators/output_utils.py,sha256=01vqke1ZfmfuLpgxNerF-QL2wA0VPv1zUdvlMw0OwUY,3508
 zenml/orchestrators/publish_utils.py,sha256=CSQKhx2f9r2knldDCuPR0lmVyRwI-Ps6Xbihfhxv21U,5477
 zenml/orchestrators/step_launcher.py,sha256=6hrLj0Dr5-FcKTm3cvLVnr3PuUpaYbyEs1MRQdAVzys,18064
-zenml/orchestrators/step_run_utils.py,sha256=jaH-WdJ8Bt6v8m4dD4LgOR0TRa4y50pYFRum14X8B2w,14831
+zenml/orchestrators/step_run_utils.py,sha256=SZjVSkmO9Vvjb2ZqQembOjg3jTU2wHXAykwlVxQNVBg,17051
 zenml/orchestrators/step_runner.py,sha256=EUgKG_g0fOQ6gnB1hPSSa6UXwUKVkguC-Yj-Q0yEQXg,26632
 zenml/orchestrators/topsort.py,sha256=D8evz3X47zwpXd90NMLsJD-_uCeXtV6ClzNfDUrq7cM,5784
 zenml/orchestrators/utils.py,sha256=6bqLc1fmdJTXg8JUwUKs8YNbmxTuMIfWmUbUpg-7hx0,12956
@@ -703,8 +703,8 @@ zenml/pipelines/__init__.py,sha256=hpIX7hN8jsQRHT5R-xSXZL88qrHwkmrvGLQeu1rWt4o,8
 zenml/pipelines/build_utils.py,sha256=DltGesybT8qYum4i23mvWZlVRgp7UxWdbHd1Y9ySv5c,27889
 zenml/pipelines/pipeline_context.py,sha256=4BixReLcPo33VtNBDrMwnJqjKTinHjmO5AOfmoeIOQM,3659
 zenml/pipelines/pipeline_decorator.py,sha256=LB21QYrbFeBdUGwKBUNbdpXAxO4OOtYl5Vs_mzJNXqU,4600
-zenml/pipelines/pipeline_definition.py,sha256=CVZDquIjq8WaiRwJuckTH_SxLnkR08g1FdZxCsJlcUU,59634
-zenml/pipelines/run_utils.py,sha256=-f4Q32YM0qM3A678qU2L9PcyqhEm6gz2bIg5BRjCmjg,12562
+zenml/pipelines/pipeline_definition.py,sha256=cQUeUcv6l9quw8xkYFA4YFNZcG_AzLHJArUXnxkFXUI,59759
+zenml/pipelines/run_utils.py,sha256=VAjfdu300AKfTuwXll3uoFrwN5dt_hesXxtylndUraQ,12515
 zenml/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 zenml/plugins/base_plugin_flavor.py,sha256=88IxFW91UB_rQ8xPlfRnIhIJh7A308NEq2epMMdlOng,2530
 zenml/plugins/plugin_flavor_registry.py,sha256=LsN2Q0K-7EQ9H4uvlEG62Y0C1_Ro1UwppX4cnGbEcOA,10862
@@ -1075,7 +1075,7 @@ zenml/zen_server/routers/webhook_endpoints.py,sha256=4Ca6k_qyE5lZpflEqV0P4mcuXnh
 zenml/zen_server/secure_headers.py,sha256=glh6QujnjyeoH1_FK-tAS-105G-qKS_34AqSzqJ6TRc,4182
 zenml/zen_server/template_execution/__init__.py,sha256=79knXLKfegsvVSVSWecpqrepq6iAavTUA4hKuiDk-WE,613
 zenml/zen_server/template_execution/runner_entrypoint_configuration.py,sha256=Y8aYJhqqs8Kv8I1q-dM1WemS5VBIfyoaaYH_YkzC7iY,1541
-zenml/zen_server/template_execution/utils.py,sha256=DIdi9ipUMT2XkqunBXk5eNUR_qXqO7Bs1OOdWlkoab0,19413
+zenml/zen_server/template_execution/utils.py,sha256=xaxG2gAaWDSFHH-R0-P_YxwVPnrZzw8imL0VM0I_LTA,19386
 zenml/zen_server/template_execution/workload_manager_interface.py,sha256=CL9c7z8ajuZE01DaHmdCDCZmsroDcTarvN-nE8jv6qQ,2590
 zenml/zen_server/utils.py,sha256=BKwaSRRWpYUaItCH4xwXOfG3JmsuFuf6Oi1B6dms0ps,24657
 zenml/zen_server/zen_server_api.py,sha256=bXriGA06xKe_Pz-Adw-dzJX9yWpQbS5Hqxbmg_7FQSA,11603
@@ -1308,7 +1308,7 @@ zenml/zen_stores/schemas/logs_schemas.py,sha256=qv6fs3JiVgzlmTXJqb_gG5NsU5q_50e0
 zenml/zen_stores/schemas/model_schemas.py,sha256=cDhWggrn3rTjaFML3iQGuW_4CpUJUxAnHieiY-dq0y4,26006
 zenml/zen_stores/schemas/pipeline_build_schemas.py,sha256=8GMdJNNcoSnbYH9daVr8zrhwQ1n-HZrpgzHoBZ7DZyA,7320
 zenml/zen_stores/schemas/pipeline_deployment_schemas.py,sha256=wCZVo8khyMOPMcO9e1itAb_3ehWFObCpgl6Pp2Yz88k,14780
-zenml/zen_stores/schemas/pipeline_run_schemas.py,sha256=k7E9VVJIQ9uYO9tR2pyATuX_F8HrPvi0uvhZ-H91Sbg,21361
+zenml/zen_stores/schemas/pipeline_run_schemas.py,sha256=nL4E5LMbB2jl_wOG6EbJp2XkaJNL_p4HkAVjoZNpmsU,21974
 zenml/zen_stores/schemas/pipeline_schemas.py,sha256=xgioTeBuFFFDOJi5eESx2j-8mW55B6hshosFylev5Mw,8213
 zenml/zen_stores/schemas/project_schemas.py,sha256=X2GClPNQz0COsEZX8xI-I8Sm68Hb6f20Obm24mQyLS0,6013
 zenml/zen_stores/schemas/run_metadata_schemas.py,sha256=G94rT4ldluMSnf9rm7R_9rw_GlgaAyq72ptkHl0gHeg,3605
@@ -1334,11 +1334,11 @@ zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py,sha256=5err1a-TrV3SR5
 zenml/zen_stores/secrets_stores/secrets_store_interface.py,sha256=Q2Jbnt2Pp7NGlR-u1YBfRZV2g8su2Fd0ArBMdksAE-Q,2819
 zenml/zen_stores/secrets_stores/service_connector_secrets_store.py,sha256=DrXGMkBxQIy2n_kkux5Xh2OM3Ks3MOpqP1D4aY8bfyY,7047
 zenml/zen_stores/secrets_stores/sql_secrets_store.py,sha256=LPFW757WCJLP1S8vrvjsrl2Tf1yo281xUTjSBsos4qk,8788
-zenml/zen_stores/sql_zen_store.py,sha256=WSTNVWbAN9YM2e5ZmyDLvlV0__EgDlL3ZLqyYGe9Ha0,479786
+zenml/zen_stores/sql_zen_store.py,sha256=vir5WfbwPyZz7bIuxRzT5WHG_qkL5ijhWpSw4nGBfIU,481114
 zenml/zen_stores/template_utils.py,sha256=GbJ7LgGVYHSCKPEA8RNTxPoVTWqpC77F_lGzjJ4O1Fw,9220
 zenml/zen_stores/zen_store_interface.py,sha256=weiSULdI9AsbCE10a5TcwtybX-BJs9hKhjPJnTapWv4,93023
-zenml_nightly-0.83.1.dev20250627.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
-zenml_nightly-0.83.1.dev20250627.dist-info/METADATA,sha256=Bj7MlfHDc0hUbq_NSa5yD-_54c8jcHgteVC-Hz5t6bk,24316
-zenml_nightly-0.83.1.dev20250627.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-zenml_nightly-0.83.1.dev20250627.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
-zenml_nightly-0.83.1.dev20250627.dist-info/RECORD,,
+zenml_nightly-0.83.1.dev20250628.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
+zenml_nightly-0.83.1.dev20250628.dist-info/METADATA,sha256=eKK8-lKVvhMh_Sr63zGnmGUHpnH70mxMMx8-AxQDMbc,24316
+zenml_nightly-0.83.1.dev20250628.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+zenml_nightly-0.83.1.dev20250628.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
+zenml_nightly-0.83.1.dev20250628.dist-info/RECORD,,