zenml-nightly 0.80.1.dev20250407__py3-none-any.whl → 0.80.1.dev20250409__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.80.1.dev20250407
+0.80.1.dev20250409

zenml/cli/base.py

@@ -87,7 +87,7 @@ ZENML_PROJECT_TEMPLATES = dict(
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",
-        github_tag="2025.01.08",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2025.04.07",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",

zenml/cli/integration.py

@@ -13,6 +13,8 @@
 #  permissions and limitations under the License.
 """Functionality to install or uninstall ZenML integrations via the CLI."""
 
+import os
+import subprocess
 import sys
 from typing import Optional, Tuple
 
@@ -131,12 +133,20 @@ def get_requirements(integration_name: Optional[str] = None) -> None:
     "environment. This can not be specified when also providing explicit "
     "integrations.",
 )
+@click.option(
+    "--poetry",
+    "poetry",
+    is_flag=True,
+    default=False,
+    help="Add the exported requirements to your current Poetry project.",
+)
 def export_requirements(
     integrations: Tuple[str],
     ignore_integration: Tuple[str],
     output_file: Optional[str] = None,
     overwrite: bool = False,
     installed_only: bool = False,
+    poetry: bool = False,
 ) -> None:
     """Exports integration requirements so they can be installed using pip.
 
@@ -150,6 +160,7 @@ def export_requirements(
         installed_only: Only export requirements for integrations installed in
             your current environment. This can not be specified when also
             providing explicit integrations.
+        poetry: Add the exported requirements to your current Poetry project.
     """
     from zenml.integrations.registry import integration_registry
 
@@ -160,6 +171,12 @@ def export_requirements(
             "environment, not both."
         )
 
+    if poetry and output_file:
+        error(
+            "You can either specify an output file or add the requirements to "
+            "the Poetry project, not both."
+        )
+
     all_integrations = set(integration_registry.integrations.keys())
 
     if integrations:
@@ -204,6 +221,25 @@ def export_requirements(
                 with open(output_file, "w") as f:
                     f.write("\n".join(requirements))
         declare(f"Requirements exported to {output_file}.")
+    if poetry:
+        res = os.popen("poetry env list").read()
+        envs = [
+            env
+            for env in res.split("\n")
+            if env.lower().find("(activated)") > 0
+        ]
+        if len(envs) == 0:
+            error(
+                "No activated Poetry environment found. Please activate one "
+                "and try again."
+            )
+        else:
+            # Use subprocess.run with shell=False to avoid command injection
+            args = ["poetry", "add"] + requirements
+            subprocess.run(args, check=True)
+            declare(
+                f"Requirements added to `{envs[0]}` environment in Poetry."
+            )
     else:
         click.echo(" ".join(requirements), nl=False)
 

zenml/client.py

@@ -3574,6 +3574,7 @@ class Client(metaclass=ClientMetaClass):
         updated: Optional[Union[datetime, str]] = None,
         id: Optional[Union[UUID, str]] = None,
         name: Optional[str] = None,
+        hidden: Optional[bool] = False,
         tag: Optional[str] = None,
         project: Optional[Union[str, UUID]] = None,
         pipeline_id: Optional[Union[str, UUID]] = None,
@@ -3596,6 +3597,7 @@ class Client(metaclass=ClientMetaClass):
             updated: Filter by the last updated date.
             id: Filter by run template ID.
             name: Filter by run template name.
+            hidden: Filter by run template hidden status.
             tag: Filter by run template tags.
             project: Filter by project name/ID.
             pipeline_id: Filter by pipeline ID.
@@ -3620,6 +3622,7 @@ class Client(metaclass=ClientMetaClass):
             updated=updated,
             id=id,
             name=name,
+            hidden=hidden,
             tag=tag,
             project=project,
             pipeline_id=pipeline_id,
@@ -3640,6 +3643,7 @@ class Client(metaclass=ClientMetaClass):
         name_id_or_prefix: Union[str, UUID],
         name: Optional[str] = None,
         description: Optional[str] = None,
+        hidden: Optional[bool] = None,
         add_tags: Optional[List[str]] = None,
         remove_tags: Optional[List[str]] = None,
         project: Optional[Union[str, UUID]] = None,
@@ -3650,6 +3654,7 @@ class Client(metaclass=ClientMetaClass):
             name_id_or_prefix: Name/ID/ID prefix of the template to update.
             name: The new name of the run template.
             description: The new description of the run template.
+            hidden: The new hidden status of the run template.
             add_tags: Tags to add to the run template.
             remove_tags: Tags to remove from the run template.
             project: The project name/ID to filter by.
@@ -3675,6 +3680,7 @@ class Client(metaclass=ClientMetaClass):
             template_update=RunTemplateUpdate(
                 name=name,
                 description=description,
+                hidden=hidden,
                 add_tags=add_tags,
                 remove_tags=remove_tags,
             ),

zenml/config/step_configurations.py

@@ -118,8 +118,9 @@ class ArtifactConfiguration(PartialArtifactConfiguration):
     @field_validator("materializer_source", mode="before")
     @classmethod
     def _convert_source(
-        cls, value: Union[Source, Dict[str, Any], str, Tuple[Source, ...]]
-    ) -> Tuple[Source, ...]:
+        cls,
+        value: Union[None, Source, Dict[str, Any], str, Tuple[Source, ...]],
+    ) -> Optional[Tuple[Source, ...]]:
         """Converts old source strings to tuples of source objects.
 
         Args:

zenml/constants.py

@@ -196,7 +196,7 @@ ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK = (
 )
 ENV_ZENML_PREVENT_CLIENT_SIDE_CACHING = "ZENML_PREVENT_CLIENT_SIDE_CACHING"
 ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING = "DISABLE_CREDENTIALS_DISK_CACHING"
-
+ENV_ZENML_RUNNER_IMAGE_DISABLE_UV = "ZENML_RUNNER_IMAGE_DISABLE_UV"
 # Logging variables
 IS_DEBUG_ENV: bool = handle_bool_env_var(ENV_ZENML_DEBUG, default=False)
 

zenml/entrypoints/step_entrypoint_configuration.py

@@ -75,6 +75,8 @@ class StepEntrypointConfiguration(BaseEntrypointConfiguration):
             self,
             deployment: "PipelineDeployment",
             stack: "Stack",
+            environment: Dict[str, str],
+            placeholder_run: Optional["PipelineRunResponse"] = None,
         ) -> Any:
             ...
 

zenml/integrations/airflow/orchestrators/airflow_orchestrator.py

@@ -51,7 +51,11 @@ if TYPE_CHECKING:
         DagConfiguration,
         TaskConfiguration,
     )
-    from zenml.models import PipelineDeploymentResponse, ScheduleResponse
+    from zenml.models import (
+        PipelineDeploymentResponse,
+        PipelineRunResponse,
+        ScheduleResponse,
+    )
     from zenml.stack import Stack
 
 logger = get_logger(__name__)
@@ -192,6 +196,7 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Creates and writes an Airflow DAG zip file.
 
@@ -200,7 +205,7 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
-
+            placeholder_run: An optional placeholder run for the deployment.
         """
         pipeline_settings = cast(
             AirflowOrchestratorSettings, self.get_settings(deployment)

zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py

@@ -267,6 +267,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Iterator[Dict[str, MetadataType]]:
         """Prepares or runs a pipeline on Sagemaker.
 
@@ -275,6 +276,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
             stack: The stack to run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If there is an error creating or scheduling the

zenml/integrations/azure/orchestrators/azureml_orchestrator.py

@@ -203,6 +203,7 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Iterator[Dict[str, MetadataType]]:
         """Prepares or runs a pipeline on AzureML.
 
@@ -211,6 +212,7 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
             stack: The stack to run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If the creation of the schedule fails.

zenml/integrations/databricks/orchestrators/databricks_orchestrator.py

@@ -58,7 +58,7 @@ from zenml.utils.pipeline_docker_image_builder import (
 )
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 
@@ -201,6 +201,7 @@ class DatabricksOrchestrator(WheeledOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Creates a wheel and uploads the pipeline to Databricks.
 
@@ -225,6 +226,7 @@ class DatabricksOrchestrator(WheeledOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             ValueError: If the schedule is not set or if the cron expression

zenml/integrations/gcp/orchestrators/vertex_orchestrator.py

@@ -357,6 +357,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Iterator[Dict[str, MetadataType]]:
         """Creates a KFP JSON pipeline.
 
@@ -390,6 +391,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             ValueError: If the attribute `pipeline_root` is not set, and it

zenml/integrations/hyperai/orchestrators/hyperai_orchestrator.py

@@ -36,7 +36,7 @@ from zenml.orchestrators import (
 from zenml.stack import Stack, StackValidator
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
 
 logger = get_logger(__name__)
 
@@ -164,6 +164,7 @@ class HyperAIOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Sequentially runs all pipeline steps in Docker containers.
 
@@ -182,6 +183,7 @@ class HyperAIOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If a step fails.

zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py

@@ -75,7 +75,7 @@ from zenml.stack import StackValidator
 from zenml.utils import io_utils, settings_utils, yaml_utils
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 
@@ -471,6 +471,7 @@ class KubeflowOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Creates a kfp yaml file.
 
@@ -501,6 +502,7 @@ class KubeflowOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If trying to run a pipeline in a notebook

zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py

@@ -47,6 +47,14 @@ class KubernetesOrchestratorSettings(BaseSettings):
         pod_settings: Pod settings to apply to pods executing the steps.
         orchestrator_pod_settings: Pod settings to apply to the pod which is
             launching the actual steps.
+        pod_startup_timeout: The maximum time to wait for a pending step pod to
+            start (in seconds).
+        pod_failure_max_retries: The maximum number of times to retry a step
+            pod if the step Kubernetes pod fails to start
+        pod_failure_retry_delay: The delay in seconds between pod
+            failure retries and pod startup retries (in seconds)
+        pod_failure_backoff: The backoff factor for pod failure retries and
+            pod startup retries.
     """
 
     synchronous: bool = True
@@ -56,6 +64,10 @@ class KubernetesOrchestratorSettings(BaseSettings):
     privileged: bool = False
     pod_settings: Optional[KubernetesPodSettings] = None
     orchestrator_pod_settings: Optional[KubernetesPodSettings] = None
+    pod_startup_timeout: int = 60 * 10  # Default 10 minutes
+    pod_failure_max_retries: int = 3
+    pod_failure_retry_delay: int = 10
+    pod_failure_backoff: float = 1.0
 
 
 class KubernetesOrchestratorConfig(

zenml/integrations/kubernetes/orchestrators/kube_utils.py

@@ -264,6 +264,9 @@ def wait_pod(
 
         resp = get_pod(core_api, pod_name, namespace)
 
+        if resp is None:
+            raise RuntimeError(f"Pod `{namespace}:{pod_name}` not found.")
+
         # Stream logs to `zenml.logger.info()`.
         # TODO: can we do this without parsing all logs every time?
         if stream_logs and pod_is_not_pending(resp):

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py

@@ -67,7 +67,7 @@ from zenml.orchestrators.utils import get_orchestrator_run_name
 from zenml.stack import StackValidator
 
 if TYPE_CHECKING:
-    from zenml.models import PipelineDeploymentResponse
+    from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
     from zenml.stack import Stack
 
 logger = get_logger(__name__)
@@ -393,6 +393,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
         deployment: "PipelineDeploymentResponse",
         stack: "Stack",
         environment: Dict[str, str],
+        placeholder_run: Optional["PipelineRunResponse"] = None,
     ) -> Any:
         """Runs the pipeline in Kubernetes.
 
@@ -401,6 +402,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
             stack: The stack the pipeline will run on.
             environment: Environment variables to set in the orchestration
                 environment.
+            placeholder_run: An optional placeholder run for the deployment.
 
         Raises:
             RuntimeError: If the Kubernetes orchestrator is not configured.
@@ -450,6 +452,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
             run_name=orchestrator_run_name,
             deployment_id=deployment.id,
             kubernetes_namespace=self.config.kubernetes_namespace,
+            run_id=placeholder_run.id if placeholder_run else None,
         )
 
         settings = cast(

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py

@@ -15,6 +15,9 @@
 
 import argparse
 import socket
+import time
+from typing import Any, Dict
+from uuid import UUID
 
 from kubernetes import client as k8s_client
 
@@ -22,6 +25,8 @@ from zenml.client import Client
 from zenml.entrypoints.step_entrypoint_configuration import (
     StepEntrypointConfiguration,
 )
+from zenml.enums import ExecutionStatus
+from zenml.exceptions import AuthorizationException
 from zenml.integrations.kubernetes.flavors.kubernetes_orchestrator_flavor import (
     KubernetesOrchestratorSettings,
 )
@@ -35,7 +40,8 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
     build_pod_manifest,
 )
 from zenml.logger import get_logger
-from zenml.orchestrators.dag_runner import ThreadedDagRunner
+from zenml.orchestrators import publish_utils
+from zenml.orchestrators.dag_runner import NodeStatus, ThreadedDagRunner
 from zenml.orchestrators.utils import get_config_environment_vars
 
 logger = get_logger(__name__)
@@ -51,6 +57,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--run_name", type=str, required=True)
     parser.add_argument("--deployment_id", type=str, required=True)
     parser.add_argument("--kubernetes_namespace", type=str, required=True)
+    parser.add_argument("--run_id", type=str, required=False)
     return parser.parse_args()
 
 
@@ -64,7 +71,9 @@ def main() -> None:
 
     orchestrator_run_id = socket.gethostname()
 
-    deployment_config = Client().get_deployment(args.deployment_id)
+    client = Client()
+
+    deployment_config = client.get_deployment(args.deployment_id)
 
     pipeline_dag = {
         step_name: step.spec.upstream_steps
@@ -72,7 +81,7 @@ def main() -> None:
     }
     step_command = StepEntrypointConfiguration.get_entrypoint_command()
 
-    active_stack = Client().active_stack
+    active_stack = client.active_stack
     mount_local_stores = active_stack.orchestrator.config.is_local
 
     # Get a Kubernetes client from the active Kubernetes orchestrator, but
@@ -91,6 +100,11 @@ def main() -> None:
 
         Args:
             step_name: Name of the step.
+
+        Raises:
+            Exception: If the pod fails to start.
+            TimeoutError: If the pod is still in a pending state after the
+                maximum wait time has elapsed.
         """
         # Define Kubernetes pod name.
         pod_name = f"{orchestrator_run_id}-{step_name}"
@@ -162,24 +176,154 @@ def main() -> None:
             mount_local_stores=mount_local_stores,
         )
 
-        # Create and run pod.
-        core_api.create_namespaced_pod(
-            namespace=args.kubernetes_namespace,
-            body=pod_manifest,
-        )
+        retries = 0
+        max_retries = settings.pod_failure_max_retries
+        delay: float = settings.pod_failure_retry_delay
+        backoff = settings.pod_failure_backoff
+
+        while retries < max_retries:
+            try:
+                # Create and run pod.
+                core_api.create_namespaced_pod(
+                    namespace=args.kubernetes_namespace,
+                    body=pod_manifest,
+                )
+                break
+            except Exception as e:
+                retries += 1
+                if retries < max_retries:
+                    logger.debug(
+                        f"Pod for step `{step_name}` failed to start: {e}"
+                    )
+                    logger.error(
+                        f"Failed to create pod for step `{step_name}`. "
+                        f"Retrying in {delay} seconds..."
+                    )
+                    time.sleep(delay)
+                    delay *= backoff
+                else:
+                    logger.error(
+                        f"Failed to create pod for step `{step_name}` after "
+                        f"{max_retries} retries. Exiting."
+                    )
+                    raise
+
+        # Wait for pod to start
+        max_wait = settings.pod_startup_timeout
+        total_wait: float = 0
+        delay = settings.pod_failure_retry_delay
+        while True:
+            pod = kube_utils.get_pod(
+                core_api, pod_name, args.kubernetes_namespace
+            )
+            if not pod or kube_utils.pod_is_not_pending(pod):
+                break
+            if total_wait >= max_wait:
+                # Have to delete the pending pod so it doesn't start running
+                # later on.
+                try:
+                    core_api.delete_namespaced_pod(
+                        name=pod_name,
+                        namespace=args.kubernetes_namespace,
+                    )
+                except Exception:
+                    pass
+                raise TimeoutError(
+                    f"Pod for step `{step_name}` is still in a pending state "
+                    f"after {total_wait} seconds. Exiting."
+                )
+
+            if total_wait + delay > max_wait:
+                delay = max_wait - total_wait
+            total_wait += delay
+            time.sleep(delay)
+            delay *= backoff
 
         # Wait for pod to finish.
-        logger.info(f"Waiting for pod of step `{step_name}` to start...")
-        kube_utils.wait_pod(
-            kube_client_fn=lambda: orchestrator.get_kube_client(
-                incluster=True
-            ),
-            pod_name=pod_name,
-            namespace=args.kubernetes_namespace,
-            exit_condition_lambda=kube_utils.pod_is_done,
-            stream_logs=True,
-        )
-        logger.info(f"Pod of step `{step_name}` completed.")
+        logger.info(f"Waiting for pod of step `{step_name}` to finish...")
+        try:
+            kube_utils.wait_pod(
+                kube_client_fn=lambda: orchestrator.get_kube_client(
+                    incluster=True
+                ),
+                pod_name=pod_name,
+                namespace=args.kubernetes_namespace,
+                exit_condition_lambda=kube_utils.pod_is_done,
+                stream_logs=True,
+            )
+
+            logger.info(f"Pod for step `{step_name}` completed.")
+        except Exception:
+            logger.error(f"Pod for step `{step_name}` failed.")
+
+            raise
+
+    def finalize_run(node_states: Dict[str, NodeStatus]) -> None:
+        """Finalize the run.
+
+        Args:
+            node_states: The states of the nodes.
+        """
+        try:
+            # Some steps may have failed because the pods could not be created.
+            # We need to check for this and mark the step run as failed if so.
+
+            # Fetch the pipeline run using any means possible.
+            list_args: Dict[str, Any] = {}
+            if args.run_id:
+                # For a run triggered outside of a schedule, we can use the
+                # placeholder run ID to find the pipeline run.
+                list_args = dict(id=UUID(args.run_id))
+            else:
+                # For a run triggered by a schedule, we can only use the
+                # orchestrator run ID to find the pipeline run.
+                list_args = dict(orchestrator_run_id=orchestrator_run_id)
+
+            pipeline_runs = client.list_pipeline_runs(
+                hydrate=True,
+                project=deployment_config.project.id,
+                deployment_id=deployment_config.id,
+                **list_args,
+            )
+            if not len(pipeline_runs):
+                # No pipeline run found, so we can't mark any step runs as failed.
+                return
+
+            pipeline_run = pipeline_runs[0]
+            pipeline_failed = False
+
+            for step_name, node_state in node_states.items():
+                if node_state != NodeStatus.FAILED:
+                    continue
+
+                pipeline_failed = True
+
+                # If steps failed for any reason, we need to mark the step run as
+                # failed, if it exists and it wasn't already in a final state.
+
+                step_run = pipeline_run.steps.get(step_name)
+
+                # Try to update the step run status, if it exists and is in
+                # a transient state.
+                if step_run and step_run.status in {
+                    ExecutionStatus.INITIALIZING,
+                    ExecutionStatus.RUNNING,
+                }:
+                    publish_utils.publish_failed_step_run(step_run.id)
+
+            # If any steps failed and the pipeline run is still in a transient
+            # state, we need to mark it as failed.
+            if pipeline_failed and pipeline_run.status in {
+                ExecutionStatus.INITIALIZING,
+                ExecutionStatus.RUNNING,
+            }:
+                publish_utils.publish_failed_pipeline_run(pipeline_run.id)
+        except AuthorizationException:
+            # If a step of the pipeline failed or all of them completed
+            # successfully, the pipeline run will be finished and the API token
+            # will be invalidated. We catch this exception and do nothing here,
+            # as the pipeline run status will already have been published.
+            pass
 
     parallel_node_startup_waiting_period = (
         orchestrator.config.parallel_step_startup_waiting_period or 0.0
@@ -188,6 +332,7 @@ def main() -> None:
         ThreadedDagRunner(
             dag=pipeline_dag,
             run_fn=run_step_on_kubernetes,
+            finalize_fn=finalize_run,
             parallel_node_startup_waiting_period=parallel_node_startup_waiting_period,
         ).run()
         logger.info("Orchestration pod completed.")