zenml-nightly 0.84.0.dev20250728__py3-none-any.whl → 0.84.0.dev20250729__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.84.0.dev20250728
+0.84.0.dev20250729

zenml/code_repositories/git/local_git_repository_context.py

@@ -13,7 +13,7 @@
 #  permissions and limitations under the License.
 """Implementation of the Local git repository context."""
 
-from typing import TYPE_CHECKING, Callable, Optional, cast
+from typing import TYPE_CHECKING, Callable, Optional
 
 from zenml.code_repositories import (
     LocalRepositoryContext,
@@ -25,7 +25,6 @@ from zenml.constants import (
 from zenml.logger import get_logger
 
 if TYPE_CHECKING:
-    from git.objects import Commit
     from git.remote import Remote
     from git.repo.base import Repo
 
@@ -180,7 +179,7 @@ class LocalGitRepositoryContext(LocalRepositoryContext):
             # Branch doesn't exist on remote
             return True
 
-        return cast("Commit", remote_commit_object) != local_commit_object
+        return remote_commit_object != local_commit_object
 
     @property
     def current_commit(self) -> str:
@@ -189,4 +188,4 @@ class LocalGitRepositoryContext(LocalRepositoryContext):
         Returns:
             The current commit sha.
         """
-        return cast(str, self.git_repo.head.object.hexsha)
+        return self.git_repo.head.object.hexsha

zenml/constants.py

@@ -181,6 +181,9 @@ ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION = (
 ENV_ZENML_CODE_REPOSITORY_IGNORE_UNTRACKED_FILES = (
     "ZENML_CODE_REPOSITORY_IGNORE_UNTRACKED_FILES"
 )
+# Environment variable that indicates whether the current environment is running
+# a step operator.
+ENV_ZENML_STEP_OPERATOR = "ZENML_STEP_OPERATOR"
 
 # Materializer environment variables
 ENV_ZENML_MATERIALIZER_ALLOW_NON_ASCII_JSON_DUMPS = (

zenml/integrations/azure/service_connectors/azure_service_connector.py

@@ -1490,7 +1490,8 @@ class AzureServiceConnector(ServiceConnector):
                 aks_clusters = [
                     cluster
                     for cluster in aks_clusters
-                    if self._get_resource_group(cluster.id)
+                    if cluster.id
+                    and self._get_resource_group(cluster.id)
                     == self.config.resource_group
                 ]
 
@@ -1507,7 +1508,8 @@ class AzureServiceConnector(ServiceConnector):
             clusters = [
                 (cluster.name, self._get_resource_group(cluster.id))
                 for cluster in aks_clusters
-                if cluster.name
+                if cluster.id
+                and cluster.name
                 and (not cluster_name or cluster.name == cluster_name)
             ]
 
@@ -1809,16 +1811,25 @@ class AzureServiceConnector(ServiceConnector):
             cluster_name, resource_group = clusters[0]
 
             try:
-                client = ContainerServiceClient(credential, subscription_id)
+                cs_client = ContainerServiceClient(credential, subscription_id)
 
-                creds = client.managed_clusters.list_cluster_admin_credentials(
-                    resource_group_name=resource_group,
-                    resource_name=cluster_name,
+                creds = (
+                    cs_client.managed_clusters.list_cluster_admin_credentials(
+                        resource_group_name=resource_group,
+                        resource_name=cluster_name,
+                    )
                 )
 
-                kubeconfig_yaml = creds.kubeconfigs[0].value.decode(
-                    encoding="UTF-8"
-                )
+                if creds.kubeconfigs and creds.kubeconfigs[0].value:
+                    kubeconfig_yaml = creds.kubeconfigs[0].value.decode(
+                        encoding="UTF-8"
+                    )
+                else:
+                    raise AuthorizationException(
+                        f"failed to list credentials for Azure Kubernetes "
+                        f"Service cluster '{cluster_name}' in resource group "
+                        f"'{resource_group}': no kubeconfig found"
+                    )
             except AzureError as e:
                 raise AuthorizationException(
                     f"failed to list credentials for Azure Kubernetes "

zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py

@@ -13,7 +13,7 @@
 #  permissions and limitations under the License.
 """Kubernetes orchestrator flavor."""
 
-from typing import TYPE_CHECKING, Any, Dict, Optional, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
 
 from pydantic import Field, NonNegativeInt, PositiveInt, field_validator
 
@@ -131,6 +131,25 @@ class KubernetesOrchestratorSettings(BaseSettings):
         "the chance of the server receiving the maximum amount of retry "
         "requests.",
     )
+    fail_on_container_waiting_reasons: Optional[List[str]] = Field(
+        default=[
+            "InvalidImageName",
+            "ErrImagePull",
+            "ImagePullBackOff",
+            "CreateContainerConfigError",
+        ],
+        description="List of container waiting reasons that should cause the "
+        "job to fail immediately. This should be set to a list of "
+        "nonrecoverable reasons, which if found in any "
+        "`pod.status.containerStatuses[*].state.waiting.reason` of a job pod, "
+        "should cause the job to fail immediately.",
+    )
+    job_monitoring_interval: int = Field(
+        default=3,
+        description="The interval in seconds to monitor the job. Each interval "
+        "is used to check for container issues and streaming logs for the "
+        "job pods.",
+    )
     pod_failure_policy: Optional[Dict[str, Any]] = Field(
         default=None,
         description="The pod failure policy to use for the job that is "

zenml/integrations/kubernetes/orchestrators/kube_utils.py

@@ -37,7 +37,7 @@ import json
 import re
 import time
 from collections import defaultdict
-from typing import Any, Callable, Dict, List, Optional, TypeVar, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast
 
 from kubernetes import client as k8s_client
 from kubernetes import config as k8s_config
@@ -657,6 +657,50 @@ def create_job(
     )
 
 
+def get_container_status(
+    pod: k8s_client.V1Pod, container_name: str
+) -> Optional[k8s_client.V1ContainerState]:
+    """Get the status of a container.
+
+    Args:
+        pod: The pod to get the container status for.
+        container_name: The container name.
+
+    Returns:
+        The container status.
+    """
+    if not pod.status or not pod.status.container_statuses:
+        return None
+
+    for container_status in pod.status.container_statuses:
+        if container_status.name == container_name:
+            return container_status.state
+
+    return None
+
+
+def get_container_termination_reason(
+    pod: k8s_client.V1Pod, container_name: str
+) -> Optional[Tuple[int, str]]:
+    """Get the termination reason for a container.
+
+    Args:
+        pod: The pod to get the termination reason for.
+        container_name: The container name.
+
+    Returns:
+        The exit code and termination reason for the container.
+    """
+    container_state = get_container_status(pod, container_name)
+    if not container_state or not container_state.terminated:
+        return None
+
+    return (
+        container_state.terminated.exit_code,
+        container_state.terminated.reason or "Unknown",
+    )
+
+
 def wait_for_job_to_finish(
     batch_api: k8s_client.BatchV1Api,
     core_api: k8s_client.CoreV1Api,
@@ -665,8 +709,9 @@ def wait_for_job_to_finish(
     backoff_interval: float = 1,
     maximum_backoff: float = 32,
     exponential_backoff: bool = False,
-    container_name: Optional[str] = None,
+    fail_on_container_waiting_reasons: Optional[List[str]] = None,
     stream_logs: bool = True,
+    container_name: Optional[str] = None,
 ) -> None:
     """Wait for a job to finish.
 
@@ -679,6 +724,8 @@ def wait_for_job_to_finish(
         maximum_backoff: The maximum interval to wait between polling the job
             status.
         exponential_backoff: Whether to use exponential backoff.
+        fail_on_container_waiting_reasons: List of container waiting reasons
+            that will cause the job to fail.
         stream_logs: Whether to stream the job logs.
         container_name: Name of the container to stream logs from.
 
@@ -703,9 +750,39 @@ def wait_for_job_to_finish(
                         f"{condition.message}"
                     )
 
+        if fail_on_container_waiting_reasons:
+            pod_list: k8s_client.V1PodList = retry_on_api_exception(
+                core_api.list_namespaced_pod
+            )(
+                namespace=namespace,
+                label_selector=f"job-name={job_name}",
+                field_selector="status.phase=Pending",
+            )
+            for pod in pod_list.items:
+                container_state = get_container_status(
+                    pod, container_name or "main"
+                )
+
+                if (
+                    container_state
+                    and (waiting_state := container_state.waiting)
+                    and waiting_state.reason
+                    in fail_on_container_waiting_reasons
+                ):
+                    retry_on_api_exception(batch_api.delete_namespaced_job)(
+                        name=job_name,
+                        namespace=namespace,
+                        propagation_policy="Foreground",
+                    )
+                    raise RuntimeError(
+                        f"Job `{namespace}:{job_name}` failed: "
+                        f"Detected container in state "
+                        f"{waiting_state.reason}"
+                    )
+
         if stream_logs:
             try:
-                pod_list: k8s_client.V1PodList = core_api.list_namespaced_pod(
+                pod_list = core_api.list_namespaced_pod(
                     namespace=namespace,
                     label_selector=f"job-name={job_name}",
                 )

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py

@@ -367,33 +367,37 @@ def main() -> None:
                     core_api=core_api,
                     namespace=namespace,
                     job_name=job_name,
+                    fail_on_container_waiting_reasons=settings.fail_on_container_waiting_reasons,
                     stream_logs=pipeline_settings.stream_step_logs,
+                    backoff_interval=settings.job_monitoring_interval,
                 )
 
                 logger.info(f"Job for step `{step_name}` completed.")
             except Exception:
-                reason = ""
+                reason = "Unknown"
                 try:
-                    pods_by_job = core_api.list_namespaced_pod(
+                    pods = core_api.list_namespaced_pod(
                         label_selector=f"job-name={job_name}",
                         namespace=namespace,
                     ).items
-                    first_pod = pods_by_job[0]
-                    if status := first_pod.status:
-                        if container_statuses := status.container_statuses:
-                            for cs in container_statuses:
-                                if cs.name == "main":
-                                    if state := cs.state:
-                                        if terminated := state.terminated:
-                                            if terminated.exit_code != 0:
-                                                reason = f"{terminated.reason}(exit_code={terminated.exit_code})"
-                                    break
+                    # Sort pods by creation timestamp, oldest first
+                    pods.sort(
+                        key=lambda pod: pod.metadata.creation_timestamp,
+                    )
+                    if pods:
+                        if (
+                            termination_reason
+                            := kube_utils.get_container_termination_reason(
+                                pods[-1], "main"
+                            )
+                        ):
+                            exit_code, reason = termination_reason
+                            if exit_code != 0:
+                                reason = f"{reason} (exit_code={exit_code})"
                 except Exception:
                     pass
                 logger.error(
                     f"Job for step `{step_name}` failed. Reason: {reason}"
-                    if reason
-                    else ""
                 )
 
                 raise

zenml/models/__init__.py

@@ -435,6 +435,7 @@ from zenml.models.v2.misc.info_models import (
     ServiceConnectorResourcesInfo,
     ResourcesInfo,
 )
+from zenml.models.v2.misc.exception_info import ExceptionInfo
 
 # ----------------------------- Forward References -----------------------------
 
@@ -840,4 +841,5 @@ __all__ = [
     "RunMetadataResource",
     "ProjectStatistics",
     "PipelineRunDAG",
+    "ExceptionInfo",
 ]

zenml/models/v2/core/step_run.py

@@ -48,6 +48,7 @@ from zenml.models.v2.base.scoped import (
 )
 from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
 from zenml.models.v2.core.model_version import ModelVersionResponse
+from zenml.models.v2.misc.exception_info import ExceptionInfo
 
 if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
@@ -143,6 +144,10 @@ class StepRunRequest(ProjectScopedRequest):
         title="Logs associated with this step run.",
         default=None,
     )
+    exception_info: Optional[ExceptionInfo] = Field(
+        default=None,
+        title="The exception information of the step run.",
+    )
 
     model_config = ConfigDict(protected_namespaces=())
 
@@ -169,6 +174,10 @@ class StepRunUpdate(BaseUpdate):
         title="The end time of the step run.",
         default=None,
     )
+    exception_info: Optional[ExceptionInfo] = Field(
+        default=None,
+        title="The exception information of the step run.",
+    )
     model_config = ConfigDict(protected_namespaces=())
 
 
@@ -237,6 +246,10 @@ class StepRunResponseMetadata(ProjectScopedResponseMetadata):
         default=None,
         max_length=TEXT_FIELD_MAX_LENGTH,
     )
+    exception_info: Optional[ExceptionInfo] = Field(
+        default=None,
+        title="The exception information of the step run.",
+    )
 
     # References
     logs: Optional["LogsResponse"] = Field(

zenml/models/v2/misc/exception_info.py

@@ -0,0 +1,30 @@
+#  Copyright (c) ZenML GmbH 2025. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Exception information models."""
+
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ExceptionInfo(BaseModel):
+    """Exception information."""
+
+    traceback: str = Field(
+        title="The traceback of the exception.",
+    )
+    step_code_line: Optional[int] = Field(
+        default=None,
+        title="The line number of the step code that raised the exception.",
+    )

zenml/orchestrators/publish_utils.py

@@ -13,12 +13,14 @@
 #  permissions and limitations under the License.
 """Utilities to publish pipeline and step runs."""
 
+from contextvars import ContextVar
 from datetime import datetime
 from typing import TYPE_CHECKING, Dict, List, Optional
 
 from zenml.client import Client
 from zenml.enums import ExecutionStatus, MetadataResourceTypes
 from zenml.models import (
+    ExceptionInfo,
     PipelineRunResponse,
     PipelineRunUpdate,
     RunMetadataResource,
@@ -32,6 +34,10 @@ if TYPE_CHECKING:
 
     from zenml.metadata.metadata_types import MetadataType
 
+step_exception_info: ContextVar[Optional[ExceptionInfo]] = ContextVar(
+    "step_exception_info", default=None
+)
+
 
 def publish_successful_step_run(
     step_run_id: "UUID", output_artifact_ids: Dict[str, List["UUID"]]
@@ -59,6 +65,7 @@ def publish_step_run_status_update(
     step_run_id: "UUID",
     status: "ExecutionStatus",
     end_time: Optional[datetime] = None,
+    exception_info: Optional[ExceptionInfo] = None,
 ) -> "StepRunResponse":
     """Publishes a step run update.
 
@@ -66,6 +73,7 @@ def publish_step_run_status_update(
         step_run_id: ID of the step run.
         status: New status of the step run.
         end_time: New end time of the step run.
+        exception_info: Exception information of the step run.
 
     Returns:
         The updated step run.
@@ -83,6 +91,7 @@ def publish_step_run_status_update(
         step_run_update=StepRunUpdate(
             status=status,
             end_time=end_time,
+            exception_info=exception_info,
         ),
     )
 
@@ -102,6 +111,7 @@ def publish_failed_step_run(step_run_id: "UUID") -> "StepRunResponse":
         step_run_id=step_run_id,
         status=ExecutionStatus.FAILED,
         end_time=utc_now(),
+        exception_info=step_exception_info.get(),
     )
 
 

zenml/orchestrators/step_launcher.py

@@ -24,6 +24,7 @@ from zenml.config.step_configurations import Step
 from zenml.config.step_run_info import StepRunInfo
 from zenml.constants import (
     ENV_ZENML_DISABLE_STEP_LOGS_STORAGE,
+    ENV_ZENML_STEP_OPERATOR,
     handle_bool_env_var,
 )
 from zenml.enums import ExecutionStatus
@@ -43,7 +44,7 @@ from zenml.orchestrators import output_utils, publish_utils, step_run_utils
 from zenml.orchestrators import utils as orchestrator_utils
 from zenml.orchestrators.step_runner import StepRunner
 from zenml.stack import Stack
-from zenml.utils import string_utils
+from zenml.utils import exception_utils, string_utils
 from zenml.utils.time_utils import utc_now
 
 if TYPE_CHECKING:
@@ -211,7 +212,9 @@ class StepLauncher:
 
         Raises:
             RunStoppedException: If the pipeline run is stopped by the user.
+            BaseException: If the step preparation or execution fails.
         """
+        publish_utils.step_exception_info.set(None)
         pipeline_run, run_was_created = self._create_or_reuse_run()
 
         # Enable or disable step logs storage
@@ -269,10 +272,13 @@ class StepLauncher:
 
             try:
                 request_factory.populate_request(request=step_run_request)
-            except:
+            except BaseException as e:
                 logger.exception(f"Failed preparing step `{self._step_name}`.")
                 step_run_request.status = ExecutionStatus.FAILED
                 step_run_request.end_time = utc_now()
+                step_run_request.exception_info = (
+                    exception_utils.collect_exception_information(e)
+                )
                 raise
             finally:
                 step_run = Client().zen_store.create_run_step(step_run_request)
@@ -453,6 +459,7 @@ class StepLauncher:
         environment = orchestrator_utils.get_config_environment_vars(
             pipeline_run_id=step_run_info.run_id,
         )
+        environment[ENV_ZENML_STEP_OPERATOR] = "True"
         logger.info(
             "Using step operator `%s` to run step `%s`.",
             step_operator.name,

zenml/orchestrators/step_runner.py

@@ -16,6 +16,7 @@
 
 import copy
 import inspect
+import os
 from contextlib import nullcontext
 from typing import (
     TYPE_CHECKING,
@@ -34,6 +35,7 @@ from zenml.config.step_configurations import StepConfiguration
 from zenml.config.step_run_info import StepRunInfo
 from zenml.constants import (
     ENV_ZENML_DISABLE_STEP_LOGS_STORAGE,
+    ENV_ZENML_STEP_OPERATOR,
     handle_bool_env_var,
 )
 from zenml.enums import ArtifactSaveType
@@ -41,10 +43,14 @@ from zenml.exceptions import StepInterfaceError
 from zenml.logger import get_logger
 from zenml.logging.step_logging import PipelineLogsStorageContext, redirected
 from zenml.materializers.base_materializer import BaseMaterializer
-from zenml.models.v2.core.step_run import StepRunInputResponse
+from zenml.models.v2.core.step_run import (
+    StepRunInputResponse,
+    StepRunUpdate,
+)
 from zenml.orchestrators.publish_utils import (
     publish_step_run_metadata,
     publish_successful_step_run,
+    step_exception_info,
 )
 from zenml.orchestrators.utils import (
     is_setting_enabled,
@@ -56,6 +62,7 @@ from zenml.steps.utils import (
     resolve_type_annotation,
 )
 from zenml.utils import (
+    exception_utils,
     materializer_utils,
     source_utils,
     string_utils,
@@ -193,6 +200,24 @@ class StepRunner:
                 )
             except BaseException as step_exception:  # noqa: E722
                 step_failed = True
+
+                exception_info = exception_utils.collect_exception_information(
+                    step_exception, step_instance
+                )
+
+                if ENV_ZENML_STEP_OPERATOR in os.environ:
+                    # We're running in a step operator environment, so we can't
+                    # depend on the step launcher to publish the exception info
+                    Client().zen_store.update_run_step(
+                        step_run_id=step_run_info.step_run_id,
+                        step_run_update=StepRunUpdate(
+                            exception_info=exception_info,
+                        ),
+                    )
+                else:
+                    # This will be published by the step launcher
+                    step_exception_info.set(exception_info)
+
                 if not step_run.is_retriable:
                     if (
                         failure_hook_source

zenml/utils/exception_utils.py

@@ -0,0 +1,90 @@
+#  Copyright (c) ZenML GmbH 2025. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Exception utilities."""
+
+import inspect
+import os
+import re
+import traceback
+from typing import TYPE_CHECKING, Optional
+
+from zenml.constants import MEDIUMTEXT_MAX_LENGTH
+from zenml.logger import get_logger
+from zenml.models import (
+    ExceptionInfo,
+)
+
+if TYPE_CHECKING:
+    from zenml.steps import BaseStep
+
+logger = get_logger(__name__)
+
+
+def collect_exception_information(
+    exception: BaseException, step_instance: Optional["BaseStep"] = None
+) -> ExceptionInfo:
+    """Collects the exception information.
+
+    Args:
+        exception: The exception to collect information from.
+        step_instance: The step instance that is currently running.
+
+    Returns:
+        The exception information.
+    """
+    tb = traceback.format_tb(exception.__traceback__)
+    line_number = None
+    start_index = None
+
+    if step_instance and (
+        source_file := inspect.getsourcefile(step_instance.entrypoint)
+    ):
+        try:
+            source_file = os.path.abspath(source_file)
+
+            lines, start_line = inspect.getsourcelines(
+                step_instance.entrypoint
+            )
+            end_line = start_line + len(lines)
+
+            line_pattern = re.compile(f'File "{source_file}", line (\d+),')
+
+            for index, line in enumerate(tb):
+                match = line_pattern.search(line)
+                if match:
+                    potential_line_number = int(match.group(1))
+                    if (
+                        potential_line_number >= start_line
+                        and potential_line_number <= end_line
+                    ):
+                        line_number = potential_line_number - start_line
+                        start_index = index
+                        break
+        except Exception as e:
+            logger.debug("Failed to detect step code line: %s", e)
+
+    if start_index is not None:
+        # If the code failed while executing user code, we remove the initial
+        # part of the traceback that is happening in the ZenML code.
+        tb = tb[start_index:]
+
+    tb_bytes = "\n".join(tb).encode()
+    tb_bytes = tb_bytes[:MEDIUMTEXT_MAX_LENGTH]
+
+    return ExceptionInfo(
+        # Ignore errors when decoding in case we cut off in the middle of an
+        # encoded character.
+        traceback=tb_bytes.decode(errors="ignore"),
+        step_code_line=line_number,
+    )

zenml/zen_stores/migrations/versions/d4591f95ac07_step_exception_info.py

@@ -0,0 +1,43 @@
+"""Step exception info [d4591f95ac07].
+
+Revision ID: d4591f95ac07
+Revises: 0.84.0
+Create Date: 2025-07-17 14:13:54.731842
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "d4591f95ac07"
+down_revision = "0.84.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("step_run", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "exception_info",
+                sa.String(length=16777215).with_variant(
+                    mysql.MEDIUMTEXT, "mysql"
+                ),
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("step_run", schema=None) as batch_op:
+        batch_op.drop_column("exception_info")
+
+    # ### end Alembic commands ###

zenml/zen_stores/schemas/step_run_schemas.py

@@ -34,6 +34,7 @@ from zenml.enums import (
     StepRunInputArtifactType,
 )
 from zenml.models import (
+    ExceptionInfo,
     StepRunRequest,
     StepRunResponse,
     StepRunResponseBody,
@@ -101,6 +102,14 @@ class StepRunSchema(NamedSchema, RunMetadataInterface, table=True):
             nullable=True,
         )
     )
+    exception_info: Optional[str] = Field(
+        sa_column=Column(
+            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
+                MEDIUMTEXT, "mysql"
+            ),
+            nullable=True,
+        )
+    )
 
     # Foreign keys
     original_step_run_id: Optional[UUID] = build_foreign_key_field(
@@ -310,6 +319,9 @@ class StepRunSchema(NamedSchema, RunMetadataInterface, table=True):
             source_code=request.source_code,
             version=version,
             is_retriable=is_retriable,
+            exception_info=json.dumps(request.exception_info)
+            if request.exception_info
+            else None,
         )
 
     def get_step_configuration(self) -> Step:
@@ -395,6 +407,11 @@ class StepRunSchema(NamedSchema, RunMetadataInterface, table=True):
                 code_hash=self.code_hash,
                 docstring=self.docstring,
                 source_code=self.source_code,
+                exception_info=ExceptionInfo.model_validate_json(
+                    self.exception_info
+                )
+                if self.exception_info
+                else None,
                 logs=self.logs.to_model() if self.logs else None,
                 deployment_id=self.deployment_id,
                 pipeline_run_id=self.pipeline_run_id,
@@ -458,6 +475,8 @@ class StepRunSchema(NamedSchema, RunMetadataInterface, table=True):
                 self.status = value.value
             if key == "end_time":
                 self.end_time = value
+            if key == "exception_info":
+                self.exception_info = json.dumps(value)
 
         self.updated = utc_now()
 

{zenml_nightly-0.84.0.dev20250728.dist-info → zenml_nightly-0.84.0.dev20250729.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.84.0.dev20250728
+Version: 0.84.0.dev20250729
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops

{zenml_nightly-0.84.0.dev20250728.dist-info → zenml_nightly-0.84.0.dev20250729.dist-info}/RECORD

@@ -1,5 +1,5 @@
 zenml/README.md,sha256=827dekbOWAs1BpW7VF1a4d7EbwPbjwccX-2zdXBENZo,1777
-zenml/VERSION,sha256=WRM_9Y6VMs8f5Mlv18OC4qR1DoA4AV-1RBz6HZkQ-fs,19
+zenml/VERSION,sha256=ig64qkkuwpgtDDUBRHxcjuv0CoEUTgcfKVFR9WA0Wt8,19
 zenml/__init__.py,sha256=r7JUg2SVDf_dPhS7iU6vudKusEqK4ics7_jFMZhq0o4,2731
 zenml/actions/__init__.py,sha256=mrt6wPo73iKRxK754_NqsGyJ3buW7RnVeIGXr1xEw8Y,681
 zenml/actions/base_action.py,sha256=UcaHev6BTuLDwuswnyaPjdA8AgUqB5xPZ-lRtuvf2FU,25553
@@ -60,7 +60,7 @@ zenml/client_lazy_loader.py,sha256=oyxKvBWVB7k2pHMavdhNEOfR2Vk4IS3XUu43SBzDPsI,7
 zenml/code_repositories/__init__.py,sha256=W5bDfzAG8OXIKZSV1L-VHuzMcSCYa9qzTdPb3jqfyYw,920
 zenml/code_repositories/base_code_repository.py,sha256=Id6VjbUu8N3ZpNvBGhOgbahtoMiCAtYXed3G7YQ_iAc,5225
 zenml/code_repositories/git/__init__.py,sha256=vU8UzMp8sv9n-R2r7VKa9LdQcYER6BhO4O-z8Ppa3kM,824
-zenml/code_repositories/git/local_git_repository_context.py,sha256=hfX7zVDQ27Le0qb4HKoJMcRB8YWfK8_yHPPgfh1SAS0,5848
+zenml/code_repositories/git/local_git_repository_context.py,sha256=PuD4GOnxBNtkSddc5QwMnnx3F2t4gClUeGmhfoLQHfs,5780
 zenml/code_repositories/local_repository_context.py,sha256=1VyiYkJBDVg0iGusgRQDToGRPJuu9lx7jTBDpplukDg,2816
 zenml/config/__init__.py,sha256=NIMS7QQo3-uVkNlzPvR5FbtgPdRibF2xFJB7Vr0fhCM,1826
 zenml/config/base_settings.py,sha256=itoLqc1cOwEYhgSGdZmSKSaBevQkvYH7NQh7PUamazc,1700
@@ -85,7 +85,7 @@ zenml/config/step_run_info.py,sha256=KiVRSTtKmZ1GbvseDTap2imr7XwMHD3jSFVpyLNEK1I
 zenml/config/store_config.py,sha256=Cla5p5dTB6nNlo8_OZDs9hod5hspi64vxwtZj882XgU,3559
 zenml/config/strict_base_model.py,sha256=t_ULrtJF2eW7TgyYBRobl1fscwwIZXATYky8ER97ev4,860
 zenml/console.py,sha256=A54wgNv_ntZkU4uGLox00IPVskDapH31xhtgAV5x4-E,1299
-zenml/constants.py,sha256=cHkG6v45NwK0XFblrS_Eoczfj80iVckI1W-PcsFqktU,16802
+zenml/constants.py,sha256=olos1s4CGUzUpTTkcEmwS4iR7GENdgA-B-9pQWsS8gk,16950
 zenml/container_registries/__init__.py,sha256=ZSPbBIOnzhg88kQSpYgKe_POLuru14m629665-kAVAA,2200
 zenml/container_registries/azure_container_registry.py,sha256=t1sfDa94Vzbyqtb1iPFNutJ2EXV5_p9CUNITasoiQ70,2667
 zenml/container_registries/base_container_registry.py,sha256=5e-49puqeNgGdsLbfALoV-J8AgnriSvD-DDwBCPCiCs,9709
@@ -166,7 +166,7 @@ zenml/integrations/azure/orchestrators/__init__.py,sha256=q4rBPIJHcuUr6dLUBdrTkQ
 zenml/integrations/azure/orchestrators/azureml_orchestrator.py,sha256=zWtRz_ymW6CZfwgcLYUGLCDvWF_SUvgB9jBPkc192A0,21072
 zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py,sha256=lacOyorsHa-HuD_kxN9M6BUiZDvs5jL9AnJWwrFCtp4,3104
 zenml/integrations/azure/service_connectors/__init__.py,sha256=yMz6bTCtIZqZwfEM6h7-PSWsd_DB8l0OD9z_bdzomZw,793
-zenml/integrations/azure/service_connectors/azure_service_connector.py,sha256=4Qma8FyKpZ5GujIa-RrGU2Lh80j9GHEyBwwDfyK7aAI,81344
+zenml/integrations/azure/service_connectors/azure_service_connector.py,sha256=B9YBr-5j2D3UZ6Lo4e7aNMy5Os35W7dR-bno0QUMfUE,81865
 zenml/integrations/azure/step_operators/__init__.py,sha256=fV4_nAO0cH53x6_-F8-CbDEZwb_Vv64oq1r0-vtigEU,819
 zenml/integrations/azure/step_operators/azureml_step_operator.py,sha256=bi6cl1PimxZF6D7PVZepC6OqHdGHTcsNGBe0njuiBCM,7397
 zenml/integrations/bentoml/__init__.py,sha256=vOlziqJ038cS7_ZtbGojWpefdQDigq8mLDALVC_lgbY,1835
@@ -334,12 +334,12 @@ zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py,sha256=lPQSQn
 zenml/integrations/kubeflow/orchestrators/local_deployment_utils.py,sha256=qszoOdvBpgIp40XkncphXAr9dRKnyZzGiz2mJ56bYmw,15448
 zenml/integrations/kubernetes/__init__.py,sha256=k1bfrdI1u5RBnAh7yT4w-m-4SWgZ7b4L5uu7dPRc0SI,1809
 zenml/integrations/kubernetes/flavors/__init__.py,sha256=a5gU45qCj3FkLwl_uVjlIkL_2F5DHk-w1gdcZrvVjBI,1266
-zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py,sha256=MqPOFJpxQ1hfzF3c_FU_431OJGHOVKT_VNwdAzNu1lc,13595
+zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py,sha256=zhcn5d3N4VmlVq-cxtyXtp63AKcwVfKWg-jAlkx2B10,14420
 zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py,sha256=Vp7kY9t2TWdWRBNSf_sJKPL5dZnhuHnskPhQ7KBuGPY,6384
 zenml/integrations/kubernetes/orchestrators/__init__.py,sha256=TJID3OTieZBox36WpQpzD0jdVRA_aZVcs_bNtfXS8ik,811
-zenml/integrations/kubernetes/orchestrators/kube_utils.py,sha256=9oPdp1jREPOn2NAu_mmdxuFeYjMartj3cWYqzb808qw,25112
+zenml/integrations/kubernetes/orchestrators/kube_utils.py,sha256=Gbw1mT0z-F_t2iQKPmLntGhdNTT-CZNHO5sDb-XxE68,27709
 zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py,sha256=Agx_Thj6nByAd5p61HTf0p6SRDPOcYKSSl-_Km2EdfU,41316
-zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py,sha256=8XZZMJB8E4HnXSUC436VRKMRJHCI57YK7a892xYGTSo,20675
+zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py,sha256=jgxnJdBVFVn6l6gSoygIoScQxyKh5cVbgXF10qFA9Nk,20809
 zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint_configuration.py,sha256=QOwQnWCfB-t_BQ2eOZN0SakurGUd0GTMCSdUlREhk6I,2324
 zenml/integrations/kubernetes/orchestrators/manifest_utils.py,sha256=52Ba6MZ4lCo6CpnW0IjnFOJQNpPoVgbvIqR-x32ubew,16085
 zenml/integrations/kubernetes/pod_settings.py,sha256=s6I0sALXB7PnRqJ-nlU71o3cTm2GM1TfqsE44qhiDJY,8022
@@ -618,7 +618,7 @@ zenml/model_deployers/__init__.py,sha256=oVBLtTfrNenl5OI1iqtQUvJ0vpocRVUN_HIt8qp
 zenml/model_deployers/base_model_deployer.py,sha256=Z5-E_zC0Ss5OQhUBz9YKeVgS2MJOzuNz2aBzqI5eo4U,24610
 zenml/model_registries/__init__.py,sha256=wA9Vzo0w_e9zuXOVURB9w8oMLSnTaimXcxg_Nb7O3b0,1238
 zenml/model_registries/base_model_registry.py,sha256=NXviKlREKv56JJaS1skG0dXgP0YOzZJN7kTxqw-cw7U,17524
-zenml/models/__init__.py,sha256=pxwOM1cJS3N5pmCPt3P_LzjStOBI0WDwmdzEo7numqM,24238
+zenml/models/__init__.py,sha256=I-5VZaOFp4-30q1kmOvIo6WUkl6iwkT_DGrvHZ8HtSk,24321
 zenml/models/v2/__init__.py,sha256=LGMIUJi19sOsvo54roZSQwDp_czNQYtenqd_frTLIhU,613
 zenml/models/v2/base/__init__.py,sha256=knhroJ2h0uHBCGzAiBBGJEiuhEA3cwI6XYBRIyXdbkQ,613
 zenml/models/v2/base/base.py,sha256=Oz6iWCaJw1ykoYe2tTvGJqalGmDp0tRS9r1NpBx8WsM,16222
@@ -660,7 +660,7 @@ zenml/models/v2/core/service.py,sha256=PeI036PIVG0zX5EiYPsx7h5LTRC8hlmfdeClKff-I
 zenml/models/v2/core/service_account.py,sha256=-1c9Et9Ma4_OHOZxIHjVnLJAaLLxhpoLEyGKkwc1Yx8,6619
 zenml/models/v2/core/service_connector.py,sha256=V1zWxTpo9ABCxRfK327w7CV21n5XWaPHgIR6CE2zHNk,38628
 zenml/models/v2/core/stack.py,sha256=3IKofEaKprAtYNbIrKvcYs5IABTbCqxcfJSXC3Q0fYM,13332
-zenml/models/v2/core/step_run.py,sha256=wM91dxaSTkOFVX9vRl4ooTlyrxmrxWrDszJydhTqjgs,22046
+zenml/models/v2/core/step_run.py,sha256=KnwZMfqxPo0xJSoRv1pkdinze5nagBhcWKcCG6cyEt4,22531
 zenml/models/v2/core/tag.py,sha256=60hCZuHAHy7lsYFkTDw7hK0zZz2eG7jx8l9VY7Hh-LQ,7100
 zenml/models/v2/core/tag_resource.py,sha256=H7CPj9oaUpym8vSkwuYSJ6_rWfHMEKVRedPw2kyxmII,2640
 zenml/models/v2/core/trigger.py,sha256=L6-pzWPM2HFsfA6-6rM1XeHx4UChZrIY58G6ZG1PA10,12782
@@ -669,6 +669,7 @@ zenml/models/v2/core/user.py,sha256=jw8HRKhZKJ3ljAbNMiFga-IV04zXhiRS33jLm13b8RA,
 zenml/models/v2/misc/__init__.py,sha256=knhroJ2h0uHBCGzAiBBGJEiuhEA3cwI6XYBRIyXdbkQ,613
 zenml/models/v2/misc/auth_models.py,sha256=j31CQt9qfH20tyCjY0depqbKP0YZEewcgGKrixqBUbg,3688
 zenml/models/v2/misc/build_item.py,sha256=66Fywatv8bDY86pf0k530fsHTk_L4IkHL6uJL7-6dAo,1938
+zenml/models/v2/misc/exception_info.py,sha256=CCTO7HYqsa3poi5hL4HK3ZS67xmkQNYQRHgLzqpUTHs,1016
 zenml/models/v2/misc/external_user.py,sha256=_prXznyVA3Dc5pCK7nVE72K3tjxslG_e3JxIXLIikBo,944
 zenml/models/v2/misc/info_models.py,sha256=3X6HFIlFcz5uuT7qgU18HqZBPkgrGZQ7QcM2yBT0qeU,2645
 zenml/models/v2/misc/loaded_visualization.py,sha256=u6lapDNZDtU9eS-_EMzl00urj0yPNhiqhZcvjIz7DSg,946
@@ -692,10 +693,10 @@ zenml/orchestrators/local/local_orchestrator.py,sha256=KCzc901_wrb1DPTDu_IY6HFxT
 zenml/orchestrators/local_docker/__init__.py,sha256=k8J68ydy6HmmvE9tWo32g761H8P_Dw4AxWNf4UMpsbs,669
 zenml/orchestrators/local_docker/local_docker_orchestrator.py,sha256=dEc1R2_xzFIzvSwPirwtH2noYzPyw0gbdDhpsqAgFMM,10032
 zenml/orchestrators/output_utils.py,sha256=01vqke1ZfmfuLpgxNerF-QL2wA0VPv1zUdvlMw0OwUY,3508
-zenml/orchestrators/publish_utils.py,sha256=nCUgjR8hl3H0sudPqojnENmALNwlFjtaZ1XKFzvPNHA,8109
-zenml/orchestrators/step_launcher.py,sha256=SIiWMgzqhJdwYNqbPgvoOxoxlM0SUg8oYEI9Gyh9wYQ,18460
+zenml/orchestrators/publish_utils.py,sha256=u0jk9Q57fLQE1QDJjrydy5KQ9xtYKUvn3CXBirDH2u8,8485
+zenml/orchestrators/step_launcher.py,sha256=ipFy7-pGKSmAUySRuBPddSusQ5LaLS0g5t49UlU77YY,18841
 zenml/orchestrators/step_run_utils.py,sha256=JXlYomV_PmC-cmjk7VWAGauKUXdijXU0ao8HBTZobmY,16802
-zenml/orchestrators/step_runner.py,sha256=sV0IkS9Rsa_TzSJNHRqwXKI0qXzMNZKRRn6Klfn-cEw,26523
+zenml/orchestrators/step_runner.py,sha256=X3sX7f5NBYxT3utFGKcOJlkvCauYyEoo0LB2TB2Pp9U,27435
 zenml/orchestrators/topsort.py,sha256=D8evz3X47zwpXd90NMLsJD-_uCeXtV6ClzNfDUrq7cM,5784
 zenml/orchestrators/utils.py,sha256=6bqLc1fmdJTXg8JUwUKs8YNbmxTuMIfWmUbUpg-7hx0,12956
 zenml/orchestrators/wheeled_orchestrator.py,sha256=eOnMcnd3sCzfhA2l6qRAzF0rOXzaojbjvvYvTkqixQo,4791
@@ -775,6 +776,7 @@ zenml/utils/docker_utils.py,sha256=occ364GUasOyiEipCcBHdK1WSvh1oVJIVnc-XXNsrcE,1
 zenml/utils/downloaded_repository_context.py,sha256=GyPOzC8M3SaEGnnbPgEgJhUW-m5hx9rLScnCiGw6_PY,1504
 zenml/utils/enum_utils.py,sha256=0fA0B9v9Wjutuqlu_owUoOit1utIw2UH5J6YHXSqhLU,1368
 zenml/utils/env_utils.py,sha256=2--2DDUt3gPOvCNVyobBtAciikQ0OEXs5WWp7NvYuKo,5276
+zenml/utils/exception_utils.py,sha256=7X--e7ERx2PYzhNwyhFMzKyriBpPPLybS5blpdZe1PE,2965
 zenml/utils/filesync_model.py,sha256=9ibsIr2HJfkEQ41upQd4uJ79ZhzB0MHS85GOGOAQ19Y,5470
 zenml/utils/function_utils.py,sha256=bznRrCrQIJ3Joc-QQ8ynDyn9AvyIcR61bzLnaXQV3tc,8109
 zenml/utils/git_utils.py,sha256=O2f6PUpDuMQrcmkcGXIhbJQpTzLM7p8PyiKWjcIuWqM,1831
@@ -1288,6 +1290,7 @@ zenml/zen_stores/migrations/versions/cc9894cb58aa_add_user_metadata.py,sha256=68
 zenml/zen_stores/migrations/versions/ccd68b7825ae_add_status_to_pipeline_and_step_run.py,sha256=MUvxaUyzcLtNOVgCgM6nBE3UTPKz5sVKsz-k071zCes,1824
 zenml/zen_stores/migrations/versions/d02b3d3464cf_add_orchestrator_run_id_column.py,sha256=b5KNEC9vWdbWkiMRpPKJDK0xE1HiOdVJT_ab5kynA7M,1157
 zenml/zen_stores/migrations/versions/d26471b6fe8f_update_build_filtering.py,sha256=VVDG2lhPzhejGm4Phzxw8sRCZd-vFXU4J3BtR96aRL0,1993
+zenml/zen_stores/migrations/versions/d4591f95ac07_step_exception_info.py,sha256=LAWlgcCgWpo4yz54JIm7ys_DdK8Jd0mOpyhUKsHNjf8,1198
 zenml/zen_stores/migrations/versions/d7b3acf9aa46_create_schedule_table.py,sha256=ntxrMe__pAjvepVi0hznANE3tIHgrfJ1Ml4gtxmmSLU,3329
 zenml/zen_stores/migrations/versions/e1d66d91a099_add_stack_and_component_spec_paths_to_.py,sha256=zAwzy9_n8s2WZ08vVyDG1WihowKpHDf-EOs9dbUmgPg,1543
 zenml/zen_stores/migrations/versions/e5225281b4d3_add_connector_skew_tolerance.py,sha256=nR2h9X_cnbJ7OUwANJuzazhN90_anXF3YRABQPofqX0,1048
@@ -1330,7 +1333,7 @@ zenml/zen_stores/schemas/server_settings_schemas.py,sha256=usBE-idRrmK-LeLN0zDtC
 zenml/zen_stores/schemas/service_connector_schemas.py,sha256=SpzFZTbfxypFtMLBujPSlGOpt-kzjcHfq3IoBghPizI,11137
 zenml/zen_stores/schemas/service_schemas.py,sha256=Fn_DyGm-PF-qN1TPZA3Q8AhrnUOTSIX0V_MD0laZ2Bk,10685
 zenml/zen_stores/schemas/stack_schemas.py,sha256=DOPoNwblTqN83RpnD6cjyF80VeplA1BGUX8yuulpzKk,6829
-zenml/zen_stores/schemas/step_run_schemas.py,sha256=nqAX5y_rlnvqXH1MBMCtvxFNebfs00iu5RtqtiCAj64,20086
+zenml/zen_stores/schemas/step_run_schemas.py,sha256=lzisqMKUsKmzfl6AiagFlmVADOZowFvyZamltgjXxaE,20740
 zenml/zen_stores/schemas/tag_schemas.py,sha256=pwQ5ogZovtkUuRtAkHakOfMh6ixw6qofvxB4i_Gd4qA,8352
 zenml/zen_stores/schemas/trigger_schemas.py,sha256=LokauFEHdN5UuYeC2kf5PtJ60WMz-E82Ngcwxcv6tDA,11936
 zenml/zen_stores/schemas/user_schemas.py,sha256=wLSHtfjjrYBWnX_PXv5KPz_DRdheTxZVjcgBe-cP__c,11497
@@ -1347,8 +1350,8 @@ zenml/zen_stores/secrets_stores/sql_secrets_store.py,sha256=LPFW757WCJLP1S8vrvjs
 zenml/zen_stores/sql_zen_store.py,sha256=cXcKBBRKiwM2GtwHzmJb6tiN5NhGWo9n8SWnL1b4_WE,491150
 zenml/zen_stores/template_utils.py,sha256=iCXrXpqzVTY7roqop4Eh9J7DmLW6PQeILZexmw_l3b8,10074
 zenml/zen_stores/zen_store_interface.py,sha256=weiSULdI9AsbCE10a5TcwtybX-BJs9hKhjPJnTapWv4,93023
-zenml_nightly-0.84.0.dev20250728.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
-zenml_nightly-0.84.0.dev20250728.dist-info/METADATA,sha256=nng2oUmzUZy3Tl0YEHaj5PH63qN78QIGAYaO1wZm_z8,24296
-zenml_nightly-0.84.0.dev20250728.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-zenml_nightly-0.84.0.dev20250728.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
-zenml_nightly-0.84.0.dev20250728.dist-info/RECORD,,
+zenml_nightly-0.84.0.dev20250729.dist-info/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
+zenml_nightly-0.84.0.dev20250729.dist-info/METADATA,sha256=0K0W8CmHG1-krP1zg_a1s4uvVVAsJy2TjFAvCo-cG9k,24296
+zenml_nightly-0.84.0.dev20250729.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+zenml_nightly-0.84.0.dev20250729.dist-info/entry_points.txt,sha256=QK3ETQE0YswAM2mWypNMOv8TLtr7EjnqAFq1br_jEFE,43
+zenml_nightly-0.84.0.dev20250729.dist-info/RECORD,,