mlrun 1.7.0rc43__py3-none-any.whl → 1.7.0rc56__py3-none-any.whl

mlrun/datastore/s3.py

@@ -36,6 +36,7 @@ class S3Store(DataStore):
 
         access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
         secret_key = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
+        token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
         endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
         force_non_anonymous = self._get_secret_or_env("S3_NON_ANONYMOUS")
         profile_name = self._get_secret_or_env("AWS_PROFILE")
@@ -94,14 +95,15 @@ class S3Store(DataStore):
             self.s3 = boto3.resource(
                 "s3", region_name=region, endpoint_url=endpoint_url
             )
-            # If not using credentials, boto will still attempt to sign the requests, and will fail any operations
-            # due to no credentials found. These commands disable signing and allow anonymous mode (same as
-            # anon in the storage_options when working with fsspec).
-            from botocore.handlers import disable_signing
-
-            self.s3.meta.client.meta.events.register(
-                "choose-signer.s3.*", disable_signing
-            )
+            if not token_file:
+                # If not using credentials, boto will still attempt to sign the requests, and will fail any operations
+                # due to no credentials found. These commands disable signing and allow anonymous mode (same as
+                # anon in the storage_options when working with fsspec).
+                from botocore.handlers import disable_signing
+
+                self.s3.meta.client.meta.events.register(
+                    "choose-signer.s3.*", disable_signing
+                )
 
     def get_spark_options(self):
         res = {}
@@ -139,6 +141,7 @@ class S3Store(DataStore):
         endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
         access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
         secret = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
+        token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
 
         if self._temp_credentials:
             access_key_id = self._temp_credentials["AccessKeyId"]
@@ -148,7 +151,7 @@ class S3Store(DataStore):
             token = None
 
         storage_options = dict(
-            anon=not (force_non_anonymous or (access_key_id and secret)),
+            anon=not (force_non_anonymous or (access_key_id and secret) or token_file),
             key=access_key_id,
             secret=secret,
             token=token,

mlrun/datastore/storeytargets.py

@@ -19,6 +19,7 @@ import mlrun
 import mlrun.model_monitoring.helpers
 from mlrun.datastore.base import DataStore
 
+from ..platforms.iguazio import parse_path
 from .utils import (
     parse_kafka_url,
 )
@@ -82,15 +83,17 @@ class StreamStoreyTarget(storey.StreamTarget):
     def __init__(self, *args, **kwargs):
         args = list(args)
 
-        path = args[0] if args else kwargs.get("stream_path")
-        endpoint, storage_options = get_url_and_storage_options(path)
+        uri = args[0] if args else kwargs.get("stream_path")
 
-        if not path:
+        if not uri:
             raise mlrun.errors.MLRunInvalidArgumentError("StreamTarget requires a path")
 
+        _, storage_options = get_url_and_storage_options(uri)
+        endpoint, path = parse_path(uri)
+
         access_key = storage_options.get("v3io_access_key")
-        storage = (
-            V3ioDriver(webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key),
+        storage = V3ioDriver(
+            webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key
         )
 
         if storage_options:
@@ -98,7 +101,7 @@ class StreamStoreyTarget(storey.StreamTarget):
         if args:
             args[0] = endpoint
         if "stream_path" in kwargs:
-            kwargs["stream_path"] = endpoint
+            kwargs["stream_path"] = path
 
         super().__init__(*args, **kwargs)
 

mlrun/db/base.py

@@ -395,6 +395,9 @@ class RunDBInterface(ABC):
         partition_order: Union[
             mlrun.common.schemas.OrderType, str
         ] = mlrun.common.schemas.OrderType.desc,
+        format_: Union[
+            str, mlrun.common.formatters.FeatureSetFormat
+        ] = mlrun.common.formatters.FeatureSetFormat.full,
     ) -> list[dict]:
         pass
 

mlrun/db/httpdb.py

@@ -525,10 +525,6 @@ class HTTPRunDB(RunDBInterface):
                 server_cfg.get("external_platform_tracking")
                 or config.external_platform_tracking
             )
-            config.model_endpoint_monitoring.store_type = (
-                server_cfg.get("model_endpoint_monitoring_store_type")
-                or config.model_endpoint_monitoring.store_type
-            )
             config.model_endpoint_monitoring.endpoint_store_connection = (
                 server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
                 or config.model_endpoint_monitoring.endpoint_store_connection
@@ -1374,20 +1370,14 @@ class HTTPRunDB(RunDBInterface):
         :returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
             that were removed.
         """
-        if grace_period is None:
-            grace_period = config.runtime_resources_deletion_grace_period
-            logger.info(
-                "Using default grace period for runtime resources deletion",
-                grace_period=grace_period,
-            )
-
         params = {
             "label-selector": label_selector,
             "kind": kind,
             "object-id": object_id,
             "force": force,
-            "grace-period": grace_period,
         }
+        if grace_period is not None:
+            params["grace-period"] = grace_period
         error = "Failed deleting runtime resources"
         project_path = project if project else "*"
         response = self.api_call(
@@ -2245,6 +2235,9 @@ class HTTPRunDB(RunDBInterface):
         partition_order: Union[
             mlrun.common.schemas.OrderType, str
         ] = mlrun.common.schemas.OrderType.desc,
+        format_: Union[
+            str, mlrun.common.formatters.FeatureSetFormat
+        ] = mlrun.common.formatters.FeatureSetFormat.full,
     ) -> list[FeatureSet]:
         """Retrieve a list of feature-sets matching the criteria provided.
 
@@ -2262,6 +2255,9 @@ class HTTPRunDB(RunDBInterface):
         :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
             Currently the only allowed value are `created` and `updated`.
         :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
+        :param format_: Format of the results. Possible values are:
+            - ``minimal`` - Return minimal feature set objects, not including stats and preview for each feature set.
+            - ``full`` - Return full feature set objects.
         :returns: List of matching :py:class:`~mlrun.feature_store.FeatureSet` objects.
         """
 
@@ -2274,6 +2270,7 @@ class HTTPRunDB(RunDBInterface):
             "entity": entities or [],
             "feature": features or [],
             "label": labels or [],
+            "format": format_,
         }
         if partition_by:
             params.update(
@@ -2757,7 +2754,7 @@ class HTTPRunDB(RunDBInterface):
         deletion_strategy: Union[
             str, mlrun.common.schemas.DeletionStrategy
         ] = mlrun.common.schemas.DeletionStrategy.default(),
-    ):
+    ) -> None:
         """Delete a project.
 
         :param name: Name of the project to delete.
@@ -2776,7 +2773,7 @@ class HTTPRunDB(RunDBInterface):
             "DELETE", f"projects/{name}", error_message, headers=headers, version="v2"
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
-            logger.info("Project is being deleted", project_name=name)
+            logger.info("Waiting for project to be deleted", project_name=name)
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
             background_task = self._wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name
@@ -2786,10 +2783,17 @@ class HTTPRunDB(RunDBInterface):
                 == mlrun.common.schemas.BackgroundTaskState.succeeded
             ):
                 logger.info("Project deleted", project_name=name)
-                return
+            elif (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.failed
+            ):
+                logger.error(
+                    "Project deletion failed",
+                    project_name=name,
+                    error=background_task.status.error,
+                )
         elif response.status_code == http.HTTPStatus.NO_CONTENT:
             logger.info("Project deleted", project_name=name)
-            return
 
     def store_project(
         self,
@@ -4193,6 +4197,9 @@ class HTTPRunDB(RunDBInterface):
         :param event_data: The data of the event.
         :param project:    The project that the event belongs to.
         """
+        if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
+            logger.warning("Alerts are disabled, event will not be generated")
+
         project = project or config.default_project
         endpoint_path = f"projects/{project}/events/{name}"
         error_message = f"post event {project}/events/{name}"
@@ -4219,6 +4226,11 @@ class HTTPRunDB(RunDBInterface):
         if not alert_data:
             raise mlrun.errors.MLRunInvalidArgumentError("Alert data must be provided")
 
+        if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
+            logger.warning(
+                "Alerts are disabled, alert will still be stored but will not be triggered"
+            )
+
         project = project or config.default_project
         endpoint_path = f"projects/{project}/alerts/{alert_name}"
         error_message = f"put alert {project}/alerts/{alert_name}"

mlrun/db/nopdb.py

@@ -21,6 +21,7 @@ import mlrun.common.formatters
 import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.errors
+import mlrun.lists
 
 from ..config import config
 from ..utils import logger
@@ -73,6 +74,22 @@ class NopDB(RunDBInterface):
     def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
         pass
 
+    def list_runtime_resources(
+        self,
+        project: Optional[str] = None,
+        label_selector: Optional[str] = None,
+        kind: Optional[str] = None,
+        object_id: Optional[str] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
+    ) -> Union[
+        mlrun.common.schemas.RuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
+    ]:
+        return []
+
     def read_run(
         self,
         uid,
@@ -108,7 +125,7 @@ class NopDB(RunDBInterface):
         max_partitions: int = 0,
         with_notifications: bool = False,
     ):
-        pass
+        return mlrun.lists.RunList()
 
     def del_run(self, uid, project="", iter=0):
         pass
@@ -149,7 +166,7 @@ class NopDB(RunDBInterface):
         format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
         limit: int = None,
     ):
-        pass
+        return mlrun.lists.ArtifactList()
 
     def del_artifact(
         self,
@@ -181,7 +198,7 @@ class NopDB(RunDBInterface):
     def list_functions(
         self, name=None, project="", tag="", labels=None, since=None, until=None
     ):
-        pass
+        return []
 
     def tag_objects(
         self,
@@ -309,6 +326,9 @@ class NopDB(RunDBInterface):
         partition_order: Union[
             mlrun.common.schemas.OrderType, str
         ] = mlrun.common.schemas.OrderType.desc,
+        format_: Union[
+            str, mlrun.common.formatters.FeatureSetFormat
+        ] = mlrun.common.formatters.FeatureSetFormat.full,
     ) -> list[dict]:
         pass
 
@@ -421,7 +441,7 @@ class NopDB(RunDBInterface):
         ] = mlrun.common.formatters.PipelineFormat.metadata_only,
         page_size: int = None,
     ) -> mlrun.common.schemas.PipelinesOutput:
-        pass
+        return mlrun.common.schemas.PipelinesOutput(runs=[], total_size=0)
 
     def create_project_secrets(
         self,

mlrun/errors.py

@@ -140,7 +140,13 @@ def err_to_str(err):
         error_strings.append(err_msg)
         err = err.__cause__
 
-    return ", caused by: ".join(error_strings)
+    err_msg = ", caused by: ".join(error_strings)
+
+    # in case the error string is longer than 32k, we truncate it
+    # the truncation takes the first 16k, then the last 16k characters
+    if len(err_msg) > 32_000:
+        err_msg = err_msg[:16_000] + "...truncated..." + err_msg[-16_000:]
+    return err_msg
 
 
 # Specific Errors

mlrun/execution.py

@@ -24,6 +24,7 @@ from dateutil import parser
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
+import mlrun.common.formatters
 from mlrun.artifacts import ModelArtifact
 from mlrun.datastore.store_resources import get_store_resource
 from mlrun.errors import MLRunInvalidArgumentError
@@ -634,7 +635,9 @@ class MLClientCtx:
         :param viewer:        Kubeflow viewer type
         :param target_path:   Absolute target path (instead of using artifact_path + local_path)
         :param src_path:      Deprecated, use local_path
-        :param upload:        Upload to datastore (default is True)
+        :param upload:        Whether to upload the artifact to the datastore. If not provided, and the `local_path`
+                              is not a directory, upload occurs by default. Directories are uploaded only when this
+                              flag is explicitly set to `True`.
         :param labels:        A set of key/value labels to tag the artifact with
         :param format:        Optional, format to use (e.g. csv, parquet, ..)
         :param db_key:        The key to use in the artifact DB table, by default its run name + '_' + key
@@ -924,12 +927,42 @@ class MLClientCtx:
                 updates, self._uid, self.project, iter=self._iteration
             )
 
-    def get_notifications(self):
-        """Get the list of notifications"""
-        return [
-            mlrun.model.Notification.from_dict(notification)
-            for notification in self._notifications
-        ]
+    def get_notifications(self, unmask_secret_params=False):
+        """
+        Get the list of notifications
+
+        :param unmask_secret_params: Used as a workaround for sending notification from workflow-runner.
+                                     When used, if the notification will be saved again a new secret will be created.
+        """
+
+        # Get the full notifications from the DB since the run context does not contain the params due to bloating
+        run = self._rundb.read_run(
+            self.uid, format_=mlrun.common.formatters.RunFormat.notifications
+        )
+
+        notifications = []
+        for notification in run["spec"]["notifications"]:
+            notification: mlrun.model.Notification = mlrun.model.Notification.from_dict(
+                notification
+            )
+            # Fill the secret params from the project secret. We cannot use the server side internal secret mechanism
+            # here as it is the client side.
+            # TODO: This is a workaround to allow the notification to get the secret params from project secret
+            #       instead of getting them from the internal project secret that should be mounted.
+            #       We should mount the internal project secret that was created to the workflow-runner
+            #       and get the secret from there.
+            if unmask_secret_params:
+                try:
+                    notification.enrich_unmasked_secret_params_from_project_secret()
+                    notifications.append(notification)
+                except mlrun.errors.MLRunValueError:
+                    logger.warning(
+                        "Failed to fill secret params from project secret for notification."
+                        "Skip this notification.",
+                        notification=notification.name,
+                    )
+
+        return notifications
 
     def to_dict(self):
         """Convert the run context to a dictionary"""

mlrun/feature_store/api.py

@@ -1051,6 +1051,7 @@ def _ingest_with_spark(
 
             spark = (
                 pyspark.sql.SparkSession.builder.appName(session_name)
+                .config("spark.driver.memory", "2g")
                 .config("spark.sql.session.timeZone", "UTC")
                 .getOrCreate()
             )

mlrun/feature_store/retrieval/spark_merger.py

@@ -188,9 +188,13 @@ class SparkFeatureMerger(BaseMerger):
 
         if self.spark is None:
             # create spark context
-            self.spark = SparkSession.builder.appName(
-                f"vector-merger-{self.vector.metadata.name}"
-            ).getOrCreate()
+            self.spark = (
+                SparkSession.builder.appName(
+                    f"vector-merger-{self.vector.metadata.name}"
+                )
+                .config("spark.driver.memory", "2g")
+                .getOrCreate()
+            )
 
     def _get_engine_df(
         self,
@@ -202,10 +206,6 @@ class SparkFeatureMerger(BaseMerger):
         time_column=None,
         additional_filters=None,
     ):
-        mlrun.utils.helpers.additional_filters_warning(
-            additional_filters, self.__class__
-        )
-
         source_kwargs = {}
         if feature_set.spec.passthrough:
             if not feature_set.spec.source:

mlrun/frameworks/_common/plan.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 from abc import ABC, abstractmethod
 
 import mlrun
 from mlrun.artifacts import Artifact
-from mlrun.utils.helpers import is_ipython
+from mlrun.utils.helpers import is_jupyter
 
 
 class Plan(ABC):
@@ -84,7 +84,7 @@ class Plan(ABC):
             return
 
         # Call the correct display method according to the kernel:
-        if is_ipython:
+        if is_jupyter:
             self._gui_display()
         else:
             self._cli_display()

mlrun/frameworks/_ml_common/plan.py

@@ -16,7 +16,7 @@ import json
 from abc import ABC, abstractmethod
 from enum import Enum
 
-from IPython.core.display import HTML, display
+from IPython.display import HTML, display
 
 import mlrun
 

mlrun/frameworks/parallel_coordinates.py

@@ -18,8 +18,7 @@ from typing import Union
 
 import numpy as np
 import pandas as pd
-from IPython.core.display import HTML
-from IPython.display import display
+from IPython.display import HTML, display
 from pandas.api.types import is_numeric_dtype, is_string_dtype
 
 import mlrun
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
                 fp.write("</body></html>")
             else:
                 fp.write(html)
-    if show or (show is None and mlrun.utils.is_ipython):
+    if show or (show is None and mlrun.utils.is_jupyter):
         display(HTML(html))
         if runs_list and len(runs_list) <= max_table_rows:
             display(HTML(html_table))

mlrun/launcher/client.py

@@ -14,7 +14,7 @@
 import abc
 from typing import Optional
 
-import IPython
+import IPython.display
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.errors
@@ -22,7 +22,7 @@ import mlrun.launcher.base as launcher
 import mlrun.lists
 import mlrun.model
 import mlrun.runtimes
-from mlrun.utils import logger
+import mlrun.utils
 
 
 class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
@@ -128,10 +128,10 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         if result:
             results_tbl.append(result)
         else:
-            logger.info("no returned result (job may still be in progress)")
+            mlrun.utils.logger.info("no returned result (job may still be in progress)")
             results_tbl.append(run.to_dict())
 
-        if mlrun.utils.is_ipython and mlrun.mlconf.ipython_widget:
+        if mlrun.utils.is_jupyter and mlrun.mlconf.ipython_widget:
             results_tbl.show()
             print()
             ui_url = mlrun.utils.get_ui_url(project, uid)
@@ -147,9 +147,9 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
             project_flag = f"-p {project}" if project else ""
             info_cmd = f"mlrun get run {uid} {project_flag}"
             logs_cmd = f"mlrun logs {uid} {project_flag}"
-            logger.info(
+            mlrun.utils.logger.info(
                 "To track results use the CLI", info_cmd=info_cmd, logs_cmd=logs_cmd
             )
             ui_url = mlrun.utils.get_ui_url(project, uid)
             if ui_url:
-                logger.info("Or click for UI", ui_url=ui_url)
+                mlrun.utils.logger.info("Or click for UI", ui_url=ui_url)

mlrun/model.py

@@ -774,6 +774,23 @@ class Notification(ModelObj):
 
         notification_class.validate_params(secret_params | params)
 
+    def enrich_unmasked_secret_params_from_project_secret(self):
+        """
+        Fill the notification secret params from the project secret.
+        We are using this function instead of unmask_secret_params_from_project_secret when we run inside the
+        workflow runner pod that doesn't have access to the k8s secrets (but have access to the project secret)
+        """
+        secret = self.secret_params.get("secret")
+        if secret:
+            secret_value = mlrun.get_secret_or_env(secret)
+            if secret_value:
+                try:
+                    self.secret_params = json.loads(secret_value)
+                except ValueError as exc:
+                    raise mlrun.errors.MLRunValueError(
+                        "Failed to parse secret value"
+                    ) from exc
+
     @staticmethod
     def validate_notification_uniqueness(notifications: list["Notification"]):
         """Validate that all notifications in the list are unique by name"""
@@ -2044,6 +2061,8 @@ class DataSource(ModelObj):
     ]
     kind = None
 
+    _fields_to_serialize = ["start_time", "end_time"]
+
     def __init__(
         self,
         name: str = None,
@@ -2072,6 +2091,16 @@ class DataSource(ModelObj):
     def set_secrets(self, secrets):
         self._secrets = secrets
 
+    def _serialize_field(
+        self, struct: dict, field_name: str = None, strip: bool = False
+    ) -> typing.Any:
+        value = super()._serialize_field(struct, field_name, strip)
+        # We pull the field from self and not from struct because it was excluded from the struct when looping over
+        # the fields to save.
+        if field_name in ("start_time", "end_time") and isinstance(value, datetime):
+            return value.isoformat()
+        return value
+
 
 class DataTargetBase(ModelObj):
     """data target spec, specify a destination for the feature set data"""

mlrun/model_monitoring/api.py

@@ -24,7 +24,6 @@ import mlrun.artifacts
 import mlrun.common.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store
-import mlrun.model_monitoring.application
 import mlrun.model_monitoring.applications as mm_app
 import mlrun.serving
 from mlrun.data_types.infer import InferOptions, get_df_stats
@@ -561,8 +560,7 @@ def _create_model_monitoring_function_base(
     func: typing.Union[str, None] = None,
     application_class: typing.Union[
         str,
-        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
-        mm_app.ModelMonitoringApplicationBaseV2,
+        mm_app.ModelMonitoringApplicationBase,
         None,
     ] = None,
     name: typing.Optional[str] = None,
@@ -576,15 +574,6 @@ def _create_model_monitoring_function_base(
     Note: this is an internal API only.
     This function does not set the labels or mounts v3io.
     """
-    if isinstance(
-        application_class,
-        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
-    ):
-        warnings.warn(
-            "The `ModelMonitoringApplicationBase` class is deprecated from version 1.7.0, "
-            "please use `ModelMonitoringApplicationBaseV2`. It will be removed in 1.9.0.",
-            FutureWarning,
-        )
     if name in mm_constants._RESERVED_FUNCTION_NAMES:
         raise mlrun.errors.MLRunInvalidArgumentError(
             "An application cannot have the following names: "

mlrun/model_monitoring/applications/__init__.py

@@ -13,12 +13,11 @@
 # limitations under the License.
 #
 
-from .base import ModelMonitoringApplicationBase, ModelMonitoringApplicationBaseV2
+from .base import ModelMonitoringApplicationBase
 from .context import MonitoringApplicationContext
 from .evidently_base import (
     _HAS_EVIDENTLY,
     SUPPORTED_EVIDENTLY_VERSION,
     EvidentlyModelMonitoringApplicationBase,
-    EvidentlyModelMonitoringApplicationBaseV2,
 )
 from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult

mlrun/model_monitoring/applications/_application_steps.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import traceback
 from typing import Any, Optional, Union
 
 import mlrun.common.schemas.alert as alert_objects
@@ -161,7 +162,10 @@ class _ApplicationErrorHandler(StepToDict):
         :param event: Application event.
         """
 
-        logger.error(f"Error in application step: {event}")
+        exception_with_trace = "".join(
+            traceback.format_exception(None, event.error, event.error.__traceback__)
+        )
+        logger.error(f"Error in application step: {exception_with_trace}")
 
         event_data = alert_objects.Event(
             kind=alert_objects.EventKind.MM_APP_FAILED,