mlrun 1.8.0rc18__py3-none-any.whl → 1.8.0rc19__py3-none-any.whl

mlrun/common/schemas/artifact.py

@@ -51,6 +51,12 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
+    @classmethod
+    def from_kind(cls, kind: str) -> "ArtifactCategories":
+        if kind in [cls.model.value, cls.dataset.value, cls.document.value]:
+            return cls(kind)
+        return cls.other
+
 
 class ArtifactIdentifier(pydantic.v1.BaseModel):
     # artifact kind

mlrun/common/schemas/model_monitoring/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from .constants import (
+    INTERSECT_DICT_KEYS,
     V3IO_MODEL_MONITORING_DB,
     ApplicationEvent,
     ControllerPolicy,

mlrun/common/schemas/model_monitoring/constants.py

@@ -248,6 +248,12 @@ class ProjectSecretKeys:
         ]
 
 
+class GetEventsFormat(MonitoringStrEnum):
+    SINGLE = "single"
+    SEPARATION = "separation"
+    INTERSECTION = "intersection"
+
+
 class ModelEndpointTargetSchemas(MonitoringStrEnum):
     V3IO = "v3io"
     MYSQL = "mysql"
@@ -448,3 +454,8 @@ FQN_REGEX = re.compile(FQN_PATTERN)
 PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
 
 MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
+INTERSECT_DICT_KEYS = {
+    ModelEndpointMonitoringMetricType.METRIC: "intersect_metrics",
+    ModelEndpointMonitoringMetricType.RESULT: "intersect_results",
+}

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -259,12 +259,12 @@ class ModelEndpointMonitoringMetric(BaseModel):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.full_name = _compose_full_name(
+        self.full_name = compose_full_name(
             project=self.project, app=self.app, name=self.name, type=self.type
         )
 
 
-def _compose_full_name(
+def compose_full_name(
     *,
     project: str,
     app: str,

mlrun/config.py

@@ -83,8 +83,8 @@ default_config = {
     "images_to_enrich_registry": "^mlrun/*,python:3.9",
     "kfp_url": "",
     "kfp_ttl": "14400",  # KFP ttl in sec, after that completed PODs will be deleted
-    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner (defaults to mlrun/mlrun-kfp)
-    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner (defaults to mlrun/ml-base)
+    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
+    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner
     "igz_version": "",  # the version of the iguazio system the API is running on
     "iguazio_api_url": "",  # the url to iguazio api
     "spark_app_image": "",  # image to use for spark operator app runtime

mlrun/db/base.py

@@ -337,6 +337,15 @@ class RunDBInterface(ABC):
     ) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
         pass
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        pass
+
     @abstractmethod
     def delete_project(
         self,

mlrun/db/httpdb.py

@@ -3524,6 +3524,48 @@ class HTTPRunDB(RunDBInterface):
             list[mm_endpoints.ModelEndpointMonitoringMetric], monitoring_metrics
         )
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        """Get application metrics/results by endpoint id and project.
+
+        :param project:         The name of the project.
+        :param endpoint_ids:    The unique id of the model endpoint. Can be a single id or a list of ids.
+        :param type:            The type of the metrics to return. "all" means "results" and "metrics".
+        :param events_format:   response format:
+
+                                separation: {"mep_id1":[...], "mep_id2":[...]}
+                                intersection {"intersect_metrics":[], "intersect_results":[]}
+        :return: A dictionary of application metrics and/or results for the model endpoints formatted by events_format.
+        """
+        path = f"projects/{project}/model-endpoints/metrics"
+        params = {
+            "type": type,
+            "endpoint-id": endpoint_ids,
+            "events_format": events_format,
+        }
+        error_message = (
+            f"Failed to get model monitoring metrics,"
+            f" endpoint_ids: {endpoint_ids}, project: {project}"
+        )
+        response = self.api_call(
+            mlrun.common.types.HTTPMethod.GET,
+            path,
+            error_message,
+            params=params,
+        )
+        monitoring_metrics_by_endpoint = response.json()
+        parsed_metrics_by_endpoint = {}
+        for endpoint, metrics in monitoring_metrics_by_endpoint.items():
+            parsed_metrics_by_endpoint[endpoint] = parse_obj_as(
+                list[mm_endpoints.ModelEndpointMonitoringMetric], metrics
+            )
+        return parsed_metrics_by_endpoint
+
     def create_user_secrets(
         self,
         user: str,

mlrun/model_monitoring/db/tsdb/base.py

@@ -234,14 +234,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoint.
+        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -252,14 +252,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoint.
+        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -341,7 +341,7 @@ class TSDBConnector(ABC):
             logger.debug("No metrics", missing_metrics=metrics_without_data.keys())
             grouped = []
         for (app_name, name), sub_df in grouped:
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project,
                 app=app_name,
                 name=name,
@@ -410,7 +410,7 @@ class TSDBConnector(ABC):
             result_kind = mlrun.model_monitoring.db.tsdb.helpers._get_result_kind(
                 sub_df
             )
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project, app=app_name, name=name
             )
             try:
@@ -467,6 +467,7 @@ class TSDBConnector(ABC):
 
         :return:        A list of mm metrics objects.
         """
+
         return list(
             map(
                 lambda record: mm_schemas.ModelEndpointMonitoringMetric(
@@ -481,6 +482,113 @@ class TSDBConnector(ABC):
             )
         )
 
+    @staticmethod
+    def df_to_metrics_grouped_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: str,
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a grouped mm metrics objects by endpoint_id.
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A grouped dict of mm metrics/results, using model_endpoints_ids as keys.
+        """
+
+        if df.empty:
+            return {}
+
+        grouped_by_fields = [mm_schemas.WriterEvent.APPLICATION_NAME]
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            grouped_by_fields.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+
+        grouped_by_fields.append(name_column)
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        grouped_by_df = df.groupby("endpoint_id")
+        grouped_dict = grouped_by_df.apply(
+            lambda group: list(
+                map(
+                    lambda record: mm_schemas.ModelEndpointMonitoringMetric(
+                        project=project,
+                        type=type,
+                        app=record.get(mm_schemas.WriterEvent.APPLICATION_NAME),
+                        name=record.get(name_column),
+                        **{"kind": record.get(mm_schemas.ResultData.RESULT_KIND)}
+                        if type == "result"
+                        else {},
+                    ),
+                    group[grouped_by_fields].to_dict(orient="records"),
+                )
+            )
+        ).to_dict()
+        return grouped_dict
+
+    @staticmethod
+    def df_to_events_intersection_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: typing.Union[str, mm_schemas.ModelEndpointMonitoringMetricType],
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a dict of intersection metrics/results by name and application
+         (and kind in results).
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A dictionary where the key is event type (as defined by `INTERSECT_DICT_KEYS`),
+                        and the value is a list containing the intersect metrics or results across all endpoint IDs.
+
+                        For example:
+                        {
+                            "intersect_metrics": [...]
+                        }
+        """
+        dict_key = mm_schemas.INTERSECT_DICT_KEYS[type]
+        metrics = []
+        if df.empty:
+            return {dict_key: []}
+
+        columns_to_zip = [mm_schemas.WriterEvent.APPLICATION_NAME]
+
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            columns_to_zip.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+        columns_to_zip.insert(1, name_column)
+
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        df["event_values"] = list(zip(*[df[col] for col in columns_to_zip]))
+        grouped_by_event_values = df.groupby("endpoint_id")["event_values"].apply(set)
+        common_event_values_combinations = set.intersection(*grouped_by_event_values)
+        result_kind = None
+        for data in common_event_values_combinations:
+            application_name, event_name = data[0], data[1]
+            if len(data) > 2:  # in result case
+                result_kind = data[2]
+            metrics.append(
+                mm_schemas.ModelEndpointMonitoringMetric(
+                    project=project,
+                    type=type,
+                    app=application_name,
+                    name=event_name,
+                    kind=result_kind,
+                )
+            )
+        return {dict_key: metrics}
+
     @staticmethod
     def _get_start_end(
         start: typing.Union[datetime, None],

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import typing
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 import pandas as pd
 import taosws
@@ -164,6 +164,17 @@ class TDEngineConnector(TSDBConnector):
     def _convert_to_datetime(val: typing.Union[str, datetime]) -> datetime:
         return datetime.fromisoformat(val) if isinstance(val, str) else val
 
+    @staticmethod
+    def _get_endpoint_filter(endpoint_id: typing.Union[str, list[str]]):
+        if isinstance(endpoint_id, str):
+            return f"endpoint_id='{endpoint_id}'"
+        elif isinstance(endpoint_id, list):
+            return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Invalid 'endpoint_id' filter: must be a string or a list."
+            )
+
     def apply_monitoring_stream_steps(self, graph, **kwarg):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -542,12 +553,11 @@ class TDEngineConnector(TSDBConnector):
             },
             inplace=True,
         )
-        df[mm_schemas.EventFieldType.LAST_REQUEST] = df[
-            mm_schemas.EventFieldType.LAST_REQUEST
-        ].map(
-            lambda last_request: datetime.strptime(
-                last_request, "%Y-%m-%d %H:%M:%S.%f %z"
-            ).astimezone(tz=timezone.utc)
+        df[mm_schemas.EventFieldType.LAST_REQUEST] = pd.to_datetime(
+            df[mm_schemas.EventFieldType.LAST_REQUEST],
+            errors="coerce",
+            format="ISO8601",
+            utc=True,
         )
         return df
 
@@ -588,7 +598,7 @@ class TDEngineConnector(TSDBConnector):
 
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
@@ -602,11 +612,12 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.MetricData.METRIC_NAME,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            filter_query=f"endpoint_id='{endpoint_id}'",
+            filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
             timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
             group_by=[
                 mm_schemas.WriterEvent.APPLICATION_NAME,
                 mm_schemas.MetricData.METRIC_NAME,
+                mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             agg_funcs=["last"],
         )
@@ -624,7 +635,7 @@ class TDEngineConnector(TSDBConnector):
 
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
@@ -639,11 +650,12 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_KIND,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            filter_query=f"endpoint_id='{endpoint_id}'",
+            filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
             timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
             group_by=[
                 mm_schemas.WriterEvent.APPLICATION_NAME,
                 mm_schemas.ResultData.RESULT_NAME,
+                mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             agg_funcs=["last"],
         )

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -33,6 +33,8 @@ _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
+V3IO_MEPS_LIMIT = 50  # TODO remove limitation after fixing ML-8886
+
 
 def _is_no_schema_error(exc: v3io_frames.Error) -> bool:
     """
@@ -577,6 +579,21 @@ class V3IOTSDBConnector(TSDBConnector):
             token=v3io_access_key,
         )
 
+    @staticmethod
+    def _get_endpoint_filter(endpoint_id: Union[str, list[str]]):
+        if isinstance(endpoint_id, str):
+            return f"endpoint_id=='{endpoint_id}'"
+        elif isinstance(endpoint_id, list):
+            if len(endpoint_id) > V3IO_MEPS_LIMIT:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"Filtering more than {V3IO_MEPS_LIMIT} model endpoints in the V3IO connector is not supported."
+                )
+            return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Invalid 'endpoint_id' filter: must be a string or a list, endpoint_id: {endpoint_id}"
+            )
+
     def read_metrics_data(
         self,
         *,
@@ -813,17 +830,18 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: Union[str, list[str]],
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.METRICS,
             start=start,
             end=end,
             columns=[mm_schemas.MetricData.METRIC_VALUE],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:
@@ -834,11 +852,12 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: Union[str, list[str]],
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
@@ -846,7 +865,7 @@ class V3IOTSDBConnector(TSDBConnector):
             columns=[
                 mm_schemas.ResultData.RESULT_KIND,
             ],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:

mlrun/model_monitoring/helpers.py

@@ -32,7 +32,7 @@ import mlrun.utils.helpers
 from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
-    _compose_full_name,
+    compose_full_name,
 )
 from mlrun.utils import logger
 
@@ -450,7 +450,7 @@ def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
 
 
 def get_invocations_fqn(project: str) -> str:
-    return _compose_full_name(
+    return compose_full_name(
         project=project,
         app=mm_constants.SpecialApps.MLRUN_INFRA,
         name=mm_constants.PredictionsQueryConstants.INVOCATIONS,

mlrun/projects/project.py

@@ -2137,18 +2137,23 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         matching_results = []
         alerts = []
-        # TODO: Refactor to use a single request to improve performance at scale, ML-8473
-        for endpoint in endpoints.endpoints:
-            results_by_endpoint = db.get_model_endpoint_monitoring_metrics(
-                project=self.name, endpoint_id=endpoint.metadata.uid, type="results"
-            )
+        endpoint_ids = [endpoint.metadata.uid for endpoint in endpoints.endpoints]
+        # using separation to group by endpoint IDs:
+        # {"mep_id1": [...], "mep_id2": [...]}
+        results_by_endpoint = db.get_metrics_by_multiple_endpoints(
+            project=self.name,
+            endpoint_ids=endpoint_ids,
+            type="results",
+            events_format=mm_constants.GetEventsFormat.SEPARATION,
+        )
+        for endpoint_uid, results in results_by_endpoint.items():
             results_fqn_by_endpoint = [
                 get_result_instance_fqn(
-                    model_endpoint_id=endpoint.metadata.uid,
+                    model_endpoint_id=endpoint_uid,
                     app_name=result.app,
                     result_name=result.name,
                 )
-                for result in results_by_endpoint
+                for result in results
             ]
             matching_results += filter_results_by_regex(
                 existing_result_names=results_fqn_by_endpoint,

mlrun/utils/helpers.py

@@ -23,6 +23,7 @@ import os
 import re
 import string
 import sys
+import traceback
 import typing
 import uuid
 import warnings
@@ -44,11 +45,16 @@ from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.helpers
+import mlrun.common.runtimes.constants as runtimes_constants
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.regex
 import mlrun.utils.version.version
+import mlrun_pipelines.common.constants
+import mlrun_pipelines.models
+import mlrun_pipelines.utils
 from mlrun.common.constants import MYSQL_MEDIUMBLOB_SIZE_BYTES
 from mlrun.config import config
 from mlrun_pipelines.models import PipelineRun
@@ -1904,3 +1910,131 @@ def join_urls(base_url: Optional[str], path: Optional[str]) -> str:
     if base_url is None:
         base_url = ""
     return f"{base_url.rstrip('/')}/{path.lstrip('/')}" if path else base_url
+
+
+class Workflow:
+    @staticmethod
+    def get_workflow_steps(workflow_id: str, project: str) -> list:
+        steps = []
+        db = mlrun.get_run_db()
+
+        def _add_run_step(_step: mlrun_pipelines.models.PipelineStep):
+            try:
+                _run = db.list_runs(
+                    project=project,
+                    labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
+                )[0]
+            except IndexError:
+                _run = {
+                    "metadata": {
+                        "name": _step.display_name,
+                        "project": project,
+                    },
+                }
+            _run["step_kind"] = _step.step_type
+            if _step.skipped:
+                _run.setdefault("status", {})["state"] = (
+                    runtimes_constants.RunStates.skipped
+                )
+            steps.append(_run)
+
+        def _add_deploy_function_step(_step: mlrun_pipelines.models.PipelineStep):
+            project, name, hash_key = Workflow._extract_function_uri(
+                _step.get_annotation("mlrun/function-uri")
+            )
+            if name:
+                try:
+                    function = db.get_function(
+                        project=project, name=name, hash_key=hash_key
+                    )
+                except mlrun.errors.MLRunNotFoundError:
+                    # If the function is not found (if build failed for example), we will create a dummy
+                    # function object for the notification to display the function name
+                    function = {
+                        "metadata": {
+                            "name": name,
+                            "project": project,
+                            "hash_key": hash_key,
+                        },
+                    }
+                pod_phase = _step.phase
+                if _step.skipped:
+                    state = mlrun.common.schemas.FunctionState.skipped
+                else:
+                    state = runtimes_constants.PodPhases.pod_phase_to_run_state(
+                        pod_phase
+                    )
+                function["status"] = {"state": state}
+                if isinstance(function["metadata"].get("updated"), datetime.datetime):
+                    function["metadata"]["updated"] = function["metadata"][
+                        "updated"
+                    ].isoformat()
+                function["step_kind"] = _step.step_type
+                steps.append(function)
+
+        step_methods = {
+            mlrun_pipelines.common.constants.PipelineRunType.run: _add_run_step,
+            mlrun_pipelines.common.constants.PipelineRunType.build: _add_deploy_function_step,
+            mlrun_pipelines.common.constants.PipelineRunType.deploy: _add_deploy_function_step,
+        }
+
+        if not workflow_id:
+            return steps
+
+        workflow_manifest = Workflow._get_workflow_manifest(workflow_id)
+        if not workflow_manifest:
+            return steps
+
+        try:
+            for step in workflow_manifest.get_steps():
+                step_method = step_methods.get(step.step_type)
+                if step_method:
+                    step_method(step)
+            return steps
+        except Exception:
+            # If we fail to read the pipeline steps, we will return the list of runs that have the same workflow id
+            logger.warning(
+                "Failed to extract workflow steps from workflow manifest, "
+                "returning all runs with the workflow id label",
+                workflow_id=workflow_id,
+                traceback=traceback.format_exc(),
+            )
+            return db.list_runs(
+                project=project,
+                labels=f"workflow={workflow_id}",
+            )
+
+    @staticmethod
+    def _extract_function_uri(function_uri: str) -> tuple[str, str, str]:
+        """
+        Extract the project, name, and hash key from a function uri.
+        Examples:
+            - "project/name@hash_key" returns project, name, hash_key
+            - "project/name returns" project, name, ""
+        """
+        project, name, hash_key = None, None, None
+        hashed_pattern = r"^(.+)/(.+)@(.+)$"
+        pattern = r"^(.+)/(.+)$"
+        match = re.match(hashed_pattern, function_uri)
+        if match:
+            project, name, hash_key = match.groups()
+        else:
+            match = re.match(pattern, function_uri)
+            if match:
+                project, name = match.groups()
+                hash_key = ""
+        return project, name, hash_key
+
+    @staticmethod
+    def _get_workflow_manifest(
+        workflow_id: str,
+    ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
+        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
+
+        # arbitrary timeout of 5 seconds, the workflow should be done by now
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        if not kfp_run:
+            return None
+
+        kfp_run = mlrun_pipelines.models.PipelineRun(kfp_run)
+        return kfp_run.workflow_manifest()

mlrun/utils/notifications/notification/webhook.py

@@ -118,6 +118,9 @@ class WebhookNotification(NotificationBase):
 
         if isinstance(override_body, dict):
             for key, value in override_body.items():
+                if not isinstance(value, str):
+                    # If the value is not a string, we don't want to parse it
+                    continue
                 if re.search(r"{{\s*runs\s*}}", value):
                     str_parsed_runs = parse_runs()
                     override_body[key] = re.sub(

mlrun/utils/notifications/notification_pusher.py

@@ -15,7 +15,6 @@
 import asyncio
 import datetime
 import os
-import re
 import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
@@ -31,11 +30,7 @@ import mlrun.model
 import mlrun.utils.helpers
 import mlrun.utils.notifications.notification as notification_module
 import mlrun.utils.notifications.notification.base as base
-import mlrun_pipelines.common.constants
-import mlrun_pipelines.common.ops
-import mlrun_pipelines.models
-import mlrun_pipelines.utils
-from mlrun.utils import logger
+from mlrun.utils import Workflow, logger
 from mlrun.utils.condition_evaluator import evaluate_condition_in_separate_process
 
 
@@ -283,7 +278,9 @@ class NotificationPusher(_NotificationPusherBase):
             custom_message = (
                 f" (workflow: {run.metadata.labels['workflow']}){custom_message}"
             )
-            runs.extend(self.get_workflow_steps(run))
+            project = run.metadata.project
+            workflow_id = run.status.results.get("workflow_id", None)
+            runs.extend(Workflow.get_workflow_steps(workflow_id, project))
 
         message = (
             self.messages.get(run.state(), "").format(resource=resource)
@@ -442,131 +439,6 @@ class NotificationPusher(_NotificationPusherBase):
             mask_params=False,
         )
 
-    def get_workflow_steps(self, run: mlrun.model.RunObject) -> list:
-        steps = []
-        db = mlrun.get_run_db()
-
-        def _add_run_step(_step: mlrun_pipelines.models.PipelineStep):
-            try:
-                _run = db.list_runs(
-                    project=run.metadata.project,
-                    labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
-                )[0]
-            except IndexError:
-                _run = {
-                    "metadata": {
-                        "name": _step.display_name,
-                        "project": run.metadata.project,
-                    },
-                }
-            _run["step_kind"] = _step.step_type
-            if _step.skipped:
-                _run.setdefault("status", {})["state"] = (
-                    runtimes_constants.RunStates.skipped
-                )
-            steps.append(_run)
-
-        def _add_deploy_function_step(_step: mlrun_pipelines.models.PipelineStep):
-            project, name, hash_key = self._extract_function_uri(
-                _step.get_annotation("mlrun/function-uri")
-            )
-            if name:
-                try:
-                    function = db.get_function(
-                        project=project, name=name, hash_key=hash_key
-                    )
-                except mlrun.errors.MLRunNotFoundError:
-                    # If the function is not found (if build failed for example), we will create a dummy
-                    # function object for the notification to display the function name
-                    function = {
-                        "metadata": {
-                            "name": name,
-                            "project": project,
-                            "hash_key": hash_key,
-                        },
-                    }
-                pod_phase = _step.phase
-                if _step.skipped:
-                    state = mlrun.common.schemas.FunctionState.skipped
-                else:
-                    state = runtimes_constants.PodPhases.pod_phase_to_run_state(
-                        pod_phase
-                    )
-                function["status"] = {"state": state}
-                if isinstance(function["metadata"].get("updated"), datetime.datetime):
-                    function["metadata"]["updated"] = function["metadata"][
-                        "updated"
-                    ].isoformat()
-                function["step_kind"] = _step.step_type
-                steps.append(function)
-
-        step_methods = {
-            mlrun_pipelines.common.constants.PipelineRunType.run: _add_run_step,
-            mlrun_pipelines.common.constants.PipelineRunType.build: _add_deploy_function_step,
-            mlrun_pipelines.common.constants.PipelineRunType.deploy: _add_deploy_function_step,
-        }
-
-        workflow_id = run.status.results.get("workflow_id", None)
-        if not workflow_id:
-            return steps
-
-        workflow_manifest = self._get_workflow_manifest(workflow_id)
-        if not workflow_manifest:
-            return steps
-
-        try:
-            for step in workflow_manifest.get_steps():
-                step_method = step_methods.get(step.step_type)
-                if step_method:
-                    step_method(step)
-            return steps
-        except Exception:
-            # If we fail to read the pipeline steps, we will return the list of runs that have the same workflow id
-            logger.warning(
-                "Failed to extract workflow steps from workflow manifest, "
-                "returning all runs with the workflow id label",
-                workflow_id=workflow_id,
-                traceback=traceback.format_exc(),
-            )
-            return db.list_runs(
-                project=run.metadata.project,
-                labels=f"workflow={workflow_id}",
-            )
-
-    @staticmethod
-    def _get_workflow_manifest(
-        workflow_id: str,
-    ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
-
-        # arbitrary timeout of 5 seconds, the workflow should be done by now
-        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
-        if not kfp_run:
-            return None
-
-        kfp_run = mlrun_pipelines.models.PipelineRun(kfp_run)
-        return kfp_run.workflow_manifest()
-
-    def _extract_function_uri(self, function_uri: str) -> tuple[str, str, str]:
-        """
-        Extract the project, name, and hash key from a function uri.
-        Examples:
-            - "project/name@hash_key" returns project, name, hash_key
-            - "project/name returns" project, name, ""
-        """
-        project, name, hash_key = None, None, None
-        hashed_pattern = r"^(.+)/(.+)@(.+)$"
-        pattern = r"^(.+)/(.+)$"
-        match = re.match(hashed_pattern, function_uri)
-        if match:
-            project, name, hash_key = match.groups()
-        else:
-            match = re.match(pattern, function_uri)
-            if match:
-                project, name = match.groups()
-                hash_key = ""
-        return project, name, hash_key
-
 
 class CustomNotificationPusher(_NotificationPusherBase):
     def __init__(self, notification_types: typing.Optional[list[str]] = None):

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "2d5cc0ee3dacf5d31a3b6a2158cd2e2b4a378b56",
-  "version": "1.8.0-rc18"
+  "git_commit": "6671b915fe08ba2b9afa054585f86b955b48b671",
+  "version": "1.8.0-rc19"
 }

{mlrun-1.8.0rc18.dist-info → mlrun-1.8.0rc19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.8.0rc18
+Version: 1.8.0rc19
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.8.0rc18.dist-info → mlrun-1.8.0rc19.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=7vuMpUiigXXDrghLRq680LKWy1faC0kQyGCZb_7cwyE,7473
 mlrun/__main__.py,sha256=o65gXHhmFA9GV_n2mqmAO80nW3MAwo_s7j80IKgCzRE,45949
-mlrun/config.py,sha256=jjjpcnTjmYPc5ptvetYMJ-BPWaOiQW1b9Phdh9A8wo0,70605
+mlrun/config.py,sha256=zHp7wtbaewFbHYsA_SLxwuwSDYwrcL6A3knWSPcRtAA,70547
 mlrun/errors.py,sha256=5raKb1PXQpTcIvWQ4sr1qn2IS7P_GT_FydBJ0dXkVuc,8097
 mlrun/execution.py,sha256=Up9U6xonTElRIaesF9Vej2JK1Isk2AZNK9ke0XcF5Dg,49030
 mlrun/features.py,sha256=ReBaNGsBYXqcbgI012n-SO_j6oHIbk_Vpv0CGPXbUmo,15842
@@ -42,7 +42,7 @@ mlrun/common/runtimes/constants.py,sha256=Mok3m9Rv182TTMp7uYNfWalm9Xcz86yva-4fTx
 mlrun/common/schemas/__init__.py,sha256=PBuIAhXSkVEVxxKcv5hR_xvTwNAUqxOXHVPugOoWTyM,5386
 mlrun/common/schemas/alert.py,sha256=G9lFTXFYDor-RVLpJxMorIPlLWr_-GYCFKRN9DkKwXs,10124
 mlrun/common/schemas/api_gateway.py,sha256=3a0QxECLmoDkD5IiOKtXJL-uiWB26Hg55WMA3nULYuI,7127
-mlrun/common/schemas/artifact.py,sha256=i29BeZ4MoOLMST3WlApX1Nli0vy-M7Zj_oTm8jySlw4,3805
+mlrun/common/schemas/artifact.py,sha256=f0NPsoZmA-WD9RtN-dcKFW6KuV0PPQB25A2psF7LbP8,4013
 mlrun/common/schemas/auth.py,sha256=AGbBNvQq_vcvhX_NLqbT-QPHL4BAJMB3xwBXW7cFvpo,6761
 mlrun/common/schemas/background_task.py,sha256=ofWRAQGGEkXEu79Dbw7tT_5GPolR09Lc3Ebg2r0fT24,1728
 mlrun/common/schemas/client_spec.py,sha256=CCdAMwRS2DFKddxNSulWcRDwp3mrE7dDdKeAD6djxLo,2856
@@ -71,10 +71,10 @@ mlrun/common/schemas/schedule.py,sha256=LTWdZ4FvKDGkmmfyqKoBQ36VFqnnyIYLnq1I6qrT
 mlrun/common/schemas/secret.py,sha256=CCxFYiPwJtDxwg2VVJH9nUG9cAZ2a34IjeuaWv-BYlc,1487
 mlrun/common/schemas/tag.py,sha256=HRZi5QZ4vVGaCr2AMk9eJgcNiAIXmH4YDc8a4fvF770,893
 mlrun/common/schemas/workflow.py,sha256=rwYzDJYxpE9k4kC88j_eUCmqK4ZsWV_h-_nli7Fs7Ow,2078
-mlrun/common/schemas/model_monitoring/__init__.py,sha256=noj7DPPia59PBWORsQZO20Ym0yhUwyoNJ3EeG8gVxlQ,1875
-mlrun/common/schemas/model_monitoring/constants.py,sha256=VA1fEETbiyTKGiAG4WBSzo69Revd1_sgr8to6N0rp6U,11543
+mlrun/common/schemas/model_monitoring/__init__.py,sha256=jz0fvdn8BEecgUCKhiSNH6QtFhSW4O19Ql9KXo0AxOg,1900
+mlrun/common/schemas/model_monitoring/constants.py,sha256=KHpZiTruqr1iQ4XfEz8Ptj9ytCBuvwOXp30YTt5YQws,11834
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=Rq10KKOyyUYr7qOQFZfwGZtUim0LY9O0LQ5uc9jmIVQ,1562
-mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=GVWePSdaKyJm7-QdBiAQdMymlp2QgZIJwXO9EA6Q7AI,11727
+mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=j60-_puybc4yLlmoWkZ04m6PuY4p5yfzVJpPo0_n1sY,11725
 mlrun/data_types/__init__.py,sha256=unRo9GGwCmj0hBKBRsXJ2P4BzpQaddlQTvIrVQaKluI,984
 mlrun/data_types/data_types.py,sha256=0_oKLC6-sXL2_nnaDMP_HSXB3fD1nJAG4J2Jq6sGNNw,4998
 mlrun/data_types/infer.py,sha256=KdaRgWcqvLkuLjXrMuDr3ik6WY7JP5wJO0Yii_Vl5kw,6173
@@ -107,9 +107,9 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=ZCvjAN68L4kRZsytkuqqz4JTLROxczmbXjJwU1vaK8c,29855
+mlrun/db/base.py,sha256=aFtDl4J_yeEovso7uvnnn9KLYMnIRiS52qM1uemdG8k,30218
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=eGuQ05kwU8h_DbtuzlPo1aBMU4ZbfIsrRPWxiqb4v_Y,226141
+mlrun/db/httpdb.py,sha256=VXp3ETu5fl_-6lEF_bsUerKESuRXbAFxLYvduH6DlKs,228070
 mlrun/db/nopdb.py,sha256=v285LHP_Onfuo8KRF078IAPHIXTeEhQsU58QoY7x-b0,26673
 mlrun/feature_store/__init__.py,sha256=AVnY2AFUNc2dKxLLUMx2K3Wo1eGviv0brDcYlDnmtf4,1506
 mlrun/feature_store/api.py,sha256=qkojZpzqGAn3r9ww0ynBRKOs8ji8URaK4DSYD4SE-CE,50395
@@ -219,7 +219,7 @@ mlrun/model_monitoring/__init__.py,sha256=ELy7njEtZnz09Dc6PGZSFFEGtnwI15bJNWM3Pj
 mlrun/model_monitoring/api.py,sha256=nH5aEUkmUEJF0CurrWJxmxVv1tQed2yzCLhQByG1L00,28561
 mlrun/model_monitoring/controller.py,sha256=dBfZQswF67vqeUFnmgsm9jU_5sOs9dLwMPEiYHG-Kk8,19786
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=Ul7-EHqzbf1GprB53Tf4WcbOHdD65sK3BgF2mpPYakg,17938
+mlrun/model_monitoring/helpers.py,sha256=6L-IO4EUAYoAf74snGNiGDln_p77OIdxVdrf392VBzY,17936
 mlrun/model_monitoring/stream_processing.py,sha256=ltCVgo_b3yay16CUbqeGkRfzCHZSn14lVeBng5m9keY,31738
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=vbL7bqTyNu8q4bNcebX72sUMybVDAoTWg-CXq4fov3Y,8429
@@ -234,15 +234,15 @@ mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJC
 mlrun/model_monitoring/db/_schedules.py,sha256=NTO1rbSyhW1JidpBDSN39ZBD0ctp5pbJFYQwxKRIRrs,5821
 mlrun/model_monitoring/db/_stats.py,sha256=VVMWLMqG3Us3ozBkLaokJF22Ewv8WKmVE1-OvS_g9vA,6943
 mlrun/model_monitoring/db/tsdb/__init__.py,sha256=_ejhGA-bi5-6kEMEVYUN5TLxND8hTj4gMG7WrYIkpxM,4585
-mlrun/model_monitoring/db/tsdb/base.py,sha256=EWm-56AzuLSH1JV6DAhSRu9sUel5YD6Ch1n5tgUHODo,20983
+mlrun/model_monitoring/db/tsdb/base.py,sha256=JjLBzZXE4ZxtBmihVXjUYZ2HKmgqX03ZhUynXp4948o,25372
 mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsVsrb9otX9a7O4,1189
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=6de8P0CJMqe7PGttoZNt9UtrbBcJnpIp82hk_MbtepA,12477
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=Vbx5u4Y2ZbzbRQDwMIyz-9S0OgkGKw_gZTvWvF8Jp7I,29599
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=IEpJknjqx_LYcZjIccPuujOfEruXsRm8c8YzYaWWNEQ,30175
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=rZGakT22t3vNO7K9Ez1Au1UM5-E8c2Fuw4B6wuIIWPM,35896
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=9j_TWS_3OXTwjiYw9jb92z8URHUXDW7hCgwZnpb2P8c,36834
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
 mlrun/package/__init__.py,sha256=v7VDyK9kDOOuDvFo4oiGV2fx-vM1KL7fdN9pGLakhUQ,7008
@@ -267,7 +267,7 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=VXUlMrouFTls-I-bMhdN5pPfQ34TR7bFQ-NUSWNvl84,20029
 mlrun/projects/pipelines.py,sha256=vZpyiERUzwPMS7NCC5ghI0KB_DItIddr7MMWGTwLawY,47437
-mlrun/projects/project.py,sha256=fApkLLMJKPQi-Q69O-pZP5sh6eVsLmBt1KqZUFwcZoM,228225
+mlrun/projects/project.py,sha256=WKIgBy1nqHxQ4d2AHId6wptnGeYzfZTY61K8pOQUrvI,228386
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
 mlrun/runtimes/base.py,sha256=Yt2l7srrXjK783cunBEKH0yQxQZRH8lkedXNOXuLbbo,37841
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
@@ -318,7 +318,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=y3zC9QS7z5mLuvyQ6vFd6sJnikbgtDwrBvieQq0sovY,7359
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=3E7cQFcvcKdXmMnjsFY5r6cY6S0LxoFOrAkN6_jYw1s,64237
+mlrun/utils/helpers.py,sha256=aPMmd5dEm7PIbx7maXtNGhNKzEHp8JK8uSmOgTtoDbc,69549
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=_v4UTv1-STzC2c6aAWAa0NNl9STQoBYbR3OHgAiL41s,14606
 mlrun/utils/regex.py,sha256=IQqwPna6Z8J31xkTUduYbGk48GkQBUJFZSuxAWm1pzU,5162
@@ -327,7 +327,7 @@ mlrun/utils/singleton.py,sha256=p1Y-X0mPSs_At092GS-pZCA8CTR62HOqPU07_ZH6-To,869
 mlrun/utils/v3io_clients.py,sha256=0aCFiQFBmgdSeLzJr_nEP6SG-zyieSgH8RdtcUq4dc0,1294
 mlrun/utils/vault.py,sha256=xUiKL17dCXjwQJ33YRzQj0oadUXATlFWPzKKYAESoQk,10447
 mlrun/utils/notifications/__init__.py,sha256=eUzQDBxSQmMZASRY-YAnYS6tL5801P0wEjycp3Dvoe0,990
-mlrun/utils/notifications/notification_pusher.py,sha256=Y30ZG8MDk0oxPPHPjqd9kDxQaOewTuE5uwZhxlM5V-s,29269
+mlrun/utils/notifications/notification_pusher.py,sha256=WN7RMfaZtCoG3bNfMjub1phbvJ2Xw8lKFiq2GStnKGw,24132
 mlrun/utils/notifications/notification/__init__.py,sha256=9Rfy6Jm8n0LaEDO1VAQb6kIbr7_uVuQhK1pS_abELIY,2581
 mlrun/utils/notifications/notification/base.py,sha256=VOgrzRakRfjYYBqvkc0cgEC5pl7KMidP7u-TL4HpGCY,5280
 mlrun/utils/notifications/notification/console.py,sha256=ICbIhOf9fEBJky_3j9TFiKAewDGyDHJr9l4VeT7G2sc,2745
@@ -335,13 +335,13 @@ mlrun/utils/notifications/notification/git.py,sha256=t2lqRrPRBO4awf_uhxJreH9Cpcb
 mlrun/utils/notifications/notification/ipython.py,sha256=9uZvI1uOLFaNuAsfJPXmL3l6dOzFoWdBK5GYNYFAfks,2282
 mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAqp1vij7C6aRJ9h2mgs,6012
 mlrun/utils/notifications/notification/slack.py,sha256=NKV4RFiY3gLsS8uPppgniPLyag8zJ9O1VhixoXkM7kw,7108
-mlrun/utils/notifications/notification/webhook.py,sha256=M-pSBM2VTKVUPRERocjORlH6mKqo1K9ihVL5Qrn2GyM,4789
+mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=fuKq-DQzj4VU91pAfeQ6y9--Wzwya8hmUG1brhepAzQ,89
+mlrun/utils/version/version.json,sha256=9e_fs0UZTGdzgD6yjggI41O7Hv5n_tbuYDMtJsSNspY,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc18.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc18.dist-info/METADATA,sha256=LmjjU6uSheHxPOGP4V2giVzBleaHV45fl9bCB_xqyBo,24885
-mlrun-1.8.0rc18.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-mlrun-1.8.0rc18.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc18.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc18.dist-info/RECORD,,
+mlrun-1.8.0rc19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc19.dist-info/METADATA,sha256=A2YOQIMx1P8pckWVn0bKsr8oEpcWxVbhkXhPxTplJMM,24885
+mlrun-1.8.0rc19.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+mlrun-1.8.0rc19.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc19.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc19.dist-info/RECORD,,