mlrun 1.8.0rc19__py3-none-any.whl → 1.8.0rc21__py3-none-any.whl

mlrun/__main__.py

@@ -32,6 +32,7 @@ from tabulate import tabulate
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
 
@@ -304,6 +305,7 @@ def run(
                 update_in(runtime, "spec.build.code_origin", url_file)
     elif runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -515,6 +517,7 @@ def build(
 
     if runtime:
         runtime = py_eval(runtime)
+        runtime = mlrun.utils.helpers.as_dict(runtime)
         if not isinstance(runtime, dict):
             print(f"Runtime parameter must be a dict, not {type(runtime)}")
             exit(1)
@@ -662,6 +665,8 @@ def deploy(
         runtime = py_eval(spec)
     else:
         runtime = {}
+
+    runtime = mlrun.utils.helpers.as_dict(runtime)
     if not isinstance(runtime, dict):
         print(f"Runtime parameter must be a dict, not {type(runtime)}")
         exit(1)

mlrun/common/runtimes/constants.py

@@ -214,6 +214,23 @@ class RunStates:
             RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
         }[run_state]
 
+    @staticmethod
+    def pipeline_run_status_to_run_state(pipeline_run_status):
+        if pipeline_run_status not in mlrun_pipelines.common.models.RunStatuses.all():
+            raise ValueError(f"Invalid pipeline run status: {pipeline_run_status}")
+        return {
+            mlrun_pipelines.common.models.RunStatuses.succeeded: RunStates.completed,
+            mlrun_pipelines.common.models.RunStatuses.failed: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.running: RunStates.running,
+            mlrun_pipelines.common.models.RunStatuses.pending: RunStates.pending,
+            mlrun_pipelines.common.models.RunStatuses.canceled: RunStates.aborted,
+            mlrun_pipelines.common.models.RunStatuses.canceling: RunStates.aborting,
+            mlrun_pipelines.common.models.RunStatuses.skipped: RunStates.skipped,
+            mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified: RunStates.unknown,
+            mlrun_pipelines.common.models.RunStatuses.error: RunStates.error,
+            mlrun_pipelines.common.models.RunStatuses.paused: RunStates.unknown,
+        }[pipeline_run_status]
+
 
 # TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):

mlrun/common/schemas/model_monitoring/constants.py

@@ -61,6 +61,7 @@ class ModelEndpointSchema(MonitoringStrEnum):
     STATE = "state"
     MONITORING_MODE = "monitoring_mode"
     FIRST_REQUEST = "first_request"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
 
     # status - operative
     LAST_REQUEST = "last_request"
@@ -137,6 +138,10 @@ class EventFieldType:
     SAMPLE_PARQUET_PATH = "sample_parquet_path"
     TIME = "time"
     TABLE_COLUMN = "table_column"
+    SAMPLING_PERCENTAGE = "sampling_percentage"
+    SAMPLING_RATE = "sampling_rate"
+    ESTIMATED_PREDICTION_COUNT = "estimated_prediction_count"
+    EFFECTIVE_SAMPLE_COUNT = "effective_sample_count"
 
 
 class FeatureSetFeatures(MonitoringStrEnum):

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -160,6 +160,7 @@ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
     state: Optional[str] = "unknown"  # will be updated according to the function state
     first_request: Optional[datetime] = None
     monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled
+    sampling_percentage: Optional[float] = 100
 
     # operative
     last_request: Optional[datetime] = None
@@ -177,6 +178,7 @@ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
             "monitoring_mode",
             "first_request",
             "last_request",
+            "sampling_percentage",
         ]
 
 

mlrun/db/base.py

@@ -68,6 +68,15 @@ class RunDBInterface(ABC):
     ):
         pass
 
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        pass
+
     @abstractmethod
     def read_run(
         self,

mlrun/db/httpdb.py

@@ -780,9 +780,84 @@ class HTTPRunDB(RunDBInterface):
         )
         if response.status_code == http.HTTPStatus.ACCEPTED:
             background_task = mlrun.common.schemas.BackgroundTask(**response.json())
-            return self._wait_for_background_task_to_reach_terminal_state(
+            background_task = self._wait_for_background_task_to_reach_terminal_state(
                 background_task.metadata.name, project=project
             )
+            if (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.succeeded
+            ):
+                logger.info(
+                    "Notifications for the run have been pushed",
+                    project=project,
+                    run_id=uid,
+                )
+            elif (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.failed
+            ):
+                logger.error(
+                    "Failed to push run notifications",
+                    project=project,
+                    run_id=uid,
+                    error=background_task.status.error,
+                )
+        return None
+
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        """
+        Push notifications for a pipeline.
+
+        :param pipeline_id: Unique ID of the pipeline(KFP).
+        :param project: Project that the run belongs to.
+        :param notifications: List of notifications to push.
+        :returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
+        """
+        if notifications is None or type(notifications) is not list:
+            raise MLRunInvalidArgumentError(
+                "The 'notifications' parameter must be a list."
+            )
+
+        project = project or config.default_project
+
+        response = self.api_call(
+            "POST",
+            path=f"projects/{project}/pipelines/{pipeline_id}/push-notifications",
+            error="Failed push notifications",
+            body=_as_json([notification.to_dict() for notification in notifications]),
+            timeout=timeout,
+        )
+        if response.status_code == http.HTTPStatus.ACCEPTED:
+            background_task = mlrun.common.schemas.BackgroundTask(**response.json())
+            background_task = self._wait_for_background_task_to_reach_terminal_state(
+                background_task.metadata.name, project=project
+            )
+            if (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.succeeded
+            ):
+                logger.info(
+                    "Pipeline notifications have been pushed",
+                    project=project,
+                    pipeline_id=pipeline_id,
+                )
+            elif (
+                background_task.status.state
+                == mlrun.common.schemas.BackgroundTaskState.failed
+            ):
+                logger.error(
+                    "Failed to push pipeline notifications",
+                    project=project,
+                    pipeline_id=pipeline_id,
+                    error=background_task.status.error,
+                )
+
         return None
 
     def read_run(

mlrun/db/nopdb.py

@@ -84,6 +84,15 @@ class NopDB(RunDBInterface):
     ):
         pass
 
+    def push_pipeline_notifications(
+        self,
+        pipeline_id,
+        project="",
+        notifications=None,
+        timeout=45,
+    ):
+        pass
+
     def list_runtime_resources(
         self,
         project: Optional[str] = None,

mlrun/frameworks/_common/model_handler.py

@@ -976,7 +976,6 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
             custom_objects_map_json,
             local_path=custom_objects_map_json,
             artifact_path=self._context.artifact_path,
-            db_key=False,
         )
 
         # Zip the custom objects directory:
@@ -997,7 +996,6 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
             custom_objects_zip,
             local_path=custom_objects_zip,
             artifact_path=self._context.artifact_path,
-            db_key=False,
         )
 
         return artifacts

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -298,6 +298,8 @@ class Predictions(TDEngineSchema):
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.EventFieldType.LATENCY: _TDEngineColumn.FLOAT,
             mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_1000,
+            mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT: _TDEngineColumn.FLOAT,
+            mm_schemas.EventFieldType.EFFECTIVE_SAMPLE_COUNT: _TDEngineColumn.INT,
         }
         tags = {
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -165,7 +165,7 @@ class TDEngineConnector(TSDBConnector):
         return datetime.fromisoformat(val) if isinstance(val, str) else val
 
     @staticmethod
-    def _get_endpoint_filter(endpoint_id: typing.Union[str, list[str]]):
+    def _get_endpoint_filter(endpoint_id: typing.Union[str, list[str]]) -> str:
         if isinstance(endpoint_id, str):
             return f"endpoint_id='{endpoint_id}'"
         elif isinstance(endpoint_id, list):
@@ -206,6 +206,8 @@ class TDEngineConnector(TSDBConnector):
                 columns=[
                     mm_schemas.EventFieldType.LATENCY,
                     mm_schemas.EventKeyMetrics.CUSTOM_METRICS,
+                    mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT,
+                    mm_schemas.EventFieldType.EFFECTIVE_SAMPLE_COUNT,
                 ],
                 tag_cols=[
                     mm_schemas.EventFieldType.ENDPOINT_ID,
@@ -483,7 +485,7 @@ class TDEngineConnector(TSDBConnector):
             table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
-            columns=[mm_schemas.EventFieldType.LATENCY],
+            columns=[mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT],
             filter_query=f"endpoint_id='{endpoint_id}'",
             agg_funcs=agg_funcs,
             interval=aggregation_window,
@@ -503,10 +505,10 @@ class TDEngineConnector(TSDBConnector):
             df["_wend"] = pd.to_datetime(df["_wend"])
             df.set_index("_wend", inplace=True)
 
-        latency_column = (
-            f"{agg_funcs[0]}({mm_schemas.EventFieldType.LATENCY})"
+        estimated_prediction_count = (
+            f"{agg_funcs[0]}({mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT})"
             if agg_funcs
-            else mm_schemas.EventFieldType.LATENCY
+            else mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT
         )
 
         return mm_schemas.ModelEndpointMonitoringMetricValues(
@@ -514,7 +516,7 @@ class TDEngineConnector(TSDBConnector):
             values=list(
                 zip(
                     df.index,
-                    df[latency_column],
+                    df[estimated_prediction_count],
                 )
             ),  # pyright: ignore[reportArgumentType]
         )
@@ -525,9 +527,7 @@ class TDEngineConnector(TSDBConnector):
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
@@ -538,7 +538,7 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.EventFieldType.TIME,
                 mm_schemas.EventFieldType.LATENCY,
             ],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             timestamp_column=mm_schemas.EventFieldType.TIME,
             agg_funcs=["last"],
             group_by=mm_schemas.EventFieldType.ENDPOINT_ID,
@@ -567,9 +567,7 @@ class TDEngineConnector(TSDBConnector):
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
         start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
@@ -580,7 +578,7 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_STATUS,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
             agg_funcs=["max"],
             group_by=mm_schemas.EventFieldType.ENDPOINT_ID,
@@ -678,9 +676,8 @@ class TDEngineConnector(TSDBConnector):
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
+        filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} = '{mm_schemas.EventFieldType.INFER_ERROR}'"
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=self.tables[mm_schemas.TDEngineSuperTables.ERRORS].super_table,
@@ -691,8 +688,7 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             agg_funcs=["count"],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]}) "
-            f"AND {mm_schemas.EventFieldType.ERROR_TYPE} = '{mm_schemas.EventFieldType.INFER_ERROR}'",
+            filter_query=filter_query,
             group_by=mm_schemas.EventFieldType.ENDPOINT_ID,
             preform_agg_columns=[mm_schemas.EventFieldType.MODEL_ERROR],
         )

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -33,7 +33,7 @@ _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
-V3IO_MEPS_LIMIT = 50  # TODO remove limitation after fixing ML-8886
+V3IO_MEPS_LIMIT = 200
 
 
 def _is_no_schema_error(exc: v3io_frames.Error) -> bool:
@@ -234,6 +234,8 @@ class V3IOTSDBConnector(TSDBConnector):
             columns=[
                 mm_schemas.EventFieldType.LATENCY,
                 mm_schemas.EventFieldType.LAST_REQUEST_TIMESTAMP,
+                mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT,
+                mm_schemas.EventFieldType.EFFECTIVE_SAMPLE_COUNT,
             ],
             index_cols=[
                 mm_schemas.EventFieldType.ENDPOINT_ID,
@@ -580,14 +582,18 @@ class V3IOTSDBConnector(TSDBConnector):
         )
 
     @staticmethod
-    def _get_endpoint_filter(endpoint_id: Union[str, list[str]]):
+    def _get_endpoint_filter(endpoint_id: Union[str, list[str]]) -> Optional[str]:
         if isinstance(endpoint_id, str):
             return f"endpoint_id=='{endpoint_id}'"
         elif isinstance(endpoint_id, list):
             if len(endpoint_id) > V3IO_MEPS_LIMIT:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"Filtering more than {V3IO_MEPS_LIMIT} model endpoints in the V3IO connector is not supported."
+                logger.info(
+                    "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
+                    "retrieving all the model endpoints from the db.",
+                    limit=V3IO_MEPS_LIMIT,
+                    amount=len(endpoint_id),
                 )
+                return None
             return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -737,7 +743,7 @@ class V3IOTSDBConnector(TSDBConnector):
             table=mm_schemas.FileTargetKind.PREDICTIONS,
             start=start,
             end=end,
-            columns=[mm_schemas.EventFieldType.LATENCY],
+            columns=[mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT],
             filter_query=f"endpoint_id=='{endpoint_id}'",
             agg_funcs=agg_funcs,
             sliding_window_step=aggregation_window,
@@ -751,10 +757,10 @@ class V3IOTSDBConnector(TSDBConnector):
                 type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
             )
 
-        latency_column = (
-            f"{agg_funcs[0]}({mm_schemas.EventFieldType.LATENCY})"
+        estimated_prediction_count = (
+            f"{agg_funcs[0]}({mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT})"
             if agg_funcs
-            else mm_schemas.EventFieldType.LATENCY
+            else mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT
         )
 
         return mm_schemas.ModelEndpointMonitoringMetricValues(
@@ -762,7 +768,7 @@ class V3IOTSDBConnector(TSDBConnector):
             values=list(
                 zip(
                     df.index,
-                    df[latency_column],
+                    df[estimated_prediction_count],
                 )
             ),  # pyright: ignore[reportArgumentType]
         )
@@ -773,15 +779,13 @@ class V3IOTSDBConnector(TSDBConnector):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.PREDICTIONS,
             start=start,
             end=end,
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:
@@ -808,9 +812,7 @@ class V3IOTSDBConnector(TSDBConnector):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
         start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
@@ -818,7 +820,7 @@ class V3IOTSDBConnector(TSDBConnector):
             start=start,
             end=end,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["max"],
             group_by="endpoint_id",
         )
@@ -883,17 +885,18 @@ class V3IOTSDBConnector(TSDBConnector):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
+        if filter_query:
+            filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'"
+        else:
+            filter_query = f"{mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}' z"
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.ERRORS,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.ERROR_COUNT],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]}) "
-            f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'",
+            filter_query=filter_query,
             agg_funcs=["count"],
         )
         if not df.empty:
@@ -912,9 +915,7 @@ class V3IOTSDBConnector(TSDBConnector):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
         start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
@@ -922,7 +923,7 @@ class V3IOTSDBConnector(TSDBConnector):
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["avg"],
         )
         if not df.empty:

mlrun/model_monitoring/stream_processing.py

@@ -430,6 +430,10 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             if not isinstance(feature, list):
                 feature = [feature]
 
+            effective_sample_count, estimated_prediction_count = (
+                self._get_effective_and_estimated_counts(event=event)
+            )
+
             events.append(
                 {
                     EventFieldType.FUNCTION_URI: function_uri,
@@ -453,6 +457,8 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
                     EventFieldType.ENTITIES: event.get("request", {}).get(
                         EventFieldType.ENTITIES, {}
                     ),
+                    EventFieldType.EFFECTIVE_SAMPLE_COUNT: effective_sample_count,
+                    EventFieldType.ESTIMATED_PREDICTION_COUNT: estimated_prediction_count,
                 }
             )
 
@@ -507,6 +513,20 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
         self.error_count[endpoint_id] += 1
         return False
 
+    @staticmethod
+    def _get_effective_and_estimated_counts(event):
+        """
+        Calculate the `effective_sample_count` and the `estimated_prediction_count` based on the event's
+        sampling percentage. These values will be stored in the TSDB target.
+        Note that In non-batch serving, the `effective_sample_count` is always set to 1. In addition, when the sampling
+        percentage is 100%, the `estimated_prediction_count` is equal to the `effective_sample_count`.
+        """
+        effective_sample_count = event.get(EventFieldType.EFFECTIVE_SAMPLE_COUNT, 1)
+        estimated_prediction_count = effective_sample_count * (
+            100 / event.get(EventFieldType.SAMPLING_PERCENTAGE, 100)
+        )
+        return effective_sample_count, estimated_prediction_count
+
 
 def is_not_none(field: typing.Any, dict_path: list[str]):
     if field is not None:
@@ -672,6 +692,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     )
                 )
             self.first_request[endpoint_id] = True
+
         if attributes_to_update:
             logger.info(
                 "Updating endpoint record",

mlrun/projects/pipelines.py

@@ -523,11 +523,12 @@ class _PipelineRunner(abc.ABC):
         text = _PipelineRunner._generate_workflow_finished_message(
             run.run_id, errors_counter, run._state
         )
-
         notifiers = notifiers or project.notifiers
         if notifiers:
             notifiers.push(text, "info", runs)
 
+        project.push_pipeline_notification_kfp_runner(run.run_id, run._state, text)
+
         if raise_error:
             raise raise_error
         return state or run._state, errors_counter, text
@@ -620,6 +621,8 @@ class _KFPRunner(_PipelineRunner):
                 params.update(notification.secret_params)
                 project.notifiers.add_notification(notification.kind, params)
 
+            project.spec.notifications = notifications
+
         run_id = _run_pipeline(
             workflow_handler,
             project=project.metadata.name,
@@ -647,13 +650,23 @@ class _KFPRunner(_PipelineRunner):
                     exc_info=err_to_str(exc),
                 )
 
-        # TODO: we should check how can we get the run uid when we don't have the context (for example on
-        #  mlrun.load_project() and later call directly to project.run)
+        # Pushing only relevant notification for the client (ipython and console)
+        project.notifiers.push_pipeline_start_message_from_client(
+            project.metadata.name, pipeline_id=run_id
+        )
+
         if context:
             project.notifiers.push_pipeline_start_message(
                 project.metadata.name,
                 context.uid,
             )
+        else:
+            project.push_pipeline_notification_kfp_runner(
+                run_id,
+                mlrun_pipelines.common.models.RunStatuses.running,
+                f"Workflow {run_id} started in project {project.metadata.name}",
+                notifications,
+            )
         pipeline_context.clear()
         return _PipelineRunStatus(run_id, cls, project=project, workflow=workflow_spec)
 

mlrun/projects/project.py

@@ -83,6 +83,7 @@ from ..artifacts import (
     ModelArtifact,
 )
 from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifact_path
+from ..common.runtimes.constants import RunStates
 from ..datastore import store_manager
 from ..features import Feature
 from ..model import EntrypointParam, ImageBuilder, ModelObj
@@ -851,6 +852,7 @@ class ProjectSpec(ModelObj):
         build=None,
         custom_packagers: Optional[list[tuple[str, bool]]] = None,
         default_function_node_selector=None,
+        notifications=None,
     ):
         self.repo = None
 
@@ -891,6 +893,7 @@ class ProjectSpec(ModelObj):
         # whether it is mandatory for a run (raise exception on collection error) or not.
         self.custom_packagers = custom_packagers or []
         self._default_function_node_selector = default_function_node_selector or None
+        self.notifications = notifications or []
 
     @property
     def source(self) -> str:
@@ -1172,7 +1175,6 @@ class MlrunProject(ModelObj):
         self._artifact_manager = None
         self._notifiers = CustomNotificationPusher(
             [
-                NotificationTypes.slack,
                 NotificationTypes.console,
                 NotificationTypes.ipython,
             ]
@@ -2670,6 +2672,36 @@ class MlrunProject(ModelObj):
             timeout=timeout,
         )
 
+    def push_pipeline_notification_kfp_runner(
+        self,
+        pipeline_id: str,
+        current_run_state: mlrun_pipelines.common.models.RunStatuses,
+        message: str,
+        notifications: Optional[list] = None,
+    ):
+        """
+        Push notifications for a pipeline run(KFP).
+
+        :param pipeline_id:         Unique ID of the pipeline run.
+        :param current_run_state:   Current run state of the pipeline.
+        :param message:             Message to send in the notification.
+        :param notifications:       List of notifications to send.
+        """
+        current_run_state = RunStates.pipeline_run_status_to_run_state(
+            current_run_state
+        )
+        db = mlrun.get_run_db()
+        notifications = notifications or self.spec.notifications
+        notifications_to_send = []
+        for notification in notifications:
+            if current_run_state in notification.when:
+                notification_copy = notification.copy()
+                notification_copy.message = message
+                notifications_to_send.append(notification_copy)
+        db.push_pipeline_notifications(
+            pipeline_id, self.metadata.name, notifications_to_send
+        )
+
     def _instantiate_function(
         self,
         func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,

mlrun/runtimes/nuclio/serving.py

@@ -309,7 +309,7 @@ class ServingRuntime(RemoteRuntime):
         self,
         stream_path: Optional[str] = None,
         batch: Optional[int] = None,
-        sample: Optional[int] = None,
+        sampling_percentage: float = 100,
         stream_args: Optional[dict] = None,
         tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
         enable_tracking: bool = True,
@@ -317,13 +317,13 @@ class ServingRuntime(RemoteRuntime):
         """Apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
         and analyze performance.
 
-        :param stream_path:         Path/url of the tracking stream e.g. v3io:///users/mike/mystream
-                                    you can use the "dummy://" path for test/simulation.
-        :param batch:               Micro batch size (send micro batches of N records at a time).
-        :param sample:              Sample size (send only one of N records).
-        :param stream_args:         Stream initialization parameters, e.g. shards, retention_in_hours, ..
-        :param enable_tracking:     Enabled/Disable model-monitoring tracking.
-                                    Default True (tracking enabled).
+        :param stream_path:                Path/url of the tracking stream e.g. v3io:///users/mike/mystream
+                                           you can use the "dummy://" path for test/simulation.
+        :param batch:                      Deprecated. Micro batch size (send micro batches of N records at a time).
+        :param sampling_percentage:        Down sampling events that will be pushed to the monitoring stream based on
+                                           a specified percentage. e.g. 50 for 50%. By default, all events are pushed.
+        :param stream_args:                Stream initialization parameters, e.g. shards, retention_in_hours, ..
+        :param enable_tracking:            Enabled/Disable model-monitoring tracking. Default True (tracking enabled).
 
         Example::
 
@@ -336,12 +336,21 @@ class ServingRuntime(RemoteRuntime):
         # Applying model monitoring configurations
         self.spec.track_models = enable_tracking
 
+        if not 0 < sampling_percentage <= 100:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "`sampling_percentage` must be greater than 0 and less or equal to 100."
+            )
+        self.spec.parameters["sampling_percentage"] = sampling_percentage
+
         if stream_path:
             self.spec.parameters["log_stream"] = stream_path
         if batch:
-            self.spec.parameters["log_stream_batch"] = batch
-        if sample:
-            self.spec.parameters["log_stream_sample"] = sample
+            warnings.warn(
+                "The `batch` size parameter was deprecated in version 1.8.0 and is no longer used. "
+                "It will be removed in 1.10.",
+                # TODO: Remove this in 1.10
+                FutureWarning,
+            )
         if stream_args:
             self.spec.parameters["stream_args"] = stream_args
         if tracking_policy is not None:

mlrun/serving/v2_serving.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import random
 import threading
 import time
 import traceback
@@ -283,7 +284,6 @@ class V2ModelServer(StepToDict):
             }
             if self.version:
                 response["model_version"] = self.version
-
         elif op == "ready" and event.method == "GET":
             # get model health operation
             setattr(event, "terminated", True)
@@ -468,13 +468,9 @@ class _ModelLogPusher:
         self.hostname = context.stream.hostname
         self.function_uri = context.stream.function_uri
         self.stream_path = context.stream.stream_uri
-        self.stream_batch = int(context.get_param("log_stream_batch", 1))
-        self.stream_sample = int(context.get_param("log_stream_sample", 1))
+        self.sampling_percentage = float(context.get_param("sampling_percentage", 100))
         self.output_stream = output_stream or context.stream.output_stream
         self._worker = context.worker_id
-        self._sample_iter = 0
-        self._batch_iter = 0
-        self._batch = []
 
     def base_data(self):
         base_data = {
@@ -485,6 +481,7 @@ class _ModelLogPusher:
             "host": self.hostname,
             "function_uri": self.function_uri,
             "endpoint_id": self.model.model_endpoint_uid,
+            "sampling_percentage": self.sampling_percentage,
         }
         if getattr(self.model, "labels", None):
             base_data["labels"] = self.model.labels
@@ -504,37 +501,55 @@ class _ModelLogPusher:
             self.output_stream.push([data], partition_key=partition_key)
             return
 
-        self._sample_iter = (self._sample_iter + 1) % self.stream_sample
-        if self.output_stream and self._sample_iter == 0:
+        if self.output_stream:
+            # Ensure that the inputs are a list of lists
+            request["inputs"] = (
+                request["inputs"]
+                if not any(not isinstance(req, list) for req in request["inputs"])
+                else [request["inputs"]]
+            )
             microsec = (now_date() - start).microseconds
 
-            if self.stream_batch > 1:
-                if self._batch_iter == 0:
-                    self._batch = []
-                self._batch.append(
-                    [request, op, resp, str(start), microsec, self.model.metrics]
+            if self.sampling_percentage != 100:
+                # Randomly select a subset of the requests based on the percentage
+                num_of_inputs = len(request["inputs"])
+                sampled_requests_indices = self._pick_random_requests(
+                    num_of_inputs, self.sampling_percentage
                 )
-                self._batch_iter = (self._batch_iter + 1) % self.stream_batch
-
-                if self._batch_iter == 0:
-                    data = self.base_data()
-                    data["headers"] = [
-                        "request",
-                        "op",
-                        "resp",
-                        "when",
-                        "microsec",
-                        "metrics",
+                if not sampled_requests_indices:
+                    # No events were selected for sampling
+                    return
+
+                request["inputs"] = [
+                    request["inputs"][i] for i in sampled_requests_indices
+                ]
+
+                if resp and "outputs" in resp and isinstance(resp["outputs"], list):
+                    resp["outputs"] = [
+                        resp["outputs"][i] for i in sampled_requests_indices
                     ]
-                    data["values"] = self._batch
-                    self.output_stream.push([data], partition_key=partition_key)
-            else:
-                data = self.base_data()
-                data["request"] = request
-                data["op"] = op
-                data["resp"] = resp
-                data["when"] = start_str
-                data["microsec"] = microsec
-                if getattr(self.model, "metrics", None):
-                    data["metrics"] = self.model.metrics
-                self.output_stream.push([data], partition_key=partition_key)
+
+            data = self.base_data()
+            data["request"] = request
+            data["op"] = op
+            data["resp"] = resp
+            data["when"] = start_str
+            data["microsec"] = microsec
+            if getattr(self.model, "metrics", None):
+                data["metrics"] = self.model.metrics
+            data["effective_sample_count"] = len(request["inputs"])
+            self.output_stream.push([data], partition_key=partition_key)
+
+    @staticmethod
+    def _pick_random_requests(num_of_reqs: int, percentage: float) -> list[int]:
+        """
+        Randomly selects indices of requests to sample based on the given percentage
+
+        :param num_of_reqs: Number of requests to select from
+        :param percentage: Sample percentage for each request
+        :return: A list containing the indices of the selected requests
+        """
+
+        return [
+            req for req in range(num_of_reqs) if random.random() < (percentage / 100)
+        ]

mlrun/utils/helpers.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import asyncio
+import base64
 import enum
 import functools
+import gzip
 import hashlib
 import inspect
 import itertools
@@ -1709,7 +1711,14 @@ def get_serving_spec():
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Failed to find serving spec in env var or config file"
             )
-    spec = json.loads(data)
+    # Attempt to decode and decompress, or use as-is for backward compatibility
+    try:
+        decoded_data = base64.b64decode(data)
+        decompressed_data = gzip.decompress(decoded_data)
+        spec = json.loads(decompressed_data.decode("utf-8"))
+    except (OSError, gzip.BadGzipFile, base64.binascii.Error, json.JSONDecodeError):
+        spec = json.loads(data)
+
     return spec
 
 
@@ -1981,7 +1990,20 @@ class Workflow:
         if not workflow_id:
             return steps
 
-        workflow_manifest = Workflow._get_workflow_manifest(workflow_id)
+        try:
+            workflow_manifest = Workflow._get_workflow_manifest(workflow_id)
+        except Exception:
+            logger.warning(
+                "Failed to extract workflow steps from workflow manifest, "
+                "returning all runs with the workflow id label",
+                workflow_id=workflow_id,
+                traceback=traceback.format_exc(),
+            )
+            return db.list_runs(
+                project=project,
+                labels=f"workflow={workflow_id}",
+            )
+
         if not workflow_manifest:
             return steps
 
@@ -2038,3 +2060,9 @@ class Workflow:
 
         kfp_run = mlrun_pipelines.models.PipelineRun(kfp_run)
         return kfp_run.workflow_manifest()
+
+
+def as_dict(data: typing.Union[dict, str]) -> dict:
+    if isinstance(data, str):
+        return json.loads(data)
+    return data

mlrun/utils/notifications/notification_pusher.py

@@ -139,15 +139,25 @@ class NotificationPusher(_NotificationPusherBase):
                     error=mlrun.errors.err_to_str(exc),
                 )
 
-    def _process_notification(self, notification, run):
-        notification.status = run.status.notifications.get(notification.name, {}).get(
+    def _process_notification(self, notification_object, run):
+        notification_object.status = run.status.notifications.get(
+            notification_object.name, {}
+        ).get(
             "status",
             mlrun.common.schemas.NotificationStatus.PENDING,
         )
-        if self._should_notify(run, notification):
-            self._load_notification(run, notification)
+        if self._should_notify(run, notification_object):
+            notification = self._load_notification(notification_object)
+            if notification.is_async:
+                self._async_notifications.append(
+                    (notification, run, notification_object)
+                )
+            else:
+                self._sync_notifications.append(
+                    (notification, run, notification_object)
+                )
 
-    def push(self):
+    def push(self, sync_push_callback=None, async_push_callback=None):
         """
         Asynchronously push notifications for all runs in the initialized runs list (if they should be pushed).
         When running from a sync environment, the notifications will be pushed asynchronously however the function will
@@ -201,8 +211,9 @@ class NotificationPusher(_NotificationPusherBase):
             notifications_amount=len(self._sync_notifications)
             + len(self._async_notifications),
         )
-
-        self._push(sync_push, async_push)
+        sync_push_callback = sync_push_callback or sync_push
+        async_push_callback = async_push_callback or async_push
+        self._push(sync_push_callback, async_push_callback)
 
     @staticmethod
     def _should_notify(
@@ -241,24 +252,19 @@ class NotificationPusher(_NotificationPusherBase):
         return False
 
     def _load_notification(
-        self, run: mlrun.model.RunObject, notification_object: mlrun.model.Notification
+        self, notification_object: mlrun.model.Notification
     ) -> base.NotificationBase:
         name = notification_object.name
         notification_type = notification_module.NotificationTypes(
             notification_object.kind or notification_module.NotificationTypes.console
         )
         params = {}
-        params.update(notification_object.secret_params)
-        params.update(notification_object.params)
+        params.update(notification_object.secret_params or {})
+        params.update(notification_object.params or {})
         default_params = self._default_params.get(notification_type.value, {})
         notification = notification_type.get_notification()(
             name, params, default_params
         )
-        if notification.is_async:
-            self._async_notifications.append((notification, run, notification_object))
-        else:
-            self._sync_notifications.append((notification, run, notification_object))
-
         logger.debug(
             "Loaded notification", notification=name, type=notification_type.value
         )
@@ -496,6 +502,14 @@ class CustomNotificationPusher(_NotificationPusherBase):
         notification_type: str,
         params: typing.Optional[dict[str, str]] = None,
     ):
+        if notification_type not in [
+            notification_module.NotificationTypes.console,
+            notification_module.NotificationTypes.ipython,
+        ]:
+            # We want that only the console and ipython notifications will be notified by the client.
+            # The rest of the notifications will be notified by the BE.
+            return
+
         if notification_type in self._async_notifications:
             self._async_notifications[notification_type].load_notification(params)
         elif notification_type in self._sync_notifications:
@@ -565,25 +579,9 @@ class CustomNotificationPusher(_NotificationPusherBase):
         pipeline_id: typing.Optional[str] = None,
         has_workflow_url: bool = False,
     ):
-        message = f"Workflow started in project {project}"
-        if pipeline_id:
-            message += f" id={pipeline_id}"
-        commit_id = (
-            commit_id or os.environ.get("GITHUB_SHA") or os.environ.get("CI_COMMIT_SHA")
+        html, message = self.generate_start_message(
+            commit_id, has_workflow_url, pipeline_id, project
         )
-        if commit_id:
-            message += f", commit={commit_id}"
-        if has_workflow_url:
-            url = mlrun.utils.helpers.get_workflow_url(project, pipeline_id)
-        else:
-            url = mlrun.utils.helpers.get_ui_url(project)
-        html = ""
-        if url:
-            html = (
-                message
-                + f'<div><a href="{url}" target="_blank">click here to view progress</a></div>'
-            )
-            message = message + f", check progress in {url}"
         self.push(message, "info", custom_html=html)
 
     def push_pipeline_run_results(
@@ -616,6 +614,30 @@ class CustomNotificationPusher(_NotificationPusherBase):
             text += f", state={state}"
         self.push(text, "info", runs=runs_list)
 
+    def generate_start_message(
+        self, commit_id=None, has_workflow_url=None, pipeline_id=None, project=None
+    ):
+        message = f"Workflow started in project {project}"
+        if pipeline_id:
+            message += f" id={pipeline_id}"
+        commit_id = (
+            commit_id or os.environ.get("GITHUB_SHA") or os.environ.get("CI_COMMIT_SHA")
+        )
+        if commit_id:
+            message += f", commit={commit_id}"
+        if has_workflow_url:
+            url = mlrun.utils.helpers.get_workflow_url(project, pipeline_id)
+        else:
+            url = mlrun.utils.helpers.get_ui_url(project)
+        html = ""
+        if url:
+            html = (
+                message
+                + f'<div><a href="{url}" target="_blank">click here to view progress</a></div>'
+            )
+            message = message + f", check progress in {url}"
+        return html, message
+
 
 def sanitize_notification(notification_dict: dict):
     notification_dict.pop("secret_params", None)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "6671b915fe08ba2b9afa054585f86b955b48b671",
-  "version": "1.8.0-rc19"
+  "git_commit": "b2650d3be127dcd121931256721bab48ab0351c2",
+  "version": "1.8.0-rc21"
 }

{mlrun-1.8.0rc19.dist-info → mlrun-1.8.0rc21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.8.0rc19
+Version: 1.8.0rc21
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.8.0rc19.dist-info → mlrun-1.8.0rc21.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mlrun/__init__.py,sha256=7vuMpUiigXXDrghLRq680LKWy1faC0kQyGCZb_7cwyE,7473
-mlrun/__main__.py,sha256=o65gXHhmFA9GV_n2mqmAO80nW3MAwo_s7j80IKgCzRE,45949
+mlrun/__main__.py,sha256=3CJdwbSQGpbEhnAnVN_-CkQmLOPUUXTKhMf7xIWNQrc,46138
 mlrun/config.py,sha256=zHp7wtbaewFbHYsA_SLxwuwSDYwrcL6A3knWSPcRtAA,70547
 mlrun/errors.py,sha256=5raKb1PXQpTcIvWQ4sr1qn2IS7P_GT_FydBJ0dXkVuc,8097
 mlrun/execution.py,sha256=Up9U6xonTElRIaesF9Vej2JK1Isk2AZNK9ke0XcF5Dg,49030
@@ -38,7 +38,7 @@ mlrun/common/formatters/project.py,sha256=0G4lhcTAsxQCxd40dKC4894cMH8nKt03BcGyp9
 mlrun/common/formatters/run.py,sha256=Gcf9lVDqxPMNfWcPX0RJasjTC_N_U0yTBkQ02jOPJ7A,1062
 mlrun/common/model_monitoring/__init__.py,sha256=kXGBqhLN0rlAx0kTXhozGzFsIdSqW0uTSKMmsLgq_is,569
 mlrun/common/model_monitoring/helpers.py,sha256=lV86teJYoE3MNDx4yhpbzO1KylWmvDbuNODw5yGZwgs,2943
-mlrun/common/runtimes/constants.py,sha256=Mok3m9Rv182TTMp7uYNfWalm9Xcz86yva-4fTxfMOPI,10988
+mlrun/common/runtimes/constants.py,sha256=07wD1g8QjXZe1fm2hSMOxZG19aAUsEZM8WeXnyoBd6Q,12127
 mlrun/common/schemas/__init__.py,sha256=PBuIAhXSkVEVxxKcv5hR_xvTwNAUqxOXHVPugOoWTyM,5386
 mlrun/common/schemas/alert.py,sha256=G9lFTXFYDor-RVLpJxMorIPlLWr_-GYCFKRN9DkKwXs,10124
 mlrun/common/schemas/api_gateway.py,sha256=3a0QxECLmoDkD5IiOKtXJL-uiWB26Hg55WMA3nULYuI,7127
@@ -72,9 +72,9 @@ mlrun/common/schemas/secret.py,sha256=CCxFYiPwJtDxwg2VVJH9nUG9cAZ2a34IjeuaWv-BYl
 mlrun/common/schemas/tag.py,sha256=HRZi5QZ4vVGaCr2AMk9eJgcNiAIXmH4YDc8a4fvF770,893
 mlrun/common/schemas/workflow.py,sha256=rwYzDJYxpE9k4kC88j_eUCmqK4ZsWV_h-_nli7Fs7Ow,2078
 mlrun/common/schemas/model_monitoring/__init__.py,sha256=jz0fvdn8BEecgUCKhiSNH6QtFhSW4O19Ql9KXo0AxOg,1900
-mlrun/common/schemas/model_monitoring/constants.py,sha256=KHpZiTruqr1iQ4XfEz8Ptj9ytCBuvwOXp30YTt5YQws,11834
+mlrun/common/schemas/model_monitoring/constants.py,sha256=kok1NFXJDicmcJ30MhOqR-vxd185sotXYU0CGLdrQxA,12082
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=Rq10KKOyyUYr7qOQFZfwGZtUim0LY9O0LQ5uc9jmIVQ,1562
-mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=j60-_puybc4yLlmoWkZ04m6PuY4p5yfzVJpPo0_n1sY,11725
+mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=0gBH-KnDDbGLOkiqHtk0_iNIdW-NPVy0TKJnZ1fDcEQ,11807
 mlrun/data_types/__init__.py,sha256=unRo9GGwCmj0hBKBRsXJ2P4BzpQaddlQTvIrVQaKluI,984
 mlrun/data_types/data_types.py,sha256=0_oKLC6-sXL2_nnaDMP_HSXB3fD1nJAG4J2Jq6sGNNw,4998
 mlrun/data_types/infer.py,sha256=KdaRgWcqvLkuLjXrMuDr3ik6WY7JP5wJO0Yii_Vl5kw,6173
@@ -107,10 +107,10 @@ mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev
 mlrun/datastore/wasbfs/fs.py,sha256=ge8NK__5vTcFT-krI155_8RDUywQw4SIRX6BWATXy9Q,6299
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
-mlrun/db/base.py,sha256=aFtDl4J_yeEovso7uvnnn9KLYMnIRiS52qM1uemdG8k,30218
+mlrun/db/base.py,sha256=Ke3atYsWfwhTlo3OoLKAz9gOkmu_a18De1lNKxV29Cc,30379
 mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
-mlrun/db/httpdb.py,sha256=VXp3ETu5fl_-6lEF_bsUerKESuRXbAFxLYvduH6DlKs,228070
-mlrun/db/nopdb.py,sha256=v285LHP_Onfuo8KRF078IAPHIXTeEhQsU58QoY7x-b0,26673
+mlrun/db/httpdb.py,sha256=n1bGlooT2OQnPmzzFyKAPJO-2pCWf5B0EHctHa1h9rQ,230865
+mlrun/db/nopdb.py,sha256=rte_AcP2itEwi7wpA26SH-AOAzm5ntPPoqXFHHtuewE,26834
 mlrun/feature_store/__init__.py,sha256=AVnY2AFUNc2dKxLLUMx2K3Wo1eGviv0brDcYlDnmtf4,1506
 mlrun/feature_store/api.py,sha256=qkojZpzqGAn3r9ww0ynBRKOs8ji8URaK4DSYD4SE-CE,50395
 mlrun/feature_store/common.py,sha256=Z7USI-d1fo0iwBMsqMBtJflJfyuiV3BLoDXQPSAoBAs,12826
@@ -130,7 +130,7 @@ mlrun/frameworks/parallel_coordinates.py,sha256=UuZ0b0ACsaaH0rDya_0YMOWwaH6zhEyD
 mlrun/frameworks/_common/__init__.py,sha256=1ovfHxNW8V9ERVVZx8lPFVGBtsXHaHli7pZPR-Ixn8g,860
 mlrun/frameworks/_common/artifacts_library.py,sha256=O0z74o3Z6k5NruTqXMLDugZ6midOmSFqNlt7WhYIRP8,8517
 mlrun/frameworks/_common/mlrun_interface.py,sha256=gjNV_siKyJ7xZdwoH8uvRmPuuQrc8WjnhoXr3GUCkAc,21093
-mlrun/frameworks/_common/model_handler.py,sha256=0hsLFxMNaLhGIeXCMeHqUp7p2L9lJgcQKI4b7ef8CLQ,55596
+mlrun/frameworks/_common/model_handler.py,sha256=xoelZloxZ1BcLX7krwWrs-TmqIn52CRoPvGsB7Hoar4,55544
 mlrun/frameworks/_common/plan.py,sha256=Yr98b5lkCV0K0u_krnU8gZJiXj14xfrFjJ6xD6QJdn0,3444
 mlrun/frameworks/_common/producer.py,sha256=R67XRbiz1bk0XNvuW7ybbA4v6o6Q5qzD0h3AkC-tM1A,5766
 mlrun/frameworks/_common/utils.py,sha256=NqoKbgj6UGPMBNhpK6mkKK4GOt5ko1lDqExFhQm9oEc,9131
@@ -220,7 +220,7 @@ mlrun/model_monitoring/api.py,sha256=nH5aEUkmUEJF0CurrWJxmxVv1tQed2yzCLhQByG1L00
 mlrun/model_monitoring/controller.py,sha256=dBfZQswF67vqeUFnmgsm9jU_5sOs9dLwMPEiYHG-Kk8,19786
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
 mlrun/model_monitoring/helpers.py,sha256=6L-IO4EUAYoAf74snGNiGDln_p77OIdxVdrf392VBzY,17936
-mlrun/model_monitoring/stream_processing.py,sha256=ltCVgo_b3yay16CUbqeGkRfzCHZSn14lVeBng5m9keY,31738
+mlrun/model_monitoring/stream_processing.py,sha256=wAPKdySwh8kGc7k6Zqu952X5E7uAom2J_YJmPXQUAlA,32859
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=vbL7bqTyNu8q4bNcebX72sUMybVDAoTWg-CXq4fov3Y,8429
 mlrun/model_monitoring/applications/__init__.py,sha256=QYvzgCutFdAkzqKPD3mvkX_3c1X4tzd-kW8ojUOE9ic,889
@@ -237,12 +237,12 @@ mlrun/model_monitoring/db/tsdb/__init__.py,sha256=_ejhGA-bi5-6kEMEVYUN5TLxND8hTj
 mlrun/model_monitoring/db/tsdb/base.py,sha256=JjLBzZXE4ZxtBmihVXjUYZ2HKmgqX03ZhUynXp4948o,25372
 mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsVsrb9otX9a7O4,1189
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
-mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=6de8P0CJMqe7PGttoZNt9UtrbBcJnpIp82hk_MbtepA,12477
+mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=qfKDUZhgteL0mp2A1aP1iMmcthgUMKmZqMUidZjQktQ,12649
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=IEpJknjqx_LYcZjIccPuujOfEruXsRm8c8YzYaWWNEQ,30175
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=Be-DdprjCdRQz-Goxy5sRxBdHp94V7d868N_IBEQxrY,30205
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
-mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=9j_TWS_3OXTwjiYw9jb92z8URHUXDW7hCgwZnpb2P8c,36834
+mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=O9j0Hnmz5LbaJO2RxKRSDJU54MOCjICap2G9tuut8JE,37044
 mlrun/model_monitoring/metrics/__init__.py,sha256=6CsTXAxeLbbf8yfCADTaxmiavqwrLEdYFJ-qc5kgDAY,569
 mlrun/model_monitoring/metrics/histogram_distance.py,sha256=E9_WIl2vd6qNvoHVHoFcnuQk3ekbFWOdi8aU7sHrfk4,4724
 mlrun/package/__init__.py,sha256=v7VDyK9kDOOuDvFo4oiGV2fx-vM1KL7fdN9pGLakhUQ,7008
@@ -266,8 +266,8 @@ mlrun/platforms/__init__.py,sha256=ZuyeHCHHUxYEoZRmaJqzFSfwhaTyUdBZXMeVp75ql1w,3
 mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13749
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=VXUlMrouFTls-I-bMhdN5pPfQ34TR7bFQ-NUSWNvl84,20029
-mlrun/projects/pipelines.py,sha256=vZpyiERUzwPMS7NCC5ghI0KB_DItIddr7MMWGTwLawY,47437
-mlrun/projects/project.py,sha256=WKIgBy1nqHxQ4d2AHId6wptnGeYzfZTY61K8pOQUrvI,228386
+mlrun/projects/pipelines.py,sha256=UwRwK2e8T0463quQ3d-ud_swmTLkBuSVUroY4YkgzZU,47903
+mlrun/projects/project.py,sha256=C1OcldXuyCUIucvPFQX-tJbwjBkht8Ocn4m7lTHNZeM,229691
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
 mlrun/runtimes/base.py,sha256=Yt2l7srrXjK783cunBEKH0yQxQZRH8lkedXNOXuLbbo,37841
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
@@ -291,7 +291,7 @@ mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVY
 mlrun/runtimes/nuclio/api_gateway.py,sha256=vH9ClKVP4Mb24rvA67xPuAvAhX-gAv6vVtjVxyplhdc,26969
 mlrun/runtimes/nuclio/function.py,sha256=Bff8Veg-eaqNrQ7yn20HpRhwAO4OA7FTnzXnAyoaBPU,52365
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
-mlrun/runtimes/nuclio/serving.py,sha256=yuBSJe8AM4ar87Tc9x0vLRc47w8f5r-joMNza9hVCOo,31399
+mlrun/runtimes/nuclio/serving.py,sha256=NG9OOTxbOGXhyVPI8BjIcZX5CDXvyk931o5DYOSfCg0,31977
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
 mlrun/runtimes/nuclio/application/application.py,sha256=HlEq4A6hbFqr3Ba3TL4m7nbmfMYI06Zb_NAKGjzkEFU,29242
 mlrun/runtimes/nuclio/application/reverse_proxy.go,sha256=JIIYae6bXzCLf3jXuu49KWPQYoXr_FDQ2Rbo1OWKAd0,3150
@@ -306,7 +306,7 @@ mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBI
 mlrun/serving/states.py,sha256=g6UIeaS6B9v8k4eDMmOxyoB8Gdqm9PiNIkeuzDyTJA8,67565
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
-mlrun/serving/v2_serving.py,sha256=B1Vmca2_YidXyit4wuxR6JGooMGdaeZI3Ja90JHCz10,21882
+mlrun/serving/v2_serving.py,sha256=Nm-mPa40JlhpAFH5JuaS4Kc38g_o70cBPGgrqo7zDRM,22525
 mlrun/track/__init__.py,sha256=yVXbT52fXvGKRlc_ByHqIVt7-9L3DRE634RSeQwgXtU,665
 mlrun/track/tracker.py,sha256=CyTU6Qd3_5GGEJ_hpocOj71wvV65EuFYUjaYEUKAL6Q,3575
 mlrun/track/tracker_manager.py,sha256=IYBl99I62IC6VCCmG1yt6JoHNOQXa53C4DURJ2sWgio,5726
@@ -318,7 +318,7 @@ mlrun/utils/azure_vault.py,sha256=IEFizrDGDbAaoWwDr1WoA88S_EZ0T--vjYtY-i0cvYQ,34
 mlrun/utils/clones.py,sha256=y3zC9QS7z5mLuvyQ6vFd6sJnikbgtDwrBvieQq0sovY,7359
 mlrun/utils/condition_evaluator.py,sha256=-nGfRmZzivn01rHTroiGY4rqEv8T1irMyhzxEei-sKc,1897
 mlrun/utils/db.py,sha256=blQgkWMfFH9lcN4sgJQcPQgEETz2Dl_zwbVA0SslpFg,2186
-mlrun/utils/helpers.py,sha256=aPMmd5dEm7PIbx7maXtNGhNKzEHp8JK8uSmOgTtoDbc,69549
+mlrun/utils/helpers.py,sha256=TYDhMDWGtkrMJLslanSAb1sg7kR92q3etOWpTcqCoJk,70499
 mlrun/utils/http.py,sha256=t6FrXQstZm9xVVjxqIGiLzrwZNCR4CSienSOuVgNIcI,8706
 mlrun/utils/logger.py,sha256=_v4UTv1-STzC2c6aAWAa0NNl9STQoBYbR3OHgAiL41s,14606
 mlrun/utils/regex.py,sha256=IQqwPna6Z8J31xkTUduYbGk48GkQBUJFZSuxAWm1pzU,5162
@@ -327,7 +327,7 @@ mlrun/utils/singleton.py,sha256=p1Y-X0mPSs_At092GS-pZCA8CTR62HOqPU07_ZH6-To,869
 mlrun/utils/v3io_clients.py,sha256=0aCFiQFBmgdSeLzJr_nEP6SG-zyieSgH8RdtcUq4dc0,1294
 mlrun/utils/vault.py,sha256=xUiKL17dCXjwQJ33YRzQj0oadUXATlFWPzKKYAESoQk,10447
 mlrun/utils/notifications/__init__.py,sha256=eUzQDBxSQmMZASRY-YAnYS6tL5801P0wEjycp3Dvoe0,990
-mlrun/utils/notifications/notification_pusher.py,sha256=WN7RMfaZtCoG3bNfMjub1phbvJ2Xw8lKFiq2GStnKGw,24132
+mlrun/utils/notifications/notification_pusher.py,sha256=9KXwuMGgA340TfYlMbudlJ-MEwOyBJbbJGu_iBrIhrM,25106
 mlrun/utils/notifications/notification/__init__.py,sha256=9Rfy6Jm8n0LaEDO1VAQb6kIbr7_uVuQhK1pS_abELIY,2581
 mlrun/utils/notifications/notification/base.py,sha256=VOgrzRakRfjYYBqvkc0cgEC5pl7KMidP7u-TL4HpGCY,5280
 mlrun/utils/notifications/notification/console.py,sha256=ICbIhOf9fEBJky_3j9TFiKAewDGyDHJr9l4VeT7G2sc,2745
@@ -337,11 +337,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=NKV4RFiY3gLsS8uPppgniPLyag8zJ9O1VhixoXkM7kw,7108
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=9e_fs0UZTGdzgD6yjggI41O7Hv5n_tbuYDMtJsSNspY,89
+mlrun/utils/version/version.json,sha256=Uil-vG-gWmMqNXcEWVV1qzfhqKErwFyS3X-ikYkxG4A,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc19.dist-info/METADATA,sha256=A2YOQIMx1P8pckWVn0bKsr8oEpcWxVbhkXhPxTplJMM,24885
-mlrun-1.8.0rc19.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-mlrun-1.8.0rc19.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc19.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc19.dist-info/RECORD,,
+mlrun-1.8.0rc21.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc21.dist-info/METADATA,sha256=OjUYCfgyCX1v5wIDojPMNCTe-gQqOxKDodJL7n-Cj_M,24885
+mlrun-1.8.0rc21.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+mlrun-1.8.0rc21.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc21.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc21.dist-info/RECORD,,