mlrun 1.8.0rc17__py3-none-any.whl → 1.8.0rc19__py3-none-any.whl

mlrun/common/schemas/artifact.py

@@ -51,6 +51,12 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
+    @classmethod
+    def from_kind(cls, kind: str) -> "ArtifactCategories":
+        if kind in [cls.model.value, cls.dataset.value, cls.document.value]:
+            return cls(kind)
+        return cls.other
+
 
 class ArtifactIdentifier(pydantic.v1.BaseModel):
     # artifact kind

mlrun/common/schemas/model_monitoring/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from .constants import (
+    INTERSECT_DICT_KEYS,
     V3IO_MODEL_MONITORING_DB,
     ApplicationEvent,
     ControllerPolicy,

mlrun/common/schemas/model_monitoring/constants.py

@@ -248,6 +248,12 @@ class ProjectSecretKeys:
         ]
 
 
+class GetEventsFormat(MonitoringStrEnum):
+    SINGLE = "single"
+    SEPARATION = "separation"
+    INTERSECTION = "intersection"
+
+
 class ModelEndpointTargetSchemas(MonitoringStrEnum):
     V3IO = "v3io"
     MYSQL = "mysql"
@@ -448,3 +454,8 @@ FQN_REGEX = re.compile(FQN_PATTERN)
 PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
 
 MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
+INTERSECT_DICT_KEYS = {
+    ModelEndpointMonitoringMetricType.METRIC: "intersect_metrics",
+    ModelEndpointMonitoringMetricType.RESULT: "intersect_results",
+}

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -259,12 +259,12 @@ class ModelEndpointMonitoringMetric(BaseModel):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.full_name = _compose_full_name(
+        self.full_name = compose_full_name(
             project=self.project, app=self.app, name=self.name, type=self.type
         )
 
 
-def _compose_full_name(
+def compose_full_name(
     *,
     project: str,
     app: str,

mlrun/config.py

@@ -83,8 +83,8 @@ default_config = {
     "images_to_enrich_registry": "^mlrun/*,python:3.9",
     "kfp_url": "",
     "kfp_ttl": "14400",  # KFP ttl in sec, after that completed PODs will be deleted
-    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner (defaults to mlrun/mlrun-kfp)
-    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner (defaults to mlrun/ml-base)
+    "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
+    "dask_kfp_image": "mlrun/ml-base",  # image to use for dask KFP runner
     "igz_version": "",  # the version of the iguazio system the API is running on
     "iguazio_api_url": "",  # the url to iguazio api
     "spark_app_image": "",  # image to use for spark operator app runtime
@@ -169,6 +169,7 @@ default_config = {
             "max_chunk_size": 1024 * 1024 * 1,  # 1MB
             "max_preview_size": 1024 * 1024 * 10,  # 10MB
             "max_download_size": 1024 * 1024 * 100,  # 100MB
+            "max_deletions": 200,
         },
     },
     # FIXME: Adding these defaults here so we won't need to patch the "installing component" (provazio-controller) to

mlrun/db/base.py

@@ -337,6 +337,15 @@ class RunDBInterface(ABC):
     ) -> list[mm_endpoints.ModelEndpointMonitoringMetric]:
         pass
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        pass
+
     @abstractmethod
     def delete_project(
         self,

mlrun/db/httpdb.py

@@ -3524,6 +3524,48 @@ class HTTPRunDB(RunDBInterface):
             list[mm_endpoints.ModelEndpointMonitoringMetric], monitoring_metrics
         )
 
+    def get_metrics_by_multiple_endpoints(
+        self,
+        project: str,
+        endpoint_ids: Union[str, list[str]],
+        type: Literal["results", "metrics", "all"] = "all",
+        events_format: mm_constants.GetEventsFormat = mm_constants.GetEventsFormat.SEPARATION,
+    ) -> dict[str, list[mm_endpoints.ModelEndpointMonitoringMetric]]:
+        """Get application metrics/results by endpoint id and project.
+
+        :param project:         The name of the project.
+        :param endpoint_ids:    The unique id of the model endpoint. Can be a single id or a list of ids.
+        :param type:            The type of the metrics to return. "all" means "results" and "metrics".
+        :param events_format:   response format:
+
+                                separation: {"mep_id1":[...], "mep_id2":[...]}
+                                intersection {"intersect_metrics":[], "intersect_results":[]}
+        :return: A dictionary of application metrics and/or results for the model endpoints formatted by events_format.
+        """
+        path = f"projects/{project}/model-endpoints/metrics"
+        params = {
+            "type": type,
+            "endpoint-id": endpoint_ids,
+            "events_format": events_format,
+        }
+        error_message = (
+            f"Failed to get model monitoring metrics,"
+            f" endpoint_ids: {endpoint_ids}, project: {project}"
+        )
+        response = self.api_call(
+            mlrun.common.types.HTTPMethod.GET,
+            path,
+            error_message,
+            params=params,
+        )
+        monitoring_metrics_by_endpoint = response.json()
+        parsed_metrics_by_endpoint = {}
+        for endpoint, metrics in monitoring_metrics_by_endpoint.items():
+            parsed_metrics_by_endpoint[endpoint] = parse_obj_as(
+                list[mm_endpoints.ModelEndpointMonitoringMetric], metrics
+            )
+        return parsed_metrics_by_endpoint
+
     def create_user_secrets(
         self,
         user: str,

mlrun/model_monitoring/applications/base.py

@@ -14,7 +14,8 @@
 
 import socket
 from abc import ABC, abstractmethod
-from datetime import datetime
+from collections.abc import Iterator
+from datetime import datetime, timedelta
 from typing import Any, Optional, Union, cast
 
 import pandas as pd
@@ -96,6 +97,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         endpoints: Optional[list[tuple[str, str]]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
+        base_period: Optional[int] = None,
     ):
         """
         A custom handler that wraps the application's logic implemented in
@@ -122,32 +124,59 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             return self.do_tracking(monitoring_context)
 
         if endpoints is not None:
-            start, end = self._validate_times(start, end)
-            for endpoint_name, endpoint_id in endpoints:
-                result = call_do_tracking(
-                    event={
-                        mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
-                        mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
-                        mm_constants.ApplicationEvent.START_INFER_TIME: start,
-                        mm_constants.ApplicationEvent.END_INFER_TIME: end,
-                    }
-                )
-                context.log_result(
-                    f"{endpoint_name}_{start.isoformat()}_{end.isoformat()}", result
-                )
+            start, end = self._validate_times(start, end, base_period)
+            for window_start, window_end in self._window_generator(
+                start, end, base_period
+            ):
+                for endpoint_name, endpoint_id in endpoints:
+                    result = call_do_tracking(
+                        event={
+                            mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
+                            mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
+                            mm_constants.ApplicationEvent.START_INFER_TIME: window_start,
+                            mm_constants.ApplicationEvent.END_INFER_TIME: window_end,
+                        }
+                    )
+                    context.log_result(
+                        f"{endpoint_name}_{window_start.isoformat()}_{window_end.isoformat()}",
+                        result,
+                    )
         else:
             return call_do_tracking()
 
     @staticmethod
     def _validate_times(
-        start: Optional[datetime], end: Optional[datetime]
+        start: Optional[datetime],
+        end: Optional[datetime],
+        base_period: Optional[int],
     ) -> tuple[datetime, datetime]:
         if (start is None) or (end is None):
             raise mlrun.errors.MLRunValueError(
                 "When `endpoint_names` is provided, you must also pass the start and end times"
             )
+        if (base_period is not None) and not (
+            isinstance(base_period, int) and base_period > 0
+        ):
+            raise mlrun.errors.MLRunValueError(
+                "`base_period` must be a nonnegative integer - the number of minutes in a monitoring window"
+            )
         return start, end
 
+    @staticmethod
+    def _window_generator(
+        start: datetime, end: datetime, base_period: Optional[int]
+    ) -> Iterator[tuple[datetime, datetime]]:
+        if base_period is None:
+            yield start, end
+            return
+
+        window_length = timedelta(minutes=base_period)
+        current_start_time = start
+        while current_start_time < end:
+            current_end_time = min(current_start_time + window_length, end)
+            yield current_start_time, current_end_time
+            current_start_time = current_end_time
+
     @classmethod
     def deploy(
         cls,
@@ -203,6 +232,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         endpoints: Optional[list[tuple[str, str]]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
+        base_period: Optional[int] = None,
     ) -> "mlrun.RunObject":
         """
         Call this function to run the application's
@@ -228,6 +258,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                                   If provided, you have to provide also the start and end times of the data to analyze.
         :param start:             The start time of the sample data.
         :param end:               The end time of the sample data.
+        :param base_period:       The window length in minutes. If ``None``, the whole window from ``start`` to ``end``
+                                  is taken. If an integer is specified, the application is run from ``start`` to ``end``
+                                  in ``base_period`` length windows, except for the last window that ends at ``end`` and
+                                  therefore may be shorter.
 
         :returns: The output of the
                   :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
@@ -253,15 +287,16 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             ),
         )
 
-        params: dict[str, Union[list[tuple[str, str]], datetime]] = {}
+        params: dict[str, Union[list[tuple[str, str]], datetime, int, None]] = {}
         if endpoints:
-            start, end = cls._validate_times(start, end)
+            start, end = cls._validate_times(start, end, base_period)
             params["endpoints"] = endpoints
             params["start"] = start
             params["end"] = end
-        elif start or end:
+            params["base_period"] = base_period
+        elif start or end or base_period:
             raise mlrun.errors.MLRunValueError(
-                "Custom start or end times are supported only with endpoints data"
+                "Custom start and end times or base_period are supported only with endpoints data"
             )
 
         inputs: dict[str, str] = {}

mlrun/model_monitoring/db/tsdb/base.py

@@ -234,14 +234,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoint.
+        Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -252,14 +252,14 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
-        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoint.
+        Fetches distinct results metadata from the app-results TSDB table for a specified model endpoints.
 
-        :param endpoint_id:        The model endpoint identifier.
+        :param endpoint_id:        The model endpoint identifier. Can be a single id or a list of ids.
         :param start:              The start time of the query.
         :param end:                The end time of the query.
 
@@ -341,7 +341,7 @@ class TSDBConnector(ABC):
             logger.debug("No metrics", missing_metrics=metrics_without_data.keys())
             grouped = []
         for (app_name, name), sub_df in grouped:
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project,
                 app=app_name,
                 name=name,
@@ -410,7 +410,7 @@ class TSDBConnector(ABC):
             result_kind = mlrun.model_monitoring.db.tsdb.helpers._get_result_kind(
                 sub_df
             )
-            full_name = mlrun.model_monitoring.helpers._compose_full_name(
+            full_name = mm_schemas.model_endpoints.compose_full_name(
                 project=project, app=app_name, name=name
             )
             try:
@@ -467,6 +467,7 @@ class TSDBConnector(ABC):
 
         :return:        A list of mm metrics objects.
         """
+
         return list(
             map(
                 lambda record: mm_schemas.ModelEndpointMonitoringMetric(
@@ -481,6 +482,113 @@ class TSDBConnector(ABC):
             )
         )
 
+    @staticmethod
+    def df_to_metrics_grouped_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: str,
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a grouped mm metrics objects by endpoint_id.
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A grouped dict of mm metrics/results, using model_endpoints_ids as keys.
+        """
+
+        if df.empty:
+            return {}
+
+        grouped_by_fields = [mm_schemas.WriterEvent.APPLICATION_NAME]
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            grouped_by_fields.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+
+        grouped_by_fields.append(name_column)
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        grouped_by_df = df.groupby("endpoint_id")
+        grouped_dict = grouped_by_df.apply(
+            lambda group: list(
+                map(
+                    lambda record: mm_schemas.ModelEndpointMonitoringMetric(
+                        project=project,
+                        type=type,
+                        app=record.get(mm_schemas.WriterEvent.APPLICATION_NAME),
+                        name=record.get(name_column),
+                        **{"kind": record.get(mm_schemas.ResultData.RESULT_KIND)}
+                        if type == "result"
+                        else {},
+                    ),
+                    group[grouped_by_fields].to_dict(orient="records"),
+                )
+            )
+        ).to_dict()
+        return grouped_dict
+
+    @staticmethod
+    def df_to_events_intersection_dict(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: typing.Union[str, mm_schemas.ModelEndpointMonitoringMetricType],
+    ) -> dict[str, list[mm_schemas.ModelEndpointMonitoringMetric]]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a dict of intersection metrics/results by name and application
+         (and kind in results).
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A dictionary where the key is event type (as defined by `INTERSECT_DICT_KEYS`),
+                        and the value is a list containing the intersect metrics or results across all endpoint IDs.
+
+                        For example:
+                        {
+                            "intersect_metrics": [...]
+                        }
+        """
+        dict_key = mm_schemas.INTERSECT_DICT_KEYS[type]
+        metrics = []
+        if df.empty:
+            return {dict_key: []}
+
+        columns_to_zip = [mm_schemas.WriterEvent.APPLICATION_NAME]
+
+        if type == "result":
+            name_column = mm_schemas.ResultData.RESULT_NAME
+            columns_to_zip.append(mm_schemas.ResultData.RESULT_KIND)
+        else:
+            name_column = mm_schemas.MetricData.METRIC_NAME
+        columns_to_zip.insert(1, name_column)
+
+        # groupby has different behavior for category columns
+        df["endpoint_id"] = df["endpoint_id"].astype(str)
+        df["event_values"] = list(zip(*[df[col] for col in columns_to_zip]))
+        grouped_by_event_values = df.groupby("endpoint_id")["event_values"].apply(set)
+        common_event_values_combinations = set.intersection(*grouped_by_event_values)
+        result_kind = None
+        for data in common_event_values_combinations:
+            application_name, event_name = data[0], data[1]
+            if len(data) > 2:  # in result case
+                result_kind = data[2]
+            metrics.append(
+                mm_schemas.ModelEndpointMonitoringMetric(
+                    project=project,
+                    type=type,
+                    app=application_name,
+                    name=event_name,
+                    kind=result_kind,
+                )
+            )
+        return {dict_key: metrics}
+
     @staticmethod
     def _get_start_end(
         start: typing.Union[datetime, None],

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import typing
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 import pandas as pd
 import taosws
@@ -164,6 +164,17 @@ class TDEngineConnector(TSDBConnector):
     def _convert_to_datetime(val: typing.Union[str, datetime]) -> datetime:
         return datetime.fromisoformat(val) if isinstance(val, str) else val
 
+    @staticmethod
+    def _get_endpoint_filter(endpoint_id: typing.Union[str, list[str]]):
+        if isinstance(endpoint_id, str):
+            return f"endpoint_id='{endpoint_id}'"
+        elif isinstance(endpoint_id, list):
+            return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Invalid 'endpoint_id' filter: must be a string or a list."
+            )
+
     def apply_monitoring_stream_steps(self, graph, **kwarg):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -542,12 +553,11 @@ class TDEngineConnector(TSDBConnector):
             },
             inplace=True,
         )
-        df[mm_schemas.EventFieldType.LAST_REQUEST] = df[
-            mm_schemas.EventFieldType.LAST_REQUEST
-        ].map(
-            lambda last_request: datetime.strptime(
-                last_request, "%Y-%m-%d %H:%M:%S.%f %z"
-            ).astimezone(tz=timezone.utc)
+        df[mm_schemas.EventFieldType.LAST_REQUEST] = pd.to_datetime(
+            df[mm_schemas.EventFieldType.LAST_REQUEST],
+            errors="coerce",
+            format="ISO8601",
+            utc=True,
         )
         return df
 
@@ -588,7 +598,7 @@ class TDEngineConnector(TSDBConnector):
 
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
@@ -602,11 +612,12 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.MetricData.METRIC_NAME,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            filter_query=f"endpoint_id='{endpoint_id}'",
+            filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
             timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
             group_by=[
                 mm_schemas.WriterEvent.APPLICATION_NAME,
                 mm_schemas.MetricData.METRIC_NAME,
+                mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             agg_funcs=["last"],
         )
@@ -624,7 +635,7 @@ class TDEngineConnector(TSDBConnector):
 
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: typing.Union[str, list[str]],
         start: typing.Optional[datetime] = None,
         end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
@@ -639,11 +650,12 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_KIND,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            filter_query=f"endpoint_id='{endpoint_id}'",
+            filter_query=self._get_endpoint_filter(endpoint_id=endpoint_id),
             timestamp_column=mm_schemas.WriterEvent.END_INFER_TIME,
             group_by=[
                 mm_schemas.WriterEvent.APPLICATION_NAME,
                 mm_schemas.ResultData.RESULT_NAME,
+                mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             agg_funcs=["last"],
         )

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -33,6 +33,8 @@ _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
+V3IO_MEPS_LIMIT = 50  # TODO remove limitation after fixing ML-8886
+
 
 def _is_no_schema_error(exc: v3io_frames.Error) -> bool:
     """
@@ -577,6 +579,21 @@ class V3IOTSDBConnector(TSDBConnector):
             token=v3io_access_key,
         )
 
+    @staticmethod
+    def _get_endpoint_filter(endpoint_id: Union[str, list[str]]):
+        if isinstance(endpoint_id, str):
+            return f"endpoint_id=='{endpoint_id}'"
+        elif isinstance(endpoint_id, list):
+            if len(endpoint_id) > V3IO_MEPS_LIMIT:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    f"Filtering more than {V3IO_MEPS_LIMIT} model endpoints in the V3IO connector is not supported."
+                )
+            return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Invalid 'endpoint_id' filter: must be a string or a list, endpoint_id: {endpoint_id}"
+            )
+
     def read_metrics_data(
         self,
         *,
@@ -813,17 +830,18 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: Union[str, list[str]],
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.METRICS,
             start=start,
             end=end,
             columns=[mm_schemas.MetricData.METRIC_VALUE],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:
@@ -834,11 +852,12 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_results_metadata(
         self,
-        endpoint_id: str,
+        endpoint_id: Union[str, list[str]],
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
@@ -846,7 +865,7 @@ class V3IOTSDBConnector(TSDBConnector):
             columns=[
                 mm_schemas.ResultData.RESULT_KIND,
             ],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:

mlrun/model_monitoring/helpers.py

@@ -32,7 +32,7 @@ import mlrun.utils.helpers
 from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
-    _compose_full_name,
+    compose_full_name,
 )
 from mlrun.utils import logger
 
@@ -450,7 +450,7 @@ def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
 
 
 def get_invocations_fqn(project: str) -> str:
-    return _compose_full_name(
+    return compose_full_name(
         project=project,
         app=mm_constants.SpecialApps.MLRUN_INFRA,
         name=mm_constants.PredictionsQueryConstants.INVOCATIONS,

mlrun/projects/pipelines.py

@@ -748,7 +748,8 @@ class _LocalRunner(_PipelineRunner):
             project.set_source(source=source)
         pipeline_context.workflow_artifact_path = artifact_path
 
-        project.notifiers.push_pipeline_start_message(
+        # TODO: we should create endpoint for sending custom notification from BE
+        project.notifiers.push_pipeline_start_message_from_client(
             project.metadata.name, pipeline_id=workflow_id
         )
         err = None