mlrun 1.7.0rc16__py3-none-any.whl → 1.7.0rc18__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import datetime
+import typing
 
 import pandas as pd
 import v3io_frames.client
@@ -21,7 +21,7 @@ from v3io.dataplane import Client as V3IOClient
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
-import mlrun.common.schemas.model_monitoring as mm_constants
+import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.feature_store.steps
 import mlrun.utils.v3io_clients
 from mlrun.model_monitoring.db import TSDBConnector
@@ -37,12 +37,14 @@ class V3IOTSDBConnector(TSDBConnector):
     Client that provides API for executing commands on the V3IO TSDB table.
     """
 
+    type: str = mm_schemas.TSDBTarget.V3IO_TSDB
+
     def __init__(
         self,
         project: str,
-        access_key: str = None,
+        access_key: typing.Optional[str] = None,
         container: str = "users",
-        v3io_framesd: str = None,
+        v3io_framesd: typing.Optional[str] = None,
         create_table: bool = False,
     ):
         super().__init__(project=project)
@@ -61,14 +63,14 @@ class V3IOTSDBConnector(TSDBConnector):
         self._init_tables_path()
 
         if create_table:
-            self.create_tsdb_application_tables()
+            self.create_tables()
 
     def _init_tables_path(self):
         self.tables = {}
 
         events_table_full_path = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=self.project,
-            kind=mm_constants.FileTargetKind.EVENTS,
+            kind=mm_schemas.FileTargetKind.EVENTS,
         )
         (
             _,
@@ -77,12 +79,12 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             events_table_full_path
         )
-        self.tables[mm_constants.MonitoringTSDBTables.EVENTS] = events_path
+        self.tables[mm_schemas.V3IOTSDBTables.EVENTS] = events_path
 
         monitoring_application_full_path = (
             mlrun.mlconf.get_model_monitoring_file_target_path(
                 project=self.project,
-                kind=mm_constants.FileTargetKind.MONITORING_APPLICATION,
+                kind=mm_schemas.FileTargetKind.MONITORING_APPLICATION,
             )
         )
         (
@@ -92,28 +94,45 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             monitoring_application_full_path
         )
-        self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS] = (
-            monitoring_application_path + mm_constants.MonitoringTSDBTables.APP_RESULTS
+        self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS] = (
+            monitoring_application_path + mm_schemas.V3IOTSDBTables.APP_RESULTS
+        )
+        self.tables[mm_schemas.V3IOTSDBTables.METRICS] = (
+            monitoring_application_path + mm_schemas.V3IOTSDBTables.METRICS
+        )
+
+        monitoring_predictions_full_path = (
+            mlrun.mlconf.get_model_monitoring_file_target_path(
+                project=self.project,
+                kind=mm_schemas.FileTargetKind.PREDICTIONS,
+            )
         )
-        self.tables[mm_constants.MonitoringTSDBTables.METRICS] = (
-            monitoring_application_path + mm_constants.MonitoringTSDBTables.METRICS
+        (
+            _,
+            _,
+            monitoring_predictions_path,
+        ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
+            monitoring_predictions_full_path
         )
+        self.tables[mm_schemas.FileTargetKind.PREDICTIONS] = monitoring_predictions_path
 
-    def create_tsdb_application_tables(self):
+    def create_tables(self) -> None:
         """
-        Create the application tables using the TSDB connector. At the moment we support 2 types of application tables:
+        Create the tables using the TSDB connector. The tables are being created in the V3IO TSDB and include:
         - app_results: a detailed result that includes status, kind, extra data, etc.
         - metrics: a basic key value that represents a single numeric metric.
+        Note that the predictions table is automatically created by the model monitoring stream pod.
         """
         application_tables = [
-            mm_constants.MonitoringTSDBTables.APP_RESULTS,
-            mm_constants.MonitoringTSDBTables.METRICS,
+            mm_schemas.V3IOTSDBTables.APP_RESULTS,
+            mm_schemas.V3IOTSDBTables.METRICS,
         ]
-        for table in application_tables:
-            logger.info("Creating table in V3IO TSDB", table=table)
+        for table_name in application_tables:
+            logger.info("Creating table in V3IO TSDB", table_name=table_name)
+            table = self.tables[table_name]
             self._frames_client.create(
                 backend=_TSDB_BE,
-                table=self.tables[table],
+                table=table,
                 if_exists=IGNORE,
                 rate=_TSDB_RATE,
             )
@@ -134,6 +153,27 @@ class V3IOTSDBConnector(TSDBConnector):
         - custom_metrics (user-defined metrics)
         """
 
+        # Write latency per prediction, labeled by endpoint ID only
+        graph.add_step(
+            "storey.TSDBTarget",
+            name="tsdb_predictions",
+            after="MapFeatureNames",
+            path=f"{self.container}/{self.tables[mm_schemas.FileTargetKind.PREDICTIONS]}",
+            rate="1/s",
+            time_col=mm_schemas.EventFieldType.TIMESTAMP,
+            container=self.container,
+            v3io_frames=self.v3io_framesd,
+            columns=["latency"],
+            index_cols=[
+                mm_schemas.EventFieldType.ENDPOINT_ID,
+            ],
+            aggr="count,avg",
+            aggr_granularity="1m",
+            max_events=tsdb_batching_max_events,
+            flush_after_seconds=tsdb_batching_timeout_secs,
+            key=mm_schemas.EventFieldType.ENDPOINT_ID,
+        )
+
         # Before writing data to TSDB, create dictionary of 2-3 dictionaries that contains
         # stats and details about the events
 
@@ -160,40 +200,40 @@ class V3IOTSDBConnector(TSDBConnector):
                 "storey.TSDBTarget",
                 name=name,
                 after=after,
-                path=f"{self.container}/{self.tables[mm_constants.MonitoringTSDBTables.EVENTS]}",
+                path=f"{self.container}/{self.tables[mm_schemas.V3IOTSDBTables.EVENTS]}",
                 rate="10/m",
-                time_col=mm_constants.EventFieldType.TIMESTAMP,
+                time_col=mm_schemas.EventFieldType.TIMESTAMP,
                 container=self.container,
                 v3io_frames=self.v3io_framesd,
                 infer_columns_from_data=True,
                 index_cols=[
-                    mm_constants.EventFieldType.ENDPOINT_ID,
-                    mm_constants.EventFieldType.RECORD_TYPE,
-                    mm_constants.EventFieldType.ENDPOINT_TYPE,
+                    mm_schemas.EventFieldType.ENDPOINT_ID,
+                    mm_schemas.EventFieldType.RECORD_TYPE,
+                    mm_schemas.EventFieldType.ENDPOINT_TYPE,
                 ],
                 max_events=tsdb_batching_max_events,
                 flush_after_seconds=tsdb_batching_timeout_secs,
-                key=mm_constants.EventFieldType.ENDPOINT_ID,
+                key=mm_schemas.EventFieldType.ENDPOINT_ID,
             )
 
         # unpacked base_metrics dictionary
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys1",
-            keys=mm_constants.EventKeyMetrics.BASE_METRICS,
+            keys=mm_schemas.EventKeyMetrics.BASE_METRICS,
         )
         apply_tsdb_target(name="tsdb1", after="FilterAndUnpackKeys1")
 
         # unpacked endpoint_features dictionary
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys2",
-            keys=mm_constants.EventKeyMetrics.ENDPOINT_FEATURES,
+            keys=mm_schemas.EventKeyMetrics.ENDPOINT_FEATURES,
         )
         apply_tsdb_target(name="tsdb2", after="FilterAndUnpackKeys2")
 
         # unpacked custom_metrics dictionary. In addition, use storey.Filter remove none values
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys3",
-            keys=mm_constants.EventKeyMetrics.CUSTOM_METRICS,
+            keys=mm_schemas.EventKeyMetrics.CUSTOM_METRICS,
         )
 
         def apply_storey_filter():
@@ -210,56 +250,55 @@ class V3IOTSDBConnector(TSDBConnector):
     def write_application_event(
         self,
         event: dict,
-        kind: mm_constants.WriterEventKind = mm_constants.WriterEventKind.RESULT,
-    ):
+        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
+    ) -> None:
         """Write a single result or metric to TSDB"""
 
-        event[mm_constants.WriterEvent.END_INFER_TIME] = (
-            datetime.datetime.fromisoformat(
-                event[mm_constants.WriterEvent.END_INFER_TIME]
-            )
+        event[mm_schemas.WriterEvent.END_INFER_TIME] = datetime.datetime.fromisoformat(
+            event[mm_schemas.WriterEvent.END_INFER_TIME]
         )
+        index_cols_base = [
+            mm_schemas.WriterEvent.END_INFER_TIME,
+            mm_schemas.WriterEvent.ENDPOINT_ID,
+            mm_schemas.WriterEvent.APPLICATION_NAME,
+        ]
 
-        if kind == mm_constants.WriterEventKind.METRIC:
-            # TODO : Implement the logic for writing metrics to V3IO TSDB
-            return
+        if kind == mm_schemas.WriterEventKind.METRIC:
+            table = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
+            index_cols = index_cols_base + [mm_schemas.MetricData.METRIC_NAME]
+        elif kind == mm_schemas.WriterEventKind.RESULT:
+            table = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
+            index_cols = index_cols_base + [mm_schemas.ResultData.RESULT_NAME]
+            del event[mm_schemas.ResultData.RESULT_EXTRA_DATA]
+        else:
+            raise ValueError(f"Invalid {kind = }")
 
-        del event[mm_constants.ResultData.RESULT_EXTRA_DATA]
         try:
             self._frames_client.write(
                 backend=_TSDB_BE,
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                table=table,
                 dfs=pd.DataFrame.from_records([event]),
-                index_cols=[
-                    mm_constants.WriterEvent.END_INFER_TIME,
-                    mm_constants.WriterEvent.ENDPOINT_ID,
-                    mm_constants.WriterEvent.APPLICATION_NAME,
-                    mm_constants.ResultData.RESULT_NAME,
-                ],
-            )
-            logger.info(
-                "Updated V3IO TSDB successfully",
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                index_cols=index_cols,
             )
+            logger.info("Updated V3IO TSDB successfully", table=table)
         except v3io_frames.errors.Error as err:
-            logger.warn(
+            logger.exception(
                 "Could not write drift measures to TSDB",
                 err=err,
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                table=table,
                 event=event,
             )
-
             raise mlrun.errors.MLRunRuntimeError(
                 f"Failed to write application result to TSDB: {err}"
             )
 
-    def delete_tsdb_resources(self, table: str = None):
+    def delete_tsdb_resources(self, table: typing.Optional[str] = None):
         if table:
             # Delete a specific table
             tables = [table]
         else:
             # Delete all tables
-            tables = mm_constants.MonitoringTSDBTables.list()
+            tables = mm_schemas.V3IOTSDBTables.list()
         for table in tables:
             try:
                 self._frames_client.delete(
@@ -282,8 +321,8 @@ class V3IOTSDBConnector(TSDBConnector):
         self,
         endpoint_id: str,
         metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
+        start: str,
+        end: str,
     ) -> dict[str, list[tuple[str, float]]]:
         """
         Getting real time metrics from the TSDB. There are pre-defined metrics for model endpoints such as
@@ -312,7 +351,7 @@ class V3IOTSDBConnector(TSDBConnector):
 
         try:
             data = self.get_records(
-                table=mm_constants.MonitoringTSDBTables.EVENTS,
+                table=mm_schemas.V3IOTSDBTables.EVENTS,
                 columns=["endpoint_id", *metrics],
                 filter_query=f"endpoint_id=='{endpoint_id}'",
                 start=start,
@@ -339,17 +378,14 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_records(
         self,
         table: str,
-        columns: list[str] = None,
+        start: str,
+        end: str,
+        columns: typing.Optional[list[str]] = None,
         filter_query: str = "",
-        start: str = "now-1h",
-        end: str = "now",
     ) -> pd.DataFrame:
         """
          Getting records from V3IO TSDB data collection.
         :param table:            Path to the collection to query.
-        :param columns:          Columns to include in the result.
-        :param filter_query:     V3IO filter expression. The expected filter expression includes different conditions,
-                                 divided by ' AND '.
         :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
                                  time, a Unix timestamp in milliseconds, a relative time (`'now'` or
                                  `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
@@ -358,6 +394,9 @@ class V3IOTSDBConnector(TSDBConnector):
                                  time, a Unix timestamp in milliseconds, a relative time (`'now'` or
                                  `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
                                  `'s'` = seconds), or 0 for the earliest time.
+        :param columns:          Columns to include in the result.
+        :param filter_query:     V3IO filter expression. The expected filter expression includes different conditions,
+                                 divided by ' AND '.
         :return: DataFrame with the provided attributes from the data collection.
         :raise:  MLRunNotFoundError if the provided table wasn't found.
         """
@@ -384,7 +423,7 @@ class V3IOTSDBConnector(TSDBConnector):
         """
         events_table_full_path = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=self.project,
-            kind=mm_constants.FileTargetKind.EVENTS,
+            kind=mm_schemas.FileTargetKind.EVENTS,
         )
 
         # Generate the main directory with the V3IO resources

mlrun/model_monitoring/db/v3io_tsdb_reader.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Move this module into the TSDB abstraction once it is in.
+# TODO: Move this module into the TSDB abstraction:
+# mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py
 
 from datetime import datetime
 from io import StringIO
+from typing import Literal, Optional, Union
 
 import pandas as pd
 
@@ -25,21 +27,27 @@ import mlrun.model_monitoring.writer as mm_writer
 import mlrun.utils.v3io_clients
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
+    ModelEndpointMonitoringMetricNoData,
     ModelEndpointMonitoringMetricType,
-    ModelEndpointMonitoringResultNoData,
+    ModelEndpointMonitoringMetricValues,
     ModelEndpointMonitoringResultValues,
     _compose_full_name,
-    _ModelEndpointMonitoringResultValuesBase,
+    _ModelEndpointMonitoringMetricValuesBase,
 )
 from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
 from mlrun.model_monitoring.db.tsdb.v3io.v3io_connector import _TSDB_BE
 from mlrun.utils import logger
 
 
-def _get_sql_query(endpoint_id: str, names: list[tuple[str, str]]) -> str:
+def _get_sql_query(
+    endpoint_id: str,
+    names: list[tuple[str, str]],
+    table_name: str = mm_constants.V3IOTSDBTables.APP_RESULTS,
+    name: str = mm_writer.ResultData.RESULT_NAME,
+) -> str:
     with StringIO() as query:
         query.write(
-            f"SELECT * FROM '{mm_constants.MonitoringTSDBTables.APP_RESULTS}' "
+            f"SELECT * FROM '{table_name}' "
             f"WHERE {mm_writer.WriterEvent.ENDPOINT_ID}='{endpoint_id}'"
         )
         if names:
@@ -48,7 +56,7 @@ def _get_sql_query(endpoint_id: str, names: list[tuple[str, str]]) -> str:
             for i, (app_name, result_name) in enumerate(names):
                 sub_cond = (
                     f"({mm_writer.WriterEvent.APPLICATION_NAME}='{app_name}' "
-                    f"AND {mm_writer.ResultData.RESULT_NAME}='{result_name}')"
+                    f"AND {name}='{result_name}')"
                 )
                 if i != 0:  # not first sub condition
                     query.write(" OR ")
@@ -73,30 +81,87 @@ def _get_result_kind(result_df: pd.DataFrame) -> mm_constants.ResultKindApp:
     return unique_kinds[0]
 
 
-def read_data(
+def read_metrics_data(
     *,
     project: str,
     endpoint_id: str,
     start: datetime,
     end: datetime,
     metrics: list[ModelEndpointMonitoringMetric],
-) -> list[_ModelEndpointMonitoringResultValuesBase]:
+    type: Literal["metrics", "results"] = "results",
+) -> Union[
+    list[
+        Union[
+            ModelEndpointMonitoringResultValues,
+            ModelEndpointMonitoringMetricNoData,
+        ],
+    ],
+    list[
+        Union[
+            ModelEndpointMonitoringMetricValues,
+            ModelEndpointMonitoringMetricNoData,
+        ],
+    ],
+]:
+    """
+    Read metrics OR results from the TSDB and return as a list.
+    Note: the type must match the actual metrics in the `metrics` parameter.
+    If the type is "results", pass only results in the `metrics` parameter.
+    """
     client = mlrun.utils.v3io_clients.get_frames_client(
         address=mlrun.mlconf.v3io_framesd,
         container=KVStoreBase.get_v3io_monitoring_apps_container(project),
     )
+
+    if type == "metrics":
+        table_name = mm_constants.V3IOTSDBTables.METRICS
+        name = mm_constants.MetricData.METRIC_NAME
+        df_handler = df_to_metrics_values
+    elif type == "results":
+        table_name = mm_constants.V3IOTSDBTables.APP_RESULTS
+        name = mm_constants.ResultData.RESULT_NAME
+        df_handler = df_to_results_values
+    else:
+        raise ValueError(f"Invalid {type = }")
+
+    query = _get_sql_query(
+        endpoint_id,
+        [(metric.app, metric.name) for metric in metrics],
+        table_name=table_name,
+        name=name,
+    )
+
+    logger.debug("Querying V3IO TSDB", query=query)
+
     df: pd.DataFrame = client.read(
         backend=_TSDB_BE,
-        query=_get_sql_query(
-            endpoint_id, [(metric.app, metric.name) for metric in metrics]
-        ),
+        query=query,
         start=start,
         end=end,
     )
 
+    logger.debug(
+        "Read a data-frame", project=project, endpoint_id=endpoint_id, is_empty=df.empty
+    )
+
+    return df_handler(df=df, metrics=metrics, project=project)
+
+
+def df_to_results_values(
+    *, df: pd.DataFrame, metrics: list[ModelEndpointMonitoringMetric], project: str
+) -> list[
+    Union[ModelEndpointMonitoringResultValues, ModelEndpointMonitoringMetricNoData]
+]:
+    """
+    Parse a time-indexed data-frame of results from the TSDB into a list of
+    results values per distinct results.
+    When a result is not found in the data-frame, it is represented in no-data object.
+    """
     metrics_without_data = {metric.full_name: metric for metric in metrics}
 
-    metrics_values: list[_ModelEndpointMonitoringResultValuesBase] = []
+    metrics_values: list[
+        Union[ModelEndpointMonitoringResultValues, ModelEndpointMonitoringMetricNoData]
+    ] = []
     if not df.empty:
         grouped = df.groupby(
             [mm_writer.WriterEvent.APPLICATION_NAME, mm_writer.ResultData.RESULT_NAME],
@@ -104,13 +169,13 @@ def read_data(
         )
     else:
         grouped = []
-    for (app_name, result_name), sub_df in grouped:
+        logger.debug("No results", missing_results=metrics_without_data.keys())
+    for (app_name, name), sub_df in grouped:
         result_kind = _get_result_kind(sub_df)
-        full_name = _compose_full_name(project=project, app=app_name, name=result_name)
+        full_name = _compose_full_name(project=project, app=app_name, name=name)
         metrics_values.append(
             ModelEndpointMonitoringResultValues(
                 full_name=full_name,
-                type=ModelEndpointMonitoringMetricType.RESULT,
                 result_kind=result_kind,
                 values=list(
                     zip(
@@ -124,11 +189,147 @@ def read_data(
         del metrics_without_data[full_name]
 
     for metric in metrics_without_data.values():
+        if metric.full_name == get_invocations_fqn(project):
+            continue
         metrics_values.append(
-            ModelEndpointMonitoringResultNoData(
+            ModelEndpointMonitoringMetricNoData(
                 full_name=metric.full_name,
                 type=ModelEndpointMonitoringMetricType.RESULT,
             )
         )
 
     return metrics_values
+
+
+def df_to_metrics_values(
+    *, df: pd.DataFrame, metrics: list[ModelEndpointMonitoringMetric], project: str
+) -> list[
+    Union[ModelEndpointMonitoringMetricValues, ModelEndpointMonitoringMetricNoData]
+]:
+    """
+    Parse a time-indexed data-frame of metrics from the TSDB into a list of
+    metrics values per distinct results.
+    When a metric is not found in the data-frame, it is represented in no-data object.
+    """
+    metrics_without_data = {metric.full_name: metric for metric in metrics}
+
+    metrics_values: list[
+        Union[ModelEndpointMonitoringMetricValues, ModelEndpointMonitoringMetricNoData]
+    ] = []
+    if not df.empty:
+        grouped = df.groupby(
+            [mm_writer.WriterEvent.APPLICATION_NAME, mm_writer.MetricData.METRIC_NAME],
+            observed=False,
+        )
+    else:
+        logger.debug("No metrics", missing_metrics=metrics_without_data.keys())
+        grouped = []
+    for (app_name, name), sub_df in grouped:
+        full_name = _compose_full_name(
+            project=project,
+            app=app_name,
+            name=name,
+            type=ModelEndpointMonitoringMetricType.METRIC,
+        )
+        metrics_values.append(
+            ModelEndpointMonitoringMetricValues(
+                full_name=full_name,
+                values=list(
+                    zip(
+                        sub_df.index,
+                        sub_df[mm_writer.MetricData.METRIC_VALUE],
+                    )
+                ),  # pyright: ignore[reportArgumentType]
+            )
+        )
+        del metrics_without_data[full_name]
+
+    for metric in metrics_without_data.values():
+        metrics_values.append(
+            ModelEndpointMonitoringMetricNoData(
+                full_name=metric.full_name,
+                type=ModelEndpointMonitoringMetricType.METRIC,
+            )
+        )
+
+    return metrics_values
+
+
+def get_invocations_fqn(project: str):
+    return mlrun.common.schemas.model_monitoring.model_endpoints._compose_full_name(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        name=mlrun.common.schemas.model_monitoring.constants.PredictionsQueryConstants.INVOCATIONS,
+        type=mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetricType.METRIC,
+    )
+
+
+def read_predictions(
+    *,
+    project: str,
+    endpoint_id: str,
+    start: Optional[Union[datetime, str]] = None,
+    end: Optional[Union[datetime, str]] = None,
+    aggregation_window: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> _ModelEndpointMonitoringMetricValuesBase:
+    client = mlrun.utils.v3io_clients.get_frames_client(
+        address=mlrun.mlconf.v3io_framesd,
+        container="users",
+    )
+    frames_client_kwargs = {}
+    if aggregation_window:
+        frames_client_kwargs["step"] = aggregation_window
+        frames_client_kwargs["aggregation_window"] = aggregation_window
+    if limit:
+        frames_client_kwargs["limit"] = limit
+    df: pd.DataFrame = client.read(
+        backend=_TSDB_BE,
+        table=f"pipelines/{project}/model-endpoints/predictions",
+        columns=["latency"],
+        filter=f"endpoint_id=='{endpoint_id}'",
+        start=start,
+        end=end,
+        aggregators="count",
+        **frames_client_kwargs,
+    )
+
+    full_name = get_invocations_fqn(project)
+
+    if df.empty:
+        return ModelEndpointMonitoringMetricNoData(
+            full_name=full_name,
+            type=ModelEndpointMonitoringMetricType.METRIC,
+        )
+
+    return ModelEndpointMonitoringMetricValues(
+        full_name=full_name,
+        values=list(
+            zip(
+                df.index,
+                df["count(latency)"],
+            )
+        ),
+    )
+
+
+def read_prediction_metric_for_endpoint_if_exists(
+    *,
+    project: str,
+    endpoint_id: str,
+) -> Optional[ModelEndpointMonitoringMetric]:
+    predictions = read_predictions(
+        project=project,
+        endpoint_id=endpoint_id,
+        start="0",
+        end="now",
+        limit=1,  # Read just one record, because we just want to check if there is any data for this endpoint_id
+    )
+    if predictions:
+        return ModelEndpointMonitoringMetric(
+            project=project,
+            app=mm_constants.SpecialApps.MLRUN_INFRA,
+            type=ModelEndpointMonitoringMetricType.METRIC,
+            name=mlrun.common.schemas.model_monitoring.constants.PredictionsQueryConstants.INVOCATIONS,
+            full_name=get_invocations_fqn(project),
+        )