mlrun 1.7.0rc17__py3-none-any.whl → 1.7.0rc19__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -12,23 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import datetime
+import typing
+from datetime import datetime
+from io import StringIO
+from typing import Literal, Optional, Union
 
 import pandas as pd
 import v3io_frames.client
 import v3io_frames.errors
-from v3io.dataplane import Client as V3IOClient
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
-import mlrun.common.schemas.model_monitoring as mm_constants
+import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.feature_store.steps
 import mlrun.utils.v3io_clients
 from mlrun.model_monitoring.db import TSDBConnector
+from mlrun.model_monitoring.helpers import get_invocations_fqn
 from mlrun.utils import logger
 
 _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
+_CONTAINER = "users"
 
 
 class V3IOTSDBConnector(TSDBConnector):
@@ -37,16 +41,16 @@ class V3IOTSDBConnector(TSDBConnector):
     Client that provides API for executing commands on the V3IO TSDB table.
     """
 
+    type: str = mm_schemas.TSDBTarget.V3IO_TSDB
+
     def __init__(
         self,
         project: str,
-        access_key: str = None,
-        container: str = "users",
-        v3io_framesd: str = None,
+        container: str = _CONTAINER,
+        v3io_framesd: typing.Optional[str] = None,
         create_table: bool = False,
-    ):
+    ) -> None:
         super().__init__(project=project)
-        self.access_key = access_key or mlrun.mlconf.get_v3io_access_key()
 
         self.container = container
 
@@ -54,21 +58,18 @@ class V3IOTSDBConnector(TSDBConnector):
         self._frames_client: v3io_frames.client.ClientBase = (
             self._get_v3io_frames_client(self.container)
         )
-        self._v3io_client: V3IOClient = mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
-        )
 
         self._init_tables_path()
 
         if create_table:
-            self.create_tsdb_application_tables()
+            self.create_tables()
 
     def _init_tables_path(self):
         self.tables = {}
 
         events_table_full_path = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=self.project,
-            kind=mm_constants.FileTargetKind.EVENTS,
+            kind=mm_schemas.FileTargetKind.EVENTS,
         )
         (
             _,
@@ -77,12 +78,12 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             events_table_full_path
         )
-        self.tables[mm_constants.MonitoringTSDBTables.EVENTS] = events_path
+        self.tables[mm_schemas.V3IOTSDBTables.EVENTS] = events_path
 
         monitoring_application_full_path = (
             mlrun.mlconf.get_model_monitoring_file_target_path(
                 project=self.project,
-                kind=mm_constants.FileTargetKind.MONITORING_APPLICATION,
+                kind=mm_schemas.FileTargetKind.MONITORING_APPLICATION,
             )
         )
         (
@@ -92,17 +93,17 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             monitoring_application_full_path
         )
-        self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS] = (
-            monitoring_application_path + mm_constants.MonitoringTSDBTables.APP_RESULTS
+        self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS] = (
+            monitoring_application_path + mm_schemas.V3IOTSDBTables.APP_RESULTS
         )
-        self.tables[mm_constants.MonitoringTSDBTables.METRICS] = (
-            monitoring_application_path + mm_constants.MonitoringTSDBTables.METRICS
+        self.tables[mm_schemas.V3IOTSDBTables.METRICS] = (
+            monitoring_application_path + mm_schemas.V3IOTSDBTables.METRICS
         )
 
         monitoring_predictions_full_path = (
             mlrun.mlconf.get_model_monitoring_file_target_path(
                 project=self.project,
-                kind=mm_constants.FileTargetKind.PREDICTIONS,
+                kind=mm_schemas.FileTargetKind.PREDICTIONS,
             )
         )
         (
@@ -112,25 +113,25 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             monitoring_predictions_full_path
         )
-        self.tables[mm_constants.FileTargetKind.PREDICTIONS] = (
-            monitoring_predictions_path
-        )
+        self.tables[mm_schemas.FileTargetKind.PREDICTIONS] = monitoring_predictions_path
 
-    def create_tsdb_application_tables(self):
+    def create_tables(self) -> None:
         """
-        Create the application tables using the TSDB connector. At the moment we support 2 types of application tables:
+        Create the tables using the TSDB connector. The tables are being created in the V3IO TSDB and include:
         - app_results: a detailed result that includes status, kind, extra data, etc.
         - metrics: a basic key value that represents a single numeric metric.
+        Note that the predictions table is automatically created by the model monitoring stream pod.
         """
         application_tables = [
-            mm_constants.MonitoringTSDBTables.APP_RESULTS,
-            mm_constants.MonitoringTSDBTables.METRICS,
+            mm_schemas.V3IOTSDBTables.APP_RESULTS,
+            mm_schemas.V3IOTSDBTables.METRICS,
         ]
-        for table in application_tables:
-            logger.info("Creating table in V3IO TSDB", table=table)
+        for table_name in application_tables:
+            logger.info("Creating table in V3IO TSDB", table_name=table_name)
+            table = self.tables[table_name]
             self._frames_client.create(
                 backend=_TSDB_BE,
-                table=self.tables[table],
+                table=table,
                 if_exists=IGNORE,
                 rate=_TSDB_RATE,
             )
@@ -156,20 +157,20 @@ class V3IOTSDBConnector(TSDBConnector):
             "storey.TSDBTarget",
             name="tsdb_predictions",
             after="MapFeatureNames",
-            path=f"{self.container}/{self.tables[mm_constants.FileTargetKind.PREDICTIONS]}",
+            path=f"{self.container}/{self.tables[mm_schemas.FileTargetKind.PREDICTIONS]}",
             rate="1/s",
-            time_col=mm_constants.EventFieldType.TIMESTAMP,
+            time_col=mm_schemas.EventFieldType.TIMESTAMP,
             container=self.container,
             v3io_frames=self.v3io_framesd,
             columns=["latency"],
             index_cols=[
-                mm_constants.EventFieldType.ENDPOINT_ID,
+                mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
             aggr="count,avg",
             aggr_granularity="1m",
             max_events=tsdb_batching_max_events,
             flush_after_seconds=tsdb_batching_timeout_secs,
-            key=mm_constants.EventFieldType.ENDPOINT_ID,
+            key=mm_schemas.EventFieldType.ENDPOINT_ID,
         )
 
         # Before writing data to TSDB, create dictionary of 2-3 dictionaries that contains
@@ -198,40 +199,40 @@ class V3IOTSDBConnector(TSDBConnector):
                 "storey.TSDBTarget",
                 name=name,
                 after=after,
-                path=f"{self.container}/{self.tables[mm_constants.MonitoringTSDBTables.EVENTS]}",
+                path=f"{self.container}/{self.tables[mm_schemas.V3IOTSDBTables.EVENTS]}",
                 rate="10/m",
-                time_col=mm_constants.EventFieldType.TIMESTAMP,
+                time_col=mm_schemas.EventFieldType.TIMESTAMP,
                 container=self.container,
                 v3io_frames=self.v3io_framesd,
                 infer_columns_from_data=True,
                 index_cols=[
-                    mm_constants.EventFieldType.ENDPOINT_ID,
-                    mm_constants.EventFieldType.RECORD_TYPE,
-                    mm_constants.EventFieldType.ENDPOINT_TYPE,
+                    mm_schemas.EventFieldType.ENDPOINT_ID,
+                    mm_schemas.EventFieldType.RECORD_TYPE,
+                    mm_schemas.EventFieldType.ENDPOINT_TYPE,
                 ],
                 max_events=tsdb_batching_max_events,
                 flush_after_seconds=tsdb_batching_timeout_secs,
-                key=mm_constants.EventFieldType.ENDPOINT_ID,
+                key=mm_schemas.EventFieldType.ENDPOINT_ID,
             )
 
         # unpacked base_metrics dictionary
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys1",
-            keys=mm_constants.EventKeyMetrics.BASE_METRICS,
+            keys=mm_schemas.EventKeyMetrics.BASE_METRICS,
         )
         apply_tsdb_target(name="tsdb1", after="FilterAndUnpackKeys1")
 
         # unpacked endpoint_features dictionary
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys2",
-            keys=mm_constants.EventKeyMetrics.ENDPOINT_FEATURES,
+            keys=mm_schemas.EventKeyMetrics.ENDPOINT_FEATURES,
         )
         apply_tsdb_target(name="tsdb2", after="FilterAndUnpackKeys2")
 
         # unpacked custom_metrics dictionary. In addition, use storey.Filter remove none values
         apply_filter_and_unpacked_keys(
             name="FilterAndUnpackKeys3",
-            keys=mm_constants.EventKeyMetrics.CUSTOM_METRICS,
+            keys=mm_schemas.EventKeyMetrics.CUSTOM_METRICS,
         )
 
         def apply_storey_filter():
@@ -248,62 +249,58 @@ class V3IOTSDBConnector(TSDBConnector):
     def write_application_event(
         self,
         event: dict,
-        kind: mm_constants.WriterEventKind = mm_constants.WriterEventKind.RESULT,
-    ):
+        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
+    ) -> None:
         """Write a single result or metric to TSDB"""
 
-        event[mm_constants.WriterEvent.END_INFER_TIME] = (
-            datetime.datetime.fromisoformat(
-                event[mm_constants.WriterEvent.END_INFER_TIME]
-            )
+        event[mm_schemas.WriterEvent.END_INFER_TIME] = datetime.fromisoformat(
+            event[mm_schemas.WriterEvent.END_INFER_TIME]
         )
+        index_cols_base = [
+            mm_schemas.WriterEvent.END_INFER_TIME,
+            mm_schemas.WriterEvent.ENDPOINT_ID,
+            mm_schemas.WriterEvent.APPLICATION_NAME,
+        ]
 
-        if kind == mm_constants.WriterEventKind.METRIC:
-            # TODO : Implement the logic for writing metrics to V3IO TSDB
-            return
+        if kind == mm_schemas.WriterEventKind.METRIC:
+            table = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
+            index_cols = index_cols_base + [mm_schemas.MetricData.METRIC_NAME]
+        elif kind == mm_schemas.WriterEventKind.RESULT:
+            table = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
+            index_cols = index_cols_base + [mm_schemas.ResultData.RESULT_NAME]
+            del event[mm_schemas.ResultData.RESULT_EXTRA_DATA]
+        else:
+            raise ValueError(f"Invalid {kind = }")
 
-        del event[mm_constants.ResultData.RESULT_EXTRA_DATA]
         try:
             self._frames_client.write(
                 backend=_TSDB_BE,
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                table=table,
                 dfs=pd.DataFrame.from_records([event]),
-                index_cols=[
-                    mm_constants.WriterEvent.END_INFER_TIME,
-                    mm_constants.WriterEvent.ENDPOINT_ID,
-                    mm_constants.WriterEvent.APPLICATION_NAME,
-                    mm_constants.ResultData.RESULT_NAME,
-                ],
-            )
-            logger.info(
-                "Updated V3IO TSDB successfully",
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                index_cols=index_cols,
             )
+            logger.info("Updated V3IO TSDB successfully", table=table)
         except v3io_frames.errors.Error as err:
-            logger.warn(
+            logger.exception(
                 "Could not write drift measures to TSDB",
                 err=err,
-                table=self.tables[mm_constants.MonitoringTSDBTables.APP_RESULTS],
+                table=table,
                 event=event,
             )
-
             raise mlrun.errors.MLRunRuntimeError(
                 f"Failed to write application result to TSDB: {err}"
             )
 
-    def delete_tsdb_resources(self, table: str = None):
+    def delete_tsdb_resources(self, table: typing.Optional[str] = None):
         if table:
             # Delete a specific table
             tables = [table]
         else:
             # Delete all tables
-            tables = mm_constants.MonitoringTSDBTables.list()
-        for table in tables:
+            tables = mm_schemas.V3IOTSDBTables.list()
+        for table_to_delete in tables:
             try:
-                self._frames_client.delete(
-                    backend=mlrun.common.schemas.model_monitoring.TimeSeriesConnector.TSDB,
-                    table=table,
-                )
+                self._frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
             except v3io_frames.errors.DeleteError as e:
                 logger.warning(
                     f"Failed to delete TSDB table '{table}'",
@@ -317,11 +314,7 @@ class V3IOTSDBConnector(TSDBConnector):
         store.rm(tsdb_path, recursive=True)
 
     def get_model_endpoint_real_time_metrics(
-        self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
+        self, endpoint_id: str, metrics: list[str], start: str, end: str
     ) -> dict[str, list[tuple[str, float]]]:
         """
         Getting real time metrics from the TSDB. There are pre-defined metrics for model endpoints such as
@@ -349,8 +342,8 @@ class V3IOTSDBConnector(TSDBConnector):
         metrics_mapping = {}
 
         try:
-            data = self.get_records(
-                table=mm_constants.MonitoringTSDBTables.EVENTS,
+            data = self._get_records(
+                table=mm_schemas.V3IOTSDBTables.EVENTS,
                 columns=["endpoint_id", *metrics],
                 filter_query=f"endpoint_id=='{endpoint_id}'",
                 start=start,
@@ -374,45 +367,74 @@ class V3IOTSDBConnector(TSDBConnector):
 
         return metrics_mapping
 
-    def get_records(
+    def _get_records(
         self,
         table: str,
-        columns: list[str] = None,
+        start: Union[datetime, str],
+        end: Union[datetime, str],
+        columns: typing.Optional[list[str]] = None,
         filter_query: str = "",
-        start: str = "now-1h",
-        end: str = "now",
+        interval: typing.Optional[str] = None,
+        agg_funcs: typing.Optional[list] = None,
+        limit: typing.Optional[int] = None,
+        sliding_window_step: typing.Optional[str] = None,
+        **kwargs,
     ) -> pd.DataFrame:
         """
          Getting records from V3IO TSDB data collection.
-        :param table:            Path to the collection to query.
-        :param columns:          Columns to include in the result.
-        :param filter_query:     V3IO filter expression. The expected filter expression includes different conditions,
-                                 divided by ' AND '.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
-                                 `'s'` = seconds), or 0 for the earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
-                                 `'s'` = seconds), or 0 for the earliest time.
+        :param table:                 Path to the collection to query.
+        :param start:                 The start time of the metrics. Can be represented by a string containing an RFC
+                                      3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                      `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
+                                      `'s'` = seconds), or 0 for the earliest time.
+        :param end:                   The end time of the metrics. Can be represented by a string containing an RFC
+                                      3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                      `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, `'d'` = days, and
+                                      `'s'` = seconds), or 0 for the earliest time.
+        :param columns:               Columns to include in the result.
+        :param filter_query:          V3IO filter expression. The expected filter expression includes different
+                                      conditions, divided by ' AND '.
+        :param interval:              The interval to aggregate the data by. Note that if interval is provided,
+                                      agg_funcs must bg provided as well. Provided as a string in the format of '1m',
+                                      '1h', etc.
+        :param agg_funcs:             The aggregation functions to apply on the columns. Note that if `agg_funcs` is
+                                      provided, `interval` must bg provided as well. Provided as a list of strings in
+                                      the format of ['sum', 'avg', 'count', ...].
+        :param limit:                 The maximum number of records to return.
+        :param sliding_window_step:   The time step for which the time window moves forward. Note that if
+                                      `sliding_window_step` is provided, interval must be provided as well. Provided
+                                      as a string in the format of '1m', '1h', etc.
+        :param kwargs:                Additional keyword arguments passed to the read method of frames client.
         :return: DataFrame with the provided attributes from the data collection.
         :raise:  MLRunNotFoundError if the provided table wasn't found.
         """
         if table not in self.tables:
             raise mlrun.errors.MLRunNotFoundError(
-                f"Table '{table}' does not exist in the tables list of the TSDB connector."
+                f"Table '{table}' does not exist in the tables list of the TSDB connector. "
                 f"Available tables: {list(self.tables.keys())}"
             )
-        return self._frames_client.read(
-            backend=mlrun.common.schemas.model_monitoring.TimeSeriesConnector.TSDB,
-            table=self.tables[table],
-            columns=columns,
-            filter=filter_query,
+
+        if agg_funcs:
+            # Frames client expects the aggregators to be a comma-separated string
+            agg_funcs = ",".join(agg_funcs)
+        table_path = self.tables[table]
+        df = self._frames_client.read(
+            backend=_TSDB_BE,
+            table=table_path,
             start=start,
             end=end,
+            columns=columns,
+            filter=filter_query,
+            aggregation_window=interval,
+            aggregators=agg_funcs,
+            step=sliding_window_step,
+            **kwargs,
         )
 
+        if limit:
+            df = df.head(limit)
+        return df
+
     def _get_v3io_source_directory(self) -> str:
         """
         Get the V3IO source directory for the current project. Usually the source directory will
@@ -422,7 +444,7 @@ class V3IOTSDBConnector(TSDBConnector):
         """
         events_table_full_path = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=self.project,
-            kind=mm_constants.FileTargetKind.EVENTS,
+            kind=mm_schemas.FileTargetKind.EVENTS,
         )
 
         # Generate the main directory with the V3IO resources
@@ -440,3 +462,169 @@ class V3IOTSDBConnector(TSDBConnector):
             address=mlrun.mlconf.v3io_framesd,
             container=v3io_container,
         )
+
+    def read_metrics_data(
+        self,
+        *,
+        endpoint_id: str,
+        start: datetime,
+        end: datetime,
+        metrics: list[mm_schemas.ModelEndpointMonitoringMetric],
+        type: Literal["metrics", "results"] = "results",
+    ) -> Union[
+        list[
+            Union[
+                mm_schemas.ModelEndpointMonitoringResultValues,
+                mm_schemas.ModelEndpointMonitoringMetricNoData,
+            ],
+        ],
+        list[
+            Union[
+                mm_schemas.ModelEndpointMonitoringMetricValues,
+                mm_schemas.ModelEndpointMonitoringMetricNoData,
+            ],
+        ],
+    ]:
+        """
+        Read metrics OR results from the TSDB and return as a list.
+        Note: the type must match the actual metrics in the `metrics` parameter.
+        If the type is "results", pass only results in the `metrics` parameter.
+        """
+
+        if type == "metrics":
+            table_path = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
+            name = mm_schemas.MetricData.METRIC_NAME
+            df_handler = self.df_to_metrics_values
+        elif type == "results":
+            table_path = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
+            name = mm_schemas.ResultData.RESULT_NAME
+            df_handler = self.df_to_results_values
+        else:
+            raise ValueError(f"Invalid {type = }")
+
+        query = self._get_sql_query(
+            endpoint_id,
+            [(metric.app, metric.name) for metric in metrics],
+            table_path=table_path,
+            name=name,
+        )
+
+        logger.debug("Querying V3IO TSDB", query=query)
+
+        df: pd.DataFrame = self._frames_client.read(
+            backend=_TSDB_BE,
+            start=start,
+            end=end,
+            query=query,  # the filter argument does not work for this complex condition
+        )
+
+        logger.debug(
+            "Converting a DataFrame to a list of metrics or results values",
+            table=table_path,
+            project=self.project,
+            endpoint_id=endpoint_id,
+            is_empty=df.empty,
+        )
+
+        return df_handler(df=df, metrics=metrics, project=self.project)
+
+    @staticmethod
+    def _get_sql_query(
+        endpoint_id: str,
+        names: list[tuple[str, str]],
+        table_path: str,
+        name: str = mm_schemas.ResultData.RESULT_NAME,
+    ) -> str:
+        """Get the SQL query for the results/metrics table"""
+        with StringIO() as query:
+            query.write(
+                f"SELECT * FROM '{table_path}' "
+                f"WHERE {mm_schemas.WriterEvent.ENDPOINT_ID}='{endpoint_id}'"
+            )
+            if names:
+                query.write(" AND (")
+
+                for i, (app_name, result_name) in enumerate(names):
+                    sub_cond = (
+                        f"({mm_schemas.WriterEvent.APPLICATION_NAME}='{app_name}' "
+                        f"AND {name}='{result_name}')"
+                    )
+                    if i != 0:  # not first sub condition
+                        query.write(" OR ")
+                    query.write(sub_cond)
+
+                query.write(")")
+
+            query.write(";")
+            return query.getvalue()
+
+    def read_predictions(
+        self,
+        *,
+        endpoint_id: str,
+        start: Union[datetime, str],
+        end: Union[datetime, str],
+        aggregation_window: Optional[str] = None,
+        agg_funcs: Optional[list[str]] = None,
+        limit: Optional[int] = None,
+    ) -> Union[
+        mm_schemas.ModelEndpointMonitoringMetricNoData,
+        mm_schemas.ModelEndpointMonitoringMetricValues,
+    ]:
+        if (agg_funcs and not aggregation_window) or (
+            aggregation_window and not agg_funcs
+        ):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "both or neither of `aggregation_window` and `agg_funcs` must be provided"
+            )
+        df = self._get_records(
+            table=mm_schemas.FileTargetKind.PREDICTIONS,
+            start=start,
+            end=end,
+            columns=[mm_schemas.EventFieldType.LATENCY],
+            filter_query=f"endpoint_id=='{endpoint_id}'",
+            interval=aggregation_window,
+            agg_funcs=agg_funcs,
+            limit=limit,
+            sliding_window_step=aggregation_window,
+        )
+
+        full_name = get_invocations_fqn(self.project)
+
+        if df.empty:
+            return mm_schemas.ModelEndpointMonitoringMetricNoData(
+                full_name=full_name,
+                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
+            )
+
+        latency_column = (
+            f"{agg_funcs[0]}({mm_schemas.EventFieldType.LATENCY})"
+            if agg_funcs
+            else mm_schemas.EventFieldType.LATENCY
+        )
+
+        return mm_schemas.ModelEndpointMonitoringMetricValues(
+            full_name=full_name,
+            values=list(
+                zip(
+                    df.index,
+                    df[latency_column],
+                )
+            ),  # pyright: ignore[reportArgumentType]
+        )
+
+    def read_prediction_metric_for_endpoint_if_exists(
+        self, endpoint_id: str
+    ) -> Optional[mm_schemas.ModelEndpointMonitoringMetric]:
+        # Read just one record, because we just want to check if there is any data for this endpoint_id
+        predictions = self.read_predictions(
+            endpoint_id=endpoint_id, start="0", end="now", limit=1
+        )
+        if predictions:
+            return mm_schemas.ModelEndpointMonitoringMetric(
+                project=self.project,
+                app=mm_schemas.SpecialApps.MLRUN_INFRA,
+                type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
+                name=mm_schemas.PredictionsQueryConstants.INVOCATIONS,
+                full_name=get_invocations_fqn(self.project),
+            )

mlrun/model_monitoring/helpers.py

@@ -24,6 +24,10 @@ import mlrun.common.schemas
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
 )
+from mlrun.common.schemas.model_monitoring.model_endpoints import (
+    ModelEndpointMonitoringMetricType,
+    _compose_full_name,
+)
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
@@ -111,6 +115,24 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
     )
 
 
+def get_tsdb_connection_string(
+    secret_provider: typing.Optional[typing.Callable] = None,
+) -> str:
+    """Get TSDB connection string from the project secret. If wasn't set, take it from the system
+    configurations.
+    :param secret_provider: An optional secret provider to get the connection string secret.
+    :return:                Valid TSDB connection string.
+    """
+
+    return (
+        mlrun.get_secret_or_env(
+            key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION,
+            secret_provider=secret_provider,
+        )
+        or mlrun.mlconf.model_endpoint_monitoring.tsdb_connection
+    )
+
+
 def batch_dict2timedelta(batch_dict: _BatchDict) -> datetime.timedelta:
     """
     Convert a batch dictionary to timedelta.
@@ -260,3 +282,26 @@ def get_endpoint_record(project: str, endpoint_id: str):
         project=project,
     )
     return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
+
+
+def get_result_instance_fqn(
+    model_endpoint_id: str, app_name: str, result_name: str
+) -> str:
+    return f"{model_endpoint_id}.{app_name}.result.{result_name}"
+
+
+def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
+    return get_result_instance_fqn(
+        model_endpoint_id,
+        mm_constants.HistogramDataDriftApplicationConstants.NAME,
+        mm_constants.HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+    )
+
+
+def get_invocations_fqn(project: str) -> str:
+    return _compose_full_name(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+    )