mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from datetime import datetime, timezone
+import math
+from datetime import datetime, timedelta
 from io import StringIO
-from typing import Literal, Optional, Union
+from typing import Callable, Literal, Optional, Union
 
 import pandas as pd
 import v3io_frames
@@ -33,6 +33,13 @@ _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
+V3IO_FRAMESD_MEPS_LIMIT = (
+    200  # Maximum number of model endpoints per single request when using V3IO Frames
+)
+V3IO_CLIENT_MEPS_LIMIT = (
+    150  # Maximum number of model endpoints per single request when using V3IO Client
+)
+
 
 def _is_no_schema_error(exc: v3io_frames.Error) -> bool:
     """
@@ -58,6 +65,7 @@ class V3IOTSDBConnector(TSDBConnector):
         project: str,
         container: str = _CONTAINER,
         v3io_framesd: Optional[str] = None,
+        v3io_access_key: str = "",
         create_table: bool = False,
     ) -> None:
         super().__init__(project=project)
@@ -65,14 +73,26 @@ class V3IOTSDBConnector(TSDBConnector):
         self.container = container
 
         self.v3io_framesd = v3io_framesd or mlrun.mlconf.v3io_framesd
+        self._v3io_access_key = v3io_access_key
         self._frames_client: Optional[v3io_frames.client.ClientBase] = None
         self._init_tables_path()
         self._create_table = create_table
+        self._v3io_client = None
+
+    @property
+    def v3io_client(self):
+        if not self._v3io_client:
+            self._v3io_client = mlrun.utils.v3io_clients.get_v3io_client(
+                endpoint=mlrun.mlconf.v3io_api, access_key=self._v3io_access_key
+            )
+        return self._v3io_client
 
     @property
     def frames_client(self) -> v3io_frames.client.ClientBase:
         if not self._frames_client:
-            self._frames_client = self._get_v3io_frames_client(self.container)
+            self._frames_client = self._get_v3io_frames_client(
+                self.container, v3io_access_key=self._v3io_access_key
+            )
             if self._create_table:
                 self.create_tables()
         return self._frames_client
@@ -129,7 +149,7 @@ class V3IOTSDBConnector(TSDBConnector):
         monitoring_predictions_full_path = (
             mlrun.mlconf.get_model_monitoring_file_target_path(
                 project=self.project,
-                kind=mm_schemas.FileTargetKind.PREDICTIONS,
+                kind=mm_schemas.V3IOTSDBTables.PREDICTIONS,
             )
         )
         (
@@ -139,28 +159,51 @@ class V3IOTSDBConnector(TSDBConnector):
         ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
             monitoring_predictions_full_path
         )
-        self.tables[mm_schemas.FileTargetKind.PREDICTIONS] = monitoring_predictions_path
+        self.tables[mm_schemas.V3IOTSDBTables.PREDICTIONS] = monitoring_predictions_path
+
+        # initialize kv table
+        last_request_full_table_path = (
+            mlrun.mlconf.get_model_monitoring_file_target_path(
+                project=self.project,
+                kind=mm_schemas.FileTargetKind.LAST_REQUEST,
+            )
+        )
+        (
+            _,
+            _,
+            self.last_request_table,
+        ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
+            last_request_full_table_path
+        )
 
     def create_tables(self) -> None:
         """
-        Create the tables using the TSDB connector. The tables are being created in the V3IO TSDB and include:
+        Create the tables using the TSDB connector. These are the tables that are stored in the V3IO TSDB:
         - app_results: a detailed result that includes status, kind, extra data, etc.
         - metrics: a basic key value that represents a single numeric metric.
-        Note that the predictions table is automatically created by the model monitoring stream pod.
+        - events: A statistics table that includes pre-aggregated metrics (such as average latency over the
+        last 5 minutes) and data samples
+        - predictions: a detailed prediction that includes latency, request timestamp, etc. This table also
+        includes pre-aggregated operations such as count and average on 1 minute granularity.
+        - errors: a detailed error that includes error desc, error type, etc.
+
         """
-        application_tables = [
-            mm_schemas.V3IOTSDBTables.APP_RESULTS,
-            mm_schemas.V3IOTSDBTables.METRICS,
-        ]
-        for table_name in application_tables:
+
+        default_configurations = {
+            "backend": _TSDB_BE,
+            "if_exists": v3io_frames.IGNORE,
+            "rate": _TSDB_RATE,
+        }
+
+        for table_name in self.tables:
+            default_configurations["table"] = self.tables[table_name]
+            if table_name == mm_schemas.V3IOTSDBTables.PREDICTIONS:
+                default_configurations["aggregates"] = "count,avg"
+                default_configurations["aggregation_granularity"] = "1m"
+            elif table_name == mm_schemas.V3IOTSDBTables.EVENTS:
+                default_configurations["rate"] = "10/m"
             logger.info("Creating table in V3IO TSDB", table_name=table_name)
-            table = self.tables[table_name]
-            self.frames_client.create(
-                backend=_TSDB_BE,
-                table=table,
-                if_exists=v3io_frames.IGNORE,
-                rate=_TSDB_RATE,
-            )
+            self.frames_client.create(**default_configurations)
 
     def apply_monitoring_stream_steps(
         self,
@@ -168,6 +211,9 @@ class V3IOTSDBConnector(TSDBConnector):
         tsdb_batching_max_events: int = 1000,
         tsdb_batching_timeout_secs: int = 30,
         sample_window: int = 10,
+        aggregate_windows: Optional[list[str]] = None,
+        aggregate_period: str = "1m",
+        **kwarg,
     ):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -178,31 +224,73 @@ class V3IOTSDBConnector(TSDBConnector):
         - endpoint_features (Prediction and feature names and values)
         - custom_metrics (user-defined metrics)
         """
+        aggregate_windows = aggregate_windows or ["5m", "1h"]
+
+        # Calculate number of predictions and average latency
+        def apply_storey_aggregations():
+            # Calculate number of predictions for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.AggregateByKey",
+                aggregates=[
+                    {
+                        "name": EventFieldType.LATENCY,
+                        "column": EventFieldType.LATENCY,
+                        "operations": ["count", "avg"],
+                        "windows": aggregate_windows,
+                        "period": aggregate_period,
+                    }
+                ],
+                name=EventFieldType.LATENCY,
+                after="FilterNOP",
+                step_name="Aggregates",
+                table=".",
+                key_field=EventFieldType.ENDPOINT_ID,
+            )
+            # Calculate average latency time for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.Rename",
+                mapping={
+                    "latency_count_5m": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_5M,
+                    "latency_count_1h": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_1H,
+                },
+                name="Rename",
+                after=EventFieldType.LATENCY,
+            )
 
+        apply_storey_aggregations()
         # Write latency per prediction, labeled by endpoint ID only
         graph.add_step(
             "storey.TSDBTarget",
             name="tsdb_predictions",
-            after="MapFeatureNames",
-            path=f"{self.container}/{self.tables[mm_schemas.FileTargetKind.PREDICTIONS]}",
-            rate="1/s",
+            after="FilterNOP",
+            path=f"{self.container}/{self.tables[mm_schemas.V3IOTSDBTables.PREDICTIONS]}",
             time_col=mm_schemas.EventFieldType.TIMESTAMP,
             container=self.container,
             v3io_frames=self.v3io_framesd,
             columns=[
                 mm_schemas.EventFieldType.LATENCY,
                 mm_schemas.EventFieldType.LAST_REQUEST_TIMESTAMP,
+                mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT,
+                mm_schemas.EventFieldType.EFFECTIVE_SAMPLE_COUNT,
             ],
             index_cols=[
                 mm_schemas.EventFieldType.ENDPOINT_ID,
             ],
-            aggr="count,avg",
-            aggr_granularity="1m",
             max_events=tsdb_batching_max_events,
             flush_after_seconds=tsdb_batching_timeout_secs,
             key=mm_schemas.EventFieldType.ENDPOINT_ID,
         )
 
+        # Write last request timestamp to KV table
+        graph.add_step(
+            "storey.NoSqlTarget",
+            name="KVLastRequest",
+            after="tsdb_predictions",
+            table=f"v3io:///users/{self.last_request_table}",
+            columns=[EventFieldType.LAST_REQUEST_TIMESTAMP],
+            index_cols=[EventFieldType.ENDPOINT_ID],
+        )
+
         # Emits the event in window size of events based on sample_window size (10 by default)
         graph.add_step(
             "storey.steps.SampleWindow",
@@ -236,7 +324,6 @@ class V3IOTSDBConnector(TSDBConnector):
                 name=name,
                 after=after,
                 path=f"{self.container}/{self.tables[mm_schemas.V3IOTSDBTables.EVENTS]}",
-                rate="10/m",
                 time_col=mm_schemas.EventFieldType.TIMESTAMP,
                 container=self.container,
                 v3io_frames=self.v3io_framesd,
@@ -300,7 +387,6 @@ class V3IOTSDBConnector(TSDBConnector):
             name="tsdb_error",
             after="error_extractor",
             path=f"{self.container}/{self.tables[mm_schemas.FileTargetKind.ERRORS]}",
-            rate="1/s",
             time_col=mm_schemas.EventFieldType.TIMESTAMP,
             container=self.container,
             v3io_frames=self.v3io_framesd,
@@ -310,6 +396,7 @@ class V3IOTSDBConnector(TSDBConnector):
             ],
             index_cols=[
                 mm_schemas.EventFieldType.ENDPOINT_ID,
+                mm_schemas.EventFieldType.ERROR_TYPE,
             ],
             max_events=tsdb_batching_max_events,
             flush_after_seconds=tsdb_batching_timeout_secs,
@@ -338,9 +425,6 @@ class V3IOTSDBConnector(TSDBConnector):
         elif kind == mm_schemas.WriterEventKind.RESULT:
             table = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
             index_cols = index_cols_base + [mm_schemas.ResultData.RESULT_NAME]
-            event.pop(mm_schemas.ResultData.CURRENT_STATS, None)
-            # TODO: remove this when extra data is supported (ML-7460)
-            event.pop(mm_schemas.ResultData.RESULT_EXTRA_DATA, None)
         else:
             raise ValueError(f"Invalid {kind = }")
 
@@ -371,12 +455,20 @@ class V3IOTSDBConnector(TSDBConnector):
             # Delete all tables
             tables = mm_schemas.V3IOTSDBTables.list()
         for table_to_delete in tables:
-            try:
-                self.frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
-            except v3io_frames.DeleteError as e:
+            if table_to_delete in self.tables:
+                try:
+                    self.frames_client.delete(
+                        backend=_TSDB_BE, table=self.tables[table_to_delete]
+                    )
+                except v3io_frames.DeleteError as e:
+                    logger.warning(
+                        f"Failed to delete TSDB table '{table_to_delete}'",
+                        err=mlrun.errors.err_to_str(e),
+                    )
+            else:
                 logger.warning(
-                    f"Failed to delete TSDB table '{table}'",
-                    err=mlrun.errors.err_to_str(e),
+                    f"Skipping deletion: table '{table_to_delete}' is not among the initialized tables.",
+                    initialized_tables=list(self.tables.keys()),
                 )
 
         # Final cleanup of tsdb path
@@ -385,6 +477,60 @@ class V3IOTSDBConnector(TSDBConnector):
         store, _, _ = mlrun.store_manager.get_or_create_store(tsdb_path)
         store.rm(tsdb_path, recursive=True)
 
+    def delete_tsdb_records(
+        self,
+        endpoint_ids: list[str],
+    ):
+        logger.debug(
+            "Deleting model endpoints resources using the V3IO TSDB connector",
+            project=self.project,
+            number_of_endpoints_to_delete=len(endpoint_ids),
+        )
+        tables = mm_schemas.V3IOTSDBTables.list()
+
+        # Split the endpoint ids into chunks to avoid exceeding the v3io-engine filter-expression limit
+        for i in range(0, len(endpoint_ids), V3IO_FRAMESD_MEPS_LIMIT):
+            endpoint_id_chunk = endpoint_ids[i : i + V3IO_FRAMESD_MEPS_LIMIT]
+            filter_query = f"endpoint_id IN({str(endpoint_id_chunk)[1:-1]}) "
+            for table in tables:
+                try:
+                    self.frames_client.delete(
+                        backend=_TSDB_BE,
+                        table=self.tables[table],
+                        filter=filter_query,
+                        start="0",
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to delete TSDB records for the provided endpoints from table '{table}'",
+                        error=mlrun.errors.err_to_str(e),
+                        project=self.project,
+                    )
+
+        # Clean the last request records from the KV table
+        self._delete_last_request_records(endpoint_ids=endpoint_ids)
+
+        logger.debug(
+            "Deleted all model endpoint resources using the V3IO connector",
+            project=self.project,
+            number_of_endpoints_to_delete=len(endpoint_ids),
+        )
+
+    def _delete_last_request_records(self, endpoint_ids: list[str]):
+        for endpoint_id in endpoint_ids:
+            try:
+                self.v3io_client.kv.delete(
+                    container=self.container,
+                    table=self.last_request_table,
+                    key=endpoint_id,
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to delete last request record for endpoint '{endpoint_id}'",
+                    error=mlrun.errors.err_to_str(e),
+                    project=self.project,
+                )
+
     def get_model_endpoint_real_time_metrics(
         self, endpoint_id: str, metrics: list[str], start: str, end: str
     ) -> dict[str, list[tuple[str, float]]]:
@@ -449,8 +595,9 @@ class V3IOTSDBConnector(TSDBConnector):
         interval: Optional[str] = None,
         agg_funcs: Optional[list[str]] = None,
         sliding_window_step: Optional[str] = None,
+        get_raw: bool = False,
         **kwargs,
-    ) -> pd.DataFrame:
+    ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
         """
          Getting records from V3IO TSDB data collection.
         :param table:                 Path to the collection to query.
@@ -475,6 +622,10 @@ class V3IOTSDBConnector(TSDBConnector):
                                       `sliding_window_step` is provided, interval must be provided as well. Provided
                                       as a string in the format of '1m', '1h', etc.
         :param kwargs:                Additional keyword arguments passed to the read method of frames client.
+        :param get_raw:               Whether to return the request as raw frames rather than a pandas dataframe.
+                                      Defaults to False. This can greatly improve performance when a dataframe isn't
+                                      needed.
+
         :return: DataFrame with the provided attributes from the data collection.
         :raise:  MLRunNotFoundError if the provided table wasn't found.
         """
@@ -488,7 +639,7 @@ class V3IOTSDBConnector(TSDBConnector):
         aggregators = ",".join(agg_funcs) if agg_funcs else None
         table_path = self.tables[table]
         try:
-            df = self.frames_client.read(
+            res = self.frames_client.read(
                 backend=_TSDB_BE,
                 table=table_path,
                 start=start,
@@ -498,15 +649,18 @@ class V3IOTSDBConnector(TSDBConnector):
                 aggregation_window=interval,
                 aggregators=aggregators,
                 step=sliding_window_step,
+                get_raw=get_raw,
                 **kwargs,
             )
+            if get_raw:
+                res = list(res)
         except v3io_frames.Error as err:
             if _is_no_schema_error(err):
-                return pd.DataFrame()
+                return [] if get_raw else pd.DataFrame()
             else:
                 raise err
 
-        return df
+        return res
 
     def _get_v3io_source_directory(self) -> str:
         """
@@ -530,12 +684,34 @@ class V3IOTSDBConnector(TSDBConnector):
         return source_directory
 
     @staticmethod
-    def _get_v3io_frames_client(v3io_container: str) -> v3io_frames.client.ClientBase:
+    def _get_v3io_frames_client(
+        v3io_container: str, v3io_access_key: str = ""
+    ) -> v3io_frames.client.ClientBase:
         return mlrun.utils.v3io_clients.get_frames_client(
             address=mlrun.mlconf.v3io_framesd,
             container=v3io_container,
+            token=v3io_access_key,
         )
 
+    @staticmethod
+    def _get_endpoint_filter(endpoint_id: Union[str, list[str]]) -> Optional[str]:
+        if isinstance(endpoint_id, str):
+            return f"endpoint_id=='{endpoint_id}'"
+        elif isinstance(endpoint_id, list):
+            if len(endpoint_id) > V3IO_FRAMESD_MEPS_LIMIT:
+                logger.info(
+                    "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
+                    "retrieving all the model endpoints from the db.",
+                    limit=V3IO_FRAMESD_MEPS_LIMIT,
+                    amount=len(endpoint_id),
+                )
+                return None
+            return f"endpoint_id IN({str(endpoint_id)[1:-1]}) "
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Invalid 'endpoint_id' filter: must be a string or a list, endpoint_id: {endpoint_id}"
+            )
+
     def read_metrics_data(
         self,
         *,
@@ -544,6 +720,7 @@ class V3IOTSDBConnector(TSDBConnector):
         end: datetime,
         metrics: list[mm_schemas.ModelEndpointMonitoringMetric],
         type: Literal["metrics", "results"] = "results",
+        with_result_extra_data: bool = False,
     ) -> Union[
         list[
             Union[
@@ -565,6 +742,12 @@ class V3IOTSDBConnector(TSDBConnector):
         """
 
         if type == "metrics":
+            if with_result_extra_data:
+                logger.warning(
+                    "The 'with_result_extra_data' parameter is not supported for metrics, just for results",
+                    project=self.project,
+                    endpoint_id=endpoint_id,
+                )
             table_path = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
             name = mm_schemas.MetricData.METRIC_NAME
             columns = [mm_schemas.MetricData.METRIC_VALUE]
@@ -577,6 +760,8 @@ class V3IOTSDBConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_STATUS,
                 mm_schemas.ResultData.RESULT_KIND,
             ]
+            if with_result_extra_data:
+                columns.append(mm_schemas.ResultData.RESULT_EXTRA_DATA)
             df_handler = self.df_to_results_values
         else:
             raise ValueError(f"Invalid {type = }")
@@ -605,6 +790,9 @@ class V3IOTSDBConnector(TSDBConnector):
             endpoint_id=endpoint_id,
             is_empty=df.empty,
         )
+        if not with_result_extra_data and type == "results":
+            # Set the extra data to an empty string if it's not requested
+            df[mm_schemas.ResultData.RESULT_EXTRA_DATA] = ""
 
         return df_handler(df=df, metrics=metrics, project=self.project)
 
@@ -653,6 +841,9 @@ class V3IOTSDBConnector(TSDBConnector):
         end: Union[datetime, str],
         aggregation_window: Optional[str] = None,
         agg_funcs: Optional[list[str]] = None,
+        limit: Optional[
+            int
+        ] = None,  # no effect, just for compatibility with the abstract method
     ) -> Union[
         mm_schemas.ModelEndpointMonitoringMetricNoData,
         mm_schemas.ModelEndpointMonitoringMetricValues,
@@ -664,10 +855,10 @@ class V3IOTSDBConnector(TSDBConnector):
                 "both or neither of `aggregation_window` and `agg_funcs` must be provided"
             )
         df = self._get_records(
-            table=mm_schemas.FileTargetKind.PREDICTIONS,
+            table=mm_schemas.V3IOTSDBTables.PREDICTIONS,
             start=start,
             end=end,
-            columns=[mm_schemas.EventFieldType.LATENCY],
+            columns=[mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT],
             filter_query=f"endpoint_id=='{endpoint_id}'",
             agg_funcs=agg_funcs,
             sliding_window_step=aggregation_window,
@@ -681,10 +872,10 @@ class V3IOTSDBConnector(TSDBConnector):
                 type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
             )
 
-        latency_column = (
-            f"{agg_funcs[0]}({mm_schemas.EventFieldType.LATENCY})"
+        estimated_prediction_count = (
+            f"{agg_funcs[0]}({mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT})"
             if agg_funcs
-            else mm_schemas.EventFieldType.LATENCY
+            else mm_schemas.EventFieldType.ESTIMATED_PREDICTION_COUNT
         )
 
         return mm_schemas.ModelEndpointMonitoringMetricValues(
@@ -692,7 +883,7 @@ class V3IOTSDBConnector(TSDBConnector):
             values=list(
                 zip(
                     df.index,
-                    df[latency_column],
+                    df[estimated_prediction_count],
                 )
             ),  # pyright: ignore[reportArgumentType]
         )
@@ -700,55 +891,78 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_last_request(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
-    ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
-        df = self._get_records(
-            table=mm_schemas.FileTargetKind.PREDICTIONS,
-            start=start,
-            end=end,
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
-            agg_funcs=["last"],
-        )
-        if not df.empty:
-            df.rename(
-                columns={
-                    f"last({mm_schemas.EventFieldType.LAST_REQUEST_TIMESTAMP})": mm_schemas.EventFieldType.LAST_REQUEST,
-                    f"last({mm_schemas.EventFieldType.LATENCY})": f"last_{mm_schemas.EventFieldType.LATENCY}",
-                },
-                inplace=True,
-            )
-            df[mm_schemas.EventFieldType.LAST_REQUEST] = df[
-                mm_schemas.EventFieldType.LAST_REQUEST
-            ].map(
-                lambda last_request: datetime.fromtimestamp(
-                    last_request, tz=timezone.utc
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+    ) -> dict[str, float]:
+        # Get the last request timestamp for each endpoint from the KV table.
+        # The result of the query is a list of dictionaries,
+        # each dictionary contains the endpoint id and the last request timestamp.
+        last_request_timestamps = {}
+        if isinstance(endpoint_ids, str):
+            endpoint_ids = [endpoint_ids]
+
+        try:
+            if len(endpoint_ids) > V3IO_CLIENT_MEPS_LIMIT:
+                logger.warning(
+                    "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
+                    "retrieving last request for all the model endpoints from the KV table.",
+                    limit=V3IO_CLIENT_MEPS_LIMIT,
+                    amount=len(endpoint_ids),
+                )
+
+                res = self.v3io_client.kv.new_cursor(
+                    container=self.container,
+                    table_path=self.last_request_table,
+                ).all()
+                last_request_timestamps.update(
+                    {d["__name"]: d["last_request_timestamp"] for d in res}
                 )
+            else:
+                filter_expression = " OR ".join(
+                    [f"__name=='{endpoint_id}'" for endpoint_id in endpoint_ids]
+                )
+                res = self.v3io_client.kv.new_cursor(
+                    container=self.container,
+                    table_path=self.last_request_table,
+                    filter_expression=filter_expression,
+                ).all()
+                last_request_timestamps.update(
+                    {d["__name"]: d["last_request_timestamp"] for d in res}
+                )
+        except Exception as e:
+            logger.warning(
+                "Failed to get last request timestamp from V3IO KV table.",
+                err=mlrun.errors.err_to_str(e),
+                project=self.project,
+                table=self.last_request_table,
             )
 
-        return df.reset_index(drop=True)
+        return last_request_timestamps
 
     def get_drift_status(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "now-24h",
-        end: Union[datetime, str] = "now",
-    ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
-        df = self._get_records(
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        get_raw: bool = False,
+    ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
+        start, end = self._get_start_end(start, end)
+        res = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
             end=end,
             columns=[mm_schemas.ResultData.RESULT_STATUS],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["max"],
             group_by="endpoint_id",
+            get_raw=get_raw,
         )
+        if get_raw:
+            return res
+
+        df = res
         if not df.empty:
             df.columns = [
                 col[len("max(") : -1] if "max(" in col else col for col in df.columns
@@ -757,16 +971,18 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_metrics_metadata(
         self,
-        endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        endpoint_id: Union[str, list[str]],
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
+        start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.METRICS,
             start=start,
             end=end,
             columns=[mm_schemas.MetricData.METRIC_VALUE],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:
@@ -777,10 +993,12 @@ class V3IOTSDBConnector(TSDBConnector):
 
     def get_results_metadata(
         self,
-        endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        endpoint_id: Union[str, list[str]],
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
+        start, end = self._get_start_end(start, end)
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_id)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
@@ -788,7 +1006,7 @@ class V3IOTSDBConnector(TSDBConnector):
             columns=[
                 mm_schemas.ResultData.RESULT_KIND,
             ],
-            filter_query=f"endpoint_id=='{endpoint_id}'",
+            filter_query=filter_query,
             agg_funcs=["last"],
         )
         if not df.empty:
@@ -803,20 +1021,30 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_error_count(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
-    ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
-        df = self._get_records(
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        get_raw: bool = False,
+    ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
+        if filter_query:
+            filter_query += f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'"
+        else:
+            filter_query = f"{mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}' z"
+        start, end = self._get_start_end(start, end)
+        res = self._get_records(
             table=mm_schemas.FileTargetKind.ERRORS,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.ERROR_COUNT],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["count"],
+            get_raw=get_raw,
         )
+
+        if get_raw:
+            return res
+
+        df = res
         if not df.empty:
             df.rename(
                 columns={
@@ -830,20 +1058,122 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_avg_latency(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
-    ) -> pd.DataFrame:
-        endpoint_ids = (
-            endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
-        )
-        df = self._get_records(
-            table=mm_schemas.FileTargetKind.PREDICTIONS,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        get_raw: bool = False,
+    ) -> Union[pd.DataFrame, list[v3io_frames.client.RawFrame]]:
+        filter_query = self._get_endpoint_filter(endpoint_id=endpoint_ids)
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
+        start, end = self._get_start_end(start, end)
+        res = self._get_records(
+            table=mm_schemas.V3IOTSDBTables.PREDICTIONS,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=filter_query,
             agg_funcs=["avg"],
+            get_raw=get_raw,
         )
+
+        if get_raw:
+            return res
+
+        df = res
         if not df.empty:
             df.dropna(inplace=True)
+            df.rename(
+                columns={
+                    f"avg({mm_schemas.EventFieldType.LATENCY})": f"avg_{mm_schemas.EventFieldType.LATENCY}"
+                },
+                inplace=True,
+            )
         return df.reset_index(drop=True)
+
+    async def add_basic_metrics(
+        self,
+        model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
+        project: str,
+        run_in_threadpool: Callable,
+        metric_list: Optional[list[str]] = None,
+    ) -> list[mlrun.common.schemas.ModelEndpoint]:
+        """
+        Fetch basic metrics from V3IO TSDB and add them to MEP objects.
+
+        :param model_endpoint_objects: A list of `ModelEndpoint` objects that will
+                                       be filled with the relevant basic metrics.
+        :param project:                The name of the project.
+        :param run_in_threadpool:      A function that runs another function in a thread pool.
+        :param metric_list:            List of metrics to include from the time series DB. Defaults to all metrics.
+
+        :return: A list of `ModelEndpointMonitoringMetric` objects.
+        """
+
+        uids = []
+        model_endpoint_objects_by_uid = {}
+        for model_endpoint_object in model_endpoint_objects:
+            uid = model_endpoint_object.metadata.uid
+            uids.append(uid)
+            model_endpoint_objects_by_uid[uid] = model_endpoint_object
+
+        metric_name_to_function_and_column_name = {
+            "error_count": (self.get_error_count, "count(error_count)"),
+            "avg_latency": (self.get_avg_latency, "avg(latency)"),
+            "result_status": (self.get_drift_status, "max(result_status)"),
+        }
+        if metric_list is not None:
+            for metric_name in list(metric_name_to_function_and_column_name):
+                if metric_name not in metric_list:
+                    del metric_name_to_function_and_column_name[metric_name]
+
+        metric_name_to_result = {}
+
+        for metric_name, (
+            function,
+            _,
+        ) in metric_name_to_function_and_column_name.items():
+            metric_name_to_result[metric_name] = await run_in_threadpool(
+                function,
+                endpoint_ids=uids,
+                get_raw=True,
+            )
+
+        def add_metric(
+            metric: str,
+            column_name: str,
+            frames: list,
+        ):
+            for frame in frames:
+                endpoint_ids = frame.column_data("endpoint_id")
+                metric_data = frame.column_data(column_name)
+                for index, endpoint_id in enumerate(endpoint_ids):
+                    mep = model_endpoint_objects_by_uid.get(endpoint_id)
+                    value = metric_data[index]
+                    if mep and value is not None and not math.isnan(value):
+                        setattr(mep.status, metric, value)
+
+        for metric_name, result in metric_name_to_result.items():
+            add_metric(
+                metric_name,
+                metric_name_to_function_and_column_name[metric_name][1],
+                result,
+            )
+        if metric_list is None or "last_request" in metric_list:
+            self._enrich_mep_with_last_request(
+                model_endpoint_objects_by_uid=model_endpoint_objects_by_uid
+            )
+
+        return list(model_endpoint_objects_by_uid.values())
+
+    def _enrich_mep_with_last_request(
+        self,
+        model_endpoint_objects_by_uid: dict[str, mlrun.common.schemas.ModelEndpoint],
+    ):
+        last_request_dictionary = self.get_last_request(
+            endpoint_ids=list(model_endpoint_objects_by_uid.keys())
+        )
+        for uid, mep in model_endpoint_objects_by_uid.items():
+            # Set the last request timestamp to the MEP object. If not found, keep the existing value from the
+            # DB (relevant for batch EP).
+            mep.status.last_request = last_request_dictionary.get(
+                uid, mep.status.last_request
+            )