mlrun 1.7.0rc15__py3-none-any.whl → 1.7.0rc16__py3-none-any.whl

mlrun/model_monitoring/__init__.py

@@ -15,7 +15,7 @@
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 # for backwards compatibility
 
-from .db import get_store_object
+from .db import get_store_object, get_tsdb_connector
 from .helpers import get_stream_path
 from .model_endpoint import ModelEndpoint
 from .tracking_policy import TrackingPolicy

mlrun/model_monitoring/applications/_application_steps.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import copy
 import json
 import typing
 from typing import Optional
@@ -138,7 +137,7 @@ class _PrepareMonitoringEvent(StepToDict):
         if not event.get("mlrun_context"):
             application_context = MonitoringApplicationContext().from_dict(
                 event,
-                context=copy.deepcopy(self.context),
+                context=self.context,
                 model_endpoint_dict=self.model_endpoints,
             )
         else:

mlrun/model_monitoring/applications/context.py

@@ -113,7 +113,7 @@ class MonitoringApplicationContext(MLClientCtx):
             attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
         )
         self._sample_df_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
+            attrs.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
         )
 
         self.endpoint_id = attrs.get(mm_constants.ApplicationEvent.ENDPOINT_ID)

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import typing
+
+import json
 from dataclasses import dataclass
-from typing import Final, Optional, Protocol, cast
+from typing import Final, Optional, Protocol, Union, cast
 
 import numpy as np
 from pandas import DataFrame, Series
@@ -90,9 +91,27 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
     """
     MLRun's default data drift application for model monitoring.
 
-    The application calculates the metrics over the features' histograms.
-    Each metric is calculated over all the features, the mean is taken,
-    and the status is returned.
+    The application expects tabular numerical data, and calculates three metrics over the features' histograms.
+    The three metrics are:
+
+    * Hellinger distance.
+    * Total variance distance.
+    * Kullback-Leibler divergence.
+
+    Each metric is calculated over all the features individually and the mean is taken as the metric value.
+    The average of Hellinger and total variance distance is taken as the result.
+
+    The application logs two artifacts:
+
+    * A JSON with the general drift per feature.
+    * A plotly table different metrics per feature.
+
+    This application is deployed by default when calling:
+
+    .. code-block:: python
+
+        project.enable_model_monitoring()
+
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
@@ -107,8 +126,6 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
     def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
         """
-        Initialize the data drift application.
-
         :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
                                  If not provided, the default `DataDriftClassifier()` is used.
         """
@@ -146,35 +163,46 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
         return metrics_per_feature
 
-    def _add_general_drift_result(
+    def _get_general_drift_result(
         self,
         metrics: list[mm_results.ModelMonitoringApplicationMetric],
-    ) -> None:
-        """Add the general drift result to the results list and log it"""
-        value = np.mean(
-            [
-                metric.value
-                for metric in metrics
-                if metric.name
-                in [
-                    f"{HellingerDistance.NAME}_mean",
-                    f"{TotalVarianceDistance.NAME}_mean",
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+    ) -> mm_results.ModelMonitoringApplicationResult:
+        """Get the general drift result from the metrics list"""
+        value = cast(
+            float,
+            np.mean(
+                [
+                    metric.value
+                    for metric in metrics
+                    if metric.name
+                    in [
+                        f"{HellingerDistance.NAME}_mean",
+                        f"{TotalVarianceDistance.NAME}_mean",
+                    ]
                 ]
-            ]
+            ),
         )
 
         status = self._value_classifier.value_to_status(value)
-        metrics.append(
-            mm_results.ModelMonitoringApplicationResult(
-                name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
-                value=value,
-                kind=ResultKindApp.data_drift,
-                status=status,
-            )
+        return mm_results.ModelMonitoringApplicationResult(
+            name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+            value=value,
+            kind=ResultKindApp.data_drift,
+            status=status,
+            extra_data={
+                EventFieldType.CURRENT_STATS: json.dumps(
+                    monitoring_context.feature_stats
+                ),
+                EventFieldType.DRIFT_MEASURES: metrics_per_feature.T.to_json(),
+                EventFieldType.DRIFT_STATUS: status.value,
+            },
         )
 
+    @staticmethod
     def _get_metrics(
-        self, metrics_per_feature: DataFrame
+        metrics_per_feature: DataFrame,
     ) -> list[mm_results.ModelMonitoringApplicationMetric]:
         """Average the metrics over the features and add the status"""
         metrics: list[mm_results.ModelMonitoringApplicationMetric] = []
@@ -206,8 +234,8 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             del sample_set_statistics[EventFieldType.TIMESTAMP]
         return sample_set_statistics
 
+    @staticmethod
     def _log_json_artifact(
-        self,
         drift_per_feature_values: Series,
         monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> None:
@@ -247,7 +275,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             mm_drift_table.FeaturesDriftTablePlot().produce(
                 sample_set_statistics=sample_set_statistics,
                 inputs_statistics=inputs_statistics,
-                metrics=metrics_per_feature.T.to_dict(),
+                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
                 drift_results=drift_results,
             )
         )
@@ -281,7 +309,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         self,
         monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> list[
-        typing.Union[
+        Union[
             mm_results.ModelMonitoringApplicationResult,
             mm_results.ModelMonitoringApplicationMetric,
         ]
@@ -308,15 +336,13 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             metrics_per_feature=metrics_per_feature,
         )
         monitoring_context.logger.debug("Computing average per metric")
-        metrics_and_result: list[
-            typing.Union[
-                mm_results.ModelMonitoringApplicationMetric,
-                mm_results.ModelMonitoringApplicationResult,
-            ]
-        ] = self._get_metrics(metrics_per_feature)
-        self._add_general_drift_result(
-            metrics=metrics_and_result,
+        metrics = self._get_metrics(metrics_per_feature)
+        result = self._get_general_drift_result(
+            metrics=metrics,
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
         )
+        metrics_and_result = metrics + [result]
         monitoring_context.logger.debug(
             "Finished running the application", results=metrics_and_result
         )

mlrun/model_monitoring/db/__init__.py

@@ -14,3 +14,5 @@
 
 from .stores import ObjectStoreFactory, get_store_object
 from .stores.base import StoreBase
+from .tsdb import get_tsdb_connector
+from .tsdb.base import TSDBConnector

mlrun/model_monitoring/db/stores/base/store.py

@@ -15,6 +15,8 @@
 import typing
 from abc import ABC, abstractmethod
 
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+
 
 class StoreBase(ABC):
     """
@@ -62,12 +64,10 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def delete_model_endpoints_resources(self, endpoints: list[dict[str, typing.Any]]):
+    def delete_model_endpoints_resources(self):
         """
         Delete all model endpoints resources.
 
-        :param endpoints: A list of model endpoints flattened dictionaries.
-
         """
         pass
 
@@ -112,45 +112,18 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def get_endpoint_real_time_metrics(
+    def write_application_event(
         self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> dict[str, list[tuple[str, float]]]:
-        """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
-        """
-
-        pass
-
-    @abstractmethod
-    def write_application_result(self, event: dict[str, typing.Any]):
+        event: dict[str, typing.Any],
+        kind: mm_constants.WriterEventKind = mm_constants.WriterEventKind.RESULT,
+    ):
         """
-        Write a new application result event in the target table.
+        Write a new event in the target table.
 
         :param event: An event dictionary that represents the application result, should be corresponded to the
                       schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
                       object.
+        :param kind: The type of the event, can be either "result" or "metric".
         """
         pass
 

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -21,12 +21,11 @@ import pandas as pd
 import sqlalchemy
 
 import mlrun.common.model_monitoring.helpers
-import mlrun.common.schemas.model_monitoring
+import mlrun.common.schemas.model_monitoring as mm_constants
 import mlrun.model_monitoring.db
 import mlrun.model_monitoring.db.stores.sqldb.models
 import mlrun.model_monitoring.helpers
 from mlrun.common.db.sql_session import create_session, get_engine
-from mlrun.utils import logger
 
 
 class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
@@ -72,9 +71,9 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
                 connection_string=self._sql_connection_string
             )
         )
-        self._tables[
-            mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_ENDPOINTS
-        ] = self.ModelEndpointsTable
+        self._tables[mm_constants.EventFieldType.MODEL_ENDPOINTS] = (
+            self.ModelEndpointsTable
+        )
 
     def _init_application_results_table(self):
         self.ApplicationResultsTable = (
@@ -82,17 +81,17 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
                 connection_string=self._sql_connection_string
             )
         )
-        self._tables[
-            mlrun.common.schemas.model_monitoring.FileTargetKind.APP_RESULTS
-        ] = self.ApplicationResultsTable
+        self._tables[mm_constants.FileTargetKind.APP_RESULTS] = (
+            self.ApplicationResultsTable
+        )
 
     def _init_monitoring_schedules_table(self):
         self.MonitoringSchedulesTable = mlrun.model_monitoring.db.stores.sqldb.models._get_monitoring_schedules_table(
             connection_string=self._sql_connection_string
         )
-        self._tables[
-            mlrun.common.schemas.model_monitoring.FileTargetKind.MONITORING_SCHEDULES
-        ] = self.MonitoringSchedulesTable
+        self._tables[mm_constants.FileTargetKind.MONITORING_SCHEDULES] = (
+            self.MonitoringSchedulesTable
+        )
 
     def _write(self, table: str, event: dict[str, typing.Any]):
         """
@@ -183,14 +182,12 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         """
 
         # Adjust timestamps fields
-        endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.FIRST_REQUEST] = (
-            endpoint
-        )[
-            mlrun.common.schemas.model_monitoring.EventFieldType.LAST_REQUEST
+        endpoint[mm_constants.EventFieldType.FIRST_REQUEST] = (endpoint)[
+            mm_constants.EventFieldType.LAST_REQUEST
         ] = mlrun.utils.datetime_now()
 
         self._write(
-            table=mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_ENDPOINTS,
+            table=mm_constants.EventFieldType.MODEL_ENDPOINTS,
             event=endpoint,
         )
 
@@ -207,13 +204,9 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         """
         self._init_model_endpoints_table()
 
-        attributes.pop(
-            mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID, None
-        )
+        attributes.pop(mm_constants.EventFieldType.ENDPOINT_ID, None)
 
-        filter_endpoint = {
-            mlrun.common.schemas.model_monitoring.EventFieldType.UID: endpoint_id
-        }
+        filter_endpoint = {mm_constants.EventFieldType.UID: endpoint_id}
 
         self._update(
             attributes=attributes, table=self.ModelEndpointsTable, **filter_endpoint
@@ -227,9 +220,7 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         """
         self._init_model_endpoints_table()
 
-        filter_endpoint = {
-            mlrun.common.schemas.model_monitoring.EventFieldType.UID: endpoint_id
-        }
+        filter_endpoint = {mm_constants.EventFieldType.UID: endpoint_id}
         # Delete the model endpoint record using sqlalchemy ORM
         self._delete(table=self.ModelEndpointsTable, **filter_endpoint)
 
@@ -249,9 +240,7 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         self._init_model_endpoints_table()
 
         # Get the model endpoint record using sqlalchemy ORM
-        filter_endpoint = {
-            mlrun.common.schemas.model_monitoring.EventFieldType.UID: endpoint_id
-        }
+        filter_endpoint = {mm_constants.EventFieldType.UID: endpoint_id}
         endpoint_record = self._get(table=self.ModelEndpointsTable, **filter_endpoint)
 
         if not endpoint_record:
@@ -303,36 +292,32 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
                 query = self._filter_values(
                     query=query,
                     model_endpoints_table=model_endpoints_table,
-                    key_filter=mlrun.common.schemas.model_monitoring.EventFieldType.MODEL,
+                    key_filter=mm_constants.EventFieldType.MODEL,
                     filtered_values=[model],
                 )
             if function:
                 query = self._filter_values(
                     query=query,
                     model_endpoints_table=model_endpoints_table,
-                    key_filter=mlrun.common.schemas.model_monitoring.EventFieldType.FUNCTION,
+                    key_filter=mm_constants.EventFieldType.FUNCTION,
                     filtered_values=[function],
                 )
             if uids:
                 query = self._filter_values(
                     query=query,
                     model_endpoints_table=model_endpoints_table,
-                    key_filter=mlrun.common.schemas.model_monitoring.EventFieldType.UID,
+                    key_filter=mm_constants.EventFieldType.UID,
                     filtered_values=uids,
                     combined=False,
                 )
             if top_level:
-                node_ep = str(
-                    mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP.value
-                )
-                router_ep = str(
-                    mlrun.common.schemas.model_monitoring.EndpointType.ROUTER.value
-                )
+                node_ep = str(mm_constants.EndpointType.NODE_EP.value)
+                router_ep = str(mm_constants.EndpointType.ROUTER.value)
                 endpoint_types = [node_ep, router_ep]
                 query = self._filter_values(
                     query=query,
                     model_endpoints_table=model_endpoints_table,
-                    key_filter=mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_TYPE,
+                    key_filter=mm_constants.EventFieldType.ENDPOINT_TYPE,
                     filtered_values=endpoint_types,
                     combined=False,
                 )
@@ -350,18 +335,28 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
 
         return endpoint_list
 
-    def write_application_result(self, event: dict[str, typing.Any]):
+    def write_application_event(
+        self,
+        event: dict[str, typing.Any],
+        kind: mm_constants.WriterEventKind = mm_constants.WriterEventKind.RESULT,
+    ):
         """
-        Write a new application result event in the target table.
+        Write a new application event in the target table.
 
         :param event: An event dictionary that represents the application result, should be corresponded to the
-                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
+                      schema defined in the :py:class:`~mm_constants.constants.WriterEvent`
                       object.
+        :param kind: The type of the event, can be either "result" or "metric".
         """
+
+        if kind == mm_constants.WriterEventKind.METRIC:
+            # TODO : Implement the logic for writing metrics to MySQL
+            return
+
         self._init_application_results_table()
 
         application_filter_dict = {
-            mlrun.common.schemas.model_monitoring.EventFieldType.UID: self._generate_application_result_uid(
+            mm_constants.EventFieldType.UID: self._generate_application_result_uid(
                 event
             )
         }
@@ -372,11 +367,11 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         if application_record:
             self._convert_to_datetime(
                 event=event,
-                key=mlrun.common.schemas.model_monitoring.WriterEvent.START_INFER_TIME,
+                key=mm_constants.WriterEvent.START_INFER_TIME,
             )
             self._convert_to_datetime(
                 event=event,
-                key=mlrun.common.schemas.model_monitoring.WriterEvent.END_INFER_TIME,
+                key=mm_constants.WriterEvent.END_INFER_TIME,
             )
             # Update an existing application result
             self._update(
@@ -386,14 +381,12 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
             )
         else:
             # Write a new application result
-            event[mlrun.common.schemas.model_monitoring.EventFieldType.UID] = (
-                application_filter_dict[
-                    mlrun.common.schemas.model_monitoring.EventFieldType.UID
-                ]
-            )
+            event[mm_constants.EventFieldType.UID] = application_filter_dict[
+                mm_constants.EventFieldType.UID
+            ]
 
             self._write(
-                table=mlrun.common.schemas.model_monitoring.FileTargetKind.APP_RESULTS,
+                table=mm_constants.FileTargetKind.APP_RESULTS,
                 event=event,
             )
 
@@ -405,11 +398,11 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
     @staticmethod
     def _generate_application_result_uid(event: dict[str, typing.Any]) -> str:
         return (
-            event[mlrun.common.schemas.model_monitoring.WriterEvent.ENDPOINT_ID]
+            event[mm_constants.WriterEvent.ENDPOINT_ID]
             + "_"
-            + event[mlrun.common.schemas.model_monitoring.WriterEvent.APPLICATION_NAME]
+            + event[mm_constants.WriterEvent.APPLICATION_NAME]
             + "_"
-            + event[mlrun.common.schemas.model_monitoring.ResultData.RESULT_NAME]
+            + event[mm_constants.ResultData.RESULT_NAME]
         )
 
     def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
@@ -459,19 +452,17 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
         if not monitoring_schedule_record:
             # Add a new record with empty last analyzed value
             self._write(
-                table=mlrun.common.schemas.model_monitoring.FileTargetKind.MONITORING_SCHEDULES,
+                table=mm_constants.FileTargetKind.MONITORING_SCHEDULES,
                 event={
-                    mlrun.common.schemas.model_monitoring.SchedulingKeys.UID: uuid.uuid4().hex,
-                    mlrun.common.schemas.model_monitoring.SchedulingKeys.APPLICATION_NAME: application_name,
-                    mlrun.common.schemas.model_monitoring.SchedulingKeys.ENDPOINT_ID: endpoint_id,
-                    mlrun.common.schemas.model_monitoring.SchedulingKeys.LAST_ANALYZED: last_analyzed,
+                    mm_constants.SchedulingKeys.UID: uuid.uuid4().hex,
+                    mm_constants.SchedulingKeys.APPLICATION_NAME: application_name,
+                    mm_constants.SchedulingKeys.ENDPOINT_ID: endpoint_id,
+                    mm_constants.SchedulingKeys.LAST_ANALYZED: last_analyzed,
                 },
             )
 
         self._update(
-            attributes={
-                mlrun.common.schemas.model_monitoring.SchedulingKeys.LAST_ANALYZED: last_analyzed
-            },
+            attributes={mm_constants.SchedulingKeys.LAST_ANALYZED: last_analyzed},
             table=self.MonitoringSchedulesTable,
             **application_filter_dict,
         )
@@ -567,9 +558,7 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
 
         # Convert endpoint labels into dictionary
         endpoint_labels = json.loads(
-            endpoint_dict.get(
-                mlrun.common.schemas.model_monitoring.EventFieldType.LABELS
-            )
+            endpoint_dict.get(mm_constants.EventFieldType.LABELS)
         )
 
         for label in labels:
@@ -596,26 +585,25 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
             )
         application_filter_dict = {}
         if endpoint_id:
-            application_filter_dict[
-                mlrun.common.schemas.model_monitoring.SchedulingKeys.ENDPOINT_ID
-            ] = endpoint_id
+            application_filter_dict[mm_constants.SchedulingKeys.ENDPOINT_ID] = (
+                endpoint_id
+            )
         if application_name:
-            application_filter_dict[
-                mlrun.common.schemas.model_monitoring.SchedulingKeys.APPLICATION_NAME
-            ] = application_name
+            application_filter_dict[mm_constants.SchedulingKeys.APPLICATION_NAME] = (
+                application_name
+            )
         return application_filter_dict
 
-    def delete_model_endpoints_resources(self, endpoints: list[dict[str, typing.Any]]):
+    def delete_model_endpoints_resources(self):
         """
         Delete all model endpoints resources in both SQL and the time series DB.
 
-        :param endpoints: A list of model endpoints flattened dictionaries.
         """
 
+        endpoints = self.list_model_endpoints()
+
         for endpoint_dict in endpoints:
-            endpoint_id = endpoint_dict[
-                mlrun.common.schemas.model_monitoring.EventFieldType.UID
-            ]
+            endpoint_id = endpoint_dict[mm_constants.EventFieldType.UID]
 
             # Delete last analyzed records
             self._delete_last_analyzed(endpoint_id=endpoint_id)
@@ -625,38 +613,3 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
 
             # Delete model endpoint record
             self.delete_model_endpoint(endpoint_id=endpoint_id)
-
-    def get_endpoint_real_time_metrics(
-        self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> dict[str, list[tuple[str, float]]]:
-        """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
-        """
-        # # TODO : Implement this method once Perometheus is supported
-        logger.warning(
-            "Real time metrics service using Prometheus will be implemented in 1.4.0"
-        )
-
-        return {}