mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/model_monitoring/controller.py

@@ -18,24 +18,19 @@ import json
 import os
 import re
 from collections.abc import Iterator
-from typing import Any, NamedTuple, Optional, Union, cast
+from typing import NamedTuple, Optional, Union, cast
 
 import nuclio
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
-import mlrun.feature_store as fstore
 import mlrun.model_monitoring.db.stores
-from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
 from mlrun.datastore import get_stream_pusher
-from mlrun.datastore.targets import ParquetTarget
 from mlrun.errors import err_to_str
 from mlrun.model_monitoring.helpers import (
     _BatchDict,
     batch_dict2timedelta,
-    calculate_inputs_statistics,
-    get_monitoring_parquet_path,
     get_stream_path,
 )
 from mlrun.utils import datetime_now, logger
@@ -218,7 +213,7 @@ class _BatchWindowGenerator:
             # If the endpoint does not have a stream, `last_updated` should be
             # the minimum between the current time and the last updated time.
             # This compensates for the bumping mechanism - see
-            # `bump_model_endpoint_last_request`.
+            # `update_model_endpoint_last_request`.
             last_updated = min(int(datetime_now().timestamp()), last_updated)
             logger.debug(
                 "The endpoint does not have a stream", last_updated=last_updated
@@ -273,26 +268,14 @@ class MonitoringApplicationController:
     Note that the MonitoringApplicationController object requires access keys along with valid project configurations.
     """
 
-    def __init__(
-        self,
-        mlrun_context: mlrun.run.MLClientCtx,
-        project: str,
-    ):
-        """
-        Initialize Monitoring Application Processor object.
+    def __init__(self) -> None:
+        """Initialize Monitoring Application Controller"""
+        self.project = cast(str, mlrun.mlconf.default_project)
+        self.project_obj = mlrun.load_project(name=self.project, url=self.project)
 
-        :param mlrun_context:               An MLRun context.
-        :param project:                     Project name.
-        """
-        self.context = mlrun_context
-        self.project = project
-        self.project_obj = mlrun.get_or_create_project(project)
+        logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
 
-        mlrun_context.logger.debug(
-            f"Initializing {self.__class__.__name__}", project=project
-        )
-
-        self.db = mlrun.model_monitoring.get_store_object(project=project)
+        self.db = mlrun.model_monitoring.get_store_object(project=self.project)
 
         self._batch_window_generator = _BatchWindowGenerator(
             batch_dict=json.loads(
@@ -303,15 +286,9 @@ class MonitoringApplicationController:
         )
 
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
-        self.parquet_directory = get_monitoring_parquet_path(
-            self.project_obj,
-            kind=mm_constants.FileTargetKind.APPS_PARQUET,
+        self.tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project
         )
-        self.storage_options = None
-        if not mlrun.mlconf.is_ce_mode():
-            self._initialize_v3io_configurations()
-        elif self.parquet_directory.startswith("s3://"):
-            self.storage_options = mlrun.mlconf.get_s3_storage_options()
 
     @staticmethod
     def _get_model_monitoring_access_key() -> Optional[str]:
@@ -321,98 +298,85 @@ class MonitoringApplicationController:
             access_key = mlrun.mlconf.get_v3io_access_key()
         return access_key
 
-    def _initialize_v3io_configurations(self) -> None:
-        self.v3io_framesd = mlrun.mlconf.v3io_framesd
-        self.v3io_api = mlrun.mlconf.v3io_api
-        self.storage_options = dict(
-            v3io_access_key=self.model_monitoring_access_key, v3io_api=self.v3io_api
-        )
-
-    def run(self, event: nuclio.Event):
+    def run(self) -> None:
         """
-        Main method for run all the relevant monitoring applications on each endpoint
-
-        :param event:   trigger event
+        Main method for run all the relevant monitoring applications on each endpoint.
+        This method handles the following:
+        1. List model endpoints
+        2. List applications
+        3. Check model monitoring windows
+        4. Send data to applications
+        5. Delete old parquets
         """
         logger.info("Start running monitoring controller")
         try:
             applications_names = []
-            endpoints = self.db.list_model_endpoints()
+            endpoints = self.db.list_model_endpoints(include_stats=True)
             if not endpoints:
-                self.context.logger.info(
-                    "No model endpoints found", project=self.project
-                )
+                logger.info("No model endpoints found", project=self.project)
                 return
             monitoring_functions = self.project_obj.list_model_monitoring_functions()
             if monitoring_functions:
-                # Gets only application in ready state
                 applications_names = list(
-                    {
-                        app.metadata.name
-                        for app in monitoring_functions
-                        if (
-                            app.status.state == "ready"
-                            # workaround for the default app, as its `status.state` is `None`
-                            or app.metadata.name
-                            == mm_constants.HistogramDataDriftApplicationConstants.NAME
-                        )
-                    }
+                    {app.metadata.name for app in monitoring_functions}
                 )
+            # if monitoring_functions: - TODO : ML-7700
+            #   Gets only application in ready state
+            #   applications_names = list(
+            #       {
+            #           app.metadata.name
+            #           for app in monitoring_functions
+            #           if (
+            #               app.status.state == "ready"
+            #               # workaround for the default app, as its `status.state` is `None`
+            #               or app.metadata.name
+            #               == mm_constants.HistogramDataDriftApplicationConstants.NAME
+            #           )
+            #       }
+            #   )
             if not applications_names:
-                self.context.logger.info(
-                    "No monitoring functions found", project=self.project
-                )
+                logger.info("No monitoring functions found", project=self.project)
                 return
-            self.context.logger.info(
+            logger.info(
                 "Starting to iterate over the applications",
                 applications=applications_names,
             )
 
         except Exception as e:
-            self.context.logger.error(
+            logger.error(
                 "Failed to list endpoints and monitoring applications",
                 exc=err_to_str(e),
             )
             return
         # Initialize a process pool that will be used to run each endpoint applications on a dedicated process
-        pool = concurrent.futures.ProcessPoolExecutor(
+        with concurrent.futures.ThreadPoolExecutor(
             max_workers=min(len(endpoints), 10),
-        )
-        futures = []
-        for endpoint in endpoints:
-            if (
-                endpoint[mm_constants.EventFieldType.ACTIVE]
-                and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
-                == mm_constants.ModelMonitoringMode.enabled.value
-            ):
-                # Skip router endpoint:
+        ) as pool:
+            for endpoint in endpoints:
                 if (
-                    int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
-                    == mm_constants.EndpointType.ROUTER
+                    endpoint[mm_constants.EventFieldType.ACTIVE]
+                    and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
+                    == mm_constants.ModelMonitoringMode.enabled.value
                 ):
-                    # Router endpoint has no feature stats
-                    logger.info(
-                        f"{endpoint[mm_constants.EventFieldType.UID]} is router skipping"
+                    # Skip router endpoint:
+                    if (
+                        int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
+                        == mm_constants.EndpointType.ROUTER
+                    ):
+                        # Router endpoint has no feature stats
+                        logger.info(
+                            f"{endpoint[mm_constants.EventFieldType.UID]} is router, skipping"
+                        )
+                        continue
+                    pool.submit(
+                        MonitoringApplicationController.model_endpoint_process,
+                        endpoint=endpoint,
+                        applications_names=applications_names,
+                        batch_window_generator=self._batch_window_generator,
+                        project=self.project,
+                        model_monitoring_access_key=self.model_monitoring_access_key,
+                        tsdb_connector=self.tsdb_connector,
                     )
-                    continue
-                future = pool.submit(
-                    MonitoringApplicationController.model_endpoint_process,
-                    endpoint=endpoint,
-                    applications_names=applications_names,
-                    batch_window_generator=self._batch_window_generator,
-                    project=self.project,
-                    parquet_directory=self.parquet_directory,
-                    storage_options=self.storage_options,
-                    model_monitoring_access_key=self.model_monitoring_access_key,
-                )
-                futures.append(future)
-
-        for future in concurrent.futures.as_completed(futures):
-            result = future.result()
-            if result:
-                self.context.log_results(result)
-
-        self._delete_old_parquet(endpoints=endpoints)
 
     @classmethod
     def model_endpoint_process(
@@ -421,10 +385,9 @@ class MonitoringApplicationController:
         applications_names: list[str],
         batch_window_generator: _BatchWindowGenerator,
         project: str,
-        parquet_directory: str,
-        storage_options: dict,
         model_monitoring_access_key: str,
-    ) -> Optional[dict[str, list[str]]]:
+        tsdb_connector: mlrun.model_monitoring.db.tsdb.TSDBConnector,
+    ) -> None:
         """
         Process a model endpoint and trigger the monitoring applications. This function running on different process
         for each endpoint. In addition, this function will generate a parquet file that includes the relevant data
@@ -434,18 +397,13 @@ class MonitoringApplicationController:
         :param applications_names:          (list[str]) List of application names to push results to.
         :param batch_window_generator:      (_BatchWindowGenerator) An object that generates _BatchWindow objects.
         :param project:                     (str) Project name.
-        :param parquet_directory:           (str) Directory to store application parquet files
-        :param storage_options:             (dict) Storage options for writing ParquetTarget.
         :param model_monitoring_access_key: (str) Access key to apply the model monitoring process.
-
+        :param tsdb_connector:              (mlrun.model_monitoring.db.tsdb.TSDBConnector) TSDB connector
         """
         endpoint_id = endpoint[mm_constants.EventFieldType.UID]
-        start_times: set[datetime.datetime] = set()
+        # if false the endpoint represent batch infer step.
+        has_stream = endpoint[mm_constants.EventFieldType.STREAM_PATH] != ""
         try:
-            m_fs = fstore.get_feature_set(
-                endpoint[mm_constants.EventFieldType.FEATURE_SET_URI]
-            )
-
             for application in applications_names:
                 batch_window = batch_window_generator.get_batch_window(
                     project=project,
@@ -453,168 +411,75 @@ class MonitoringApplicationController:
                     application=application,
                     first_request=endpoint[mm_constants.EventFieldType.FIRST_REQUEST],
                     last_request=endpoint[mm_constants.EventFieldType.LAST_REQUEST],
-                    has_stream=endpoint[mm_constants.EventFieldType.STREAM_PATH] != "",
+                    has_stream=has_stream,
                 )
 
                 for start_infer_time, end_infer_time in batch_window.get_intervals():
-                    # start - TODO : delete in 1.9.0 (V1 app deprecation)
-                    try:
-                        # Get application sample data
-                        offline_response = cls._get_sample_df(
-                            feature_set=m_fs,
+                    prediction_metric = tsdb_connector.read_predictions(
+                        endpoint_id=endpoint_id,
+                        start=start_infer_time,
+                        end=end_infer_time,
+                    )
+                    if not prediction_metric.data and has_stream:
+                        logger.info(
+                            "No data found for the given interval",
+                            start=start_infer_time,
+                            end=end_infer_time,
                             endpoint_id=endpoint_id,
+                        )
+                    else:
+                        logger.info(
+                            "Data found for the given interval",
+                            start=start_infer_time,
+                            end=end_infer_time,
+                            endpoint_id=endpoint_id,
+                        )
+                        cls._push_to_applications(
                             start_infer_time=start_infer_time,
                             end_infer_time=end_infer_time,
-                            parquet_directory=parquet_directory,
-                            storage_options=storage_options,
-                            application_name=application,
-                        )
-
-                        df = offline_response.to_dataframe()
-                        parquet_target_path = offline_response.vector.get_target_path()
-
-                        if len(df) == 0:
-                            logger.info(
-                                "During this time window, the endpoint has not received any data",
-                                endpoint=endpoint[mm_constants.EventFieldType.UID],
-                                start_time=start_infer_time,
-                                end_time=end_infer_time,
-                            )
-                            continue
-
-                    except FileNotFoundError:
-                        logger.warn(
-                            "No parquets were written yet",
-                            endpoint=endpoint[mm_constants.EventFieldType.UID],
+                            endpoint_id=endpoint_id,
+                            project=project,
+                            applications_names=[application],
+                            model_monitoring_access_key=model_monitoring_access_key,
                         )
-                        continue
-
-                    # Get the timestamp of the latest request:
-                    latest_request = df[mm_constants.EventFieldType.TIMESTAMP].iloc[-1]
-
-                    # Get the feature stats from the model endpoint for reference data
-                    feature_stats = json.loads(
-                        endpoint[mm_constants.EventFieldType.FEATURE_STATS]
-                    )
-
-                    # Pad the original feature stats to accommodate current
-                    # data out of the original range (unless already padded)
-                    pad_features_hist(FeatureStats(feature_stats))
-
-                    # Get the current stats:
-                    current_stats = calculate_inputs_statistics(
-                        sample_set_statistics=feature_stats, inputs=df
-                    )
-                    # end - TODO : delete in 1.9.0 (V1 app deprecation)
-                    cls._push_to_applications(
-                        current_stats=current_stats,
-                        feature_stats=feature_stats,
-                        start_infer_time=start_infer_time,
-                        end_infer_time=end_infer_time,
-                        endpoint_id=endpoint_id,
-                        latest_request=latest_request,
-                        project=project,
-                        applications_names=[application],
-                        model_monitoring_access_key=model_monitoring_access_key,
-                        parquet_target_path=parquet_target_path,
-                    )
-                    start_times.add(start_infer_time)
         except Exception:
             logger.exception(
                 "Encountered an exception",
                 endpoint_id=endpoint[mm_constants.EventFieldType.UID],
             )
 
-        if start_times:
-            return {endpoint_id: [str(t) for t in sorted(list(start_times))]}
-
-    def _delete_old_parquet(self, endpoints: list[dict[str, Any]], days: int = 1):
-        """
-        Delete application parquets older than the argument days.
-
-        :param endpoints: A list of dictionaries of model endpoints records.
-        """
-        if self.parquet_directory.startswith("v3io:///"):
-            # create fs with access to the user side (under projects)
-            store, _, _ = mlrun.store_manager.get_or_create_store(
-                self.parquet_directory,
-                {"V3IO_ACCESS_KEY": self.model_monitoring_access_key},
-            )
-            fs = store.filesystem
-
-            # calculate time threshold (keep only files from the last 24 hours)
-            time_to_keep = (
-                datetime.datetime.now(tz=datetime.timezone.utc)
-                - datetime.timedelta(days=days)
-            ).timestamp()
-
-            for endpoint in endpoints:
-                try:
-                    apps_parquet_directories = fs.listdir(
-                        path=f"{self.parquet_directory}"
-                        f"/key={endpoint[mm_constants.EventFieldType.UID]}"
-                    )
-                    for directory in apps_parquet_directories:
-                        if directory["mtime"] < time_to_keep:
-                            # Delete files
-                            fs.rm(path=directory["name"], recursive=True)
-                            # Delete directory
-                            fs.rmdir(path=directory["name"])
-                except FileNotFoundError:
-                    logger.info(
-                        "Application parquet directory is empty, "
-                        "probably parquets have not yet been created for this app",
-                        endpoint=endpoint[mm_constants.EventFieldType.UID],
-                        path=f"{self.parquet_directory}"
-                        f"/key={endpoint[mm_constants.EventFieldType.UID]}",
-                    )
-
     @staticmethod
     def _push_to_applications(
-        current_stats,
-        feature_stats,
-        start_infer_time,
-        end_infer_time,
-        endpoint_id,
-        latest_request,
-        project,
-        applications_names,
-        model_monitoring_access_key,
-        parquet_target_path,
+        start_infer_time: datetime.datetime,
+        end_infer_time: datetime.datetime,
+        endpoint_id: str,
+        project: str,
+        applications_names: list[str],
+        model_monitoring_access_key: str,
     ):
         """
         Pushes data to multiple stream applications.
 
-        :param current_stats:       Current statistics of input data.
-        :param feature_stats:       Statistics of train features.
-        :param start_infer_time:    The beginning of the infer interval window.
-        :param end_infer_time:      The end of the infer interval window.
-        :param endpoint_id:         Identifier for the model endpoint.
-        :param latest_request:      Timestamp of the latest model request.
-        :param project: mlrun       Project name.
-        :param applications_names:  List of application names to which data will be pushed.
+        :param start_infer_time:            The beginning of the infer interval window.
+        :param end_infer_time:              The end of the infer interval window.
+        :param endpoint_id:                 Identifier for the model endpoint.
+        :param project: mlrun               Project name.
+        :param applications_names:          List of application names to which data will be pushed.
+        :param model_monitoring_access_key: Access key to apply the model monitoring process.
 
         """
-
         data = {
-            mm_constants.ApplicationEvent.CURRENT_STATS: json.dumps(current_stats),
-            mm_constants.ApplicationEvent.FEATURE_STATS: json.dumps(feature_stats),
-            mm_constants.ApplicationEvent.SAMPLE_PARQUET_PATH: parquet_target_path,
             mm_constants.ApplicationEvent.START_INFER_TIME: start_infer_time.isoformat(
                 sep=" ", timespec="microseconds"
             ),
             mm_constants.ApplicationEvent.END_INFER_TIME: end_infer_time.isoformat(
                 sep=" ", timespec="microseconds"
             ),
-            mm_constants.ApplicationEvent.LAST_REQUEST: latest_request.isoformat(
-                sep=" ", timespec="microseconds"
-            ),
             mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
             mm_constants.ApplicationEvent.OUTPUT_STREAM_URI: get_stream_path(
                 project=project,
                 function_name=mm_constants.MonitoringFunctionNames.WRITER,
             ),
-            mm_constants.ApplicationEvent.MLRUN_CONTEXT: {},  # TODO : for future use by ad-hoc batch infer
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})
@@ -627,49 +492,12 @@ class MonitoringApplicationController:
                 [data]
             )
 
-    @staticmethod
-    def _get_sample_df(
-        feature_set: mlrun.common.schemas.FeatureSet,
-        endpoint_id: str,
-        start_infer_time: datetime.datetime,
-        end_infer_time: datetime.datetime,
-        parquet_directory: str,
-        storage_options: dict,
-        application_name: str,
-    ) -> mlrun.feature_store.OfflineVectorResponse:
-        """
-        Retrieves a sample DataFrame of the current input according to the provided infer interval window.
-
-        :param feature_set:         The main feature set.
-        :param endpoint_id:         Identifier for the model endpoint.
-        :param start_infer_time:    The beginning of the infer interval window.
-        :param end_infer_time:      The end of the infer interval window.
-        :param parquet_directory:   Directory where Parquet files are stored.
-        :param storage_options:     Storage options for accessing the data.
-        :param application_name:    Current application name.
 
-        :return: OfflineVectorResponse that can be used for generating a sample DataFrame for the specified endpoint.
+def handler(context: nuclio.Context, event: nuclio.Event) -> None:
+    """
+    Run model monitoring application processor
 
-        """
-        features = [f"{feature_set.metadata.name}.*"]
-        vector = fstore.FeatureVector(
-            name=f"{endpoint_id}_vector",
-            features=features,
-            with_indexes=True,
-        )
-        vector.metadata.tag = application_name
-        vector.feature_set_objects = {feature_set.metadata.name: feature_set}
-
-        # get offline features based on application start and end time.
-        # store the result parquet by partitioning by controller end processing time
-        offline_response = vector.get_offline_features(
-            start_time=start_infer_time,
-            end_time=end_infer_time,
-            timestamp_for_filtering=mm_constants.EventFieldType.TIMESTAMP,
-            target=ParquetTarget(
-                path=parquet_directory
-                + f"/key={endpoint_id}/{int(start_infer_time.timestamp())}/{application_name}.parquet",
-                storage_options=storage_options,
-            ),
-        )
-        return offline_response
+    :param context: the Nuclio context
+    :param event:   trigger event
+    """
+    MonitoringApplicationController().run()

mlrun/model_monitoring/db/stores/__init__.py

@@ -63,7 +63,7 @@ class ObjectStoreFactory(enum.Enum):
         :param value: Provided enum (invalid) value.
         """
         valid_values = list(cls.__members__.keys())
-        raise mlrun.errors.MLRunInvalidMMStoreType(
+        raise mlrun.errors.MLRunInvalidMMStoreTypeError(
             f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
         )
 
@@ -100,7 +100,9 @@ def get_store_object(
     :param store_connection_string: Optional explicit connection string of the store.
 
     :return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
-    and delete a model endpoint record.
+             and delete a model endpoint record.
+    :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide store connection
+             or the provided store connection is invalid.
     """
 
     store_connection_string = (
@@ -121,7 +123,10 @@ def get_store_object(
             mlrun.common.schemas.model_monitoring.ModelEndpointTarget.V3IO_NOSQL
         )
     else:
-        store_type = None
+        raise mlrun.errors.MLRunInvalidMMStoreTypeError(
+            "You must provide a valid store connection by using "
+            "set_model_monitoring_credentials API."
+        )
     # Get store type value from ObjectStoreFactory enum class
     store_type_fact = ObjectStoreFactory(store_type)
 

mlrun/model_monitoring/db/stores/base/store.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
 import typing
 from abc import ABC, abstractmethod
@@ -94,6 +95,7 @@ class StoreBase(ABC):
         labels: list[str] = None,
         top_level: bool = None,
         uids: list = None,
+        include_stats: bool = None,
     ) -> list[dict[str, typing.Any]]:
         """
         Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
@@ -107,6 +109,7 @@ class StoreBase(ABC):
                                 key (i.e. "key").
         :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
         :param uids:             List of model endpoint unique ids to include in the result.
+        :param include_stats:   If True, will include model endpoint statistics in the result.
 
         :return: A list of model endpoint dictionaries.
         """

mlrun/model_monitoring/db/stores/sqldb/models/base.py

@@ -11,8 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from sqlalchemy import (
-    TIMESTAMP,
+    DATETIME,
+    TIMESTAMP,  # TODO: migrate to DATETIME, see ML-6921
     Boolean,
     Column,
     Float,
@@ -90,11 +92,11 @@ class ModelEndpointsBaseTable(BaseModel):
     metrics = Column(EventFieldType.METRICS, Text)
     first_request = Column(
         EventFieldType.FIRST_REQUEST,
-        TIMESTAMP(timezone=True),
+        TIMESTAMP(timezone=True),  # TODO: migrate to DATETIME, see ML-6921
     )
     last_request = Column(
         EventFieldType.LAST_REQUEST,
-        TIMESTAMP(timezone=True),
+        TIMESTAMP(timezone=True),  # TODO: migrate to DATETIME, see ML-6921
     )
 
 
@@ -122,11 +124,11 @@ class ApplicationResultBaseTable(BaseModel):
 
     start_infer_time = Column(
         WriterEvent.START_INFER_TIME,
-        TIMESTAMP(timezone=True),
+        DATETIME(timezone=True),
     )
     end_infer_time = Column(
         WriterEvent.END_INFER_TIME,
-        TIMESTAMP(timezone=True),
+        DATETIME(timezone=True),
     )
 
     result_status = Column(ResultData.RESULT_STATUS, String(10))
@@ -152,11 +154,11 @@ class ApplicationMetricsBaseTable(BaseModel):
     )
     start_infer_time = Column(
         WriterEvent.START_INFER_TIME,
-        TIMESTAMP(timezone=True),
+        DATETIME(timezone=True),
     )
     end_infer_time = Column(
         WriterEvent.END_INFER_TIME,
-        TIMESTAMP(timezone=True),
+        DATETIME(timezone=True),
     )
     metric_name = Column(
         MetricData.METRIC_NAME,