mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/model_monitoring/db/_schedules.py

@@ -0,0 +1,156 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from contextlib import AbstractContextManager
+from types import TracebackType
+from typing import Final, Optional
+
+import botocore.exceptions
+
+import mlrun.common.schemas
+import mlrun.errors
+import mlrun.model_monitoring.helpers
+from mlrun.utils import logger
+
+
+class ModelMonitoringSchedulesFile(AbstractContextManager):
+    DEFAULT_SCHEDULES: Final = {}
+    INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
+    ENCODING = "utf-8"
+
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize applications monitoring schedules file object.
+        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        :param endpoint_id: The endpoint ID.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._item = mlrun.model_monitoring.helpers.get_monitoring_schedules_data(
+            project=project, endpoint_id=endpoint_id
+        )
+        self._path = self._item.url
+        self._fs = self._item.store.filesystem
+        # `self._schedules` is an in-memory copy of the DB for all the applications for
+        # the same model endpoint.
+        self._schedules: dict[str, int] = self.DEFAULT_SCHEDULES.copy()
+        # Does `self._schedules` hold the content of `self._item`?
+        self._open_schedules = False
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringSchedulesFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+    def create(self) -> None:
+        """Create a schedules file with initial content - an empty dictionary"""
+        logger.debug("Creating model monitoring schedules file", path=self._item.url)
+        self._item.put(self.INITIAL_CONTENT)
+
+    def delete(self) -> None:
+        """Delete schedules file if it exists"""
+        if (
+            self._fs is None  # In-memory store
+            or self._fs.exists(self._path)
+        ):
+            logger.debug(
+                "Deleting model monitoring schedules file", path=self._item.url
+            )
+            self._item.delete()
+        else:
+            logger.debug(
+                "Model monitoring schedules file does not exist, nothing to delete",
+                path=self._item.url,
+            )
+
+    def _open(self) -> None:
+        try:
+            content = self._item.get()
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ) as err:
+            if (
+                isinstance(err, botocore.exceptions.ClientError)
+                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
+                and err.response["Error"]["Code"] != "NoSuchKey"
+            ):
+                raise
+
+            logger.exception(
+                "The schedules file was not found. It should have been created "
+                "as a part of the model endpoint's creation",
+                path=self._path,
+            )
+            raise
+
+        if isinstance(content, bytes):
+            content = content.decode(encoding=self.ENCODING)
+        self._schedules = json.loads(content)
+        self._open_schedules = True
+
+    def _close(self) -> None:
+        self._item.put(json.dumps(self._schedules))
+        self._schedules = self.DEFAULT_SCHEDULES
+        self._open_schedules = False
+
+    def __enter__(self) -> "ModelMonitoringSchedulesFile":
+        self._open()
+        return super().__enter__()
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Optional[bool]:
+        self._close()
+
+    def _check_open_schedules(self) -> None:
+        if not self._open_schedules:
+            raise mlrun.errors.MLRunValueError(
+                "Open the schedules file as a context manager first"
+            )
+
+    def get_application_time(self, application: str) -> Optional[int]:
+        self._check_open_schedules()
+        return self._schedules.get(application)
+
+    def update_application_time(self, application: str, timestamp: int) -> None:
+        self._check_open_schedules()
+        self._schedules[application] = timestamp
+
+
+def delete_model_monitoring_schedules_folder(project: str) -> None:
+    """Delete the model monitoring schedules folder of the project"""
+    folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(
+        project
+    )
+    fs = mlrun.datastore.store_manager.object(folder).store.filesystem
+    if fs and fs.exists(folder):
+        logger.debug("Deleting model monitoring schedules folder", folder=folder)
+        fs.rm(folder, recursive=True)
+    elif fs is None:  # In-memory store
+        raise mlrun.errors.MLRunValueError(
+            "Cannot delete a folder without a file-system"
+        )

mlrun/model_monitoring/db/_stats.py

@@ -0,0 +1,189 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import abc
+import json
+from abc import abstractmethod
+from datetime import datetime, timezone
+from typing import cast
+
+import botocore.exceptions
+import fsspec
+
+import mlrun.datastore.base
+from mlrun.common.schemas.model_monitoring.constants import StatsKind
+from mlrun.model_monitoring.helpers import (
+    get_monitoring_current_stats_data,
+    get_monitoring_drift_measures_data,
+    get_monitoring_stats_directory_path,
+)
+from mlrun.utils import logger
+
+
+class ModelMonitoringStatsFile(abc.ABC):
+    """
+    Abstract class
+    Initialize applications monitoring stats file object.
+    The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+    When working with the schedules data, use this class as a context manager to read and write the data.
+    """
+
+    def __init__(self, item: mlrun.datastore.base.DataItem, file_type: str):
+        self._path = item.url
+        self._item = item
+        self._file_type = file_type
+        self._fs = cast(fsspec.AbstractFileSystem, self._item.store.filesystem)
+
+    def create(self) -> None:
+        """Create a json file with initial content - an empty dictionary"""
+        logger.debug(
+            f"Creating model monitoring {self._file_type} file", path=self._item.url
+        )
+        self._item.put(
+            json.dumps(
+                {
+                    "data": dict(),
+                    "timestamp": mlrun.utils.datetime_now().isoformat(
+                        sep=" ", timespec="microseconds"
+                    ),
+                }
+            )
+        )
+
+    def delete(self) -> None:
+        """Delete json file if it exists"""
+        if self._fs.exists(self._path):
+            logger.debug(
+                f"Deleting model monitoring {self._file_type} file", path=self._item.url
+            )
+            self._item.delete()
+        else:
+            logger.debug(
+                f"Model monitoring {self._file_type} file does not exist, nothing to delete",
+                path=self._item.url,
+            )
+
+    def read(self) -> tuple[dict, datetime]:
+        """
+        Read the stats data and timestamp saved in file
+        :return: tuple[dict, str] dictionary with stats data and timestamp saved in file
+        """
+        try:
+            content = json.loads(self._item.get().decode())
+            timestamp = content.get("timestamp")
+            if timestamp is not None:
+                timestamp = datetime.fromisoformat(timestamp).astimezone(
+                    tz=timezone.utc
+                )
+            return content.get("data"), timestamp
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ) as err:
+            if (
+                isinstance(err, botocore.exceptions.ClientError)
+                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
+                and err.response["Error"]["Code"] != "NoSuchKey"
+            ):
+                raise
+
+            logger.exception(
+                "The Stats file was not found. It should have been created "
+                "as a part of the model endpoint's creation",
+                path=self._path,
+                error=err,
+            )
+            raise
+
+    def write(self, stats: dict, timestamp: datetime) -> None:
+        """
+        Write stats data to file overwrite the existing file
+        :param stats: dictionary with the stats data
+        :param timestamp: datetime object with the timestamp of last entry point for the stats calculation
+        """
+        content = {
+            "data": stats,
+            "timestamp": timestamp.isoformat(sep=" ", timespec="microseconds"),
+        }
+        self._item.put(json.dumps(content))
+
+    @classmethod
+    @abstractmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringStatsFile":
+        """
+        Return ModelMonitoringStatsFile child object using ModelEndpoint metadata
+        :param model_endpoint: The current model endpoint to get a stats object for
+        :return: ModelMonitoringStatsFile child object instance
+        """
+        pass
+
+
+class ModelMonitoringCurrentStatsFile(ModelMonitoringStatsFile):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize File object specific for current stats.
+        :param project:         (str) Project name
+        :param endpoint_id:     (str) Endpoint name
+        """
+        super().__init__(
+            get_monitoring_current_stats_data(project, endpoint_id),
+            StatsKind.CURRENT_STATS.value,
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringCurrentStatsFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+
+class ModelMonitoringDriftMeasuresFile(ModelMonitoringStatsFile):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize File object specific for drift measures.
+        :param project:         (str) Project name
+        :param endpoint_id:     (str) Endpoint name
+        """
+        super().__init__(
+            get_monitoring_drift_measures_data(project, endpoint_id),
+            StatsKind.DRIFT_MEASURES.value,
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringDriftMeasuresFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+
+def delete_model_monitoring_stats_folder(project: str) -> None:
+    """Delete the model monitoring schedules folder of the project"""
+    folder = get_monitoring_stats_directory_path(project)
+    fs = mlrun.datastore.store_manager.object(folder).store.filesystem
+    if fs and fs.exists(folder):
+        logger.debug("Deleting model monitoring stats folder", folder=folder)
+        fs.rm(folder, recursive=True)
+    elif fs is None:  # In-memory store
+        raise mlrun.errors.MLRunValueError(
+            "Cannot delete a folder without a file-system"
+        )

mlrun/model_monitoring/db/tsdb/base.py

@@ -15,10 +15,9 @@
 import typing
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import Union
 
 import pandas as pd
-import pydantic
+import pydantic.v1
 
 import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.model_monitoring.db.tsdb.helpers
@@ -48,7 +47,7 @@ class TSDBConnector(ABC):
         self.project = project
 
     @abstractmethod
-    def apply_monitoring_stream_steps(self, graph) -> None:
+    def apply_monitoring_stream_steps(self, graph, **kwargs) -> None:
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
         different key metric dictionaries. This data is being used by the monitoring dashboards in
@@ -132,6 +131,7 @@ class TSDBConnector(ABC):
         end: datetime,
         metrics: list[mm_schemas.ModelEndpointMonitoringMetric],
         type: typing.Literal["metrics", "results"],
+        with_result_extra_data: bool,
     ) -> typing.Union[
         list[
             typing.Union[
@@ -150,11 +150,13 @@ class TSDBConnector(ABC):
         Read metrics OR results from the TSDB and return as a list.
 
         :param endpoint_id: The model endpoint identifier.
-        :param start:       The start time of the query.
-        :param end:         The end time of the query.
-        :param metrics:     The list of metrics to get the values for.
-        :param type:        "metrics" or "results" - the type of each item in metrics.
-        :return:            A list of result values or a list of metric values.
+        :param start:                  The start time of the query.
+        :param end:                    The end time of the query.
+        :param metrics:                The list of metrics to get the values for.
+        :param type:                   "metrics" or "results" - the type of each item in metrics.
+        :param with_result_extra_data: Whether to include the extra data in the results, relevant only when
+                                       `type="results"`.
+        :return:                        A list of result values or a list of metric values.
         """
 
     @abstractmethod
@@ -193,9 +195,9 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_last_request(
         self,
-        endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        endpoint_ids: typing.Union[str, list[str]],
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches data from the predictions TSDB table and returns the most recent request
@@ -212,9 +214,9 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_drift_status(
         self,
-        endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "now-24h",
-        end: Union[datetime, str] = "now",
+        endpoint_ids: typing.Union[str, list[str]],
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches data from the app-results TSDB table and returns the highest status among all
@@ -233,8 +235,8 @@ class TSDBConnector(ABC):
     def get_metrics_metadata(
         self,
         endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches distinct metrics metadata from the metrics TSDB table for a specified model endpoint.
@@ -251,8 +253,8 @@ class TSDBConnector(ABC):
     def get_results_metadata(
         self,
         endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches distinct results metadata from the app-results TSDB table for a specified model endpoint.
@@ -268,9 +270,9 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_error_count(
         self,
-        endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        endpoint_ids: typing.Union[str, list[str]],
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches data from the error TSDB table and returns the error count for each specified endpoint.
@@ -286,12 +288,13 @@ class TSDBConnector(ABC):
     @abstractmethod
     def get_avg_latency(
         self,
-        endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        endpoint_ids: typing.Union[str, list[str]],
+        start: typing.Optional[datetime] = None,
+        end: typing.Optional[datetime] = None,
     ) -> pd.DataFrame:
         """
         Fetches data from the predictions TSDB table and returns the average latency for each specified endpoint
+        in the provided time range, which by default is the last 24 hours.
 
         :param endpoint_ids:    A list of model endpoint identifiers.
         :param start:           The start time for the query.
@@ -420,11 +423,12 @@ class TSDBConnector(ABC):
                                 sub_df.index,
                                 sub_df[mm_schemas.ResultData.RESULT_VALUE],
                                 sub_df[mm_schemas.ResultData.RESULT_STATUS],
+                                sub_df[mm_schemas.ResultData.RESULT_EXTRA_DATA],
                             )
                         ),  # pyright: ignore[reportArgumentType]
                     )
                 )
-            except pydantic.ValidationError:
+            except pydantic.v1.ValidationError:
                 logger.exception(
                     "Failed to convert data-frame into `ModelEndpointMonitoringResultValues`",
                     full_name=full_name,
@@ -446,3 +450,52 @@ class TSDBConnector(ABC):
             )
 
         return metrics_values
+
+    @staticmethod
+    def df_to_metrics_list(
+        *,
+        df: pd.DataFrame,
+        project: str,
+        type: str,
+    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
+        """
+        Parse a DataFrame of metrics from the TSDB into a list of mm metrics objects.
+
+        :param df:      The DataFrame to parse.
+        :param project: The project name.
+        :param type:    The type of the metrics (either "result" or "metric").
+
+        :return:        A list of mm metrics objects.
+        """
+        return list(
+            map(
+                lambda record: mm_schemas.ModelEndpointMonitoringMetric(
+                    project=project,
+                    type=type,
+                    app=record.get(mm_schemas.WriterEvent.APPLICATION_NAME),
+                    name=record.get(mm_schemas.ResultData.RESULT_NAME)
+                    or record.get(mm_schemas.MetricData.METRIC_NAME),
+                    kind=record.get(mm_schemas.ResultData.RESULT_KIND),
+                ),
+                df.to_dict("records"),
+            )
+        )
+
+    @staticmethod
+    def _get_start_end(
+        start: typing.Union[datetime, None],
+        end: typing.Union[datetime, None],
+    ) -> tuple[datetime, datetime]:
+        """
+        static utils function for tsdb start end format
+        :param start:       Either None or datetime, None is handled as datetime.min(tz=timezone.utc)
+        :param end:         Either None or datetime, None is handled as datetime.now(tz=timezone.utc)
+        :return:            start datetime, end datetime
+        """
+        start = start or mlrun.utils.datetime_min()
+        end = end or mlrun.utils.datetime_now()
+        if not (isinstance(start, datetime) and isinstance(end, datetime)):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Both start and end must be datetime objects"
+            )
+        return start, end