mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/model_monitoring/db/_schedules.py

@@ -0,0 +1,156 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from contextlib import AbstractContextManager
+from types import TracebackType
+from typing import Final, Optional
+
+import botocore.exceptions
+
+import mlrun.common.schemas
+import mlrun.errors
+import mlrun.model_monitoring.helpers
+from mlrun.utils import logger
+
+
+class ModelMonitoringSchedulesFile(AbstractContextManager):
+    DEFAULT_SCHEDULES: Final = {}
+    INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
+    ENCODING = "utf-8"
+
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize applications monitoring schedules file object.
+        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        :param endpoint_id: The endpoint ID.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._item = mlrun.model_monitoring.helpers.get_monitoring_schedules_data(
+            project=project, endpoint_id=endpoint_id
+        )
+        self._path = self._item.url
+        self._fs = self._item.store.filesystem
+        # `self._schedules` is an in-memory copy of the DB for all the applications for
+        # the same model endpoint.
+        self._schedules: dict[str, int] = self.DEFAULT_SCHEDULES.copy()
+        # Does `self._schedules` hold the content of `self._item`?
+        self._open_schedules = False
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringSchedulesFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+    def create(self) -> None:
+        """Create a schedules file with initial content - an empty dictionary"""
+        logger.debug("Creating model monitoring schedules file", path=self._item.url)
+        self._item.put(self.INITIAL_CONTENT)
+
+    def delete(self) -> None:
+        """Delete schedules file if it exists"""
+        if (
+            self._fs is None  # In-memory store
+            or self._fs.exists(self._path)
+        ):
+            logger.debug(
+                "Deleting model monitoring schedules file", path=self._item.url
+            )
+            self._item.delete()
+        else:
+            logger.debug(
+                "Model monitoring schedules file does not exist, nothing to delete",
+                path=self._item.url,
+            )
+
+    def _open(self) -> None:
+        try:
+            content = self._item.get()
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ) as err:
+            if (
+                isinstance(err, botocore.exceptions.ClientError)
+                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
+                and err.response["Error"]["Code"] != "NoSuchKey"
+            ):
+                raise
+
+            logger.exception(
+                "The schedules file was not found. It should have been created "
+                "as a part of the model endpoint's creation",
+                path=self._path,
+            )
+            raise
+
+        if isinstance(content, bytes):
+            content = content.decode(encoding=self.ENCODING)
+        self._schedules = json.loads(content)
+        self._open_schedules = True
+
+    def _close(self) -> None:
+        self._item.put(json.dumps(self._schedules))
+        self._schedules = self.DEFAULT_SCHEDULES
+        self._open_schedules = False
+
+    def __enter__(self) -> "ModelMonitoringSchedulesFile":
+        self._open()
+        return super().__enter__()
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> Optional[bool]:
+        self._close()
+
+    def _check_open_schedules(self) -> None:
+        if not self._open_schedules:
+            raise mlrun.errors.MLRunValueError(
+                "Open the schedules file as a context manager first"
+            )
+
+    def get_application_time(self, application: str) -> Optional[int]:
+        self._check_open_schedules()
+        return self._schedules.get(application)
+
+    def update_application_time(self, application: str, timestamp: int) -> None:
+        self._check_open_schedules()
+        self._schedules[application] = timestamp
+
+
+def delete_model_monitoring_schedules_folder(project: str) -> None:
+    """Delete the model monitoring schedules folder of the project"""
+    folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(
+        project
+    )
+    fs = mlrun.datastore.store_manager.object(folder).store.filesystem
+    if fs and fs.exists(folder):
+        logger.debug("Deleting model monitoring schedules folder", folder=folder)
+        fs.rm(folder, recursive=True)
+    elif fs is None:  # In-memory store
+        raise mlrun.errors.MLRunValueError(
+            "Cannot delete a folder without a file-system"
+        )

mlrun/model_monitoring/db/_stats.py

@@ -0,0 +1,189 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import abc
+import json
+from abc import abstractmethod
+from datetime import datetime, timezone
+from typing import cast
+
+import botocore.exceptions
+import fsspec
+
+import mlrun.datastore.base
+from mlrun.common.schemas.model_monitoring.constants import StatsKind
+from mlrun.model_monitoring.helpers import (
+    get_monitoring_current_stats_data,
+    get_monitoring_drift_measures_data,
+    get_monitoring_stats_directory_path,
+)
+from mlrun.utils import logger
+
+
+class ModelMonitoringStatsFile(abc.ABC):
+    """
+    Abstract class
+    Initialize applications monitoring stats file object.
+    The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+    When working with the schedules data, use this class as a context manager to read and write the data.
+    """
+
+    def __init__(self, item: mlrun.datastore.base.DataItem, file_type: str):
+        self._path = item.url
+        self._item = item
+        self._file_type = file_type
+        self._fs = cast(fsspec.AbstractFileSystem, self._item.store.filesystem)
+
+    def create(self) -> None:
+        """Create a json file with initial content - an empty dictionary"""
+        logger.debug(
+            f"Creating model monitoring {self._file_type} file", path=self._item.url
+        )
+        self._item.put(
+            json.dumps(
+                {
+                    "data": dict(),
+                    "timestamp": mlrun.utils.datetime_now().isoformat(
+                        sep=" ", timespec="microseconds"
+                    ),
+                }
+            )
+        )
+
+    def delete(self) -> None:
+        """Delete json file if it exists"""
+        if self._fs.exists(self._path):
+            logger.debug(
+                f"Deleting model monitoring {self._file_type} file", path=self._item.url
+            )
+            self._item.delete()
+        else:
+            logger.debug(
+                f"Model monitoring {self._file_type} file does not exist, nothing to delete",
+                path=self._item.url,
+            )
+
+    def read(self) -> tuple[dict, datetime]:
+        """
+        Read the stats data and timestamp saved in file
+        :return: tuple[dict, str] dictionary with stats data and timestamp saved in file
+        """
+        try:
+            content = json.loads(self._item.get().decode())
+            timestamp = content.get("timestamp")
+            if timestamp is not None:
+                timestamp = datetime.fromisoformat(timestamp).astimezone(
+                    tz=timezone.utc
+                )
+            return content.get("data"), timestamp
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ) as err:
+            if (
+                isinstance(err, botocore.exceptions.ClientError)
+                # Add a log only to "NoSuchKey" errors codes - equivalent to `FileNotFoundError`
+                and err.response["Error"]["Code"] != "NoSuchKey"
+            ):
+                raise
+
+            logger.exception(
+                "The Stats file was not found. It should have been created "
+                "as a part of the model endpoint's creation",
+                path=self._path,
+                error=err,
+            )
+            raise
+
+    def write(self, stats: dict, timestamp: datetime) -> None:
+        """
+        Write stats data to file overwrite the existing file
+        :param stats: dictionary with the stats data
+        :param timestamp: datetime object with the timestamp of last entry point for the stats calculation
+        """
+        content = {
+            "data": stats,
+            "timestamp": timestamp.isoformat(sep=" ", timespec="microseconds"),
+        }
+        self._item.put(json.dumps(content))
+
+    @classmethod
+    @abstractmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringStatsFile":
+        """
+        Return ModelMonitoringStatsFile child object using ModelEndpoint metadata
+        :param model_endpoint: The current model endpoint to get a stats object for
+        :return: ModelMonitoringStatsFile child object instance
+        """
+        pass
+
+
+class ModelMonitoringCurrentStatsFile(ModelMonitoringStatsFile):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize File object specific for current stats.
+        :param project:         (str) Project name
+        :param endpoint_id:     (str) Endpoint name
+        """
+        super().__init__(
+            get_monitoring_current_stats_data(project, endpoint_id),
+            StatsKind.CURRENT_STATS.value,
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringCurrentStatsFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+
+class ModelMonitoringDriftMeasuresFile(ModelMonitoringStatsFile):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize File object specific for drift measures.
+        :param project:         (str) Project name
+        :param endpoint_id:     (str) Endpoint name
+        """
+        super().__init__(
+            get_monitoring_drift_measures_data(project, endpoint_id),
+            StatsKind.DRIFT_MEASURES.value,
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
+    ) -> "ModelMonitoringDriftMeasuresFile":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
+
+def delete_model_monitoring_stats_folder(project: str) -> None:
+    """Delete the model monitoring schedules folder of the project"""
+    folder = get_monitoring_stats_directory_path(project)
+    fs = mlrun.datastore.store_manager.object(folder).store.filesystem
+    if fs and fs.exists(folder):
+        logger.debug("Deleting model monitoring stats folder", folder=folder)
+        fs.rm(folder, recursive=True)
+    elif fs is None:  # In-memory store
+        raise mlrun.errors.MLRunValueError(
+            "Cannot delete a folder without a file-system"
+        )

mlrun/model_monitoring/db/stores/__init__.py

@@ -70,7 +70,7 @@ class ObjectStoreFactory(enum.Enum):
 
 def get_model_endpoint_store(
     project: str,
-    access_key: str = None,
+    access_key: typing.Optional[str] = None,
     secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
 ) -> StoreBase:
     # Leaving here for backwards compatibility

mlrun/model_monitoring/db/stores/base/store.py

@@ -90,12 +90,12 @@ class StoreBase(ABC):
     @abstractmethod
     def list_model_endpoints(
         self,
-        model: str = None,
-        function: str = None,
-        labels: list[str] = None,
-        top_level: bool = None,
-        uids: list = None,
-        include_stats: bool = None,
+        model: typing.Optional[str] = None,
+        function: typing.Optional[str] = None,
+        labels: typing.Optional[list[str]] = None,
+        top_level: typing.Optional[bool] = None,
+        uids: typing.Optional[list] = None,
+        include_stats: typing.Optional[bool] = None,
     ) -> list[dict[str, typing.Any]]:
         """
         Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
@@ -115,65 +115,6 @@ class StoreBase(ABC):
         """
         pass
 
-    @abstractmethod
-    def write_application_event(
-        self,
-        event: dict[str, typing.Any],
-        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
-    ) -> None:
-        """
-        Write a new event in the target table.
-
-        :param event: An event dictionary that represents the application result, should be corresponded to the
-                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
-                      object.
-        :param kind: The type of the event, can be either "result" or "metric".
-        """
-
-    @abstractmethod
-    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
-        """
-        Get the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-
-        :return: Timestamp as a Unix time.
-        :raise:  MLRunNotFoundError if last analyzed value is not found.
-        """
-        pass
-
-    @abstractmethod
-    def update_last_analyzed(
-        self,
-        endpoint_id: str,
-        application_name: str,
-        last_analyzed: int,
-    ):
-        """
-        Update the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
-                                 application and model endpoint.
-
-        """
-        pass
-
-    @abstractmethod
-    def get_model_endpoint_metrics(
-        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
-    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
-        """
-        Get the model monitoring results and metrics of the requested model endpoint.
-
-        :param: endpoint_id: The model endpoint identifier.
-        :param: type:        The type of the requested metrics ("result" or "metric").
-
-        :return:             A list of the available metrics.
-        """
-
     @staticmethod
     def _validate_labels(
         endpoint_dict: dict,

mlrun/model_monitoring/db/stores/sqldb/models/__init__.py

@@ -15,14 +15,8 @@
 from functools import partial
 from typing import Optional, TypeVar, Union
 
-from .mysql import ApplicationMetricsTable as MySQLApplicationMetricsTable
-from .mysql import ApplicationResultTable as MySQLApplicationResultTable
 from .mysql import ModelEndpointsTable as MySQLModelEndpointsTable
-from .mysql import MonitoringSchedulesTable as MySQLMonitoringSchedulesTable
-from .sqlite import ApplicationMetricsTable as SQLiteApplicationMetricsTable
-from .sqlite import ApplicationResultTable as SQLiteApplicationResultTable
 from .sqlite import ModelEndpointsTable as SQLiteModelEndpointsTable
-from .sqlite import MonitoringSchedulesTable as SQLiteMonitoringSchedulesTable
 
 MySQLTableType = TypeVar("MySQLTableType")
 SQLiteTableType = TypeVar("SQLiteTableType")
@@ -50,22 +44,3 @@ _get_model_endpoints_table = partial(
     mysql_table=MySQLModelEndpointsTable,
     sqlite_table=SQLiteModelEndpointsTable,
 )
-
-
-_get_application_result_table = partial(
-    _get_sql_table,
-    mysql_table=MySQLApplicationResultTable,
-    sqlite_table=SQLiteApplicationResultTable,
-)
-
-_get_application_metrics_table = partial(
-    _get_sql_table,
-    mysql_table=MySQLApplicationMetricsTable,
-    sqlite_table=SQLiteApplicationMetricsTable,
-)
-
-_get_monitoring_schedules_table = partial(
-    _get_sql_table,
-    mysql_table=MySQLMonitoringSchedulesTable,
-    sqlite_table=SQLiteMonitoringSchedulesTable,
-)

mlrun/model_monitoring/db/stores/sqldb/models/base.py

@@ -13,11 +13,9 @@
 # limitations under the License.
 
 from sqlalchemy import (
-    DATETIME,
     TIMESTAMP,  # TODO: migrate to DATETIME, see ML-6921
     Boolean,
     Column,
-    Float,
     Integer,
     String,
     Text,
@@ -25,11 +23,6 @@ from sqlalchemy import (
 
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
-    FileTargetKind,
-    MetricData,
-    ResultData,
-    SchedulingKeys,
-    WriterEvent,
 )
 from mlrun.utils.db import BaseModel
 
@@ -98,93 +91,3 @@ class ModelEndpointsBaseTable(BaseModel):
         EventFieldType.LAST_REQUEST,
         TIMESTAMP(timezone=True),  # TODO: migrate to DATETIME, see ML-6921
     )
-
-
-class ApplicationResultBaseTable(BaseModel):
-    __tablename__ = FileTargetKind.APP_RESULTS
-
-    uid = Column(EventFieldType.UID, String(120), primary_key=True)
-
-    application_name = Column(
-        WriterEvent.APPLICATION_NAME,
-        String(40),
-        nullable=True,
-    )
-
-    endpoint_id = Column(
-        WriterEvent.ENDPOINT_ID,
-        String(40),
-        nullable=True,
-    )
-
-    result_name = Column(
-        ResultData.RESULT_NAME,
-        String(40),
-    )
-
-    start_infer_time = Column(
-        WriterEvent.START_INFER_TIME,
-        DATETIME(timezone=True),
-    )
-    end_infer_time = Column(
-        WriterEvent.END_INFER_TIME,
-        DATETIME(timezone=True),
-    )
-
-    result_status = Column(ResultData.RESULT_STATUS, String(10))
-    result_kind = Column(ResultData.RESULT_KIND, String(40))
-    result_value = Column(ResultData.RESULT_VALUE, Float)
-    result_extra_data = Column(ResultData.RESULT_EXTRA_DATA, Text)
-    current_stats = Column(ResultData.CURRENT_STATS, Text)
-
-
-class ApplicationMetricsBaseTable(BaseModel):
-    __tablename__ = FileTargetKind.APP_METRICS
-
-    uid = Column(EventFieldType.UID, String(120), primary_key=True)
-    application_name = Column(
-        WriterEvent.APPLICATION_NAME,
-        String(40),
-        nullable=True,
-    )
-    endpoint_id = Column(
-        WriterEvent.ENDPOINT_ID,
-        String(40),
-        nullable=True,
-    )
-    start_infer_time = Column(
-        WriterEvent.START_INFER_TIME,
-        DATETIME(timezone=True),
-    )
-    end_infer_time = Column(
-        WriterEvent.END_INFER_TIME,
-        DATETIME(timezone=True),
-    )
-    metric_name = Column(
-        MetricData.METRIC_NAME,
-        String(40),
-    )
-    metric_value = Column(MetricData.METRIC_VALUE, Float)
-
-
-class MonitoringSchedulesBaseTable(BaseModel):
-    __tablename__ = FileTargetKind.MONITORING_SCHEDULES
-
-    uid = Column(SchedulingKeys.UID, String(32), primary_key=True)
-
-    application_name = Column(
-        SchedulingKeys.APPLICATION_NAME,
-        String(40),
-        nullable=False,
-    )
-
-    endpoint_id = Column(
-        SchedulingKeys.ENDPOINT_ID,
-        String(40),
-        nullable=False,
-    )
-
-    last_analyzed = Column(
-        SchedulingKeys.LAST_ANALYZED,
-        Integer,
-    )

mlrun/model_monitoring/db/stores/sqldb/models/mysql.py

@@ -13,20 +13,15 @@
 # limitations under the License.
 
 import sqlalchemy.dialects.mysql
-from sqlalchemy import Column, ForeignKey, String
-from sqlalchemy.ext.declarative import declarative_base, declared_attr
+from sqlalchemy import Column
+from sqlalchemy.ext.declarative import declarative_base
 
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
-    ResultData,
-    WriterEvent,
 )
 
 from .base import (
-    ApplicationMetricsBaseTable,
-    ApplicationResultBaseTable,
     ModelEndpointsBaseTable,
-    MonitoringSchedulesBaseTable,
 )
 
 Base = declarative_base()
@@ -50,54 +45,3 @@ class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
         # TODO: migrate to DATETIME, see ML-6921
         sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
     )
-
-
-class _ApplicationResultOrMetric:
-    """
-    This class sets common columns of `ApplicationResultTable` and `ApplicationMetricsTable`
-    to the correct values in MySQL.
-    Note: This class must come before the base tables in the inheritance order to override
-    the relevant columns.
-    """
-
-    start_infer_time = Column(
-        WriterEvent.START_INFER_TIME,
-        sqlalchemy.dialects.mysql.DATETIME(fsp=3, timezone=True),
-    )
-    end_infer_time = Column(
-        WriterEvent.END_INFER_TIME,
-        sqlalchemy.dialects.mysql.DATETIME(fsp=3, timezone=True),
-    )
-
-    @declared_attr
-    def endpoint_id(self):
-        return Column(
-            String(40),
-            ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),
-        )
-
-
-class ApplicationResultTable(
-    Base, _ApplicationResultOrMetric, ApplicationResultBaseTable
-):
-    result_extra_data = Column(
-        ResultData.RESULT_EXTRA_DATA, sqlalchemy.dialects.mysql.MEDIUMTEXT
-    )
-    current_stats = Column(
-        ResultData.CURRENT_STATS, sqlalchemy.dialects.mysql.MEDIUMTEXT
-    )
-
-
-class ApplicationMetricsTable(
-    Base, _ApplicationResultOrMetric, ApplicationMetricsBaseTable
-):
-    pass
-
-
-class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
-    @declared_attr
-    def endpoint_id(self):
-        return Column(
-            String(40),
-            ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),
-        )

mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py

@@ -15,10 +15,7 @@
 from sqlalchemy.ext.declarative import declarative_base
 
 from .base import (
-    ApplicationMetricsBaseTable,
-    ApplicationResultBaseTable,
     ModelEndpointsBaseTable,
-    MonitoringSchedulesBaseTable,
 )
 
 Base = declarative_base()
@@ -26,15 +23,3 @@ Base = declarative_base()
 
 class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
     pass
-
-
-class ApplicationResultTable(Base, ApplicationResultBaseTable):
-    pass
-
-
-class ApplicationMetricsTable(Base, ApplicationMetricsBaseTable):
-    pass
-
-
-class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
-    pass