mlrun 1.10.0rc13__py3-none-any.whl → 1.10.0rc42__py3-none-any.whl

mlrun/model_monitoring/db/_schedules.py

@@ -13,25 +13,36 @@
 # limitations under the License.
 
 import json
+import sys
 from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
+from datetime import datetime
 from types import TracebackType
-from typing import Final, Optional
+from typing import TYPE_CHECKING, Final, Optional
 
 import botocore.exceptions
 
+import mlrun
 import mlrun.common.schemas as schemas
 import mlrun.errors
 import mlrun.model_monitoring.helpers
+import mlrun.utils.helpers
 from mlrun.utils import logger
 
+if TYPE_CHECKING:
+    if sys.version_info >= (3, 11):
+        from typing import Self
+    else:
+        from typing_extensions import Self
+
 
 class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
     DEFAULT_SCHEDULES: Final = {}
     INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
     ENCODING = "utf-8"
 
-    def __init__(self):
+    def __init__(self) -> None:
+        # `self._item` is the persistent version of the monitoring schedules.
         self._item = self.get_data_item_object()
         if self._item:
             self._path = self._item.url
@@ -43,9 +54,16 @@ class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
             self._open_schedules = False
 
     @abstractmethod
-    def get_data_item_object(self) -> mlrun.DataItem:
+    def get_data_item_object(self) -> "mlrun.DataItem":
         pass
 
+    def _exists(self) -> bool:
+        """Return whether the file exists or not"""
+        return (
+            self._fs is None  # In-memory store
+            or self._fs.exists(self._path)
+        )
+
     def create(self) -> None:
         """Create a schedules file with initial content - an empty dictionary"""
         logger.debug("Creating model monitoring schedules file", path=self._item.url)
@@ -53,10 +71,7 @@ class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
 
     def delete(self) -> None:
         """Delete schedules file if it exists"""
-        if (
-            self._fs is None  # In-memory store
-            or self._fs.exists(self._path)
-        ):
+        if self._exists():
             logger.debug(
                 "Deleting model monitoring schedules file", path=self._item.url
             )
@@ -100,7 +115,7 @@ class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
         self._schedules = self.DEFAULT_SCHEDULES
         self._open_schedules = False
 
-    def __enter__(self) -> "ModelMonitoringSchedulesFileBase":
+    def __enter__(self) -> "Self":
         self._open()
         return super().__enter__()
 
@@ -129,12 +144,11 @@ class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
         :param project:     The project name.
         :param endpoint_id: The endpoint ID.
         """
-        # `self._item` is the persistent version of the monitoring schedules.
         self._project = project
         self._endpoint_id = endpoint_id
         super().__init__()
 
-    def get_data_item_object(self) -> mlrun.DataItem:
+    def get_data_item_object(self) -> "mlrun.DataItem":
         return mlrun.model_monitoring.helpers.get_monitoring_schedules_endpoint_data(
             project=self._project, endpoint_id=self._endpoint_id
         )
@@ -148,19 +162,29 @@ class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
             endpoint_id=model_endpoint.metadata.uid,
         )
 
-    def get_application_time(self, application: str) -> Optional[int]:
+    def get_application_time(self, application: str) -> Optional[float]:
         self._check_open_schedules()
         return self._schedules.get(application)
 
-    def update_application_time(self, application: str, timestamp: int) -> None:
+    def update_application_time(self, application: str, timestamp: float) -> None:
         self._check_open_schedules()
-        self._schedules[application] = timestamp
+        self._schedules[application] = float(timestamp)
+
+    def delete_application_time(self, application: str) -> None:
+        self._check_open_schedules()
+        if application in self._schedules:
+            logger.debug(
+                "Deleting application time from schedules",
+                application=application,
+                endpoint_id=self._endpoint_id,
+            )
+            del self._schedules[application]
 
     def get_application_list(self) -> set[str]:
         self._check_open_schedules()
         return set(self._schedules.keys())
 
-    def get_min_timestamp(self) -> Optional[int]:
+    def get_min_timestamp(self) -> Optional[float]:
         self._check_open_schedules()
         return min(self._schedules.values(), default=None)
 
@@ -179,12 +203,12 @@ class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
         self._project = project
         super().__init__()
 
-    def get_data_item_object(self) -> mlrun.DataItem:
+    def get_data_item_object(self) -> "mlrun.DataItem":
         return mlrun.model_monitoring.helpers.get_monitoring_schedules_chief_data(
             project=self._project
         )
 
-    def get_endpoint_last_request(self, endpoint_uid: str) -> Optional[int]:
+    def get_endpoint_last_request(self, endpoint_uid: str) -> Optional[float]:
         self._check_open_schedules()
         if endpoint_uid in self._schedules:
             return self._schedules[endpoint_uid].get(
@@ -194,15 +218,19 @@ class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
             return None
 
     def update_endpoint_timestamps(
-        self, endpoint_uid: str, last_request: int, last_analyzed: int
+        self, endpoint_uid: str, last_request: float, last_analyzed: float
     ) -> None:
         self._check_open_schedules()
         self._schedules[endpoint_uid] = {
-            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST: last_request,
-            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED: last_analyzed,
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST: float(
+                last_request
+            ),
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED: float(
+                last_analyzed
+            ),
         }
 
-    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[int]:
+    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[float]:
         self._check_open_schedules()
         if endpoint_uid in self._schedules:
             return self._schedules[endpoint_uid].get(
@@ -216,22 +244,58 @@ class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
         return set(self._schedules.keys())
 
     def get_or_create(self) -> None:
-        try:
-            self._open()
-        except (
-            mlrun.errors.MLRunNotFoundError,
-            # Different errors are raised for S3 or local storage, see ML-8042
-            botocore.exceptions.ClientError,
-            FileNotFoundError,
-        ):
+        if not self._exists():
             self.create()
 
 
-def delete_model_monitoring_schedules_folder(project: str) -> None:
-    """Delete the model monitoring schedules folder of the project"""
-    folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(
-        project
-    )
+class ModelMonitoringSchedulesFileApplication(ModelMonitoringSchedulesFileBase):
+    def __init__(self, out_path: str, application: str) -> None:
+        self._out_path = out_path
+        self._application = application
+        super().__init__()
+
+    def get_data_item_object(self) -> "mlrun.DataItem":
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_user_application_data(
+            out_path=self._out_path, application=self._application
+        )
+
+    def _open(self) -> None:
+        if not self._exists():
+            # Create the file when it is needed the first time
+            logger.info(
+                "Creating the application schedules file",
+                application=self._application,
+                path=self._path,
+            )
+            self.create()
+        super()._open()
+
+    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[datetime]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return datetime.fromisoformat(self._schedules[endpoint_uid])
+        else:
+            return None
+
+    def update_endpoint_last_analyzed(
+        self, endpoint_uid: str, last_analyzed: datetime
+    ) -> None:
+        self._check_open_schedules()
+        self._schedules[endpoint_uid] = last_analyzed.isoformat()
+
+    def delete_endpoints_last_analyzed(self, endpoint_uids: list[str]) -> None:
+        self._check_open_schedules()
+        for endpoint_uid in endpoint_uids:
+            if endpoint_uid in self._schedules:
+                logger.debug(
+                    "Deleting endpoint last analyzed from schedules",
+                    endpoint_uid=endpoint_uid,
+                    application=self._application,
+                )
+                del self._schedules[endpoint_uid]
+
+
+def _delete_folder(folder: str) -> None:
     fs = mlrun.datastore.store_manager.object(folder).store.filesystem
     if fs and fs.exists(folder):
         logger.debug("Deleting model monitoring schedules folder", folder=folder)
@@ -240,3 +304,22 @@ def delete_model_monitoring_schedules_folder(project: str) -> None:
         raise mlrun.errors.MLRunValueError(
             "Cannot delete a folder without a file-system"
         )
+
+
+def delete_model_monitoring_schedules_folder(project: str) -> None:
+    """Delete the model monitoring schedules folder of the project"""
+    folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(
+        project
+    )
+    _delete_folder(folder)
+
+
+def delete_model_monitoring_schedules_user_folder(project: str) -> None:
+    """Delete the user created schedules folder (created through `app.evaluate`)"""
+    out_path = mlrun.utils.helpers.template_artifact_path(
+        mlrun.mlconf.artifact_path, project=project
+    )
+    folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_user_folder_path(
+        out_path
+    )
+    _delete_folder(folder)

mlrun/model_monitoring/db/_stats.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import abc
 import json
+import typing
 from abc import abstractmethod
 from datetime import datetime, timezone
 from typing import cast
@@ -73,7 +74,7 @@ class ModelMonitoringStatsFile(abc.ABC):
                 path=self._item.url,
             )
 
-    def read(self) -> tuple[dict, datetime]:
+    def read(self) -> tuple[dict, typing.Optional[datetime]]:
         """
         Read the stats data and timestamp saved in file
         :return: tuple[dict, str] dictionary with stats data and timestamp saved in file
@@ -99,13 +100,13 @@ class ModelMonitoringStatsFile(abc.ABC):
             ):
                 raise
 
-            logger.exception(
+            logger.warning(
                 "The Stats file was not found. It should have been created "
                 "as a part of the model endpoint's creation",
                 path=self._path,
                 error=err,
             )
-            raise
+            return {}, None
 
     def write(self, stats: dict, timestamp: datetime) -> None:
         """

mlrun/model_monitoring/db/tsdb/base.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 from abc import ABC, abstractmethod
-from datetime import datetime
-from typing import Callable, ClassVar, Literal, Optional, Union
+from datetime import datetime, timedelta
+from typing import ClassVar, Literal, Optional, Union
 
 import pandas as pd
 import pydantic.v1
@@ -60,6 +60,16 @@ class TSDBConnector(ABC):
         """
         pass
 
+    def apply_writer_steps(self, graph, after, **kwargs) -> None:
+        """
+        Apply TSDB steps on the provided writer graph. Throughout these steps, the graph stores metrics / results.
+        This data is being used by mlrun UI and the monitoring dashboards in grafana.
+        There are 2 different key metric dictionaries that are being generated throughout these steps:
+        - metrics (user-defined metrics) - model monitoring application metrics
+        - results (user-defined results) - model monitoring application results
+        """
+        pass
+
     @abstractmethod
     def handle_model_error(self, graph, **kwargs) -> None:
         """
@@ -81,14 +91,38 @@ class TSDBConnector(ABC):
         """
 
     @abstractmethod
-    def delete_tsdb_records(
+    def get_drift_data(
         self,
-        endpoint_ids: list[str],
-    ) -> None:
+        start: datetime,
+        end: datetime,
+    ) -> mm_schemas.ModelEndpointDriftValues:
+        """
+        Fetches drift counts per interval in the specified time range.
+
+        :param start: The start time of the query.
+        :param end:   The end time of the query.
+
+        :return: A ModelEndpointDriftValues object containing drift data.
+        """
+
+    @abstractmethod
+    def delete_tsdb_records(self, endpoint_ids: list[str]) -> None:
         """
         Delete model endpoint records from the TSDB connector.
+
         :param endpoint_ids: List of model endpoint unique identifiers.
-        :param delete_timeout: The timeout in seconds to wait for the deletion to complete.
+        """
+        pass
+
+    @abstractmethod
+    def delete_application_records(
+        self, application_name: str, endpoint_ids: Optional[list[str]] = None
+    ) -> None:
+        """
+        Delete application records from the TSDB for the given model endpoints or all if ``None``.
+
+        :param application_name: The name of the application to delete records for.
+        :param endpoint_ids:     List of model endpoint unique identifiers.
         """
         pass
 
@@ -410,11 +444,9 @@ class TSDBConnector(ABC):
                                    ]
         """
 
-    async def add_basic_metrics(
+    def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
-        project: str,
-        run_in_threadpool: Callable,
         metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         raise NotImplementedError()
@@ -703,3 +735,62 @@ class TSDBConnector(ABC):
                 )
             )
         return {dict_key: metrics}
+
+    @staticmethod
+    def _prepare_aligned_start_end(
+        start: datetime, end: datetime
+    ) -> tuple[datetime, datetime, str]:
+        delta = end - start
+        if delta <= timedelta(hours=6):
+            interval = "10m"
+            start = start.replace(
+                minute=start.minute // 10 * 10, second=0, microsecond=0
+            )
+        elif delta <= timedelta(hours=72):
+            interval = "1h"
+            start = start.replace(minute=0, second=0, microsecond=0)
+        else:
+            interval = "1d"
+            start = start.replace(hour=0, minute=0, second=0, microsecond=0)
+
+        interval_map = {
+            "10m": timedelta(minutes=10),
+            "1h": timedelta(hours=1),
+            "1d": timedelta(days=1),
+        }
+        delta = end - start
+        interval_td = interval_map[interval]
+        end = start + (delta // interval_td) * interval_td
+        return start, end, interval
+
+    @staticmethod
+    def _df_to_drift_data(df: pd.DataFrame) -> mm_schemas.ModelEndpointDriftValues:
+        suspected_val = mm_schemas.constants.ResultStatusApp.potential_detection.value
+        detected_val = mm_schemas.constants.ResultStatusApp.detected.value
+        aggregated_df = (
+            df.groupby(["_wstart", f"max({mm_schemas.ResultData.RESULT_STATUS})"])
+            .size()  # add size column for each interval x result-status combination
+            .unstack()  # create a size column for each result-status
+            .reindex(
+                columns=[suspected_val, detected_val], fill_value=0
+            )  # ensure both columns exists
+            .fillna(0)
+            .astype(int)
+            .rename(
+                columns={
+                    suspected_val: "count_suspected",
+                    detected_val: "count_detected",
+                }
+            )
+        )
+        values = list(
+            zip(
+                aggregated_df.index,
+                aggregated_df["count_suspected"],
+                aggregated_df["count_detected"],
+            )
+        )
+        return mm_schemas.ModelEndpointDriftValues(values=values)
+
+    def add_pre_writer_steps(self, graph, after):
+        return None

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -122,10 +122,7 @@ class TDEngineSchema:
             )
         return f"DELETE FROM {self.database}.{subtable} WHERE {values};"
 
-    def drop_subtable_query(
-        self,
-        subtable: str,
-    ) -> str:
+    def drop_subtable_query(self, subtable: str) -> str:
         return f"DROP TABLE if EXISTS {self.database}.`{subtable}`;"
 
     def drop_supertable_query(self) -> str:
@@ -145,8 +142,10 @@ class TDEngineSchema:
         values = f" {operator} ".join(
             f"{filter_tag} LIKE '{val}'" for val in filter_values
         )
+        return self._get_tables_query_by_condition(values)
 
-        return f"SELECT DISTINCT tbname FROM {self.database}.{self.super_table} WHERE {values};"
+    def _get_tables_query_by_condition(self, condition: str) -> str:
+        return f"SELECT DISTINCT TBNAME FROM {self.database}.{self.super_table} WHERE {condition};"
 
     @staticmethod
     def _get_records_query(
@@ -165,6 +164,7 @@ class TDEngineSchema:
         preform_agg_funcs_columns: Optional[list[str]] = None,
         order_by: Optional[str] = None,
         desc: Optional[bool] = None,
+        partition_by: Optional[str] = None,
     ) -> str:
         if agg_funcs and not columns_to_filter:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -176,7 +176,10 @@ class TDEngineSchema:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "`agg_funcs` must be provided when using interval"
             )
-
+        if partition_by and not agg_funcs:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "`agg_funcs` must be provided when using partition by"
+            )
         if sliding_window_step and not interval:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "`interval` must be provided when using sliding window"
@@ -232,6 +235,8 @@ class TDEngineSchema:
                 if isinstance(group_by, list):
                     group_by = ", ".join(group_by)
                 query.write(f" GROUP BY {group_by}")
+            if partition_by:
+                query.write(f" PARTITION BY {partition_by}")
             if order_by:
                 desc = " DESC" if desc else ""
                 query.write(f" ORDER BY {order_by}{desc}")