mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/model_monitoring/tracking_policy.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
+import warnings
 from typing import Union
 
 import mlrun.common.schemas.schedule
@@ -55,6 +55,12 @@ class TrackingPolicy(mlrun.model.ModelObj):
                                             writer function, which is a real time nuclio functino, will be deployed
                                             with the same image. By default, the image is mlrun/mlrun.
         """
+        warnings.warn(
+            "The `TrackingPolicy` class is deprecated from version 1.7.0 and is not "
+            "used anymore. It will be removed in 1.9.0.",
+            FutureWarning,
+        )
+
         if isinstance(default_batch_intervals, str):
             default_batch_intervals = (
                 mlrun.common.schemas.schedule.ScheduleCronTrigger.from_crontab(
@@ -96,12 +102,13 @@ class TrackingPolicy(mlrun.model.ModelObj):
                 )
         return new_obj
 
-    def to_dict(self, fields=None, exclude=None):
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         struct = super().to_dict(
             fields,
             exclude=[
                 mlrun.common.schemas.model_monitoring.EventFieldType.DEFAULT_BATCH_INTERVALS
             ],
+            strip=strip,
         )
         if self.default_batch_intervals:
             struct[

mlrun/model_monitoring/writer.py

@@ -12,29 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import datetime
 import json
-from http import HTTPStatus
-from typing import Any, NewType
-
-import pandas as pd
-from v3io.dataplane import Client as V3IOClient
-from v3io_frames.client import ClientBase as V3IOFramesClient
-from v3io_frames.errors import Error as V3IOFramesError
-from v3io_frames.frames_pb2 import IGNORE
+from typing import Any, Callable, NewType
 
 import mlrun.common.model_monitoring
+import mlrun.common.schemas
+import mlrun.common.schemas.alert as alert_objects
 import mlrun.model_monitoring
-import mlrun.utils.v3io_clients
-from mlrun.common.schemas.model_monitoring.constants import ResultStatusApp, WriterEvent
+from mlrun.common.schemas.model_monitoring.constants import (
+    EventFieldType,
+    HistogramDataDriftApplicationConstants,
+    MetricData,
+    ResultData,
+    ResultKindApp,
+    ResultStatusApp,
+    WriterEvent,
+    WriterEventKind,
+)
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
+from mlrun.model_monitoring.helpers import get_result_instance_fqn
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 from mlrun.utils.notifications.notification_pusher import CustomNotificationPusher
 
-_TSDB_BE = "tsdb"
-_TSDB_RATE = "1/s"
-_TSDB_TABLE = "app-results"
 _RawEvent = dict[str, Any]
 _AppResultEvent = NewType("_AppResultEvent", _RawEvent)
 
@@ -69,20 +69,20 @@ class _Notifier:
         self._severity = severity
 
     def _should_send_event(self) -> bool:
-        return self._event[WriterEvent.RESULT_STATUS] >= ResultStatusApp.detected
+        return self._event[ResultData.RESULT_STATUS] >= ResultStatusApp.detected.value
 
     def _generate_message(self) -> str:
         return f"""\
 The monitoring app `{self._event[WriterEvent.APPLICATION_NAME]}` \
-of kind `{self._event[WriterEvent.RESULT_KIND]}` \
+of kind `{self._event[ResultData.RESULT_KIND]}` \
 detected a problem in model endpoint ID `{self._event[WriterEvent.ENDPOINT_ID]}` \
 at time `{self._event[WriterEvent.START_INFER_TIME]}`.
 
 Result data:
-Name: `{self._event[WriterEvent.RESULT_NAME]}`
-Value: `{self._event[WriterEvent.RESULT_VALUE]}`
-Status: `{self._event[WriterEvent.RESULT_STATUS]}`
-Extra data: `{self._event[WriterEvent.RESULT_EXTRA_DATA]}`\
+Name: `{self._event[ResultData.RESULT_NAME]}`
+Value: `{self._event[ResultData.RESULT_VALUE]}`
+Status: `{self._event[ResultData.RESULT_STATUS]}`
+Extra data: `{self._event[ResultData.RESULT_EXTRA_DATA]}`\
 """
 
     def notify(self) -> None:
@@ -97,140 +97,176 @@ Extra data: `{self._event[WriterEvent.RESULT_EXTRA_DATA]}`\
 
 class ModelMonitoringWriter(StepToDict):
     """
-    Write monitoring app events to V3IO KV storage
+    Write monitoring application results to the target databases
     """
 
     kind = "monitoring_application_stream_pusher"
 
-    def __init__(self, project: str) -> None:
+    def __init__(
+        self,
+        project: str,
+        secret_provider: Callable = None,
+    ) -> None:
         self.project = project
         self.name = project  # required for the deployment process
-        self._v3io_container = self.get_v3io_container(self.name)
-        self._kv_client = self._get_v3io_client().kv
-        self._tsdb_client = self._get_v3io_frames_client(self._v3io_container)
+
         self._custom_notifier = CustomNotificationPusher(
             notification_types=[NotificationKind.slack]
         )
-        self._create_tsdb_table()
-        self._kv_schemas = []
-
-    @staticmethod
-    def get_v3io_container(project_name: str) -> str:
-        return f"users/pipelines/{project_name}/monitoring-apps"
 
-    @staticmethod
-    def _get_v3io_client() -> V3IOClient:
-        return mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
+        self._app_result_store = mlrun.model_monitoring.get_store_object(
+            project=self.project, secret_provider=secret_provider
         )
-
-    @staticmethod
-    def _get_v3io_frames_client(v3io_container: str) -> V3IOFramesClient:
-        return mlrun.utils.v3io_clients.get_frames_client(
-            address=mlrun.mlconf.v3io_framesd,
-            container=v3io_container,
+        self._tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project, secret_provider=secret_provider
         )
+        self._endpoints_records = {}
 
-    def _create_tsdb_table(self) -> None:
-        self._tsdb_client.create(
-            backend=_TSDB_BE,
-            table=_TSDB_TABLE,
-            if_exists=IGNORE,
-            rate=_TSDB_RATE,
+    def _generate_event_on_drift(
+        self,
+        entity_id: str,
+        result_status: int,
+        event_value: dict,
+        project_name: str,
+        result_kind: int,
+    ) -> None:
+        entity = mlrun.common.schemas.alert.EventEntities(
+            kind=alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT,
+            project=project_name,
+            ids=[entity_id],
         )
 
-    def _update_kv_db(self, event: _AppResultEvent) -> None:
-        event = _AppResultEvent(event.copy())
-        endpoint_id = event.pop(WriterEvent.ENDPOINT_ID)
-        app_name = event.pop(WriterEvent.APPLICATION_NAME)
-        metric_name = event.pop(WriterEvent.RESULT_NAME)
-        attributes = {metric_name: json.dumps(event)}
-        self._kv_client.update(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=app_name,
-            attributes=attributes,
+        event_kind = self._generate_alert_event_kind(
+            result_status=result_status, result_kind=result_kind
         )
-        if endpoint_id not in self._kv_schemas:
-            self._generate_kv_schema(endpoint_id)
-        logger.info("Updated V3IO KV successfully", key=app_name)
-
-    def _generate_kv_schema(self, endpoint_id: str):
-        """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
-        fields = [
-            {"name": WriterEvent.RESULT_NAME, "type": "string", "nullable": False}
-        ]
-        res = self._kv_client.create_schema(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=WriterEvent.APPLICATION_NAME,
-            fields=fields,
+
+        event_data = mlrun.common.schemas.Event(
+            kind=alert_objects.EventKind(value=event_kind),
+            entity=entity,
+            value_dict=event_value,
         )
-        if res.status_code != HTTPStatus.OK.value:
-            raise mlrun.errors.MLRunBadRequestError(
-                f"Couldn't infer schema for endpoint {endpoint_id} which is required for Grafana dashboards"
-            )
-        else:
-            logger.info(
-                "Generated V3IO KV schema successfully", endpoint_id=endpoint_id
-            )
-            self._kv_schemas.append(endpoint_id)
+        logger.info("Sending a drift event")
+        mlrun.get_run_db().generate_event(event_kind, event_data)
+        logger.info("Drift event sent successfully")
 
-    def _update_tsdb(self, event: _AppResultEvent) -> None:
-        event = _AppResultEvent(event.copy())
-        event[WriterEvent.END_INFER_TIME] = datetime.datetime.fromisoformat(
-            event[WriterEvent.END_INFER_TIME]
+    @staticmethod
+    def _generate_alert_event_kind(
+        result_kind: int, result_status: int
+    ) -> alert_objects.EventKind:
+        """Generate the required Event Kind format for the alerting system"""
+        event_kind = ResultKindApp(value=result_kind).name
+
+        if result_status == ResultStatusApp.detected.value:
+            event_kind = f"{event_kind}_detected"
+        else:
+            event_kind = f"{event_kind}_suspected"
+        return alert_objects.EventKind(
+            value=mlrun.utils.helpers.normalize_name(event_kind)
         )
-        del event[WriterEvent.RESULT_EXTRA_DATA]
-        try:
-            self._tsdb_client.write(
-                backend=_TSDB_BE,
-                table=_TSDB_TABLE,
-                dfs=pd.DataFrame.from_records([event]),
-                index_cols=[
-                    WriterEvent.END_INFER_TIME,
-                    WriterEvent.ENDPOINT_ID,
-                    WriterEvent.APPLICATION_NAME,
-                    WriterEvent.RESULT_NAME,
-                ],
-            )
-            logger.info("Updated V3IO TSDB successfully", table=_TSDB_TABLE)
-        except V3IOFramesError as err:
-            logger.warn(
-                "Could not write drift measures to TSDB",
-                err=err,
-                table=_TSDB_TABLE,
-                event=event,
-            )
 
     @staticmethod
-    def _reconstruct_event(event: _RawEvent) -> _AppResultEvent:
+    def _reconstruct_event(event: _RawEvent) -> tuple[_AppResultEvent, WriterEventKind]:
         """
         Modify the raw event into the expected monitoring application event
         schema as defined in `mlrun.common.schemas.model_monitoring.constants.WriterEvent`
         """
-        try:
-            result_event = _AppResultEvent(
-                {key: event[key] for key in WriterEvent.list()}
+        if not isinstance(event, dict):
+            raise _WriterEventTypeError(
+                f"The event is of type: {type(event)}, expected a dictionary"
             )
-            result_event[WriterEvent.CURRENT_STATS] = json.loads(
-                event[WriterEvent.CURRENT_STATS]
+        kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
+        result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
+        if not result_event:  # BC for < 1.7.0, can be removed in 1.9.0
+            result_event = _AppResultEvent(event)
+        else:
+            result_event.update(_AppResultEvent(event))
+
+        expected_keys = list(
+            set(WriterEvent.list()).difference(
+                [WriterEvent.EVENT_KIND, WriterEvent.DATA]
             )
-            return result_event
-        except KeyError as err:
+        )
+        if kind == WriterEventKind.METRIC:
+            expected_keys.extend(MetricData.list())
+        elif kind == WriterEventKind.RESULT:
+            expected_keys.extend(ResultData.list())
+        else:
             raise _WriterEventValueError(
-                "The received event misses some keys compared to the expected "
-                "monitoring application event schema"
-            ) from err
-        except TypeError as err:
-            raise _WriterEventTypeError(
-                f"The event is of type: {type(event)}, expected a dictionary"
-            ) from err
+                f"Unknown event kind: {kind}, expected one of: {WriterEventKind.list()}"
+            )
+        missing_keys = [key for key in expected_keys if key not in result_event]
+        if missing_keys:
+            raise _WriterEventValueError(
+                f"The received event misses some keys compared to the expected "
+                f"monitoring application event schema: {missing_keys}"
+            )
+
+        return result_event, kind
 
     def do(self, event: _RawEvent) -> None:
-        event = self._reconstruct_event(event)
+        event, kind = self._reconstruct_event(event)
         logger.info("Starting to write event", event=event)
-        self._update_tsdb(event)
-        self._update_kv_db(event)
-        _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+        self._tsdb_connector.write_application_event(event=event.copy(), kind=kind)
+        self._app_result_store.write_application_event(event=event.copy(), kind=kind)
+
         logger.info("Completed event DB writes")
+
+        if kind == WriterEventKind.RESULT:
+            _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+
+        if (
+            mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled
+            and kind == WriterEventKind.RESULT
+            and (
+                event[ResultData.RESULT_STATUS] == ResultStatusApp.detected.value
+                or event[ResultData.RESULT_STATUS]
+                == ResultStatusApp.potential_detection.value
+            )
+        ):
+            endpoint_id = event[WriterEvent.ENDPOINT_ID]
+            endpoint_record = self._endpoints_records.setdefault(
+                endpoint_id,
+                self._app_result_store.get_model_endpoint(endpoint_id=endpoint_id),
+            )
+            event_value = {
+                "app_name": event[WriterEvent.APPLICATION_NAME],
+                "model": endpoint_record.get(EventFieldType.MODEL),
+                "model_endpoint_id": event[WriterEvent.ENDPOINT_ID],
+                "result_name": event[ResultData.RESULT_NAME],
+                "result_value": event[ResultData.RESULT_VALUE],
+            }
+            self._generate_event_on_drift(
+                entity_id=get_result_instance_fqn(
+                    event[WriterEvent.ENDPOINT_ID],
+                    event[WriterEvent.APPLICATION_NAME],
+                    event[ResultData.RESULT_NAME],
+                ),
+                result_status=event[ResultData.RESULT_STATUS],
+                event_value=event_value,
+                project_name=self.project,
+                result_kind=event[ResultData.RESULT_KIND],
+            )
+
+        if (
+            kind == WriterEventKind.RESULT
+            and event[WriterEvent.APPLICATION_NAME]
+            == HistogramDataDriftApplicationConstants.NAME
+            and event[ResultData.RESULT_NAME]
+            == HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME
+        ):
+            endpoint_id = event[WriterEvent.ENDPOINT_ID]
+            logger.info(
+                "Updating the model endpoint with metadata specific to the histogram "
+                "data drift app",
+                endpoint_id=endpoint_id,
+            )
+            attributes = json.loads(event[ResultData.RESULT_EXTRA_DATA])
+            attributes[EventFieldType.DRIFT_STATUS] = str(
+                attributes[EventFieldType.DRIFT_STATUS]
+            )
+            self._app_result_store.update_model_endpoint(
+                endpoint_id=endpoint_id,
+                attributes=attributes,
+            )
+
+        logger.info("Model monitoring writer finished handling event")

mlrun/package/__init__.py

@@ -18,7 +18,7 @@
 import functools
 import inspect
 from collections import OrderedDict
-from typing import Callable, Dict, List, Type, Union
+from typing import Callable, Union
 
 from ..config import config
 from .context_handler import ContextHandler
@@ -40,9 +40,9 @@ from .utils import (
 
 
 def handler(
-    labels: Dict[str, str] = None,
-    outputs: List[Union[str, Dict[str, str]]] = None,
-    inputs: Union[bool, Dict[str, Union[str, Type]]] = True,
+    labels: dict[str, str] = None,
+    outputs: list[Union[str, dict[str, str]]] = None,
+    inputs: Union[bool, dict[str, Union[str, type]]] = True,
 ):
     """
     MLRun's handler is a decorator to wrap a function and enable setting labels, parsing inputs (`mlrun.DataItem`) using
@@ -58,7 +58,7 @@ def handler(
                     * `str` - A string in the format of '{key}:{artifact_type}'. If a string was given without ':' it
                       will indicate the key, and the artifact type will be according to the returned value type's
                       default artifact type. The artifact types supported are listed in the relevant type packager.
-                    * `Dict[str, str]` - A dictionary of logging configuration. the key 'key' is mandatory for the
+                    * `dict[str, str]` - A dictionary of logging configuration. the key 'key' is mandatory for the
                       logged artifact key.
                     * None - Do not log the output.
 
@@ -73,7 +73,7 @@ def handler(
                    * True - Parse all found inputs to the assigned type hint in the function's signature. If there is no
                      type hint assigned, the value will remain an `mlrun.DataItem`.
                    * False - Do not parse inputs, leaving the inputs as `mlrun.DataItem`.
-                   * Dict[str, Union[Type, str]] - A dictionary with argument name as key and the expected type to parse
+                   * dict[str, Union[Type, str]] - A dictionary with argument name as key and the expected type to parse
                      the `mlrun.DataItem` to. The expected type can be a string as well, idicating the full module path.
 
                    Default: True - meaning inputs will be parsed from DataItem's as long as they are type hinted.

mlrun/package/context_handler.py

@@ -15,7 +15,7 @@
 import inspect
 import os
 from collections import OrderedDict
-from typing import Dict, List, Union
+from typing import Union
 
 from mlrun.datastore import DataItem
 from mlrun.errors import MLRunInvalidArgumentError
@@ -181,7 +181,7 @@ class ContextHandler:
     def log_outputs(
         self,
         outputs: list,
-        log_hints: List[Union[Dict[str, str], str, None]],
+        log_hints: list[Union[dict[str, str], str, None]],
     ):
         """
         Log the given outputs as artifacts (or results) with the stored context. Errors raised during the packing will
@@ -229,7 +229,7 @@ class ContextHandler:
         # Clear packagers outputs:
         self._packagers_manager.clear_packagers_outputs()
 
-    def set_labels(self, labels: Dict[str, str]):
+    def set_labels(self, labels: dict[str, str]):
         """
         Set the given labels with the stored context.
 
@@ -239,7 +239,7 @@ class ContextHandler:
             self._context.set_label(key=key, value=value)
 
     def _collect_packagers(
-        self, packagers: List[str], is_mandatory: bool, is_custom_packagers: bool
+        self, packagers: list[str], is_mandatory: bool, is_custom_packagers: bool
     ):
         """
         Collect packagers with the stored manager. The collection can ignore errors raised by setting the mandatory flag
@@ -310,7 +310,7 @@ class ContextHandler:
     def _validate_objects_to_log_hints_length(
         self,
         outputs: list,
-        log_hints: List[Union[Dict[str, str], str, None]],
+        log_hints: list[Union[dict[str, str], str, None]],
     ):
         """
         Validate the outputs and log hints are the same length. If they are not, warnings will be printed on what will

mlrun/package/packager.py

@@ -14,7 +14,7 @@
 #
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, List, Tuple, Type, Union
+from typing import Any, Union
 
 from mlrun.artifacts import Artifact
 from mlrun.datastore import DataItem
@@ -93,7 +93,7 @@ class Packager(ABC):
     """
 
     #: The type of object this packager can pack and unpack.
-    PACKABLE_OBJECT_TYPE: Type = ...
+    PACKABLE_OBJECT_TYPE: type = ...
 
     #: The priority of this packager in the packagers collection of the manager (lower is better).
     PRIORITY: int = ...
@@ -104,7 +104,7 @@ class Packager(ABC):
         self._priority = Packager.PRIORITY
 
         # List of all paths to be deleted by the manager of this packager after logging the packages:
-        self._future_clearing_path_list: List[str] = []
+        self._future_clearing_path_list: list[str] = []
 
     @abstractmethod
     def get_default_packing_artifact_type(self, obj: Any) -> str:
@@ -132,7 +132,7 @@ class Packager(ABC):
         pass
 
     @abstractmethod
-    def get_supported_artifact_types(self) -> List[str]:
+    def get_supported_artifact_types(self) -> list[str]:
         """
         Get all the supported artifact types on this packager.
 
@@ -147,7 +147,7 @@ class Packager(ABC):
         key: str = None,
         artifact_type: str = None,
         configurations: dict = None,
-    ) -> Union[Tuple[Artifact, dict], dict]:
+    ) -> Union[tuple[Artifact, dict], dict]:
         """
         Pack an object as the given artifact type using the provided configurations.
 
@@ -212,7 +212,7 @@ class Packager(ABC):
         return True
 
     def is_unpackable(
-        self, data_item: DataItem, type_hint: Type, artifact_type: str = None
+        self, data_item: DataItem, type_hint: type, artifact_type: str = None
     ) -> bool:
         """
         Check if this packager can unpack an input according to the user-given type hint and the provided artifact type.
@@ -269,7 +269,7 @@ class Packager(ABC):
         self._priority = priority
 
     @property
-    def future_clearing_path_list(self) -> List[str]:
+    def future_clearing_path_list(self) -> list[str]:
         """
         Get the packager's future clearing path list.
 

mlrun/package/packagers/default_packager.py

@@ -15,7 +15,7 @@
 import inspect
 from abc import ABCMeta
 from types import MethodType
-from typing import Any, List, Tuple, Type, Union
+from typing import Any, Union
 
 import docstring_parser
 
@@ -34,7 +34,7 @@ class _DefaultPackagerMeta(ABCMeta):
     dynamically generated docstring that will include a summary of the packager.
     """
 
-    def __new__(mcls, name: str, bases: tuple, namespace: dict, **kwargs):
+    def __new__(cls, name: str, bases: tuple, namespace: dict, **kwargs):
         """
         Create a new DefaultPackager metaclass that saves the original packager docstring to another attribute named
         `_packager_doc`.
@@ -48,10 +48,10 @@ class _DefaultPackagerMeta(ABCMeta):
         namespace["_packager_doc"] = namespace.get("__doc__", "")
 
         # Continue creating the metaclass:
-        return super().__new__(mcls, name, bases, namespace, **kwargs)
+        return super().__new__(cls, name, bases, namespace, **kwargs)
 
     @property
-    def __doc__(cls: Type["DefaultPackager"]) -> str:
+    def __doc__(cls: type["DefaultPackager"]) -> str:
         """
         Override the `__doc__` attribute of a `DefaultPackager` to be a property in order to auto-summarize the
         packager's class docstring. The summary is concatenated after the original class doc string.
@@ -273,7 +273,7 @@ class DefaultPackager(Packager, metaclass=_DefaultPackagerMeta):
     """
 
     #: The type of object this packager can pack and unpack.
-    PACKABLE_OBJECT_TYPE: Type = ...
+    PACKABLE_OBJECT_TYPE: type = ...
 
     #: A flag for indicating whether to also pack all subclasses of the `PACKABLE_OBJECT_TYPE`.
     PACK_SUBCLASSES = False
@@ -306,7 +306,7 @@ class DefaultPackager(Packager, metaclass=_DefaultPackagerMeta):
         """
         return self.DEFAULT_UNPACKING_ARTIFACT_TYPE
 
-    def get_supported_artifact_types(self) -> List[str]:
+    def get_supported_artifact_types(self) -> list[str]:
         """
         Get all the supported artifact types on this packager.
 
@@ -326,7 +326,7 @@ class DefaultPackager(Packager, metaclass=_DefaultPackagerMeta):
         key: str = None,
         artifact_type: str = None,
         configurations: dict = None,
-    ) -> Union[Tuple[Artifact, dict], dict]:
+    ) -> Union[tuple[Artifact, dict], dict]:
         """
         Pack an object as the given artifact type using the provided configurations.
 
@@ -442,7 +442,7 @@ class DefaultPackager(Packager, metaclass=_DefaultPackagerMeta):
         obj: Any,
         key: str,
         pickle_module_name: str = DEFAULT_PICKLE_MODULE,
-    ) -> Tuple[Artifact, dict]:
+    ) -> tuple[Artifact, dict]:
         """
         Pack a python object, pickling it into a pkl file and store it in an artifact.