mlrun 1.7.0rc5__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/model_monitoring/tracking_policy.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
+import warnings
 from typing import Union
 
 import mlrun.common.schemas.schedule
@@ -55,6 +55,12 @@ class TrackingPolicy(mlrun.model.ModelObj):
                                             writer function, which is a real time nuclio functino, will be deployed
                                             with the same image. By default, the image is mlrun/mlrun.
         """
+        warnings.warn(
+            "The `TrackingPolicy` class is deprecated from version 1.7.0 and is not "
+            "used anymore. It will be removed in 1.9.0.",
+            FutureWarning,
+        )
+
         if isinstance(default_batch_intervals, str):
             default_batch_intervals = (
                 mlrun.common.schemas.schedule.ScheduleCronTrigger.from_crontab(

mlrun/model_monitoring/writer.py

@@ -12,29 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import datetime
 import json
-from http import HTTPStatus
-from typing import Any, NewType
-
-import pandas as pd
-from v3io.dataplane import Client as V3IOClient
-from v3io_frames.client import ClientBase as V3IOFramesClient
-from v3io_frames.errors import Error as V3IOFramesError
-from v3io_frames.frames_pb2 import IGNORE
+from typing import Any, Callable, NewType
 
 import mlrun.common.model_monitoring
+import mlrun.common.schemas
+import mlrun.common.schemas.alert as alert_objects
 import mlrun.model_monitoring
-import mlrun.utils.v3io_clients
-from mlrun.common.schemas.model_monitoring.constants import ResultStatusApp, WriterEvent
+from mlrun.common.schemas.model_monitoring.constants import (
+    EventFieldType,
+    HistogramDataDriftApplicationConstants,
+    MetricData,
+    ResultData,
+    ResultKindApp,
+    ResultStatusApp,
+    WriterEvent,
+    WriterEventKind,
+)
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
+from mlrun.model_monitoring.helpers import get_result_instance_fqn
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 from mlrun.utils.notifications.notification_pusher import CustomNotificationPusher
 
-_TSDB_BE = "tsdb"
-_TSDB_RATE = "1/s"
-_TSDB_TABLE = "app-results"
 _RawEvent = dict[str, Any]
 _AppResultEvent = NewType("_AppResultEvent", _RawEvent)
 
@@ -69,20 +69,20 @@ class _Notifier:
         self._severity = severity
 
     def _should_send_event(self) -> bool:
-        return self._event[WriterEvent.RESULT_STATUS] >= ResultStatusApp.detected
+        return self._event[ResultData.RESULT_STATUS] >= ResultStatusApp.detected.value
 
     def _generate_message(self) -> str:
         return f"""\
 The monitoring app `{self._event[WriterEvent.APPLICATION_NAME]}` \
-of kind `{self._event[WriterEvent.RESULT_KIND]}` \
+of kind `{self._event[ResultData.RESULT_KIND]}` \
 detected a problem in model endpoint ID `{self._event[WriterEvent.ENDPOINT_ID]}` \
 at time `{self._event[WriterEvent.START_INFER_TIME]}`.
 
 Result data:
-Name: `{self._event[WriterEvent.RESULT_NAME]}`
-Value: `{self._event[WriterEvent.RESULT_VALUE]}`
-Status: `{self._event[WriterEvent.RESULT_STATUS]}`
-Extra data: `{self._event[WriterEvent.RESULT_EXTRA_DATA]}`\
+Name: `{self._event[ResultData.RESULT_NAME]}`
+Value: `{self._event[ResultData.RESULT_VALUE]}`
+Status: `{self._event[ResultData.RESULT_STATUS]}`
+Extra data: `{self._event[ResultData.RESULT_EXTRA_DATA]}`\
 """
 
     def notify(self) -> None:
@@ -97,140 +97,176 @@ Extra data: `{self._event[WriterEvent.RESULT_EXTRA_DATA]}`\
 
 class ModelMonitoringWriter(StepToDict):
     """
-    Write monitoring app events to V3IO KV storage
+    Write monitoring application results to the target databases
     """
 
     kind = "monitoring_application_stream_pusher"
 
-    def __init__(self, project: str) -> None:
+    def __init__(
+        self,
+        project: str,
+        secret_provider: Callable = None,
+    ) -> None:
         self.project = project
         self.name = project  # required for the deployment process
-        self._v3io_container = self.get_v3io_container(self.name)
-        self._kv_client = self._get_v3io_client().kv
-        self._tsdb_client = self._get_v3io_frames_client(self._v3io_container)
+
         self._custom_notifier = CustomNotificationPusher(
             notification_types=[NotificationKind.slack]
         )
-        self._create_tsdb_table()
-        self._kv_schemas = []
-
-    @staticmethod
-    def get_v3io_container(project_name: str) -> str:
-        return f"users/pipelines/{project_name}/monitoring-apps"
 
-    @staticmethod
-    def _get_v3io_client() -> V3IOClient:
-        return mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
+        self._app_result_store = mlrun.model_monitoring.get_store_object(
+            project=self.project, secret_provider=secret_provider
         )
-
-    @staticmethod
-    def _get_v3io_frames_client(v3io_container: str) -> V3IOFramesClient:
-        return mlrun.utils.v3io_clients.get_frames_client(
-            address=mlrun.mlconf.v3io_framesd,
-            container=v3io_container,
+        self._tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project, secret_provider=secret_provider
         )
+        self._endpoints_records = {}
 
-    def _create_tsdb_table(self) -> None:
-        self._tsdb_client.create(
-            backend=_TSDB_BE,
-            table=_TSDB_TABLE,
-            if_exists=IGNORE,
-            rate=_TSDB_RATE,
+    def _generate_event_on_drift(
+        self,
+        entity_id: str,
+        result_status: int,
+        event_value: dict,
+        project_name: str,
+        result_kind: int,
+    ) -> None:
+        entity = mlrun.common.schemas.alert.EventEntities(
+            kind=alert_objects.EventEntityKind.MODEL_ENDPOINT_RESULT,
+            project=project_name,
+            ids=[entity_id],
         )
 
-    def _update_kv_db(self, event: _AppResultEvent) -> None:
-        event = _AppResultEvent(event.copy())
-        endpoint_id = event.pop(WriterEvent.ENDPOINT_ID)
-        app_name = event.pop(WriterEvent.APPLICATION_NAME)
-        metric_name = event.pop(WriterEvent.RESULT_NAME)
-        attributes = {metric_name: json.dumps(event)}
-        self._kv_client.update(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=app_name,
-            attributes=attributes,
+        event_kind = self._generate_alert_event_kind(
+            result_status=result_status, result_kind=result_kind
         )
-        if endpoint_id not in self._kv_schemas:
-            self._generate_kv_schema(endpoint_id)
-        logger.info("Updated V3IO KV successfully", key=app_name)
-
-    def _generate_kv_schema(self, endpoint_id: str):
-        """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
-        fields = [
-            {"name": WriterEvent.RESULT_NAME, "type": "string", "nullable": False}
-        ]
-        res = self._kv_client.create_schema(
-            container=self._v3io_container,
-            table_path=endpoint_id,
-            key=WriterEvent.APPLICATION_NAME,
-            fields=fields,
+
+        event_data = mlrun.common.schemas.Event(
+            kind=alert_objects.EventKind(value=event_kind),
+            entity=entity,
+            value_dict=event_value,
         )
-        if res.status_code != HTTPStatus.OK.value:
-            raise mlrun.errors.MLRunBadRequestError(
-                f"Couldn't infer schema for endpoint {endpoint_id} which is required for Grafana dashboards"
-            )
-        else:
-            logger.info(
-                "Generated V3IO KV schema successfully", endpoint_id=endpoint_id
-            )
-            self._kv_schemas.append(endpoint_id)
+        logger.info("Sending a drift event")
+        mlrun.get_run_db().generate_event(event_kind, event_data)
+        logger.info("Drift event sent successfully")
 
-    def _update_tsdb(self, event: _AppResultEvent) -> None:
-        event = _AppResultEvent(event.copy())
-        event[WriterEvent.END_INFER_TIME] = datetime.datetime.fromisoformat(
-            event[WriterEvent.END_INFER_TIME]
+    @staticmethod
+    def _generate_alert_event_kind(
+        result_kind: int, result_status: int
+    ) -> alert_objects.EventKind:
+        """Generate the required Event Kind format for the alerting system"""
+        event_kind = ResultKindApp(value=result_kind).name
+
+        if result_status == ResultStatusApp.detected.value:
+            event_kind = f"{event_kind}_detected"
+        else:
+            event_kind = f"{event_kind}_suspected"
+        return alert_objects.EventKind(
+            value=mlrun.utils.helpers.normalize_name(event_kind)
         )
-        del event[WriterEvent.RESULT_EXTRA_DATA]
-        try:
-            self._tsdb_client.write(
-                backend=_TSDB_BE,
-                table=_TSDB_TABLE,
-                dfs=pd.DataFrame.from_records([event]),
-                index_cols=[
-                    WriterEvent.END_INFER_TIME,
-                    WriterEvent.ENDPOINT_ID,
-                    WriterEvent.APPLICATION_NAME,
-                    WriterEvent.RESULT_NAME,
-                ],
-            )
-            logger.info("Updated V3IO TSDB successfully", table=_TSDB_TABLE)
-        except V3IOFramesError as err:
-            logger.warn(
-                "Could not write drift measures to TSDB",
-                err=err,
-                table=_TSDB_TABLE,
-                event=event,
-            )
 
     @staticmethod
-    def _reconstruct_event(event: _RawEvent) -> _AppResultEvent:
+    def _reconstruct_event(event: _RawEvent) -> tuple[_AppResultEvent, WriterEventKind]:
         """
         Modify the raw event into the expected monitoring application event
         schema as defined in `mlrun.common.schemas.model_monitoring.constants.WriterEvent`
         """
-        try:
-            result_event = _AppResultEvent(
-                {key: event[key] for key in WriterEvent.list()}
+        if not isinstance(event, dict):
+            raise _WriterEventTypeError(
+                f"The event is of type: {type(event)}, expected a dictionary"
             )
-            result_event[WriterEvent.CURRENT_STATS] = json.loads(
-                event[WriterEvent.CURRENT_STATS]
+        kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
+        result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
+        if not result_event:  # BC for < 1.7.0, can be removed in 1.9.0
+            result_event = _AppResultEvent(event)
+        else:
+            result_event.update(_AppResultEvent(event))
+
+        expected_keys = list(
+            set(WriterEvent.list()).difference(
+                [WriterEvent.EVENT_KIND, WriterEvent.DATA]
             )
-            return result_event
-        except KeyError as err:
+        )
+        if kind == WriterEventKind.METRIC:
+            expected_keys.extend(MetricData.list())
+        elif kind == WriterEventKind.RESULT:
+            expected_keys.extend(ResultData.list())
+        else:
             raise _WriterEventValueError(
-                "The received event misses some keys compared to the expected "
-                "monitoring application event schema"
-            ) from err
-        except TypeError as err:
-            raise _WriterEventTypeError(
-                f"The event is of type: {type(event)}, expected a dictionary"
-            ) from err
+                f"Unknown event kind: {kind}, expected one of: {WriterEventKind.list()}"
+            )
+        missing_keys = [key for key in expected_keys if key not in result_event]
+        if missing_keys:
+            raise _WriterEventValueError(
+                f"The received event misses some keys compared to the expected "
+                f"monitoring application event schema: {missing_keys}"
+            )
+
+        return result_event, kind
 
     def do(self, event: _RawEvent) -> None:
-        event = self._reconstruct_event(event)
+        event, kind = self._reconstruct_event(event)
         logger.info("Starting to write event", event=event)
-        self._update_tsdb(event)
-        self._update_kv_db(event)
-        _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+        self._tsdb_connector.write_application_event(event=event.copy(), kind=kind)
+        self._app_result_store.write_application_event(event=event.copy(), kind=kind)
+
         logger.info("Completed event DB writes")
+
+        if kind == WriterEventKind.RESULT:
+            _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+
+        if (
+            mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled
+            and kind == WriterEventKind.RESULT
+            and (
+                event[ResultData.RESULT_STATUS] == ResultStatusApp.detected.value
+                or event[ResultData.RESULT_STATUS]
+                == ResultStatusApp.potential_detection.value
+            )
+        ):
+            endpoint_id = event[WriterEvent.ENDPOINT_ID]
+            endpoint_record = self._endpoints_records.setdefault(
+                endpoint_id,
+                self._app_result_store.get_model_endpoint(endpoint_id=endpoint_id),
+            )
+            event_value = {
+                "app_name": event[WriterEvent.APPLICATION_NAME],
+                "model": endpoint_record.get(EventFieldType.MODEL),
+                "model_endpoint_id": event[WriterEvent.ENDPOINT_ID],
+                "result_name": event[ResultData.RESULT_NAME],
+                "result_value": event[ResultData.RESULT_VALUE],
+            }
+            self._generate_event_on_drift(
+                entity_id=get_result_instance_fqn(
+                    event[WriterEvent.ENDPOINT_ID],
+                    event[WriterEvent.APPLICATION_NAME],
+                    event[ResultData.RESULT_NAME],
+                ),
+                result_status=event[ResultData.RESULT_STATUS],
+                event_value=event_value,
+                project_name=self.project,
+                result_kind=event[ResultData.RESULT_KIND],
+            )
+
+        if (
+            kind == WriterEventKind.RESULT
+            and event[WriterEvent.APPLICATION_NAME]
+            == HistogramDataDriftApplicationConstants.NAME
+            and event[ResultData.RESULT_NAME]
+            == HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME
+        ):
+            endpoint_id = event[WriterEvent.ENDPOINT_ID]
+            logger.info(
+                "Updating the model endpoint with metadata specific to the histogram "
+                "data drift app",
+                endpoint_id=endpoint_id,
+            )
+            attributes = json.loads(event[ResultData.RESULT_EXTRA_DATA])
+            attributes[EventFieldType.DRIFT_STATUS] = str(
+                attributes[EventFieldType.DRIFT_STATUS]
+            )
+            self._app_result_store.update_model_endpoint(
+                endpoint_id=endpoint_id,
+                attributes=attributes,
+            )
+
+        logger.info("Model monitoring writer finished handling event")

mlrun/package/packagers/default_packager.py

@@ -34,7 +34,7 @@ class _DefaultPackagerMeta(ABCMeta):
     dynamically generated docstring that will include a summary of the packager.
     """
 
-    def __new__(mcls, name: str, bases: tuple, namespace: dict, **kwargs):
+    def __new__(cls, name: str, bases: tuple, namespace: dict, **kwargs):
         """
         Create a new DefaultPackager metaclass that saves the original packager docstring to another attribute named
         `_packager_doc`.
@@ -48,7 +48,7 @@ class _DefaultPackagerMeta(ABCMeta):
         namespace["_packager_doc"] = namespace.get("__doc__", "")
 
         # Continue creating the metaclass:
-        return super().__new__(mcls, name, bases, namespace, **kwargs)
+        return super().__new__(cls, name, bases, namespace, **kwargs)
 
     @property
     def __doc__(cls: type["DefaultPackager"]) -> str:

mlrun/package/packagers_manager.py

@@ -92,6 +92,7 @@ class PackagersManager:
             from mlrun import Packager
             from x import XPackager
 
+
             class YPackager(Packager):
                 pass
 

mlrun/package/utils/_formatter.py

@@ -142,11 +142,11 @@ class _YAMLFormatter(_Formatter):
 
         :param obj:         The object to write.
         :param file_path:   The file path to write to.
-        :param dump_kwargs: Additional keyword arguments to pass to the `yaml.dump` method of the formatter in use.
+        :param dump_kwargs: Additional keyword arguments to pass to the `yaml.safe_dump` method of the formatter in use.
         """
         dump_kwargs = dump_kwargs or cls.DEFAULT_DUMP_KWARGS
         with open(file_path, "w") as file:
-            yaml.dump(obj, file, **dump_kwargs)
+            yaml.safe_dump(obj, file, **dump_kwargs)
 
     @classmethod
     def read(cls, file_path: str) -> Union[list, dict]:

mlrun/platforms/__init__.py

@@ -17,22 +17,23 @@ import json
 from pprint import pprint
 from time import sleep
 
-from .iguazio import (
-    V3ioStreamClient,
-    VolumeMount,
-    add_or_refresh_credentials,
-    is_iguazio_session_cookie,
-    mount_v3io,
-    v3io_cred,
-)
-from .other import (
+from mlrun_pipelines.common.mounts import VolumeMount
+from mlrun_pipelines.mounts import (
     auto_mount,
     mount_configmap,
     mount_hostpath,
     mount_pvc,
     mount_s3,
     mount_secret,
+    mount_v3io,
     set_env_variables,
+    v3io_cred,
+)
+
+from .iguazio import (
+    V3ioStreamClient,
+    add_or_refresh_credentials,
+    is_iguazio_session_cookie,
 )
 
 
@@ -48,7 +49,7 @@ def watch_stream(
 
     example::
 
-        watch_stream('v3io:///users/admin/mystream')
+        watch_stream("v3io:///users/admin/mystream")
 
     :param url:        stream url
     :param shard_ids:  range or list of shard IDs