mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/model_monitoring/application.py

@@ -12,299 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import dataclasses
-import json
-import re
-from abc import ABC, abstractmethod
-from typing import Any, Optional, Tuple, Union
-
-import numpy as np
-import pandas as pd
-
-import mlrun.common.helpers
-import mlrun.common.schemas.model_monitoring
-import mlrun.common.schemas.model_monitoring.constants as mm_constant
-import mlrun.utils.v3io_clients
-from mlrun.datastore import get_stream_pusher
-from mlrun.datastore.targets import ParquetTarget
-from mlrun.model_monitoring.helpers import get_stream_path
-from mlrun.serving.utils import StepToDict
-from mlrun.utils import logger
-
-
-@dataclasses.dataclass
-class ModelMonitoringApplicationResult:
-    """
-    Class representing the result of a custom model monitoring application.
-
-    :param name:           (str) Name of the application result. This name must be
-                            unique for each metric in a single application
-                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
-    :param value:          (float) Value of the application result.
-    :param kind:           (ResultKindApp) Kind of application result.
-    :param status:         (ResultStatusApp) Status of the application result.
-    :param extra_data:     (dict) Extra data associated with the application result.
-    """
-
-    name: str
-    value: float
-    kind: mm_constant.ResultKindApp
-    status: mm_constant.ResultStatusApp
-    extra_data: dict = dataclasses.field(default_factory=dict)
-
-    def __post_init__(self):
-        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
-        if not re.fullmatch(pat, self.name):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Attribute name must be of the format [a-zA-Z_][a-zA-Z0-9_]*"
-            )
-
-    def to_dict(self):
-        """
-        Convert the object to a dictionary format suitable for writing.
-
-        :returns:    (dict) Dictionary representation of the result.
-        """
-        return {
-            mm_constant.WriterEvent.RESULT_NAME: self.name,
-            mm_constant.WriterEvent.RESULT_VALUE: self.value,
-            mm_constant.WriterEvent.RESULT_KIND: self.kind,
-            mm_constant.WriterEvent.RESULT_STATUS: self.status,
-            mm_constant.WriterEvent.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
-        }
-
-
-class ModelMonitoringApplicationBase(StepToDict, ABC):
-    """
-    A base class for a model monitoring application.
-    Inherit from this class to create a custom model monitoring application.
-
-    example for very simple custom application::
-        # mlrun: start-code
-        class MyApp(ApplicationBase):
-            def do_tracking(
-                self,
-                sample_df_stats: pd.DataFrame,
-                feature_stats: pd.DataFrame,
-                start_infer_time: pd.Timestamp,
-                end_infer_time: pd.Timestamp,
-                schedule_time: pd.Timestamp,
-                latest_request: pd.Timestamp,
-                endpoint_id: str,
-                output_stream_uri: str,
-            ) -> ModelMonitoringApplicationResult:
-                self.context.log_artifact(TableArtifact("sample_df_stats", df=sample_df_stats))
-                return ModelMonitoringApplicationResult(
-                    name="data_drift_test",
-                    value=0.5,
-                    kind=mm_constant.ResultKindApp.data_drift,
-                    status=mm_constant.ResultStatusApp.detected,
-                )
-
-        # mlrun: end-code
-    """
-
-    kind = "monitoring_application"
-
-    def do(
-        self, event: dict[str, Any]
-    ) -> Tuple[list[ModelMonitoringApplicationResult], dict]:
-        """
-        Process the monitoring event and return application results.
-
-        :param event:   (dict) The monitoring event to process.
-        :returns:       (list[ModelMonitoringApplicationResult], dict) The application results
-                        and the original event for the application.
-        """
-        resolved_event = self._resolve_event(event)
-        if not (
-            hasattr(self, "context") and isinstance(self.context, mlrun.MLClientCtx)
-        ):
-            self._lazy_init(app_name=resolved_event[0])
-        results = self.do_tracking(*resolved_event)
-        results = results if isinstance(results, list) else [results]
-        return results, event
-
-    def _lazy_init(self, app_name: str):
-        self.context = self._create_context_for_logging(app_name=app_name)
-
-    @abstractmethod
-    def do_tracking(
-        self,
-        application_name: str,
-        sample_df_stats: pd.DataFrame,
-        feature_stats: pd.DataFrame,
-        sample_df: pd.DataFrame,
-        start_infer_time: pd.Timestamp,
-        end_infer_time: pd.Timestamp,
-        latest_request: pd.Timestamp,
-        endpoint_id: str,
-        output_stream_uri: str,
-    ) -> Union[
-        ModelMonitoringApplicationResult, list[ModelMonitoringApplicationResult]
-    ]:
-        """
-        Implement this method with your custom monitoring logic.
-
-        :param application_name:         (str) the app name
-        :param sample_df_stats:         (pd.DataFrame) The new sample distribution DataFrame.
-        :param feature_stats:           (pd.DataFrame) The train sample distribution DataFrame.
-        :param sample_df:               (pd.DataFrame) The new sample DataFrame.
-        :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
-        :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
-        :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
-        :param endpoint_id:             (str) ID of the monitored model endpoint
-        :param output_stream_uri:       (str) URI of the output stream for results
-
-        :returns:                       (ModelMonitoringApplicationResult) or
-                                        (list[ModelMonitoringApplicationResult]) of the application results.
-        """
-        raise NotImplementedError
-
-    @classmethod
-    def _resolve_event(
-        cls,
-        event: dict[str, Any],
-    ) -> Tuple[
-        str,
-        pd.DataFrame,
-        pd.DataFrame,
-        pd.DataFrame,
-        pd.Timestamp,
-        pd.Timestamp,
-        pd.Timestamp,
-        str,
-        str,
-    ]:
-        """
-        Converting the event into a single tuple that will be used for passing the event arguments to the running
-        application
-
-        :param event: dictionary with all the incoming data
-
-        :return: A tuple of:
-                     [0] = (str) application name
-                     [1] = (pd.DataFrame) current input statistics
-                     [2] = (pd.DataFrame) train statistics
-                     [3] = (pd.DataFrame) current input data
-                     [4] = (pd.Timestamp) start time of the monitoring schedule
-                     [5] = (pd.Timestamp) end time of the monitoring schedule
-                     [6] = (pd.Timestamp) timestamp of the latest request
-                     [7] = (str) endpoint id
-                     [8] = (str) output stream uri
-        """
-        start_time = pd.Timestamp(event[mm_constant.ApplicationEvent.START_INFER_TIME])
-        end_time = pd.Timestamp(event[mm_constant.ApplicationEvent.END_INFER_TIME])
-        return (
-            event[mm_constant.ApplicationEvent.APPLICATION_NAME],
-            cls._dict_to_histogram(
-                json.loads(event[mm_constant.ApplicationEvent.CURRENT_STATS])
-            ),
-            cls._dict_to_histogram(
-                json.loads(event[mm_constant.ApplicationEvent.FEATURE_STATS])
-            ),
-            ParquetTarget(
-                path=event[mm_constant.ApplicationEvent.SAMPLE_PARQUET_PATH]
-            ).as_df(start_time=start_time, end_time=end_time, time_column="timestamp"),
-            start_time,
-            end_time,
-            pd.Timestamp(event[mm_constant.ApplicationEvent.LAST_REQUEST]),
-            event[mm_constant.ApplicationEvent.ENDPOINT_ID],
-            event[mm_constant.ApplicationEvent.OUTPUT_STREAM_URI],
-        )
-
-    @staticmethod
-    def _create_context_for_logging(app_name: str):
-        context = mlrun.get_or_create_ctx(
-            f"{app_name}-logger",
-            upload_artifacts=True,
-            labels={"workflow": "model-monitoring-app-logger"},
-        )
-        return context
-
-    @staticmethod
-    def _dict_to_histogram(histogram_dict: dict[str, dict[str, Any]]) -> pd.DataFrame:
-        """
-        Convert histogram dictionary to pandas DataFrame with feature histograms as columns
-
-        :param histogram_dict: Histogram dictionary
-
-        :returns: Histogram dataframe
-        """
-
-        # Create a dictionary with feature histograms as values
-        histograms = {}
-        for feature, stats in histogram_dict.items():
-            if "hist" in stats:
-                # Normalize to probability distribution of each feature
-                histograms[feature] = np.array(stats["hist"][0]) / stats["count"]
-
-        # Convert the dictionary to pandas DataFrame
-        histograms = pd.DataFrame(histograms)
-
-        return histograms
-
-
-class PushToMonitoringWriter(StepToDict):
-    kind = "monitoring_application_stream_pusher"
-
-    def __init__(
-        self,
-        project: Optional[str] = None,
-        writer_application_name: Optional[str] = None,
-        stream_uri: Optional[str] = None,
-        name: Optional[str] = None,
-    ):
-        """
-        Class for pushing application results to the monitoring writer stream.
-
-        :param project:                     Project name.
-        :param writer_application_name:     Writer application name.
-        :param stream_uri:                  Stream URI for pushing results.
-        :param name:                        Name of the PushToMonitoringWriter
-                                            instance default to PushToMonitoringWriter.
-        """
-        self.project = project
-        self.application_name_to_push = writer_application_name
-        self.stream_uri = stream_uri or get_stream_path(
-            project=self.project, application_name=self.application_name_to_push
-        )
-        self.output_stream = None
-        self.name = name or "PushToMonitoringWriter"
-
-    def do(self, event: Tuple[list[ModelMonitoringApplicationResult], dict]) -> None:
-        """
-        Push application results to the monitoring writer stream.
-
-        :param event: Monitoring result(s) to push and the original event from the controller.
-        """
-        self._lazy_init()
-        application_results, application_event = event
-        metadata = {
-            mm_constant.WriterEvent.APPLICATION_NAME: application_event[
-                mm_constant.ApplicationEvent.APPLICATION_NAME
-            ],
-            mm_constant.WriterEvent.ENDPOINT_ID: application_event[
-                mm_constant.ApplicationEvent.ENDPOINT_ID
-            ],
-            mm_constant.WriterEvent.START_INFER_TIME: application_event[
-                mm_constant.ApplicationEvent.START_INFER_TIME
-            ],
-            mm_constant.WriterEvent.END_INFER_TIME: application_event[
-                mm_constant.ApplicationEvent.END_INFER_TIME
-            ],
-            mm_constant.WriterEvent.CURRENT_STATS: json.dumps(
-                application_event[mm_constant.ApplicationEvent.CURRENT_STATS]
-            ),
-        }
-        for result in application_results:
-            data = result.to_dict()
-            data.update(metadata)
-            logger.info(f"Pushing data = {data} \n to stream = {self.stream_uri}")
-            self.output_stream.push([data])
-
-    def _lazy_init(self):
-        if self.output_stream is None:
-            self.output_stream = get_stream_pusher(
-                self.stream_uri,
-            )
+# TODO : delete this file in 1.9.0
+from mlrun.model_monitoring.applications import (  # noqa: F401
+    ModelMonitoringApplicationBase,
+    ModelMonitoringApplicationResult,
+)

mlrun/model_monitoring/applications/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from .base import ModelMonitoringApplicationBase, ModelMonitoringApplicationBaseV2
+from .context import MonitoringApplicationContext
+from .evidently_base import (
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+    EvidentlyModelMonitoringApplicationBaseV2,
+)
+from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult

mlrun/model_monitoring/applications/_application_steps.py

@@ -0,0 +1,157 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import typing
+from typing import Optional
+
+import mlrun.common.helpers
+import mlrun.common.model_monitoring.helpers
+import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.datastore
+import mlrun.utils.v3io_clients
+from mlrun.model_monitoring.helpers import get_stream_path
+from mlrun.serving.utils import StepToDict
+from mlrun.utils import logger
+
+from .context import MonitoringApplicationContext
+from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult
+
+
+class _PushToMonitoringWriter(StepToDict):
+    kind = "monitoring_application_stream_pusher"
+
+    def __init__(
+        self,
+        project: Optional[str] = None,
+        writer_application_name: Optional[str] = None,
+        stream_uri: Optional[str] = None,
+        name: Optional[str] = None,
+    ):
+        """
+        Class for pushing application results to the monitoring writer stream.
+
+        :param project:                     Project name.
+        :param writer_application_name:     Writer application name.
+        :param stream_uri:                  Stream URI for pushing results.
+        :param name:                        Name of the PushToMonitoringWriter
+                                            instance default to PushToMonitoringWriter.
+        """
+        self.project = project
+        self.application_name_to_push = writer_application_name
+        self.stream_uri = stream_uri or get_stream_path(
+            project=self.project, function_name=self.application_name_to_push
+        )
+        self.output_stream = None
+        self.name = name or "PushToMonitoringWriter"
+
+    def do(
+        self,
+        event: tuple[
+            list[
+                typing.Union[
+                    ModelMonitoringApplicationResult, ModelMonitoringApplicationMetric
+                ]
+            ],
+            MonitoringApplicationContext,
+        ],
+    ) -> None:
+        """
+        Push application results to the monitoring writer stream.
+
+        :param event: Monitoring result(s) to push and the original event from the controller.
+        """
+        self._lazy_init()
+        application_results, application_context = event
+        writer_event = {
+            mm_constant.WriterEvent.APPLICATION_NAME: application_context.application_name,
+            mm_constant.WriterEvent.ENDPOINT_ID: application_context.endpoint_id,
+            mm_constant.WriterEvent.START_INFER_TIME: application_context.start_infer_time.isoformat(
+                sep=" ", timespec="microseconds"
+            ),
+            mm_constant.WriterEvent.END_INFER_TIME: application_context.end_infer_time.isoformat(
+                sep=" ", timespec="microseconds"
+            ),
+        }
+        for result in application_results:
+            data = result.to_dict()
+            if isinstance(result, ModelMonitoringApplicationResult):
+                writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                    mm_constant.WriterEventKind.RESULT
+                )
+                data[mm_constant.ResultData.CURRENT_STATS] = json.dumps(
+                    application_context.sample_df_stats
+                )
+                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
+            else:
+                writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                    mm_constant.WriterEventKind.METRIC
+                )
+                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
+
+            writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                mm_constant.WriterEventKind.RESULT
+                if isinstance(result, ModelMonitoringApplicationResult)
+                else mm_constant.WriterEventKind.METRIC
+            )
+            logger.info(
+                f"Pushing data = {writer_event} \n to stream = {self.stream_uri}"
+            )
+            self.output_stream.push([writer_event])
+
+    def _lazy_init(self):
+        if self.output_stream is None:
+            self.output_stream = mlrun.datastore.get_stream_pusher(
+                self.stream_uri,
+            )
+
+
+class _PrepareMonitoringEvent(StepToDict):
+    def __init__(self, application_name: str):
+        """
+        Class for preparing the application event for the application step.
+
+        :param application_name: Application name.
+        """
+
+        self.context = self._create_mlrun_context(application_name)
+        self.model_endpoints = {}
+
+    def do(self, event: dict[str, dict]) -> MonitoringApplicationContext:
+        """
+        Prepare the application event for the application step.
+
+        :param event: Application event.
+        :return: Application event.
+        """
+        if not event.get("mlrun_context"):
+            application_context = MonitoringApplicationContext().from_dict(
+                event,
+                context=self.context,
+                model_endpoint_dict=self.model_endpoints,
+            )
+        else:
+            application_context = MonitoringApplicationContext().from_dict(event)
+        self.model_endpoints.setdefault(
+            application_context.endpoint_id, application_context.model_endpoint
+        )
+        return application_context
+
+    @staticmethod
+    def _create_mlrun_context(app_name: str):
+        context = mlrun.get_or_create_ctx(
+            f"{app_name}-logger",
+            upload_artifacts=True,
+        )
+        context.__class__ = MonitoringApplicationContext
+        return context