mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/model_monitoring/api.py

@@ -24,7 +24,6 @@ import mlrun.artifacts
 import mlrun.common.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store
-import mlrun.model_monitoring.application
 import mlrun.model_monitoring.applications as mm_app
 import mlrun.serving
 from mlrun.data_types.infer import InferOptions, get_df_stats
@@ -147,8 +146,7 @@ def record_results(
                                      on the provided `endpoint_id`.
     :param function_name:            If a new model endpoint is created, use this function name for generating the
                                      function URI.
-    :param context:                  MLRun context. Note that the context is required for logging the artifacts
-                                     following the batch drift job.
+    :param context:                  MLRun context. Note that the context is required generating the model endpoint.
     :param infer_results_df:         DataFrame that will be stored under the model endpoint parquet target. Will be
                                      used for doing the drift analysis. Please make sure that the dataframe includes
                                      both feature names and label columns.
@@ -252,14 +250,31 @@ def _model_endpoint_validations(
                                      In case of discrepancy between the provided `sample_set_statistics` and the
                                      `model_endpoints.spec.feature_stats`, a warning will be presented to the user.
     """
-    # Model path
-    if model_path and model_endpoint.spec.model_uri != model_path:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"provided model store path {model_path} does not match "
-            f"the path that is stored under the existing model "
-            f"endpoint record: {model_endpoint.spec.model_uri}"
+
+    # Model Path
+    if model_path:
+        # Generate the parsed model uri that is based on hash, key, iter, and tree
+        model_obj = mlrun.datastore.get_store_resource(model_path)
+
+        model_artifact_uri = mlrun.utils.helpers.generate_artifact_uri(
+            project=model_endpoint.metadata.project,
+            key=model_obj.key,
+            iter=model_obj.iter,
+            tree=model_obj.tree,
+        )
+
+        # Enrich the uri schema with the store prefix
+        model_artifact_uri = mlrun.datastore.get_store_uri(
+            kind=mlrun.utils.helpers.StorePrefix.Model, uri=model_artifact_uri
         )
 
+        if model_endpoint.spec.model_uri != model_artifact_uri:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"provided model store path {model_path} does not match "
+                f"the path that is stored under the existing model "
+                f"endpoint record: {model_endpoint.spec.model_uri}"
+            )
+
     # Feature stats
     if (
         sample_set_statistics
@@ -545,8 +560,7 @@ def _create_model_monitoring_function_base(
     func: typing.Union[str, None] = None,
     application_class: typing.Union[
         str,
-        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
-        mm_app.ModelMonitoringApplicationBaseV2,
+        mm_app.ModelMonitoringApplicationBase,
         None,
     ] = None,
     name: typing.Optional[str] = None,
@@ -560,19 +574,10 @@ def _create_model_monitoring_function_base(
     Note: this is an internal API only.
     This function does not set the labels or mounts v3io.
     """
-    if isinstance(
-        application_class,
-        mlrun.model_monitoring.application.ModelMonitoringApplicationBase,
-    ):
-        warnings.warn(
-            "The `ModelMonitoringApplicationBase` class is deprecated from version 1.7.0, "
-            "please use `ModelMonitoringApplicationBaseV2`. It will be removed in 1.9.0.",
-            FutureWarning,
-        )
-    if name in mm_constants.MonitoringFunctionNames.list():
+    if name in mm_constants._RESERVED_FUNCTION_NAMES:
         raise mlrun.errors.MLRunInvalidArgumentError(
-            f"An application cannot have the following names: "
-            f"{mm_constants.MonitoringFunctionNames.list()}"
+            "An application cannot have the following names: "
+            f"{mm_constants._RESERVED_FUNCTION_NAMES}"
         )
     if func is None:
         func = ""
@@ -599,11 +604,20 @@ def _create_model_monitoring_function_base(
         app_step = prepare_step.to(class_name=application_class, **application_kwargs)
     else:
         app_step = prepare_step.to(class_name=application_class)
+
     app_step.__class__ = mlrun.serving.MonitoringApplicationStep
+
+    app_step.error_handler(
+        name="ApplicationErrorHandler",
+        class_name="mlrun.model_monitoring.applications._application_steps._ApplicationErrorHandler",
+        full_event=True,
+        project=project,
+    )
+
     app_step.to(
         class_name="mlrun.model_monitoring.applications._application_steps._PushToMonitoringWriter",
         name="PushToMonitoringWriter",
         project=project,
         writer_application_name=mm_constants.MonitoringFunctionNames.WRITER,
-    ).respond()
+    )
     return func_obj

mlrun/model_monitoring/applications/__init__.py

@@ -13,12 +13,11 @@
 # limitations under the License.
 #
 
-from .base import ModelMonitoringApplicationBase, ModelMonitoringApplicationBaseV2
+from .base import ModelMonitoringApplicationBase
 from .context import MonitoringApplicationContext
 from .evidently_base import (
     _HAS_EVIDENTLY,
     SUPPORTED_EVIDENTLY_VERSION,
     EvidentlyModelMonitoringApplicationBase,
-    EvidentlyModelMonitoringApplicationBaseV2,
 )
 from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult

mlrun/model_monitoring/applications/_application_steps.py

@@ -11,16 +11,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
-import typing
-from typing import Optional
+import traceback
+from typing import Any, Optional, Union
 
-import mlrun.common.helpers
-import mlrun.common.model_monitoring.helpers
+import mlrun.common.schemas.alert as alert_objects
 import mlrun.common.schemas.model_monitoring.constants as mm_constant
 import mlrun.datastore
-import mlrun.utils.v3io_clients
+import mlrun.model_monitoring
 from mlrun.model_monitoring.helpers import get_stream_path
+from mlrun.serving import GraphContext
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 
@@ -33,8 +34,8 @@ class _PushToMonitoringWriter(StepToDict):
 
     def __init__(
         self,
-        project: Optional[str] = None,
-        writer_application_name: Optional[str] = None,
+        project: str,
+        writer_application_name: str,
         stream_uri: Optional[str] = None,
         name: Optional[str] = None,
     ):
@@ -59,7 +60,7 @@ class _PushToMonitoringWriter(StepToDict):
         self,
         event: tuple[
             list[
-                typing.Union[
+                Union[
                     ModelMonitoringApplicationResult, ModelMonitoringApplicationMetric
                 ]
             ],
@@ -108,6 +109,7 @@ class _PushToMonitoringWriter(StepToDict):
                 f"Pushing data = {writer_event} \n to stream = {self.stream_uri}"
             )
             self.output_stream.push([writer_event])
+            logger.info(f"Pushed data to {self.stream_uri} successfully")
 
     def _lazy_init(self):
         if self.output_stream is None:
@@ -117,41 +119,70 @@ class _PushToMonitoringWriter(StepToDict):
 
 
 class _PrepareMonitoringEvent(StepToDict):
-    def __init__(self, application_name: str):
+    def __init__(self, context: GraphContext, application_name: str) -> None:
         """
         Class for preparing the application event for the application step.
 
         :param application_name: Application name.
         """
+        self.graph_context = context
+        self.application_name = application_name
+        self.model_endpoints: dict[str, mlrun.model_monitoring.ModelEndpoint] = {}
 
-        self.context = self._create_mlrun_context(application_name)
-        self.model_endpoints = {}
-
-    def do(self, event: dict[str, dict]) -> MonitoringApplicationContext:
+    def do(self, event: dict[str, Any]) -> MonitoringApplicationContext:
         """
         Prepare the application event for the application step.
 
         :param event: Application event.
-        :return: Application event.
+        :return: Application context.
         """
-        if not event.get("mlrun_context"):
-            application_context = MonitoringApplicationContext().from_dict(
-                event,
-                context=self.context,
-                model_endpoint_dict=self.model_endpoints,
-            )
-        else:
-            application_context = MonitoringApplicationContext().from_dict(event)
+        application_context = MonitoringApplicationContext(
+            graph_context=self.graph_context,
+            application_name=self.application_name,
+            event=event,
+            model_endpoint_dict=self.model_endpoints,
+        )
+
         self.model_endpoints.setdefault(
             application_context.endpoint_id, application_context.model_endpoint
         )
+
         return application_context
 
-    @staticmethod
-    def _create_mlrun_context(app_name: str):
-        context = mlrun.get_or_create_ctx(
-            f"{app_name}-logger",
-            upload_artifacts=True,
+
+class _ApplicationErrorHandler(StepToDict):
+    def __init__(self, project: str, name: Optional[str] = None):
+        self.project = project
+        self.name = name or "ApplicationErrorHandler"
+
+    def do(self, event):
+        """
+        Handle model monitoring application error. This step will generate an event, describing the error.
+
+        :param event: Application event.
+        """
+
+        exception_with_trace = "".join(
+            traceback.format_exception(None, event.error, event.error.__traceback__)
+        )
+        logger.error(f"Error in application step: {exception_with_trace}")
+
+        event_data = alert_objects.Event(
+            kind=alert_objects.EventKind.MM_APP_FAILED,
+            entity=alert_objects.EventEntities(
+                kind=alert_objects.EventEntityKind.MODEL_MONITORING_APPLICATION,
+                project=self.project,
+                ids=[f"{self.project}_{event.body.application_name}"],
+            ),
+            value_dict={
+                "Error": event.error,
+                "Timestamp": event.timestamp,
+                "Application Class": event.body.application_name,
+                "Endpoint ID": event.body.endpoint_id,
+            },
+        )
+
+        mlrun.get_run_db().generate_event(
+            name=alert_objects.EventKind.MM_APP_FAILED, event_data=event_data
         )
-        context.__class__ = MonitoringApplicationContext
-        return context
+        logger.info("Event generated successfully")

mlrun/model_monitoring/applications/base.py

@@ -13,18 +13,14 @@
 # limitations under the License.
 
 from abc import ABC, abstractmethod
-from typing import Any, Union, cast
+from typing import Any, Union
 
-import numpy as np
-import pandas as pd
-
-import mlrun
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
 from mlrun.serving.utils import MonitoringApplicationToDict
 
 
-class ModelMonitoringApplicationBaseV2(MonitoringApplicationToDict, ABC):
+class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     """
     A base class for a model monitoring application.
     Inherit from this class to create a custom model monitoring application.
@@ -110,171 +106,3 @@ class ModelMonitoringApplicationBaseV2(MonitoringApplicationToDict, ABC):
                                         each metric name is the key and the metric value is the corresponding value).
         """
         raise NotImplementedError
-
-
-class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
-    """
-    A base class for a model monitoring application.
-    Inherit from this class to create a custom model monitoring application.
-
-    example for very simple custom application::
-
-        class MyApp(ApplicationBase):
-            def do_tracking(
-                self,
-                sample_df_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-                feature_stats: mlrun.common.model_monitoring.helpers.FeatureStats,
-                start_infer_time: pd.Timestamp,
-                end_infer_time: pd.Timestamp,
-                schedule_time: pd.Timestamp,
-                latest_request: pd.Timestamp,
-                endpoint_id: str,
-                output_stream_uri: str,
-            ) -> ModelMonitoringApplicationResult:
-                self.context.log_artifact(
-                    TableArtifact(
-                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
-                    )
-                )
-                return ModelMonitoringApplicationResult(
-                    name="data_drift_test",
-                    value=0.5,
-                    kind=mm_constant.ResultKindApp.data_drift,
-                    status=mm_constant.ResultStatusApp.detected,
-                )
-
-
-    """
-
-    kind = "monitoring_application"
-
-    def do(
-        self, monitoring_context: mm_context.MonitoringApplicationContext
-    ) -> tuple[
-        list[mm_results.ModelMonitoringApplicationResult],
-        mm_context.MonitoringApplicationContext,
-    ]:
-        """
-        Process the monitoring event and return application results.
-
-        :param monitoring_context:   (MonitoringApplicationContext) The monitoring context to process.
-        :returns:                    A tuple of:
-                                        [0] = list of application results that can be either from type
-                                        `ModelMonitoringApplicationResult` or from type
-                                        `ModelMonitoringApplicationResult`.
-                                        [1] = the original application event, wrapped in `MonitoringApplicationContext`
-                                         object
-        """
-        resolved_event = self._resolve_event(monitoring_context)
-        if not (
-            hasattr(self, "context") and isinstance(self.context, mlrun.MLClientCtx)
-        ):
-            self._lazy_init(monitoring_context)
-        results = self.do_tracking(*resolved_event)
-        results = results if isinstance(results, list) else [results]
-        return results, monitoring_context
-
-    def _lazy_init(self, monitoring_context: mm_context.MonitoringApplicationContext):
-        self.context = cast(mlrun.MLClientCtx, monitoring_context)
-
-    @abstractmethod
-    def do_tracking(
-        self,
-        application_name: str,
-        sample_df_stats: pd.DataFrame,
-        feature_stats: pd.DataFrame,
-        sample_df: pd.DataFrame,
-        start_infer_time: pd.Timestamp,
-        end_infer_time: pd.Timestamp,
-        latest_request: pd.Timestamp,
-        endpoint_id: str,
-        output_stream_uri: str,
-    ) -> Union[
-        mm_results.ModelMonitoringApplicationResult,
-        list[mm_results.ModelMonitoringApplicationResult],
-    ]:
-        """
-        Implement this method with your custom monitoring logic.
-
-        :param application_name:        (str) the app name
-        :param sample_df_stats:         (pd.DataFrame) The new sample distribution.
-        :param feature_stats:           (pd.DataFrame) The train sample distribution.
-        :param sample_df:               (pd.DataFrame) The new sample DataFrame.
-        :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
-        :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
-        :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
-        :param endpoint_id:             (str) ID of the monitored model endpoint
-        :param output_stream_uri:       (str) URI of the output stream for results
-
-        :returns:                       (ModelMonitoringApplicationResult) or
-                                        (list[ModelMonitoringApplicationResult]) of the application results.
-        """
-        raise NotImplementedError
-
-    @classmethod
-    def _resolve_event(
-        cls,
-        monitoring_context: mm_context.MonitoringApplicationContext,
-    ) -> tuple[
-        str,
-        pd.DataFrame,
-        pd.DataFrame,
-        pd.DataFrame,
-        pd.Timestamp,
-        pd.Timestamp,
-        pd.Timestamp,
-        str,
-        str,
-    ]:
-        """
-        Converting the event into a single tuple that will be used for passing the event arguments to the running
-        application
-
-        :param monitoring_context: (MonitoringApplicationContext) The monitoring context to process.
-
-        :return: A tuple of:
-                     [0] = (str) application name
-                     [1] = (pd.DataFrame) current input statistics
-                     [2] = (pd.DataFrame) train statistics
-                     [3] = (pd.DataFrame) current input data
-                     [4] = (pd.Timestamp) start time of the monitoring schedule
-                     [5] = (pd.Timestamp) end time of the monitoring schedule
-                     [6] = (pd.Timestamp) timestamp of the latest request
-                     [7] = (str) endpoint id
-                     [8] = (str) output stream uri
-        """
-        return (
-            monitoring_context.application_name,
-            cls.dict_to_histogram(monitoring_context.sample_df_stats),
-            cls.dict_to_histogram(monitoring_context.feature_stats),
-            monitoring_context.sample_df,
-            monitoring_context.start_infer_time,
-            monitoring_context.end_infer_time,
-            monitoring_context.latest_request,
-            monitoring_context.endpoint_id,
-            monitoring_context.output_stream_uri,
-        )
-
-    @staticmethod
-    def dict_to_histogram(
-        histogram_dict: mlrun.common.model_monitoring.helpers.FeatureStats,
-    ) -> pd.DataFrame:
-        """
-        Convert histogram dictionary to pandas DataFrame with feature histograms as columns
-
-        :param histogram_dict: Histogram dictionary
-
-        :returns: Histogram dataframe
-        """
-
-        # Create a dictionary with feature histograms as values
-        histograms = {}
-        for feature, stats in histogram_dict.items():
-            if "hist" in stats:
-                # Normalize to probability distribution of each feature
-                histograms[feature] = np.array(stats["hist"][0]) / stats["count"]
-
-        # Convert the dictionary to pandas DataFrame
-        histograms = pd.DataFrame(histograms)
-
-        return histograms