mlrun 1.7.0rc38__py3-none-any.whl → 1.7.0rc41__py3-none-any.whl

mlrun/model_monitoring/api.py

@@ -147,8 +147,7 @@ def record_results(
                                      on the provided `endpoint_id`.
     :param function_name:            If a new model endpoint is created, use this function name for generating the
                                      function URI.
-    :param context:                  MLRun context. Note that the context is required for logging the artifacts
-                                     following the batch drift job.
+    :param context:                  MLRun context. Note that the context is required generating the model endpoint.
     :param infer_results_df:         DataFrame that will be stored under the model endpoint parquet target. Will be
                                      used for doing the drift analysis. Please make sure that the dataframe includes
                                      both feature names and label columns.
@@ -616,7 +615,16 @@ def _create_model_monitoring_function_base(
         app_step = prepare_step.to(class_name=application_class, **application_kwargs)
     else:
         app_step = prepare_step.to(class_name=application_class)
+
     app_step.__class__ = mlrun.serving.MonitoringApplicationStep
+
+    app_step.error_handler(
+        name="ApplicationErrorHandler",
+        class_name="mlrun.model_monitoring.applications._application_steps._ApplicationErrorHandler",
+        full_event=True,
+        project=project,
+    )
+
     app_step.to(
         class_name="mlrun.model_monitoring.applications._application_steps._PushToMonitoringWriter",
         name="PushToMonitoringWriter",

mlrun/model_monitoring/applications/_application_steps.py

@@ -11,18 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
-import typing
-from typing import Optional
+from typing import Any, Optional, Union
 
-import mlrun.common.helpers
-import mlrun.common.model_monitoring.helpers
+import mlrun.common.schemas.alert as alert_objects
 import mlrun.common.schemas.model_monitoring.constants as mm_constant
 import mlrun.datastore
-import mlrun.serving
-import mlrun.utils.helpers
-import mlrun.utils.v3io_clients
+import mlrun.model_monitoring
 from mlrun.model_monitoring.helpers import get_stream_path
+from mlrun.serving import GraphContext
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 
@@ -61,7 +59,7 @@ class _PushToMonitoringWriter(StepToDict):
         self,
         event: tuple[
             list[
-                typing.Union[
+                Union[
                     ModelMonitoringApplicationResult, ModelMonitoringApplicationMetric
                 ]
             ],
@@ -120,47 +118,67 @@ class _PushToMonitoringWriter(StepToDict):
 
 
 class _PrepareMonitoringEvent(StepToDict):
-    def __init__(self, application_name: str):
+    def __init__(self, context: GraphContext, application_name: str) -> None:
         """
         Class for preparing the application event for the application step.
 
         :param application_name: Application name.
         """
+        self.graph_context = context
+        self.application_name = application_name
+        self.model_endpoints: dict[str, mlrun.model_monitoring.ModelEndpoint] = {}
 
-        self.context = self._create_mlrun_context(application_name)
-        self.model_endpoints = {}
-
-    def do(self, event: dict[str, dict]) -> MonitoringApplicationContext:
+    def do(self, event: dict[str, Any]) -> MonitoringApplicationContext:
         """
         Prepare the application event for the application step.
 
         :param event: Application event.
-        :return: Application event.
+        :return: Application context.
         """
-        if not event.get("mlrun_context"):
-            application_context = MonitoringApplicationContext().from_dict(
-                event,
-                context=self.context,
-                model_endpoint_dict=self.model_endpoints,
-            )
-        else:
-            application_context = MonitoringApplicationContext().from_dict(event)
+        application_context = MonitoringApplicationContext(
+            graph_context=self.graph_context,
+            application_name=self.application_name,
+            event=event,
+            model_endpoint_dict=self.model_endpoints,
+        )
+
         self.model_endpoints.setdefault(
             application_context.endpoint_id, application_context.model_endpoint
         )
+
         return application_context
 
-    @staticmethod
-    def _create_mlrun_context(app_name: str):
-        artifact_path = mlrun.utils.helpers.template_artifact_path(
-            mlrun.mlconf.artifact_path, mlrun.mlconf.default_project
-        )
-        context = mlrun.get_or_create_ctx(
-            f"{app_name}-logger",
-            spec={
-                "metadata": {"labels": {"kind": mlrun.runtimes.RuntimeKinds.serving}},
-                "spec": {mlrun.utils.helpers.RunKeys.output_path: artifact_path},
+
+class _ApplicationErrorHandler(StepToDict):
+    def __init__(self, project: str, name: Optional[str] = None):
+        self.project = project
+        self.name = name or "ApplicationErrorHandler"
+
+    def do(self, event):
+        """
+        Handle model monitoring application error. This step will generate an event, describing the error.
+
+        :param event: Application event.
+        """
+
+        logger.error(f"Error in application step: {event}")
+
+        event_data = alert_objects.Event(
+            kind=alert_objects.EventKind.MM_APP_FAILED,
+            entity=alert_objects.EventEntities(
+                kind=alert_objects.EventEntityKind.MODEL_MONITORING_APPLICATION,
+                project=self.project,
+                ids=[f"{self.project}_{event.body.application_name}"],
+            ),
+            value_dict={
+                "Error": event.error,
+                "Timestamp": event.timestamp,
+                "Application Class": event.body.application_name,
+                "Endpoint ID": event.body.endpoint_id,
             },
         )
-        context.__class__ = MonitoringApplicationContext
-        return context
+
+        mlrun.get_run_db().generate_event(
+            name=alert_objects.EventKind.MM_APP_FAILED, event_data=event_data
+        )
+        logger.info("Event generated successfully")

mlrun/model_monitoring/applications/context.py

@@ -11,19 +11,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
-import typing
+import socket
+from typing import Any, Optional, cast
 
 import numpy as np
 import pandas as pd
 
-import mlrun.common.helpers
-import mlrun.common.model_monitoring.helpers
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store as fstore
-from mlrun.artifacts.model import ModelArtifact, get_model
+import mlrun.features
+import mlrun.serving
+import mlrun.utils
+from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
 from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
-from mlrun.execution import MLClientCtx
 from mlrun.model_monitoring.helpers import (
     calculate_inputs_statistics,
     get_endpoint_record,
@@ -31,13 +34,17 @@ from mlrun.model_monitoring.helpers import (
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 
 
-class MonitoringApplicationContext(MLClientCtx):
+class MonitoringApplicationContext:
     """
     The monitoring context holds all the relevant information for the monitoring application,
     and also it can be used for logging artifacts and results.
     The monitoring context has the following attributes:
 
-    :param application_name:        (str) the app name
+    :param application_name:        (str) The model monitoring application name.
+    :param project_name:            (str) The project name.
+    :param project:                 (MlrunProject) The project object.
+    :param logger:                  (mlrun.utils.Logger) MLRun logger.
+    :param nuclio_logger:           (nuclio.request.Logger) Nuclio logger.
     :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
     :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
     :param sample_df:               (pd.DataFrame) The new sample DataFrame.
@@ -49,79 +56,89 @@ class MonitoringApplicationContext(MLClientCtx):
     :param model_endpoint:          (ModelEndpoint) The model endpoint object.
     :param feature_names:           (list[str]) List of models feature names.
     :param label_names:             (list[str]) List of models label names.
-    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object, and list of
-
+    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object,
+                                    and a list of extra data items.
     """
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def _enrich_data(self):
-        self.application_name: typing.Optional[str] = None
-        self.start_infer_time: typing.Optional[pd.Timestamp] = None
-        self.end_infer_time: typing.Optional[pd.Timestamp] = None
-        self.latest_request: typing.Optional[pd.Timestamp] = None
-        self.endpoint_id: typing.Optional[str] = None
-        self.output_stream_uri: typing.Optional[str] = None
-
-        self._sample_df: typing.Optional[pd.DataFrame] = None
-        self._model_endpoint: typing.Optional[ModelEndpoint] = None
-        self._feature_stats: typing.Optional[FeatureStats] = None
-        self._sample_df_stats: typing.Optional[FeatureStats] = None
-
-    @classmethod
-    def from_dict(
-        cls,
-        attrs: dict,
-        context=None,
-        model_endpoint_dict=None,
-        **kwargs,
-    ) -> "MonitoringApplicationContext":
+    def __init__(
+        self,
+        *,
+        graph_context: mlrun.serving.GraphContext,
+        application_name: str,
+        event: dict[str, Any],
+        model_endpoint_dict: dict[str, ModelEndpoint],
+    ) -> None:
         """
-        Create an instance of the MonitoringApplicationContext from a dictionary.
+        Initialize a `MonitoringApplicationContext` object.
+        Note: this object should not be instantiated manually.
 
-        :param attrs:               The instance data dictionary.
-        :param context:             The current application context.
+        :param application_name:    The application name.
+        :param event:               The instance data dictionary.
         :param model_endpoint_dict: Dictionary of model endpoints.
-
         """
+        self.application_name = application_name
 
-        if not context:
-            ctx = (
-                super().from_dict(
-                    attrs=attrs.get(mm_constants.ApplicationEvent.MLRUN_CONTEXT, {}),
-                    **kwargs,
-                ),
-            )
-        else:
-            ctx = context
-            cls._enrich_data(ctx)
+        self.project_name = graph_context.project
+        self.project = mlrun.load_project(url=self.project_name)
+
+        # MLRun Logger
+        self.logger = mlrun.utils.create_logger(
+            level=mlrun.mlconf.log_level,
+            formatter_kind=mlrun.mlconf.log_formatter,
+            name="monitoring-application",
+        )
+        # Nuclio logger - `nuclio.request.Logger`.
+        # Note: this logger does not accept keyword arguments.
+        self.nuclio_logger = graph_context.logger
 
-        ctx.start_infer_time = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.START_INFER_TIME)
+        # event data
+        self.start_infer_time = pd.Timestamp(
+            cast(str, event.get(mm_constants.ApplicationEvent.START_INFER_TIME))
         )
-        ctx.end_infer_time = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.END_INFER_TIME)
+        self.end_infer_time = pd.Timestamp(
+            cast(str, event.get(mm_constants.ApplicationEvent.END_INFER_TIME))
         )
-        ctx.latest_request = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.LAST_REQUEST)
+        self.latest_request = pd.Timestamp(
+            cast(str, event.get(mm_constants.ApplicationEvent.LAST_REQUEST))
         )
-        ctx.application_name = attrs.get(mm_constants.ApplicationEvent.APPLICATION_NAME)
-        ctx._feature_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
+        self.endpoint_id = cast(
+            str, event.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
         )
-        ctx._sample_df_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
+        self.output_stream_uri = cast(
+            str, event.get(mm_constants.ApplicationEvent.OUTPUT_STREAM_URI)
         )
 
-        ctx.endpoint_id = attrs.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
-        ctx._model_endpoint = model_endpoint_dict.get(ctx.endpoint_id)
+        self._feature_stats: Optional[FeatureStats] = json.loads(
+            event.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
+        )
+        self._sample_df_stats: Optional[FeatureStats] = json.loads(
+            event.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
+        )
 
-        return ctx
+        # Default labels for the artifacts
+        self._default_labels = self._get_default_labels()
+
+        # Persistent data - fetched when needed
+        self._sample_df: Optional[pd.DataFrame] = None
+        self._model_endpoint: Optional[ModelEndpoint] = model_endpoint_dict.get(
+            self.endpoint_id
+        )
+
+    def _get_default_labels(self) -> dict[str, str]:
+        return {
+            mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
+            mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
+            mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
+            mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
+        }
+
+    def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
+        """Add the default labels to logged artifacts labels"""
+        return (labels or {}) | self._default_labels
 
     @property
     def sample_df(self) -> pd.DataFrame:
-        if not hasattr(self, "_sample_df") or self._sample_df is None:
+        if self._sample_df is None:
             feature_set = fstore.get_feature_set(
                 self.model_endpoint.status.monitoring_feature_set_uri
             )
@@ -144,15 +161,15 @@ class MonitoringApplicationContext(MLClientCtx):
 
     @property
     def model_endpoint(self) -> ModelEndpoint:
-        if not hasattr(self, "_model_endpoint") or not self._model_endpoint:
+        if not self._model_endpoint:
             self._model_endpoint = ModelEndpoint.from_flat_dict(
-                get_endpoint_record(self.project, self.endpoint_id)
+                get_endpoint_record(self.project_name, self.endpoint_id)
             )
         return self._model_endpoint
 
     @property
     def feature_stats(self) -> FeatureStats:
-        if not hasattr(self, "_feature_stats") or not self._feature_stats:
+        if not self._feature_stats:
             self._feature_stats = json.loads(self.model_endpoint.status.feature_stats)
             pad_features_hist(self._feature_stats)
         return self._feature_stats
@@ -160,7 +177,7 @@ class MonitoringApplicationContext(MLClientCtx):
     @property
     def sample_df_stats(self) -> FeatureStats:
         """statistics of the sample dataframe"""
-        if not hasattr(self, "_sample_df_stats") or not self._sample_df_stats:
+        if not self._sample_df_stats:
             self._sample_df_stats = calculate_inputs_statistics(
                 self.feature_stats, self.sample_df
             )
@@ -184,13 +201,11 @@ class MonitoringApplicationContext(MLClientCtx):
 
     @property
     def model(self) -> tuple[str, ModelArtifact, dict]:
-        """return model file, model spec object, and list of extra data items"""
+        """The model file, model spec object, and a list of extra data items"""
         return get_model(self.model_endpoint.spec.model_uri)
 
     @staticmethod
-    def dict_to_histogram(
-        histogram_dict: mlrun.common.model_monitoring.helpers.FeatureStats,
-    ) -> pd.DataFrame:
+    def dict_to_histogram(histogram_dict: FeatureStats) -> pd.DataFrame:
         """
         Convert histogram dictionary to pandas DataFrame with feature histograms as columns
 
@@ -210,3 +225,124 @@ class MonitoringApplicationContext(MLClientCtx):
         histograms = pd.DataFrame(histograms)
 
         return histograms
+
+    def log_artifact(
+        self,
+        item,
+        body=None,
+        tag: str = "",
+        local_path: str = "",
+        artifact_path: Optional[str] = None,
+        format: Optional[str] = None,
+        upload: Optional[bool] = None,
+        labels: Optional[dict[str, str]] = None,
+        target_path: Optional[str] = None,
+        **kwargs,
+    ) -> Artifact:
+        """
+        Log an artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_artifact(
+            item,
+            body=body,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            format=format,
+            upload=upload,
+            labels=labels,
+            target_path=target_path,
+            **kwargs,
+        )
+
+    def log_dataset(
+        self,
+        key,
+        df,
+        tag="",
+        local_path=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        format="",
+        preview=None,
+        stats=None,
+        target_path="",
+        extra_data=None,
+        label_column: Optional[str] = None,
+        **kwargs,
+    ) -> DatasetArtifact:
+        """
+        Log a dataset artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_dataset(
+            key,
+            df,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            format=format,
+            preview=preview,
+            stats=stats,
+            target_path=target_path,
+            extra_data=extra_data,
+            label_column=label_column,
+            **kwargs,
+        )
+
+    def log_model(
+        self,
+        key,
+        body=None,
+        framework="",
+        tag="",
+        model_dir=None,
+        model_file=None,
+        algorithm=None,
+        metrics=None,
+        parameters=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        inputs: Optional[list[mlrun.features.Feature]] = None,
+        outputs: Optional[list[mlrun.features.Feature]] = None,
+        feature_vector: Optional[str] = None,
+        feature_weights: Optional[list] = None,
+        training_set=None,
+        label_column=None,
+        extra_data=None,
+        **kwargs,
+    ) -> ModelArtifact:
+        """
+        Log a model artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_model(
+            key,
+            body=body,
+            framework=framework,
+            tag=tag,
+            model_dir=model_dir,
+            model_file=model_file,
+            algorithm=algorithm,
+            metrics=metrics,
+            parameters=parameters,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            inputs=inputs,
+            outputs=outputs,
+            feature_vector=feature_vector,
+            feature_weights=feature_weights,
+            training_set=training_set,
+            label_column=label_column,
+            extra_data=extra_data,
+            **kwargs,
+        )

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -91,7 +91,9 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
     """
     MLRun's default data drift application for model monitoring.
 
-    The application expects tabular numerical data, and calculates three metrics over the features' histograms.
+    The application expects tabular numerical data, and calculates three metrics over the shared features' histograms.
+    The metrics are calculated on features that have reference data from the training dataset. When there is no
+    reference data (`feature_stats`), this application send a warning log and does nothing.
     The three metrics are:
 
     * Hellinger distance.
@@ -112,6 +114,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
         project.enable_model_monitoring()
 
+    To avoid it, pass `deploy_histogram_data_drift_app=False`.
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
@@ -223,19 +226,18 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         return metrics
 
     @staticmethod
-    def _remove_timestamp_feature(
-        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+    def _get_shared_features_sample_stats(
+        monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> mlrun.common.model_monitoring.helpers.FeatureStats:
         """
-        Drop the 'timestamp' feature if it exists, as it is irrelevant
-        in the plotly artifact
+        Filter out features without reference data in `feature_stats`, e.g. `timestamp`.
         """
-        sample_set_statistics = mlrun.common.model_monitoring.helpers.FeatureStats(
-            sample_set_statistics.copy()
+        return mlrun.common.model_monitoring.helpers.FeatureStats(
+            {
+                key: monitoring_context.sample_df_stats[key]
+                for key in monitoring_context.feature_stats
+            }
         )
-        if EventFieldType.TIMESTAMP in sample_set_statistics:
-            del sample_set_statistics[EventFieldType.TIMESTAMP]
-        return sample_set_statistics
 
     @staticmethod
     def _log_json_artifact(
@@ -299,8 +301,8 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             self._log_json_artifact(drift_per_feature_values, monitoring_context)
 
         self._log_plotly_table_artifact(
-            sample_set_statistics=self._remove_timestamp_feature(
-                monitoring_context.sample_df_stats
+            sample_set_statistics=self._get_shared_features_sample_stats(
+                monitoring_context
             ),
             inputs_statistics=monitoring_context.feature_stats,
             metrics_per_feature=metrics_per_feature,
@@ -325,7 +327,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         """
         monitoring_context.logger.debug("Starting to run the application")
         if not monitoring_context.feature_stats:
-            monitoring_context.logger.info(
+            monitoring_context.logger.warning(
                 "No feature statistics found, skipping the application. \n"
                 "In order to run the application, training set must be provided when logging the model."
             )

mlrun/model_monitoring/controller.py

@@ -335,19 +335,23 @@ class MonitoringApplicationController:
                 return
             monitoring_functions = self.project_obj.list_model_monitoring_functions()
             if monitoring_functions:
-                # Gets only application in ready state
                 applications_names = list(
-                    {
-                        app.metadata.name
-                        for app in monitoring_functions
-                        if (
-                            app.status.state == "ready"
-                            # workaround for the default app, as its `status.state` is `None`
-                            or app.metadata.name
-                            == mm_constants.HistogramDataDriftApplicationConstants.NAME
-                        )
-                    }
+                    {app.metadata.name for app in monitoring_functions}
                 )
+            # if monitoring_functions: - TODO : ML-7700
+            #   Gets only application in ready state
+            #   applications_names = list(
+            #       {
+            #           app.metadata.name
+            #           for app in monitoring_functions
+            #           if (
+            #               app.status.state == "ready"
+            #               # workaround for the default app, as its `status.state` is `None`
+            #               or app.metadata.name
+            #               == mm_constants.HistogramDataDriftApplicationConstants.NAME
+            #           )
+            #       }
+            #   )
             if not applications_names:
                 logger.info("No monitoring functions found", project=self.project)
                 return
@@ -592,7 +596,6 @@ class MonitoringApplicationController:
                 project=project,
                 function_name=mm_constants.MonitoringFunctionNames.WRITER,
             ),
-            mm_constants.ApplicationEvent.MLRUN_CONTEXT: {},  # TODO : for future use by ad-hoc batch infer
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})