mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py}

@@ -14,15 +14,16 @@
 
 import uuid
 import warnings
-from typing import Union
+from abc import ABC
 
 import pandas as pd
 import semver
 
+import mlrun.model_monitoring.applications.base as mm_base
+import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
-from mlrun.model_monitoring.application import ModelMonitoringApplicationBase
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.39")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -56,15 +57,15 @@ except ModuleNotFoundError:
 
 
 if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
-    from evidently.report.report import Report
-    from evidently.suite.base_suite import Suite
+    from evidently.suite.base_suite import Display
     from evidently.ui.type_aliases import STR_UUID
     from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
+    from evidently.utils.dashboard import TemplateParams, file_html_template
 
 
-class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
+class EvidentlyModelMonitoringApplicationBase(
+    mm_base.ModelMonitoringApplicationBase, ABC
+):
     def __init__(
         self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
     ) -> None:
@@ -77,6 +78,8 @@ class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
         :param evidently_project_id:        (str) The ID of the Evidently project.
 
         """
+
+        # TODO : more then one project (mep -> project)
         if not _HAS_EVIDENTLY:
             raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
         self.evidently_workspace = Workspace.create(evidently_workspace_path)
@@ -85,32 +88,38 @@ class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
             evidently_project_id
         )
 
+    @staticmethod
     def log_evidently_object(
-        self, evidently_object: Union["Report", "Suite"], artifact_name: str
-    ):
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        evidently_object: "Display",
+        artifact_name: str,
+    ) -> None:
         """
          Logs an Evidently report or suite as an artifact.
 
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
+        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
+        :param evidently_object:    (Display) The Evidently display to log, e.g. a report or a test suite object.
         :param artifact_name:       (str) The name for the logged artifact.
         """
         evidently_object_html = evidently_object.get_html()
-        self.context.log_artifact(
+        monitoring_context.log_artifact(
             artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
         )
 
     def log_project_dashboard(
         self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
         timestamp_start: pd.Timestamp,
         timestamp_end: pd.Timestamp,
         artifact_name: str = "dashboard",
-    ):
+    ) -> None:
         """
         Logs an Evidently project dashboard.
 
-        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
-        :param artifact_name:   (str) The name for the logged artifact.
+        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
+        :param timestamp_start:     (pd.Timestamp) The start timestamp for the dashboard data.
+        :param timestamp_end:       (pd.Timestamp) The end timestamp for the dashboard data.
+        :param artifact_name:       (str) The name for the logged artifact.
         """
 
         dashboard_info = self.evidently_project.build_dashboard_info(
@@ -122,11 +131,7 @@ class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
             additional_graphs={},
         )
 
-        dashboard_html = self._render(determine_template("inline"), template_params)
-        self.context.log_artifact(
+        dashboard_html = file_html_template(params=template_params)
+        monitoring_context.log_artifact(
             artifact_name, body=dashboard_html.encode("utf-8"), format="html"
         )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -12,22 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 from dataclasses import dataclass
-from typing import Final, Optional, Protocol
+from typing import Final, Optional, Protocol, Union, cast
 
 import numpy as np
-from pandas import DataFrame, Timestamp
+from pandas import DataFrame, Series
 
+import mlrun.artifacts
+import mlrun.common.model_monitoring.helpers
+import mlrun.model_monitoring.applications.context as mm_context
+import mlrun.model_monitoring.applications.results as mm_results
+import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
-    MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME,
+    EventFieldType,
+    HistogramDataDriftApplicationConstants,
     ResultKindApp,
     ResultStatusApp,
 )
-from mlrun.model_monitoring.application import (
+from mlrun.model_monitoring.applications import (
     ModelMonitoringApplicationBase,
-    ModelMonitoringApplicationResult,
 )
-from mlrun.model_monitoring.batch import (
+from mlrun.model_monitoring.metrics.histogram_distance import (
     HellingerDistance,
     HistogramDistanceMetric,
     KullbackLeiblerDivergence,
@@ -85,13 +91,33 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     """
     MLRun's default data drift application for model monitoring.
 
-    The application calculates the metrics over the features' histograms.
-    Each metric is calculated over all the features, the mean is taken,
-    and the status is returned.
+    The application expects tabular numerical data, and calculates three metrics over the shared features' histograms.
+    The metrics are calculated on features that have reference data from the training dataset. When there is no
+    reference data (`feature_stats`), this application send a warning log and does nothing.
+    The three metrics are:
+
+    * Hellinger distance.
+    * Total variance distance.
+    * Kullback-Leibler divergence.
+
+    Each metric is calculated over all the features individually and the mean is taken as the metric value.
+    The average of Hellinger and total variance distance is taken as the result.
+
+    The application logs two artifacts:
+
+    * A JSON with the general drift per feature.
+    * A plotly table different metrics per feature.
+
+    This application is deployed by default when calling:
+
+    .. code-block:: python
+
+        project.enable_model_monitoring()
+
+    To avoid it, pass `deploy_histogram_data_drift_app=False`.
     """
 
-    NAME: Final[str] = MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
-    METRIC_KIND: Final[ResultKindApp] = ResultKindApp.data_drift
+    NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
 
@@ -103,8 +129,6 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
 
     def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
         """
-        Initialize the data drift application.
-
         :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
                                  If not provided, the default `DataDriftClassifier()` is used.
         """
@@ -114,105 +138,217 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         ), "TVD and Hellinger distance are required for the general data drift result"
 
     def _compute_metrics_per_feature(
-        self, sample_df_stats: DataFrame, feature_stats: DataFrame
-    ) -> dict[type[HistogramDistanceMetric], list[float]]:
+        self, monitoring_context: mm_context.MonitoringApplicationContext
+    ) -> DataFrame:
         """Compute the metrics for the different features and labels"""
-        metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]] = {
-            metric_class: [] for metric_class in self.metrics
-        }
-
-        for (sample_feat, sample_hist), (reference_feat, reference_hist) in zip(
-            sample_df_stats.items(), feature_stats.items()
-        ):
-            assert sample_feat == reference_feat, "The features do not match"
-            self.context.logger.info(
-                "Computing metrics for feature", feature_name=sample_feat
+        metrics_per_feature = DataFrame(
+            columns=[metric_class.NAME for metric_class in self.metrics]
+        )
+        feature_stats = monitoring_context.dict_to_histogram(
+            monitoring_context.feature_stats
+        )
+        sample_df_stats = monitoring_context.dict_to_histogram(
+            monitoring_context.sample_df_stats
+        )
+        for feature_name in feature_stats:
+            sample_hist = np.asarray(sample_df_stats[feature_name])
+            reference_hist = np.asarray(feature_stats[feature_name])
+            monitoring_context.logger.info(
+                "Computing metrics for feature", feature_name=feature_name
             )
-            sample_arr = np.asarray(sample_hist)
-            reference_arr = np.asarray(reference_hist)
-            for metric in self.metrics:
-                metric_name = metric.NAME
-                self.context.logger.debug(
-                    "Computing data drift metric",
-                    metric_name=metric_name,
-                    feature_name=sample_feat,
-                )
-                metrics_per_feature[metric].append(
-                    metric(distrib_t=sample_arr, distrib_u=reference_arr).compute()
-                )
-        self.context.logger.info("Finished computing the metrics")
+            metrics_per_feature.loc[feature_name] = {  # pyright: ignore[reportCallIssue,reportArgumentType]
+                metric.NAME: metric(
+                    distrib_t=sample_hist, distrib_u=reference_hist
+                ).compute()
+                for metric in self.metrics
+            }
+        monitoring_context.logger.info("Finished computing the metrics")
 
         return metrics_per_feature
 
-    def _add_general_drift_result(
-        self, results: list[ModelMonitoringApplicationResult], value: float
-    ) -> None:
-        results.append(
-            ModelMonitoringApplicationResult(
-                name="general_drift",
-                value=value,
-                kind=self.METRIC_KIND,
-                status=self._value_classifier.value_to_status(value),
-            )
+    def _get_general_drift_result(
+        self,
+        metrics: list[mm_results.ModelMonitoringApplicationMetric],
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+    ) -> mm_results.ModelMonitoringApplicationResult:
+        """Get the general drift result from the metrics list"""
+        value = cast(
+            float,
+            np.mean(
+                [
+                    metric.value
+                    for metric in metrics
+                    if metric.name
+                    in [
+                        f"{HellingerDistance.NAME}_mean",
+                        f"{TotalVarianceDistance.NAME}_mean",
+                    ]
+                ]
+            ),
+        )
+
+        status = self._value_classifier.value_to_status(value)
+        return mm_results.ModelMonitoringApplicationResult(
+            name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+            value=value,
+            kind=ResultKindApp.data_drift,
+            status=status,
+            extra_data={
+                EventFieldType.CURRENT_STATS: json.dumps(
+                    monitoring_context.sample_df_stats
+                ),
+                EventFieldType.DRIFT_MEASURES: json.dumps(
+                    metrics_per_feature.T.to_dict()
+                    | {metric.name: metric.value for metric in metrics}
+                ),
+                EventFieldType.DRIFT_STATUS: status.value,
+            },
         )
 
-    def _get_results(
-        self, metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]]
-    ) -> list[ModelMonitoringApplicationResult]:
+    @staticmethod
+    def _get_metrics(
+        metrics_per_feature: DataFrame,
+    ) -> list[mm_results.ModelMonitoringApplicationMetric]:
         """Average the metrics over the features and add the status"""
-        results: list[ModelMonitoringApplicationResult] = []
-        hellinger_tvd_values: list[float] = []
-        for metric_class, metric_values in metrics_per_feature.items():
-            self.context.logger.debug(
-                "Averaging metric over the features", metric_name=metric_class.NAME
-            )
-            value = np.mean(metric_values)
-            if metric_class == KullbackLeiblerDivergence:
-                # This metric is not bounded from above [0, inf).
-                # No status is currently reported for KL divergence
-                status = ResultStatusApp.irrelevant
-            else:
-                status = self._value_classifier.value_to_status(value)
-            if metric_class in self._REQUIRED_METRICS:
-                hellinger_tvd_values.append(value)
-            results.append(
-                ModelMonitoringApplicationResult(
-                    name=f"{metric_class.NAME}_mean",
+        metrics: list[mm_results.ModelMonitoringApplicationMetric] = []
+
+        metrics_mean = metrics_per_feature.mean().to_dict()
+
+        for name, value in metrics_mean.items():
+            metrics.append(
+                mm_results.ModelMonitoringApplicationMetric(
+                    name=f"{name}_mean",
                     value=value,
-                    kind=self.METRIC_KIND,
-                    status=status,
                 )
             )
 
-        self._add_general_drift_result(
-            results=results, value=np.mean(hellinger_tvd_values)
+        return metrics
+
+    @staticmethod
+    def _get_shared_features_sample_stats(
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> mlrun.common.model_monitoring.helpers.FeatureStats:
+        """
+        Filter out features without reference data in `feature_stats`, e.g. `timestamp`.
+        """
+        return mlrun.common.model_monitoring.helpers.FeatureStats(
+            {
+                key: monitoring_context.sample_df_stats[key]
+                for key in monitoring_context.feature_stats
+            }
+        )
+
+    @staticmethod
+    def _log_json_artifact(
+        drift_per_feature_values: Series,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> None:
+        """Log the drift values as a JSON artifact"""
+        monitoring_context.logger.debug("Logging drift value per feature JSON artifact")
+        monitoring_context.log_artifact(
+            mlrun.artifacts.Artifact(
+                body=drift_per_feature_values.to_json(),
+                format="json",
+                key="features_drift_results",
+            )
         )
+        monitoring_context.logger.debug("Logged JSON artifact successfully")
 
-        return results
+    def _log_plotly_table_artifact(
+        self,
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        inputs_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        metrics_per_feature: DataFrame,
+        drift_per_feature_values: Series,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> None:
+        """Log the Plotly drift table artifact"""
+        monitoring_context.logger.debug(
+            "Feature stats",
+            sample_set_statistics=sample_set_statistics,
+            inputs_statistics=inputs_statistics,
+        )
+
+        monitoring_context.logger.debug("Computing drift results per feature")
+        drift_results = {
+            cast(str, key): (self._value_classifier.value_to_status(value), value)
+            for key, value in drift_per_feature_values.items()
+        }
+        monitoring_context.logger.debug("Logging plotly artifact")
+        monitoring_context.log_artifact(
+            mm_drift_table.FeaturesDriftTablePlot().produce(
+                sample_set_statistics=sample_set_statistics,
+                inputs_statistics=inputs_statistics,
+                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
+                drift_results=drift_results,
+            )
+        )
+        monitoring_context.logger.debug("Logged plotly artifact successfully")
+
+    def _log_drift_artifacts(
+        self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+        log_json_artifact: bool = True,
+    ) -> None:
+        """Log JSON and Plotly drift data per feature artifacts"""
+        drift_per_feature_values = metrics_per_feature[
+            [HellingerDistance.NAME, TotalVarianceDistance.NAME]
+        ].mean(axis=1)
+
+        if log_json_artifact:
+            self._log_json_artifact(drift_per_feature_values, monitoring_context)
+
+        self._log_plotly_table_artifact(
+            sample_set_statistics=self._get_shared_features_sample_stats(
+                monitoring_context
+            ),
+            inputs_statistics=monitoring_context.feature_stats,
+            metrics_per_feature=metrics_per_feature,
+            drift_per_feature_values=drift_per_feature_values,
+            monitoring_context=monitoring_context,
+        )
 
     def do_tracking(
         self,
-        application_name: str,
-        sample_df_stats: DataFrame,
-        feature_stats: DataFrame,
-        sample_df: DataFrame,
-        start_infer_time: Timestamp,
-        end_infer_time: Timestamp,
-        latest_request: Timestamp,
-        endpoint_id: str,
-        output_stream_uri: str,
-    ) -> list[ModelMonitoringApplicationResult]:
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> list[
+        Union[
+            mm_results.ModelMonitoringApplicationResult,
+            mm_results.ModelMonitoringApplicationMetric,
+        ]
+    ]:
         """
         Calculate and return the data drift metrics, averaged over the features.
 
-        Refer to `ModelMonitoringApplicationBase` for the meaning of the
+        Refer to `ModelMonitoringApplicationBaseV2` for the meaning of the
         function arguments.
         """
-        self.context.logger.debug("Starting to run the application")
+        monitoring_context.logger.debug("Starting to run the application")
+        if not monitoring_context.feature_stats:
+            monitoring_context.logger.warning(
+                "No feature statistics found, skipping the application. \n"
+                "In order to run the application, training set must be provided when logging the model."
+            )
+            return []
         metrics_per_feature = self._compute_metrics_per_feature(
-            sample_df_stats=sample_df_stats, feature_stats=feature_stats
+            monitoring_context=monitoring_context
+        )
+        monitoring_context.logger.debug("Saving artifacts")
+        self._log_drift_artifacts(
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
+        )
+        monitoring_context.logger.debug("Computing average per metric")
+        metrics = self._get_metrics(metrics_per_feature)
+        result = self._get_general_drift_result(
+            metrics=metrics,
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
+        )
+        metrics_and_result = metrics + [result]
+        monitoring_context.logger.debug(
+            "Finished running the application", results=metrics_and_result
         )
-        self.context.logger.debug("Computing average per metric")
-        results = self._get_results(metrics_per_feature)
-        self.context.logger.debug("Finished running the application", results=results)
-        return results
+        return metrics_and_result

mlrun/model_monitoring/applications/results.py

@@ -0,0 +1,99 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import dataclasses
+import json
+import re
+from abc import ABC, abstractmethod
+
+import mlrun.common.helpers
+import mlrun.common.model_monitoring.helpers
+import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.utils.v3io_clients
+
+
+class _ModelMonitoringApplicationDataRes(ABC):
+    name: str
+
+    def __post_init__(self):
+        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
+        if not re.fullmatch(pat, self.name):
+            raise mlrun.errors.MLRunValueError(
+                "Attribute name must comply with the regex `[a-zA-Z_][a-zA-Z0-9_]*`"
+            )
+
+    @abstractmethod
+    def to_dict(self):
+        raise NotImplementedError
+
+
+@dataclasses.dataclass
+class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing the result of a custom model monitoring application.
+
+    :param name:           (str) Name of the application result. This name must be
+                            unique for each metric in a single application
+                            (name must be of the format :code:`[a-zA-Z_][a-zA-Z0-9_]*`).
+    :param value:          (float) Value of the application result.
+    :param kind:           (ResultKindApp) Kind of application result.
+    :param status:         (ResultStatusApp) Status of the application result.
+    :param extra_data:     (dict) Extra data associated with the application result.
+    """
+
+    name: str
+    value: float
+    kind: mm_constant.ResultKindApp
+    status: mm_constant.ResultStatusApp
+    extra_data: dict = dataclasses.field(default_factory=dict)
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constant.ResultData.RESULT_NAME: self.name,
+            mm_constant.ResultData.RESULT_VALUE: self.value,
+            mm_constant.ResultData.RESULT_KIND: self.kind.value,
+            mm_constant.ResultData.RESULT_STATUS: self.status.value,
+            mm_constant.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
+        }
+
+
+@dataclasses.dataclass
+class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing a single metric of a custom model monitoring application.
+
+    :param name:           (str) Name of the application metric. This name must be
+                            unique for each metric in a single application
+                            (name must be of the format :code:`[a-zA-Z_][a-zA-Z0-9_]*`).
+    :param value:          (float) Value of the application metric.
+    """
+
+    name: str
+    value: float
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constant.MetricData.METRIC_NAME: self.name,
+            mm_constant.MetricData.METRIC_VALUE: self.value,
+        }