mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 from dataclasses import dataclass
 from typing import Final, Optional, Protocol, Union, cast
 
@@ -25,10 +24,10 @@ import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
 import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
-    EventFieldType,
     HistogramDataDriftApplicationConstants,
     ResultKindApp,
     ResultStatusApp,
+    StatsKind,
 )
 from mlrun.model_monitoring.applications import (
     ModelMonitoringApplicationBase,
@@ -103,23 +102,28 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     Each metric is calculated over all the features individually and the mean is taken as the metric value.
     The average of Hellinger and total variance distance is taken as the result.
 
-    The application logs two artifacts:
+    The application can log two artifacts (disabled by default due to performance issues):
 
-    * A JSON with the general drift per feature.
-    * A plotly table different metrics per feature.
+    * JSON with the general drift value per feature.
+    * Plotly table with the various metrics and histograms per feature.
 
-    This application is deployed by default when calling:
+    This application is deployed by default when calling
+    :py:func:`~mlrun.projects.MlrunProject.enable_model_monitoring`.
+    To avoid it, pass :code:`deploy_histogram_data_drift_app=False`.
 
-    .. code-block:: python
-
-        project.enable_model_monitoring()
-
-    To avoid it, pass `deploy_histogram_data_drift_app=False`.
+    If you want to change the application defaults, such as the classifier or which artifacts to produce, you
+    need to inherit from this class and deploy it as any other model monitoring application.
+    Please make sure to keep the default application name. This ensures that the full functionality of the application,
+    including the statistics view in the UI, is available.
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
+    _STATS_TYPES: tuple[StatsKind, StatsKind] = (
+        StatsKind.CURRENT_STATS,
+        StatsKind.DRIFT_MEASURES,
+    )
 
     metrics: list[type[HistogramDistanceMetric]] = [
         HellingerDistance,
@@ -127,16 +131,26 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         TotalVarianceDistance,
     ]
 
-    def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
+    def __init__(
+        self,
+        value_classifier: Optional[ValueClassifier] = None,
+        produce_json_artifact: bool = False,
+        produce_plotly_artifact: bool = False,
+    ) -> None:
         """
-        :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
-                                 If not provided, the default `DataDriftClassifier()` is used.
+        :param value_classifier:        Classifier object that adheres to the :py:class:`~ValueClassifier` protocol.
+                                        If not provided, the default :py:class:`~DataDriftClassifier` is used.
+        :param produce_json_artifact:   Whether to produce the JSON artifact or not, ``False`` by default.
+        :param produce_plotly_artifact: Whether to produce the Plotly artifact or not, ``False`` by default.
         """
         self._value_classifier = value_classifier or DataDriftClassifier()
         assert self._REQUIRED_METRICS <= set(
             self.metrics
         ), "TVD and Hellinger distance are required for the general data drift result"
 
+        self._produce_json_artifact = produce_json_artifact
+        self._produce_plotly_artifact = produce_plotly_artifact
+
     def _compute_metrics_per_feature(
         self, monitoring_context: mm_context.MonitoringApplicationContext
     ) -> DataFrame:
@@ -167,10 +181,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         return metrics_per_feature
 
     def _get_general_drift_result(
-        self,
-        metrics: list[mm_results.ModelMonitoringApplicationMetric],
-        monitoring_context: mm_context.MonitoringApplicationContext,
-        metrics_per_feature: DataFrame,
+        self, metrics: list[mm_results.ModelMonitoringApplicationMetric]
     ) -> mm_results.ModelMonitoringApplicationResult:
         """Get the general drift result from the metrics list"""
         value = cast(
@@ -189,21 +200,12 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         )
 
         status = self._value_classifier.value_to_status(value)
+
         return mm_results.ModelMonitoringApplicationResult(
             name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
             value=value,
             kind=ResultKindApp.data_drift,
             status=status,
-            extra_data={
-                EventFieldType.CURRENT_STATS: json.dumps(
-                    monitoring_context.sample_df_stats
-                ),
-                EventFieldType.DRIFT_MEASURES: json.dumps(
-                    metrics_per_feature.T.to_dict()
-                    | {metric.name: metric.value for metric in metrics}
-                ),
-                EventFieldType.DRIFT_STATUS: status.value,
-            },
         )
 
     @staticmethod
@@ -225,6 +227,36 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
 
         return metrics
 
+    @staticmethod
+    def _get_stats(
+        metrics: list[mm_results.ModelMonitoringApplicationMetric],
+        metrics_per_feature: DataFrame,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> list[mm_results._ModelMonitoringApplicationStats]:
+        """
+        Return a list of the statistics.
+
+        :param metrics: the calculated metrics
+        :param metrics_per_feature: metric calculated per feature
+        :param monitoring_context:  context object for current monitoring application
+        :returns: list of mm_results._ModelMonitoringApplicationStats for histogram data drift application
+        """
+        stats = []
+        for stats_type in HistogramDataDriftApplication._STATS_TYPES:
+            stats.append(
+                mm_results._ModelMonitoringApplicationStats(
+                    name=stats_type,
+                    stats=metrics_per_feature.T.to_dict()
+                    | {metric.name: metric.value for metric in metrics}
+                    if stats_type == StatsKind.DRIFT_MEASURES
+                    else monitoring_context.sample_df_stats,
+                    timestamp=monitoring_context.end_infer_time.isoformat(
+                        sep=" ", timespec="microseconds"
+                    ),
+                )
+            )
+        return stats
+
     @staticmethod
     def _get_shared_features_sample_stats(
         monitoring_context: mm_context.MonitoringApplicationContext,
@@ -275,55 +307,55 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
             cast(str, key): (self._value_classifier.value_to_status(value), value)
             for key, value in drift_per_feature_values.items()
         }
-        monitoring_context.logger.debug("Logging plotly artifact")
-        monitoring_context.log_artifact(
-            mm_drift_table.FeaturesDriftTablePlot().produce(
-                sample_set_statistics=sample_set_statistics,
-                inputs_statistics=inputs_statistics,
-                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
-                drift_results=drift_results,
-            )
+        monitoring_context.logger.debug("Producing plotly artifact")
+        artifact = mm_drift_table.FeaturesDriftTablePlot().produce(
+            sample_set_statistics=sample_set_statistics,
+            inputs_statistics=inputs_statistics,
+            metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
+            drift_results=drift_results,
         )
+        monitoring_context.logger.debug("Logging plotly artifact")
+        monitoring_context.log_artifact(artifact)
         monitoring_context.logger.debug("Logged plotly artifact successfully")
 
     def _log_drift_artifacts(
         self,
         monitoring_context: mm_context.MonitoringApplicationContext,
         metrics_per_feature: DataFrame,
-        log_json_artifact: bool = True,
     ) -> None:
         """Log JSON and Plotly drift data per feature artifacts"""
+        if not self._produce_json_artifact and not self._produce_plotly_artifact:
+            return
+
         drift_per_feature_values = metrics_per_feature[
             [HellingerDistance.NAME, TotalVarianceDistance.NAME]
         ].mean(axis=1)
 
-        if log_json_artifact:
+        if self._produce_json_artifact:
             self._log_json_artifact(drift_per_feature_values, monitoring_context)
 
-        self._log_plotly_table_artifact(
-            sample_set_statistics=self._get_shared_features_sample_stats(
-                monitoring_context
-            ),
-            inputs_statistics=monitoring_context.feature_stats,
-            metrics_per_feature=metrics_per_feature,
-            drift_per_feature_values=drift_per_feature_values,
-            monitoring_context=monitoring_context,
-        )
+        if self._produce_plotly_artifact:
+            self._log_plotly_table_artifact(
+                sample_set_statistics=self._get_shared_features_sample_stats(
+                    monitoring_context
+                ),
+                inputs_statistics=monitoring_context.feature_stats,
+                metrics_per_feature=metrics_per_feature,
+                drift_per_feature_values=drift_per_feature_values,
+                monitoring_context=monitoring_context,
+            )
 
     def do_tracking(
-        self,
-        monitoring_context: mm_context.MonitoringApplicationContext,
+        self, monitoring_context: mm_context.MonitoringApplicationContext
     ) -> list[
         Union[
             mm_results.ModelMonitoringApplicationResult,
             mm_results.ModelMonitoringApplicationMetric,
+            mm_results._ModelMonitoringApplicationStats,
         ]
     ]:
         """
         Calculate and return the data drift metrics, averaged over the features.
-
-        Refer to `ModelMonitoringApplicationBaseV2` for the meaning of the
-        function arguments.
         """
         monitoring_context.logger.debug("Starting to run the application")
         if not monitoring_context.feature_stats:
@@ -342,13 +374,14 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         )
         monitoring_context.logger.debug("Computing average per metric")
         metrics = self._get_metrics(metrics_per_feature)
-        result = self._get_general_drift_result(
+        result = self._get_general_drift_result(metrics=metrics)
+        stats = self._get_stats(
             metrics=metrics,
             monitoring_context=monitoring_context,
             metrics_per_feature=metrics_per_feature,
         )
-        metrics_and_result = metrics + [result]
+        metrics_result_and_stats = metrics + [result] + stats
         monitoring_context.logger.debug(
-            "Finished running the application", results=metrics_and_result
+            "Finished running the application", results=metrics_result_and_stats
         )
-        return metrics_and_result
+        return metrics_result_and_stats

mlrun/model_monitoring/applications/results.py

@@ -17,20 +17,26 @@ import json
 import re
 from abc import ABC, abstractmethod
 
+from pydantic.v1 import validator
+from pydantic.v1.dataclasses import dataclass
+
 import mlrun.common.helpers
 import mlrun.common.model_monitoring.helpers
-import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.utils.v3io_clients
+from mlrun.utils import logger
+
+_RESULT_EXTRA_DATA_MAX_SIZE = 998
 
 
 class _ModelMonitoringApplicationDataRes(ABC):
     name: str
 
     def __post_init__(self):
-        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
+        pat = re.compile(mm_constants.RESULT_NAME_PATTERN)
         if not re.fullmatch(pat, self.name):
             raise mlrun.errors.MLRunValueError(
-                "Attribute name must comply with the regex `[a-zA-Z_][a-zA-Z0-9_]*`"
+                f"Attribute name must comply with the regex `{mm_constants.RESULT_NAME_PATTERN}`"
             )
 
     @abstractmethod
@@ -38,7 +44,7 @@ class _ModelMonitoringApplicationDataRes(ABC):
         raise NotImplementedError
 
 
-@dataclasses.dataclass
+@dataclass
 class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
     """
     Class representing the result of a custom model monitoring application.
@@ -49,13 +55,16 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
     :param value:          (float) Value of the application result.
     :param kind:           (ResultKindApp) Kind of application result.
     :param status:         (ResultStatusApp) Status of the application result.
-    :param extra_data:     (dict) Extra data associated with the application result.
+    :param extra_data:     (dict) Extra data associated with the application result. Note that if the extra data is
+                                  exceeding the maximum size of 998 characters, it will be ignored and a message will
+                                  be logged. In this case, we recommend logging the extra data as a separate artifact or
+                                  shortening it.
     """
 
     name: str
     value: float
-    kind: mm_constant.ResultKindApp
-    status: mm_constant.ResultStatusApp
+    kind: mm_constants.ResultKindApp
+    status: mm_constants.ResultStatusApp
     extra_data: dict = dataclasses.field(default_factory=dict)
 
     def to_dict(self):
@@ -65,15 +74,30 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
         :returns:    (dict) Dictionary representation of the result.
         """
         return {
-            mm_constant.ResultData.RESULT_NAME: self.name,
-            mm_constant.ResultData.RESULT_VALUE: self.value,
-            mm_constant.ResultData.RESULT_KIND: self.kind.value,
-            mm_constant.ResultData.RESULT_STATUS: self.status.value,
-            mm_constant.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
+            mm_constants.ResultData.RESULT_NAME: self.name,
+            mm_constants.ResultData.RESULT_VALUE: self.value,
+            mm_constants.ResultData.RESULT_KIND: self.kind.value,
+            mm_constants.ResultData.RESULT_STATUS: self.status.value,
+            mm_constants.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
         }
 
+    @validator("extra_data")
+    @classmethod
+    def validate_extra_data_len(cls, result_extra_data: dict):
+        """Ensure that the extra data is not exceeding the maximum size which is important to avoid
+        possible storage issues."""
+        extra_data_len = len(json.dumps(result_extra_data))
+        if extra_data_len > _RESULT_EXTRA_DATA_MAX_SIZE:
+            logger.warning(
+                f"Extra data is too long and won't be stored: {extra_data_len} characters while the maximum "
+                f"is {_RESULT_EXTRA_DATA_MAX_SIZE} characters."
+                f"Please shorten the extra data or log it as a separate artifact."
+            )
+            return {}
+        return result_extra_data
 
-@dataclasses.dataclass
+
+@dataclass
 class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
     """
     Class representing a single metric of a custom model monitoring application.
@@ -94,6 +118,34 @@ class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
         :returns:    (dict) Dictionary representation of the result.
         """
         return {
-            mm_constant.MetricData.METRIC_NAME: self.name,
-            mm_constant.MetricData.METRIC_VALUE: self.value,
+            mm_constants.MetricData.METRIC_NAME: self.name,
+            mm_constants.MetricData.METRIC_VALUE: self.value,
+        }
+
+
+@dataclasses.dataclass
+class _ModelMonitoringApplicationStats(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing the stats of histogram data drift application.
+
+    :param name             (mm_constant.StatsKind) Enum mm_constant.StatsData of the stats data kind of the event
+    :param                  (str) iso format representation of the timestamp the event took place
+    :param stats            (dict) Dictionary representation of the stats calculated for the event
+
+    """
+
+    name: mm_constants.StatsKind
+    timestamp: str
+    stats: dict = dataclasses.field(default_factory=dict)
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constants.StatsData.STATS_NAME: self.name,
+            mm_constants.StatsData.STATS: self.stats,
+            mm_constants.StatsData.TIMESTAMP: self.timestamp,
         }