mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 from dataclasses import dataclass
 from typing import Final, Optional, Protocol, Union, cast
 
@@ -25,10 +24,10 @@ import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
 import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
-    EventFieldType,
     HistogramDataDriftApplicationConstants,
     ResultKindApp,
     ResultStatusApp,
+    StatsKind,
 )
 from mlrun.model_monitoring.applications import (
     ModelMonitoringApplicationBase,
@@ -114,12 +113,13 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
 
         project.enable_model_monitoring()
 
-    To avoid it, pass `deploy_histogram_data_drift_app=False`.
+    To avoid it, pass :code:`deploy_histogram_data_drift_app=False`.
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
+    _STATS_TYPES: tuple[StatsKind] = (StatsKind.CURRENT_STATS, StatsKind.DRIFT_MEASURES)
 
     metrics: list[type[HistogramDistanceMetric]] = [
         HellingerDistance,
@@ -189,21 +189,12 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         )
 
         status = self._value_classifier.value_to_status(value)
+
         return mm_results.ModelMonitoringApplicationResult(
             name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
             value=value,
             kind=ResultKindApp.data_drift,
             status=status,
-            extra_data={
-                EventFieldType.CURRENT_STATS: json.dumps(
-                    monitoring_context.sample_df_stats
-                ),
-                EventFieldType.DRIFT_MEASURES: json.dumps(
-                    metrics_per_feature.T.to_dict()
-                    | {metric.name: metric.value for metric in metrics}
-                ),
-                EventFieldType.DRIFT_STATUS: status.value,
-            },
         )
 
     @staticmethod
@@ -225,6 +216,35 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
 
         return metrics
 
+    @staticmethod
+    def _get_stats(
+        metrics: list[mm_results.ModelMonitoringApplicationMetric],
+        metrics_per_feature: DataFrame,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> list[mm_results._ModelMonitoringApplicationStats]:
+        """
+        list the application calculated stats
+        :param metrics: the calculated metrics
+        :param metrics_per_feature: metric calculated per feature
+        :param monitoring_context:  context object for current monitoring application
+        :returns: list of mm_results._ModelMonitoringApplicationStats for histogram data drift application
+        """
+        stats = []
+        for stats_type in HistogramDataDriftApplication._STATS_TYPES:
+            stats.append(
+                mm_results._ModelMonitoringApplicationStats(
+                    name=stats_type,
+                    stats=metrics_per_feature.T.to_dict()
+                    | {metric.name: metric.value for metric in metrics}
+                    if stats_type == StatsKind.DRIFT_MEASURES
+                    else monitoring_context.sample_df_stats,
+                    timestamp=monitoring_context.end_infer_time.isoformat(
+                        sep=" ", timespec="microseconds"
+                    ),
+                )
+            )
+        return stats
+
     @staticmethod
     def _get_shared_features_sample_stats(
         monitoring_context: mm_context.MonitoringApplicationContext,
@@ -311,19 +331,16 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         )
 
     def do_tracking(
-        self,
-        monitoring_context: mm_context.MonitoringApplicationContext,
+        self, monitoring_context: mm_context.MonitoringApplicationContext
     ) -> list[
         Union[
             mm_results.ModelMonitoringApplicationResult,
             mm_results.ModelMonitoringApplicationMetric,
+            mm_results._ModelMonitoringApplicationStats,
         ]
     ]:
         """
         Calculate and return the data drift metrics, averaged over the features.
-
-        Refer to `ModelMonitoringApplicationBaseV2` for the meaning of the
-        function arguments.
         """
         monitoring_context.logger.debug("Starting to run the application")
         if not monitoring_context.feature_stats:
@@ -347,8 +364,13 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
             monitoring_context=monitoring_context,
             metrics_per_feature=metrics_per_feature,
         )
-        metrics_and_result = metrics + [result]
+        stats = self._get_stats(
+            metrics=metrics,
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
+        )
+        metrics_result_and_stats = metrics + [result] + stats
         monitoring_context.logger.debug(
-            "Finished running the application", results=metrics_and_result
+            "Finished running the application", results=metrics_result_and_stats
         )
-        return metrics_and_result
+        return metrics_result_and_stats

mlrun/model_monitoring/applications/results.py

@@ -17,10 +17,16 @@ import json
 import re
 from abc import ABC, abstractmethod
 
+from pydantic.v1 import validator
+from pydantic.v1.dataclasses import dataclass
+
 import mlrun.common.helpers
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constant
 import mlrun.utils.v3io_clients
+from mlrun.utils import logger
+
+_RESULT_EXTRA_DATA_MAX_SIZE = 998
 
 
 class _ModelMonitoringApplicationDataRes(ABC):
@@ -38,7 +44,7 @@ class _ModelMonitoringApplicationDataRes(ABC):
         raise NotImplementedError
 
 
-@dataclasses.dataclass
+@dataclass
 class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
     """
     Class representing the result of a custom model monitoring application.
@@ -49,7 +55,10 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
     :param value:          (float) Value of the application result.
     :param kind:           (ResultKindApp) Kind of application result.
     :param status:         (ResultStatusApp) Status of the application result.
-    :param extra_data:     (dict) Extra data associated with the application result.
+    :param extra_data:     (dict) Extra data associated with the application result. Note that if the extra data is
+                                  exceeding the maximum size of 998 characters, it will be ignored and a message will
+                                  be logged. In this case, we recommend logging the extra data as a separate artifact or
+                                  shortening it.
     """
 
     name: str
@@ -72,8 +81,23 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
             mm_constant.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
         }
 
+    @validator("extra_data")
+    @classmethod
+    def validate_extra_data_len(cls, result_extra_data: dict):
+        """Ensure that the extra data is not exceeding the maximum size which is important to avoid
+        possible storage issues."""
+        extra_data_len = len(json.dumps(result_extra_data))
+        if extra_data_len > _RESULT_EXTRA_DATA_MAX_SIZE:
+            logger.warning(
+                f"Extra data is too long and won't be stored: {extra_data_len} characters while the maximum "
+                f"is {_RESULT_EXTRA_DATA_MAX_SIZE} characters."
+                f"Please shorten the extra data or log it as a separate artifact."
+            )
+            return {}
+        return result_extra_data
 
-@dataclasses.dataclass
+
+@dataclass
 class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
     """
     Class representing a single metric of a custom model monitoring application.
@@ -97,3 +121,31 @@ class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
             mm_constant.MetricData.METRIC_NAME: self.name,
             mm_constant.MetricData.METRIC_VALUE: self.value,
         }
+
+
+@dataclasses.dataclass
+class _ModelMonitoringApplicationStats(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing the stats of histogram data drift application.
+
+    :param name             (mm_constant.StatsKind) Enum mm_constant.StatsData of the stats data kind of the event
+    :param                  (str) iso format representation of the timestamp the event took place
+    :param stats            (dict) Dictionary representation of the stats calculated for the event
+
+    """
+
+    name: mm_constant.StatsKind
+    timestamp: str
+    stats: dict = dataclasses.field(default_factory=dict)
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constant.StatsData.STATS_NAME: self.name,
+            mm_constant.StatsData.STATS: self.stats,
+            mm_constant.StatsData.TIMESTAMP: self.timestamp,
+        }