mlrun 1.7.0rc3__py3-none-any.whl → 1.7.0rc4__py3-none-any.whl

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -0,0 +1,218 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from typing import Final, Optional, Protocol
+
+import numpy as np
+from pandas import DataFrame, Timestamp
+
+from mlrun.common.schemas.model_monitoring.constants import (
+    MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME,
+    ResultKindApp,
+    ResultStatusApp,
+)
+from mlrun.model_monitoring.application import (
+    ModelMonitoringApplicationBase,
+    ModelMonitoringApplicationResult,
+)
+from mlrun.model_monitoring.batch import (
+    HellingerDistance,
+    HistogramDistanceMetric,
+    KullbackLeiblerDivergence,
+    TotalVarianceDistance,
+)
+
+
+class InvalidMetricValueError(ValueError):
+    pass
+
+
+class InvalidThresholdValueError(ValueError):
+    pass
+
+
+class ValueClassifier(Protocol):
+    def value_to_status(self, value: float) -> ResultStatusApp: ...
+
+
+@dataclass
+class DataDriftClassifier:
+    """
+    Classify data drift numeric values into categorical status.
+    """
+
+    potential: float = 0.5
+    detected: float = 0.7
+
+    def __post_init__(self) -> None:
+        """Catch erroneous threshold values"""
+        if not 0 < self.potential < self.detected < 1:
+            raise InvalidThresholdValueError(
+                "The provided thresholds do not comply with the rules"
+            )
+
+    def value_to_status(self, value: float) -> ResultStatusApp:
+        """
+        Translate the numeric value into status category.
+
+        :param value: The numeric value of the data drift metric, between 0 and 1.
+        :returns:     `ResultStatusApp` according to the classification.
+        """
+        if value > 1 or value < 0:
+            raise InvalidMetricValueError(
+                f"{value = } is invalid, must be in the range [0, 1]."
+            )
+        if value >= self.detected:
+            return ResultStatusApp.detected
+        if value >= self.potential:
+            return ResultStatusApp.potential_detection
+        return ResultStatusApp.no_detection
+
+
+class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
+    """
+    MLRun's default data drift application for model monitoring.
+
+    The application calculates the metrics over the features' histograms.
+    Each metric is calculated over all the features, the mean is taken,
+    and the status is returned.
+    """
+
+    NAME: Final[str] = MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+    METRIC_KIND: Final[ResultKindApp] = ResultKindApp.data_drift
+
+    _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
+
+    metrics: list[type[HistogramDistanceMetric]] = [
+        HellingerDistance,
+        KullbackLeiblerDivergence,
+        TotalVarianceDistance,
+    ]
+
+    def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
+        """
+        Initialize the data drift application.
+
+        :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
+                                 If not provided, the default `DataDriftClassifier()` is used.
+        """
+        self._value_classifier = value_classifier or DataDriftClassifier()
+        assert self._REQUIRED_METRICS <= set(
+            self.metrics
+        ), "TVD and Hellinger distance are required for the general data drift result"
+
+    def _compute_metrics_per_feature(
+        self, sample_df_stats: DataFrame, feature_stats: DataFrame
+    ) -> dict[type[HistogramDistanceMetric], list[float]]:
+        """Compute the metrics for the different features and labels"""
+        metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]] = {
+            metric_class: [] for metric_class in self.metrics
+        }
+
+        for (sample_feat, sample_hist), (reference_feat, reference_hist) in zip(
+            sample_df_stats.items(), feature_stats.items()
+        ):
+            assert sample_feat == reference_feat, "The features do not match"
+            self.context.logger.info(
+                "Computing metrics for feature", feature_name=sample_feat
+            )
+            sample_arr = np.asarray(sample_hist)
+            reference_arr = np.asarray(reference_hist)
+            for metric in self.metrics:
+                metric_name = metric.NAME
+                self.context.logger.debug(
+                    "Computing data drift metric",
+                    metric_name=metric_name,
+                    feature_name=sample_feat,
+                )
+                metrics_per_feature[metric].append(
+                    metric(distrib_t=sample_arr, distrib_u=reference_arr).compute()
+                )
+        self.context.logger.info("Finished computing the metrics")
+
+        return metrics_per_feature
+
+    def _add_general_drift_result(
+        self, results: list[ModelMonitoringApplicationResult], value: float
+    ) -> None:
+        results.append(
+            ModelMonitoringApplicationResult(
+                name="general_drift",
+                value=value,
+                kind=self.METRIC_KIND,
+                status=self._value_classifier.value_to_status(value),
+            )
+        )
+
+    def _get_results(
+        self, metrics_per_feature: dict[type[HistogramDistanceMetric], list[float]]
+    ) -> list[ModelMonitoringApplicationResult]:
+        """Average the metrics over the features and add the status"""
+        results: list[ModelMonitoringApplicationResult] = []
+        hellinger_tvd_values: list[float] = []
+        for metric_class, metric_values in metrics_per_feature.items():
+            self.context.logger.debug(
+                "Averaging metric over the features", metric_name=metric_class.NAME
+            )
+            value = np.mean(metric_values)
+            if metric_class == KullbackLeiblerDivergence:
+                # This metric is not bounded from above [0, inf).
+                # No status is currently reported for KL divergence
+                status = ResultStatusApp.irrelevant
+            else:
+                status = self._value_classifier.value_to_status(value)
+            if metric_class in self._REQUIRED_METRICS:
+                hellinger_tvd_values.append(value)
+            results.append(
+                ModelMonitoringApplicationResult(
+                    name=f"{metric_class.NAME}_mean",
+                    value=value,
+                    kind=self.METRIC_KIND,
+                    status=status,
+                )
+            )
+
+        self._add_general_drift_result(
+            results=results, value=np.mean(hellinger_tvd_values)
+        )
+
+        return results
+
+    def do_tracking(
+        self,
+        application_name: str,
+        sample_df_stats: DataFrame,
+        feature_stats: DataFrame,
+        sample_df: DataFrame,
+        start_infer_time: Timestamp,
+        end_infer_time: Timestamp,
+        latest_request: Timestamp,
+        endpoint_id: str,
+        output_stream_uri: str,
+    ) -> list[ModelMonitoringApplicationResult]:
+        """
+        Calculate and return the data drift metrics, averaged over the features.
+
+        Refer to `ModelMonitoringApplicationBase` for the meaning of the
+        function arguments.
+        """
+        self.context.logger.debug("Starting to run the application")
+        metrics_per_feature = self._compute_metrics_per_feature(
+            sample_df_stats=sample_df_stats, feature_stats=feature_stats
+        )
+        self.context.logger.debug("Computing average per metric")
+        results = self._get_results(metrics_per_feature)
+        self.context.logger.debug("Finished running the application", results=results)
+        return results

mlrun/model_monitoring/batch.py

@@ -11,15 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-import abc
+
 import collections
-import dataclasses
 import datetime
 import json
 import os
 import re
-from typing import Any, ClassVar, Optional, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -35,118 +33,18 @@ import mlrun.common.schemas.model_monitoring
 import mlrun.data_types.infer
 import mlrun.feature_store as fstore
 import mlrun.utils.v3io_clients
+from mlrun.model_monitoring.metrics.histogram_distance import (
+    HellingerDistance,
+    HistogramDistanceMetric,
+    KullbackLeiblerDivergence,
+    TotalVarianceDistance,
+)
 from mlrun.utils import logger
 
 # A type for representing a drift result, a tuple of the status and the drift mean:
 DriftResultType = tuple[mlrun.common.schemas.model_monitoring.DriftStatus, float]
 
 
-@dataclasses.dataclass
-class HistogramDistanceMetric(abc.ABC):
-    """
-    An abstract base class for distance metrics between histograms.
-
-    :args distrib_t: array of distribution t (usually the latest dataset distribution)
-    :args distrib_u: array of distribution u (usually the sample dataset distribution)
-
-    Each distribution must contain nonnegative floats that sum up to 1.0.
-    """
-
-    distrib_t: np.ndarray
-    distrib_u: np.ndarray
-
-    NAME: ClassVar[str]
-
-    # noinspection PyMethodOverriding
-    def __init_subclass__(cls, *, metric_name: str, **kwargs) -> None:
-        super().__init_subclass__(**kwargs)
-        cls.NAME = metric_name
-
-    @abc.abstractmethod
-    def compute(self) -> float:
-        raise NotImplementedError
-
-
-class TotalVarianceDistance(HistogramDistanceMetric, metric_name="tvd"):
-    """
-    Provides a symmetric drift distance between two periods t and u
-    Z - vector of random variables
-    Pt - Probability distribution over time span t
-    """
-
-    def compute(self) -> float:
-        """
-        Calculate Total Variance distance.
-
-        :returns:  Total Variance Distance.
-        """
-        return np.sum(np.abs(self.distrib_t - self.distrib_u)) / 2
-
-
-class HellingerDistance(HistogramDistanceMetric, metric_name="hellinger"):
-    """
-    Hellinger distance is an f divergence measure, similar to the Kullback-Leibler (KL) divergence.
-    It used to quantify the difference between two probability distributions.
-    However, unlike KL Divergence the Hellinger divergence is symmetric and bounded over a probability space.
-    The output range of Hellinger distance is [0,1]. The closer to 0, the more similar the two distributions.
-    """
-
-    def compute(self) -> float:
-        """
-        Calculate Hellinger Distance
-
-        :returns: Hellinger Distance
-        """
-        return np.sqrt(
-            max(
-                1 - np.sum(np.sqrt(self.distrib_u * self.distrib_t)),
-                0,  # numerical errors may produce small negative numbers, e.g. -1e-16.
-                # However, Cauchy-Schwarz inequality assures this number is in the range [0, 1]
-            )
-        )
-
-
-class KullbackLeiblerDivergence(HistogramDistanceMetric, metric_name="kld"):
-    """
-    KL Divergence (or relative entropy) is a measure of how one probability distribution differs from another.
-    It is an asymmetric measure (thus it's not a metric) and it doesn't satisfy the triangle inequality.
-    KL Divergence of 0, indicates two identical distributions.
-    """
-
-    @staticmethod
-    def _calc_kl_div(
-        actual_dist: np.array, expected_dist: np.array, kld_scaling: float
-    ) -> float:
-        """Return the asymmetric KL divergence"""
-        # We take 0*log(0) == 0 for this calculation
-        mask = actual_dist != 0
-        actual_dist = actual_dist[mask]
-        expected_dist = expected_dist[mask]
-        return np.sum(
-            actual_dist
-            * np.log(
-                actual_dist / np.where(expected_dist != 0, expected_dist, kld_scaling)
-            ),
-        )
-
-    def compute(
-        self, capping: Optional[float] = None, kld_scaling: float = 1e-4
-    ) -> float:
-        """
-        :param capping:      A bounded value for the KL Divergence. For infinite distance, the result is replaced with
-                             the capping value which indicates a huge differences between the distributions.
-        :param kld_scaling:  Will be used to replace 0 values for executing the logarithmic operation.
-
-        :returns: symmetric KL Divergence
-        """
-        t_u = self._calc_kl_div(self.distrib_t, self.distrib_u, kld_scaling)
-        u_t = self._calc_kl_div(self.distrib_u, self.distrib_t, kld_scaling)
-        result = t_u + u_t
-        if capping and result == float("inf"):
-            return capping
-        return result
-
-
 class VirtualDrift:
     """
     Virtual Drift object is used for handling the drift calculations.
@@ -992,7 +890,7 @@ class BatchProcessor:
         """
         stream_http_path = (
             mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
-                project=self.project
+                project=self.project, namespace=mlrun.mlconf.namespace
             )
         )
 

mlrun/model_monitoring/controller.py

@@ -20,6 +20,7 @@ import re
 from collections.abc import Iterator
 from typing import Any, NamedTuple, Optional, Union, cast
 
+import nuclio
 from v3io.dataplane.response import HttpResponseError
 
 import mlrun
@@ -29,6 +30,7 @@ import mlrun.feature_store as fstore
 from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
 from mlrun.datastore import get_stream_pusher
 from mlrun.datastore.targets import ParquetTarget
+from mlrun.errors import err_to_str
 from mlrun.model_monitoring.batch import calculate_inputs_statistics
 from mlrun.model_monitoring.helpers import (
     _BatchDict,
@@ -282,33 +284,33 @@ class MonitoringApplicationController:
 
     def __init__(
         self,
-        context: mlrun.run.MLClientCtx,
+        mlrun_context: mlrun.run.MLClientCtx,
         project: str,
     ):
         """
         Initialize Monitoring Application Processor object.
 
-        :param context:                     An MLRun context.
+        :param mlrun_context:               An MLRun context.
         :param project:                     Project name.
         """
-        self.context = context
+        self.context = mlrun_context
         self.project = project
         self.project_obj = mlrun.get_or_create_project(project)
 
-        context.logger.debug(f"Initializing {self.__class__.__name__}", project=project)
+        mlrun_context.logger.debug(
+            f"Initializing {self.__class__.__name__}", project=project
+        )
 
         self.db = mlrun.model_monitoring.get_model_endpoint_store(project=project)
 
         self._batch_window_generator = _BatchWindowGenerator(
-            batch_dict=context.parameters[
-                mm_constants.EventFieldType.BATCH_INTERVALS_DICT
-            ]
+            batch_dict=json.loads(
+                mlrun.get_secret_or_env(
+                    mm_constants.EventFieldType.BATCH_INTERVALS_DICT
+                )
+            )
         )
 
-        # If provided, only model endpoints in that that list will be analyzed
-        self.model_endpoints = context.parameters.get(
-            mm_constants.EventFieldType.MODEL_ENDPOINTS, None
-        )
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
         self.parquet_directory = get_monitoring_parquet_path(
             self.project_obj,
@@ -335,66 +337,82 @@ class MonitoringApplicationController:
             v3io_access_key=self.model_monitoring_access_key, v3io_api=self.v3io_api
         )
 
-    def run(self):
+    def run(self, event: nuclio.Event):
         """
         Main method for run all the relevant monitoring applications on each endpoint
+
+        :param event:   trigger event
         """
+        logger.info("Start running monitoring controller")
         try:
-            endpoints = self.db.list_model_endpoints(uids=self.model_endpoints)
+            applications_names = []
+            endpoints = self.db.list_model_endpoints()
+            if not endpoints:
+                self.context.logger.info(
+                    "No model endpoints found", project=self.project
+                )
+                return
             monitoring_functions = self.project_obj.list_model_monitoring_functions()
             if monitoring_functions:
+                # Gets only application in ready state
                 applications_names = list(
-                    {app.metadata.name for app in monitoring_functions}
+                    {
+                        app.metadata.name
+                        for app in monitoring_functions
+                        if app.status.state == "ready"
+                    }
                 )
-            else:
+            if not applications_names:
                 self.context.logger.info(
                     "No monitoring functions found", project=self.project
                 )
-                applications_names = []
+                return
 
         except Exception as e:
-            self.context.logger.error("Failed to list endpoints", exc=e)
-            return
-        if endpoints and applications_names:
-            # Initialize a process pool that will be used to run each endpoint applications on a dedicated process
-            pool = concurrent.futures.ProcessPoolExecutor(
-                max_workers=min(len(endpoints), 10),
+            self.context.logger.error(
+                "Failed to list endpoints and monitoring applications",
+                exc=err_to_str(e),
             )
-            futures = []
-            for endpoint in endpoints:
+            return
+        # Initialize a process pool that will be used to run each endpoint applications on a dedicated process
+        pool = concurrent.futures.ProcessPoolExecutor(
+            max_workers=min(len(endpoints), 10),
+        )
+        futures = []
+        for endpoint in endpoints:
+            if (
+                endpoint[mm_constants.EventFieldType.ACTIVE]
+                and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
+                == mm_constants.ModelMonitoringMode.enabled.value
+            ):
+                # Skip router endpoint:
                 if (
-                    endpoint[mm_constants.EventFieldType.ACTIVE]
-                    and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
-                    == mm_constants.ModelMonitoringMode.enabled.value
+                    int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
+                    == mm_constants.EndpointType.ROUTER
                 ):
-                    # Skip router endpoint:
-                    if (
-                        int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
-                        == mm_constants.EndpointType.ROUTER
-                    ):
-                        # Router endpoint has no feature stats
-                        logger.info(
-                            f"{endpoint[mm_constants.EventFieldType.UID]} is router skipping"
-                        )
-                        continue
-                    future = pool.submit(
-                        MonitoringApplicationController.model_endpoint_process,
-                        endpoint=endpoint,
-                        applications_names=applications_names,
-                        batch_window_generator=self._batch_window_generator,
-                        project=self.project,
-                        parquet_directory=self.parquet_directory,
-                        storage_options=self.storage_options,
-                        model_monitoring_access_key=self.model_monitoring_access_key,
+                    # Router endpoint has no feature stats
+                    logger.info(
+                        f"{endpoint[mm_constants.EventFieldType.UID]} is router skipping"
                     )
-                    futures.append(future)
+                    continue
+                future = pool.submit(
+                    MonitoringApplicationController.model_endpoint_process,
+                    endpoint=endpoint,
+                    applications_names=applications_names,
+                    batch_window_generator=self._batch_window_generator,
+                    project=self.project,
+                    parquet_directory=self.parquet_directory,
+                    storage_options=self.storage_options,
+                    model_monitoring_access_key=self.model_monitoring_access_key,
+                )
+                futures.append(future)
 
-            for future in concurrent.futures.as_completed(futures):
-                result = future.result()
-                if result:
-                    self.context.log_results(result)
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result:
+                self.context.log_results(result)
 
-            self._delete_old_parquet(endpoints=endpoints)
+        self._delete_old_parquet(endpoints=endpoints)
 
     @classmethod
     def model_endpoint_process(
@@ -525,7 +543,7 @@ class MonitoringApplicationController:
         """
         if self.parquet_directory.startswith("v3io:///"):
             # create fs with access to the user side (under projects)
-            store, _ = mlrun.store_manager.get_or_create_store(
+            store, _, _ = mlrun.store_manager.get_or_create_store(
                 self.parquet_directory,
                 {"V3IO_ACCESS_KEY": self.model_monitoring_access_key},
             )
@@ -601,12 +619,12 @@ class MonitoringApplicationController:
             mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
             mm_constants.ApplicationEvent.OUTPUT_STREAM_URI: get_stream_path(
                 project=project,
-                application_name=mm_constants.MonitoringFunctionNames.WRITER,
+                function_name=mm_constants.MonitoringFunctionNames.WRITER,
             ),
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})
-            stream_uri = get_stream_path(project=project, application_name=app_name)
+            stream_uri = get_stream_path(project=project, function_name=app_name)
 
             logger.info(
                 f"push endpoint_id {endpoint_id} to {app_name} by stream :{stream_uri}"

mlrun/model_monitoring/controller_handler.py

@@ -11,19 +11,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import nuclio
 
 import mlrun
 from mlrun.model_monitoring.controller import MonitoringApplicationController
 
 
-def handler(context: mlrun.run.MLClientCtx) -> None:
+def handler(context: nuclio.Context, event: nuclio.Event) -> None:
     """
     Run model monitoring application processor
 
-    :param context: the MLRun context
+    :param context: the Nuclio context
+    :param event:   trigger event
     """
+    context.user_data.monitor_app_controller.run(event)
+
+
+def init_context(context):
+    mlrun_context = mlrun.get_or_create_ctx("model_monitoring_controller")
+    mlrun_context.logger.info("Initialize monitoring app controller")
     monitor_app_controller = MonitoringApplicationController(
-        context=context,
-        project=context.project,
+        mlrun_context=mlrun_context,
+        project=mlrun_context.project,
     )
-    monitor_app_controller.run()
+    setattr(context.user_data, "monitor_app_controller", monitor_app_controller)