mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/model_monitoring/applications/evidently_base.py

@@ -0,0 +1,211 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+import warnings
+from typing import Union
+
+import pandas as pd
+import semver
+
+import mlrun.model_monitoring.applications.base as mm_base
+import mlrun.model_monitoring.applications.context as mm_context
+from mlrun.errors import MLRunIncompatibleVersionError
+
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
+
+
+def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
+    if ref.is_compatible(cur) or (
+        cur.major == ref.major == 0 and cur.minor == ref.minor and cur.patch > ref.patch
+    ):
+        return
+    if cur.major == ref.major == 0 and cur.minor > ref.minor:
+        warnings.warn(
+            f"Evidently version {cur} is not compatible with the tested "
+            f"version {ref}, use at your own risk."
+        )
+    else:
+        raise MLRunIncompatibleVersionError(
+            f"Evidently version {cur} is not supported, please change to "
+            f"{ref} (or another compatible version)."
+        )
+
+
+_HAS_EVIDENTLY = False
+try:
+    import evidently  # noqa: F401
+
+    _check_evidently_version(
+        cur=semver.Version.parse(evidently.__version__),
+        ref=SUPPORTED_EVIDENTLY_VERSION,
+    )
+    _HAS_EVIDENTLY = True
+except ModuleNotFoundError:
+    pass
+
+
+if _HAS_EVIDENTLY:
+    from evidently.renderers.notebook_utils import determine_template
+    from evidently.report.report import Report
+    from evidently.suite.base_suite import Suite
+    from evidently.ui.type_aliases import STR_UUID
+    from evidently.ui.workspace import Workspace
+    from evidently.utils.dashboard import TemplateParams
+
+
+class EvidentlyModelMonitoringApplicationBase(mm_base.ModelMonitoringApplicationBase):
+    def __init__(
+        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
+    ) -> None:
+        """
+        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
+        Note: evidently is not installed by default in the mlrun/mlrun image.
+        It must be installed separately to use this class.
+
+        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
+        :param evidently_project_id:        (str) The ID of the Evidently project.
+
+        """
+        if not _HAS_EVIDENTLY:
+            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
+        self.evidently_workspace = Workspace.create(evidently_workspace_path)
+        self.evidently_project_id = evidently_project_id
+        self.evidently_project = self.evidently_workspace.get_project(
+            evidently_project_id
+        )
+
+    def log_evidently_object(
+        self, evidently_object: Union["Report", "Suite"], artifact_name: str
+    ):
+        """
+         Logs an Evidently report or suite as an artifact.
+
+        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
+        :param artifact_name:       (str) The name for the logged artifact.
+        """
+        evidently_object_html = evidently_object.get_html()
+        self.context.log_artifact(
+            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
+        )
+
+    def log_project_dashboard(
+        self,
+        timestamp_start: pd.Timestamp,
+        timestamp_end: pd.Timestamp,
+        artifact_name: str = "dashboard",
+    ):
+        """
+        Logs an Evidently project dashboard.
+
+        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
+        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
+        :param artifact_name:   (str) The name for the logged artifact.
+        """
+
+        dashboard_info = self.evidently_project.build_dashboard_info(
+            timestamp_start, timestamp_end
+        )
+        template_params = TemplateParams(
+            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
+            dashboard_info=dashboard_info,
+            additional_graphs={},
+        )
+
+        dashboard_html = self._render(determine_template("inline"), template_params)
+        self.context.log_artifact(
+            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
+        )
+
+    @staticmethod
+    def _render(temple_func, template_params: "TemplateParams"):
+        return temple_func(params=template_params)
+
+
+class EvidentlyModelMonitoringApplicationBaseV2(
+    mm_base.ModelMonitoringApplicationBaseV2
+):
+    def __init__(
+        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
+    ) -> None:
+        """
+        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
+        Note: evidently is not installed by default in the mlrun/mlrun image.
+        It must be installed separately to use this class.
+
+        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
+        :param evidently_project_id:        (str) The ID of the Evidently project.
+
+        """
+
+        # TODO : more then one project (mep -> project)
+        if not _HAS_EVIDENTLY:
+            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
+        self.evidently_workspace = Workspace.create(evidently_workspace_path)
+        self.evidently_project_id = evidently_project_id
+        self.evidently_project = self.evidently_workspace.get_project(
+            evidently_project_id
+        )
+
+    @staticmethod
+    def log_evidently_object(
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        evidently_object: Union["Report", "Suite"],
+        artifact_name: str,
+    ):
+        """
+         Logs an Evidently report or suite as an artifact.
+
+        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
+        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
+        :param artifact_name:       (str) The name for the logged artifact.
+        """
+        evidently_object_html = evidently_object.get_html()
+        monitoring_context.log_artifact(
+            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
+        )
+
+    def log_project_dashboard(
+        self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        timestamp_start: pd.Timestamp,
+        timestamp_end: pd.Timestamp,
+        artifact_name: str = "dashboard",
+    ):
+        """
+        Logs an Evidently project dashboard.
+
+        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
+        :param timestamp_start:     (pd.Timestamp) The start timestamp for the dashboard data.
+        :param timestamp_end:       (pd.Timestamp) The end timestamp for the dashboard data.
+        :param artifact_name:       (str) The name for the logged artifact.
+        """
+
+        dashboard_info = self.evidently_project.build_dashboard_info(
+            timestamp_start, timestamp_end
+        )
+        template_params = TemplateParams(
+            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
+            dashboard_info=dashboard_info,
+            additional_graphs={},
+        )
+
+        dashboard_html = self._render(determine_template("inline"), template_params)
+        monitoring_context.log_artifact(
+            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
+        )
+
+    @staticmethod
+    def _render(temple_func, template_params: "TemplateParams"):
+        return temple_func(params=template_params)

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -0,0 +1,349 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from dataclasses import dataclass
+from typing import Final, Optional, Protocol, Union, cast
+
+import numpy as np
+from pandas import DataFrame, Series
+
+import mlrun.artifacts
+import mlrun.common.model_monitoring.helpers
+import mlrun.model_monitoring.applications.context as mm_context
+import mlrun.model_monitoring.applications.results as mm_results
+import mlrun.model_monitoring.features_drift_table as mm_drift_table
+from mlrun.common.schemas.model_monitoring.constants import (
+    EventFieldType,
+    HistogramDataDriftApplicationConstants,
+    ResultKindApp,
+    ResultStatusApp,
+)
+from mlrun.model_monitoring.applications import (
+    ModelMonitoringApplicationBaseV2,
+)
+from mlrun.model_monitoring.metrics.histogram_distance import (
+    HellingerDistance,
+    HistogramDistanceMetric,
+    KullbackLeiblerDivergence,
+    TotalVarianceDistance,
+)
+
+
+class InvalidMetricValueError(ValueError):
+    pass
+
+
+class InvalidThresholdValueError(ValueError):
+    pass
+
+
+class ValueClassifier(Protocol):
+    def value_to_status(self, value: float) -> ResultStatusApp: ...
+
+
+@dataclass
+class DataDriftClassifier:
+    """
+    Classify data drift numeric values into categorical status.
+    """
+
+    potential: float = 0.5
+    detected: float = 0.7
+
+    def __post_init__(self) -> None:
+        """Catch erroneous threshold values"""
+        if not 0 < self.potential < self.detected < 1:
+            raise InvalidThresholdValueError(
+                "The provided thresholds do not comply with the rules"
+            )
+
+    def value_to_status(self, value: float) -> ResultStatusApp:
+        """
+        Translate the numeric value into status category.
+
+        :param value: The numeric value of the data drift metric, between 0 and 1.
+        :returns:     `ResultStatusApp` according to the classification.
+        """
+        if value > 1 or value < 0:
+            raise InvalidMetricValueError(
+                f"{value = } is invalid, must be in the range [0, 1]."
+            )
+        if value >= self.detected:
+            return ResultStatusApp.detected
+        if value >= self.potential:
+            return ResultStatusApp.potential_detection
+        return ResultStatusApp.no_detection
+
+
+class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
+    """
+    MLRun's default data drift application for model monitoring.
+
+    The application expects tabular numerical data, and calculates three metrics over the features' histograms.
+    The three metrics are:
+
+    * Hellinger distance.
+    * Total variance distance.
+    * Kullback-Leibler divergence.
+
+    Each metric is calculated over all the features individually and the mean is taken as the metric value.
+    The average of Hellinger and total variance distance is taken as the result.
+
+    The application logs two artifacts:
+
+    * A JSON with the general drift per feature.
+    * A plotly table different metrics per feature.
+
+    This application is deployed by default when calling:
+
+    .. code-block:: python
+
+        project.enable_model_monitoring()
+
+    """
+
+    NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
+
+    _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
+
+    metrics: list[type[HistogramDistanceMetric]] = [
+        HellingerDistance,
+        KullbackLeiblerDivergence,
+        TotalVarianceDistance,
+    ]
+
+    def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
+        """
+        :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
+                                 If not provided, the default `DataDriftClassifier()` is used.
+        """
+        self._value_classifier = value_classifier or DataDriftClassifier()
+        assert self._REQUIRED_METRICS <= set(
+            self.metrics
+        ), "TVD and Hellinger distance are required for the general data drift result"
+
+    def _compute_metrics_per_feature(
+        self, monitoring_context: mm_context.MonitoringApplicationContext
+    ) -> DataFrame:
+        """Compute the metrics for the different features and labels"""
+        metrics_per_feature = DataFrame(
+            columns=[metric_class.NAME for metric_class in self.metrics]
+        )
+        feature_stats = monitoring_context.dict_to_histogram(
+            monitoring_context.feature_stats
+        )
+        sample_df_stats = monitoring_context.dict_to_histogram(
+            monitoring_context.sample_df_stats
+        )
+        for feature_name in feature_stats:
+            sample_hist = np.asarray(sample_df_stats[feature_name])
+            reference_hist = np.asarray(feature_stats[feature_name])
+            monitoring_context.logger.info(
+                "Computing metrics for feature", feature_name=feature_name
+            )
+            metrics_per_feature.loc[feature_name] = {  # pyright: ignore[reportCallIssue,reportArgumentType]
+                metric.NAME: metric(
+                    distrib_t=sample_hist, distrib_u=reference_hist
+                ).compute()
+                for metric in self.metrics
+            }
+        monitoring_context.logger.info("Finished computing the metrics")
+
+        return metrics_per_feature
+
+    def _get_general_drift_result(
+        self,
+        metrics: list[mm_results.ModelMonitoringApplicationMetric],
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+    ) -> mm_results.ModelMonitoringApplicationResult:
+        """Get the general drift result from the metrics list"""
+        value = cast(
+            float,
+            np.mean(
+                [
+                    metric.value
+                    for metric in metrics
+                    if metric.name
+                    in [
+                        f"{HellingerDistance.NAME}_mean",
+                        f"{TotalVarianceDistance.NAME}_mean",
+                    ]
+                ]
+            ),
+        )
+
+        status = self._value_classifier.value_to_status(value)
+        return mm_results.ModelMonitoringApplicationResult(
+            name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+            value=value,
+            kind=ResultKindApp.data_drift,
+            status=status,
+            extra_data={
+                EventFieldType.CURRENT_STATS: json.dumps(
+                    monitoring_context.feature_stats
+                ),
+                EventFieldType.DRIFT_MEASURES: metrics_per_feature.T.to_json(),
+                EventFieldType.DRIFT_STATUS: status.value,
+            },
+        )
+
+    @staticmethod
+    def _get_metrics(
+        metrics_per_feature: DataFrame,
+    ) -> list[mm_results.ModelMonitoringApplicationMetric]:
+        """Average the metrics over the features and add the status"""
+        metrics: list[mm_results.ModelMonitoringApplicationMetric] = []
+
+        metrics_mean = metrics_per_feature.mean().to_dict()
+
+        for name, value in metrics_mean.items():
+            metrics.append(
+                mm_results.ModelMonitoringApplicationMetric(
+                    name=f"{name}_mean",
+                    value=value,
+                )
+            )
+
+        return metrics
+
+    @staticmethod
+    def _remove_timestamp_feature(
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+    ) -> mlrun.common.model_monitoring.helpers.FeatureStats:
+        """
+        Drop the 'timestamp' feature if it exists, as it is irrelevant
+        in the plotly artifact
+        """
+        sample_set_statistics = mlrun.common.model_monitoring.helpers.FeatureStats(
+            sample_set_statistics.copy()
+        )
+        if EventFieldType.TIMESTAMP in sample_set_statistics:
+            del sample_set_statistics[EventFieldType.TIMESTAMP]
+        return sample_set_statistics
+
+    @staticmethod
+    def _log_json_artifact(
+        drift_per_feature_values: Series,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> None:
+        """Log the drift values as a JSON artifact"""
+        monitoring_context.logger.debug("Logging drift value per feature JSON artifact")
+        monitoring_context.log_artifact(
+            mlrun.artifacts.Artifact(
+                body=drift_per_feature_values.to_json(),
+                format="json",
+                key="features_drift_results",
+            )
+        )
+        monitoring_context.logger.debug("Logged JSON artifact successfully")
+
+    def _log_plotly_table_artifact(
+        self,
+        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        inputs_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+        metrics_per_feature: DataFrame,
+        drift_per_feature_values: Series,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> None:
+        """Log the Plotly drift table artifact"""
+        monitoring_context.logger.debug(
+            "Feature stats",
+            sample_set_statistics=sample_set_statistics,
+            inputs_statistics=inputs_statistics,
+        )
+
+        monitoring_context.logger.debug("Computing drift results per feature")
+        drift_results = {
+            cast(str, key): (self._value_classifier.value_to_status(value), value)
+            for key, value in drift_per_feature_values.items()
+        }
+        monitoring_context.logger.debug("Logging plotly artifact")
+        monitoring_context.log_artifact(
+            mm_drift_table.FeaturesDriftTablePlot().produce(
+                sample_set_statistics=sample_set_statistics,
+                inputs_statistics=inputs_statistics,
+                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
+                drift_results=drift_results,
+            )
+        )
+        monitoring_context.logger.debug("Logged plotly artifact successfully")
+
+    def _log_drift_artifacts(
+        self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+        log_json_artifact: bool = True,
+    ) -> None:
+        """Log JSON and Plotly drift data per feature artifacts"""
+        drift_per_feature_values = metrics_per_feature[
+            [HellingerDistance.NAME, TotalVarianceDistance.NAME]
+        ].mean(axis=1)
+
+        if log_json_artifact:
+            self._log_json_artifact(drift_per_feature_values, monitoring_context)
+
+        self._log_plotly_table_artifact(
+            sample_set_statistics=self._remove_timestamp_feature(
+                monitoring_context.sample_df_stats
+            ),
+            inputs_statistics=monitoring_context.feature_stats,
+            metrics_per_feature=metrics_per_feature,
+            drift_per_feature_values=drift_per_feature_values,
+            monitoring_context=monitoring_context,
+        )
+
+    def do_tracking(
+        self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> list[
+        Union[
+            mm_results.ModelMonitoringApplicationResult,
+            mm_results.ModelMonitoringApplicationMetric,
+        ]
+    ]:
+        """
+        Calculate and return the data drift metrics, averaged over the features.
+
+        Refer to `ModelMonitoringApplicationBaseV2` for the meaning of the
+        function arguments.
+        """
+        monitoring_context.logger.debug("Starting to run the application")
+        if not monitoring_context.feature_stats:
+            monitoring_context.logger.info(
+                "No feature statistics found, skipping the application. \n"
+                "In order to run the application, training set must be provided when logging the model."
+            )
+            return []
+        metrics_per_feature = self._compute_metrics_per_feature(
+            monitoring_context=monitoring_context
+        )
+        monitoring_context.logger.debug("Saving artifacts")
+        self._log_drift_artifacts(
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
+        )
+        monitoring_context.logger.debug("Computing average per metric")
+        metrics = self._get_metrics(metrics_per_feature)
+        result = self._get_general_drift_result(
+            metrics=metrics,
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
+        )
+        metrics_and_result = metrics + [result]
+        monitoring_context.logger.debug(
+            "Finished running the application", results=metrics_and_result
+        )
+        return metrics_and_result

mlrun/model_monitoring/applications/results.py

@@ -0,0 +1,99 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import dataclasses
+import json
+import re
+from abc import ABC, abstractmethod
+
+import mlrun.common.helpers
+import mlrun.common.model_monitoring.helpers
+import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.utils.v3io_clients
+
+
+class _ModelMonitoringApplicationDataRes(ABC):
+    name: str
+
+    def __post_init__(self):
+        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
+        if not re.fullmatch(pat, self.name):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Attribute name must be of the format [a-zA-Z_][a-zA-Z0-9_]*"
+            )
+
+    @abstractmethod
+    def to_dict(self):
+        raise NotImplementedError
+
+
+@dataclasses.dataclass
+class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing the result of a custom model monitoring application.
+
+    :param name:           (str) Name of the application result. This name must be
+                            unique for each metric in a single application
+                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
+    :param value:          (float) Value of the application result.
+    :param kind:           (ResultKindApp) Kind of application result.
+    :param status:         (ResultStatusApp) Status of the application result.
+    :param extra_data:     (dict) Extra data associated with the application result.
+    """
+
+    name: str
+    value: float
+    kind: mm_constant.ResultKindApp
+    status: mm_constant.ResultStatusApp
+    extra_data: dict = dataclasses.field(default_factory=dict)
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constant.ResultData.RESULT_NAME: self.name,
+            mm_constant.ResultData.RESULT_VALUE: self.value,
+            mm_constant.ResultData.RESULT_KIND: self.kind.value,
+            mm_constant.ResultData.RESULT_STATUS: self.status.value,
+            mm_constant.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
+        }
+
+
+@dataclasses.dataclass
+class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
+    """
+    Class representing a single metric of a custom model monitoring application.
+
+    :param name:           (str) Name of the application metric. This name must be
+                            unique for each metric in a single application
+                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
+    :param value:          (float) Value of the application metric.
+    """
+
+    name: str
+    value: float
+
+    def to_dict(self):
+        """
+        Convert the object to a dictionary format suitable for writing.
+
+        :returns:    (dict) Dictionary representation of the result.
+        """
+        return {
+            mm_constant.MetricData.METRIC_NAME: self.name,
+            mm_constant.MetricData.METRIC_VALUE: self.value,
+        }