mlrun 1.7.0rc5__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/model_monitoring/applications/_application_steps.py

@@ -0,0 +1,190 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import traceback
+from typing import Any, Optional, Union
+
+import mlrun.common.schemas.alert as alert_objects
+import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.datastore
+import mlrun.model_monitoring
+from mlrun.model_monitoring.helpers import get_stream_path
+from mlrun.serving import GraphContext
+from mlrun.serving.utils import StepToDict
+from mlrun.utils import logger
+
+from .context import MonitoringApplicationContext
+from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult
+
+
+class _PushToMonitoringWriter(StepToDict):
+    kind = "monitoring_application_stream_pusher"
+
+    def __init__(
+        self,
+        project: str,
+        writer_application_name: str,
+        stream_uri: Optional[str] = None,
+        name: Optional[str] = None,
+    ):
+        """
+        Class for pushing application results to the monitoring writer stream.
+
+        :param project:                     Project name.
+        :param writer_application_name:     Writer application name.
+        :param stream_uri:                  Stream URI for pushing results.
+        :param name:                        Name of the PushToMonitoringWriter
+                                            instance default to PushToMonitoringWriter.
+        """
+        self.project = project
+        self.application_name_to_push = writer_application_name
+        self.stream_uri = stream_uri or get_stream_path(
+            project=self.project, function_name=self.application_name_to_push
+        )
+        self.output_stream = None
+        self.name = name or "PushToMonitoringWriter"
+
+    def do(
+        self,
+        event: tuple[
+            list[
+                Union[
+                    ModelMonitoringApplicationResult, ModelMonitoringApplicationMetric
+                ]
+            ],
+            MonitoringApplicationContext,
+        ],
+    ) -> None:
+        """
+        Push application results to the monitoring writer stream.
+
+        :param event: Monitoring result(s) to push and the original event from the controller.
+        """
+        self._lazy_init()
+        application_results, application_context = event
+        writer_event = {
+            mm_constant.WriterEvent.APPLICATION_NAME: application_context.application_name,
+            mm_constant.WriterEvent.ENDPOINT_ID: application_context.endpoint_id,
+            mm_constant.WriterEvent.START_INFER_TIME: application_context.start_infer_time.isoformat(
+                sep=" ", timespec="microseconds"
+            ),
+            mm_constant.WriterEvent.END_INFER_TIME: application_context.end_infer_time.isoformat(
+                sep=" ", timespec="microseconds"
+            ),
+        }
+        for result in application_results:
+            data = result.to_dict()
+            if isinstance(result, ModelMonitoringApplicationResult):
+                writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                    mm_constant.WriterEventKind.RESULT
+                )
+                data[mm_constant.ResultData.CURRENT_STATS] = json.dumps(
+                    application_context.sample_df_stats
+                )
+                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
+            else:
+                writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                    mm_constant.WriterEventKind.METRIC
+                )
+                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
+
+            writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                mm_constant.WriterEventKind.RESULT
+                if isinstance(result, ModelMonitoringApplicationResult)
+                else mm_constant.WriterEventKind.METRIC
+            )
+            logger.info(
+                f"Pushing data = {writer_event} \n to stream = {self.stream_uri}"
+            )
+            self.output_stream.push([writer_event])
+            logger.info(f"Pushed data to {self.stream_uri} successfully")
+
+    def _lazy_init(self):
+        if self.output_stream is None:
+            self.output_stream = mlrun.datastore.get_stream_pusher(
+                self.stream_uri,
+            )
+
+
+class _PrepareMonitoringEvent(StepToDict):
+    def __init__(self, context: GraphContext, application_name: str) -> None:
+        """
+        Class for preparing the application event for the application step.
+
+        :param application_name: Application name.
+        """
+        self.graph_context = context
+        self.application_name = application_name
+        self.model_endpoints: dict[str, mlrun.model_monitoring.ModelEndpoint] = {}
+
+    def do(self, event: dict[str, Any]) -> MonitoringApplicationContext:
+        """
+        Prepare the application event for the application step.
+
+        :param event: Application event.
+        :return: Application context.
+        """
+        application_context = MonitoringApplicationContext(
+            graph_context=self.graph_context,
+            application_name=self.application_name,
+            event=event,
+            model_endpoint_dict=self.model_endpoints,
+        )
+
+        self.model_endpoints.setdefault(
+            application_context.endpoint_id, application_context.model_endpoint
+        )
+
+        return application_context
+
+
+class _ApplicationErrorHandler(StepToDict):
+    def __init__(self, project: str, name: Optional[str] = None):
+        self.project = project
+        self.name = name or "ApplicationErrorHandler"
+
+    def do(self, event):
+        """
+        Handle model monitoring application error. This step will generate an event, describing the error.
+
+        :param event: Application event.
+        """
+
+        error_data = {
+            "Endpoint ID": event.body.endpoint_id,
+            "Application Class": event.body.application_name,
+            "Error": "".join(
+                traceback.format_exception(None, event.error, event.error.__traceback__)
+            ),
+            "Timestamp": event.timestamp,
+        }
+        logger.error("Error in application step", **error_data)
+
+        error_data["Error"] = event.error
+
+        event_data = alert_objects.Event(
+            kind=alert_objects.EventKind.MM_APP_FAILED,
+            entity=alert_objects.EventEntities(
+                kind=alert_objects.EventEntityKind.MODEL_MONITORING_APPLICATION,
+                project=self.project,
+                ids=[f"{self.project}_{event.body.application_name}"],
+            ),
+            value_dict=error_data,
+        )
+
+        mlrun.get_run_db().generate_event(
+            name=alert_objects.EventKind.MM_APP_FAILED, event_data=event_data
+        )
+        logger.info("Event generated successfully")

mlrun/model_monitoring/applications/base.py

@@ -0,0 +1,108 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Any, Union
+
+import mlrun.model_monitoring.applications.context as mm_context
+import mlrun.model_monitoring.applications.results as mm_results
+from mlrun.serving.utils import MonitoringApplicationToDict
+
+
+class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
+    """
+    A base class for a model monitoring application.
+    Inherit from this class to create a custom model monitoring application.
+
+    example for very simple custom application::
+
+        class MyApp(ApplicationBase):
+            def do_tracking(
+                self,
+                monitoring_context: mm_context.MonitoringApplicationContext,
+            ) -> ModelMonitoringApplicationResult:
+                monitoring_context.log_artifact(
+                    TableArtifact(
+                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
+                    )
+                )
+                return ModelMonitoringApplicationResult(
+                    name="data_drift_test",
+                    value=0.5,
+                    kind=mm_constant.ResultKindApp.data_drift,
+                    status=mm_constant.ResultStatusApp.detected,
+                )
+
+
+    """
+
+    kind = "monitoring_application"
+
+    def do(
+        self, monitoring_context: mm_context.MonitoringApplicationContext
+    ) -> tuple[
+        list[
+            Union[
+                mm_results.ModelMonitoringApplicationResult,
+                mm_results.ModelMonitoringApplicationMetric,
+            ]
+        ],
+        mm_context.MonitoringApplicationContext,
+    ]:
+        """
+        Process the monitoring event and return application results & metrics.
+
+        :param monitoring_context:   (MonitoringApplicationContext) The monitoring application context.
+        :returns:                    A tuple of:
+                                        [0] = list of application results that can be either from type
+                                        `ModelMonitoringApplicationResult`
+                                        or from type `ModelMonitoringApplicationResult`.
+                                        [1] = the original application event, wrapped in `MonitoringApplicationContext`
+                                         object
+        """
+        results = self.do_tracking(monitoring_context=monitoring_context)
+        if isinstance(results, dict):
+            results = [
+                mm_results.ModelMonitoringApplicationMetric(name=key, value=value)
+                for key, value in results.items()
+            ]
+        results = results if isinstance(results, list) else [results]
+        return results, monitoring_context
+
+    @abstractmethod
+    def do_tracking(
+        self,
+        monitoring_context: mm_context.MonitoringApplicationContext,
+    ) -> Union[
+        mm_results.ModelMonitoringApplicationResult,
+        list[
+            Union[
+                mm_results.ModelMonitoringApplicationResult,
+                mm_results.ModelMonitoringApplicationMetric,
+            ]
+        ],
+        dict[str, Any],
+    ]:
+        """
+        Implement this method with your custom monitoring logic.
+
+        :param monitoring_context:      (MonitoringApplicationContext) The monitoring context to process.
+
+        :returns:                       (ModelMonitoringApplicationResult) or
+                                        (list[Union[ModelMonitoringApplicationResult,
+                                        ModelMonitoringApplicationMetric]])
+                                        or dict that contains the application metrics only (in this case the name of
+                                        each metric name is the key and the metric value is the corresponding value).
+        """
+        raise NotImplementedError

mlrun/model_monitoring/applications/context.py

@@ -0,0 +1,341 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import socket
+from typing import Any, Optional, cast
+
+import numpy as np
+import pandas as pd
+
+import mlrun.common.constants as mlrun_constants
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.feature_store as fstore
+import mlrun.features
+import mlrun.serving
+import mlrun.utils
+from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
+from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
+from mlrun.model_monitoring.helpers import (
+    calculate_inputs_statistics,
+    get_endpoint_record,
+)
+from mlrun.model_monitoring.model_endpoint import ModelEndpoint
+
+
+class MonitoringApplicationContext:
+    """
+    The monitoring context holds all the relevant information for the monitoring application,
+    and also it can be used for logging artifacts and results.
+    The monitoring context has the following attributes:
+
+    :param application_name:        (str) The model monitoring application name.
+    :param project_name:            (str) The project name.
+    :param project:                 (MlrunProject) The project object.
+    :param logger:                  (mlrun.utils.Logger) MLRun logger.
+    :param nuclio_logger:           (nuclio.request.Logger) Nuclio logger.
+    :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
+    :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
+    :param sample_df:               (pd.DataFrame) The new sample DataFrame.
+    :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
+    :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
+    :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
+    :param endpoint_id:             (str) ID of the monitored model endpoint
+    :param output_stream_uri:       (str) URI of the output stream for results
+    :param model_endpoint:          (ModelEndpoint) The model endpoint object.
+    :param feature_names:           (list[str]) List of models feature names.
+    :param label_names:             (list[str]) List of models label names.
+    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object,
+                                    and a list of extra data items.
+    """
+
+    def __init__(
+        self,
+        *,
+        graph_context: mlrun.serving.GraphContext,
+        application_name: str,
+        event: dict[str, Any],
+        model_endpoint_dict: dict[str, ModelEndpoint],
+    ) -> None:
+        """
+        Initialize a `MonitoringApplicationContext` object.
+        Note: this object should not be instantiated manually.
+
+        :param application_name:    The application name.
+        :param event:               The instance data dictionary.
+        :param model_endpoint_dict: Dictionary of model endpoints.
+        """
+        self.application_name = application_name
+
+        self.project_name = graph_context.project
+        self.project = mlrun.load_project(url=self.project_name)
+
+        # MLRun Logger
+        self.logger = mlrun.utils.create_logger(
+            level=mlrun.mlconf.log_level,
+            formatter_kind=mlrun.mlconf.log_formatter,
+            name="monitoring-application",
+        )
+        # Nuclio logger - `nuclio.request.Logger`.
+        # Note: this logger does not accept keyword arguments.
+        self.nuclio_logger = graph_context.logger
+
+        # event data
+        self.start_infer_time = pd.Timestamp(
+            cast(str, event.get(mm_constants.ApplicationEvent.START_INFER_TIME))
+        )
+        self.end_infer_time = pd.Timestamp(
+            cast(str, event.get(mm_constants.ApplicationEvent.END_INFER_TIME))
+        )
+        self.endpoint_id = cast(
+            str, event.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
+        )
+        self.output_stream_uri = cast(
+            str, event.get(mm_constants.ApplicationEvent.OUTPUT_STREAM_URI)
+        )
+
+        self._feature_stats: Optional[FeatureStats] = None
+        self._sample_df_stats: Optional[FeatureStats] = None
+
+        # Default labels for the artifacts
+        self._default_labels = self._get_default_labels()
+
+        # Persistent data - fetched when needed
+        self._sample_df: Optional[pd.DataFrame] = None
+        self._model_endpoint: Optional[ModelEndpoint] = model_endpoint_dict.get(
+            self.endpoint_id
+        )
+
+    def _get_default_labels(self) -> dict[str, str]:
+        return {
+            mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
+            mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
+            mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
+            mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
+        }
+
+    def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
+        """Add the default labels to logged artifacts labels"""
+        return (labels or {}) | self._default_labels
+
+    @property
+    def sample_df(self) -> pd.DataFrame:
+        if self._sample_df is None:
+            feature_set = fstore.get_feature_set(
+                self.model_endpoint.status.monitoring_feature_set_uri
+            )
+            features = [f"{feature_set.metadata.name}.*"]
+            vector = fstore.FeatureVector(
+                name=f"{self.endpoint_id}_vector",
+                features=features,
+                with_indexes=True,
+            )
+            vector.metadata.tag = self.application_name
+            vector.feature_set_objects = {feature_set.metadata.name: feature_set}
+
+            offline_response = vector.get_offline_features(
+                start_time=self.start_infer_time,
+                end_time=self.end_infer_time,
+                timestamp_for_filtering=mm_constants.FeatureSetFeatures.time_stamp(),
+            )
+            self._sample_df = offline_response.to_dataframe().reset_index(drop=True)
+        return self._sample_df
+
+    @property
+    def model_endpoint(self) -> ModelEndpoint:
+        if not self._model_endpoint:
+            self._model_endpoint = ModelEndpoint.from_flat_dict(
+                get_endpoint_record(self.project_name, self.endpoint_id)
+            )
+        return self._model_endpoint
+
+    @property
+    def feature_stats(self) -> FeatureStats:
+        if not self._feature_stats:
+            self._feature_stats = json.loads(self.model_endpoint.status.feature_stats)
+            pad_features_hist(self._feature_stats)
+        return self._feature_stats
+
+    @property
+    def sample_df_stats(self) -> FeatureStats:
+        """statistics of the sample dataframe"""
+        if not self._sample_df_stats:
+            self._sample_df_stats = calculate_inputs_statistics(
+                self.feature_stats, self.sample_df
+            )
+        return self._sample_df_stats
+
+    @property
+    def feature_names(self) -> list[str]:
+        """The feature names of the model"""
+        feature_names = self.model_endpoint.spec.feature_names
+        return (
+            feature_names
+            if isinstance(feature_names, list)
+            else json.loads(feature_names)
+        )
+
+    @property
+    def label_names(self) -> list[str]:
+        """The label names of the model"""
+        label_names = self.model_endpoint.spec.label_names
+        return label_names if isinstance(label_names, list) else json.loads(label_names)
+
+    @property
+    def model(self) -> tuple[str, ModelArtifact, dict]:
+        """The model file, model spec object, and a list of extra data items"""
+        return get_model(self.model_endpoint.spec.model_uri)
+
+    @staticmethod
+    def dict_to_histogram(histogram_dict: FeatureStats) -> pd.DataFrame:
+        """
+        Convert histogram dictionary to pandas DataFrame with feature histograms as columns
+
+        :param histogram_dict: Histogram dictionary
+
+        :returns: Histogram dataframe
+        """
+
+        # Create a dictionary with feature histograms as values
+        histograms = {}
+        for feature, stats in histogram_dict.items():
+            if "hist" in stats:
+                # Normalize to probability distribution of each feature
+                histograms[feature] = np.array(stats["hist"][0]) / stats["count"]
+
+        # Convert the dictionary to pandas DataFrame
+        histograms = pd.DataFrame(histograms)
+
+        return histograms
+
+    def log_artifact(
+        self,
+        item,
+        body=None,
+        tag: str = "",
+        local_path: str = "",
+        artifact_path: Optional[str] = None,
+        format: Optional[str] = None,
+        upload: Optional[bool] = None,
+        labels: Optional[dict[str, str]] = None,
+        target_path: Optional[str] = None,
+        **kwargs,
+    ) -> Artifact:
+        """
+        Log an artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_artifact(
+            item,
+            body=body,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            format=format,
+            upload=upload,
+            labels=labels,
+            target_path=target_path,
+            **kwargs,
+        )
+
+    def log_dataset(
+        self,
+        key,
+        df,
+        tag="",
+        local_path=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        format="",
+        preview=None,
+        stats=None,
+        target_path="",
+        extra_data=None,
+        label_column: Optional[str] = None,
+        **kwargs,
+    ) -> DatasetArtifact:
+        """
+        Log a dataset artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_dataset(
+            key,
+            df,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            format=format,
+            preview=preview,
+            stats=stats,
+            target_path=target_path,
+            extra_data=extra_data,
+            label_column=label_column,
+            **kwargs,
+        )
+
+    def log_model(
+        self,
+        key,
+        body=None,
+        framework="",
+        tag="",
+        model_dir=None,
+        model_file=None,
+        algorithm=None,
+        metrics=None,
+        parameters=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        inputs: Optional[list[mlrun.features.Feature]] = None,
+        outputs: Optional[list[mlrun.features.Feature]] = None,
+        feature_vector: Optional[str] = None,
+        feature_weights: Optional[list] = None,
+        training_set=None,
+        label_column=None,
+        extra_data=None,
+        **kwargs,
+    ) -> ModelArtifact:
+        """
+        Log a model artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_model(
+            key,
+            body=body,
+            framework=framework,
+            tag=tag,
+            model_dir=model_dir,
+            model_file=model_file,
+            algorithm=algorithm,
+            metrics=metrics,
+            parameters=parameters,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            inputs=inputs,
+            outputs=outputs,
+            feature_vector=feature_vector,
+            feature_weights=feature_weights,
+            training_set=training_set,
+            label_column=label_column,
+            extra_data=extra_data,
+            **kwargs,
+        )