mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/model_monitoring/applications/_application_steps.py

@@ -16,6 +16,7 @@ import json
 import traceback
 from typing import Any, Optional, Union
 
+import mlrun.common.schemas
 import mlrun.common.schemas.alert as alert_objects
 import mlrun.common.schemas.model_monitoring.constants as mm_constant
 import mlrun.datastore
@@ -26,7 +27,11 @@ from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 
 from .context import MonitoringApplicationContext
-from .results import ModelMonitoringApplicationMetric, ModelMonitoringApplicationResult
+from .results import (
+    ModelMonitoringApplicationMetric,
+    ModelMonitoringApplicationResult,
+    _ModelMonitoringApplicationStats,
+)
 
 
 class _PushToMonitoringWriter(StepToDict):
@@ -61,7 +66,9 @@ class _PushToMonitoringWriter(StepToDict):
         event: tuple[
             list[
                 Union[
-                    ModelMonitoringApplicationResult, ModelMonitoringApplicationMetric
+                    ModelMonitoringApplicationResult,
+                    ModelMonitoringApplicationMetric,
+                    _ModelMonitoringApplicationStats,
                 ]
             ],
             MonitoringApplicationContext,
@@ -75,6 +82,7 @@ class _PushToMonitoringWriter(StepToDict):
         self._lazy_init()
         application_results, application_context = event
         writer_event = {
+            mm_constant.WriterEvent.ENDPOINT_NAME: application_context.endpoint_name,
             mm_constant.WriterEvent.APPLICATION_NAME: application_context.application_name,
             mm_constant.WriterEvent.ENDPOINT_ID: application_context.endpoint_id,
             mm_constant.WriterEvent.START_INFER_TIME: application_context.start_infer_time.isoformat(
@@ -90,21 +98,15 @@ class _PushToMonitoringWriter(StepToDict):
                 writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
                     mm_constant.WriterEventKind.RESULT
                 )
-                data[mm_constant.ResultData.CURRENT_STATS] = json.dumps(
-                    application_context.sample_df_stats
+            elif isinstance(result, _ModelMonitoringApplicationStats):
+                writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
+                    mm_constant.WriterEventKind.STATS
                 )
-                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
             else:
                 writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
                     mm_constant.WriterEventKind.METRIC
                 )
-                writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
-
-            writer_event[mm_constant.WriterEvent.EVENT_KIND] = (
-                mm_constant.WriterEventKind.RESULT
-                if isinstance(result, ModelMonitoringApplicationResult)
-                else mm_constant.WriterEventKind.METRIC
-            )
+            writer_event[mm_constant.WriterEvent.DATA] = json.dumps(data)
             logger.info(
                 f"Pushing data = {writer_event} \n to stream = {self.stream_uri}"
             )
@@ -113,9 +115,7 @@ class _PushToMonitoringWriter(StepToDict):
 
     def _lazy_init(self):
         if self.output_stream is None:
-            self.output_stream = mlrun.datastore.get_stream_pusher(
-                self.stream_uri,
-            )
+            self.output_stream = mlrun.datastore.get_stream_pusher(self.stream_uri)
 
 
 class _PrepareMonitoringEvent(StepToDict):
@@ -127,7 +127,7 @@ class _PrepareMonitoringEvent(StepToDict):
         """
         self.graph_context = context
         self.application_name = application_name
-        self.model_endpoints: dict[str, mlrun.model_monitoring.ModelEndpoint] = {}
+        self.model_endpoints: dict[str, mlrun.common.schemas.ModelEndpoint] = {}
 
     def do(self, event: dict[str, Any]) -> MonitoringApplicationContext:
         """
@@ -137,10 +137,10 @@ class _PrepareMonitoringEvent(StepToDict):
         :return: Application context.
         """
         application_context = MonitoringApplicationContext(
-            graph_context=self.graph_context,
             application_name=self.application_name,
             event=event,
             model_endpoint_dict=self.model_endpoints,
+            graph_context=self.graph_context,
         )
 
         self.model_endpoints.setdefault(

mlrun/model_monitoring/applications/base.py

@@ -12,9 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import socket
 from abc import ABC, abstractmethod
-from typing import Any, Union
+from datetime import datetime
+from typing import Any, Optional, Union, cast
 
+import pandas as pd
+
+import mlrun
+import mlrun.common.constants as mlrun_constants
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.errors
+import mlrun.model_monitoring.api as mm_api
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
 from mlrun.serving.utils import MonitoringApplicationToDict
@@ -22,12 +31,12 @@ from mlrun.serving.utils import MonitoringApplicationToDict
 
 class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     """
-    A base class for a model monitoring application.
+    The base class for a model monitoring application.
     Inherit from this class to create a custom model monitoring application.
 
-    example for very simple custom application::
+    For example, :code:`MyApp` below is a simplistic custom application::
 
-        class MyApp(ApplicationBase):
+        class MyApp(ModelMonitoringApplicationBase):
             def do_tracking(
                 self,
                 monitoring_context: mm_context.MonitoringApplicationContext,
@@ -43,8 +52,6 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                     kind=mm_constant.ResultKindApp.data_drift,
                     status=mm_constant.ResultStatusApp.detected,
                 )
-
-
     """
 
     kind = "monitoring_application"
@@ -62,6 +69,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     ]:
         """
         Process the monitoring event and return application results & metrics.
+        Note: this method is internal and should not be called directly or overridden.
 
         :param monitoring_context:   (MonitoringApplicationContext) The monitoring application context.
         :returns:                    A tuple of:
@@ -80,6 +88,157 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         results = results if isinstance(results, list) else [results]
         return results, monitoring_context
 
+    def _handler(
+        self,
+        context: "mlrun.MLClientCtx",
+        sample_data: Optional[pd.DataFrame] = None,
+        reference_data: Optional[pd.DataFrame] = None,
+        endpoint_names: Optional[list[str]] = None,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+    ):
+        """
+        A custom handler that wraps the application's logic implemented in
+        :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
+        for an MLRun job.
+        This method should not be called directly.
+        """
+        feature_stats = (
+            mm_api.get_sample_set_statistics(reference_data)
+            if reference_data is not None
+            else None
+        )
+
+        def call_do_tracking(event: Optional[dict] = None):
+            if event is None:
+                event = {}
+            monitoring_context = mm_context.MonitoringApplicationContext(
+                event=event,
+                application_name=self.__class__.__name__,
+                logger=context.logger,
+                artifacts_logger=context,
+                sample_df=sample_data,
+                feature_stats=feature_stats,
+            )
+            return self.do_tracking(monitoring_context)
+
+        if endpoint_names is not None:
+            start, end = self._validate_times(start, end)
+            for endpoint_name in endpoint_names:
+                result = call_do_tracking(
+                    event={
+                        mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
+                        mm_constants.ApplicationEvent.START_INFER_TIME: start,
+                        mm_constants.ApplicationEvent.END_INFER_TIME: end,
+                    }
+                )
+                context.log_result(
+                    f"{endpoint_name}_{start.isoformat()}_{end.isoformat()}", result
+                )
+        else:
+            return call_do_tracking()
+
+    @staticmethod
+    def _validate_times(
+        start: Optional[datetime], end: Optional[datetime]
+    ) -> tuple[datetime, datetime]:
+        if (start is None) or (end is None):
+            raise mlrun.errors.MLRunValueError(
+                "When `endpoint_names` is provided, you must also pass the start and end times"
+            )
+        return start, end
+
+    @classmethod
+    def evaluate(
+        cls,
+        func_path: Optional[str] = None,
+        func_name: Optional[str] = None,
+        *,
+        tag: Optional[str] = None,
+        run_local: bool = True,
+        sample_data: Optional[pd.DataFrame] = None,
+        reference_data: Optional[pd.DataFrame] = None,
+        image: Optional[str] = None,
+        with_repo: Optional[bool] = False,
+        requirements: Optional[Union[str, list[str]]] = None,
+        requirements_file: str = "",
+        endpoint_names: Optional[list[str]] = None,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+    ) -> "mlrun.RunObject":
+        """
+        Call this function to run the application's
+        :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
+        model monitoring logic as a :py:class:`~mlrun.runtimes.KubejobRuntime`, which is an MLRun function.
+
+        :param func_path: The path to the function. If not passed, the current notebook is used.
+        :param func_name: The name of the function. If not passed, the class name is used.
+        :param tag:       An optional tag for the function.
+        :param run_local: Whether to run the function locally or remotely.
+        :param sample_df: Optional - pandas data-frame as the current dataset.
+                          When set, it replaces the data read from the model endpoint's offline source.
+        :param feature_stats: Optional - statistics dictionary of the reference data.
+                              When set, it overrides the model endpoint's feature stats.
+        :param image:             Docker image to run the job on.
+        :param with_repo:         Whether to clone the current repo to the build source.
+        :param requirements:      List of Python requirements to be installed in the image.
+        :param requirements_file: Path to a Python requirements file to be installed in the image.
+        :param endpoint_names:    The model endpoint names to get the data from. When the names are passed,
+                                  you have to provide also the start and end times of the data to analyze.
+        :param start:             The start time of the sample data.
+        :param end:               The end time of the sample data.
+
+        :returns: The output of the
+                  :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
+                  method with the given parameters and inputs, wrapped in a :py:class:`~mlrun.model.RunObject`.
+        """
+        project = cast("mlrun.MlrunProject", mlrun.get_current_project())
+        class_name = cls.__name__
+        job_name = func_name if func_name is not None else class_name
+        handler = f"{class_name}::{cls._handler.__name__}"
+
+        job = cast(
+            mlrun.runtimes.KubejobRuntime,
+            project.set_function(
+                func=func_path,
+                name=job_name,
+                kind=mlrun.runtimes.KubejobRuntime.kind,
+                handler=handler,
+                tag=tag,
+                image=image,
+                with_repo=with_repo,
+                requirements=requirements,
+                requirements_file=requirements_file,
+            ),
+        )
+
+        params: dict[str, Union[list[str], datetime]] = {}
+        if endpoint_names:
+            start, end = cls._validate_times(start, end)
+            params["endpoint_names"] = endpoint_names
+            params["start"] = start
+            params["end"] = end
+
+        inputs: dict[str, str] = {}
+        for data, identifier in [
+            (sample_data, "sample_data"),
+            (reference_data, "reference_data"),
+        ]:
+            if data is not None:
+                key = f"{job_name}_{identifier}"
+                inputs[identifier] = project.log_dataset(
+                    key,
+                    data,
+                    labels={
+                        mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
+                        mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-job",
+                        mlrun_constants.MLRunInternalLabels.app_name: class_name,
+                    },
+                ).uri
+
+        run_result = job.run(local=run_local, params=params, inputs=inputs)
+        return run_result
+
     @abstractmethod
     def do_tracking(
         self,

mlrun/model_monitoring/applications/context.py

@@ -12,26 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import socket
-from typing import Any, Optional, cast
+from typing import Any, Optional, Protocol, cast
 
+import nuclio.request
 import numpy as np
 import pandas as pd
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.errors
 import mlrun.feature_store as fstore
 import mlrun.features
 import mlrun.serving
 import mlrun.utils
 from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
-from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
+from mlrun.common.model_monitoring.helpers import FeatureStats
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.model_monitoring.helpers import (
     calculate_inputs_statistics,
-    get_endpoint_record,
 )
-from mlrun.model_monitoring.model_endpoint import ModelEndpoint
+
+
+class _ArtifactsLogger(Protocol):
+    """
+    Classes that implement this protocol are :code:`MlrunProject` and :code:`MLClientCtx`.
+    """
+
+    def log_artifact(self, *args, **kwargs) -> Artifact: ...
+    def log_dataset(self, *args, **kwargs) -> DatasetArtifact: ...
+    def log_model(self, *args, **kwargs) -> ModelArtifact: ...
 
 
 class MonitoringApplicationContext:
@@ -52,6 +62,7 @@ class MonitoringApplicationContext:
     :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
     :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
     :param endpoint_id:             (str) ID of the monitored model endpoint
+    :param endpoint_name:           (str) Name of the monitored model endpoint
     :param output_stream_uri:       (str) URI of the output stream for results
     :param model_endpoint:          (ModelEndpoint) The model endpoint object.
     :param feature_names:           (list[str]) List of models feature names.
@@ -60,36 +71,71 @@ class MonitoringApplicationContext:
                                     and a list of extra data items.
     """
 
+    _logger_name = "monitoring-application"
+
     def __init__(
         self,
         *,
-        graph_context: mlrun.serving.GraphContext,
         application_name: str,
         event: dict[str, Any],
-        model_endpoint_dict: dict[str, ModelEndpoint],
+        model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
+        logger: Optional[mlrun.utils.Logger] = None,
+        graph_context: Optional[mlrun.serving.GraphContext] = None,
+        context: Optional["mlrun.MLClientCtx"] = None,
+        artifacts_logger: Optional[_ArtifactsLogger] = None,
+        sample_df: Optional[pd.DataFrame] = None,
+        feature_stats: Optional[FeatureStats] = None,
     ) -> None:
         """
-        Initialize a `MonitoringApplicationContext` object.
+        The :code:`__init__` method initializes a :code:`MonitoringApplicationContext` object
+        and has the following attributes.
         Note: this object should not be instantiated manually.
 
         :param application_name:    The application name.
         :param event:               The instance data dictionary.
-        :param model_endpoint_dict: Dictionary of model endpoints.
+        :param model_endpoint_dict: Optional - dictionary of model endpoints.
+        :param logger:              Optional - MLRun logger instance.
+        :param graph_context:       Optional - GraphContext instance.
+        :param context:             Optional - MLClientCtx instance.
+        :param artifacts_logger:    Optional - an object that can log artifacts,
+                                    typically :py:class:`~mlrun.projects.MlrunProject` or
+                                    :py:class:`~mlrun.execution.MLClientCtx`.
+        :param sample_df:           Optional - pandas data-frame as the current dataset.
+                                    When set, it replaces the data read from the offline source.
+        :param feature_stats:       Optional - statistics dictionary of the reference data.
+                                    When set, it overrides the model endpoint's feature stats.
         """
         self.application_name = application_name
 
-        self.project_name = graph_context.project
-        self.project = mlrun.load_project(url=self.project_name)
+        if graph_context:
+            self.project_name = graph_context.project
+            self.project = mlrun.load_project(url=self.project_name)
+        elif context:
+            potential_project = context.get_project_object()
+            if not potential_project:
+                raise mlrun.errors.MLRunValueError(
+                    "Could not load project from context"
+                )
+            self.project = potential_project
+            self.project_name = self.project.name
+
+        self._artifacts_logger: _ArtifactsLogger = artifacts_logger or self.project
 
         # MLRun Logger
-        self.logger = mlrun.utils.create_logger(
+        self.logger = logger or mlrun.utils.create_logger(
             level=mlrun.mlconf.log_level,
             formatter_kind=mlrun.mlconf.log_formatter,
-            name="monitoring-application",
+            name=self._logger_name,
         )
         # Nuclio logger - `nuclio.request.Logger`.
-        # Note: this logger does not accept keyword arguments.
-        self.nuclio_logger = graph_context.logger
+        # Note: this logger accepts keyword arguments only in its `_with` methods, e.g. `info_with`.
+        self.nuclio_logger = (
+            graph_context.logger
+            if graph_context
+            else nuclio.request.Logger(
+                level=mlrun.mlconf.log_level, name=self._logger_name
+            )
+        )
 
         # event data
         self.start_infer_time = pd.Timestamp(
@@ -101,29 +147,38 @@ class MonitoringApplicationContext:
         self.endpoint_id = cast(
             str, event.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
         )
+        self.endpoint_name = cast(
+            str, event.get(mm_constants.ApplicationEvent.ENDPOINT_NAME)
+        )
         self.output_stream_uri = cast(
             str, event.get(mm_constants.ApplicationEvent.OUTPUT_STREAM_URI)
         )
 
-        self._feature_stats: Optional[FeatureStats] = None
+        self._feature_stats: Optional[FeatureStats] = feature_stats
         self._sample_df_stats: Optional[FeatureStats] = None
 
         # Default labels for the artifacts
         self._default_labels = self._get_default_labels()
 
         # Persistent data - fetched when needed
-        self._sample_df: Optional[pd.DataFrame] = None
-        self._model_endpoint: Optional[ModelEndpoint] = model_endpoint_dict.get(
-            self.endpoint_id
+        self._sample_df: Optional[pd.DataFrame] = sample_df
+        self._model_endpoint: Optional[ModelEndpoint] = (
+            model_endpoint_dict.get(self.endpoint_id) if model_endpoint_dict else None
         )
 
     def _get_default_labels(self) -> dict[str, str]:
-        return {
+        labels = {
             mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
             mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
             mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
-            mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
         }
+        for key, value in [
+            (mlrun_constants.MLRunInternalLabels.endpoint_id, self.endpoint_id),
+            (mlrun_constants.MLRunInternalLabels.endpoint_name, self.endpoint_name),
+        ]:
+            if value:
+                labels[key] = value
+        return labels
 
     def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
         """Add the default labels to logged artifacts labels"""
@@ -133,7 +188,7 @@ class MonitoringApplicationContext:
     def sample_df(self) -> pd.DataFrame:
         if self._sample_df is None:
             feature_set = fstore.get_feature_set(
-                self.model_endpoint.status.monitoring_feature_set_uri
+                self.model_endpoint.spec.monitoring_feature_set_uri
             )
             features = [f"{feature_set.metadata.name}.*"]
             vector = fstore.FeatureVector(
@@ -155,16 +210,18 @@ class MonitoringApplicationContext:
     @property
     def model_endpoint(self) -> ModelEndpoint:
         if not self._model_endpoint:
-            self._model_endpoint = ModelEndpoint.from_flat_dict(
-                get_endpoint_record(self.project_name, self.endpoint_id)
+            self._model_endpoint = mlrun.db.get_run_db().get_model_endpoint(
+                name=self.endpoint_name,
+                project=self.project_name,
+                endpoint_id=self.endpoint_id,
+                feature_analysis=True,
             )
         return self._model_endpoint
 
     @property
     def feature_stats(self) -> FeatureStats:
         if not self._feature_stats:
-            self._feature_stats = json.loads(self.model_endpoint.status.feature_stats)
-            pad_features_hist(self._feature_stats)
+            self._feature_stats = self.model_endpoint.spec.feature_stats
         return self._feature_stats
 
     @property
@@ -179,18 +236,12 @@ class MonitoringApplicationContext:
     @property
     def feature_names(self) -> list[str]:
         """The feature names of the model"""
-        feature_names = self.model_endpoint.spec.feature_names
-        return (
-            feature_names
-            if isinstance(feature_names, list)
-            else json.loads(feature_names)
-        )
+        return self.model_endpoint.spec.feature_names
 
     @property
     def label_names(self) -> list[str]:
         """The label names of the model"""
-        label_names = self.model_endpoint.spec.label_names
-        return label_names if isinstance(label_names, list) else json.loads(label_names)
+        return self.model_endpoint.spec.label_names
 
     @property
     def model(self) -> tuple[str, ModelArtifact, dict]:
@@ -237,7 +288,7 @@ class MonitoringApplicationContext:
         See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
         """
         labels = self._add_default_labels(labels)
-        return self.project.log_artifact(
+        return self._artifacts_logger.log_artifact(
             item,
             body=body,
             tag=tag,
@@ -272,7 +323,7 @@ class MonitoringApplicationContext:
         See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
         """
         labels = self._add_default_labels(labels)
-        return self.project.log_dataset(
+        return self._artifacts_logger.log_dataset(
             key,
             df,
             tag=tag,
@@ -317,7 +368,7 @@ class MonitoringApplicationContext:
         See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
         """
         labels = self._add_default_labels(labels)
-        return self.project.log_model(
+        return self._artifacts_logger.log_model(
             key,
             body=body,
             framework=framework,

mlrun/model_monitoring/applications/evidently_base.py

@@ -23,7 +23,7 @@ import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.32")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.39")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -76,7 +76,6 @@ class EvidentlyModelMonitoringApplicationBase(
 
         :param evidently_workspace_path:    (str) The path to the Evidently workspace.
         :param evidently_project_id:        (str) The ID of the Evidently project.
-
         """
 
         # TODO : more then one project (mep -> project)