mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/model_monitoring/applications/context.py

@@ -11,19 +11,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
-import typing
+import socket
+from typing import Any, Optional, cast
 
 import numpy as np
 import pandas as pd
 
-import mlrun.common.helpers
-import mlrun.common.model_monitoring.helpers
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store as fstore
-from mlrun.artifacts.model import ModelArtifact, get_model
+import mlrun.features
+import mlrun.serving
+import mlrun.utils
+from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
 from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
-from mlrun.execution import MLClientCtx
 from mlrun.model_monitoring.helpers import (
     calculate_inputs_statistics,
     get_endpoint_record,
@@ -31,13 +34,17 @@ from mlrun.model_monitoring.helpers import (
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 
 
-class MonitoringApplicationContext(MLClientCtx):
+class MonitoringApplicationContext:
     """
     The monitoring context holds all the relevant information for the monitoring application,
     and also it can be used for logging artifacts and results.
     The monitoring context has the following attributes:
 
-    :param application_name:        (str) the app name
+    :param application_name:        (str) The model monitoring application name.
+    :param project_name:            (str) The project name.
+    :param project:                 (MlrunProject) The project object.
+    :param logger:                  (mlrun.utils.Logger) MLRun logger.
+    :param nuclio_logger:           (nuclio.request.Logger) Nuclio logger.
     :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
     :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
     :param sample_df:               (pd.DataFrame) The new sample DataFrame.
@@ -49,81 +56,82 @@ class MonitoringApplicationContext(MLClientCtx):
     :param model_endpoint:          (ModelEndpoint) The model endpoint object.
     :param feature_names:           (list[str]) List of models feature names.
     :param label_names:             (list[str]) List of models label names.
-    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object, and list of
-
+    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object,
+                                    and a list of extra data items.
     """
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def __post_init__(self):
-        self.application_name: typing.Optional[str] = None
-        self.start_infer_time: typing.Optional[pd.Timestamp] = None
-        self.end_infer_time: typing.Optional[pd.Timestamp] = None
-        self.latest_request: typing.Optional[pd.Timestamp] = None
-        self.endpoint_id: typing.Optional[str] = None
-        self.output_stream_uri: typing.Optional[str] = None
-
-        self._sample_df: typing.Optional[pd.DataFrame] = None
-        self._model_endpoint: typing.Optional[ModelEndpoint] = None
-        self._feature_stats: typing.Optional[FeatureStats] = None
-        self._sample_df_stats: typing.Optional[FeatureStats] = None
-
-    @classmethod
-    def from_dict(
-        cls,
-        attrs: dict,
-        context=None,
-        model_endpoint_dict=None,
-        **kwargs,
-    ) -> "MonitoringApplicationContext":
+    def __init__(
+        self,
+        *,
+        graph_context: mlrun.serving.GraphContext,
+        application_name: str,
+        event: dict[str, Any],
+        model_endpoint_dict: dict[str, ModelEndpoint],
+    ) -> None:
         """
-        Create an instance of the MonitoringApplicationContext from a dictionary.
+        Initialize a `MonitoringApplicationContext` object.
+        Note: this object should not be instantiated manually.
 
-        :param attrs:               The instance data dictionary.
-        :param context:             The current application context.
+        :param application_name:    The application name.
+        :param event:               The instance data dictionary.
         :param model_endpoint_dict: Dictionary of model endpoints.
-
         """
+        self.application_name = application_name
 
-        if not context:
-            self = (
-                super().from_dict(
-                    attrs=attrs.get(mm_constants.ApplicationEvent.MLRUN_CONTEXT, {}),
-                    **kwargs,
-                ),
-            )
-        else:
-            self = context
-            self.__post_init__()
+        self.project_name = graph_context.project
+        self.project = mlrun.load_project(url=self.project_name)
+
+        # MLRun Logger
+        self.logger = mlrun.utils.create_logger(
+            level=mlrun.mlconf.log_level,
+            formatter_kind=mlrun.mlconf.log_formatter,
+            name="monitoring-application",
+        )
+        # Nuclio logger - `nuclio.request.Logger`.
+        # Note: this logger does not accept keyword arguments.
+        self.nuclio_logger = graph_context.logger
 
+        # event data
         self.start_infer_time = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.START_INFER_TIME)
+            cast(str, event.get(mm_constants.ApplicationEvent.START_INFER_TIME))
         )
         self.end_infer_time = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.END_INFER_TIME)
+            cast(str, event.get(mm_constants.ApplicationEvent.END_INFER_TIME))
         )
-        self.latest_request = pd.Timestamp(
-            attrs.get(mm_constants.ApplicationEvent.LAST_REQUEST)
+        self.endpoint_id = cast(
+            str, event.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
         )
-        self.application_name = attrs.get(
-            mm_constants.ApplicationEvent.APPLICATION_NAME
+        self.output_stream_uri = cast(
+            str, event.get(mm_constants.ApplicationEvent.OUTPUT_STREAM_URI)
         )
-        self._feature_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
-        )
-        self._sample_df_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
+
+        self._feature_stats: Optional[FeatureStats] = None
+        self._sample_df_stats: Optional[FeatureStats] = None
+
+        # Default labels for the artifacts
+        self._default_labels = self._get_default_labels()
+
+        # Persistent data - fetched when needed
+        self._sample_df: Optional[pd.DataFrame] = None
+        self._model_endpoint: Optional[ModelEndpoint] = model_endpoint_dict.get(
+            self.endpoint_id
         )
 
-        self.endpoint_id = attrs.get(mm_constants.ApplicationEvent.ENDPOINT_ID)
-        self._model_endpoint = model_endpoint_dict.get(self.endpoint_id)
+    def _get_default_labels(self) -> dict[str, str]:
+        return {
+            mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
+            mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
+            mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
+            mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
+        }
 
-        return self
+    def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
+        """Add the default labels to logged artifacts labels"""
+        return (labels or {}) | self._default_labels
 
     @property
     def sample_df(self) -> pd.DataFrame:
-        if not hasattr(self, "_sample_df") or self._sample_df is None:
+        if self._sample_df is None:
             feature_set = fstore.get_feature_set(
                 self.model_endpoint.status.monitoring_feature_set_uri
             )
@@ -146,15 +154,15 @@ class MonitoringApplicationContext(MLClientCtx):
 
     @property
     def model_endpoint(self) -> ModelEndpoint:
-        if not hasattr(self, "_model_endpoint") or not self._model_endpoint:
+        if not self._model_endpoint:
             self._model_endpoint = ModelEndpoint.from_flat_dict(
-                get_endpoint_record(self.project, self.endpoint_id)
+                get_endpoint_record(self.project_name, self.endpoint_id)
             )
         return self._model_endpoint
 
     @property
     def feature_stats(self) -> FeatureStats:
-        if not hasattr(self, "_feature_stats") or not self._feature_stats:
+        if not self._feature_stats:
             self._feature_stats = json.loads(self.model_endpoint.status.feature_stats)
             pad_features_hist(self._feature_stats)
         return self._feature_stats
@@ -162,7 +170,7 @@ class MonitoringApplicationContext(MLClientCtx):
     @property
     def sample_df_stats(self) -> FeatureStats:
         """statistics of the sample dataframe"""
-        if not hasattr(self, "_sample_df_stats") or not self._sample_df_stats:
+        if not self._sample_df_stats:
             self._sample_df_stats = calculate_inputs_statistics(
                 self.feature_stats, self.sample_df
             )
@@ -186,13 +194,11 @@ class MonitoringApplicationContext(MLClientCtx):
 
     @property
     def model(self) -> tuple[str, ModelArtifact, dict]:
-        """return model file, model spec object, and list of extra data items"""
+        """The model file, model spec object, and a list of extra data items"""
         return get_model(self.model_endpoint.spec.model_uri)
 
     @staticmethod
-    def dict_to_histogram(
-        histogram_dict: mlrun.common.model_monitoring.helpers.FeatureStats,
-    ) -> pd.DataFrame:
+    def dict_to_histogram(histogram_dict: FeatureStats) -> pd.DataFrame:
         """
         Convert histogram dictionary to pandas DataFrame with feature histograms as columns
 
@@ -212,3 +218,124 @@ class MonitoringApplicationContext(MLClientCtx):
         histograms = pd.DataFrame(histograms)
 
         return histograms
+
+    def log_artifact(
+        self,
+        item,
+        body=None,
+        tag: str = "",
+        local_path: str = "",
+        artifact_path: Optional[str] = None,
+        format: Optional[str] = None,
+        upload: Optional[bool] = None,
+        labels: Optional[dict[str, str]] = None,
+        target_path: Optional[str] = None,
+        **kwargs,
+    ) -> Artifact:
+        """
+        Log an artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_artifact(
+            item,
+            body=body,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            format=format,
+            upload=upload,
+            labels=labels,
+            target_path=target_path,
+            **kwargs,
+        )
+
+    def log_dataset(
+        self,
+        key,
+        df,
+        tag="",
+        local_path=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        format="",
+        preview=None,
+        stats=None,
+        target_path="",
+        extra_data=None,
+        label_column: Optional[str] = None,
+        **kwargs,
+    ) -> DatasetArtifact:
+        """
+        Log a dataset artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_dataset(
+            key,
+            df,
+            tag=tag,
+            local_path=local_path,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            format=format,
+            preview=preview,
+            stats=stats,
+            target_path=target_path,
+            extra_data=extra_data,
+            label_column=label_column,
+            **kwargs,
+        )
+
+    def log_model(
+        self,
+        key,
+        body=None,
+        framework="",
+        tag="",
+        model_dir=None,
+        model_file=None,
+        algorithm=None,
+        metrics=None,
+        parameters=None,
+        artifact_path=None,
+        upload=None,
+        labels=None,
+        inputs: Optional[list[mlrun.features.Feature]] = None,
+        outputs: Optional[list[mlrun.features.Feature]] = None,
+        feature_vector: Optional[str] = None,
+        feature_weights: Optional[list] = None,
+        training_set=None,
+        label_column=None,
+        extra_data=None,
+        **kwargs,
+    ) -> ModelArtifact:
+        """
+        Log a model artifact.
+        See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
+        """
+        labels = self._add_default_labels(labels)
+        return self.project.log_model(
+            key,
+            body=body,
+            framework=framework,
+            tag=tag,
+            model_dir=model_dir,
+            model_file=model_file,
+            algorithm=algorithm,
+            metrics=metrics,
+            parameters=parameters,
+            artifact_path=artifact_path,
+            upload=upload,
+            labels=labels,
+            inputs=inputs,
+            outputs=outputs,
+            feature_vector=feature_vector,
+            feature_weights=feature_weights,
+            training_set=training_set,
+            label_column=label_column,
+            extra_data=extra_data,
+            **kwargs,
+        )

mlrun/model_monitoring/applications/evidently_base.py

@@ -14,7 +14,7 @@
 
 import uuid
 import warnings
-from typing import Union
+from abc import ABC
 
 import pandas as pd
 import semver
@@ -23,7 +23,7 @@ import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.32")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -57,84 +57,14 @@ except ModuleNotFoundError:
 
 
 if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
-    from evidently.report.report import Report
-    from evidently.suite.base_suite import Suite
+    from evidently.suite.base_suite import Display
     from evidently.ui.type_aliases import STR_UUID
     from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
+    from evidently.utils.dashboard import TemplateParams, file_html_template
 
 
-class EvidentlyModelMonitoringApplicationBase(mm_base.ModelMonitoringApplicationBase):
-    def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
-    ) -> None:
-        """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
-
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
-        :param evidently_project_id:        (str) The ID of the Evidently project.
-
-        """
-        if not _HAS_EVIDENTLY:
-            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
-        self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
-
-    def log_evidently_object(
-        self, evidently_object: Union["Report", "Suite"], artifact_name: str
-    ):
-        """
-         Logs an Evidently report or suite as an artifact.
-
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        """
-        evidently_object_html = evidently_object.get_html()
-        self.context.log_artifact(
-            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
-        )
-
-    def log_project_dashboard(
-        self,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-    ):
-        """
-        Logs an Evidently project dashboard.
-
-        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
-        :param artifact_name:   (str) The name for the logged artifact.
-        """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = self._render(determine_template("inline"), template_params)
-        self.context.log_artifact(
-            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
-        )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)
-
-
-class EvidentlyModelMonitoringApplicationBaseV2(
-    mm_base.ModelMonitoringApplicationBaseV2
+class EvidentlyModelMonitoringApplicationBase(
+    mm_base.ModelMonitoringApplicationBase, ABC
 ):
     def __init__(
         self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
@@ -161,14 +91,14 @@ class EvidentlyModelMonitoringApplicationBaseV2(
     @staticmethod
     def log_evidently_object(
         monitoring_context: mm_context.MonitoringApplicationContext,
-        evidently_object: Union["Report", "Suite"],
+        evidently_object: "Display",
         artifact_name: str,
-    ):
+    ) -> None:
         """
          Logs an Evidently report or suite as an artifact.
 
         :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
+        :param evidently_object:    (Display) The Evidently display to log, e.g. a report or a test suite object.
         :param artifact_name:       (str) The name for the logged artifact.
         """
         evidently_object_html = evidently_object.get_html()
@@ -182,7 +112,7 @@ class EvidentlyModelMonitoringApplicationBaseV2(
         timestamp_start: pd.Timestamp,
         timestamp_end: pd.Timestamp,
         artifact_name: str = "dashboard",
-    ):
+    ) -> None:
         """
         Logs an Evidently project dashboard.
 
@@ -201,11 +131,7 @@ class EvidentlyModelMonitoringApplicationBaseV2(
             additional_graphs={},
         )
 
-        dashboard_html = self._render(determine_template("inline"), template_params)
+        dashboard_html = file_html_template(params=template_params)
         monitoring_context.log_artifact(
             artifact_name, body=dashboard_html.encode("utf-8"), format="html"
         )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -31,7 +31,7 @@ from mlrun.common.schemas.model_monitoring.constants import (
     ResultStatusApp,
 )
 from mlrun.model_monitoring.applications import (
-    ModelMonitoringApplicationBaseV2,
+    ModelMonitoringApplicationBase,
 )
 from mlrun.model_monitoring.metrics.histogram_distance import (
     HellingerDistance,
@@ -87,11 +87,13 @@ class DataDriftClassifier:
         return ResultStatusApp.no_detection
 
 
-class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
+class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     """
     MLRun's default data drift application for model monitoring.
 
-    The application expects tabular numerical data, and calculates three metrics over the features' histograms.
+    The application expects tabular numerical data, and calculates three metrics over the shared features' histograms.
+    The metrics are calculated on features that have reference data from the training dataset. When there is no
+    reference data (`feature_stats`), this application send a warning log and does nothing.
     The three metrics are:
 
     * Hellinger distance.
@@ -112,6 +114,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
         project.enable_model_monitoring()
 
+    To avoid it, pass `deploy_histogram_data_drift_app=False`.
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
@@ -195,7 +198,10 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
                 EventFieldType.CURRENT_STATS: json.dumps(
                     monitoring_context.sample_df_stats
                 ),
-                EventFieldType.DRIFT_MEASURES: metrics_per_feature.T.to_json(),
+                EventFieldType.DRIFT_MEASURES: json.dumps(
+                    metrics_per_feature.T.to_dict()
+                    | {metric.name: metric.value for metric in metrics}
+                ),
                 EventFieldType.DRIFT_STATUS: status.value,
             },
         )
@@ -220,19 +226,18 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         return metrics
 
     @staticmethod
-    def _remove_timestamp_feature(
-        sample_set_statistics: mlrun.common.model_monitoring.helpers.FeatureStats,
+    def _get_shared_features_sample_stats(
+        monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> mlrun.common.model_monitoring.helpers.FeatureStats:
         """
-        Drop the 'timestamp' feature if it exists, as it is irrelevant
-        in the plotly artifact
+        Filter out features without reference data in `feature_stats`, e.g. `timestamp`.
         """
-        sample_set_statistics = mlrun.common.model_monitoring.helpers.FeatureStats(
-            sample_set_statistics.copy()
+        return mlrun.common.model_monitoring.helpers.FeatureStats(
+            {
+                key: monitoring_context.sample_df_stats[key]
+                for key in monitoring_context.feature_stats
+            }
         )
-        if EventFieldType.TIMESTAMP in sample_set_statistics:
-            del sample_set_statistics[EventFieldType.TIMESTAMP]
-        return sample_set_statistics
 
     @staticmethod
     def _log_json_artifact(
@@ -296,8 +301,8 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             self._log_json_artifact(drift_per_feature_values, monitoring_context)
 
         self._log_plotly_table_artifact(
-            sample_set_statistics=self._remove_timestamp_feature(
-                monitoring_context.sample_df_stats
+            sample_set_statistics=self._get_shared_features_sample_stats(
+                monitoring_context
             ),
             inputs_statistics=monitoring_context.feature_stats,
             metrics_per_feature=metrics_per_feature,
@@ -322,7 +327,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         """
         monitoring_context.logger.debug("Starting to run the application")
         if not monitoring_context.feature_stats:
-            monitoring_context.logger.info(
+            monitoring_context.logger.warning(
                 "No feature statistics found, skipping the application. \n"
                 "In order to run the application, training set must be provided when logging the model."
             )

mlrun/model_monitoring/applications/results.py

@@ -29,8 +29,8 @@ class _ModelMonitoringApplicationDataRes(ABC):
     def __post_init__(self):
         pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
         if not re.fullmatch(pat, self.name):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Attribute name must be of the format [a-zA-Z_][a-zA-Z0-9_]*"
+            raise mlrun.errors.MLRunValueError(
+                "Attribute name must comply with the regex `[a-zA-Z_][a-zA-Z0-9_]*`"
             )
 
     @abstractmethod
@@ -45,7 +45,7 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
 
     :param name:           (str) Name of the application result. This name must be
                             unique for each metric in a single application
-                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
+                            (name must be of the format :code:`[a-zA-Z_][a-zA-Z0-9_]*`).
     :param value:          (float) Value of the application result.
     :param kind:           (ResultKindApp) Kind of application result.
     :param status:         (ResultStatusApp) Status of the application result.
@@ -80,7 +80,7 @@ class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
 
     :param name:           (str) Name of the application metric. This name must be
                             unique for each metric in a single application
-                            (name must be of the format [a-zA-Z_][a-zA-Z0-9_]*).
+                            (name must be of the format :code:`[a-zA-Z_][a-zA-Z0-9_]*`).
     :param value:          (float) Value of the application metric.
     """