mlrun 1.7.0rc4__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/model_monitoring/evidently_application.py

@@ -12,121 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import uuid
-import warnings
-from typing import Union
-
-import pandas as pd
-import semver
-
-from mlrun.errors import MLRunIncompatibleVersionError
-from mlrun.model_monitoring.application import ModelMonitoringApplicationBase
-
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
-
-
-def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
-    if ref.is_compatible(cur) or (
-        cur.major == ref.major == 0 and cur.minor == ref.minor and cur.patch > ref.patch
-    ):
-        return
-    if cur.major == ref.major == 0 and cur.minor > ref.minor:
-        warnings.warn(
-            f"Evidently version {cur} is not compatible with the tested "
-            f"version {ref}, use at your own risk."
-        )
-    else:
-        raise MLRunIncompatibleVersionError(
-            f"Evidently version {cur} is not supported, please change to "
-            f"{ref} (or another compatible version)."
-        )
-
-
-_HAS_EVIDENTLY = False
-try:
-    import evidently  # noqa: F401
-
-    _check_evidently_version(
-        cur=semver.Version.parse(evidently.__version__),
-        ref=SUPPORTED_EVIDENTLY_VERSION,
-    )
-    _HAS_EVIDENTLY = True
-except ModuleNotFoundError:
-    pass
-
-
-if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
-    from evidently.report.report import Report
-    from evidently.suite.base_suite import Suite
-    from evidently.ui.type_aliases import STR_UUID
-    from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
-
-
-class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
-    def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
-    ) -> None:
-        """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
-
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
-        :param evidently_project_id:        (str) The ID of the Evidently project.
-
-        """
-        if not _HAS_EVIDENTLY:
-            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
-        self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
-
-    def log_evidently_object(
-        self, evidently_object: Union["Report", "Suite"], artifact_name: str
-    ):
-        """
-         Logs an Evidently report or suite as an artifact.
-
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        """
-        evidently_object_html = evidently_object.get_html()
-        self.context.log_artifact(
-            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
-        )
-
-    def log_project_dashboard(
-        self,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-    ):
-        """
-        Logs an Evidently project dashboard.
-
-        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
-        :param artifact_name:   (str) The name for the logged artifact.
-        """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = self._render(determine_template("inline"), template_params)
-        self.context.log_artifact(
-            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
-        )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)
+# TODO : delete this file in 1.9.0
+from mlrun.model_monitoring.applications import (  # noqa: F401
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+)

mlrun/model_monitoring/features_drift_table.py

@@ -21,9 +21,34 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
 import mlrun.common.schemas.model_monitoring
+from mlrun.artifacts import PlotlyArtifact
 
 # A type for representing a drift result, a tuple of the status and the drift mean:
-DriftResultType = tuple[mlrun.common.schemas.model_monitoring.DriftStatus, float]
+DriftResultType = tuple[
+    mlrun.common.schemas.model_monitoring.constants.ResultStatusApp, float
+]
+
+
+class _PlotlyTableArtifact(PlotlyArtifact):
+    """A custom class for plotly table artifacts"""
+
+    @staticmethod
+    def _disable_table_dragging(figure_html: str) -> str:
+        """
+        Disable the table columns dragging by adding the following
+        JavaScript code
+        """
+        start, end = figure_html.rsplit(";", 1)
+        middle = (
+            ';for (const element of document.getElementsByClassName("table")) '
+            '{element.style.pointerEvents = "none";}'
+        )
+        figure_html = start + middle + end
+        return figure_html
+
+    def get_body(self) -> str:
+        """Get the adjusted HTML representation of the figure"""
+        return self._disable_table_dragging(super().get_body())
 
 
 class FeaturesDriftTablePlot:
@@ -62,9 +87,9 @@ class FeaturesDriftTablePlot:
 
     # Status configurations:
     _STATUS_COLORS = {
-        mlrun.common.schemas.model_monitoring.DriftStatus.NO_DRIFT: "rgb(0,176,80)",  # Green
-        mlrun.common.schemas.model_monitoring.DriftStatus.POSSIBLE_DRIFT: "rgb(255,192,0)",  # Orange
-        mlrun.common.schemas.model_monitoring.DriftStatus.DRIFT_DETECTED: "rgb(208,0,106)",  # Magenta
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.no_detection: "rgb(0,176,80)",  # Green
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.potential_detection: "rgb(255,192,0)",  # Orange
+        mlrun.common.schemas.model_monitoring.constants.ResultStatusApp.detected: "rgb(208,0,106)",  # Magenta
     }
 
     # Font configurations:
@@ -97,7 +122,7 @@ class FeaturesDriftTablePlot:
         inputs_statistics: dict,
         metrics: dict[str, Union[dict, float]],
         drift_results: dict[str, DriftResultType],
-    ) -> str:
+    ) -> _PlotlyTableArtifact:
         """
         Produce the html code of the table plot with the given information and the stored configurations in the class.
 
@@ -106,9 +131,8 @@ class FeaturesDriftTablePlot:
         :param metrics:               The drift detection metrics calculated on the sample set and inputs.
         :param drift_results:         The drift results per feature according to the rules of the monitor.
 
-        :return: The full path to the html file of the plot.
+        :return: The drift table as a plotly artifact.
         """
-        # Plot the drift table:
         figure = self._plot(
             features=list(inputs_statistics.keys()),
             sample_set_statistics=sample_set_statistics,
@@ -116,19 +140,7 @@ class FeaturesDriftTablePlot:
             metrics=metrics,
             drift_results=drift_results,
         )
-
-        # Get its HTML representation:
-        figure_html = figure.to_html()
-
-        # Turn off the table columns dragging by injecting the following JavaScript code:
-        start, end = figure_html.rsplit(";", 1)
-        middle = (
-            ';for (const element of document.getElementsByClassName("table")) '
-            '{element.style.pointerEvents = "none";}'
-        )
-        figure_html = start + middle + end
-
-        return figure_html
+        return _PlotlyTableArtifact(figure=figure, key="drift_table_plot")
 
     def _read_columns_names(self, statistics_dictionary: dict, drift_metrics: dict):
         """
@@ -366,10 +378,10 @@ class FeaturesDriftTablePlot:
             bins = np.array(bins)
             if bins[0] == -sys.float_info.max:
                 bins[0] = bins[1] - (bins[2] - bins[1])
-                hovertext[0] = f"(-∞, {bins[1]})"
+                hovertext[0] = f"(-inf, {bins[1]})"
             if bins[-1] == sys.float_info.max:
                 bins[-1] = bins[-2] + (bins[-2] - bins[-3])
-                hovertext[-1] = f"({bins[-2]}, ∞)"
+                hovertext[-1] = f"({bins[-2]}, inf)"
             # Center the bins (leave the first one):
             bins = 0.5 * (bins[:-1] + bins[1:])
             # Plot the histogram as a line with filled background below it:

mlrun/model_monitoring/helpers.py

@@ -15,12 +15,19 @@
 import datetime
 import typing
 
+import numpy as np
+import pandas as pd
+
 import mlrun
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
 )
+from mlrun.common.schemas.model_monitoring.model_endpoints import (
+    ModelEndpointMonitoringMetricType,
+    _compose_full_name,
+)
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
@@ -36,14 +43,10 @@ class _BatchDict(typing.TypedDict):
     days: int
 
 
-class _MLRunNoRunsFoundError(Exception):
-    pass
-
-
 def get_stream_path(
     project: str = None,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
-):
+) -> str:
     """
     Get stream path from the project secret. If wasn't set, take it from the system configurations
 
@@ -55,8 +58,6 @@ def get_stream_path(
 
     stream_uri = mlrun.get_secret_or_env(
         mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
-        if function_name is mm_constants.MonitoringFunctionNames.STREAM
-        else ""
     ) or mlrun.mlconf.get_model_monitoring_file_target_path(
         project=project,
         kind=mlrun.common.schemas.model_monitoring.FileTargetKind.STREAM,
@@ -64,6 +65,8 @@ def get_stream_path(
         function_name=function_name,
     )
 
+    if isinstance(stream_uri, list):  # ML-6043 - user side gets only the new stream uri
+        stream_uri = stream_uri[1]  # get new stream path, under projects
     return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
         stream_uri=stream_uri, project=project, function_name=function_name
     )
@@ -112,6 +115,24 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
     )
 
 
+def get_tsdb_connection_string(
+    secret_provider: typing.Optional[typing.Callable] = None,
+) -> str:
+    """Get TSDB connection string from the project secret. If wasn't set, take it from the system
+    configurations.
+    :param secret_provider: An optional secret provider to get the connection string secret.
+    :return:                Valid TSDB connection string.
+    """
+
+    return (
+        mlrun.get_secret_or_env(
+            key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION,
+            secret_provider=secret_provider,
+        )
+        or mlrun.mlconf.model_endpoint_monitoring.tsdb_connection
+    )
+
+
 def batch_dict2timedelta(batch_dict: _BatchDict) -> datetime.timedelta:
     """
     Convert a batch dictionary to timedelta.
@@ -212,3 +233,75 @@ def update_model_endpoint_last_request(
             endpoint_id=model_endpoint.metadata.uid,
             attributes={EventFieldType.LAST_REQUEST: bumped_last_request},
         )
+
+
+def calculate_inputs_statistics(
+    sample_set_statistics: dict, inputs: pd.DataFrame
+) -> mlrun.common.model_monitoring.helpers.FeatureStats:
+    """
+    Calculate the inputs data statistics for drift monitoring purpose.
+
+    :param sample_set_statistics: The sample set (stored end point's dataset to reference) statistics. The bins of the
+                                  histograms of each feature will be used to recalculate the histograms of the inputs.
+    :param inputs:                The inputs to calculate their statistics and later on - the drift with respect to the
+                                  sample set.
+
+    :returns: The calculated statistics of the inputs data.
+    """
+
+    # Use `DFDataInfer` to calculate the statistics over the inputs:
+    inputs_statistics = mlrun.data_types.infer.DFDataInfer.get_stats(
+        df=inputs,
+        options=mlrun.data_types.infer.InferOptions.Histogram,
+    )
+
+    # Recalculate the histograms over the bins that are set in the sample-set of the end point:
+    for feature in inputs_statistics.keys():
+        if feature in sample_set_statistics:
+            counts, bins = np.histogram(
+                inputs[feature].to_numpy(),
+                bins=sample_set_statistics[feature]["hist"][1],
+            )
+            inputs_statistics[feature]["hist"] = [
+                counts.tolist(),
+                bins.tolist(),
+            ]
+        elif "hist" in inputs_statistics[feature]:
+            # Comply with the other common features' histogram length
+            mlrun.common.model_monitoring.helpers.pad_hist(
+                mlrun.common.model_monitoring.helpers.Histogram(
+                    inputs_statistics[feature]["hist"]
+                )
+            )
+
+    return inputs_statistics
+
+
+def get_endpoint_record(project: str, endpoint_id: str):
+    model_endpoint_store = mlrun.model_monitoring.get_store_object(
+        project=project,
+    )
+    return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
+
+
+def get_result_instance_fqn(
+    model_endpoint_id: str, app_name: str, result_name: str
+) -> str:
+    return f"{model_endpoint_id}.{app_name}.result.{result_name}"
+
+
+def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
+    return get_result_instance_fqn(
+        model_endpoint_id,
+        mm_constants.HistogramDataDriftApplicationConstants.NAME,
+        mm_constants.HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+    )
+
+
+def get_invocations_fqn(project: str) -> str:
+    return _compose_full_name(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+    )

mlrun/model_monitoring/model_endpoint.py

@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
 from typing import Any
 
 import mlrun.model
+from mlrun.common.model_monitoring.helpers import FeatureStats
 from mlrun.common.schemas.model_monitoring.constants import (
     EndpointType,
     EventKeyMetrics,
@@ -42,8 +43,8 @@ class ModelEndpointSpec(mlrun.model.ModelObj):
 
 @dataclass
 class ModelEndpointStatus(mlrun.model.ModelObj):
-    feature_stats: dict = field(default_factory=dict)
-    current_stats: dict = field(default_factory=dict)
+    feature_stats: FeatureStats = field(default_factory=dict)
+    current_stats: FeatureStats = field(default_factory=dict)
     first_request: str = ""
     last_request: str = ""
     error_count: int = 0