mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc22__py3-none-any.whl

mlrun/model_monitoring/evidently_application.py

@@ -12,121 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import uuid
-import warnings
-from typing import Union
-
-import pandas as pd
-import semver
-
-from mlrun.errors import MLRunIncompatibleVersionError
-from mlrun.model_monitoring.application import ModelMonitoringApplicationBase
-
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
-
-
-def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
-    if ref.is_compatible(cur) or (
-        cur.major == ref.major == 0 and cur.minor == ref.minor and cur.patch > ref.patch
-    ):
-        return
-    if cur.major == ref.major == 0 and cur.minor > ref.minor:
-        warnings.warn(
-            f"Evidently version {cur} is not compatible with the tested "
-            f"version {ref}, use at your own risk."
-        )
-    else:
-        raise MLRunIncompatibleVersionError(
-            f"Evidently version {cur} is not supported, please change to "
-            f"{ref} (or another compatible version)."
-        )
-
-
-_HAS_EVIDENTLY = False
-try:
-    import evidently  # noqa: F401
-
-    _check_evidently_version(
-        cur=semver.Version.parse(evidently.__version__),
-        ref=SUPPORTED_EVIDENTLY_VERSION,
-    )
-    _HAS_EVIDENTLY = True
-except ModuleNotFoundError:
-    pass
-
-
-if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
-    from evidently.report.report import Report
-    from evidently.suite.base_suite import Suite
-    from evidently.ui.type_aliases import STR_UUID
-    from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
-
-
-class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
-    def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
-    ) -> None:
-        """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
-
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
-        :param evidently_project_id:        (str) The ID of the Evidently project.
-
-        """
-        if not _HAS_EVIDENTLY:
-            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
-        self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
-
-    def log_evidently_object(
-        self, evidently_object: Union["Report", "Suite"], artifact_name: str
-    ):
-        """
-         Logs an Evidently report or suite as an artifact.
-
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        """
-        evidently_object_html = evidently_object.get_html()
-        self.context.log_artifact(
-            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
-        )
-
-    def log_project_dashboard(
-        self,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-    ):
-        """
-        Logs an Evidently project dashboard.
-
-        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
-        :param artifact_name:   (str) The name for the logged artifact.
-        """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = self._render(determine_template("inline"), template_params)
-        self.context.log_artifact(
-            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
-        )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)
+# TODO : delete this file in 1.9.0
+from mlrun.model_monitoring.applications import (  # noqa: F401
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+)

mlrun/model_monitoring/helpers.py

@@ -24,6 +24,11 @@ import mlrun.common.schemas
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
 )
+from mlrun.common.schemas.model_monitoring.model_endpoints import (
+    ModelEndpointMonitoringMetric,
+    ModelEndpointMonitoringMetricType,
+    _compose_full_name,
+)
 from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
@@ -111,6 +116,24 @@ def get_connection_string(secret_provider: typing.Callable = None) -> str:
     )
 
 
+def get_tsdb_connection_string(
+    secret_provider: typing.Optional[typing.Callable] = None,
+) -> str:
+    """Get TSDB connection string from the project secret. If wasn't set, take it from the system
+    configurations.
+    :param secret_provider: An optional secret provider to get the connection string secret.
+    :return:                Valid TSDB connection string.
+    """
+
+    return (
+        mlrun.get_secret_or_env(
+            key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION,
+            secret_provider=secret_provider,
+        )
+        or mlrun.mlconf.model_endpoint_monitoring.tsdb_connection
+    )
+
+
 def batch_dict2timedelta(batch_dict: _BatchDict) -> datetime.timedelta:
     """
     Convert a batch dictionary to timedelta.
@@ -215,7 +238,7 @@ def update_model_endpoint_last_request(
 
 def calculate_inputs_statistics(
     sample_set_statistics: dict, inputs: pd.DataFrame
-) -> dict:
+) -> mlrun.common.model_monitoring.helpers.FeatureStats:
     """
     Calculate the inputs data statistics for drift monitoring purpose.
 
@@ -260,3 +283,42 @@ def get_endpoint_record(project: str, endpoint_id: str):
         project=project,
     )
     return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
+
+
+def get_result_instance_fqn(
+    model_endpoint_id: str, app_name: str, result_name: str
+) -> str:
+    return f"{model_endpoint_id}.{app_name}.result.{result_name}"
+
+
+def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
+    return get_result_instance_fqn(
+        model_endpoint_id,
+        mm_constants.HistogramDataDriftApplicationConstants.NAME,
+        mm_constants.HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+    )
+
+
+def get_invocations_fqn(project: str) -> str:
+    return _compose_full_name(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+    )
+
+
+def get_invocations_metric(project: str) -> ModelEndpointMonitoringMetric:
+    """
+    Return the invocations metric of any model endpoint in the given project.
+
+    :param project: The project name.
+    :returns:       The model monitoring metric object.
+    """
+    return ModelEndpointMonitoringMetric(
+        project=project,
+        app=mm_constants.SpecialApps.MLRUN_INFRA,
+        type=ModelEndpointMonitoringMetricType.METRIC,
+        name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
+        full_name=get_invocations_fqn(project),
+    )

mlrun/model_monitoring/model_endpoint.py

@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
 from typing import Any
 
 import mlrun.model
+from mlrun.common.model_monitoring.helpers import FeatureStats
 from mlrun.common.schemas.model_monitoring.constants import (
     EndpointType,
     EventKeyMetrics,
@@ -42,8 +43,8 @@ class ModelEndpointSpec(mlrun.model.ModelObj):
 
 @dataclass
 class ModelEndpointStatus(mlrun.model.ModelObj):
-    feature_stats: dict = field(default_factory=dict)
-    current_stats: dict = field(default_factory=dict)
+    feature_stats: FeatureStats = field(default_factory=dict)
+    current_stats: FeatureStats = field(default_factory=dict)
     first_request: str = ""
     last_request: str = ""
     error_count: int = 0