mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -409,7 +409,7 @@ class SQLStoreBase(mlrun.model_monitoring.db.StoreBase):
             + "_"
             + event[mlrun.common.schemas.model_monitoring.WriterEvent.APPLICATION_NAME]
             + "_"
-            + event[mlrun.common.schemas.model_monitoring.WriterEvent.RESULT_NAME]
+            + event[mlrun.common.schemas.model_monitoring.ResultData.RESULT_NAME]
         )
 
     def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import json
 import os
@@ -41,7 +40,7 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
     client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
     """
 
-    def __init__(self, project: str, access_key: str):
+    def __init__(self, project: str, access_key: typing.Optional[str] = None) -> None:
         super().__init__(project=project)
         # Initialize a V3IO client instance
         self.access_key = access_key or os.environ.get("V3IO_ACCESS_KEY")
@@ -410,7 +409,7 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
             mlrun.common.schemas.model_monitoring.WriterEvent.APPLICATION_NAME
         )
         metric_name = event.pop(
-            mlrun.common.schemas.model_monitoring.WriterEvent.RESULT_NAME
+            mlrun.common.schemas.model_monitoring.ResultData.RESULT_NAME
         )
         attributes = {metric_name: json.dumps(event)}
 
@@ -446,7 +445,7 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
         """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
         fields = [
             {
-                "name": mlrun.common.schemas.model_monitoring.WriterEvent.RESULT_NAME,
+                "name": mlrun.common.schemas.model_monitoring.ResultData.RESULT_NAME,
                 "type": "string",
                 "nullable": False,
             }
@@ -703,3 +702,67 @@ class KVStoreBase(mlrun.model_monitoring.db.StoreBase):
     @staticmethod
     def _get_monitoring_schedules_container(project_name: str) -> str:
         return f"users/pipelines/{project_name}/monitoring-schedules/functions"
+
+    def _extract_metrics_from_items(
+        self, app_items: list[dict[str, str]]
+    ) -> list[mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetric]:
+        metrics: list[
+            mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetric
+        ] = []
+        for app_item in app_items:
+            # See https://www.iguazio.com/docs/latest-release/services/data-layer/reference/system-attributes/#sys-attr-__name
+            app_name = app_item.pop("__name")
+            if app_name == ".#schema":
+                continue
+            for result_name in app_item:
+                metrics.append(
+                    mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetric(
+                        project=self.project,
+                        app=app_name,
+                        type=mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetricType.RESULT,
+                        name=result_name,
+                        full_name=".".join(
+                            [
+                                self.project,
+                                app_name,
+                                mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetricType.RESULT,
+                                result_name,
+                            ]
+                        ),
+                    )
+                )
+        return metrics
+
+    def get_model_endpoint_metrics(
+        self, endpoint_id: str
+    ) -> list[mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetric]:
+        """Get model monitoring results and metrics on the endpoint"""
+        metrics: list[
+            mlrun.common.schemas.model_monitoring.ModelEndpointMonitoringMetric
+        ] = []
+        container = self.get_v3io_monitoring_apps_container(self.project)
+        try:
+            response = self.client.kv.scan(container=container, table_path=endpoint_id)
+        except v3io.dataplane.response.HttpResponseError as err:
+            if err.status_code == HTTPStatus.NOT_FOUND:
+                logger.warning(
+                    "Attempt getting metrics and results - no data. Check the "
+                    "project name, endpoint, or wait for the applications to start.",
+                    container=container,
+                    table_path=endpoint_id,
+                )
+                return []
+            raise
+
+        while True:
+            metrics.extend(self._extract_metrics_from_items(response.output.items))
+            if response.output.last:
+                break
+            # TODO: Use AIO client: `v3io.aio.dataplane.client.Client`
+            response = self.client.kv.scan(
+                container=container,
+                table_path=endpoint_id,
+                marker=response.output.next_marker,
+            )
+
+        return metrics

mlrun/model_monitoring/evidently_application.py

@@ -12,121 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import uuid
-import warnings
-from typing import Union
-
-import pandas as pd
-import semver
-
-from mlrun.errors import MLRunIncompatibleVersionError
-from mlrun.model_monitoring.application import ModelMonitoringApplicationBase
-
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
-
-
-def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
-    if ref.is_compatible(cur) or (
-        cur.major == ref.major == 0 and cur.minor == ref.minor and cur.patch > ref.patch
-    ):
-        return
-    if cur.major == ref.major == 0 and cur.minor > ref.minor:
-        warnings.warn(
-            f"Evidently version {cur} is not compatible with the tested "
-            f"version {ref}, use at your own risk."
-        )
-    else:
-        raise MLRunIncompatibleVersionError(
-            f"Evidently version {cur} is not supported, please change to "
-            f"{ref} (or another compatible version)."
-        )
-
-
-_HAS_EVIDENTLY = False
-try:
-    import evidently  # noqa: F401
-
-    _check_evidently_version(
-        cur=semver.Version.parse(evidently.__version__),
-        ref=SUPPORTED_EVIDENTLY_VERSION,
-    )
-    _HAS_EVIDENTLY = True
-except ModuleNotFoundError:
-    pass
-
-
-if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
-    from evidently.report.report import Report
-    from evidently.suite.base_suite import Suite
-    from evidently.ui.type_aliases import STR_UUID
-    from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
-
-
-class EvidentlyModelMonitoringApplicationBase(ModelMonitoringApplicationBase):
-    def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
-    ) -> None:
-        """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
-
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
-        :param evidently_project_id:        (str) The ID of the Evidently project.
-
-        """
-        if not _HAS_EVIDENTLY:
-            raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
-        self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
-
-    def log_evidently_object(
-        self, evidently_object: Union["Report", "Suite"], artifact_name: str
-    ):
-        """
-         Logs an Evidently report or suite as an artifact.
-
-        :param evidently_object:    (Union[Report, Suite]) The Evidently report or suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        """
-        evidently_object_html = evidently_object.get_html()
-        self.context.log_artifact(
-            artifact_name, body=evidently_object_html.encode("utf-8"), format="html"
-        )
-
-    def log_project_dashboard(
-        self,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-    ):
-        """
-        Logs an Evidently project dashboard.
-
-        :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:   (pd.Timestamp) The end timestamp for the dashboard data.
-        :param artifact_name:   (str) The name for the logged artifact.
-        """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = self._render(determine_template("inline"), template_params)
-        self.context.log_artifact(
-            artifact_name, body=dashboard_html.encode("utf-8"), format="html"
-        )
-
-    @staticmethod
-    def _render(temple_func, template_params: "TemplateParams"):
-        return temple_func(params=template_params)
+# TODO : delete this file in 1.9.0
+from mlrun.model_monitoring.applications import (  # noqa: F401
+    _HAS_EVIDENTLY,
+    SUPPORTED_EVIDENTLY_VERSION,
+    EvidentlyModelMonitoringApplicationBase,
+)

mlrun/model_monitoring/helpers.py

@@ -215,7 +215,7 @@ def update_model_endpoint_last_request(
 
 def calculate_inputs_statistics(
     sample_set_statistics: dict, inputs: pd.DataFrame
-) -> dict:
+) -> mlrun.common.model_monitoring.helpers.FeatureStats:
     """
     Calculate the inputs data statistics for drift monitoring purpose.
 

mlrun/model_monitoring/model_endpoint.py

@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
 from typing import Any
 
 import mlrun.model
+from mlrun.common.model_monitoring.helpers import FeatureStats
 from mlrun.common.schemas.model_monitoring.constants import (
     EndpointType,
     EventKeyMetrics,
@@ -42,8 +43,8 @@ class ModelEndpointSpec(mlrun.model.ModelObj):
 
 @dataclass
 class ModelEndpointStatus(mlrun.model.ModelObj):
-    feature_stats: dict = field(default_factory=dict)
-    current_stats: dict = field(default_factory=dict)
+    feature_stats: FeatureStats = field(default_factory=dict)
+    current_stats: FeatureStats = field(default_factory=dict)
     first_request: str = ""
     last_request: str = ""
     error_count: int = 0

mlrun/model_monitoring/stream_processing.py

@@ -40,7 +40,6 @@ from mlrun.common.schemas.model_monitoring.constants import (
     ProjectSecretKeys,
     PrometheusEndpoints,
 )
-from mlrun.model_monitoring.helpers import get_endpoint_record
 from mlrun.utils import logger
 
 
@@ -807,7 +806,7 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
         # left them
         if endpoint_id not in self.endpoints:
             logger.info("Trying to resume state", endpoint_id=endpoint_id)
-            endpoint_record = get_endpoint_record(
+            endpoint_record = mlrun.model_monitoring.helpers.get_endpoint_record(
                 project=self.project,
                 endpoint_id=endpoint_id,
             )
@@ -940,7 +939,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
         label_values = event[EventFieldType.PREDICTION]
         # Get feature names and label columns
         if endpoint_id not in self.feature_names:
-            endpoint_record = get_endpoint_record(
+            endpoint_record = mlrun.model_monitoring.helpers.get_endpoint_record(
                 project=self.project,
                 endpoint_id=endpoint_id,
             )

mlrun/model_monitoring/writer.py

@@ -23,14 +23,18 @@ from v3io_frames.errors import Error as V3IOFramesError
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
+import mlrun.common.schemas
 import mlrun.common.schemas.alert as alert_constants
 import mlrun.model_monitoring
 import mlrun.model_monitoring.db.stores
 import mlrun.utils.v3io_clients
 from mlrun.common.schemas.model_monitoring.constants import (
     EventFieldType,
+    MetricData,
+    ResultData,
     ResultStatusApp,
     WriterEvent,
+    WriterEventKind,
 )
 from mlrun.common.schemas.notification import NotificationKind, NotificationSeverity
 from mlrun.model_monitoring.helpers import get_endpoint_record
@@ -75,20 +79,20 @@ class _Notifier:
         self._severity = severity
 
     def _should_send_event(self) -> bool:
-        return self._event[WriterEvent.RESULT_STATUS] >= ResultStatusApp.detected
+        return self._event[ResultData.RESULT_STATUS] >= ResultStatusApp.detected.value
 
     def _generate_message(self) -> str:
         return f"""\
 The monitoring app `{self._event[WriterEvent.APPLICATION_NAME]}` \
-of kind `{self._event[WriterEvent.RESULT_KIND]}` \
+of kind `{self._event[ResultData.RESULT_KIND]}` \
 detected a problem in model endpoint ID `{self._event[WriterEvent.ENDPOINT_ID]}` \
 at time `{self._event[WriterEvent.START_INFER_TIME]}`.
 
 Result data:
-Name: `{self._event[WriterEvent.RESULT_NAME]}`
-Value: `{self._event[WriterEvent.RESULT_VALUE]}`
-Status: `{self._event[WriterEvent.RESULT_STATUS]}`
-Extra data: `{self._event[WriterEvent.RESULT_EXTRA_DATA]}`\
+Name: `{self._event[ResultData.RESULT_NAME]}`
+Value: `{self._event[ResultData.RESULT_VALUE]}`
+Status: `{self._event[ResultData.RESULT_STATUS]}`
+Extra data: `{self._event[ResultData.RESULT_EXTRA_DATA]}`\
 """
 
     def notify(self) -> None:
@@ -144,19 +148,25 @@ class ModelMonitoringWriter(StepToDict):
             rate=_TSDB_RATE,
         )
 
-    def _update_kv_db(self, event: _AppResultEvent) -> None:
+    def _update_kv_db(self, event: _AppResultEvent, kind: str = "result") -> None:
+        if kind == "metric":
+            # TODO : Implement the logic for writing metrics to KV
+            return
         event = _AppResultEvent(event.copy())
         application_result_store = mlrun.model_monitoring.get_store_object(
             project=self.project
         )
         application_result_store.write_application_result(event=event)
 
-    def _update_tsdb(self, event: _AppResultEvent) -> None:
+    def _update_tsdb(self, event: _AppResultEvent, kind: str = "result") -> None:
+        if kind == "metric":
+            # TODO : Implement the logic for writing metrics to TSDB
+            return
         event = _AppResultEvent(event.copy())
         event[WriterEvent.END_INFER_TIME] = datetime.datetime.fromisoformat(
             event[WriterEvent.END_INFER_TIME]
         )
-        del event[WriterEvent.RESULT_EXTRA_DATA]
+        del event[ResultData.RESULT_EXTRA_DATA]
         try:
             self._tsdb_client.write(
                 backend=_TSDB_BE,
@@ -166,7 +176,7 @@ class ModelMonitoringWriter(StepToDict):
                     WriterEvent.END_INFER_TIME,
                     WriterEvent.ENDPOINT_ID,
                     WriterEvent.APPLICATION_NAME,
-                    WriterEvent.RESULT_NAME,
+                    ResultData.RESULT_NAME,
                 ],
             )
             logger.info("Updated V3IO TSDB successfully", table=_TSDB_TABLE)
@@ -180,20 +190,21 @@ class ModelMonitoringWriter(StepToDict):
 
     @staticmethod
     def _generate_event_on_drift(
-        uid: str, drift_status: str, event_value: dict, project_name: str
-    ):
+        model_endpoint: str, drift_status: str, event_value: dict, project_name: str
+    ) -> None:
         if (
-            drift_status == ResultStatusApp.detected
-            or drift_status == ResultStatusApp.potential_detection
+            drift_status == ResultStatusApp.detected.value
+            or drift_status == ResultStatusApp.potential_detection.value
         ):
+            logger.info("Sending an alert")
             entity = {
                 "kind": alert_constants.EventEntityKind.MODEL,
                 "project": project_name,
-                "id": uid,
+                "model_endpoint": model_endpoint,
             }
             event_kind = (
                 alert_constants.EventKind.DRIFT_DETECTED
-                if drift_status == ResultStatusApp.detected
+                if drift_status == ResultStatusApp.detected.value
                 else alert_constants.EventKind.DRIFT_SUSPECTED
             )
             event_data = mlrun.common.schemas.Event(
@@ -202,37 +213,57 @@ class ModelMonitoringWriter(StepToDict):
             mlrun.get_run_db().generate_event(event_kind, event_data)
 
     @staticmethod
-    def _reconstruct_event(event: _RawEvent) -> _AppResultEvent:
+    def _reconstruct_event(event: _RawEvent) -> tuple[_AppResultEvent, str]:
         """
         Modify the raw event into the expected monitoring application event
         schema as defined in `mlrun.common.schemas.model_monitoring.constants.WriterEvent`
         """
-        try:
-            result_event = _AppResultEvent(
-                {key: event[key] for key in WriterEvent.list()}
+        if not isinstance(event, dict):
+            raise _WriterEventTypeError(
+                f"The event is of type: {type(event)}, expected a dictionary"
             )
-            result_event[WriterEvent.CURRENT_STATS] = json.loads(
-                event[WriterEvent.CURRENT_STATS]
+        kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
+        result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
+        if not result_event:  # BC for < 1.7.0, can be removed in 1.9.0
+            result_event = _AppResultEvent(event)
+        else:
+            result_event.update(_AppResultEvent(event))
+
+        expected_keys = list(
+            set(WriterEvent.list()).difference(
+                [WriterEvent.EVENT_KIND, WriterEvent.DATA]
             )
-            return result_event
-        except KeyError as err:
+        )
+        if kind == WriterEventKind.METRIC:
+            expected_keys.extend(MetricData.list())
+        elif kind == WriterEventKind.RESULT:
+            expected_keys.extend(ResultData.list())
+        else:
             raise _WriterEventValueError(
-                "The received event misses some keys compared to the expected "
-                "monitoring application event schema"
-            ) from err
-        except TypeError as err:
-            raise _WriterEventTypeError(
-                f"The event is of type: {type(event)}, expected a dictionary"
-            ) from err
+                f"Unknown event kind: {kind}, expected one of: {WriterEventKind.list()}"
+            )
+        missing_keys = [key for key in expected_keys if key not in result_event]
+        if missing_keys:
+            raise _WriterEventValueError(
+                f"The received event misses some keys compared to the expected "
+                f"monitoring application event schema: {missing_keys}"
+            )
+
+        return result_event, kind
 
     def do(self, event: _RawEvent) -> None:
-        event = self._reconstruct_event(event)
+        event, kind = self._reconstruct_event(event)
         logger.info("Starting to write event", event=event)
-        self._update_tsdb(event)
-        self._update_kv_db(event)
+
+        self._update_tsdb(event, kind)
+        self._update_kv_db(event, kind)
+        logger.info("Completed event DB writes")
         _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
 
-        if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled:
+        if (
+            mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled
+            and kind == WriterEventKind.RESULT
+        ):
             endpoint_id = event[WriterEvent.ENDPOINT_ID]
             endpoint_record = self._endpoints_records.setdefault(
                 endpoint_id,
@@ -242,13 +273,12 @@ class ModelMonitoringWriter(StepToDict):
                 "app_name": event[WriterEvent.APPLICATION_NAME],
                 "model": endpoint_record.get(EventFieldType.MODEL),
                 "model_endpoint_id": event[WriterEvent.ENDPOINT_ID],
-                "result_name": event[WriterEvent.RESULT_NAME],
-                "result_value": event[WriterEvent.RESULT_VALUE],
+                "result_name": event[ResultData.RESULT_NAME],
+                "result_value": event[ResultData.RESULT_VALUE],
             }
             self._generate_event_on_drift(
                 event[WriterEvent.ENDPOINT_ID],
-                event[WriterEvent.RESULT_STATUS],
+                event[ResultData.RESULT_STATUS],
                 event_value,
                 self.project,
             )
-        logger.info("Completed event DB writes")

mlrun/platforms/iguazio.py

@@ -525,8 +525,8 @@ def add_or_refresh_credentials(
     # different access keys for the 2 usages
     token = (
         token
-        # can't use mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session cause this is running in the
-        # import execution path (when we're initializing the run db) and therefore we can't import mlrun.runtimes
+        # can't use mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session cause this is running
+        # in the import execution path (when we're initializing the run db) and therefore we can't import mlrun.runtimes
         or os.environ.get("MLRUN_AUTH_SESSION")
         or os.environ.get("V3IO_ACCESS_KEY")
     )

mlrun/projects/pipelines.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import abc
 import builtins
+import http
 import importlib.util as imputil
 import os
 import tempfile
@@ -521,7 +522,7 @@ class _PipelineRunner(abc.ABC):
     @staticmethod
     def _get_handler(workflow_handler, workflow_spec, project, secrets):
         if not (workflow_handler and callable(workflow_handler)):
-            workflow_file = workflow_spec.get_source_file(project.spec.context)
+            workflow_file = workflow_spec.get_source_file(project.spec.get_code_path())
             workflow_handler = create_pipeline(
                 project,
                 workflow_file,
@@ -553,7 +554,7 @@ class _KFPRunner(_PipelineRunner):
     @classmethod
     def save(cls, project, workflow_spec: WorkflowSpec, target, artifact_path=None):
         pipeline_context.set(project, workflow_spec)
-        workflow_file = workflow_spec.get_source_file(project.spec.context)
+        workflow_file = workflow_spec.get_source_file(project.spec.get_code_path())
         functions = FunctionsDict(project)
         pipeline = create_pipeline(
             project,
@@ -882,17 +883,33 @@ class _RemoteRunner(_PipelineRunner):
                 get_workflow_id_timeout=get_workflow_id_timeout,
             )
 
+            def _get_workflow_id_or_bail():
+                try:
+                    return run_db.get_workflow_id(
+                        project=project.name,
+                        name=workflow_response.name,
+                        run_id=workflow_response.run_id,
+                        engine=workflow_spec.engine,
+                    )
+                except mlrun.errors.MLRunHTTPStatusError as get_wf_exc:
+                    # fail fast on specific errors
+                    if get_wf_exc.error_status_code in [
+                        http.HTTPStatus.PRECONDITION_FAILED
+                    ]:
+                        raise mlrun.errors.MLRunFatalFailureError(
+                            original_exception=get_wf_exc
+                        )
+
+                    # raise for a retry (on other errors)
+                    raise
+
             # Getting workflow id from run:
             response = retry_until_successful(
                 1,
                 get_workflow_id_timeout,
                 logger,
                 False,
-                run_db.get_workflow_id,
-                project=project.name,
-                name=workflow_response.name,
-                run_id=workflow_response.run_id,
-                engine=workflow_spec.engine,
+                _get_workflow_id_or_bail,
             )
             workflow_id = response.workflow_id
             # After fetching the workflow_id the workflow executed successfully