mlrun 1.7.0rc15__py3-none-any.whl → 1.7.0rc17__py3-none-any.whl

mlrun/db/nopdb.py

@@ -16,6 +16,7 @@
 import datetime
 from typing import Optional, Union
 
+import mlrun.alerts
 import mlrun.common.schemas
 import mlrun.errors
 
@@ -128,7 +129,18 @@ class NopDB(RunDBInterface):
     ):
         pass
 
-    def del_artifact(self, key, tag="", project="", tree=None, uid=None):
+    def del_artifact(
+        self,
+        key,
+        tag="",
+        project="",
+        tree=None,
+        uid=None,
+        deletion_strategy: mlrun.common.schemas.artifact.ArtifactsDeletionStrategies = (
+            mlrun.common.schemas.artifact.ArtifactsDeletionStrategies.metadata_only
+        ),
+        secrets: dict = None,
+    ):
         pass
 
     def del_artifacts(self, name="", project="", tag="", labels=None):
@@ -508,8 +520,11 @@ class NopDB(RunDBInterface):
 
     def store_api_gateway(
         self,
-        project: str,
-        api_gateway: mlrun.runtimes.nuclio.APIGateway,
+        api_gateway: Union[
+            mlrun.common.schemas.APIGateway,
+            mlrun.runtimes.nuclio.api_gateway.APIGateway,
+        ],
+        project: str = None,
     ) -> mlrun.common.schemas.APIGateway:
         pass
 
@@ -671,7 +686,7 @@ class NopDB(RunDBInterface):
     def store_alert_config(
         self,
         alert_name: str,
-        alert_data: Union[dict, mlrun.common.schemas.AlertConfig],
+        alert_data: Union[dict, mlrun.alerts.alert.AlertConfig],
         project="",
     ):
         pass
@@ -687,3 +702,9 @@ class NopDB(RunDBInterface):
 
     def reset_alert_config(self, alert_name: str, project=""):
         pass
+
+    def get_alert_template(self, template_name: str):
+        pass
+
+    def list_alert_templates(self):
+        pass

mlrun/errors.py

@@ -183,6 +183,10 @@ class MLRunInternalServerError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.INTERNAL_SERVER_ERROR.value
 
 
+class MLRunNotImplementedServerError(MLRunHTTPStatusError):
+    error_status_code = HTTPStatus.NOT_IMPLEMENTED.value
+
+
 class MLRunServiceUnavailableError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.SERVICE_UNAVAILABLE.value
 
@@ -234,4 +238,5 @@ STATUS_ERRORS = {
     HTTPStatus.PRECONDITION_FAILED.value: MLRunPreconditionFailedError,
     HTTPStatus.INTERNAL_SERVER_ERROR.value: MLRunInternalServerError,
     HTTPStatus.SERVICE_UNAVAILABLE.value: MLRunServiceUnavailableError,
+    HTTPStatus.NOT_IMPLEMENTED.value: MLRunNotImplementedServerError,
 }

mlrun/launcher/base.py

@@ -18,10 +18,11 @@ import os
 import uuid
 from typing import Any, Callable, Optional, Union
 
+import mlrun_pipelines.common.ops
+
 import mlrun.common.schemas
 import mlrun.config
 import mlrun.errors
-import mlrun.kfpops
 import mlrun.lists
 import mlrun.model
 import mlrun.runtimes
@@ -390,7 +391,7 @@ class BaseLauncher(abc.ABC):
             return
 
         if result and runtime.kfp and err is None:
-            mlrun.kfpops.write_kfpmeta(result)
+            mlrun_pipelines.common.ops.write_kfpmeta(result)
 
         self._log_track_results(runtime.is_child, result, run)
 

mlrun/lists.py

@@ -29,12 +29,14 @@ list_header = [
     "iter",
     "start",
     "state",
+    "kind",
     "name",
     "labels",
     "inputs",
     "parameters",
     "results",
     "artifacts",
+    "artifact_uris",
     "error",
 ]
 
@@ -56,12 +58,14 @@ class RunList(list):
                 get_in(run, "metadata.iteration", ""),
                 get_in(run, "status.start_time", ""),
                 get_in(run, "status.state", ""),
+                get_in(run, "step_kind", get_in(run, "kind", "")),
                 get_in(run, "metadata.name", ""),
                 get_in(run, "metadata.labels", ""),
                 get_in(run, "spec.inputs", ""),
                 get_in(run, "spec.parameters", ""),
                 get_in(run, "status.results", ""),
                 get_in(run, "status.artifacts", []),
+                get_in(run, "status.artifact_uris", {}),
                 get_in(run, "status.error", ""),
             ]
             if extend_iterations and iterations:

mlrun/model.py

@@ -681,10 +681,14 @@ class Notification(ModelObj):
 
     def __init__(
         self,
-        kind=None,
+        kind: mlrun.common.schemas.notification.NotificationKind = (
+            mlrun.common.schemas.notification.NotificationKind.slack
+        ),
         name=None,
         message=None,
-        severity=None,
+        severity: mlrun.common.schemas.notification.NotificationSeverity = (
+            mlrun.common.schemas.notification.NotificationSeverity.INFO
+        ),
         when=None,
         condition=None,
         secret_params=None,
@@ -693,12 +697,10 @@ class Notification(ModelObj):
         sent_time=None,
         reason=None,
     ):
-        self.kind = kind or mlrun.common.schemas.notification.NotificationKind.slack
+        self.kind = kind
         self.name = name or ""
         self.message = message or ""
-        self.severity = (
-            severity or mlrun.common.schemas.notification.NotificationSeverity.INFO
-        )
+        self.severity = severity
         self.when = when or ["completed"]
         self.condition = condition or ""
         self.secret_params = secret_params or {}
@@ -1207,6 +1209,7 @@ class RunStatus(ModelObj):
         ui_url=None,
         reason: str = None,
         notifications: dict[str, Notification] = None,
+        artifact_uris: dict[str, str] = None,
     ):
         self.state = state or "created"
         self.status_text = status_text
@@ -1221,6 +1224,8 @@ class RunStatus(ModelObj):
         self.ui_url = ui_url
         self.reason = reason
         self.notifications = notifications or {}
+        # Artifact key -> URI mapping, since the full artifacts are not stored in the runs DB table
+        self.artifact_uris = artifact_uris or {}
 
     def is_failed(self) -> Optional[bool]:
         """
@@ -1539,8 +1544,10 @@ class RunObject(RunTemplate):
             iter=self.metadata.iteration,
         )
         if run:
-            self.status = RunStatus.from_dict(run.get("status", {}))
-            self.status.from_dict(run.get("status", {}))
+            run_status = run.get("status", {})
+            # Artifacts are not stored in the DB, so we need to preserve them here
+            run_status["artifacts"] = self.status.artifacts
+            self.status = RunStatus.from_dict(run_status)
             return self
 
     def show(self):

mlrun/model_monitoring/__init__.py

@@ -15,7 +15,7 @@
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 # for backwards compatibility
 
-from .db import get_store_object
+from .db import get_store_object, get_tsdb_connector
 from .helpers import get_stream_path
 from .model_endpoint import ModelEndpoint
 from .tracking_policy import TrackingPolicy

mlrun/model_monitoring/applications/_application_steps.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import copy
 import json
 import typing
 from typing import Optional
@@ -138,7 +137,7 @@ class _PrepareMonitoringEvent(StepToDict):
         if not event.get("mlrun_context"):
             application_context = MonitoringApplicationContext().from_dict(
                 event,
-                context=copy.deepcopy(self.context),
+                context=self.context,
                 model_endpoint_dict=self.model_endpoints,
             )
         else:

mlrun/model_monitoring/applications/context.py

@@ -113,7 +113,7 @@ class MonitoringApplicationContext(MLClientCtx):
             attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
         )
         self._sample_df_stats = json.loads(
-            attrs.get(mm_constants.ApplicationEvent.FEATURE_STATS, "{}")
+            attrs.get(mm_constants.ApplicationEvent.CURRENT_STATS, "{}")
         )
 
         self.endpoint_id = attrs.get(mm_constants.ApplicationEvent.ENDPOINT_ID)

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import typing
+
+import json
 from dataclasses import dataclass
-from typing import Final, Optional, Protocol, cast
+from typing import Final, Optional, Protocol, Union, cast
 
 import numpy as np
 from pandas import DataFrame, Series
@@ -90,9 +91,27 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
     """
     MLRun's default data drift application for model monitoring.
 
-    The application calculates the metrics over the features' histograms.
-    Each metric is calculated over all the features, the mean is taken,
-    and the status is returned.
+    The application expects tabular numerical data, and calculates three metrics over the features' histograms.
+    The three metrics are:
+
+    * Hellinger distance.
+    * Total variance distance.
+    * Kullback-Leibler divergence.
+
+    Each metric is calculated over all the features individually and the mean is taken as the metric value.
+    The average of Hellinger and total variance distance is taken as the result.
+
+    The application logs two artifacts:
+
+    * A JSON with the general drift per feature.
+    * A plotly table different metrics per feature.
+
+    This application is deployed by default when calling:
+
+    .. code-block:: python
+
+        project.enable_model_monitoring()
+
     """
 
     NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
@@ -107,8 +126,6 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
     def __init__(self, value_classifier: Optional[ValueClassifier] = None) -> None:
         """
-        Initialize the data drift application.
-
         :param value_classifier: Classifier object that adheres to the `ValueClassifier` protocol.
                                  If not provided, the default `DataDriftClassifier()` is used.
         """
@@ -146,35 +163,46 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
 
         return metrics_per_feature
 
-    def _add_general_drift_result(
+    def _get_general_drift_result(
         self,
         metrics: list[mm_results.ModelMonitoringApplicationMetric],
-    ) -> None:
-        """Add the general drift result to the results list and log it"""
-        value = np.mean(
-            [
-                metric.value
-                for metric in metrics
-                if metric.name
-                in [
-                    f"{HellingerDistance.NAME}_mean",
-                    f"{TotalVarianceDistance.NAME}_mean",
+        monitoring_context: mm_context.MonitoringApplicationContext,
+        metrics_per_feature: DataFrame,
+    ) -> mm_results.ModelMonitoringApplicationResult:
+        """Get the general drift result from the metrics list"""
+        value = cast(
+            float,
+            np.mean(
+                [
+                    metric.value
+                    for metric in metrics
+                    if metric.name
+                    in [
+                        f"{HellingerDistance.NAME}_mean",
+                        f"{TotalVarianceDistance.NAME}_mean",
+                    ]
                 ]
-            ]
+            ),
         )
 
         status = self._value_classifier.value_to_status(value)
-        metrics.append(
-            mm_results.ModelMonitoringApplicationResult(
-                name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
-                value=value,
-                kind=ResultKindApp.data_drift,
-                status=status,
-            )
+        return mm_results.ModelMonitoringApplicationResult(
+            name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
+            value=value,
+            kind=ResultKindApp.data_drift,
+            status=status,
+            extra_data={
+                EventFieldType.CURRENT_STATS: json.dumps(
+                    monitoring_context.feature_stats
+                ),
+                EventFieldType.DRIFT_MEASURES: metrics_per_feature.T.to_json(),
+                EventFieldType.DRIFT_STATUS: status.value,
+            },
         )
 
+    @staticmethod
     def _get_metrics(
-        self, metrics_per_feature: DataFrame
+        metrics_per_feature: DataFrame,
     ) -> list[mm_results.ModelMonitoringApplicationMetric]:
         """Average the metrics over the features and add the status"""
         metrics: list[mm_results.ModelMonitoringApplicationMetric] = []
@@ -206,8 +234,8 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             del sample_set_statistics[EventFieldType.TIMESTAMP]
         return sample_set_statistics
 
+    @staticmethod
     def _log_json_artifact(
-        self,
         drift_per_feature_values: Series,
         monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> None:
@@ -247,7 +275,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             mm_drift_table.FeaturesDriftTablePlot().produce(
                 sample_set_statistics=sample_set_statistics,
                 inputs_statistics=inputs_statistics,
-                metrics=metrics_per_feature.T.to_dict(),
+                metrics=metrics_per_feature.T.to_dict(),  # pyright: ignore[reportArgumentType]
                 drift_results=drift_results,
             )
         )
@@ -281,7 +309,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
         self,
         monitoring_context: mm_context.MonitoringApplicationContext,
     ) -> list[
-        typing.Union[
+        Union[
             mm_results.ModelMonitoringApplicationResult,
             mm_results.ModelMonitoringApplicationMetric,
         ]
@@ -308,15 +336,13 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
             metrics_per_feature=metrics_per_feature,
         )
         monitoring_context.logger.debug("Computing average per metric")
-        metrics_and_result: list[
-            typing.Union[
-                mm_results.ModelMonitoringApplicationMetric,
-                mm_results.ModelMonitoringApplicationResult,
-            ]
-        ] = self._get_metrics(metrics_per_feature)
-        self._add_general_drift_result(
-            metrics=metrics_and_result,
+        metrics = self._get_metrics(metrics_per_feature)
+        result = self._get_general_drift_result(
+            metrics=metrics,
+            monitoring_context=monitoring_context,
+            metrics_per_feature=metrics_per_feature,
         )
+        metrics_and_result = metrics + [result]
         monitoring_context.logger.debug(
             "Finished running the application", results=metrics_and_result
         )

mlrun/model_monitoring/db/__init__.py

@@ -14,3 +14,5 @@
 
 from .stores import ObjectStoreFactory, get_store_object
 from .stores.base import StoreBase
+from .tsdb import get_tsdb_connector
+from .tsdb.base import TSDBConnector

mlrun/model_monitoring/db/stores/base/store.py

@@ -15,6 +15,8 @@
 import typing
 from abc import ABC, abstractmethod
 
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+
 
 class StoreBase(ABC):
     """
@@ -62,12 +64,10 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def delete_model_endpoints_resources(self, endpoints: list[dict[str, typing.Any]]):
+    def delete_model_endpoints_resources(self):
         """
         Delete all model endpoints resources.
 
-        :param endpoints: A list of model endpoints flattened dictionaries.
-
         """
         pass
 
@@ -112,45 +112,18 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def get_endpoint_real_time_metrics(
+    def write_application_event(
         self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> dict[str, list[tuple[str, float]]]:
-        """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
-        """
-
-        pass
-
-    @abstractmethod
-    def write_application_result(self, event: dict[str, typing.Any]):
+        event: dict[str, typing.Any],
+        kind: mm_constants.WriterEventKind = mm_constants.WriterEventKind.RESULT,
+    ):
         """
-        Write a new application result event in the target table.
+        Write a new event in the target table.
 
         :param event: An event dictionary that represents the application result, should be corresponded to the
                       schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
                       object.
+        :param kind: The type of the event, can be either "result" or "metric".
         """
         pass