mlrun 1.9.0rc3__py3-none-any.whl → 1.9.0rc5__py3-none-any.whl

mlrun/db/httpdb.py

@@ -21,7 +21,7 @@ import typing
 import warnings
 from copy import deepcopy
 from datetime import datetime, timedelta
-from os import path, remove
+from os import environ, path, remove
 from typing import Literal, Optional, Union
 from urllib.parse import urlparse
 
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
         self._wait_for_background_task_terminal_state_retry_interval = 3
         self._wait_for_project_deletion_interval = 3
         self.client_version = version.Version().get()["version"]
-        self.python_version = str(version.Version().get_python_version())
+        self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
+            version.Version().get_python_version()
+        )
 
         self._enrich_and_validate(url)
 
@@ -945,7 +947,7 @@ class HTTPRunDB(RunDBInterface):
             or just `"label"` for key existence.
             - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
             the specified key-value pairs or key existence.
-        :param state: Deprecated - List only runs whose state is specified (will be removed in 1.9.0)
+        :param state: Deprecated - List only runs whose state is specified (will be removed in 1.10.0)
         :param states: List only runs whose state is one of the provided states.
         :param sort: Whether to sort the result according to their start time. Otherwise, results will be
             returned by their internal order in the DB (order will not be guaranteed).
@@ -1276,8 +1278,8 @@ class HTTPRunDB(RunDBInterface):
         :param producer_uri:    Return artifacts produced by the requested producer URI. Producer URI usually
             points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
             is a workflow id (artifact was created as part of a workflow).
-        :param format_:         The format in which to return the artifacts. Default is 'full'.
-        :param limit:           Maximum number of artifacts to return.
+        :param format_: The format in which to return the artifacts. Default is 'full'.
+        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
             parameter must be provided as well.
         :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -2221,18 +2223,20 @@ class HTTPRunDB(RunDBInterface):
         elif pipe_file.endswith(".zip"):
             headers = {"content-type": "application/zip"}
         else:
-            raise ValueError("pipeline file must be .yaml or .zip")
+            raise ValueError("'pipeline' file must be .yaml or .zip")
         if arguments:
             if not isinstance(arguments, dict):
-                raise ValueError("arguments must be dict type")
+                raise ValueError("'arguments' must be dict type")
             headers[mlrun.common.schemas.HeaderNames.pipeline_arguments] = str(
                 arguments
             )
 
         if not path.isfile(pipe_file):
-            raise OSError(f"file {pipe_file} doesnt exist")
+            raise OSError(f"File {pipe_file} doesnt exist")
         with open(pipe_file, "rb") as fp:
             data = fp.read()
+            if not data:
+                raise ValueError("The compiled pipe file is empty")
         if not isinstance(pipeline, str):
             remove(pipe_file)
 
@@ -3584,7 +3588,7 @@ class HTTPRunDB(RunDBInterface):
         params = {
             "type": type,
             "endpoint-id": endpoint_ids,
-            "events_format": events_format,
+            "events-format": events_format,
         }
         error_message = (
             f"Failed to get model monitoring metrics,"
@@ -3720,7 +3724,7 @@ class HTTPRunDB(RunDBInterface):
             path=path,
             body=model_endpoint.json(),
             params={
-                "creation_strategy": creation_strategy,
+                "creation-strategy": creation_strategy,
             },
         )
         return mlrun.common.schemas.ModelEndpoint(**response.json())
@@ -3750,9 +3754,9 @@ class HTTPRunDB(RunDBInterface):
             method=mlrun.common.types.HTTPMethod.DELETE,
             path=path,
             params={
-                "function_name": function_name,
-                "function_tag": function_tag,
-                "endpoint_id": endpoint_id,
+                "function-name": function_name,
+                "function-tag": function_tag,
+                "endpoint-id": endpoint_id,
             },
         )
 
@@ -3767,7 +3771,8 @@ class HTTPRunDB(RunDBInterface):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
+        metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
@@ -3785,6 +3790,9 @@ class HTTPRunDB(RunDBInterface):
         :param start:           The start time to filter by.Corresponding to the `created` field.
         :param end:             The end time to filter by. Corresponding to the `created` field.
         :param tsdb_metrics:    Whether to include metrics from the time series DB.
+        :param metric_list:     List of metrics to include from the time series DB. Defaults to all metrics.
+                                If tsdb_metrics=False, this parameter will be ignored and no tsdb metrics
+                                will be included.
         :param top_level:       Whether to return only top level model endpoints.
         :param uids:            A list of unique ids to filter by.
         :param latest_only:     Whether to return only the latest model endpoint version.
@@ -3799,17 +3807,18 @@ class HTTPRunDB(RunDBInterface):
             path=path,
             params={
                 "name": names,
-                "model_name": model_name,
-                "model_tag": model_tag,
-                "function_name": function_name,
-                "function_tag": function_tag,
+                "model-name": model_name,
+                "model-tag": model_tag,
+                "function-name": function_name,
+                "function-tag": function_tag,
                 "label": labels,
                 "start": datetime_to_iso(start),
                 "end": datetime_to_iso(end),
-                "tsdb_metrics": tsdb_metrics,
+                "tsdb-metrics": tsdb_metrics,
+                "metric": metric_list,
                 "top-level": top_level,
                 "uid": uids,
-                "latest_only": latest_only,
+                "latest-only": latest_only,
             },
         )
 
@@ -3823,6 +3832,7 @@ class HTTPRunDB(RunDBInterface):
         function_tag: Optional[str] = None,
         endpoint_id: Optional[str] = None,
         tsdb_metrics: bool = True,
+        metric_list: Optional[list[str]] = None,
         feature_analysis: bool = False,
     ) -> mlrun.common.schemas.ModelEndpoint:
         """
@@ -3834,6 +3844,9 @@ class HTTPRunDB(RunDBInterface):
         :param function_tag:               The tag of the function
         :param endpoint_id:                The id of the endpoint
         :param tsdb_metrics:               Whether to include metrics from the time series DB.
+        :param metric_list:                List of metrics to include from the time series DB. Defaults to all metrics.
+                                           If tsdb_metrics=False, this parameter will be ignored and no tsdb metrics
+                                           will be included.
         :param feature_analysis:           Whether to include feature analysis data (feature_stats,
                                             current_stats & drift_measures).
 
@@ -3847,11 +3860,12 @@ class HTTPRunDB(RunDBInterface):
             method=mlrun.common.types.HTTPMethod.GET,
             path=path,
             params={
-                "function_name": function_name,
-                "function_tag": function_tag,
-                "endpoint_id": endpoint_id,
-                "tsdb_metrics": tsdb_metrics,
-                "feature_analysis": feature_analysis,
+                "function-name": function_name,
+                "function-tag": function_tag,
+                "endpoint-id": endpoint_id,
+                "tsdb-metrics": tsdb_metrics,
+                "metric": metric_list,
+                "feature-analysis": feature_analysis,
             },
         )
 
@@ -5089,6 +5103,13 @@ class HTTPRunDB(RunDBInterface):
         project = project or config.default_project
         labels = self._parse_labels(labels)
 
+        if limit:
+            # TODO: Remove this in 1.11.0
+            warnings.warn(
+                "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
+                FutureWarning,
+            )
+
         params = {
             "name": name,
             "tag": tag,
@@ -5225,9 +5246,9 @@ class HTTPRunDB(RunDBInterface):
             )
 
         if state:
-            # TODO: Remove this in 1.9.0
+            # TODO: Remove this in 1.10.0
             warnings.warn(
-                "'state' is deprecated and will be removed in 1.9.0. Use 'states' instead.",
+                "'state' is deprecated in 1.7.0 and will be removed in 1.10.0. Use 'states' instead.",
                 FutureWarning,
             )
 

mlrun/db/nopdb.py

@@ -631,7 +631,8 @@ class NopDB(RunDBInterface):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
+        metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
@@ -646,6 +647,7 @@ class NopDB(RunDBInterface):
         function_tag: Optional[str] = None,
         endpoint_id: Optional[str] = None,
         tsdb_metrics: bool = True,
+        metric_list: Optional[list[str]] = None,
         feature_analysis: bool = False,
     ) -> mlrun.common.schemas.ModelEndpoint:
         pass

mlrun/execution.py

@@ -976,7 +976,7 @@ class MLClientCtx:
     def get_cached_artifact(self, key):
         """Return a logged artifact from cache (for potential updates)"""
         warnings.warn(
-            "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.10.0. Use get_artifact instead.",
+            "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
             FutureWarning,
         )
         return self.get_artifact(key)

mlrun/frameworks/_common/model_handler.py

@@ -690,10 +690,10 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
         }
         self._registered_artifacts = {}
 
-        # Get the model artifact. If the model was logged during this run, use the cached artifact, otherwise use the
+        # Get the model artifact. If the model was logged during this run, use the artifact, otherwise use the
         # user's given model path:
         model_artifact = (
-            self._context.get_cached_artifact(self._model_name)
+            self._context.get_artifact(self._model_name)
             if self._is_logged
             else self._model_path
         )

mlrun/launcher/client.py

@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     ):
         run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if run.spec.output_path:
             run.spec.output_path = run.spec.output_path.replace(

mlrun/model_monitoring/api.py

@@ -181,7 +181,7 @@ def record_results(
     if drift_threshold is not None or possible_drift_threshold is not None:
         warnings.warn(
             "Custom drift threshold arguments are deprecated since version "
-            "1.7.0 and have no effect. They will be removed in version 1.9.0.\n"
+            "1.7.0 and have no effect. They will be removed in version 1.10.0.\n"
             "To enable the default histogram data drift application, run:\n"
             "`project.enable_model_monitoring()`.",
             FutureWarning,
@@ -189,7 +189,7 @@ def record_results(
     if trigger_monitoring_job is not False:
         warnings.warn(
             "`trigger_monitoring_job` argument is deprecated since version "
-            "1.7.0 and has no effect. It will be removed in version 1.9.0.\n"
+            "1.7.0 and has no effect. It will be removed in version 1.10.0.\n"
             "To enable the default histogram data drift application, run:\n"
             "`project.enable_model_monitoring()`.",
             FutureWarning,
@@ -197,13 +197,13 @@ def record_results(
     if artifacts_tag != "":
         warnings.warn(
             "`artifacts_tag` argument is deprecated since version "
-            "1.7.0 and has no effect. It will be removed in version 1.9.0.",
+            "1.7.0 and has no effect. It will be removed in version 1.10.0.",
             FutureWarning,
         )
     if default_batch_image != "mlrun/mlrun":
         warnings.warn(
             "`default_batch_image` argument is deprecated since version "
-            "1.7.0 and has no effect. It will be removed in version 1.9.0.",
+            "1.7.0 and has no effect. It will be removed in version 1.10.0.",
             FutureWarning,
         )
 

mlrun/model_monitoring/applications/_application_steps.py

@@ -96,7 +96,9 @@ class _PushToMonitoringWriter(StepToDict):
             logger.debug(
                 "Pushing data to output stream", writer_event=str(writer_event)
             )
-            self.output_stream.push([writer_event])
+            self.output_stream.push(
+                [writer_event], partition_key=application_context.endpoint_id
+            )
             logger.debug("Pushed data to output stream successfully")
 
     def _lazy_init(self):

mlrun/model_monitoring/applications/evidently/base.py

@@ -14,19 +14,18 @@
 
 import json
 import posixpath
-import uuid
 import warnings
 from abc import ABC
+from tempfile import NamedTemporaryFile
+from typing import Optional
 
-import pandas as pd
 import semver
-from evidently.ui.storage.local.base import METADATA_PATH, FSLocation
 
 import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
-from mlrun.errors import MLRunIncompatibleVersionError
+from mlrun.errors import MLRunIncompatibleVersionError, MLRunValueError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.6.0")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.7.5")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -60,36 +59,66 @@ except ModuleNotFoundError:
 
 
 if _HAS_EVIDENTLY:
-    from evidently.suite.base_suite import Display
-    from evidently.ui.type_aliases import STR_UUID
-    from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams, file_html_template
+    from evidently.core.report import Snapshot
+    from evidently.legacy.ui.storage.local.base import METADATA_PATH, FSLocation
+    from evidently.ui.workspace import (
+        STR_UUID,
+        CloudWorkspace,
+        Project,
+        Workspace,
+        WorkspaceBase,
+    )
 
 
 class EvidentlyModelMonitoringApplicationBase(
     mm_base.ModelMonitoringApplicationBase, ABC
 ):
     def __init__(
-        self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
+        self,
+        evidently_project_id: "STR_UUID",
+        evidently_workspace_path: Optional[str] = None,
+        cloud_workspace: bool = False,
     ) -> None:
         """
-        A class for integrating Evidently for mlrun model monitoring within a monitoring application.
-        Note: evidently is not installed by default in the mlrun/mlrun image.
-        It must be installed separately to use this class.
+        A class for integrating Evidently for MLRun model monitoring within a monitoring application.
+
+        .. note::
+
+            The ``evidently`` package is not installed by default in the mlrun/mlrun image.
+            It must be installed separately to use this class.
 
-        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
         :param evidently_project_id:        (str) The ID of the Evidently project.
+        :param evidently_workspace_path:    (str) The path to the Evidently workspace.
+        :param cloud_workspace:             (bool) Whether the workspace is an Evidently Cloud workspace.
         """
-
-        # TODO : more then one project (mep -> project)
         if not _HAS_EVIDENTLY:
             raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
-        self._log_location(evidently_workspace_path)
-        self.evidently_workspace = Workspace.create(evidently_workspace_path)
+        self.evidently_workspace_path = evidently_workspace_path
+        if cloud_workspace:
+            self.get_workspace = self.get_cloud_workspace
+        self.evidently_workspace = self.get_workspace()
         self.evidently_project_id = evidently_project_id
-        self.evidently_project = self.evidently_workspace.get_project(
-            evidently_project_id
-        )
+        self.evidently_project = self.load_project()
+
+    def load_project(self) -> Project:
+        """Load the Evidently project."""
+        return self.evidently_workspace.get_project(self.evidently_project_id)
+
+    def get_workspace(self) -> WorkspaceBase:
+        """Get the Evidently workspace. Override this method for customize access to the workspace."""
+        if self.evidently_workspace_path:
+            self._log_location(self.evidently_workspace_path)
+            return Workspace.create(self.evidently_workspace_path)
+        else:
+            raise MLRunValueError(
+                "A local workspace could not be created as `evidently_workspace_path` is not set.\n"
+                "If you intend to use a cloud workspace, please use `cloud_workspace=True` and set the "
+                "`EVIDENTLY_API_KEY` environment variable. In other cases, override this method."
+            )
+
+    def get_cloud_workspace(self) -> CloudWorkspace:
+        """Load the Evidently cloud workspace according to the `EVIDENTLY_API_KEY` environment variable."""
+        return CloudWorkspace()
 
     @staticmethod
     def _log_location(evidently_workspace_path):
@@ -128,7 +157,7 @@ class EvidentlyModelMonitoringApplicationBase(
     @staticmethod
     def log_evidently_object(
         monitoring_context: mm_context.MonitoringApplicationContext,
-        evidently_object: "Display",
+        evidently_object: "Snapshot",
         artifact_name: str,
         unique_per_endpoint: bool = True,
     ) -> None:
@@ -141,56 +170,15 @@ class EvidentlyModelMonitoringApplicationBase(
             This method should be called on special occasions only.
 
         :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
-        :param evidently_object:    (Display) The Evidently display to log, e.g. a report or a test suite object.
-        :param artifact_name:       (str) The name for the logged artifact.
-        :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
-                                    set to ``False`` without changing item key will cause artifact override.
-        """
-        evidently_object_html = evidently_object.get_html()
-        monitoring_context.log_artifact(
-            artifact_name,
-            body=evidently_object_html.encode("utf-8"),
-            format="html",
-            unique_per_endpoint=unique_per_endpoint,
-        )
-
-    def log_project_dashboard(
-        self,
-        monitoring_context: mm_context.MonitoringApplicationContext,
-        timestamp_start: pd.Timestamp,
-        timestamp_end: pd.Timestamp,
-        artifact_name: str = "dashboard",
-        unique_per_endpoint: bool = True,
-    ) -> None:
-        """
-        Logs an Evidently project dashboard.
-
-        .. caution::
-
-            Logging Evidently dashboards in every model monitoring window may cause scale issues.
-            This method should be called on special occasions only.
-
-        :param monitoring_context:  (MonitoringApplicationContext) The monitoring context to process.
-        :param timestamp_start:     (pd.Timestamp) The start timestamp for the dashboard data.
-        :param timestamp_end:       (pd.Timestamp) The end timestamp for the dashboard data.
+        :param evidently_object:    (Snapshot) The Evidently run to log, e.g. a report run.
         :param artifact_name:       (str) The name for the logged artifact.
         :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
                                     set to ``False`` without changing item key will cause artifact override.
         """
-
-        dashboard_info = self.evidently_project.build_dashboard_info(
-            timestamp_start, timestamp_end
-        )
-        template_params = TemplateParams(
-            dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
-            dashboard_info=dashboard_info,
-            additional_graphs={},
-        )
-
-        dashboard_html = file_html_template(params=template_params)
-        monitoring_context.log_artifact(
-            artifact_name,
-            body=dashboard_html.encode("utf-8"),
-            format="html",
-            unique_per_endpoint=unique_per_endpoint,
-        )
+        with NamedTemporaryFile(suffix=".html") as file:
+            evidently_object.save_html(filename=file.name)
+            monitoring_context.log_artifact(
+                artifact_name,
+                local_path=file.name,
+                unique_per_endpoint=unique_per_endpoint,
+            )

mlrun/model_monitoring/controller.py

@@ -25,6 +25,7 @@ from types import TracebackType
 from typing import Any, NamedTuple, Optional, Union, cast
 
 import nuclio_sdk
+import pandas as pd
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
@@ -673,7 +674,15 @@ class MonitoringApplicationController:
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
-        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=True).endpoints
+        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=False).endpoints
+        last_request_dict = self.tsdb_connector.get_last_request(
+            endpoint_ids=[mep.metadata.uid for mep in endpoints]
+        )
+        if isinstance(last_request_dict, pd.DataFrame):
+            last_request_dict = last_request_dict.set_index(
+                mm_constants.EventFieldType.ENDPOINT_ID
+            )[mm_constants.ModelEndpointSchema.LAST_REQUEST].to_dict()
+
         if not endpoints:
             logger.info("No model endpoints found", project=self.project)
             return
@@ -719,16 +728,22 @@ class MonitoringApplicationController:
             with schedules.ModelMonitoringSchedulesFileChief(
                 self.project
             ) as schedule_file:
-                futures = {
-                    pool.submit(
-                        self.endpoint_to_regular_event,
-                        endpoint,
-                        policy,
-                        set(applications_names),
-                        schedule_file,
-                    ): endpoint
-                    for endpoint in endpoints
-                }
+                for endpoint in endpoints:
+                    last_request = last_request_dict.get(endpoint.metadata.uid, None)
+                    if isinstance(last_request, float):
+                        last_request = pd.to_datetime(last_request, unit="s", utc=True)
+                    endpoint.status.last_request = (
+                        last_request or endpoint.status.last_request
+                    )
+                    futures = {
+                        pool.submit(
+                            self.endpoint_to_regular_event,
+                            endpoint,
+                            policy,
+                            set(applications_names),
+                            schedule_file,
+                        ): endpoint
+                    }
                 for future in concurrent.futures.as_completed(futures):
                     if future.exception():
                         exception = future.exception()

mlrun/model_monitoring/db/tsdb/base.py

@@ -82,7 +82,8 @@ class TSDBConnector(ABC):
 
     @abstractmethod
     def delete_tsdb_records(
-        self, endpoint_ids: list[str], delete_timeout: Optional[int] = None
+        self,
+        endpoint_ids: list[str],
     ) -> None:
         """
         Delete model endpoint records from the TSDB connector.
@@ -332,6 +333,7 @@ class TSDBConnector(ABC):
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],
         project: str,
         run_in_threadpool: Callable,
+        metric_list: Optional[list[str]] = None,
     ) -> list[mlrun.common.schemas.ModelEndpoint]:
         raise NotImplementedError()