mlrun 1.10.0rc5__py3-none-any.whl → 1.10.0rc7__py3-none-any.whl

mlrun/db/nopdb.py

@@ -126,9 +126,6 @@ class NopDB(RunDBInterface):
         uid: Optional[Union[str, list[str]]] = None,
         project: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-        state: Optional[
-            mlrun.common.runtimes.constants.RunStates
-        ] = None,  # Backward compatibility
         states: Optional[list[mlrun.common.runtimes.constants.RunStates]] = None,
         sort: bool = True,
         iter: bool = False,
@@ -374,16 +371,6 @@ class NopDB(RunDBInterface):
     ) -> dict:
         pass
 
-    def list_features(
-        self,
-        project: str,
-        name: Optional[str] = None,
-        tag: Optional[str] = None,
-        entities: Optional[list[str]] = None,
-        labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
-    ) -> mlrun.common.schemas.FeaturesOutput:
-        pass
-
     def list_features_v2(
         self,
         project: str,
@@ -893,6 +880,18 @@ class NopDB(RunDBInterface):
     ) -> None:
         pass
 
+    def get_monitoring_function_summaries(
+        self,
+        project: str,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+        names: Optional[Union[list[str], str]] = None,
+        labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
+        include_stats: bool = False,
+        include_infra: bool = True,
+    ) -> [mlrun.common.schemas.model_monitoring.FunctionSummary]:
+        pass
+
     def generate_event(
         self, name: str, event_data: Union[dict, mlrun.common.schemas.Event], project=""
     ):

mlrun/launcher/base.py

@@ -82,6 +82,7 @@ class BaseLauncher(abc.ABC):
         runtime: "mlrun.runtimes.base.BaseRuntime",
         project_name: Optional[str] = "",
         full: bool = True,
+        client_version: str = "",
     ):
         pass
 

mlrun/launcher/client.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import abc
+import warnings
 from typing import Optional
 
 import IPython.display
@@ -23,6 +24,7 @@ import mlrun.lists
 import mlrun.model
 import mlrun.runtimes
 import mlrun.utils
+import mlrun.utils.version
 
 
 class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
@@ -35,6 +37,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         runtime: "mlrun.runtimes.base.BaseRuntime",
         project_name: Optional[str] = "",
         full: bool = True,
+        client_version: str = "",
     ):
         runtime.try_auto_mount_based_on_config()
         runtime._fill_credentials()
@@ -60,6 +63,27 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         ):
             image = mlrun.mlconf.function_defaults.image_by_kind.to_dict()[runtime.kind]
 
+        # Warn if user explicitly set the deprecated mlrun/ml-base image
+        if image and "mlrun/ml-base" in image:
+            client_version = mlrun.utils.version.Version().get()["version"]
+            auto_replaced = mlrun.utils.validate_component_version_compatibility(
+                "mlrun-client", "1.10.0", mlrun_client_version=client_version
+            )
+            message = (
+                "'mlrun/ml-base' image is deprecated in 1.10.0 and will be removed in 1.12.0, "
+                "use 'mlrun/mlrun' instead."
+            )
+            if auto_replaced:
+                message += (
+                    " Since your client version is >= 1.10.0, the image will be automatically "
+                    "replaced with mlrun/mlrun."
+                )
+            warnings.warn(
+                message,
+                # TODO: Remove this in 1.12.0
+                FutureWarning,
+            )
+
         # TODO: need a better way to decide whether a function requires a build
         if require_build and image and not runtime.spec.build.base_image:
             # when the function require build use the image as the base_image for the build

mlrun/launcher/local.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import os
 import pathlib
+from os import environ
 from typing import Callable, Optional, Union
 
 import mlrun.common.constants as mlrun_constants
@@ -251,6 +252,9 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
             # copy the code/base-spec to the local function (for the UI and code logging)
             fn.spec.description = runtime.spec.description
             fn.spec.build = runtime.spec.build
+            serving_spec = getattr(runtime.spec, "serving_spec", None)
+            if serving_spec:
+                environ["SERVING_SPEC_ENV"] = serving_spec
 
         run.spec.handler = handler
         run.spec.reset_on_run = reset_on_run

mlrun/model_monitoring/applications/_application_steps.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import collections
-import json
 import traceback
 from collections import OrderedDict
 from datetime import datetime
@@ -23,10 +22,12 @@ import mlrun.common.schemas
 import mlrun.common.schemas.alert as alert_objects
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.model_monitoring.helpers
+import mlrun.platforms.iguazio
 from mlrun.serving import GraphContext
 from mlrun.serving.utils import StepToDict
 from mlrun.utils import logger
 
+from .base import _serialize_context_and_result
 from .context import MonitoringApplicationContext
 from .results import (
     ModelMonitoringApplicationMetric,
@@ -45,7 +46,7 @@ class _PushToMonitoringWriter(StepToDict):
         :param project: Project name.
         """
         self.project = project
-        self.output_stream = None
+        self._output_stream = None
 
     def do(
         self,
@@ -65,48 +66,31 @@ class _PushToMonitoringWriter(StepToDict):
 
         :param event: Monitoring result(s) to push and the original event from the controller.
         """
-        self._lazy_init()
         application_results, application_context = event
-        writer_event = {
-            mm_constants.WriterEvent.ENDPOINT_NAME: application_context.endpoint_name,
-            mm_constants.WriterEvent.APPLICATION_NAME: application_context.application_name,
-            mm_constants.WriterEvent.ENDPOINT_ID: application_context.endpoint_id,
-            mm_constants.WriterEvent.START_INFER_TIME: application_context.start_infer_time.isoformat(
-                sep=" ", timespec="microseconds"
-            ),
-            mm_constants.WriterEvent.END_INFER_TIME: application_context.end_infer_time.isoformat(
-                sep=" ", timespec="microseconds"
-            ),
-        }
-        for result in application_results:
-            data = result.to_dict()
-            if isinstance(result, ModelMonitoringApplicationResult):
-                writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
-                    mm_constants.WriterEventKind.RESULT
-                )
-            elif isinstance(result, _ModelMonitoringApplicationStats):
-                writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
-                    mm_constants.WriterEventKind.STATS
-                )
-            else:
-                writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
-                    mm_constants.WriterEventKind.METRIC
-                )
-            writer_event[mm_constants.WriterEvent.DATA] = json.dumps(data)
-            logger.debug(
-                "Pushing data to output stream", writer_event=str(writer_event)
-            )
-            self.output_stream.push(
-                [writer_event], partition_key=application_context.endpoint_id
-            )
-            logger.debug("Pushed data to output stream successfully")
 
-    def _lazy_init(self):
-        if self.output_stream is None:
-            self.output_stream = mlrun.model_monitoring.helpers.get_output_stream(
+        writer_events = [
+            _serialize_context_and_result(context=application_context, result=result)
+            for result in application_results
+        ]
+
+        logger.debug("Pushing data to output stream", writer_events=str(writer_events))
+        self.output_stream.push(
+            writer_events, partition_key=application_context.endpoint_id
+        )
+        logger.debug("Pushed data to output stream successfully")
+
+    @property
+    def output_stream(
+        self,
+    ) -> Union[
+        mlrun.platforms.iguazio.OutputStream, mlrun.platforms.iguazio.KafkaOutputStream
+    ]:
+        if self._output_stream is None:
+            self._output_stream = mlrun.model_monitoring.helpers.get_output_stream(
                 project=self.project,
                 function_name=mm_constants.MonitoringFunctionNames.WRITER,
             )
+        return self._output_stream
 
 
 class _PrepareMonitoringEvent(StepToDict):

mlrun/model_monitoring/applications/base.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import socket
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from collections.abc import Iterator
+from contextlib import contextmanager
 from datetime import datetime, timedelta
 from typing import Any, Optional, Union, cast
 
@@ -23,14 +26,58 @@ import pandas as pd
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.datastore.datastore_profile as ds_profile
 import mlrun.errors
 import mlrun.model_monitoring.api as mm_api
 import mlrun.model_monitoring.applications.context as mm_context
 import mlrun.model_monitoring.applications.results as mm_results
+import mlrun.model_monitoring.helpers as mm_helpers
 from mlrun.serving.utils import MonitoringApplicationToDict
 from mlrun.utils import logger
 
 
+def _serialize_context_and_result(
+    *,
+    context: mm_context.MonitoringApplicationContext,
+    result: Union[
+        mm_results.ModelMonitoringApplicationResult,
+        mm_results.ModelMonitoringApplicationMetric,
+        mm_results._ModelMonitoringApplicationStats,
+    ],
+) -> dict[mm_constants.WriterEvent, str]:
+    """
+    Serialize the returned result from a model monitoring application and its context
+    for the writer.
+    """
+    writer_event = {
+        mm_constants.WriterEvent.ENDPOINT_NAME: context.endpoint_name,
+        mm_constants.WriterEvent.APPLICATION_NAME: context.application_name,
+        mm_constants.WriterEvent.ENDPOINT_ID: context.endpoint_id,
+        mm_constants.WriterEvent.START_INFER_TIME: context.start_infer_time.isoformat(
+            sep=" ", timespec="microseconds"
+        ),
+        mm_constants.WriterEvent.END_INFER_TIME: context.end_infer_time.isoformat(
+            sep=" ", timespec="microseconds"
+        ),
+    }
+
+    if isinstance(result, mm_results.ModelMonitoringApplicationResult):
+        writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
+            mm_constants.WriterEventKind.RESULT
+        )
+    elif isinstance(result, mm_results._ModelMonitoringApplicationStats):
+        writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
+            mm_constants.WriterEventKind.STATS
+        )
+    else:
+        writer_event[mm_constants.WriterEvent.EVENT_KIND] = (
+            mm_constants.WriterEventKind.METRIC
+        )
+    writer_event[mm_constants.WriterEvent.DATA] = json.dumps(result.to_dict())
+
+    return writer_event
+
+
 class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
     """
     The base class for a model monitoring application.
@@ -118,6 +165,43 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             ]
         return result
 
+    @staticmethod
+    @contextmanager
+    def _push_to_writer(
+        *,
+        write_output: bool,
+        stream_profile: Optional[ds_profile.DatastoreProfile],
+    ) -> Iterator[dict[str, list[tuple]]]:
+        endpoints_output: dict[str, list[tuple]] = defaultdict(list)
+        try:
+            yield endpoints_output
+        finally:
+            if write_output:
+                logger.debug(
+                    "Pushing model monitoring application job data to the writer stream",
+                    passed_stream_profile=str(stream_profile),
+                )
+                project_name = (
+                    mlrun.mlconf.active_project or mlrun.get_current_project().name
+                )
+                writer_stream = mm_helpers.get_output_stream(
+                    project=project_name,
+                    function_name=mm_constants.MonitoringFunctionNames.WRITER,
+                    profile=stream_profile,
+                )
+                for endpoint_id, outputs in endpoints_output.items():
+                    writer_stream.push(
+                        [
+                            _serialize_context_and_result(context=ctx, result=res)
+                            for ctx, res in outputs
+                        ],
+                        partition_key=endpoint_id,
+                    )
+                logger.debug(
+                    "Pushed the data to all the relevant model endpoints successfully",
+                    endpoints_output=endpoints_output,
+                )
+
     def _handler(
         self,
         context: "mlrun.MLClientCtx",
@@ -127,6 +211,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         start: Optional[str] = None,
         end: Optional[str] = None,
         base_period: Optional[int] = None,
+        write_output: bool = False,
+        stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ):
         """
         A custom handler that wraps the application's logic implemented in
@@ -134,46 +220,69 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         for an MLRun job.
         This method should not be called directly.
         """
+
+        if write_output and (
+            not endpoints or sample_data is not None or reference_data is not None
+        ):
+            raise mlrun.errors.MLRunValueError(
+                "Writing the results of an application to the TSDB is possible only when "
+                "working with endpoints, without any custom data-frame input"
+            )
+
         feature_stats = (
             mm_api.get_sample_set_statistics(reference_data)
             if reference_data is not None
             else None
         )
 
-        def call_do_tracking(event: Optional[dict] = None):
-            if event is None:
-                event = {}
-            monitoring_context = mm_context.MonitoringApplicationContext._from_ml_ctx(
-                event=event,
-                application_name=self.__class__.__name__,
-                context=context,
-                sample_df=sample_data,
-                feature_stats=feature_stats,
-            )
-            return self.do_tracking(monitoring_context)
-
-        if endpoints is not None:
-            for window_start, window_end in self._window_generator(
-                start, end, base_period
-            ):
-                for endpoint_name, endpoint_id in endpoints:
-                    result = call_do_tracking(
-                        event={
-                            mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
-                            mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
-                            mm_constants.ApplicationEvent.START_INFER_TIME: window_start,
-                            mm_constants.ApplicationEvent.END_INFER_TIME: window_end,
-                        }
-                    )
-                    result_key = (
-                        f"{endpoint_name}-{endpoint_id}_{window_start.isoformat()}_{window_end.isoformat()}"
-                        if window_start and window_end
-                        else f"{endpoint_name}-{endpoint_id}"
+        with self._push_to_writer(
+            write_output=write_output, stream_profile=stream_profile
+        ) as endpoints_output:
+
+            def call_do_tracking(event: Optional[dict] = None):
+                nonlocal endpoints_output
+
+                if event is None:
+                    event = {}
+                monitoring_context = (
+                    mm_context.MonitoringApplicationContext._from_ml_ctx(
+                        event=event,
+                        application_name=self.__class__.__name__,
+                        context=context,
+                        sample_df=sample_data,
+                        feature_stats=feature_stats,
                     )
+                )
+                result = self.do_tracking(monitoring_context)
+                endpoints_output[monitoring_context.endpoint_id].append(
+                    (monitoring_context, result)
+                )
+                return result
+
+            if endpoints is not None:
+                for window_start, window_end in self._window_generator(
+                    start, end, base_period
+                ):
+                    for endpoint_name, endpoint_id in endpoints:
+                        result = call_do_tracking(
+                            event={
+                                mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
+                                mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
+                                mm_constants.ApplicationEvent.START_INFER_TIME: window_start,
+                                mm_constants.ApplicationEvent.END_INFER_TIME: window_end,
+                            }
+                        )
+                        result_key = (
+                            f"{endpoint_name}-{endpoint_id}_{window_start.isoformat()}_{window_end.isoformat()}"
+                            if window_start and window_end
+                            else f"{endpoint_name}-{endpoint_id}"
+                        )
 
-                    context.log_result(result_key, self._flatten_data_result(result))
-        else:
-            return self._flatten_data_result(call_do_tracking())
+                        context.log_result(
+                            result_key, self._flatten_data_result(result)
+                        )
+            else:
+                return self._flatten_data_result(call_do_tracking())
 
     @staticmethod
     def _handle_endpoints_type_evaluate(
@@ -338,6 +447,7 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         * ``start``, ``datetime``
         * ``end``, ``datetime``
         * ``base_period``, ``int``
+        * ``write_output``, ``bool``
 
         For Git sources, add the source archive to the returned job and change the handler:
 
@@ -420,6 +530,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
         base_period: Optional[int] = None,
+        write_output: bool = False,
+        stream_profile: Optional[ds_profile.DatastoreProfile] = None,
     ) -> "mlrun.RunObject":
         """
         Call this function to run the application's
@@ -470,6 +582,14 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                                   ..., (\\operatorname{start} +
                                   m\\cdot\\operatorname{base\\_period}, \\operatorname{end}]`,
                                   where :math:`m` is some positive integer.
+        :param write_output:      Whether to write the results and metrics to the time-series DB. Can be ``True`` only
+                                  if ``endpoints`` are passed.
+                                  Note: the model monitoring infrastructure must be up for the writing to work.
+        :param stream_profile:    The stream datastore profile. It should be provided only when running locally and
+                                  writing the outputs to the database (i.e., when both ``run_local`` and
+                                  ``write_output`` are set to ``True``).
+                                  For more details on configuring the stream profile, see
+                                  :py:meth:`~mlrun.projects.MlrunProject.set_model_monitoring_credentials`.
 
         :returns: The output of the
                   :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
@@ -507,10 +627,25 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                 )
                 params["end"] = end.isoformat() if isinstance(end, datetime) else end
                 params["base_period"] = base_period
+                params["write_output"] = write_output
+                if stream_profile:
+                    if not run_local:
+                        raise mlrun.errors.MLRunValueError(
+                            "Passing a `stream_profile` is relevant only when running locally"
+                        )
+                    if not write_output:
+                        raise mlrun.errors.MLRunValueError(
+                            "Passing a `stream_profile` is relevant only when writing the outputs"
+                        )
+                params["stream_profile"] = stream_profile
         elif start or end or base_period:
             raise mlrun.errors.MLRunValueError(
                 "Custom `start` and `end` times or base_period are supported only with endpoints data"
             )
+        elif write_output or stream_profile:
+            raise mlrun.errors.MLRunValueError(
+                "Writing the application output or passing `stream_profile` are supported only with endpoints data"
+            )
 
         inputs: dict[str, str] = {}
         for data, identifier in [

mlrun/model_monitoring/db/tsdb/base.py

@@ -328,6 +328,36 @@ class TSDBConnector(ABC):
         If an endpoint has not been invoked within the specified time range, it will not appear in the result.
         """
 
+    @abstractmethod
+    def count_results_by_status(
+        self,
+        start: Optional[Union[datetime, str]] = None,
+        end: Optional[Union[datetime, str]] = None,
+        endpoint_ids: Optional[Union[str, list[str]]] = None,
+        application_names: Optional[Union[str, list[str]]] = None,
+        result_status_list: Optional[list[int]] = None,
+    ) -> dict[tuple[str, int], int]:
+        """
+        Read results status from the TSDB and return a dictionary of results statuses by application name.
+
+        :param start:              The start time in which to read the results. By default, the last 24 hours are read.
+        :param end:                The end time in which to read the results. Default is the current time (now).
+        :param endpoint_ids:       Optional list of endpoint ids to filter the results by. By default, all
+                                   endpoint ids are included.
+        :param application_names:  Optional list of application names to filter the results by. By default, all
+                                   application are included.
+        :param result_status_list: Optional list of result statuses to filter the results by. By default, all
+                                   result statuses are included.
+
+        :return: A dictionary where the key is a tuple of (application_name, result_status) and the value is the total
+                 number of results with that status for that application.
+                 For example:
+                 {
+                    ('app1', 1): 10,
+                    ('app1', 2): 5
+                 }
+        """
+
     async def add_basic_metrics(
         self,
         model_endpoint_objects: list[mlrun.common.schemas.ModelEndpoint],