mlrun 1.8.0rc28__py3-none-any.whl → 1.8.0rc30__py3-none-any.whl

mlrun/model_monitoring/helpers.py

@@ -16,7 +16,7 @@ import datetime
 import functools
 import os
 from fnmatch import fnmatchcase
-from typing import TYPE_CHECKING, Callable, Optional, TypedDict, cast
+from typing import TYPE_CHECKING, Callable, Optional, TypedDict, Union, cast
 
 import numpy as np
 import pandas as pd
@@ -28,6 +28,7 @@ import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
 import mlrun.datastore.datastore_profile
 import mlrun.model_monitoring
+import mlrun.platforms.iguazio
 import mlrun.utils.helpers
 from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
@@ -294,6 +295,112 @@ _get_stream_profile = functools.partial(
 )
 
 
+def _get_v3io_output_stream(
+    *,
+    v3io_profile: mlrun.datastore.datastore_profile.DatastoreProfileV3io,
+    project: str,
+    function_name: str,
+    v3io_access_key: Optional[str],
+    mock: bool = False,
+) -> mlrun.platforms.iguazio.OutputStream:
+    stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
+        project=project,
+        kind=mm_constants.FileTargetKind.STREAM,
+        target="online",
+        function_name=function_name,
+    )
+    endpoint, stream_path = mlrun.platforms.iguazio.parse_path(stream_uri)
+    return mlrun.platforms.iguazio.OutputStream(
+        stream_path,
+        endpoint=endpoint,
+        access_key=v3io_access_key or v3io_profile.v3io_access_key,
+        mock=mock,
+    )
+
+
+def _get_kafka_output_stream(
+    *,
+    kafka_profile: mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource,
+    project: str,
+    function_name: str,
+    mock: bool = False,
+) -> mlrun.platforms.iguazio.KafkaOutputStream:
+    topic = mlrun.common.model_monitoring.helpers.get_kafka_topic(
+        project=project, function_name=function_name
+    )
+    profile_attributes = kafka_profile.attributes()
+    producer_options = profile_attributes.get("producer_options", {})
+    if "sasl" in profile_attributes:
+        sasl = profile_attributes["sasl"]
+        producer_options.update(
+            {
+                "security_protocol": "SASL_PLAINTEXT",
+                "sasl_mechanism": sasl["mechanism"],
+                "sasl_plain_username": sasl["user"],
+                "sasl_plain_password": sasl["password"],
+            },
+        )
+    return mlrun.platforms.iguazio.KafkaOutputStream(
+        brokers=kafka_profile.brokers,
+        topic=topic,
+        producer_options=producer_options,
+        mock=mock,
+    )
+
+
+def get_output_stream(
+    project: str,
+    function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
+    secret_provider: Optional[Callable[[str], str]] = None,
+    profile: Optional[mlrun.datastore.datastore_profile.DatastoreProfile] = None,
+    v3io_access_key: Optional[str] = None,
+    mock: bool = False,
+) -> Union[
+    mlrun.platforms.iguazio.OutputStream, mlrun.platforms.iguazio.KafkaOutputStream
+]:
+    """
+    Get stream path from the project secret. If wasn't set, take it from the system configurations
+
+    :param project:             Project name.
+    :param function_name:       Application name. Default is model_monitoring_stream.
+    :param secret_provider:     Optional secret provider to get the connection string secret.
+                                If not set, the env vars are used.
+    :param profile:             Optional datastore profile of the stream (V3IO/KafkaSource profile).
+    :param v3io_access_key:     Optional V3IO access key.
+    :param mock:                Should the output stream be mocked or not.
+    :return:                    Monitoring stream path to the relevant application.
+    """
+
+    profile = profile or _get_stream_profile(
+        project=project, secret_provider=secret_provider
+    )
+
+    if isinstance(profile, mlrun.datastore.datastore_profile.DatastoreProfileV3io):
+        return _get_v3io_output_stream(
+            v3io_profile=profile,
+            project=project,
+            function_name=function_name,
+            v3io_access_key=v3io_access_key,
+            mock=mock,
+        )
+
+    elif isinstance(
+        profile, mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource
+    ):
+        return _get_kafka_output_stream(
+            kafka_profile=profile,
+            project=project,
+            function_name=function_name,
+            mock=mock,
+        )
+
+    else:
+        raise mlrun.errors.MLRunValueError(
+            f"Received an unexpected stream profile type: {type(profile)}\n"
+            "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaSource`."
+        )
+
+
 def batch_dict2timedelta(batch_dict: _BatchDict) -> datetime.timedelta:
     """
     Convert a batch dictionary to timedelta.

mlrun/model_monitoring/stream_processing.py

@@ -20,11 +20,8 @@ import storey
 
 import mlrun
 import mlrun.common.model_monitoring.helpers
-import mlrun.config
-import mlrun.datastore.targets
 import mlrun.feature_store as fstore
 import mlrun.feature_store.steps
-import mlrun.model_monitoring.db
 import mlrun.serving.states
 import mlrun.utils
 from mlrun.common.schemas.model_monitoring.constants import (
@@ -392,10 +389,6 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
         if not is_not_none(model, [EventFieldType.MODEL]):
             return None
 
-        version = full_event.body.get(EventFieldType.VERSION)
-        versioned_model = f"{model}:{version}" if version else f"{model}:latest"
-
-        full_event.body[EventFieldType.VERSIONED_MODEL] = versioned_model
         endpoint_id = event[EventFieldType.ENDPOINT_ID]
 
         # In case this process fails, resume state from existing record
@@ -493,7 +486,6 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
             events.append(
                 {
                     EventFieldType.FUNCTION_URI: function_uri,
-                    EventFieldType.MODEL: versioned_model,
                     EventFieldType.ENDPOINT_NAME: event.get(EventFieldType.MODEL),
                     EventFieldType.MODEL_CLASS: model_class,
                     EventFieldType.TIMESTAMP: timestamp,

mlrun/projects/project.py

@@ -1254,8 +1254,14 @@ class MlrunProject(ModelObj):
         mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
 
         self.spec.load_source_on_run = pull_at_runtime
+
+        source_has_changed = source != self.spec.source
         self.spec.source = source or self.spec.source
 
+        # new source should not relay on old workdir
+        if source_has_changed:
+            self.spec.workdir = workdir
+
         if self.spec.source.startswith("git://"):
             source, reference, branch = resolve_git_reference_from_source(source)
             if not branch and not reference:
@@ -1264,7 +1270,6 @@ class MlrunProject(ModelObj):
                     "'git://<url>/org/repo.git#<branch-name or refs/heads/..>'"
                 )
 
-        self.spec.workdir = workdir or self.spec.workdir
         try:
             # reset function objects (to recalculate build attributes)
             self.sync_functions()
@@ -2217,6 +2222,12 @@ class MlrunProject(ModelObj):
                     reset_policy=reset_policy,
                 )
             )
+        if not alerts:
+            warnings.warn(
+                "No alert config has been created. "
+                "Try specifying a result name explicitly or verifying that results are available"
+            )
+
         return alerts
 
     def set_model_monitoring_function(
@@ -3456,12 +3467,13 @@ class MlrunProject(ModelObj):
 
         arguments = arguments or {}
         need_repo = self.spec._need_repo()
-        if self.spec.repo and self.spec.repo.is_dirty():
-            msg = "You seem to have uncommitted git changes, use .push()"
-            if dirty or not need_repo:
-                logger.warning("WARNING!, " + msg)
-            else:
-                raise ProjectError(msg + " or dirty=True")
+        if not dirty:
+            if self.spec.repo and self.spec.repo.is_dirty():
+                msg = "You seem to have uncommitted git changes, use .push()"
+                if not need_repo:
+                    logger.warning("WARNING!, " + msg)
+                else:
+                    raise ProjectError(msg + " or dirty=True")
 
         if need_repo and self.spec.repo and not self.spec.source:
             raise ProjectError(
@@ -4120,7 +4132,7 @@ class MlrunProject(ModelObj):
 
         :param image: target image name/path. If not specified the project's existing `default_image` name will be
                         used. If not set, the `mlconf.default_project_image_name` value will be used
-        :param set_as_default: set `image` to be the project's default image (default False)
+        :param set_as_default: set `image` to be the project's default image (default True)
         :param with_mlrun:      add the current mlrun package to the container build
         :param base_image:      base image name/path (commands and source code will be added to it) defaults to
                                 mlrun.mlconf.default_base_image

mlrun/run.py

@@ -911,7 +911,7 @@ def _run_pipeline(
 
 def retry_pipeline(
     run_id: str,
-    project: Optional[str] = None,
+    project: str,
     namespace: Optional[str] = None,
 ) -> str:
     """Retry a pipeline run.
@@ -920,7 +920,7 @@ def retry_pipeline(
     retryable state, a new run is created as a clone of the original run.
 
     :param run_id: ID of the pipeline run to retry.
-    :param project: Optional; name of the project associated with the pipeline run.
+    :param project: name of the project associated with the pipeline run.
     :param namespace: Optional; Kubernetes namespace to use if not the default.
 
     :returns: ID of the retried pipeline run or the ID of a cloned run if the original run is not retryable.

mlrun/runtimes/nuclio/function.py

@@ -369,8 +369,9 @@ class RemoteRuntime(KubeResource):
                 )
         """
         self.spec.build.source = source
-        # update handler in function_handler
-        self.spec.function_handler = handler
+        # update handler in function_handler if needed
+        if handler:
+            self.spec.function_handler = handler
         if workdir:
             self.spec.workdir = workdir
         if runtime:
@@ -600,13 +601,15 @@ class RemoteRuntime(KubeResource):
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
         # check if there are any background tasks related to creating model endpoints
-        background_tasks = mlrun.common.schemas.BackgroundTaskList(
-            **data.pop("background_tasks", {"background_tasks": []})
-        ).background_tasks
-        if background_tasks:
+        model_endpoints_creation_background_tasks = (
+            mlrun.common.schemas.BackgroundTaskList(
+                **data.pop("background_tasks", {"background_tasks": []})
+            ).background_tasks
+        )
+        if model_endpoints_creation_background_tasks:
             self._check_model_endpoint_task_state(
                 db=db,
-                background_task=background_tasks[0],
+                background_task=model_endpoints_creation_background_tasks[0],
                 wait_for_completion=False,
             )
 
@@ -1014,6 +1017,7 @@ class RemoteRuntime(KubeResource):
     ):
         """
         Add a sidecar container to the function pod
+
         :param name:    Sidecar container name.
         :param image:   Sidecar container image.
         :param ports:   Sidecar container ports to expose. Can be a single port or a list of ports.

mlrun/serving/routers.py

@@ -596,7 +596,6 @@ class VotingEnsemble(ParallelRun):
         self.vote_type = vote_type
         self.vote_flag = True if self.vote_type is not None else False
         self.weights = weights
-        self.version = kwargs.get("version", "v1")
         self.log_router = True
         self.prediction_col_name = prediction_col_name or "prediction"
         self.format_response_with_col_name_flag = format_response_with_col_name_flag
@@ -927,14 +926,14 @@ class VotingEnsemble(ParallelRun):
                     "model_name": self.name,
                     "outputs": votes,
                 }
-                if self.version:
-                    response_body["model_version"] = self.version
+                if self.model_endpoint_uid:
+                    response_body["model_endpoint_uid"] = self.model_endpoint_uid
                 response.body = response_body
             elif name == self.name and event.method == "GET" and not subpath:
                 response = copy.copy(event)
                 response_body = {
                     "name": self.name,
-                    "version": self.version or "",
+                    "model_endpoint_uid": self.model_endpoint_uid or "",
                     "inputs": [],
                     "outputs": [],
                 }

mlrun/serving/server.py

@@ -65,7 +65,7 @@ class _StreamContext:
         self.hostname = socket.gethostname()
         self.function_uri = function_uri
         self.output_stream = None
-        self.stream_uri = None
+        stream_uri = None
         log_stream = parameters.get(FileTargetKind.LOG_STREAM, "")
 
         if (enabled or log_stream) and function_uri:
@@ -78,17 +78,19 @@ class _StreamContext:
 
             if log_stream == DUMMY_STREAM:
                 # Dummy stream used for testing, see tests/serving/test_serving.py
-                self.stream_uri = DUMMY_STREAM
+                stream_uri = DUMMY_STREAM
             elif not stream_args.get("mock"):  # if not a mock: `context.is_mock = True`
-                self.stream_uri = mlrun.model_monitoring.get_stream_path(
-                    project=project
-                )
+                stream_uri = mlrun.model_monitoring.get_stream_path(project=project)
 
             if log_stream:
                 # Update the stream path to the log stream value
-                self.stream_uri = log_stream.format(project=project)
-
-            self.output_stream = get_stream_pusher(self.stream_uri, **stream_args)
+                stream_uri = log_stream.format(project=project)
+                self.output_stream = get_stream_pusher(stream_uri, **stream_args)
+            else:
+                # Get the output stream from the profile
+                self.output_stream = mlrun.model_monitoring.helpers.get_output_stream(
+                    project=project, mock=stream_args.get("mock", False)
+                )
 
 
 class GraphServer(ModelObj):

mlrun/serving/states.py

@@ -652,6 +652,14 @@ class TaskStep(BaseStep):
             if isinstance(self.endpoint_type, schemas.EndpointType)
             else self.endpoint_type
         )
+        self.model_endpoint_creation_strategy = (
+            self.model_endpoint_creation_strategy.value
+            if isinstance(
+                self.model_endpoint_creation_strategy,
+                schemas.ModelEndpointCreationStrategy,
+            )
+            else self.model_endpoint_creation_strategy
+        )
         return super().to_dict(fields, exclude, strip)
 
 
@@ -755,9 +763,11 @@ class RouterStep(TaskStep):
         self._routes: ObjectDict = None
         self.routes = routes
         self.endpoint_type = schemas.EndpointType.ROUTER
+        if isinstance(class_name, type):
+            class_name = class_name.__name__
         self.model_endpoint_creation_strategy = (
             schemas.ModelEndpointCreationStrategy.INPLACE
-            if class_name and "serving.VotingEnsemble" in class_name
+            if class_name and "VotingEnsemble" in class_name
             else schemas.ModelEndpointCreationStrategy.SKIP
         )
 

mlrun/serving/v2_serving.py

@@ -95,9 +95,6 @@ class V2ModelServer(StepToDict):
         :param kwargs:     extra arguments (can be accessed using self.get_param(key))
         """
         self.name = name
-        self.version = ""
-        if name and ":" in name:
-            self.version = name.split(":", 1)[-1]
         self.context = context
         self.ready = False
         self.error = ""
@@ -325,8 +322,8 @@ class V2ModelServer(StepToDict):
                 "outputs": outputs,
                 "timestamp": start.isoformat(sep=" ", timespec="microseconds"),
             }
-            if self.version:
-                response["model_version"] = self.version
+            if self.model_endpoint_uid:
+                response["model_endpoint_uid"] = self.model_endpoint_uid
         elif op == "ready" and event.method == "GET":
             # get model health operation
             setattr(event, "terminated", True)
@@ -352,7 +349,7 @@ class V2ModelServer(StepToDict):
             setattr(event, "terminated", True)
             event_body = {
                 "name": self.name.split(":")[0],
-                "version": self.version or "",
+                "model_endpoint_uid": self.model_endpoint_uid or "",
                 "inputs": [],
                 "outputs": [],
             }
@@ -386,8 +383,8 @@ class V2ModelServer(StepToDict):
                 "model_name": self.name,
                 "outputs": outputs,
             }
-            if self.version:
-                response["model_version"] = self.version
+            if self.model_endpoint_uid:
+                response["model_endpoint_uid"] = self.model_endpoint_uid
 
         elif hasattr(self, "op_" + op):
             # custom operation (child methods starting with "op_")
@@ -510,7 +507,6 @@ class _ModelLogPusher:
         self.verbose = context.verbose
         self.hostname = context.stream.hostname
         self.function_uri = context.stream.function_uri
-        self.stream_path = context.stream.stream_uri
         self.sampling_percentage = float(context.get_param("sampling_percentage", 100))
         self.output_stream = output_stream or context.stream.output_stream
         self._worker = context.worker_id
@@ -520,7 +516,6 @@ class _ModelLogPusher:
             "class": self.model.__class__.__name__,
             "worker": self._worker,
             "model": self.model.name,
-            "version": self.model.version,
             "host": self.hostname,
             "function_uri": self.function_uri,
             "endpoint_id": self.model.model_endpoint_uid,

mlrun/utils/async_http.py

@@ -24,10 +24,17 @@ from aiohttp_retry import ExponentialRetry, RequestParams, RetryClient, RetryOpt
 from aiohttp_retry.client import _RequestContext
 
 from mlrun.config import config
-from mlrun.errors import err_to_str, raise_for_status
+from mlrun.errors import err_to_str
+from mlrun.errors import raise_for_status as ml_raise_for_status
 
 from .helpers import logger as mlrun_logger
 
+DEFAULT_BLACKLISTED_METHODS = [
+    "POST",
+    "PUT",
+    "PATCH",
+]
+
 
 class AsyncClientWithRetry(RetryClient):
     """
@@ -46,16 +53,6 @@ class AsyncClientWithRetry(RetryClient):
         *args,
         **kwargs,
     ):
-        # do not retry on PUT / PATCH as they might have side effects (not truly idempotent)
-        blacklisted_methods = (
-            blacklisted_methods
-            if blacklisted_methods is not None
-            else [
-                "POST",
-                "PUT",
-                "PATCH",
-            ]
-        )
         super().__init__(
             *args,
             retry_options=ExponentialRetryOverride(
@@ -72,9 +69,13 @@ class AsyncClientWithRetry(RetryClient):
             **kwargs,
         )
 
+    @property
+    def retry_options(self) -> "ExponentialRetryOverride":
+        return typing.cast(ExponentialRetryOverride, self._retry_options)
+
     def methods_blacklist_update_required(self, new_blacklist: str):
         self._retry_options: ExponentialRetryOverride
-        return set(self._retry_options.blacklisted_methods).difference(
+        return set(self.retry_options.blacklisted_methods).difference(
             set(new_blacklist)
         )
 
@@ -114,8 +115,8 @@ class ExponentialRetryOverride(ExponentialRetry):
 
     def __init__(
         self,
-        retry_on_exception: bool,
-        blacklisted_methods: list[str],
+        retry_on_exception: typing.Optional[bool] = True,
+        blacklisted_methods: typing.Optional[list[str]] = None,
         *args,
         **kwargs,
     ):
@@ -123,7 +124,12 @@ class ExponentialRetryOverride(ExponentialRetry):
         self.retry_on_exception = retry_on_exception
 
         # methods that should not be retried
-        self.blacklisted_methods = blacklisted_methods
+        # do not retry on PUT / PATCH as they might have side effects (not truly idempotent)
+        self.blacklisted_methods = (
+            blacklisted_methods
+            if blacklisted_methods is not None
+            else DEFAULT_BLACKLISTED_METHODS
+        )
 
         # default exceptions that should be retried on (when retry_on_exception is True)
         if "exceptions" not in kwargs:
@@ -136,6 +142,10 @@ class _CustomRequestContext(_RequestContext):
     Extends by adding retry logic on both error statuses and certain exceptions.
     """
 
+    @property
+    def retry_options(self) -> ExponentialRetryOverride:
+        return typing.cast(ExponentialRetryOverride, self._retry_options)
+
     async def _do_request(self) -> aiohttp.ClientResponse:
         current_attempt = 0
         while True:
@@ -186,9 +196,9 @@ class _CustomRequestContext(_RequestContext):
                     return response
 
                 last_attempt = current_attempt == self._retry_options.attempts
-                if self._is_status_code_ok(response.status) or last_attempt:
+                if not self._is_status_retryable(response.status) or last_attempt:
                     if self._raise_for_status:
-                        raise_for_status(response)
+                        ml_raise_for_status(response)
 
                     self._response = response
                     return response
@@ -204,7 +214,7 @@ class _CustomRequestContext(_RequestContext):
                     )
 
             except Exception as exc:
-                if not self._retry_options.retry_on_exception:
+                if not self.retry_options.retry_on_exception:
                     self._logger.warning(
                         "Request failed, retry on exception is disabled",
                         method=params.method,
@@ -250,7 +260,10 @@ class _CustomRequestContext(_RequestContext):
             await asyncio.sleep(retry_wait)
 
     def _is_method_retryable(self, method: str):
-        return method not in self._retry_options.blacklisted_methods
+        return method not in self.retry_options.blacklisted_methods
+
+    def _is_status_retryable(self, status: int):
+        return status in self._retry_options.statuses
 
     async def _check_response_callback(
         self,

mlrun/utils/helpers.py

@@ -34,6 +34,7 @@ from importlib import import_module, reload
 from os import path
 from types import ModuleType
 from typing import Any, Optional
+from urllib.parse import urlparse
 
 import git
 import inflection
@@ -1736,7 +1737,9 @@ setting partitioned=False"""
 
 def is_ecr_url(registry: str) -> bool:
     # example URL: <aws_account_id>.dkr.ecr.<region>.amazonaws.com
-    return ".ecr." in registry and ".amazonaws.com" in registry
+    parsed_url = urlparse(f"https://{registry}")
+    hostname = parsed_url.hostname
+    return hostname and ".ecr." in hostname and hostname.endswith(".amazonaws.com")
 
 
 def get_local_file_schema() -> list:

mlrun/utils/logger.py

@@ -53,6 +53,7 @@ class _BaseFormatter(logging.Formatter):
             json_object,
             option=orjson.OPT_NAIVE_UTC
             | orjson.OPT_SERIALIZE_NUMPY
+            | orjson.OPT_NON_STR_KEYS
             | orjson.OPT_SORT_KEYS,
             default=default,
         ).decode()

mlrun/utils/notifications/notification_pusher.py

@@ -55,6 +55,7 @@ class _NotificationPusherBase:
                 event_loop = asyncio.get_event_loop()
             except RuntimeError:
                 event_loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(event_loop)
 
             if not event_loop.is_running():
                 event_loop.run_until_complete(async_push_callback())

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "f7d4248834e0690d0e02b88fe62c16f9b40378e0",
-  "version": "1.8.0-rc28"
+  "git_commit": "d77b75367fbca42653b1bcfcee75ff821848202b",
+  "version": "1.8.0-rc30"
 }

{mlrun-1.8.0rc28.dist-info → mlrun-1.8.0rc30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mlrun
-Version: 1.8.0rc28
+Version: 1.8.0rc30
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -23,8 +23,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: urllib3<1.27,>=1.26.9
 Requires-Dist: GitPython>=3.1.41,~=3.1
-Requires-Dist: aiohttp~=3.10.0
-Requires-Dist: aiohttp-retry~=2.8.0
+Requires-Dist: aiohttp~=3.11
+Requires-Dist: aiohttp-retry~=2.9
 Requires-Dist: click~=8.1
 Requires-Dist: nest-asyncio~=1.0
 Requires-Dist: ipython~=8.10
@@ -51,7 +51,7 @@ Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.3.6
+Requires-Dist: mlrun-pipelines-kfp-common~=0.3.7
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.3.5; python_version < "3.11"
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0