mlrun 1.10.0rc16__tar.gz → 1.10.0rc18__tar.gz

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/Makefile

@@ -752,22 +752,6 @@ run-api: api ## Run mlrun api (dockerized)
 		--env MLRUN_HTTPDB__REAL_PATH=$(MLRUN_HTTPDB__REAL_PATH) \
 		$(MLRUN_API_IMAGE_NAME_TAGGED)
 
-.PHONY: run-test-db
-run-test-db:
-	# clean up any previous test db container. Don't remove it after run to be able to debug failures
-	docker rm test-db --force || true
-	docker run \
-		--name=test-db \
-		--volume $(shell pwd):/mlrun \
-		--publish 3306:3306 \
-		--env MYSQL_ROOT_PASSWORD="" \
-		--env MYSQL_ALLOW_EMPTY_PASSWORD="true" \
-		--env MYSQL_ROOT_HOST=% \
-		--env MYSQL_DATABASE="mlrun" \
-		--detach \
-		gcr.io/iguazio/mlrun-mysql:8.0 \
-		--character-set-server=utf8 \
-		--collation-server=utf8_bin
 
 .PHONY: clean-html-docs
 clean-html-docs: ## Clean html docs

{mlrun-1.10.0rc16/mlrun.egg-info → mlrun-1.10.0rc18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc16
+Version: 1.10.0rc18
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.8
+Requires-Dist: storey~=1.10.9
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/dev-requirements.txt

@@ -12,7 +12,8 @@ pytest-alembic~=0.11.0
 pytest-httpserver~=1.0
 pytest-timeout~=2.0
 pytest-mock-resources[mysql]~=2.12
-testcontainers[k3s]~=4.10
+# pin to 4.10.0 to avoid issues with k3s compatibility
+testcontainers[k3s]~=4.10.0
 aioresponses~=0.7
 requests-mock~=1.8
 httpx~=0.27.0

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/constants.py

@@ -84,6 +84,8 @@ class MLRunInternalLabels:
     original_workflow_id = "original-workflow-id"
     workflow_id = "workflow-id"
     retrying = "retrying"
+    rerun_counter = "rerun-counter"
+    rerun_index = "rerun-index"
 
     owner = "owner"
     v3io_user = "v3io_user"

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/formatters/artifact.py

@@ -41,6 +41,7 @@ class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
                     "spec.metrics",
                     "spec.target_path",
                     "spec.parent_uri",
+                    "spec.has_children",
                 ]
             ),
         }[_format]

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/schemas/__init__.py

@@ -133,6 +133,7 @@ from .k8s import NodeSelectorOperator, Resources, ResourceSpec
 from .memory_reports import MostCommonObjectTypesReport, ObjectTypeReport
 from .model_monitoring import (
     DriftStatus,
+    EndpointMode,
     EndpointType,
     EndpointUID,
     EventFieldType,

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/schemas/model_monitoring/__init__.py

@@ -16,6 +16,7 @@ from .constants import (
     INTERSECT_DICT_KEYS,
     ApplicationEvent,
     DriftStatus,
+    EndpointMode,
     EndpointType,
     EndpointUID,
     EventFieldType,

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/schemas/model_monitoring/constants.py

@@ -205,6 +205,11 @@ class ControllerEvent(MonitoringStrEnum):
     FIRST_REQUEST = "first_request"
     FEATURE_SET_URI = "feature_set_uri"
     ENDPOINT_TYPE = "endpoint_type"
+
+    # first_timestamp and last_timestamp are used to batch completed events
+    FIRST_TIMESTAMP = "first_timestamp"
+    LAST_TIMESTAMP = "last_timestamp"
+
     ENDPOINT_POLICY = "endpoint_policy"
     # Note: currently under endpoint policy we will have a dictionary including the keys: "application_names"
     # "base_period", and "updated_endpoint" stand for when the MEP was updated
@@ -219,6 +224,7 @@ class ControllerEventEndpointPolicy(MonitoringStrEnum):
 class ControllerEventKind(MonitoringStrEnum):
     NOP_EVENT = "nop_event"
     REGULAR_EVENT = "regular_event"
+    BATCH_COMPLETE = "batch_complete"
 
 
 class MetricData(MonitoringStrEnum):
@@ -320,6 +326,19 @@ class EndpointType(IntEnum):
     def top_level_list(cls):
         return [cls.NODE_EP, cls.ROUTER, cls.BATCH_EP]
 
+    @classmethod
+    def real_time_list(cls):
+        return [cls.NODE_EP, cls.ROUTER, cls.LEAF_EP]
+
+    @classmethod
+    def batch_list(cls):
+        return [cls.BATCH_EP]
+
+
+class EndpointMode(StrEnum):
+    REAL_TIME = "real_time"
+    BATCH = "batch"
+
 
 class MonitoringFunctionNames(MonitoringStrEnum):
     STREAM = "model-monitoring-stream"
@@ -468,25 +487,33 @@ class ModelMonitoringLabels:
 
 _RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
 
+_RESERVED_EVALUATE_FUNCTION_SUFFIX = "-batch"
+
 
 class ModelEndpointMonitoringMetricType(StrEnum):
     RESULT = "result"
     METRIC = "metric"
 
 
+# refer to `mlrun.utils.regex.project_name`
+_INNER_PROJECT_PATTERN = r"[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?"
+PROJECT_PATTERN = rf"^{_INNER_PROJECT_PATTERN}$"
+
+MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
 _FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
+_RESULT_NAME_PATTERN = r"[a-zA-Z_][a-zA-Z0-9_]*"
+
 FQN_PATTERN = (
-    rf"^(?P<project>{_FQN_PART_PATTERN})\."
+    rf"^(?P<project>{_INNER_PROJECT_PATTERN})\."
     rf"(?P<app>{_FQN_PART_PATTERN})\."
     rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
-    rf"(?P<name>{_FQN_PART_PATTERN})$"
+    rf"(?P<name>{_RESULT_NAME_PATTERN})$"
 )
 FQN_REGEX = re.compile(FQN_PATTERN)
+APP_NAME_REGEX = re.compile(_FQN_PART_PATTERN)
+RESULT_NAME_REGEX = re.compile(_RESULT_NAME_PATTERN)
 
-# refer to `mlrun.utils.regex.project_name`
-PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
-MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"
-RESULT_NAME_PATTERN = r"[a-zA-Z_][a-zA-Z0-9_]*"
 
 INTERSECT_DICT_KEYS = {
     ModelEndpointMonitoringMetricType.METRIC: "intersect_metrics",

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/schemas/serving.py

@@ -47,3 +47,6 @@ class MonitoringData(StrEnum):
 class ModelsData(enum.Enum):
     MODEL_CLASS = 0
     MODEL_PARAMETERS = 1
+
+
+MAX_BATCH_JOB_DURATION = "1w"

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/common/schemas/workflow.py

@@ -53,6 +53,7 @@ class RerunWorkflowRequest(pydantic.v1.BaseModel):
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
     original_workflow_runner_uid: typing.Optional[str] = None
     original_workflow_name: typing.Optional[str] = None
+    rerun_index: typing.Optional[int] = None
 
 
 class WorkflowResponse(pydantic.v1.BaseModel):

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/config.py

@@ -194,6 +194,7 @@ default_config = {
     "v3io_framesd": "http://framesd:8080",
     "model_providers": {
         "openai_default_model": "gpt-4o",
+        "huggingface_default_model": "microsoft/Phi-3-mini-4k-instruct",
     },
     # default node selector to be applied to all functions - json string base64 encoded format
     "default_function_node_selector": "e30=",
@@ -406,11 +407,7 @@ default_config = {
                 #
                 # if set to "nil" or "none", nothing would be set
                 "modes": (
-                    "STRICT_TRANS_TABLES"
-                    ",NO_ZERO_IN_DATE"
-                    ",NO_ZERO_DATE"
-                    ",ERROR_FOR_DIVISION_BY_ZERO"
-                    ",NO_ENGINE_SUBSTITUTION",
+                    "STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"
                 )
             },
         },
@@ -1242,6 +1239,19 @@ class Config:
         """
         return self.is_running_on_iguazio()
 
+    @staticmethod
+    def get_run_retry_staleness_threshold_timedelta() -> timedelta:
+        """
+        Get the staleness threshold in timedelta for run retries.
+        This is used to determine if a run is stale and should be retried.
+
+        :return: The staleness threshold in timedelta.
+        """
+        staleness_threshold = int(
+            mlrun.mlconf.monitoring.runs.retry.staleness_threshold
+        )
+        return timedelta(minutes=staleness_threshold)
+
     def to_dict(self):
         return copy.deepcopy(self._cfg)
 

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/datastore/datastore.py

@@ -38,6 +38,7 @@ from ..utils import DB_SCHEMA, RunKeys
 from .base import DataItem, DataStore, HttpStore
 from .filestore import FileStore
 from .inmem import InMemoryStore
+from .model_provider.huggingface_provider import HuggingFaceProvider
 from .model_provider.openai_provider import OpenAIProvider
 from .store_resources import get_store_resource, is_store_uri
 from .v3io import V3ioStore
@@ -102,8 +103,7 @@ def schema_to_store(schema) -> DataStore.__subclasses__():
 def schema_to_model_provider(
     schema: str, raise_missing_schema_exception=True
 ) -> type[ModelProvider]:
-    #  TODO add hugging face and http
-    schema_dict = {"openai": OpenAIProvider}
+    schema_dict = {"openai": OpenAIProvider, "huggingface": HuggingFaceProvider}
     provider_class = schema_dict.get(schema, None)
     if not provider_class:
         if raise_missing_schema_exception:
@@ -247,7 +247,7 @@ class StoreManager:
 
         if schema == "ds":
             datastore_profile = datastore_profile_read(url, project_name, secrets)
-            secrets = merge(secrets or {}, datastore_profile.secrets() or {})
+            secrets = merge({}, secrets or {}, datastore_profile.secrets() or {})
             url = datastore_profile.url(subpath)
             schema, endpoint, parsed_url = parse_url(url)
             subpath = parsed_url.path
@@ -281,7 +281,7 @@ class StoreManager:
                 endpoint, subpath
             )
             remote_client = remote_client_class(
-                self, schema, cache_key, parsed_url.netloc, secrets=secrets, **kwargs
+                self, schema, cache_key, endpoint, secrets=secrets, **kwargs
             )
             if not secrets and not mlrun.config.is_running_as_api():
                 cache[cache_key] = remote_client

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/datastore/datastore_profile.py

@@ -486,6 +486,31 @@ class OpenAIProfile(DatastoreProfile):
         return f"{self.type}://{subpath.lstrip('/')}"
 
 
+class HuggingFaceProfile(DatastoreProfile):
+    type: str = pydantic.v1.Field("huggingface")
+    _private_attributes = ("token", "model_kwargs")
+    task: typing.Optional[str] = None
+    token: typing.Optional[str] = None
+    device: typing.Optional[typing.Union[int, str]] = None
+    device_map: typing.Union[str, dict[str, typing.Union[int, str]], None] = None
+    trust_remote_code: bool = None
+    model_kwargs: typing.Optional[dict[str, typing.Any]] = None
+
+    def secrets(self) -> dict:
+        keys = {
+            "HF_TASK": self.task,
+            "HF_TOKEN": self.token,
+            "HF_DEVICE": self.device,
+            "HF_DEVICE_MAP": self.device_map,
+            "HF_TRUST_REMOTE_CODE": self.trust_remote_code,
+            "HF_MODEL_KWARGS": self.model_kwargs,
+        }
+        return {k: v for k, v in keys.items() if v}
+
+    def url(self, subpath):
+        return f"{self.type}://{subpath.lstrip('/')}"
+
+
 _DATASTORE_TYPE_TO_PROFILE_CLASS: dict[str, type[DatastoreProfile]] = {
     "v3io": DatastoreProfileV3io,
     "s3": DatastoreProfileS3,
@@ -500,6 +525,7 @@ _DATASTORE_TYPE_TO_PROFILE_CLASS: dict[str, type[DatastoreProfile]] = {
     "taosws": DatastoreProfileTDEngine,
     "config": ConfigProfile,
     "openai": OpenAIProfile,
+    "huggingface": HuggingFaceProfile,
 }
 
 

mlrun-1.10.0rc18/mlrun/datastore/model_provider/huggingface_provider.py

@@ -0,0 +1,183 @@
+# Copyright 2025 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Optional, TypeVar, Union
+
+import mlrun
+from mlrun.datastore.model_provider.model_provider import ModelProvider
+
+if TYPE_CHECKING:
+    from transformers.pipelines.base import Pipeline
+
+T = TypeVar("T")
+ChatType = list[dict[str, str]]  # according to transformers.pipelines.text_generation
+
+
+class HuggingFaceProvider(ModelProvider):
+    """
+    HuggingFaceProvider is a wrapper around the Hugging Face Transformers pipeline
+    that provides an interface for interacting with a wide range of Hugging Face models.
+
+    It supports synchronous operations, enabling flexible integration into various workflows.
+
+    This class extends the ModelProvider base class and implements Hugging Face-specific
+    functionality, including pipeline initialization, default text generation operations,
+    and custom operations tailored to the Hugging Face Transformers pipeline API.
+    """
+
+    def __init__(
+        self,
+        parent,
+        schema,
+        name,
+        endpoint="",
+        secrets: Optional[dict] = None,
+        default_invoke_kwargs: Optional[dict] = None,
+    ):
+        endpoint = endpoint or mlrun.mlconf.model_providers.huggingface_default_model
+        if schema != "huggingface":
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "HuggingFaceProvider supports only 'huggingface' as the provider kind."
+            )
+        super().__init__(
+            parent=parent,
+            kind=schema,
+            name=name,
+            endpoint=endpoint,
+            secrets=secrets,
+            default_invoke_kwargs=default_invoke_kwargs,
+        )
+        self.options = self.get_client_options()
+        self._expected_operation_type = None
+        self.load_client()
+
+    @staticmethod
+    def _extract_string_output(result) -> str:
+        """
+        Extracts the first generated string from Hugging Face pipeline output,
+        regardless of whether it's plain text-generation or chat-style output.
+        """
+        if not isinstance(result, list) or len(result) == 0:
+            raise ValueError("Empty or invalid pipeline output")
+
+        return result[0].get("generated_text")
+
+    @classmethod
+    def parse_endpoint_and_path(cls, endpoint, subpath) -> (str, str):
+        if endpoint and subpath:
+            endpoint = endpoint + subpath
+            # In HuggingFace, "/" in a model name is part of the name — `subpath` is not used.
+            subpath = ""
+        return endpoint, subpath
+
+    def load_client(self) -> None:
+        """
+        Initializes the Hugging Face pipeline using the provided options.
+
+        This method imports the `pipeline` function from the `transformers` package,
+        creates a pipeline instance with the specified task and model (from `self.options`),
+        and assigns it to `self._client`.
+
+        Note: Hugging Face pipelines are synchronous and do not support async invocation.
+
+        Raises:
+            ImportError: If the `transformers` package is not installed.
+        """
+        try:
+            from transformers import pipeline, AutoModelForCausalLM  # noqa
+            from transformers import AutoTokenizer  # noqa
+            from transformers.pipelines.base import Pipeline  # noqa
+
+            self._client = pipeline(model=self.model, **self.options)
+            self._expected_operation_type = Pipeline
+        except ImportError as exc:
+            raise ImportError("transformers package is not installed") from exc
+
+    def get_client_options(self):
+        res = dict(
+            task=self._get_secret_or_env("HF_TASK") or "text-generation",
+            token=self._get_secret_or_env("HF_TOKEN"),
+            device=self._get_secret_or_env("HF_DEVICE"),
+            device_map=self._get_secret_or_env("HF_DEVICE_MAP"),
+            trust_remote_code=self._get_secret_or_env("HF_TRUST_REMOTE_CODE"),
+            model_kwargs=self._get_secret_or_env("HF_MODEL_KWARGS"),
+        )
+        return self._sanitize_options(res)
+
+    def custom_invoke(
+        self, operation: Optional["Pipeline"] = None, **invoke_kwargs
+    ) -> Optional[T]:
+        """
+        HuggingFace implementation of `ModelProvider.custom_invoke`.
+        Use the default config in provider client/ user defined client:
+
+        Example:
+        ```python
+            image = Image.open(image_path)
+            pipeline_object =  pipeline("image-classification", model="microsoft/resnet-50")
+            result = hf_provider.custom_invoke(
+                pipeline_object,
+                inputs=image,
+            )
+        ```
+
+
+        :param operation:               A pipeline object
+        :param invoke_kwargs:           Keyword arguments to pass to the operation.
+        :return:                        The full response returned by the operation.
+
+        """
+        invoke_kwargs = self.get_invoke_kwargs(invoke_kwargs)
+        if operation:
+            if not isinstance(operation, self._expected_operation_type):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Huggingface operation must inherit" " from 'Pipeline' object"
+                )
+            return operation(**invoke_kwargs)
+        else:
+            return self.client(**invoke_kwargs)
+
+    def invoke(
+        self,
+        messages: Union[str, list[str], ChatType, list[ChatType]] = None,
+        as_str: bool = False,
+        **invoke_kwargs,
+    ) -> Optional[Union[str, list, T]]:
+        """
+        HuggingFace-specific implementation of `ModelProvider.invoke`.
+        Invokes a HuggingFace model operation using the synchronous client.
+        For complete usage details, refer to `ModelProvider.invoke`.
+        :param messages:
+                            Same as ModelProvider.invoke.
+
+        :param as_str:
+                            If `True`, returns only the main content from a single response
+                            (intended for single-response use cases).
+                            If `False`, returns the full response object, whose type depends on
+                            the client (e.g., `pipeline`).
+
+        :param invoke_kwargs:
+                            Same as ModelProvider.invoke.
+        :return:            Same as ModelProvider.invoke.
+        """
+        if self.client.task != "text-generation":
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "HuggingFaceProvider.invoke supports text-generation task only"
+            )
+        if as_str:
+            invoke_kwargs["return_full_text"] = False
+        response = self.custom_invoke(text_inputs=messages, **invoke_kwargs)
+        if as_str:
+            return self._extract_string_output(response)
+        return response

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/datastore/model_provider/model_provider.py

@@ -148,7 +148,12 @@ class ModelProvider(BaseRemoteClient):
 
     @property
     def model(self) -> Optional[str]:
-        return None
+        """
+        Returns the model identifier used by the underlying SDK.
+
+        :return: A string representing the model ID, or None if not set.
+        """
+        return self.endpoint
 
     def get_invoke_kwargs(self, invoke_kwargs) -> dict:
         kwargs = self.default_invoke_kwargs.copy()

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/datastore/model_provider/openai_provider.py

@@ -11,11 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import inspect
 from collections.abc import Awaitable
 from typing import Callable, Optional, TypeVar, Union
 
 import mlrun
 from mlrun.datastore.model_provider.model_provider import ModelProvider
+from mlrun.datastore.utils import accepts_param
 
 T = TypeVar("T")
 
@@ -68,10 +70,6 @@ class OpenAIProvider(ModelProvider):
             subpath = ""
         return endpoint, subpath
 
-    @property
-    def model(self) -> Optional[str]:
-        return self.endpoint
-
     def load_client(self) -> None:
         """
         Initializes the OpenAI SDK client using the provided options.
@@ -126,12 +124,18 @@ class OpenAIProvider(ModelProvider):
 
         """
         invoke_kwargs = self.get_invoke_kwargs(invoke_kwargs)
+        model_kwargs = {"model": invoke_kwargs.pop("model", None) or self.model}
+
         if operation:
-            return operation(**invoke_kwargs, model=self.model)
+            if not callable(operation):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "OpenAI custom_invoke operation must be a callable"
+                )
+            if not accepts_param(operation, "model"):
+                model_kwargs = {}
+            return operation(**invoke_kwargs, **model_kwargs)
         else:
-            return self.client.chat.completions.create(
-                **invoke_kwargs, model=self.model
-            )
+            return self.client.chat.completions.create(**invoke_kwargs, **model_kwargs)
 
     async def async_custom_invoke(
         self,
@@ -145,25 +149,33 @@ class OpenAIProvider(ModelProvider):
         `ModelProvider.async_custom_invoke`.
 
         Example:
-            ```python
+        ```python
             result = openai_model_provider.invoke(
                 openai_model_provider.async_client.images.generate,
                 prompt="A futuristic cityscape at sunset",
                 n=1,
                 size="1024x1024",
             )
-            ```
+        ```
+
         :param operation:       Same as ModelProvider.async_custom_invoke.
         :param invoke_kwargs:   Same as ModelProvider.async_custom_invoke.
         :return:                Same as ModelProvider.async_custom_invoke.
 
         """
         invoke_kwargs = self.get_invoke_kwargs(invoke_kwargs)
+        model_kwargs = {"model": invoke_kwargs.pop("model", None) or self.model}
         if operation:
-            return await operation(**invoke_kwargs, model=self.model)
+            if not inspect.iscoroutinefunction(operation):
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "OpenAI async_custom_invoke operation must be a coroutine function"
+                )
+            if not accepts_param(operation, "model"):
+                model_kwargs = {}
+            return await operation(**invoke_kwargs, **model_kwargs)
         else:
             return await self.async_client.chat.completions.create(
-                **invoke_kwargs, model=self.model
+                **invoke_kwargs, **model_kwargs
             )
 
     def invoke(

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/datastore/utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import inspect
 import math
 import tarfile
 import tempfile
@@ -333,3 +334,8 @@ def parse_url(url):
     if parsed_url.port:
         endpoint += f":{parsed_url.port}"
     return schema, endpoint, parsed_url
+
+
+def accepts_param(func: callable, param_name):
+    sig = inspect.signature(func)
+    return param_name in sig.parameters

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/db/base.py

@@ -741,6 +741,7 @@ class RunDBInterface(ABC):
         tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
+        mode: Optional[mlrun.common.schemas.EndpointMode] = None,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
     ) -> mlrun.common.schemas.ModelEndpointList:

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/db/httpdb.py

@@ -3813,6 +3813,7 @@ class HTTPRunDB(RunDBInterface):
         tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
+        mode: mm_constants.EndpointMode = None,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
     ) -> mlrun.common.schemas.ModelEndpointList:
@@ -3833,6 +3834,8 @@ class HTTPRunDB(RunDBInterface):
                                 If tsdb_metrics=False, this parameter will be ignored and no tsdb metrics
                                 will be included.
         :param top_level:       Whether to return only top level model endpoints.
+        :param mode:            Specifies the mode of the model endpoint. Can be "real-time", "batch", or both if set
+                                to None.
         :param uids:            A list of unique ids to filter by.
         :param latest_only:     Whether to return only the latest model endpoint version.
         :return:                A list of model endpoints.
@@ -3856,6 +3859,7 @@ class HTTPRunDB(RunDBInterface):
                 "tsdb-metrics": tsdb_metrics,
                 "metric": metric_list,
                 "top-level": top_level,
+                "mode": mode,
                 "uid": uids,
                 "latest-only": latest_only,
             },

{mlrun-1.10.0rc16 → mlrun-1.10.0rc18}/mlrun/model_monitoring/api.py

@@ -19,8 +19,6 @@ from datetime import datetime
 import numpy as np
 import pandas as pd
 
-import mlrun.artifacts
-import mlrun.common.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.datastore.base
 import mlrun.feature_store
@@ -538,10 +536,14 @@ def _create_model_monitoring_function_base(
     This function does not set the labels or mounts v3io.
     """
     if name in mm_constants._RESERVED_FUNCTION_NAMES:
-        raise mlrun.errors.MLRunInvalidArgumentError(
+        raise mlrun.errors.MLRunValueError(
             "An application cannot have the following names: "
             f"{mm_constants._RESERVED_FUNCTION_NAMES}"
         )
+    if name and name.endswith(mm_constants._RESERVED_EVALUATE_FUNCTION_SUFFIX):
+        raise mlrun.errors.MLRunValueError(
+            "Model monitoring application names cannot end with `-batch`"
+        )
     if func is None:
         func = ""
     func_obj = typing.cast(