mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/serving/states.py

@@ -27,6 +27,8 @@ from copy import copy, deepcopy
 from inspect import getfullargspec, signature
 from typing import Any, Union
 
+import storey.utils
+
 import mlrun
 
 from ..config import config
@@ -82,6 +84,9 @@ _task_step_fields = [
 ]
 
 
+MAX_ALLOWED_STEPS = 4500
+
+
 def new_model_endpoint(class_name, model_path, handler=None, **class_args):
     class_args = deepcopy(class_args)
     class_args["model_path"] = model_path
@@ -386,6 +391,9 @@ class BaseStep(ModelObj):
         """
         raise NotImplementedError("set_flow() can only be called on a FlowStep")
 
+    def supports_termination(self):
+        return False
+
 
 class TaskStep(BaseStep):
     """task execution step, runs a class or handler"""
@@ -728,6 +736,11 @@ class RouterStep(TaskStep):
         if not route:
             route = TaskStep(class_name, class_args, handler=handler)
         route.function = function or route.function
+
+        if len(self._routes) >= MAX_ALLOWED_STEPS:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Cannot create the serving graph: the maximum number of steps is {MAX_ALLOWED_STEPS}"
+            )
         route = self._routes.update(key, route)
         route.set_parent(self)
         return route
@@ -867,7 +880,10 @@ class QueueStep(BaseStep):
             return event
 
         if self._stream:
-            self._stream.push({"id": event.id, "body": data, "path": event.path})
+            full_event = self.options.get("full_event")
+            if full_event or full_event is None and self.next:
+                data = storey.utils.wrap_event_for_serialization(event, data)
+            self._stream.push(data)
             event.terminated = True
             event.body = None
         return event
@@ -1273,6 +1289,8 @@ class FlowStep(BaseStep):
             event.body = {"id": event.id}
             return event
 
+        event = storey.utils.unpack_event_if_wrapped(event)
+
         if len(self._start_steps) == 0:
             return event
         next_obj = self._start_steps[0]
@@ -1380,6 +1398,9 @@ class FlowStep(BaseStep):
 
         return step
 
+    def supports_termination(self):
+        return self.engine != "sync"
+
 
 class RootFlowStep(FlowStep):
     """root flow step"""
@@ -1618,7 +1639,11 @@ def _init_async_objects(context, steps):
                 if step.path and not skip_stream:
                     stream_path = step.path
                     endpoint = None
-                    options = {}
+                    # in case of a queue, we default to a full_event=True
+                    full_event = step.options.get("full_event")
+                    options = {
+                        "full_event": full_event or full_event is None and step.next
+                    }
                     options.update(step.options)
 
                     kafka_brokers = get_kafka_brokers_from_dict(options, pop=True)
@@ -1672,7 +1697,9 @@ def _init_async_objects(context, steps):
                 wait_for_result = True
 
     source_args = context.get_param("source_args", {})
-    explicit_ack = is_explicit_ack_supported(context) and mlrun.mlconf.is_explicit_ack()
+    explicit_ack = (
+        is_explicit_ack_supported(context) and mlrun.mlconf.is_explicit_ack_enabled()
+    )
 
     # TODO: Change to AsyncEmitSource once we can drop support for nuclio<1.12.10
     default_source = storey.SyncEmitSource(

mlrun/serving/v2_serving.py

@@ -15,12 +15,12 @@
 import threading
 import time
 import traceback
-from typing import Union
+from typing import Optional, Union
 
-import mlrun.common.model_monitoring
+import mlrun.artifacts
+import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring
-from mlrun.artifacts import ModelArtifact  # noqa: F401
-from mlrun.config import config
+import mlrun.model_monitoring
 from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
@@ -102,7 +102,7 @@ class V2ModelServer(StepToDict):
         self.error = ""
         self.protocol = protocol or "v2"
         self.model_path = model_path
-        self.model_spec: mlrun.artifacts.ModelArtifact = None
+        self.model_spec: Optional[mlrun.artifacts.ModelArtifact] = None
         self._input_path = input_path
         self._result_path = result_path
         self._kwargs = kwargs  # for to_dict()
@@ -148,7 +148,7 @@ class V2ModelServer(StepToDict):
             logger.warn("GraphServer not initialized for VotingEnsemble instance")
             return
 
-        if not self.context.is_mock or self.context.server.track_models:
+        if not self.context.is_mock or self.context.monitoring_mock:
             self.model_endpoint_uid = _init_endpoint_record(
                 graph_server=server, model=self
             )
@@ -258,6 +258,7 @@ class V2ModelServer(StepToDict):
                 "id": event_id,
                 "model_name": self.name,
                 "outputs": outputs,
+                "timestamp": start.isoformat(sep=" ", timespec="microseconds"),
             }
             if self.version:
                 response["model_version"] = self.version
@@ -335,6 +336,7 @@ class V2ModelServer(StepToDict):
             else:
                 track_request = {"id": event_id, "inputs": inputs or []}
                 track_response = {"outputs": outputs or []}
+                # TODO : check dict/list
                 self._model_logger.push(start, track_request, track_response, op)
         event.body = _update_result_body(self._result_path, original_body, response)
         return event
@@ -376,8 +378,10 @@ class V2ModelServer(StepToDict):
         """postprocess, before returning response"""
         return request
 
-    def predict(self, request: dict) -> dict:
-        """model prediction operation"""
+    def predict(self, request: dict) -> list:
+        """model prediction operation
+        :return: list with the model prediction results (can be multi-port) or list of lists for multiple predictions
+        """
         raise NotImplementedError()
 
     def explain(self, request: dict) -> dict:
@@ -551,13 +555,13 @@ def _init_endpoint_record(
     except mlrun.errors.MLRunNotFoundError:
         model_ep = None
     except mlrun.errors.MLRunBadRequestError as err:
-        logger.debug(
-            f"Cant reach to model endpoints store, due to  : {err}",
+        logger.info(
+            "Cannot get the model endpoints store", err=mlrun.errors.err_to_str(err)
         )
         return
 
     if model.context.server.track_models and not model_ep:
-        logger.debug("Creating a new model endpoint record", endpoint_id=uid)
+        logger.info("Creating a new model endpoint record", endpoint_id=uid)
         model_endpoint = mlrun.common.schemas.ModelEndpoint(
             metadata=mlrun.common.schemas.ModelEndpointMetadata(
                 project=project, labels=model.labels, uid=uid
@@ -567,9 +571,7 @@ def _init_endpoint_record(
                 model=versioned_model_name,
                 model_class=model.__class__.__name__,
                 model_uri=model.model_path,
-                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                    project=project, kind="stream"
-                ),
+                stream_path=model.context.stream.stream_uri,
                 active=True,
                 monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
             ),
@@ -585,28 +587,35 @@ def _init_endpoint_record(
             model_endpoint=model_endpoint.dict(),
         )
 
-    elif (
-        model_ep
-        and (
+    elif model_ep:
+        attributes = {}
+        old_model_uri = model_ep.spec.model_uri
+        mlrun.model_monitoring.helpers.enrich_model_endpoint_with_model_uri(
+            model_endpoint=model_ep,
+            model_obj=model.model_spec,
+        )
+        if model_ep.spec.model_uri != old_model_uri:
+            attributes["model_uri"] = model_ep.spec.model_uri
+        if (
             model_ep.spec.monitoring_mode
             == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-        )
-        != model.context.server.track_models
-    ):
-        monitoring_mode = (
-            mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-            if model.context.server.track_models
-            else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
-        )
-        db = mlrun.get_run_db()
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=uid,
-            attributes={"monitoring_mode": monitoring_mode},
-        )
-        logger.debug(
-            f"Updating model endpoint monitoring_mode to {monitoring_mode}",
-            endpoint_id=uid,
-        )
+        ) != model.context.server.track_models:
+            attributes["monitoring_mode"] = (
+                mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+                if model.context.server.track_models
+                else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
+            )
+        if attributes:
+            db = mlrun.get_run_db()
+            db.patch_model_endpoint(
+                project=project,
+                endpoint_id=uid,
+                attributes=attributes,
+            )
+            logger.info(
+                "Updating model endpoint attributes",
+                attributes=attributes,
+                endpoint_id=uid,
+            )
 
     return uid

mlrun/track/trackers/mlflow_tracker.py

@@ -442,6 +442,11 @@ class MLFlowTracker(Tracker):
         # Prepare the archive path:
         model_uri = pathlib.Path(model_uri)
         archive_path = pathlib.Path(tmp_path) / f"{model_uri.stem}.zip"
+        if not os.path.exists(model_uri):
+            local_path = mlflow.artifacts.download_artifacts(
+                artifact_uri=str(model_uri)
+            )
+            model_uri = pathlib.Path(local_path)
 
         # TODO add progress bar for the case of large files
         # Zip the artifact:

mlrun/utils/async_http.py

@@ -237,7 +237,7 @@ class _CustomRequestContext(_RequestContext):
                 retry_wait = self._retry_options.get_timeout(
                     attempt=current_attempt, response=None
                 )
-                self._logger.debug(
+                self._logger.warning(
                     "Request failed on retryable exception, retrying",
                     retry_wait_secs=retry_wait,
                     method=params.method,

mlrun/utils/db.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import abc
 import pickle
 from datetime import datetime
 
@@ -28,12 +29,22 @@ class BaseModel:
         columns = [column.key for column in mapper.columns if column.key not in exclude]
 
         def get_key_value(c):
+            # all (never say never) DB classes have "object" defined as "full_object"
+            if c == "object":
+                c = "full_object"
             if isinstance(getattr(self, c), datetime):
                 return c, getattr(self, c).isoformat()
             return c, getattr(self, c)
 
         return dict(map(get_key_value, columns))
 
+    @abc.abstractmethod
+    def get_identifier_string(self):
+        """
+        This method must be implemented by any subclass.
+        """
+        pass
+
 
 class HasStruct(BaseModel):
     @property
@@ -51,3 +62,10 @@ class HasStruct(BaseModel):
         exclude = exclude or []
         exclude.append("body")
         return super().to_dict(exclude, strip=strip)
+
+    @abc.abstractmethod
+    def get_identifier_string(self):
+        """
+        This method must be implemented by any subclass.
+        """
+        pass

mlrun/utils/helpers.py

@@ -24,6 +24,7 @@ import re
 import string
 import sys
 import typing
+import uuid
 import warnings
 from datetime import datetime, timezone
 from importlib import import_module, reload
@@ -40,7 +41,7 @@ import semver
 import yaml
 from dateutil import parser
 from mlrun_pipelines.models import PipelineRun
-from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
+from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
 import mlrun
@@ -110,14 +111,15 @@ def get_artifact_target(item: dict, project=None):
     project_str = project or item["metadata"].get("project")
     tree = item["metadata"].get("tree")
     tag = item["metadata"].get("tag")
-
     kind = item.get("kind")
-    if kind in ["dataset", "model", "artifact"] and db_key:
-        target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
-        if tag:
-            target = f"{target}:{tag}"
+
+    if kind in {"dataset", "model", "artifact"} and db_key:
+        target = (
+            f"{DB_SCHEMA}://{StorePrefix.kind_to_prefix(kind)}/{project_str}/{db_key}"
+        )
+        target += f":{tag}" if tag else ":latest"
         if tree:
-            target = f"{target}@{tree}"
+            target += f"@{tree}"
         return target
 
     return item["spec"].get("target_path")
@@ -134,18 +136,25 @@ def is_legacy_artifact(artifact):
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
 missing = object()
 
-is_ipython = False
+is_ipython = False  # is IPython terminal, including Jupyter
+is_jupyter = False  # is Jupyter notebook/lab terminal
 try:
-    import IPython
+    import IPython.core.getipython
+
+    ipy = IPython.core.getipython.get_ipython()
+
+    is_ipython = ipy is not None
+    is_jupyter = (
+        is_ipython
+        # not IPython
+        and "Terminal" not in str(type(ipy))
+    )
 
-    ipy = IPython.get_ipython()
-    # if its IPython terminal ignore (cant show html)
-    if ipy and "Terminal" not in str(type(ipy)):
-        is_ipython = True
-except ImportError:
+    del ipy
+except ModuleNotFoundError:
     pass
 
-if is_ipython and config.nest_asyncio_enabled in ["1", "True"]:
+if is_jupyter and config.nest_asyncio_enabled in ["1", "True"]:
     # bypass Jupyter asyncio bug
     import nest_asyncio
 
@@ -819,7 +828,6 @@ def enrich_image_url(
     tag += resolve_image_tag_suffix(
         mlrun_version=mlrun_version, python_version=client_python_version
     )
-    registry = config.images_registry
 
     # it's an mlrun image if the repository is mlrun
     is_mlrun_image = image_url.startswith("mlrun/") or "/mlrun/" in image_url
@@ -827,6 +835,10 @@ def enrich_image_url(
     if is_mlrun_image and tag and ":" not in image_url:
         image_url = f"{image_url}:{tag}"
 
+    registry = (
+        config.images_registry if is_mlrun_image else config.vendor_images_registry
+    )
+
     enrich_registry = False
     # enrich registry only if images_to_enrich_registry provided
     # example: "^mlrun/*" means enrich only if the image repository is mlrun and registry is not specified (in which
@@ -1005,6 +1017,23 @@ def get_workflow_url(project, id=None):
     return url
 
 
+def get_kfp_project_filter(project_name: str) -> str:
+    """
+    Generates a filter string for KFP runs, using a substring predicate
+    on the run's 'name' field. This is used as a heuristic to retrieve runs that are associated
+    with a specific project. The 'op: 9' operator indicates that the filter checks if the
+    project name appears as a substring in the run's name, ensuring that we can identify
+    runs belonging to the desired project.
+    """
+    is_substring_op = 9
+    project_name_filter = {
+        "predicates": [
+            {"key": "name", "op": is_substring_op, "string_value": project_name}
+        ]
+    }
+    return json.dumps(project_name_filter)
+
+
 def are_strings_in_exception_chain_messages(
     exception: Exception, strings_list: list[str]
 ) -> bool:
@@ -1402,11 +1431,27 @@ def is_running_in_jupyter_notebook() -> bool:
     Check if the code is running inside a Jupyter Notebook.
     :return: True if running inside a Jupyter Notebook, False otherwise.
     """
-    import IPython
+    return is_jupyter
 
-    ipy = IPython.get_ipython()
-    # if its IPython terminal, it isn't a Jupyter ipython
-    return ipy and "Terminal" not in str(type(ipy))
+
+def create_ipython_display():
+    """
+    Create an IPython display object and fill it with initial content.
+    We can later use the returned display_id with the update_display method to update the content.
+    If IPython is not installed, a warning will be logged and None will be returned.
+    """
+    if is_ipython:
+        import IPython
+
+        display_id = uuid.uuid4().hex
+        content = IPython.display.HTML(
+            f'<div id="{display_id}">Temporary Display Content</div>'
+        )
+        IPython.display.display(content, display_id=display_id)
+        return display_id
+
+    # returning None if IPython is not installed, this method shouldn't be called in that case but logging for sanity
+    logger.debug("IPython is not installed, cannot create IPython display")
 
 
 def as_number(field_name, field_value):
@@ -1617,28 +1662,25 @@ def additional_filters_warning(additional_filters, class_name):
         )
 
 
-def merge_with_precedence(first_dict: dict, second_dict: dict) -> dict:
+def merge_dicts_with_precedence(*dicts: dict) -> dict:
     """
-    Merge two dictionaries with precedence given to keys from the second dictionary.
+    Merge multiple dictionaries with precedence given to keys from later dictionaries.
 
-    This function merges two dictionaries, `first_dict` and `second_dict`, where keys from `second_dict`
-    take precedence in case of conflicts. If both dictionaries contain the same key,
-    the value from `second_dict` will overwrite the value from `first_dict`.
+    This function merges an arbitrary number of dictionaries, where keys from dictionaries later
+    in the argument list take precedence over keys from dictionaries earlier in the list. If all
+    dictionaries contain the same key, the value from the last dictionary with that key will
+    overwrite the values from earlier dictionaries.
 
     Example:
         >>> first_dict = {"key1": "value1", "key2": "value2"}
         >>> second_dict = {"key2": "new_value2", "key3": "value3"}
-        >>> merge_with_precedence(first_dict, second_dict)
-        {'key1': 'value1', 'key2': 'new_value2', 'key3': 'value3'}
+        >>> third_dict = {"key3": "new_value3", "key4": "value4"}
+        >>> merge_dicts_with_precedence(first_dict, second_dict, third_dict)
+        {'key1': 'value1', 'key2': 'new_value2', 'key3': 'new_value3', 'key4': 'value4'}
 
-    Note:
-    - The merge operation uses the ** operator in Python, which combines key-value pairs
-      from each dictionary. Later dictionaries take precedence when there are conflicting keys.
+    - If no dictionaries are provided, the function returns an empty dictionary.
     """
-    return {
-        **(first_dict or {}),
-        **(second_dict or {}),
-    }
+    return {k: v for d in dicts if d for k, v in d.items()}
 
 
 def validate_component_version_compatibility(
@@ -1682,11 +1724,21 @@ def validate_component_version_compatibility(
             )
         return True
 
+    # Feature might have been back-ported e.g. nuclio node selection is supported from
+    # 1.5.20 and 1.6.10 but not in 1.6.9 - therefore we reverse sort to validate against 1.6.x 1st and
+    # then against 1.5.x
     parsed_min_versions.sort(reverse=True)
     for parsed_min_version in parsed_min_versions:
-        if parsed_current_version < parsed_min_version:
+        if (
+            parsed_current_version.major == parsed_min_version.major
+            and parsed_current_version.minor == parsed_min_version.minor
+            and parsed_current_version.patch < parsed_min_version.patch
+        ):
             return False
-    return True
+
+        if parsed_current_version >= parsed_min_version:
+            return True
+    return False
 
 
 def format_alert_summary(
@@ -1698,6 +1750,28 @@ def format_alert_summary(
     return result
 
 
+def is_parquet_file(file_path, format_=None):
+    return (file_path and file_path.endswith((".parquet", ".pq"))) or (
+        format_ == "parquet"
+    )
+
+
+def validate_single_def_handler(function_kind: str, code: str):
+    # The name of MLRun's wrapper is 'handler', which is why the handler function name cannot be 'handler'
+    # it would override MLRun's wrapper
+    if function_kind == "mlrun":
+        # Find all lines that start with "def handler("
+        pattern = re.compile(r"^def handler\(", re.MULTILINE)
+        matches = pattern.findall(code)
+
+        # Only MLRun's wrapper handler (footer) can be in the code
+        if len(matches) > 1:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "The code file contains a function named “handler“, which is reserved. "
+                + "Use a different name for your function."
+            )
+
+
 def _reload(module, max_recursion_depth):
     """Recursively reload modules."""
     if max_recursion_depth <= 0:
@@ -1708,3 +1782,43 @@ def _reload(module, max_recursion_depth):
         attribute = getattr(module, attribute_name)
         if type(attribute) is ModuleType:
             _reload(attribute, max_recursion_depth - 1)
+
+
+def run_with_retry(
+    retry_count: int,
+    func: typing.Callable,
+    retry_on_exceptions: typing.Union[
+        type[Exception],
+        tuple[type[Exception]],
+    ] = None,
+    *args,
+    **kwargs,
+):
+    """
+    Executes a function with retry logic upon encountering specified exceptions.
+
+    :param retry_count: The number of times to retry the function execution.
+    :param func: The function to execute.
+    :param retry_on_exceptions: Exception(s) that trigger a retry. Can be a single exception or a tuple of exceptions.
+    :param args: Positional arguments to pass to the function.
+    :param kwargs: Keyword arguments to pass to the function.
+    :return: The result of the function execution if successful.
+    :raises Exception: Re-raises the last exception encountered after all retries are exhausted.
+    """
+    if retry_on_exceptions is None:
+        retry_on_exceptions = (Exception,)
+    elif isinstance(retry_on_exceptions, list):
+        retry_on_exceptions = tuple(retry_on_exceptions)
+
+    last_exception = None
+    for attempt in range(retry_count + 1):
+        try:
+            return func(*args, **kwargs)
+        except retry_on_exceptions as exc:
+            last_exception = exc
+            logger.warning(
+                f"Attempt {{{attempt}/ {retry_count}}} failed with exception: {exc}",
+            )
+            if attempt == retry_count:
+                raise
+    raise last_exception

mlrun/utils/http.py

@@ -95,7 +95,7 @@ class HTTPSessionWithRetry(requests.Session):
                     total=self.max_retries,
                     backoff_factor=self.retry_backoff_factor,
                     status_forcelist=config.http_retry_defaults.status_codes,
-                    method_whitelist=self._retry_methods,
+                    allowed_methods=self._retry_methods,
                     # we want to retry but not to raise since we do want that last response (to parse details on the
                     # error from response body) we'll handle raising ourselves
                     raise_on_status=False,

mlrun/utils/notifications/notification/__init__.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import enum
-import typing
 
 from mlrun.common.schemas.notification import NotificationKind
 

mlrun/utils/notifications/notification/webhook.py

@@ -60,7 +60,14 @@ class WebhookNotification(NotificationBase):
             request_body["runs"] = runs
 
         if alert:
-            request_body["alert"] = alert.dict()
+            request_body["name"] = alert.name
+            request_body["project"] = alert.project
+            request_body["severity"] = alert.severity
+            if alert.summary:
+                request_body["summary"] = mlrun.utils.helpers.format_alert_summary(
+                    alert, event_data
+                )
+
             if event_data:
                 request_body["value"] = event_data.value_dict
                 request_body["id"] = event_data.entity.ids[0]

mlrun/utils/notifications/notification_pusher.py

@@ -484,7 +484,7 @@ class NotificationPusher(_NotificationPusherBase):
     def _get_workflow_manifest(
         workflow_id: str,
     ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf)
+        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
 
         # arbitrary timeout of 5 seconds, the workflow should be done by now
         kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)

mlrun/utils/v3io_clients.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from v3io.dataplane import Client as V3IOClient
-from v3io_frames import Client as get_client
+from v3io_frames import Client as V3IOFramesClient
 from v3io_frames.client import ClientBase
 
 _v3io_clients: dict[frozenset, V3IOClient] = {}
@@ -24,7 +24,7 @@ def get_frames_client(**kwargs) -> ClientBase:
     global _frames_clients
     kw_set = frozenset(kwargs.items())
     if kw_set not in _frames_clients:
-        _frames_clients[kw_set] = get_client(**kwargs)
+        _frames_clients[kw_set] = V3IOFramesClient(**kwargs)
 
     return _frames_clients[kw_set]
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "71818c39b83a7e8c396aa2c37f1cd4f3e192dd2f",
-  "version": "1.7.0-rc28"
+  "git_commit": "e1c65a391e9fcd99886541d4223d8603402180ce",
+  "version": "1.7.0-rc55"
 }