mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

mlrun/serving/states.py

@@ -12,7 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__all__ = ["TaskStep", "RouterStep", "RootFlowStep", "ErrorStep"]
+__all__ = [
+    "TaskStep",
+    "RouterStep",
+    "RootFlowStep",
+    "ErrorStep",
+    "MonitoringApplicationStep",
+]
 
 import os
 import pathlib
@@ -55,6 +61,7 @@ class StepKinds:
     choice = "choice"
     root = "root"
     error_step = "error_step"
+    monitoring_application = "monitoring_application"
 
 
 _task_step_fields = [
@@ -485,13 +492,15 @@ class TaskStep(BaseStep):
                 class_args[key] = arg
         class_args.update(extra_kwargs)
 
-        # add common args (name, context, ..) only if target class can accept them
-        argspec = getfullargspec(class_object)
-        for key in ["name", "context", "input_path", "result_path", "full_event"]:
-            if argspec.varkw or key in argspec.args:
-                class_args[key] = getattr(self, key)
-        if argspec.varkw or "graph_step" in argspec.args:
-            class_args["graph_step"] = self
+        if not isinstance(self, MonitoringApplicationStep):
+            # add common args (name, context, ..) only if target class can accept them
+            argspec = getfullargspec(class_object)
+
+            for key in ["name", "context", "input_path", "result_path", "full_event"]:
+                if argspec.varkw or key in argspec.args:
+                    class_args[key] = getattr(self, key)
+            if argspec.varkw or "graph_step" in argspec.args:
+                class_args["graph_step"] = self
         return class_args
 
     def get_step_class_object(self, namespace):
@@ -582,6 +591,39 @@ class TaskStep(BaseStep):
         return event
 
 
+class MonitoringApplicationStep(TaskStep):
+    """monitoring application execution step, runs users class code"""
+
+    kind = "monitoring_application"
+    _default_class = ""
+
+    def __init__(
+        self,
+        class_name: Union[str, type] = None,
+        class_args: dict = None,
+        handler: str = None,
+        name: str = None,
+        after: list = None,
+        full_event: bool = None,
+        function: str = None,
+        responder: bool = None,
+        input_path: str = None,
+        result_path: str = None,
+    ):
+        super().__init__(
+            class_name=class_name,
+            class_args=class_args,
+            handler=handler,
+            name=name,
+            after=after,
+            full_event=full_event,
+            function=function,
+            responder=responder,
+            input_path=input_path,
+            result_path=result_path,
+        )
+
+
 class ErrorStep(TaskStep):
     """error execution step, runs a class or handler"""
 
@@ -790,6 +832,35 @@ class QueueStep(BaseStep):
     def async_object(self):
         return self._async_object
 
+    def to(
+        self,
+        class_name: Union[str, StepToDict] = None,
+        name: str = None,
+        handler: str = None,
+        graph_shape: str = None,
+        function: str = None,
+        full_event: bool = None,
+        input_path: str = None,
+        result_path: str = None,
+        **class_args,
+    ):
+        if not function:
+            name = get_name(name, class_name)
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"step '{name}' must specify a function, because it follows a queue step"
+            )
+        return super().to(
+            class_name,
+            name,
+            handler,
+            graph_shape,
+            function,
+            full_event,
+            input_path,
+            result_path,
+            **class_args,
+        )
+
     def run(self, event, *args, **kwargs):
         data = event.body
         if not data:
@@ -1323,6 +1394,7 @@ classes_map = {
     "flow": FlowStep,
     "queue": QueueStep,
     "error_step": ErrorStep,
+    "monitoring_application": MonitoringApplicationStep,
 }
 
 

mlrun/serving/utils.py

@@ -46,6 +46,15 @@ def _update_result_body(result_path, event_body, result):
 class StepToDict:
     """auto serialization of graph steps to a python dictionary"""
 
+    meta_keys = [
+        "context",
+        "name",
+        "input_path",
+        "result_path",
+        "full_event",
+        "kwargs",
+    ]
+
     def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         """convert the step object to a python dictionary"""
         fields = fields or getattr(self, "_dict_fields", None)
@@ -54,24 +63,16 @@ class StepToDict:
         if exclude:
             fields = [field for field in fields if field not in exclude]
 
-        meta_keys = [
-            "context",
-            "name",
-            "input_path",
-            "result_path",
-            "full_event",
-            "kwargs",
-        ]
         args = {
             key: getattr(self, key)
             for key in fields
-            if getattr(self, key, None) is not None and key not in meta_keys
+            if getattr(self, key, None) is not None and key not in self.meta_keys
         }
         # add storey kwargs or extra kwargs
         if "kwargs" in fields and (hasattr(self, "kwargs") or hasattr(self, "_kwargs")):
             kwargs = getattr(self, "kwargs", {}) or getattr(self, "_kwargs", {})
             for key, value in kwargs.items():
-                if key not in meta_keys:
+                if key not in self.meta_keys:
                     args[key] = value
 
         mod_name = self.__class__.__module__
@@ -80,7 +81,9 @@ class StepToDict:
             class_path = f"{mod_name}.{class_path}"
         struct = {
             "class_name": class_path,
-            "name": self.name or self.__class__.__name__,
+            "name": self.name
+            if hasattr(self, "name") and self.name
+            else self.__class__.__name__,
             "class_args": args,
         }
         if hasattr(self, "_STEP_KIND"):
@@ -94,6 +97,11 @@ class StepToDict:
         return struct
 
 
+class MonitoringApplicationToDict(StepToDict):
+    _STEP_KIND = "monitoring_application"
+    meta_keys = []
+
+
 class RouterToDict(StepToDict):
     _STEP_KIND = "router"
 

mlrun/serving/v2_serving.py

@@ -528,7 +528,13 @@ def _init_endpoint_record(
         return None
 
     # Generating version model value based on the model name and model version
-    if model.version:
+    if model.model_path and model.model_path.startswith("store://"):
+        # Enrich the model server with the model artifact metadata
+        model.get_model()
+        if not model.version:
+            # Enrich the model version with the model artifact tag
+            model.version = model.model_spec.tag
+        model.labels = model.model_spec.labels
         versioned_model_name = f"{model.name}:{model.version}"
     else:
         versioned_model_name = f"{model.name}:latest"
@@ -538,48 +544,69 @@ def _init_endpoint_record(
         function_uri=graph_server.function_uri, versioned_model=versioned_model_name
     ).uid
 
-    # If model endpoint object was found in DB, skip the creation process.
     try:
-        mlrun.get_run_db().get_model_endpoint(project=project, endpoint_id=uid)
-
+        model_ep = mlrun.get_run_db().get_model_endpoint(
+            project=project, endpoint_id=uid
+        )
     except mlrun.errors.MLRunNotFoundError:
-        logger.info("Creating a new model endpoint record", endpoint_id=uid)
-
-        try:
-            model_endpoint = mlrun.common.schemas.ModelEndpoint(
-                metadata=mlrun.common.schemas.ModelEndpointMetadata(
-                    project=project, labels=model.labels, uid=uid
-                ),
-                spec=mlrun.common.schemas.ModelEndpointSpec(
-                    function_uri=graph_server.function_uri,
-                    model=versioned_model_name,
-                    model_class=model.__class__.__name__,
-                    model_uri=model.model_path,
-                    stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
-                        project=project, kind="stream"
-                    ),
-                    active=True,
-                    monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-                    if model.context.server.track_models
-                    else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled,
-                ),
-                status=mlrun.common.schemas.ModelEndpointStatus(
-                    endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+        model_ep = None
+    except mlrun.errors.MLRunBadRequestError as err:
+        logger.debug(
+            f"Cant reach to model endpoints store, due to  : {err}",
+        )
+        return
+
+    if model.context.server.track_models and not model_ep:
+        logger.debug("Creating a new model endpoint record", endpoint_id=uid)
+        model_endpoint = mlrun.common.schemas.ModelEndpoint(
+            metadata=mlrun.common.schemas.ModelEndpointMetadata(
+                project=project, labels=model.labels, uid=uid
+            ),
+            spec=mlrun.common.schemas.ModelEndpointSpec(
+                function_uri=graph_server.function_uri,
+                model=versioned_model_name,
+                model_class=model.__class__.__name__,
+                model_uri=model.model_path,
+                stream_path=config.model_endpoint_monitoring.store_prefixes.default.format(
+                    project=project, kind="stream"
                 ),
-            )
-
-            db = mlrun.get_run_db()
-
-            db.create_model_endpoint(
-                project=project,
-                endpoint_id=uid,
-                model_endpoint=model_endpoint.dict(),
-            )
+                active=True,
+                monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
+            ),
+            status=mlrun.common.schemas.ModelEndpointStatus(
+                endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP
+            ),
+        )
 
-        except Exception as e:
-            logger.error("Failed to create endpoint record", exc=err_to_str(e))
+        db = mlrun.get_run_db()
+        db.create_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            model_endpoint=model_endpoint.dict(),
+        )
 
-    except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
+    elif (
+        model_ep
+        and (
+            model_ep.spec.monitoring_mode
+            == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+        )
+        != model.context.server.track_models
+    ):
+        monitoring_mode = (
+            mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+            if model.context.server.track_models
+            else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
+        )
+        db = mlrun.get_run_db()
+        db.patch_model_endpoint(
+            project=project,
+            endpoint_id=uid,
+            attributes={"monitoring_mode": monitoring_mode},
+        )
+        logger.debug(
+            f"Updating model endpoint monitoring_mode to {monitoring_mode}",
+            endpoint_id=uid,
+        )
 
     return uid

mlrun/utils/helpers.py

@@ -26,7 +26,7 @@ import sys
 import typing
 import warnings
 from datetime import datetime, timezone
-from importlib import import_module
+from importlib import import_module, reload
 from os import path
 from types import ModuleType
 from typing import Any, Optional
@@ -109,10 +109,13 @@ def get_artifact_target(item: dict, project=None):
     db_key = item["spec"].get("db_key")
     project_str = project or item["metadata"].get("project")
     tree = item["metadata"].get("tree")
+    tag = item["metadata"].get("tag")
 
     kind = item.get("kind")
     if kind in ["dataset", "model", "artifact"] and db_key:
         target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
+        if tag:
+            target = f"{target}:{tag}"
         if tree:
             target = f"{target}@{tree}"
         return target
@@ -149,7 +152,7 @@ if is_ipython and config.nest_asyncio_enabled in ["1", "True"]:
     nest_asyncio.apply()
 
 
-class run_keys:
+class RunKeys:
     input_path = "input_path"
     output_path = "output_path"
     inputs = "inputs"
@@ -160,6 +163,10 @@ class run_keys:
     secrets = "secret_sources"
 
 
+# for Backward compatibility
+run_keys = RunKeys
+
+
 def verify_field_regex(
     field_name,
     field_value,
@@ -659,7 +666,7 @@ def parse_artifact_uri(uri, default_project=""):
         [3] = tag
         [4] = tree
     """
-    uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?(:(?P<tag>.*?))?(@(?P<tree>.*))?$"
+    uri_pattern = mlrun.utils.regex.artifact_uri_pattern
     match = re.match(uri_pattern, uri)
     if not match:
         raise ValueError(
@@ -674,6 +681,8 @@ def parse_artifact_uri(uri, default_project=""):
             raise ValueError(
                 f"illegal store path '{uri}', iteration must be integer value"
             )
+    else:
+        iteration = 0
     return (
         group_dict["project"] or default_project,
         group_dict["key"],
@@ -1019,16 +1028,35 @@ def create_class(pkg_class: str):
     return class_
 
 
-def create_function(pkg_func: str):
+def create_function(pkg_func: str, reload_modules: bool = False):
     """Create a function from a package.module.function string
 
     :param pkg_func:  full function location,
                       e.g. "sklearn.feature_selection.f_classif"
+    :param reload_modules: reload the function again.
     """
     splits = pkg_func.split(".")
     pkg_module = ".".join(splits[:-1])
     cb_fname = splits[-1]
     pkg_module = __import__(pkg_module, fromlist=[cb_fname])
+
+    if reload_modules:
+        # Even though the function appears in the modules list, we need to reload
+        # the code again because it may have changed
+        try:
+            logger.debug("Reloading module", module=pkg_func)
+            _reload(
+                pkg_module,
+                max_recursion_depth=mlrun.mlconf.function.spec.reload_max_recursion_depth,
+            )
+        except Exception as exc:
+            logger.warning(
+                "Failed to reload module. Not all associated modules can be reloaded, import them manually."
+                "Or, with Jupyter, restart the Python kernel.",
+                module=pkg_func,
+                err=mlrun.errors.err_to_str(exc),
+            )
+
     function_ = getattr(pkg_module, cb_fname)
     return function_
 
@@ -1086,8 +1114,14 @@ def get_class(class_name, namespace=None):
     return class_object
 
 
-def get_function(function, namespace):
-    """return function callable object from function name string"""
+def get_function(function, namespaces, reload_modules: bool = False):
+    """Return function callable object from function name string
+
+    :param function: path to the function ([class_name::]function)
+    :param namespaces: one or list of namespaces/modules to search the function in
+    :param reload_modules: reload the function again
+    :return: function handler (callable)
+    """
     if callable(function):
         return function
 
@@ -1096,12 +1130,12 @@ def get_function(function, namespace):
         if not function.endswith(")"):
             raise ValueError('function expression must start with "(" and end with ")"')
         return eval("lambda event: " + function[1:-1], {}, {})
-    function_object = _search_in_namespaces(function, namespace)
+    function_object = _search_in_namespaces(function, namespaces)
     if function_object is not None:
         return function_object
 
     try:
-        function_object = create_function(function)
+        function_object = create_function(function, reload_modules)
     except (ImportError, ValueError) as exc:
         raise ImportError(
             f"state/function init failed, handler '{function}' not found"
@@ -1110,19 +1144,24 @@ def get_function(function, namespace):
 
 
 def get_handler_extended(
-    handler_path: str, context=None, class_args: dict = None, namespaces=None
+    handler_path: str,
+    context=None,
+    class_args: dict = None,
+    namespaces=None,
+    reload_modules: bool = False,
 ):
-    """get function handler from [class_name::]handler string
+    """Get function handler from [class_name::]handler string
 
     :param handler_path:  path to the function ([class_name::]handler)
     :param context:       MLRun function/job client context
     :param class_args:    optional dict of class init kwargs
     :param namespaces:    one or list of namespaces/modules to search the handler in
+    :param reload_modules: reload the function again
     :return: function handler (callable)
     """
     class_args = class_args or {}
     if "::" not in handler_path:
-        return get_function(handler_path, namespaces)
+        return get_function(handler_path, namespaces, reload_modules)
 
     splitted = handler_path.split("::")
     class_path = splitted[0].strip()
@@ -1227,6 +1266,10 @@ def _fill_project_path_template(artifact_path, project):
     return artifact_path
 
 
+def to_non_empty_values_dict(input_dict: dict) -> dict:
+    return {key: value for key, value in input_dict.items() if value}
+
+
 def str_to_timestamp(time_str: str, now_time: Timestamp = None):
     """convert fixed/relative time string to Pandas Timestamp
 
@@ -1284,6 +1327,7 @@ def format_run(run: PipelineRun, with_project=False) -> dict:
         "scheduled_at",
         "finished_at",
         "description",
+        "experiment_id",
     ]
 
     if with_project:
@@ -1573,6 +1617,30 @@ def additional_filters_warning(additional_filters, class_name):
         )
 
 
+def merge_with_precedence(first_dict: dict, second_dict: dict) -> dict:
+    """
+    Merge two dictionaries with precedence given to keys from the second dictionary.
+
+    This function merges two dictionaries, `first_dict` and `second_dict`, where keys from `second_dict`
+    take precedence in case of conflicts. If both dictionaries contain the same key,
+    the value from `second_dict` will overwrite the value from `first_dict`.
+
+    Example:
+        >>> first_dict = {"key1": "value1", "key2": "value2"}
+        >>> second_dict = {"key2": "new_value2", "key3": "value3"}
+        >>> merge_with_precedence(first_dict, second_dict)
+        {'key1': 'value1', 'key2': 'new_value2', 'key3': 'value3'}
+
+    Note:
+    - The merge operation uses the ** operator in Python, which combines key-value pairs
+      from each dictionary. Later dictionaries take precedence when there are conflicting keys.
+    """
+    return {
+        **(first_dict or {}),
+        **(second_dict or {}),
+    }
+
+
 def validate_component_version_compatibility(
     component_name: typing.Literal["iguazio", "nuclio"], *min_versions: str
 ):
@@ -1628,3 +1696,15 @@ def format_alert_summary(
     result = result.replace("{{name}}", alert.name)
     result = result.replace("{{entity}}", event_data.entity.ids[0])
     return result
+
+
+def _reload(module, max_recursion_depth):
+    """Recursively reload modules."""
+    if max_recursion_depth <= 0:
+        return
+
+    reload(module)
+    for attribute_name in dir(module):
+        attribute = getattr(module, attribute_name)
+        if type(attribute) is ModuleType:
+            _reload(attribute, max_recursion_depth - 1)

mlrun/utils/logger.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import logging
+import os
 import typing
 from enum import Enum
+from functools import cached_property
 from sys import stdout
 from traceback import format_exception
 from typing import IO, Optional, Union
@@ -92,6 +94,16 @@ class HumanReadableFormatter(_BaseFormatter):
 
 
 class HumanReadableExtendedFormatter(HumanReadableFormatter):
+    _colors = {
+        logging.NOTSET: "",
+        logging.DEBUG: "\x1b[34m",
+        logging.INFO: "\x1b[36m",
+        logging.WARNING: "\x1b[33m",
+        logging.ERROR: "\x1b[0;31m",
+        logging.CRITICAL: "\x1b[1;31m",
+    }
+    _color_reset = "\x1b[0m"
+
     def format(self, record) -> str:
         more = ""
         record_with = self._record_with(record)
@@ -113,12 +125,34 @@ class HumanReadableExtendedFormatter(HumanReadableFormatter):
                 [f"{key}: {_format_value(val)}" for key, val in record_with.items()]
             )
         return (
-            "> "
+            f"{self._get_message_color(record.levelno)}> "
             f"{self.formatTime(record, self.datefmt)} "
             f"[{record.name}:{record.levelname.lower()}] "
-            f"{record.getMessage()}{more}"
+            f"{record.getMessage()}{more}{self._get_color_reset()}"
         )
 
+    def _get_color_reset(self):
+        if not self._have_color_support:
+            return ""
+
+        return self._color_reset
+
+    def _get_message_color(self, levelno):
+        if not self._have_color_support:
+            return ""
+
+        return self._colors[levelno]
+
+    @cached_property
+    def _have_color_support(self):
+        if os.environ.get("PYCHARM_HOSTED"):
+            return True
+        if os.environ.get("NO_COLOR"):
+            return False
+        if os.environ.get("CLICOLOR_FORCE"):
+            return True
+        return stdout.isatty()
+
 
 class Logger:
     def __init__(

mlrun/utils/notifications/notification/base.py

@@ -28,6 +28,10 @@ class NotificationBase:
         self.name = name
         self.params = params or {}
 
+    @classmethod
+    def validate_params(cls, params):
+        pass
+
     @property
     def active(self) -> bool:
         return True
@@ -69,16 +73,27 @@ class NotificationBase:
         if custom_html:
             return custom_html
 
-        if self.name:
-            message = f"{self.name}: {message}"
-
         if alert:
             if not event_data:
                 return f"[{severity}] {message}"
-            return (
-                f"[{severity}] {message} for project {alert.project} "
-                f"UID {event_data.entity.ids[0]}. Values {event_data.value_dict}"
-            )
+
+            html = f"<h3>[{severity}] {message}</h3>"
+            html += f"<br>{alert.name} alert has occurred<br>"
+            html += f"<br><h4>Project:</h4>{alert.project}<br>"
+            html += f"<br><h4>ID:</h4>{event_data.entity.ids[0]}<br>"
+            html += f"<br><h4>Summary:</h4>{mlrun.utils.helpers.format_alert_summary(alert, event_data)}<br>"
+
+            if event_data.value_dict:
+                html += "<br><h4>Event data:</h4>"
+                for key, value in event_data.value_dict.items():
+                    html += f"{key}: {value}<br>"
+
+            overview_type, url = self._get_overview_type_and_url(alert, event_data)
+            html += f"<br><h4>Overview:</h4><a href={url}>{overview_type}</a>"
+            return html
+
+        if self.name:
+            message = f"{self.name}: {message}"
 
         if not runs:
             return f"[{severity}] {message}"
@@ -90,3 +105,24 @@ class NotificationBase:
         html += "<br>click the hyper links below to see detailed results<br>"
         html += runs.show(display=False, short=True)
         return html
+
+    def _get_overview_type_and_url(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> (str, str):
+        if (
+            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
+        ):  # JOB entity
+            uid = event_data.value_dict.get("uid")
+            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
+            overview_type = "Job overview"
+        else:  # MODEL entity
+            model_name = event_data.value_dict.get("model")
+            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
+            url = mlrun.utils.helpers.get_model_endpoint_url(
+                alert.project, model_name, model_endpoint_id
+            )
+            overview_type = "Model endpoint"
+
+        return overview_type, url