mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/utils/notifications/notification/webhook.py

@@ -28,6 +28,12 @@ class WebhookNotification(NotificationBase):
     API/Client notification for sending run statuses in a http request
     """
 
+    @classmethod
+    def validate_params(cls, params):
+        url = params.get("url", None)
+        if not url:
+            raise ValueError("Parameter 'url' is required for WebhookNotification")
+
     async def push(
         self,
         message: str,
@@ -36,6 +42,8 @@ class WebhookNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()
@@ -46,14 +54,29 @@ class WebhookNotification(NotificationBase):
         request_body = {
             "message": message,
             "severity": severity,
-            "runs": runs,
         }
 
+        if runs:
+            request_body["runs"] = runs
+
+        if alert:
+            request_body["name"] = alert.name
+            request_body["project"] = alert.project
+            request_body["severity"] = alert.severity
+            if alert.summary:
+                request_body["summary"] = mlrun.utils.helpers.format_alert_summary(
+                    alert, event_data
+                )
+
+            if event_data:
+                request_body["value"] = event_data.value_dict
+                request_body["id"] = event_data.entity.ids[0]
+
         if custom_html:
             request_body["custom_html"] = custom_html
 
         if override_body:
-            request_body = override_body
+            request_body = self._serialize_runs_in_request_body(override_body, runs)
 
         # Specify the `verify_ssl` parameter value only for HTTPS urls.
         # The `ClientSession` allows using `ssl=None` for the default SSL check,
@@ -67,3 +90,37 @@ class WebhookNotification(NotificationBase):
                 url, headers=headers, json=request_body, ssl=verify_ssl
             )
             response.raise_for_status()
+
+    @staticmethod
+    def _serialize_runs_in_request_body(override_body, runs):
+        str_parsed_runs = ""
+        runs = runs or []
+
+        def parse_runs():
+            parsed_runs = []
+            for run in runs:
+                if hasattr(run, "to_dict"):
+                    run = run.to_dict()
+                if isinstance(run, dict):
+                    parsed_run = {
+                        "project": run["metadata"]["project"],
+                        "name": run["metadata"]["name"],
+                        "host": run["metadata"]["labels"]["host"],
+                        "status": {"state": run["status"]["state"]},
+                    }
+                    if run["status"].get("error", None):
+                        parsed_run["status"]["error"] = run["status"]["error"]
+                    elif run["status"].get("results", None):
+                        parsed_run["status"]["results"] = run["status"]["results"]
+                    parsed_runs.append(parsed_run)
+            return str(parsed_runs)
+
+        if isinstance(override_body, dict):
+            for key, value in override_body.items():
+                if "{{ runs }}" or "{{runs}}" in value:
+                    if not str_parsed_runs:
+                        str_parsed_runs = parse_runs()
+                    override_body[key] = value.replace(
+                        "{{ runs }}", str_parsed_runs
+                    ).replace("{{runs}}", str_parsed_runs)
+        return override_body

mlrun/utils/notifications/notification_pusher.py

@@ -15,10 +15,17 @@
 import asyncio
 import datetime
 import os
+import re
 import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
+import mlrun_pipelines.common.ops
+import mlrun_pipelines.models
+import mlrun_pipelines.utils
+
+import mlrun.common.constants as mlrun_constants
+import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.config
 import mlrun.db.base
@@ -32,7 +39,7 @@ from mlrun.utils.condition_evaluator import evaluate_condition_in_separate_proce
 from .notification import NotificationBase, NotificationTypes
 
 
-class _NotificationPusherBase(object):
+class _NotificationPusherBase:
     def _push(
         self, sync_push_callback: typing.Callable, async_push_callback: typing.Callable
     ):
@@ -95,15 +102,11 @@ class NotificationPusher(_NotificationPusherBase):
 
     def __init__(self, runs: typing.Union[mlrun.lists.RunList, list]):
         self._runs = runs
-        self._sync_notifications: typing.List[
-            typing.Tuple[
-                NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
-            ]
+        self._sync_notifications: list[
+            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
         ] = []
-        self._async_notifications: typing.List[
-            typing.Tuple[
-                NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
-            ]
+        self._async_notifications: list[
+            tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
         ] = []
 
         for run in self._runs:
@@ -237,25 +240,12 @@ class NotificationPusher(_NotificationPusherBase):
         resource = "Run"
         runs = [run.to_dict()]
 
-        if "workflow" in run.metadata.labels:
-            resource = "Workflow"
+        if mlrun_constants.MLRunInternalLabels.workflow in run.metadata.labels:
+            resource = mlrun_constants.MLRunInternalLabels.workflow
             custom_message = (
                 f" (workflow: {run.metadata.labels['workflow']}){custom_message}"
             )
-            db = mlrun.get_run_db()
-
-            workflow_id = run.status.results.get("workflow_id", None)
-            if workflow_id:
-                workflow_runs = db.list_runs(
-                    project=run.metadata.project,
-                    labels=f"workflow={workflow_id}",
-                )
-                logger.debug(
-                    "Found workflow runs, extending notification runs",
-                    workflow_id=workflow_id,
-                    workflow_runs_amount=len(workflow_runs),
-                )
-                runs.extend(workflow_runs)
+            runs.extend(self.get_workflow_steps(run))
 
         message = (
             self.messages.get(run.state(), "").format(resource=resource)
@@ -399,9 +389,134 @@ class NotificationPusher(_NotificationPusherBase):
             mask_params=False,
         )
 
+    def get_workflow_steps(self, run: mlrun.model.RunObject) -> list:
+        steps = []
+        db = mlrun.get_run_db()
+
+        def _add_run_step(_step: mlrun_pipelines.models.PipelineStep):
+            try:
+                _run = db.list_runs(
+                    project=run.metadata.project,
+                    labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
+                )[0]
+            except IndexError:
+                _run = {
+                    "metadata": {
+                        "name": _step.display_name,
+                        "project": run.metadata.project,
+                    },
+                }
+            _run["step_kind"] = _step.step_type
+            if _step.skipped:
+                _run.setdefault("status", {})["state"] = (
+                    mlrun.common.runtimes.constants.RunStates.skipped
+                )
+            steps.append(_run)
+
+        def _add_deploy_function_step(_step: mlrun_pipelines.models.PipelineStep):
+            project, name, hash_key = self._extract_function_uri(
+                _step.get_annotation("mlrun/function-uri")
+            )
+            if name:
+                try:
+                    function = db.get_function(
+                        project=project, name=name, hash_key=hash_key
+                    )
+                except mlrun.errors.MLRunNotFoundError:
+                    # If the function is not found (if build failed for example), we will create a dummy
+                    # function object for the notification to display the function name
+                    function = {
+                        "metadata": {
+                            "name": name,
+                            "project": project,
+                            "hash_key": hash_key,
+                        },
+                    }
+                pod_phase = _step.phase
+                if _step.skipped:
+                    state = mlrun.common.schemas.FunctionState.skipped
+                else:
+                    state = mlrun.common.runtimes.constants.PodPhases.pod_phase_to_run_state(
+                        pod_phase
+                    )
+                function["status"] = {"state": state}
+                if isinstance(function["metadata"].get("updated"), datetime.datetime):
+                    function["metadata"]["updated"] = function["metadata"][
+                        "updated"
+                    ].isoformat()
+                function["step_kind"] = _step.step_type
+                steps.append(function)
+
+        step_methods = {
+            mlrun_pipelines.common.ops.PipelineRunType.run: _add_run_step,
+            mlrun_pipelines.common.ops.PipelineRunType.build: _add_deploy_function_step,
+            mlrun_pipelines.common.ops.PipelineRunType.deploy: _add_deploy_function_step,
+        }
+
+        workflow_id = run.status.results.get("workflow_id", None)
+        if not workflow_id:
+            return steps
+
+        workflow_manifest = self._get_workflow_manifest(workflow_id)
+        if not workflow_manifest:
+            return steps
+
+        try:
+            for step in workflow_manifest.get_steps():
+                step_method = step_methods.get(step.step_type)
+                if step_method:
+                    step_method(step)
+            return steps
+        except Exception:
+            # If we fail to read the pipeline steps, we will return the list of runs that have the same workflow id
+            logger.warning(
+                "Failed to extract workflow steps from workflow manifest, "
+                "returning all runs with the workflow id label",
+                workflow_id=workflow_id,
+                traceback=traceback.format_exc(),
+            )
+            return db.list_runs(
+                project=run.metadata.project,
+                labels=f"workflow={workflow_id}",
+            )
+
+    @staticmethod
+    def _get_workflow_manifest(
+        workflow_id: str,
+    ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
+        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf.kfp_url)
+
+        # arbitrary timeout of 5 seconds, the workflow should be done by now
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        if not kfp_run:
+            return None
+
+        kfp_run = mlrun_pipelines.models.PipelineRun(kfp_run)
+        return kfp_run.workflow_manifest()
+
+    def _extract_function_uri(self, function_uri: str) -> tuple[str, str, str]:
+        """
+        Extract the project, name, and hash key from a function uri.
+        Examples:
+            - "project/name@hash_key" returns project, name, hash_key
+            - "project/name returns" project, name, ""
+        """
+        project, name, hash_key = None, None, None
+        hashed_pattern = r"^(.+)/(.+)@(.+)$"
+        pattern = r"^(.+)/(.+)$"
+        match = re.match(hashed_pattern, function_uri)
+        if match:
+            project, name, hash_key = match.groups()
+        else:
+            match = re.match(pattern, function_uri)
+            if match:
+                project, name = match.groups()
+                hash_key = ""
+        return project, name, hash_key
+
 
 class CustomNotificationPusher(_NotificationPusherBase):
-    def __init__(self, notification_types: typing.List[str] = None):
+    def __init__(self, notification_types: list[str] = None):
         notifications = {
             notification_type: NotificationTypes(notification_type).get_notification()()
             for notification_type in notification_types
@@ -417,6 +532,12 @@ class CustomNotificationPusher(_NotificationPusherBase):
             if notification.is_async
         }
 
+    @property
+    def notifications(self):
+        notifications = self._sync_notifications.copy()
+        notifications.update(self._async_notifications)
+        return notifications
+
     def push(
         self,
         message: str,
@@ -446,7 +567,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
     def add_notification(
         self,
         notification_type: str,
-        params: typing.Dict[str, str] = None,
+        params: dict[str, str] = None,
     ):
         if notification_type in self._async_notifications:
             self._async_notifications[notification_type].load_notification(params)
@@ -471,9 +592,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
         else:
             logger.warning(f"No notification of type {notification_type} in project")
 
-    def edit_notification(
-        self, notification_type: str, params: typing.Dict[str, str] = None
-    ):
+    def edit_notification(self, notification_type: str, params: dict[str, str] = None):
         self.remove_notification(notification_type)
         self.add_notification(notification_type, params)
 

mlrun/utils/regex.py

@@ -92,3 +92,12 @@ artifact_key = [r"[^\/\\]+$"]
 # must be alphanumeric or _
 # max 256 length
 v3io_stream_consumer_group = [r"^(?!_)[a-zA-Z0-9_]{1,256}$"]
+
+# URI patterns
+run_uri_pattern = r"^(?P<project>.*)@(?P<uid>.*)\#(?P<iteration>.*?)(:(?P<tag>.*))?$"
+
+artifact_uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?(:(?P<tag>.*?))?(@(?P<tree>.*))?$"
+
+artifact_producer_uri_pattern = (
+    r"^((?P<project>.*)/)?(?P<uid>.*?)(\-(?P<iteration>.*?))?$"
+)

mlrun/utils/retryer.py

@@ -0,0 +1,208 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import time
+
+import mlrun.errors
+
+
+def create_linear_backoff(base=2, coefficient=2, stop_value=120):
+    """
+    Create a generator of linear backoff. Check out usage example in test_helpers.py
+    """
+    x = 0
+    comparison = min if coefficient >= 0 else max
+
+    while True:
+        next_value = comparison(base + x * coefficient, stop_value)
+        yield next_value
+        x += 1
+
+
+def create_step_backoff(steps=None):
+    """
+    Create a generator of steps backoff.
+    Example: steps = [[2, 5], [20, 10], [120, None]] will produce a generator in which the first 5
+    values will be 2, the next 10 values will be 20 and the rest will be 120.
+    :param steps: a list of lists [step_value, number_of_iteration_in_this_step]
+    """
+    steps = steps if steps is not None else [[2, 10], [10, 10], [120, None]]
+    steps = iter(steps)
+
+    # Get first step
+    step = next(steps)
+    while True:
+        current_step_value, current_step_remain = step
+        if current_step_remain == 0:
+            # No more in this step, moving on
+            step = next(steps)
+        elif current_step_remain is None:
+            # We are in the last step, staying here forever
+            yield current_step_value
+        elif current_step_remain > 0:
+            # Still more remains in this step, just reduce the remaining number
+            step[1] -= 1
+            yield current_step_value
+
+
+def create_exponential_backoff(base=2, max_value=120, scale_factor=1):
+    """
+    Create a generator of exponential backoff. Check out usage example in test_helpers.py
+    :param base: exponent base
+    :param max_value: max limit on the result
+    :param scale_factor: factor to be used as linear scaling coefficient
+    """
+    exponent = 1
+    while True:
+        # This "complex" implementation (unlike the one in linear backoff) is to avoid exponent growing too fast and
+        # risking going behind max_int
+        next_value = scale_factor * (base**exponent)
+        if next_value < max_value:
+            exponent += 1
+            yield next_value
+        else:
+            yield max_value
+
+
+class Retryer:
+    def __init__(self, backoff, timeout, logger, verbose, function, *args, **kwargs):
+        """
+        Initialize function retryer with given *args and **kwargs.
+        Tries to run it until success or timeout reached (timeout is optional)
+        :param backoff: can either be a:
+                - number (int / float) that will be used as interval.
+                - generator of waiting intervals. (support next())
+        :param timeout: pass None if timeout is not wanted, number of seconds if it is
+        :param logger: a logger so we can log the failures
+        :param verbose: whether to log the failure on each retry
+        :param _function: function to run
+        :param args: functions args
+        :param kwargs: functions kwargs
+        """
+        self.backoff = backoff
+        self.timeout = timeout
+        self.logger = logger
+        self.verbose = verbose
+        self.function = function
+        self.args = args
+        self.kwargs = kwargs
+        self.start_time = None
+        self.last_exception = None
+        self.first_interval = None
+
+    def run(self):
+        self._prepare()
+        while not self._timeout_exceeded():
+            next_interval = self.first_interval or next(self.backoff)
+            result, exc, retry = self._perform_call(next_interval)
+            if retry:
+                time.sleep(next_interval)
+            elif not exc:
+                return result
+            else:
+                break
+
+        self._raise_last_exception()
+
+    def _prepare(self):
+        self.start_time = time.monotonic()
+        self.last_exception = None
+
+        # Check if backoff is just a simple interval
+        if isinstance(self.backoff, int) or isinstance(self.backoff, float):
+            self.backoff = create_linear_backoff(base=self.backoff, coefficient=0)
+
+        self.first_interval = next(self.backoff)
+        if self.timeout and self.timeout <= self.first_interval:
+            self.logger.warning(
+                f"Timeout ({self.timeout}) must be higher than backoff ({self.first_interval})."
+                f" Set timeout to be higher than backoff."
+            )
+
+    def _perform_call(self, next_interval):
+        try:
+            result = self.function(*self.args, **self.kwargs)
+            return result, None, False
+        except mlrun.errors.MLRunFatalFailureError as exc:
+            raise exc.original_exception
+        except Exception as exc:
+            self.last_exception = exc
+            return (
+                None,
+                self.last_exception,
+                self._assert_failure_timeout(next_interval, exc),
+            )
+
+    def _assert_failure_timeout(self, next_interval, exc):
+        self.last_exception = exc
+
+        # If next interval is within allowed time period - wait on interval, abort otherwise
+        if not self._timeout_exceeded(next_interval):
+            if self.logger is not None and self.verbose:
+                self.logger.debug(
+                    f"Operation not yet successful, Retrying in {next_interval} seconds."
+                    f" exc: {mlrun.errors.err_to_str(exc)}"
+                )
+            return True
+        else:
+            return False
+
+    def _raise_last_exception(self):
+        if self.logger is not None:
+            self.logger.warning(
+                f"Operation did not complete on time. last exception: {self.last_exception}"
+            )
+
+        raise mlrun.errors.MLRunRetryExhaustedError(
+            f"Failed to execute command by the given deadline."
+            f" last_exception: {self.last_exception},"
+            f" function_name: {self.function.__name__},"
+            f" timeout: {self.timeout}"
+        ) from self.last_exception
+
+    def _timeout_exceeded(self, next_interval=None):
+        now = time.monotonic()
+        if next_interval:
+            now = now + next_interval
+        return self.timeout is not None and now >= self.start_time + self.timeout
+
+
+class AsyncRetryer(Retryer):
+    async def run(self):
+        self._prepare()
+        while not self._timeout_exceeded():
+            next_interval = self.first_interval or next(self.backoff)
+            result, exc, retry = await self._perform_call(next_interval)
+            if retry:
+                await asyncio.sleep(next_interval)
+            elif not exc:
+                return result
+            else:
+                break
+
+        self._raise_last_exception()
+
+    async def _perform_call(self, next_interval):
+        try:
+            result = await self.function(*self.args, **self.kwargs)
+            return result, None, False
+        except mlrun.errors.MLRunFatalFailureError as exc:
+            raise exc.original_exception
+        except Exception as exc:
+            return (
+                None,
+                self.last_exception,
+                self._assert_failure_timeout(next_interval, exc),
+            )

mlrun/utils/singleton.py

@@ -20,7 +20,7 @@ class Singleton(type):
 
     def __call__(cls, *args, **kwargs):
         if cls not in cls._instances:
-            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 

mlrun/utils/v3io_clients.py

@@ -11,22 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-from typing import Dict, FrozenSet
 
 from v3io.dataplane import Client as V3IOClient
-from v3io_frames import Client as get_client
+from v3io_frames import Client as V3IOFramesClient
 from v3io_frames.client import ClientBase
 
-_v3io_clients: Dict[FrozenSet, V3IOClient] = {}
-_frames_clients: Dict[FrozenSet, ClientBase] = {}
+_v3io_clients: dict[frozenset, V3IOClient] = {}
+_frames_clients: dict[frozenset, ClientBase] = {}
 
 
 def get_frames_client(**kwargs) -> ClientBase:
     global _frames_clients
     kw_set = frozenset(kwargs.items())
     if kw_set not in _frames_clients:
-        _frames_clients[kw_set] = get_client(**kwargs)
+        _frames_clients[kw_set] = V3IOFramesClient(**kwargs)
 
     return _frames_clients[kw_set]
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "0ee0a62bfe73dbc8ad04a3a1c6ad958e40a6f56a",
-  "version": "1.6.4-rc8"
+  "git_commit": "8b31f05f7defd779cd119acd2ba3f7ae66e81316",
+  "version": "1.7.0"
 }

mlrun/utils/version/version.py

@@ -14,15 +14,11 @@
 #
 import json
 import sys
+from importlib.resources import read_text
 
 import mlrun.utils
 from mlrun.utils.singleton import Singleton
 
-if sys.version_info >= (3, 7):
-    from importlib.resources import read_text
-else:
-    from importlib_resources import read_text
-
 
 class _VersionInfo:
     def __init__(self, major, minor, patch):
@@ -56,5 +52,5 @@ class Version(metaclass=Singleton):
         return self.python_version
 
     @staticmethod
-    def _resolve_python_version() -> sys.version_info:
+    def _resolve_python_version() -> _VersionInfo:
         return _VersionInfo(*sys.version_info[:3])