mlrun 1.7.0rc9__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl

mlrun/secrets.py

@@ -163,15 +163,19 @@ def get_secret_or_env(
 
     Example::
 
-        secrets = { "KEY1": "VALUE1" }
+        secrets = {"KEY1": "VALUE1"}
         secret = get_secret_or_env("KEY1", secret_provider=secrets)
 
+
         # Using a function to retrieve a secret
         def my_secret_provider(key):
             # some internal logic to retrieve secret
             return value
 
-        secret = get_secret_or_env("KEY1", secret_provider=my_secret_provider, default="TOO-MANY-SECRETS")
+
+        secret = get_secret_or_env(
+            "KEY1", secret_provider=my_secret_provider, default="TOO-MANY-SECRETS"
+        )
 
     :param key: Secret key to look for
     :param secret_provider: Dictionary, callable or `SecretsStore` to extract the secret value from. If using a

mlrun/serving/routers.py

@@ -272,7 +272,9 @@ class ParallelRun(BaseModelRouter):
             fn = mlrun.new_function("parallel", kind="serving")
             graph = fn.set_topology(
                 "router",
-                mlrun.serving.routers.ParallelRun(extend_event=True, executor_type=executor),
+                mlrun.serving.routers.ParallelRun(
+                    extend_event=True, executor_type=executor
+                ),
             )
             graph.add_route("child1", class_name="Cls1")
             graph.add_route("child2", class_name="Cls2", my_arg={"c": 7})

mlrun/serving/states.py

@@ -17,6 +17,7 @@ __all__ = ["TaskStep", "RouterStep", "RootFlowStep", "ErrorStep"]
 import os
 import pathlib
 import traceback
+import warnings
 from copy import copy, deepcopy
 from inspect import getfullargspec, signature
 from typing import Union
@@ -590,7 +591,7 @@ class RouterStep(TaskStep):
 
     kind = "router"
     default_shape = "doubleoctagon"
-    _dict_fields = _task_step_fields + ["routes", "engine"]
+    _dict_fields = _task_step_fields + ["routes"]
     _default_class = "mlrun.serving.ModelRouter"
 
     def __init__(
@@ -603,7 +604,6 @@ class RouterStep(TaskStep):
         function: str = None,
         input_path: str = None,
         result_path: str = None,
-        engine: str = None,
     ):
         super().__init__(
             class_name,
@@ -616,8 +616,6 @@ class RouterStep(TaskStep):
         )
         self._routes: ObjectDict = None
         self.routes = routes
-        self.engine = engine
-        self._controller = None
 
     def get_children(self):
         """get child steps (routes)"""
@@ -687,33 +685,6 @@ class RouterStep(TaskStep):
         self._set_error_handler()
         self._post_init(mode)
 
-        if self.engine == "async":
-            self._build_async_flow()
-            self._run_async_flow()
-
-    def _build_async_flow(self):
-        """initialize and build the async/storey DAG"""
-
-        self.respond()
-        source, self._wait_for_result = _init_async_objects(self.context, [self])
-        source.to(self.async_object)
-
-        self._async_flow = source
-
-    def _run_async_flow(self):
-        self._controller = self._async_flow.run()
-
-    def run(self, event, *args, **kwargs):
-        if self._controller:
-            # async flow (using storey)
-            event._awaitable_result = None
-            resp = self._controller.emit(
-                event, return_awaitable_result=self._wait_for_result
-            )
-            return resp.await_result()
-
-        return super().run(event, *args, **kwargs)
-
     def __getitem__(self, name):
         return self._routes[name]
 
@@ -1524,13 +1495,21 @@ def _init_async_objects(context, steps):
                     endpoint = None
                     options = {}
                     options.update(step.options)
+
                     kafka_brokers = options.pop("kafka_brokers", None)
+                    if not kafka_brokers and "kafka_bootstrap_servers" in options:
+                        kafka_brokers = options.pop("kafka_bootstrap_servers")
+                        warnings.warn(
+                            "The 'kafka_bootstrap_servers' parameter is deprecated and will be removed in "
+                            "1.9.0. Please pass the 'kafka_brokers' parameter instead.",
+                            FutureWarning,
+                        )
+
                     if stream_path.startswith("kafka://") or kafka_brokers:
                         topic, brokers = parse_kafka_url(stream_path, kafka_brokers)
 
                         kafka_producer_options = options.pop(
-                            "kafka_producer_options",
-                            options.pop("kafka_bootstrap_servers", None),
+                            "kafka_producer_options", None
                         )
 
                         step._async_object = storey.KafkaTarget(

mlrun/serving/v2_serving.py

@@ -63,11 +63,11 @@ class V2ModelServer(StepToDict):
             class MyClass(V2ModelServer):
                 def load(self):
                     # load and initialize the model and/or other elements
-                    model_file, extra_data = self.get_model(suffix='.pkl')
+                    model_file, extra_data = self.get_model(suffix=".pkl")
                     self.model = load(open(model_file, "rb"))
 
                 def predict(self, request):
-                    events = np.array(request['inputs'])
+                    events = np.array(request["inputs"])
                     dmatrix = xgb.DMatrix(events)
                     result: xgb.DMatrix = self.model.predict(dmatrix)
                     return {"outputs": result.tolist()}
@@ -176,9 +176,9 @@ class V2ModelServer(StepToDict):
         ::
 
             def load(self):
-                model_file, extra_data = self.get_model(suffix='.pkl')
+                model_file, extra_data = self.get_model(suffix=".pkl")
                 self.model = load(open(model_file, "rb"))
-                categories = extra_data['categories'].as_df()
+                categories = extra_data["categories"].as_df()
 
         Parameters
         ----------

mlrun/utils/helpers.py

@@ -437,7 +437,7 @@ class LogBatchWriter:
 
 def get_in(obj, keys, default=None):
     """
-    >>> get_in({'a': {'b': 1}}, 'a.b')
+    >>> get_in({"a": {"b": 1}}, "a.b")
     1
     """
     if isinstance(keys, str):

mlrun/utils/notifications/notification/base.py

@@ -44,6 +44,8 @@ class NotificationBase:
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         raise NotImplementedError()
 
@@ -61,6 +63,8 @@ class NotificationBase:
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ) -> str:
         if custom_html:
             return custom_html
@@ -68,6 +72,14 @@ class NotificationBase:
         if self.name:
             message = f"{self.name}: {message}"
 
+        if alert:
+            if not event_data:
+                return f"[{severity}] {message}"
+            return (
+                f"[{severity}] {message} for project {alert.project} "
+                f"UID {event_data.entity.id}. Value {event_data.value}"
+            )
+
         if not runs:
             return f"[{severity}] {message}"
 

mlrun/utils/notifications/notification/console.py

@@ -36,6 +36,8 @@ class ConsoleNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         severity = self._resolve_severity(severity)
         print(f"[{severity}] {message}")

mlrun/utils/notifications/notification/git.py

@@ -38,6 +38,8 @@ class GitNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         git_repo = self.params.get("repo", None)
         git_issue = self.params.get("issue", None)
@@ -50,7 +52,7 @@ class GitNotification(NotificationBase):
         server = self.params.get("server", None)
         gitlab = self.params.get("gitlab", False)
         await self._pr_comment(
-            self._get_html(message, severity, runs, custom_html),
+            self._get_html(message, severity, runs, custom_html, alert, event_data),
             git_repo,
             git_issue,
             merge_request=git_merge_request,

mlrun/utils/notifications/notification/ipython.py

@@ -53,6 +53,8 @@ class IPythonNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         if not self._ipython:
             mlrun.utils.helpers.logger.debug(

mlrun/utils/notifications/notification/slack.py

@@ -42,6 +42,8 @@ class SlackNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
             "SLACK_WEBHOOK"
@@ -53,7 +55,7 @@ class SlackNotification(NotificationBase):
             )
             return
 
-        data = self._generate_slack_data(message, severity, runs)
+        data = self._generate_slack_data(message, severity, runs, alert, event_data)
 
         async with aiohttp.ClientSession() as session:
             async with session.post(webhook, json=data) as response:
@@ -66,12 +68,14 @@ class SlackNotification(NotificationBase):
             mlrun.common.schemas.NotificationSeverity, str
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ) -> dict:
         data = {
             "blocks": [
                 {
-                    "type": "section",
-                    "text": self._get_slack_row(f"[{severity}] {message}"),
+                    "type": "header",
+                    "text": {"type": "plain_text", "text": f"[{severity}] {message}"},
                 },
             ]
         }
@@ -80,22 +84,46 @@ class SlackNotification(NotificationBase):
                 {"type": "section", "text": self._get_slack_row(self.name)}
             )
 
-        if not runs:
-            return data
+        if alert:
+            fields = self._get_alert_fields(alert, event_data)
 
-        if isinstance(runs, list):
-            runs = mlrun.lists.RunList(runs)
+            for i in range(len(fields)):
+                data["blocks"].append({"type": "section", "text": fields[i]})
+        else:
+            if not runs:
+                return data
+
+            if isinstance(runs, list):
+                runs = mlrun.lists.RunList(runs)
 
-        fields = [self._get_slack_row("*Runs*"), self._get_slack_row("*Results*")]
-        for run in runs:
-            fields.append(self._get_run_line(run))
-            fields.append(self._get_run_result(run))
+            fields = [self._get_slack_row("*Runs*"), self._get_slack_row("*Results*")]
+            for run in runs:
+                fields.append(self._get_run_line(run))
+                fields.append(self._get_run_result(run))
 
-        for i in range(0, len(fields), 8):
-            data["blocks"].append({"type": "section", "fields": fields[i : i + 8]})
+            for i in range(0, len(fields), 8):
+                data["blocks"].append({"type": "section", "fields": fields[i : i + 8]})
 
         return data
 
+    def _get_alert_fields(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> list:
+        line = [
+            self._get_slack_row(f":bell: {alert.name} alert has occurred"),
+            self._get_slack_row(f"*Project:*\n{alert.project}"),
+            self._get_slack_row(f"*UID:*\n{event_data.entity.id}"),
+        ]
+        if event_data.value is not None:
+            line.append(self._get_slack_row(f"*Event data:*\n{event_data.value}"))
+
+        if url := mlrun.utils.helpers.get_ui_url(alert.project, event_data.entity.id):
+            line.append(self._get_slack_row(f"*Overview:*\n<{url}|*Job overview*>"))
+
+        return line
+
     def _get_run_line(self, run: dict) -> dict:
         meta = run["metadata"]
         url = mlrun.utils.helpers.get_ui_url(meta.get("project"), meta.get("uid"))

mlrun/utils/notifications/notification/webhook.py

@@ -36,6 +36,8 @@ class WebhookNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()
@@ -46,9 +48,17 @@ class WebhookNotification(NotificationBase):
         request_body = {
             "message": message,
             "severity": severity,
-            "runs": runs,
         }
 
+        if runs:
+            request_body["runs"] = runs
+
+        if alert:
+            request_body["alert"] = alert.dict()
+            if event_data:
+                request_body["value"] = event_data.value
+                request_body["id"] = event_data.entity.id
+
         if custom_html:
             request_body["custom_html"] = custom_html
 

mlrun/utils/retryer.py

@@ -117,7 +117,7 @@ class Retryer:
         self._raise_last_exception()
 
     def _prepare(self):
-        self.start_time = time.time()
+        self.start_time = time.monotonic()
         self.last_exception = None
 
         # Check if backoff is just a simple interval
@@ -173,7 +173,7 @@ class Retryer:
         ) from self.last_exception
 
     def _timeout_exceeded(self, next_interval=None):
-        now = time.time()
+        now = time.monotonic()
         if next_interval:
             now = now + next_interval
         return self.timeout is not None and now >= self.start_time + self.timeout

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "3396f9ef4c278411ca6227b1792ff7df20005a42",
-  "version": "1.7.0-rc9"
+  "git_commit": "fbea0ad0a63c1a8af06793354df5d3c467ca606d",
+  "version": "1.7.0-rc11"
 }

{mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc9
+Version: 1.7.0rc11
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv