mlrun 1.7.0rc9__py3-none-any.whl → 1.7.0rc12__py3-none-any.whl

mlrun/runtimes/nuclio/function.py

@@ -345,17 +345,21 @@ class RemoteRuntime(KubeResource):
 
             git::
 
-                fn.with_source_archive("git://github.com/org/repo#my-branch",
-                        handler="main:handler",
-                        workdir="path/inside/repo")
+                fn.with_source_archive(
+                    "git://github.com/org/repo#my-branch",
+                    handler="main:handler",
+                    workdir="path/inside/repo",
+                )
 
             s3::
 
                 fn.spec.nuclio_runtime = "golang"
-                fn.with_source_archive("s3://my-bucket/path/in/bucket/my-functions-archive",
+                fn.with_source_archive(
+                    "s3://my-bucket/path/in/bucket/my-functions-archive",
                     handler="my_func:Handler",
                     workdir="path/inside/functions/archive",
-                    runtime="golang")
+                    runtime="golang",
+                )
         """
         self.spec.build.source = source
         # update handler in function_handler

mlrun/runtimes/nuclio/serving.py

@@ -23,7 +23,7 @@ from nuclio import KafkaTrigger
 
 import mlrun
 import mlrun.common.schemas
-from mlrun.datastore import parse_kafka_url
+from mlrun.datastore import get_kafka_brokers_from_dict, parse_kafka_url
 from mlrun.model import ObjectList
 from mlrun.runtimes.function_reference import FunctionReference
 from mlrun.secrets import SecretsStore
@@ -295,9 +295,7 @@ class ServingRuntime(RemoteRuntime):
                         "provided class is not a router step, must provide a router class in router topology"
                     )
             else:
-                step = RouterStep(
-                    class_name=class_name, class_args=class_args, engine=engine
-                )
+                step = RouterStep(class_name=class_name, class_args=class_args)
             self.spec.graph = step
         elif topology == StepKinds.flow:
             self.spec.graph = RootFlowStep(engine=engine)
@@ -367,8 +365,8 @@ class ServingRuntime(RemoteRuntime):
 
         Example, create a function (from the notebook), add a model class, and deploy::
 
-            fn = code_to_function(kind='serving')
-            fn.add_model('boost', model_path, model_class='MyClass', my_arg=5)
+            fn = code_to_function(kind="serving")
+            fn.add_model("boost", model_path, model_class="MyClass", my_arg=5)
             fn.deploy()
 
         only works with router topology, for nested topologies (model under router under flow)
@@ -450,7 +448,7 @@ class ServingRuntime(RemoteRuntime):
 
         example::
 
-            fn.add_child_function('enrich', './enrich.ipynb', 'mlrun/mlrun')
+            fn.add_child_function("enrich", "./enrich.ipynb", "mlrun/mlrun")
 
         :param name:   child function name
         :param url:    function/code url, support .py, .ipynb, .yaml extensions
@@ -489,11 +487,8 @@ class ServingRuntime(RemoteRuntime):
                         "worker_allocation_mode", "static"
                     )
 
-                if (
-                    stream.path.startswith("kafka://")
-                    or "kafka_brokers" in stream.options
-                ):
-                    brokers = stream.options.get("kafka_brokers")
+                brokers = get_kafka_brokers_from_dict(stream.options)
+                if stream.path.startswith("kafka://") or brokers:
                     if brokers:
                         brokers = brokers.split(",")
                     topic, brokers = parse_kafka_url(stream.path, brokers)
@@ -731,8 +726,11 @@ class ServingRuntime(RemoteRuntime):
         example::
 
             serving_fn = mlrun.new_function("serving", image="mlrun/mlrun", kind="serving")
-            serving_fn.add_model('my-classifier',model_path=model_path,
-                                  class_name='mlrun.frameworks.sklearn.SklearnModelServer')
+            serving_fn.add_model(
+                "my-classifier",
+                model_path=model_path,
+                class_name="mlrun.frameworks.sklearn.SklearnModelServer",
+            )
             serving_fn.plot(rankdir="LR")
 
         :param filename:  target filepath for the image (None for the notebook)

mlrun/runtimes/pod.py

@@ -1278,9 +1278,9 @@ class KubeResource(BaseRuntime):
             from kubernetes import client as k8s_client
 
             security_context = k8s_client.V1SecurityContext(
-                        run_as_user=1000,
-                        run_as_group=3000,
-                    )
+                run_as_user=1000,
+                run_as_group=3000,
+            )
             function.with_security_context(security_context)
 
         More info:

mlrun/secrets.py

@@ -163,15 +163,19 @@ def get_secret_or_env(
 
     Example::
 
-        secrets = { "KEY1": "VALUE1" }
+        secrets = {"KEY1": "VALUE1"}
         secret = get_secret_or_env("KEY1", secret_provider=secrets)
 
+
         # Using a function to retrieve a secret
         def my_secret_provider(key):
             # some internal logic to retrieve secret
             return value
 
-        secret = get_secret_or_env("KEY1", secret_provider=my_secret_provider, default="TOO-MANY-SECRETS")
+
+        secret = get_secret_or_env(
+            "KEY1", secret_provider=my_secret_provider, default="TOO-MANY-SECRETS"
+        )
 
     :param key: Secret key to look for
     :param secret_provider: Dictionary, callable or `SecretsStore` to extract the secret value from. If using a

mlrun/serving/routers.py

@@ -272,7 +272,9 @@ class ParallelRun(BaseModelRouter):
             fn = mlrun.new_function("parallel", kind="serving")
             graph = fn.set_topology(
                 "router",
-                mlrun.serving.routers.ParallelRun(extend_event=True, executor_type=executor),
+                mlrun.serving.routers.ParallelRun(
+                    extend_event=True, executor_type=executor
+                ),
             )
             graph.add_route("child1", class_name="Cls1")
             graph.add_route("child2", class_name="Cls2", my_arg={"c": 7})

mlrun/serving/states.py

@@ -25,7 +25,10 @@ import mlrun
 
 from ..config import config
 from ..datastore import get_stream_pusher
-from ..datastore.utils import parse_kafka_url
+from ..datastore.utils import (
+    get_kafka_brokers_from_dict,
+    parse_kafka_url,
+)
 from ..errors import MLRunInvalidArgumentError, err_to_str
 from ..model import ModelObj, ObjectDict
 from ..platforms.iguazio import parse_path
@@ -590,7 +593,7 @@ class RouterStep(TaskStep):
 
     kind = "router"
     default_shape = "doubleoctagon"
-    _dict_fields = _task_step_fields + ["routes", "engine"]
+    _dict_fields = _task_step_fields + ["routes"]
     _default_class = "mlrun.serving.ModelRouter"
 
     def __init__(
@@ -603,7 +606,6 @@ class RouterStep(TaskStep):
         function: str = None,
         input_path: str = None,
         result_path: str = None,
-        engine: str = None,
     ):
         super().__init__(
             class_name,
@@ -616,8 +618,6 @@ class RouterStep(TaskStep):
         )
         self._routes: ObjectDict = None
         self.routes = routes
-        self.engine = engine
-        self._controller = None
 
     def get_children(self):
         """get child steps (routes)"""
@@ -687,33 +687,6 @@ class RouterStep(TaskStep):
         self._set_error_handler()
         self._post_init(mode)
 
-        if self.engine == "async":
-            self._build_async_flow()
-            self._run_async_flow()
-
-    def _build_async_flow(self):
-        """initialize and build the async/storey DAG"""
-
-        self.respond()
-        source, self._wait_for_result = _init_async_objects(self.context, [self])
-        source.to(self.async_object)
-
-        self._async_flow = source
-
-    def _run_async_flow(self):
-        self._controller = self._async_flow.run()
-
-    def run(self, event, *args, **kwargs):
-        if self._controller:
-            # async flow (using storey)
-            event._awaitable_result = None
-            resp = self._controller.emit(
-                event, return_awaitable_result=self._wait_for_result
-            )
-            return resp.await_result()
-
-        return super().run(event, *args, **kwargs)
-
     def __getitem__(self, name):
         return self._routes[name]
 
@@ -1524,13 +1497,14 @@ def _init_async_objects(context, steps):
                     endpoint = None
                     options = {}
                     options.update(step.options)
-                    kafka_brokers = options.pop("kafka_brokers", None)
+
+                    kafka_brokers = get_kafka_brokers_from_dict(options, pop=True)
+
                     if stream_path.startswith("kafka://") or kafka_brokers:
                         topic, brokers = parse_kafka_url(stream_path, kafka_brokers)
 
                         kafka_producer_options = options.pop(
-                            "kafka_producer_options",
-                            options.pop("kafka_bootstrap_servers", None),
+                            "kafka_producer_options", None
                         )
 
                         step._async_object = storey.KafkaTarget(

mlrun/serving/v2_serving.py

@@ -63,11 +63,11 @@ class V2ModelServer(StepToDict):
             class MyClass(V2ModelServer):
                 def load(self):
                     # load and initialize the model and/or other elements
-                    model_file, extra_data = self.get_model(suffix='.pkl')
+                    model_file, extra_data = self.get_model(suffix=".pkl")
                     self.model = load(open(model_file, "rb"))
 
                 def predict(self, request):
-                    events = np.array(request['inputs'])
+                    events = np.array(request["inputs"])
                     dmatrix = xgb.DMatrix(events)
                     result: xgb.DMatrix = self.model.predict(dmatrix)
                     return {"outputs": result.tolist()}
@@ -176,9 +176,9 @@ class V2ModelServer(StepToDict):
         ::
 
             def load(self):
-                model_file, extra_data = self.get_model(suffix='.pkl')
+                model_file, extra_data = self.get_model(suffix=".pkl")
                 self.model = load(open(model_file, "rb"))
-                categories = extra_data['categories'].as_df()
+                categories = extra_data["categories"].as_df()
 
         Parameters
         ----------

mlrun/utils/helpers.py

@@ -437,7 +437,7 @@ class LogBatchWriter:
 
 def get_in(obj, keys, default=None):
     """
-    >>> get_in({'a': {'b': 1}}, 'a.b')
+    >>> get_in({"a": {"b": 1}}, "a.b")
     1
     """
     if isinstance(keys, str):

mlrun/utils/notifications/notification/base.py

@@ -44,6 +44,8 @@ class NotificationBase:
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         raise NotImplementedError()
 
@@ -61,6 +63,8 @@ class NotificationBase:
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ) -> str:
         if custom_html:
             return custom_html
@@ -68,6 +72,14 @@ class NotificationBase:
         if self.name:
             message = f"{self.name}: {message}"
 
+        if alert:
+            if not event_data:
+                return f"[{severity}] {message}"
+            return (
+                f"[{severity}] {message} for project {alert.project} "
+                f"UID {event_data.entity.id}. Value {event_data.value}"
+            )
+
         if not runs:
             return f"[{severity}] {message}"
 

mlrun/utils/notifications/notification/console.py

@@ -36,6 +36,8 @@ class ConsoleNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         severity = self._resolve_severity(severity)
         print(f"[{severity}] {message}")

mlrun/utils/notifications/notification/git.py

@@ -38,6 +38,8 @@ class GitNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         git_repo = self.params.get("repo", None)
         git_issue = self.params.get("issue", None)
@@ -50,7 +52,7 @@ class GitNotification(NotificationBase):
         server = self.params.get("server", None)
         gitlab = self.params.get("gitlab", False)
         await self._pr_comment(
-            self._get_html(message, severity, runs, custom_html),
+            self._get_html(message, severity, runs, custom_html, alert, event_data),
             git_repo,
             git_issue,
             merge_request=git_merge_request,

mlrun/utils/notifications/notification/ipython.py

@@ -53,6 +53,8 @@ class IPythonNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         if not self._ipython:
             mlrun.utils.helpers.logger.debug(

mlrun/utils/notifications/notification/slack.py

@@ -42,6 +42,8 @@ class SlackNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
             "SLACK_WEBHOOK"
@@ -53,7 +55,7 @@ class SlackNotification(NotificationBase):
             )
             return
 
-        data = self._generate_slack_data(message, severity, runs)
+        data = self._generate_slack_data(message, severity, runs, alert, event_data)
 
         async with aiohttp.ClientSession() as session:
             async with session.post(webhook, json=data) as response:
@@ -66,12 +68,14 @@ class SlackNotification(NotificationBase):
             mlrun.common.schemas.NotificationSeverity, str
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ) -> dict:
         data = {
             "blocks": [
                 {
-                    "type": "section",
-                    "text": self._get_slack_row(f"[{severity}] {message}"),
+                    "type": "header",
+                    "text": {"type": "plain_text", "text": f"[{severity}] {message}"},
                 },
             ]
         }
@@ -80,22 +84,46 @@ class SlackNotification(NotificationBase):
                 {"type": "section", "text": self._get_slack_row(self.name)}
             )
 
-        if not runs:
-            return data
+        if alert:
+            fields = self._get_alert_fields(alert, event_data)
 
-        if isinstance(runs, list):
-            runs = mlrun.lists.RunList(runs)
+            for i in range(len(fields)):
+                data["blocks"].append({"type": "section", "text": fields[i]})
+        else:
+            if not runs:
+                return data
+
+            if isinstance(runs, list):
+                runs = mlrun.lists.RunList(runs)
 
-        fields = [self._get_slack_row("*Runs*"), self._get_slack_row("*Results*")]
-        for run in runs:
-            fields.append(self._get_run_line(run))
-            fields.append(self._get_run_result(run))
+            fields = [self._get_slack_row("*Runs*"), self._get_slack_row("*Results*")]
+            for run in runs:
+                fields.append(self._get_run_line(run))
+                fields.append(self._get_run_result(run))
 
-        for i in range(0, len(fields), 8):
-            data["blocks"].append({"type": "section", "fields": fields[i : i + 8]})
+            for i in range(0, len(fields), 8):
+                data["blocks"].append({"type": "section", "fields": fields[i : i + 8]})
 
         return data
 
+    def _get_alert_fields(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> list:
+        line = [
+            self._get_slack_row(f":bell: {alert.name} alert has occurred"),
+            self._get_slack_row(f"*Project:*\n{alert.project}"),
+            self._get_slack_row(f"*UID:*\n{event_data.entity.id}"),
+        ]
+        if event_data.value is not None:
+            line.append(self._get_slack_row(f"*Event data:*\n{event_data.value}"))
+
+        if url := mlrun.utils.helpers.get_ui_url(alert.project, event_data.entity.id):
+            line.append(self._get_slack_row(f"*Overview:*\n<{url}|*Job overview*>"))
+
+        return line
+
     def _get_run_line(self, run: dict) -> dict:
         meta = run["metadata"]
         url = mlrun.utils.helpers.get_ui_url(meta.get("project"), meta.get("uid"))

mlrun/utils/notifications/notification/webhook.py

@@ -36,6 +36,8 @@ class WebhookNotification(NotificationBase):
         ] = mlrun.common.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
+        alert: mlrun.common.schemas.AlertConfig = None,
+        event_data: mlrun.common.schemas.Event = None,
     ):
         url = self.params.get("url", None)
         method = self.params.get("method", "post").lower()
@@ -46,9 +48,17 @@ class WebhookNotification(NotificationBase):
         request_body = {
             "message": message,
             "severity": severity,
-            "runs": runs,
         }
 
+        if runs:
+            request_body["runs"] = runs
+
+        if alert:
+            request_body["alert"] = alert.dict()
+            if event_data:
+                request_body["value"] = event_data.value
+                request_body["id"] = event_data.entity.id
+
         if custom_html:
             request_body["custom_html"] = custom_html
 

mlrun/utils/retryer.py

@@ -117,7 +117,7 @@ class Retryer:
         self._raise_last_exception()
 
     def _prepare(self):
-        self.start_time = time.time()
+        self.start_time = time.monotonic()
         self.last_exception = None
 
         # Check if backoff is just a simple interval
@@ -173,7 +173,7 @@ class Retryer:
         ) from self.last_exception
 
     def _timeout_exceeded(self, next_interval=None):
-        now = time.time()
+        now = time.monotonic()
         if next_interval:
             now = now + next_interval
         return self.timeout is not None and now >= self.start_time + self.timeout

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "3396f9ef4c278411ca6227b1792ff7df20005a42",
-  "version": "1.7.0-rc9"
+  "git_commit": "d99c139a3045c51860077414b2076ea95b77a46c",
+  "version": "1.7.0-rc12"
 }

{mlrun-1.7.0rc9.dist-info → mlrun-1.7.0rc12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc9
+Version: 1.7.0rc12
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv