mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/utils/notifications/notification_pusher.py

@@ -13,20 +13,30 @@
 # limitations under the License.
 
 import ast
+import asyncio
+import datetime
 import os
 import typing
 
+from fastapi.concurrency import run_in_threadpool
+
+import mlrun.api.db.base
+import mlrun.api.db.session
+import mlrun.api.schemas
+import mlrun.api.utils.singletons.k8s
+import mlrun.config
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
+from mlrun.utils import logger
 
-from .notification import NotificationBase, NotificationSeverity, NotificationTypes
+from .notification import NotificationBase, NotificationTypes
 
 
 class NotificationPusher(object):
 
     messages = {
-        "success": "Run completed",
+        "completed": "Run completed",
         "error": "Run failed",
     }
 
@@ -36,55 +46,163 @@ class NotificationPusher(object):
         self._notifications = {}
 
         for run in self._runs:
-            for notification_config in run.get("spec", {}).get("notifications", []):
-                if self._should_notify(run, notification_config):
-                    self._notification_data.append((run, notification_config))
-
-    def push(self):
-        for notification_data in self._notification_data:
-            self._send_notification(
-                self._load_notification(notification_data[1]), *notification_data
-            )
+            if isinstance(run, dict):
+                run = mlrun.model.RunObject.from_dict(run)
+
+            for notification in run.spec.notifications:
+                if self._should_notify(run, notification):
+                    self._notification_data.append((run, notification))
+
+    def push(
+        self,
+        db: mlrun.api.db.base.DBInterface = None,
+    ):
+        """
+        Asynchronously push notifications for all runs in the initialized runs list (if they should be pushed).
+        When running from a sync environment, the notifications will be pushed asynchronously however the function will
+        wait for all notifications to be pushed before returning.
+        """
+
+        if not len(self._notification_data):
+            return
+
+        async def _push():
+            tasks = []
+            for notification_data in self._notification_data:
+                tasks.append(
+                    self._push_notification(
+                        self._load_notification(*notification_data),
+                        notification_data[0],
+                        notification_data[1],
+                        db,
+                    )
+                )
+            await asyncio.gather(*tasks)
+
+        logger.debug(
+            "Pushing notifications", notifications_amount=len(self._notification_data)
+        )
+        main_event_loop = asyncio.get_event_loop()
+        if main_event_loop.is_running():
+
+            # If running from the api or from jupyter notebook, we are already in an event loop.
+            # We add the async push function to the loop and run it.
+            asyncio.run_coroutine_threadsafe(_push(), main_event_loop)
+        else:
+
+            # If running mlrun SDK locally (not from jupyter), there isn't necessarily an event loop.
+            # We create a new event loop and run the async push function in it.
+            main_event_loop.run_until_complete(_push())
 
     @staticmethod
     def _should_notify(
-        run: typing.Union[mlrun.model.RunObject, dict], notification_config: dict
+        run: mlrun.model.RunObject,
+        notification: mlrun.model.Notification,
     ) -> bool:
-        when_states = notification_config.get("when", [])
-        condition = notification_config.get("condition", "")
-        run_state = run.get("status", {}).get("state", "")
+        when_states = notification.when
+        condition = notification.condition
+        run_state = run.state()
+
+        # if the notification isn't pending, don't push it
+        if (
+            notification.status
+            and notification.status != mlrun.api.schemas.NotificationStatus.PENDING
+        ):
+            return False
 
         # if at least one condition is met, notify
         for when_state in when_states:
             if (
-                when_state == "success"
-                and run_state == "success"
+                when_state == run_state == "completed"
                 and (not condition or ast.literal_eval(condition))
-            ) or (when_state == "failure" and run_state == "error"):
+            ) or when_state == run_state == "error":
                 return True
 
         return False
 
-    def _load_notification(self, notification_config: dict) -> NotificationBase:
-        params = notification_config.get("params", {})
+    def _load_notification(
+        self, run: mlrun.model.RunObject, notification: mlrun.model.Notification
+    ) -> NotificationBase:
+        name = notification.name
         notification_type = NotificationTypes(
-            notification_config.get("type", "console")
+            notification.kind or NotificationTypes.console
         )
-        if notification_type not in self._notifications:
+        notification_key = f"{run.metadata.uid}-{name or notification_type}"
+        if notification_key not in self._notifications:
             self._notifications[
-                notification_type
-            ] = notification_type.get_notification()(params)
+                notification_key
+            ] = notification_type.get_notification()(name, notification.params)
         else:
-            self._notifications[notification_type].load_notification(params)
+            self._notifications[notification_key].load_notification(notification.params)
 
-        return self._notifications[notification_type]
+        logger.debug(
+            "Loaded notification", notification=self._notifications[notification_key]
+        )
+        return self._notifications[notification_key]
+
+    async def _push_notification(
+        self,
+        notification: NotificationBase,
+        run: mlrun.model.RunObject,
+        notification_object: mlrun.model.Notification,
+        db: mlrun.api.db.base.DBInterface,
+    ):
+        message = self.messages.get(run.state(), "")
+        severity = (
+            notification_object.severity or mlrun.api.schemas.NotificationSeverity.INFO
+        )
+        logger.debug(
+            "Pushing notification",
+            notification=notification_object.to_dict(),
+            run_uid=run.metadata.uid,
+        )
+        try:
+            if asyncio.iscoroutinefunction(notification.push):
+                await notification.push(message, severity, [run.to_dict()])
+            else:
+                notification.push(message, severity, [run.to_dict()])
+
+            if mlrun.config.is_running_as_api():
+                await self._update_notification_status(
+                    db,
+                    run.metadata.uid,
+                    run.metadata.project,
+                    notification_object,
+                    status=mlrun.api.schemas.NotificationStatus.SENT,
+                    sent_time=datetime.datetime.now(tz=datetime.timezone.utc),
+                )
+        except Exception as exc:
+            if mlrun.config.is_running_as_api():
+                await self._update_notification_status(
+                    db,
+                    run.metadata.uid,
+                    run.metadata.project,
+                    notification_object,
+                    status=mlrun.api.schemas.NotificationStatus.ERROR,
+                )
+            raise exc
 
-    def _send_notification(
-        self, notification: NotificationBase, run: dict, notification_config: dict
+    @staticmethod
+    async def _update_notification_status(
+        db: mlrun.api.db.base.DBInterface,
+        run_uid: str,
+        project: str,
+        notification: mlrun.model.Notification,
+        status: str = None,
+        sent_time: datetime.datetime = None,
     ):
-        message = self.messages.get(run.get("status", {}).get("state", ""), "")
-        severity = notification_config.get("severity", NotificationSeverity.INFO)
-        notification.send(message, severity, [run])
+        db_session = mlrun.api.db.session.create_session()
+        notification.status = status or notification.status
+        notification.sent_time = sent_time or notification.sent_time
+
+        # store directly in db, no need to use crud as the secrets are already loaded
+        await run_in_threadpool(
+            db.store_run_notifications,
+            db_session,
+            [notification],
+            run_uid,
+            project,
+        )
 
 
 class CustomNotificationPusher(object):
@@ -97,13 +215,43 @@ class CustomNotificationPusher(object):
     def push(
         self,
         message: str,
-        severity: typing.Union[NotificationSeverity, str] = NotificationSeverity.INFO,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
+        runs: typing.Union[mlrun.lists.RunList, list] = None,
+        custom_html: str = None,
+    ):
+        async def _push():
+            tasks = []
+            for notification_type, notification in self._notifications.items():
+                if self.should_push_notification(notification_type):
+                    tasks.append(
+                        self._push_notification(
+                            notification, message, severity, runs, custom_html
+                        )
+                    )
+            await asyncio.gather(*tasks)
+
+        main_event_loop = asyncio.get_event_loop()
+        if main_event_loop.is_running():
+            asyncio.run_coroutine_threadsafe(_push(), main_event_loop)
+        else:
+            main_event_loop.run_until_complete(_push())
+
+    @staticmethod
+    async def _push_notification(
+        notification: NotificationBase,
+        message: str,
+        severity: typing.Union[
+            mlrun.api.schemas.NotificationSeverity, str
+        ] = mlrun.api.schemas.NotificationSeverity.INFO,
         runs: typing.Union[mlrun.lists.RunList, list] = None,
         custom_html: str = None,
     ):
-        for notification_type, notification in self._notifications.items():
-            if self.should_send_notification(notification_type):
-                notification.send(message, severity, runs, custom_html)
+        if asyncio.iscoroutinefunction(notification.push):
+            await notification.push(message, severity, runs, custom_html)
+        else:
+            notification.push(message, severity, runs, custom_html)
 
     def add_notification(
         self, notification_type: str, params: typing.Dict[str, str] = None
@@ -115,12 +263,12 @@ class CustomNotificationPusher(object):
                 notification_type
             ).get_notification()(params)
 
-    def should_send_notification(self, notification_type):
+    def should_push_notification(self, notification_type):
         notification = self._notifications.get(notification_type)
         if not notification or not notification.active:
             return False
 
-        # get notification's inverse dependencies, and only send the notification if
+        # get notification's inverse dependencies, and only push the notification if
         # none of its inverse dependencies are being sent
         inverse_dependencies = NotificationTypes(
             notification_type

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "b3de4b32ad5396aa395e94a0f46cb992619e9b28",
-  "version": "1.3.1-rc5"
+  "git_commit": "11fee9f04e66041b9883f73dbdc61be0b4c6498b",
+  "version": "1.4.0-rc2"
 }

{mlrun-1.3.1rc5.dist-info → mlrun-1.4.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.3.1rc5
+Version: 1.4.0rc2
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -34,7 +34,7 @@ Requires-Dist: ipython (<9.0,>=7.0)
 Requires-Dist: nuclio-jupyter (~=0.9.9)
 Requires-Dist: numpy (<1.23.0,>=1.16.5)
 Requires-Dist: pandas (<1.5.0,~=1.2)
-Requires-Dist: pyarrow (<11,>=10)
+Requires-Dist: pyarrow (<12,>=10.0)
 Requires-Dist: pyyaml (~=5.1)
 Requires-Dist: requests (~=2.22)
 Requires-Dist: sqlalchemy (~=1.4)
@@ -63,6 +63,7 @@ Requires-Dist: deprecated (~=1.2)
 Provides-Extra: all
 Requires-Dist: adlfs (~=2021.8.1) ; extra == 'all'
 Requires-Dist: aiobotocore (~=1.4.0) ; extra == 'all'
+Requires-Dist: avro (~=1.11) ; extra == 'all'
 Requires-Dist: azure-core (~=1.24) ; extra == 'all'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'all'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'all'
@@ -102,6 +103,7 @@ Requires-Dist: bokeh (>=2.4.2,~=2.4) ; extra == 'bokeh'
 Provides-Extra: complete
 Requires-Dist: adlfs (~=2021.8.1) ; extra == 'complete'
 Requires-Dist: aiobotocore (~=1.4.0) ; extra == 'complete'
+Requires-Dist: avro (~=1.11) ; extra == 'complete'
 Requires-Dist: azure-core (~=1.24) ; extra == 'complete'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'complete'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete'
@@ -121,6 +123,7 @@ Provides-Extra: complete-api
 Requires-Dist: adlfs (~=2021.8.1) ; extra == 'complete-api'
 Requires-Dist: aiobotocore (~=1.4.0) ; extra == 'complete-api'
 Requires-Dist: apscheduler (~=3.6) ; extra == 'complete-api'
+Requires-Dist: avro (~=1.11) ; extra == 'complete-api'
 Requires-Dist: azure-core (~=1.24) ; extra == 'complete-api'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'complete-api'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete-api'
@@ -152,6 +155,7 @@ Provides-Extra: graphviz
 Requires-Dist: graphviz (~=0.20.0) ; extra == 'graphviz'
 Provides-Extra: kafka
 Requires-Dist: kafka-python (~=2.0) ; extra == 'kafka'
+Requires-Dist: avro (~=1.11) ; extra == 'kafka'
 Provides-Extra: plotly
 Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'plotly'
 Provides-Extra: redis

{mlrun-1.3.1rc5.dist-info → mlrun-1.4.0rc2.dist-info}/RECORD

@@ -1,27 +1,27 @@
 mlrun/__init__.py,sha256=cCTbgyTFsXvEdaq1Cvwz8s8FE75YGIJKX58um-fnfFs,8727
-mlrun/__main__.py,sha256=7rP5R0xv1YziN-yfFqy9IJM1Z16vehUNPVz_PUU0zqQ,47778
-mlrun/builder.py,sha256=4PnePQzFy69RHgzFksdRNskExrjnmGCGPLXMHVPUdwM,24314
-mlrun/config.py,sha256=UbxmDWCDpIMZuPUANsmF0xl3_Ihey9s6K4gVn_SYfcM,49992
+mlrun/__main__.py,sha256=7AXWZ6AYLQ1xGIHgrSnPORYU8ZERr3pHGsRdJpCxpHA,49967
+mlrun/builder.py,sha256=5YKq67e3nHlVtxW0Z9f6j86qnrhclHEvy8OxmlDVyJ4,24965
+mlrun/config.py,sha256=83S9K8pNvlIknMtpTNaVW0LGbFJ37qLtrF4CJHi6dtg,50570
 mlrun/errors.py,sha256=XEb1pxwy5Lzr9q0VT4z_SjP74jy37HFSmdd2hddWgvY,6780
-mlrun/execution.py,sha256=jL7GuRPa4ZHxroW7kGLtBFeiOzDQoPv3bHZ2BGGSaUo,38928
+mlrun/execution.py,sha256=RMuw1D78GOWxDk3aQfFtFRZZ5maM4T7BiXaApoQA_6E,39096
 mlrun/features.py,sha256=pDUvV6HlkxZnUwW3lg7oq3vBwdcJQBprvdyGPNv57X8,15624
-mlrun/k8s_utils.py,sha256=5Ly-7Pgis6W1dHL1fOHSYkLcWEqZLdfGuDgDK-bsmIM,31384
+mlrun/k8s_utils.py,sha256=fGObRopN5-7iS1TzIUzMVggN5-yb5wieLTDdesbhm-8,31688
 mlrun/kfpops.py,sha256=fCq9kqsIO5six8AmHkiRLngyDclRymd7V6U5YXnX_ek,29645
 mlrun/lists.py,sha256=yOKwpS71Fua08sV825Anth8aaR9H5bHZyV544YbnXj4,8360
-mlrun/model.py,sha256=hb7Zwbl9fXp6BAmYt-VBHZRMzhp3Ql8vKzHHqWgsShI,52194
+mlrun/model.py,sha256=kxctSqX1Uqpy_YFd0kzMPppo5PamiM5o0sSd1JnADgk,53402
 mlrun/render.py,sha256=983QAYawKTFlwXLpUwGC1gWEjUQUFxl1lo8bPQuXIA0,11828
-mlrun/run.py,sha256=AoVyOqBTm1Hm3uIlSEkh_cqv0RuGsUFfBKpn085-w6I,64321
+mlrun/run.py,sha256=ccokIwQ1yodQxVBieNl3x9GxDx-Ztsi3O6m0Lh9KCYs,64451
 mlrun/secrets.py,sha256=yboWaNO33RCXmopLL2bGCkDUbHuwJ7IgJaFv1lkoN_I,7731
 mlrun/api/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
 mlrun/api/alembic.ini,sha256=oeSJRHwBbF3c3-N_RJilcZD-diq3F-oPv21gDIkrZGs,2105
 mlrun/api/constants.py,sha256=fyTQXJe4sx4GK8vZxklivgJgaSyOftk_NezGKA1J-mo,685
 mlrun/api/initial_data.py,sha256=Zj9Oan9Qeg45AOv8Dfg3Gvsymuz5cFR7zgLskuxUTFs,23530
-mlrun/api/main.py,sha256=PKT5kdQTSZVKl8-miQwl--V0rc2HPCkuk_Zpn_EVqQM,24019
+mlrun/api/main.py,sha256=1VtkWdILYXSY32i8DjO1Ost7_DrQN1tLO0CLTP9NWbo,26180
 mlrun/api/middlewares.py,sha256=VTW5VV7AGOigXf-ru2V3XSZzw_bMBr8j0uH2aMIGGfk,5235
 mlrun/api/api/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
 mlrun/api/api/api.py,sha256=suDQI8HL_zjtqaY7fBdm49SP73zJN2imdBiOh1hA4fQ,4267
 mlrun/api/api/deps.py,sha256=KTllUn0OHT-gGfnWPcz47QQwiHtxeHZBLEWMMbYWPBc,3143
-mlrun/api/api/utils.py,sha256=R2fVBlahNevk5IshtohBqibmS3Rnc4Ph3xViUoEKNfU,32788
+mlrun/api/api/utils.py,sha256=ia6xAEdGHwL2Z6t8jhI0_qFX4BeHz-qFxlnoe6vqR5s,36224
 mlrun/api/api/endpoints/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
 mlrun/api/api/endpoints/artifacts.py,sha256=oGy-key8BmAUNWvCiOqAcatjV0pVd_0atcn8ZGftgjM,9458
 mlrun/api/api/endpoints/auth.py,sha256=8-vv61tPX_RFabJPPyXAWbSb8i3apcpbqJSs0GXINYM,1190
@@ -35,12 +35,12 @@ mlrun/api/api/endpoints/functions.py,sha256=dOTy57CyTSqFI7mOB7utl8JUG7pR9idQn18h
 mlrun/api/api/endpoints/grafana_proxy.py,sha256=NFgIpoxIXAmI_3bKXTNvC0Z0wEu7WcqsBH3zqfhbyZo,5798
 mlrun/api/api/endpoints/healthz.py,sha256=3UHtyOFBgUooSmpWGiErav3_yJh1ZrgGrtPEmBZGWrM,1335
 mlrun/api/api/endpoints/logs.py,sha256=1a73ZwNNMDZaaYRqFbL3XZ6OrK3dKCpQI7HC-QJxI78,2429
-mlrun/api/api/endpoints/marketplace.py,sha256=PXc5OftoSoILyHCgSgNdNBEh4JjR8PmcvTQ6uB3nYfo,8283
+mlrun/api/api/endpoints/marketplace.py,sha256=FdgtlvewBVGJ6F5IFlffU6Q8eoVIgcxe82rjbnVRX34,10035
 mlrun/api/api/endpoints/model_endpoints.py,sha256=vlKqhQUJZISBWujGT3BQLWQD0aVyihrMcl8Fb-W8PXo,15724
 mlrun/api/api/endpoints/operations.py,sha256=Fxtp2jY3dlgzYG-2vm1AqNnpxQ-7WaJuibyKX569stg,3700
 mlrun/api/api/endpoints/pipelines.py,sha256=wV3ZvX7LPgb6bBsryXHXkwiiX_oxpK-Cwskj88W3KyE,9099
 mlrun/api/api/endpoints/projects.py,sha256=lFHvfDeeKdNIZ-DhmGKRSqQOuQuOWLu8a0uNay2tyDA,11392
-mlrun/api/api/endpoints/runs.py,sha256=2a_x6-IOLeQwhvLSH7dJOG3Xn-4aO2P4Vjt5T6J3cb8,8956
+mlrun/api/api/endpoints/runs.py,sha256=WyMFtZKU79BhkYwcOyejXyK9D_YX7U7bc7fZwRBUxm0,9076
 mlrun/api/api/endpoints/runtime_resources.py,sha256=Eer8nrP8Hh0y-pXkYr0dOXL7wjw4leE9Q-cxOMdIw08,8988
 mlrun/api/api/endpoints/schedules.py,sha256=Yw3tZNnjfZBgwS-Rsx-i8i8dzi-UNsbfNNvja8eN41I,10579
 mlrun/api/api/endpoints/secrets.py,sha256=l6IzLPMEJr_e7wrTNxLQIn-x8LGBlxeV7diYSAD55t4,6057
@@ -49,36 +49,35 @@ mlrun/api/api/endpoints/tags.py,sha256=bnsWCKoOUnVHL42qqUzbXvs8gdmfT-8kcGwiT1svJ
 mlrun/api/api/endpoints/internal/__init__.py,sha256=Lq7QbwU5hFGCq6hzU28fA5t1soMesi7DVo99T21blU8,1203
 mlrun/api/api/endpoints/internal/config.py,sha256=kXnraXbXv029QZMMEjwvhsUWof69wTvFBLcRyB32JFc,1043
 mlrun/api/api/endpoints/internal/memory_reports.py,sha256=k9cA6NhvAPfA-CYRubpjSrhsHJ7Ig_wTFMIMTEZxS4E,1709
-mlrun/api/crud/__init__.py,sha256=WMo1AvY_XqJJQWd_9hgw0B35Rvnx50eN0uB4HRAms3E,1184
+mlrun/api/crud/__init__.py,sha256=2ea5kCz3AGmRX2uRHCv2oRRIWX4LL0j7RPn8BVKjcTA,1225
 mlrun/api/crud/artifacts.py,sha256=p-CJZKokYwy2ej0K2vEp5EEy5xM-FD7ytSWMHBd7J9Y,5433
 mlrun/api/crud/client_spec.py,sha256=VapHAcFSKCmBq7AarJkpwoHns6fXLJ-VCEQcBuBczmU,7341
 mlrun/api/crud/clusterization_spec.py,sha256=wf9uiLb3DRwegDGlRu-dgfahoyp51bxtAuB8TEnklfc,1049
 mlrun/api/crud/feature_store.py,sha256=SAWhliY9UJAQbaJYKb7CmXTuVbovq3fDl-_5-RMxfxI,18449
 mlrun/api/crud/functions.py,sha256=pqCTL7rsHawfynV6DfqRrWKhnWxw1MNHJhLY_og6cJo,3550
 mlrun/api/crud/logs.py,sha256=r5JWMAtmxA9kIQ0RCi-DZG7njyh9GbmHL-2w_hu-r9M,9578
-mlrun/api/crud/marketplace.py,sha256=dDJG6puO_asotN8gRgkjwVXQwkLFR7MoTghAcb7TRY4,8180
+mlrun/api/crud/marketplace.py,sha256=zS0rQexzKNA2Lhjfao2H_-O4fpzLqSD827x46E8qh8o,11769
+mlrun/api/crud/notifications.py,sha256=aEIx2ByVQzHxdmjePiYXxzUpZA1yZ8YXhR-nvIbet3o,2751
 mlrun/api/crud/pipelines.py,sha256=aW0RDWAyzK0vnCzyuVLEHDsW-MY-Cy5RvvuqbUhh6ow,13982
 mlrun/api/crud/projects.py,sha256=uLLSMZD0Z8BLkm2naHdNyZK1XyF51IAGVqWN6ydI_9Y,13326
-mlrun/api/crud/runs.py,sha256=E_FP0tQcpBQbisVpDmGZ3Y-2Z0wzoQxkJ73V0ioNOZg,5931
+mlrun/api/crud/runs.py,sha256=Vi6RshVl9RWYef8xPdPcpnx-Vw8SzSXanqlERrtXUpQ,6005
 mlrun/api/crud/runtime_resources.py,sha256=XJRxzrh0NKKBqQq-wXAXFTxWkEMqAX_t3buJsVYOfT8,5539
-mlrun/api/crud/secrets.py,sha256=_aIjtNOBI3fNM_2Tz0_W6UMM3gbMfapDFsBaQU7P_WU,19885
+mlrun/api/crud/secrets.py,sha256=SvaOsSFfnk4OKGr-hXTv34XRoR7SOpdLbCxElgW9dVo,19921
 mlrun/api/crud/tags.py,sha256=TKi6u--McIdmcFEXNrbwgsX5wAaENil7FVwi8mnKqok,3265
 mlrun/api/crud/model_monitoring/__init__.py,sha256=CAW_Xhm94IVid0yos0tLQUd_UdCRYz-_2s_rAloZEZw,722
 mlrun/api/crud/model_monitoring/grafana.py,sha256=uyZOa7KsqCkDWdfcWnV8xg2vlrzmPOQX_rbQq2X9bZ0,14943
 mlrun/api/crud/model_monitoring/model_endpoints.py,sha256=VOpm83yvmgycy-pKzin3CcixgDVOFpitqJHE80RLICQ,40998
 mlrun/api/db/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
-mlrun/api/db/base.py,sha256=Kxj491ODRvO7X6sq5gl7oP1wPMPU38mbQPsgNYudTAQ,13684
+mlrun/api/db/base.py,sha256=Cxseet80T_PPFxKA29vNGsMP7x1L3Y3FPU1LP38vTCw,14365
 mlrun/api/db/init_db.py,sha256=5WSrojwOh79O3guN_7IEi9wjqzKj3H74DU2xbGtrElk,870
-mlrun/api/db/session.py,sha256=g2-5rlAE9PENCro6KBMrsvk5IZPrfHdKgEjjcwtDMro,1226
-mlrun/api/db/filedb/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
-mlrun/api/db/filedb/db.py,sha256=iTehkcdD488diqx4oTng8AJcL2fsYKFJ61mdlMlhywg,14555
+mlrun/api/db/session.py,sha256=_5DVL4YyGGFG9sbtHx_Bq2C1aHrZC5V8MS7TJfNeWIk,993
 mlrun/api/db/sqldb/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
-mlrun/api/db/sqldb/db.py,sha256=_i3NmUJcHm3p1NYqGzOavfrak8Vot69kI8A2nkR-IR4,136433
+mlrun/api/db/sqldb/db.py,sha256=C4NUyVn5e1VR_N4sgfe8WPSD16P62zAsUY7t9jB9Lpg,142645
 mlrun/api/db/sqldb/helpers.py,sha256=j8VK-l_qZ5vLUjGkFK3D3fBgDgPmi-U8hU1MkMIAhC0,2340
 mlrun/api/db/sqldb/session.py,sha256=97QcZc7EOhypk3ZMl0F5k1ogsgK834S3SxCj3UNry-s,2542
 mlrun/api/db/sqldb/models/__init__.py,sha256=0cHZKMpzqkC6u3WGPEYE8zPcahY6KxBrK2pwkzTCCME,1063
-mlrun/api/db/sqldb/models/models_mysql.py,sha256=WOyZDnQRvvXPrPWqJoJ5x-HULSXPssVy3GPru6JAgIM,18687
-mlrun/api/db/sqldb/models/models_sqlite.py,sha256=8tHiSP5ruRjenfZzoDvjnp7kLD7wfEi3XhZx1iXPHd8,16908
+mlrun/api/db/sqldb/models/models_mysql.py,sha256=ZRerU-oWljKFkzXZ7nTb22y-qWpX_RXxvDH_gFhcbK4,20624
+mlrun/api/db/sqldb/models/models_sqlite.py,sha256=uMUkomR_YOa7d14g2XZcaIGtq1OlQ_fqkbOG9MbcVfE,18286
 mlrun/api/migrations_mysql/env.py,sha256=U5Fu1WLV5vggRYy4sa7d1ItJr1H3-T71BnkOw4ZjWIs,2906
 mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py,sha256=MF0DwcRcbyRXdWNeWieEWIu5zMpxZ5IQPmF__werbkU,4614
 mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py,sha256=BthqApYJLuzuHZEJfrOaERD8M-2FRGsZ3e5tRFvMk78,1936
@@ -87,6 +86,7 @@ mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_i
 mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py,sha256=1U5dpO6k_SLNZ0MAd5AukDddcxp8eCUTEx-2o3b_U9k,1504
 mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py,sha256=lppZKpOBbGoyzP855Q8Y5hWfcqCzCjOlz-FefAqenUQ,1649
 mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py,sha256=CGh1ihmMJ4GNpDXNoiB9ksSe1P7PyrV3NqWTiq6GTIg,24057
+mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py,sha256=IU4QdjJrnqJK-fzf5SL3mtpXduTxieoog_HH9aahq3M,2457
 mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py,sha256=18_SGHfgvNE6TcjsP2_bmqAjL3YI70B0J5-NojmxRr4,1351
 mlrun/api/migrations_sqlite/env.py,sha256=U5Fu1WLV5vggRYy4sa7d1ItJr1H3-T71BnkOw4ZjWIs,2906
 mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py,sha256=Kp1VTDMqS72UMODn1GboBo-IwUEZ8IWuflxrBTbATj0,11473
@@ -96,6 +96,7 @@ mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to
 mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py,sha256=pJqfbcEjG7I8yacspyFnbBZvVVpJ4mPLXc4VcJ8X1oQ,1767
 mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py,sha256=3Q_PnbBjaD-04YfjH2MW4IUocOWVUfa-a1_Y-snzBHA,1360
 mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py,sha256=cNFtnxsL4tTnfZiEo3uhATcqmgU6aqh2jEf521G0jH8,1259
+mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py,sha256=Lth-_bXTG-8l-vUi_YxwfWdMVC6aJAKVHQnQ609GmsQ,2147
 mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py,sha256=Q7KfNsbnnPWhXUnLMmbGFZIaItOS_Hvr2bY_td-K3vw,1482
 mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py,sha256=1indyv5TlsDK4haAC9xxNznRjXgqjNIjgfFOakgBpNo,1892
 mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py,sha256=JyUhI_H02D0fkBDZr3Hv4-KAiHEOL62z4-YE25ik5Rs,1882
@@ -106,7 +107,7 @@ mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py,
 mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py,sha256=lQ0lPq6xZD59iupkbAo0wCV-otE0G9lmjEpqIBZEguA,1831
 mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py,sha256=AWkrtG6jWOMJHyLqR8jrmbp95Fk9qB6qhjjYLb1mezo,3936
 mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py,sha256=KrIm5pHwHlJr5e3DdtMcBiEjgbopOniWzDi9T1XTSHQ,2588
-mlrun/api/schemas/__init__.py,sha256=9qJuHKGc6nLZzUNuKZImwqMRXQSTwfrT_gB6l-SAhNQ,3962
+mlrun/api/schemas/__init__.py,sha256=XwgaWHdXLIASbqnw7YPEfW43AO6PmHxJbzXaBZY7VzA,4029
 mlrun/api/schemas/artifact.py,sha256=gpAdViV_3TQ-W2KPPNvdOArEfQZdd427YUW9bml9vE4,2070
 mlrun/api/schemas/auth.py,sha256=9MnT6Ckuy8_kAI7edsAqwN9-wp3noh5xZlL40utYXqA,5311
 mlrun/api/schemas/background_task.py,sha256=RayGDZ3_KYCYVx-n7CgMNDQJQp78ajrb5unQW6wFzhU,1563
@@ -118,9 +119,10 @@ mlrun/api/schemas/frontend_spec.py,sha256=-kYKekdyrUAx-dOhhyvhh54y-KWV3bGfFAxqJQ
 mlrun/api/schemas/function.py,sha256=5Iy0aZYwXw9_vLSPBUwhuD_YMuQn95_11oX7DSTFIeo,3991
 mlrun/api/schemas/http.py,sha256=q9l7LBK1pIWlax2gHK9rPQfeugHjBMUrLDWAxdGxUII,715
 mlrun/api/schemas/k8s.py,sha256=R6Ufggp8MqA0lQ68AMsh_qiRcbsovaWKJg1a40XzcZg,1405
-mlrun/api/schemas/marketplace.py,sha256=BD0Gb_VFxcvJS7_6gQqxb524TL2ZPn9xwbUhwvleoTo,4258
+mlrun/api/schemas/marketplace.py,sha256=fpvHrAkmtpJEu4-6UNtnvSQz_Rrb87en5VBMvLZFPok,4592
 mlrun/api/schemas/memory_reports.py,sha256=SlNJ-uya5RywEysm_IZMwF1t4j-zYsjOj8oPYwLRO08,920
 mlrun/api/schemas/model_endpoints.py,sha256=j1U0bqYSv_ytjsQ7ZM0qEk4ebUdk-1Yks3h9dNEJijo,12343
+mlrun/api/schemas/notification.py,sha256=sCVgEsQUWANr4zxXtvTjS2lve8eaTKv3me7dy5zLoDI,894
 mlrun/api/schemas/object.py,sha256=bLMt30Kgch6_4Q_KML68NerU_HuTqW8K-4LUc8KfyVc,1965
 mlrun/api/schemas/pipeline.py,sha256=qT-4NW2vOXmLGxwbGhSPbmDRoeoOhCA2I-j52pop4dU,1194
 mlrun/api/schemas/project.py,sha256=EhGRhSa_Pzzy8cxl_vtu93v2bydX9VHo12rM5rCSeOY,3935
@@ -164,7 +166,7 @@ mlrun/api/utils/projects/remotes/leader.py,sha256=D0V41b5rcr-w4AgEOCp0THG7ve2v4B
 mlrun/api/utils/projects/remotes/nop_follower.py,sha256=lMnq7uZcNbNc4A3ghhTdYzJjSo6V3sYnyqnFTbzpbP0,4601
 mlrun/api/utils/projects/remotes/nop_leader.py,sha256=0sLDBZcGtacQwygS1p7GD--EGFLFWohOGFyXmIiy5x0,3801
 mlrun/api/utils/singletons/__init__.py,sha256=B-yvMk7vTs3QqtlPkM4pnZk_B4QKTGnfNhYr2n4Z30M,571
-mlrun/api/utils/singletons/db.py,sha256=4jzYXBZYHGjEARXCbHFwIcvlqW-05efUZVvrsnXiLf8,1439
+mlrun/api/utils/singletons/db.py,sha256=y-v3JdLIHb_GtPvdzKH137jsflTav5TY9CafsSXfkN8,1202
 mlrun/api/utils/singletons/k8s.py,sha256=fxQuohr-iIGKQEAR6V5iZ3dka0438Tfm0xq_j2D5OGo,694
 mlrun/api/utils/singletons/logs_dir.py,sha256=zCALZ1N8IylkfvATfsbWfBtEPvWKl2AEQ2-QF-J0VWs,840
 mlrun/api/utils/singletons/project_member.py,sha256=1xlvJjbY-HvQuUV5rsQg9rMt-Y9MKU77Az5f98v-D28,1279
@@ -178,17 +180,17 @@ mlrun/artifacts/plots.py,sha256=0Jq5OgDB1srtUlcISF696E_q3S_D-mqeMe6Wyn1S6ZQ,1538
 mlrun/data_types/__init__.py,sha256=hBd2-QsTqesw3HUDoIUaUtCR1mWEcRc4V7Z1V8lNP0s,1087
 mlrun/data_types/data_types.py,sha256=xjsnUw144Lbkg4JqYLYym8qQpucYQUjphxay_BIc7VA,4647
 mlrun/data_types/infer.py,sha256=ErGpXigp8bG6hkPJfbJt8cJZxsZvj-bK0dLSiTlUYzk,5813
-mlrun/data_types/spark.py,sha256=f7ila08Hnckxs-RjhzubRbg2_ZzyNpvCV_Vrgak7yUM,9118
+mlrun/data_types/spark.py,sha256=BVM_FeWh1mKYLu6QILYfZRCOGefQzRJ2t0LeHo-pqwA,9426
 mlrun/datastore/__init__.py,sha256=lEcQKc3_Q2dfK6CTg2hwlTH3IoKQg-xE0WeMOanmxfo,3300
 mlrun/datastore/azure_blob.py,sha256=XFBpXZfwMKvv0rrYVP2J6RDqziNPHW6KqCbsY-VLzto,6946
-mlrun/datastore/base.py,sha256=5njgvevYPBEAAKJ0AFAh-fzebietmifi4sNuFYWAxa0,17304
+mlrun/datastore/base.py,sha256=5jODaKrJHSeCFQFHVBeREj3k9LaDLgyh5kW20-0ePGw,17575
 mlrun/datastore/datastore.py,sha256=l6CfmIYn7rdPQyRZ8kZu45cfyDZcDXDH5CwLKvsFlI8,7995
 mlrun/datastore/filestore.py,sha256=v9-BTJMgamGN_neoXNkIbNNr56UpgOewMEX70P8MTbc,3791
 mlrun/datastore/google_cloud_storage.py,sha256=9dtZt7Opytrra45urj6ObYa1fxf7UIOHXSoog4G9l7I,5114
 mlrun/datastore/inmem.py,sha256=o6Svv12k5PxJNT24nMgV9FBYEPCzD3OQzuuvU6OeCYY,2627
 mlrun/datastore/redis.py,sha256=pywWgjGJ51pmvKuwX1MI3ILWqh6WtgDs9igOQAhmIhc,4873
 mlrun/datastore/s3.py,sha256=fn1Yg5Rs33ugHlwTKvBfu_jMKqkSZWgHNw0ocQ5nO7A,7035
-mlrun/datastore/sources.py,sha256=89ckC7p5wmLdKCtZjSdWog6On97k8-JfhsUPKnRsucE,34025
+mlrun/datastore/sources.py,sha256=HqcD7cmI5tL5vsIA31g6lQ9qKv-P_QFN9V5NiNUowxc,34023
 mlrun/datastore/spark_udf.py,sha256=LFICcRzj7Yx03ttgQZImodEXjOZVvkcQd0OKwHuMLCg,1402
 mlrun/datastore/store_resources.py,sha256=u60_0LGXbagkG2UIm2o2pntWioF4kPu7FLMd0tlrR-8,6889
 mlrun/datastore/targets.py,sha256=3pA6jyHPzEHwfoMEDK3SObDAnLakpZU2AcJ30URO0KQ,62054
@@ -196,18 +198,18 @@ mlrun/datastore/utils.py,sha256=WNKBhgG2w9xAviyieGCB6hTnESEYzCwzgjcjz6uaWC4,1660
 mlrun/datastore/v3io.py,sha256=ArEYq78irv5bwlRwcLeVfFJvcwtXEamGsZ2I0sVt7Qg,8097
 mlrun/datastore/wasbfs/__init__.py,sha256=4CK7vZeJmC4Ak9cJjG07cGnk7jlkBlpCxw-ANfbBmzk,1343
 mlrun/datastore/wasbfs/fs.py,sha256=DEZgwrmQSPeR93MNlC9OCwuU9wvTSmIM534p5ccQzAE,6152
-mlrun/db/__init__.py,sha256=4_UTWWoVZU5B2D70NLTjYlMM7GVmLofnPRo5nGzT7XY,2810
-mlrun/db/base.py,sha256=FR1LnEP0Y4UlarcxM6wki4HeYnL0raccPkt_2tkEngU,14796
-mlrun/db/filedb.py,sha256=c-mpjJL_hcx2paRPcb4XdoXGyX6Dlv17tMNdsjULVJY,28411
-mlrun/db/httpdb.py,sha256=7pa5nwfwsptKakNBOFCWI7TQ-J_VoU3wv97KHzkC7b0,128262
-mlrun/db/sqldb.py,sha256=CdiF5D2t3oQLtphKNUzBY4DJSY-ThGSxS5ks108uxIQ,24292
+mlrun/db/__init__.py,sha256=HA6ahbEFrRd5I81eJQ1xrHmUch6JOy3JeEKMf7KrOTM,2816
+mlrun/db/base.py,sha256=eOWjqPEVq6UnffIw4s9XM_ESakcaVxLJHZzMUI2bOGU,14771
+mlrun/db/httpdb.py,sha256=OOgn0D6lPM1vHuzOM2oVpVTIGwHXD4HIoLCYNOHH1G4,129128
+mlrun/db/nopdb.py,sha256=J_7mlHfF9rSwGK1WL12_f4ciUcrRD3ZZ48AYisaNBmQ,12257
+mlrun/db/sqldb.py,sha256=sk2sJZpAAootYnTCDsUPo5JvPgqVSC6PanSODFelnh8,25370
 mlrun/feature_store/__init__.py,sha256=guoak0hQTMoE4YsWwyoyso0CYE9NSPoP0eUal4crfVA,1501
-mlrun/feature_store/api.py,sha256=2HGosxeChN_OZD04t3OX3x6pop1wPgJKcRze2mRgMRg,43335
-mlrun/feature_store/common.py,sha256=bFpPyFgGFuh-SwTl5AMUGD1mJr3GhVsFcDu-yIP5HAc,12807
+mlrun/feature_store/api.py,sha256=N2Vr1Z_hoQudXbYZ4_WmjjwdYbllfrAP5n1vodqs3_U,43761
+mlrun/feature_store/common.py,sha256=vgBs-jQKffjwOLNnOzrNnaU3yhzQTu-eh3X4yCy2ExQ,12807
 mlrun/feature_store/feature_set.py,sha256=SE6qCVpNCn2aImrVXRWLaE3SmD17iF49SOHj9uzltt4,47059
 mlrun/feature_store/feature_vector.py,sha256=hLOyejUK-itqZRzbqOQEQcijJHsJlypuHD7ZWQ6B5aE,21937
 mlrun/feature_store/ingestion.py,sha256=7qcBfNkDZsD4IXRdRN4OCSPIlpQDVo3WaO8pnAAYY6Q,11813
-mlrun/feature_store/steps.py,sha256=sqwY_IsUUA0ATNQryExjjzRwqgRKN4Ez66OktXVSoFY,23868
+mlrun/feature_store/steps.py,sha256=khDqxTusXDtBNfS7ESZh-QEsWu3_8FTR5ep_AY_Nvdo,28635
 mlrun/feature_store/retrieval/__init__.py,sha256=2b2WoPmH4TqcSSJ2OQYSjsDIcUe1KA5DnG7-wkrIayM,1232
 mlrun/feature_store/retrieval/base.py,sha256=38n5qY60V-wvn5vAVSd4qQlfkMMTMchX0c2TdZ8XqqE,25911
 mlrun/feature_store/retrieval/dask_merger.py,sha256=-xRgRPWUblu3KtVhEoL0YXEAcgkZrSUykDe31xnEzRI,4261
@@ -323,23 +325,23 @@ mlrun/platforms/__init__.py,sha256=NNh9MSxgjdi7QftO3DHp4axQnBeO5IjxD8oR13_jp3A,2
 mlrun/platforms/iguazio.py,sha256=gPOXoIRGcUCNmyMOddeHQT9DXW3835kB9AduZm_S0E0,23113
 mlrun/platforms/other.py,sha256=DwqbdbMNlkC2nEl1yJ0jF9WZJzjVu2IVPTC3bQ_W9Y0,11852
 mlrun/projects/__init__.py,sha256=dsXSDyqBhLi8OOfpdjLJpYOwC2fy_Q_Ki95Bk6gVSwc,1153
-mlrun/projects/operations.py,sha256=RZAyhsi3VuqvJjA3gwknuSpWhM1BfOFsZ-lGX9fIXCQ,17302
-mlrun/projects/pipelines.py,sha256=6iYVViKMUaEfqGIrVgQdl4UjtbDHe0nzlNXkLKte8fo,37882
-mlrun/projects/project.py,sha256=I0ll0JGWf13OfEeA2XgHgSBigLHHpbCCQtBBlegTsRM,103146
-mlrun/runtimes/__init__.py,sha256=1rNfzU3G76BxDkqvTVVrxlQkhLDSmlzv838_56OID7E,8973
-mlrun/runtimes/base.py,sha256=M5bJyOLqhEfSlkOD49SCpd_wH1bd4dMZle_03otYiG4,107571
+mlrun/projects/operations.py,sha256=rxxpSemZIp4_V6IC2gm2MnUH84aB4Q_7n_Y_BcbjK_c,17485
+mlrun/projects/pipelines.py,sha256=845_9berjVxxGB2OHkuGO2m5xRI-69_cp7Y2aqb5t9U,37899
+mlrun/projects/project.py,sha256=BP0hZ0DoflSkUNwJpp2eRVZvZwYen0DZkgSyw7AvCF0,104532
+mlrun/runtimes/__init__.py,sha256=NpPeD7XxDNxmeOqLSEmb7tUN59cYZlvw-DnJxla_peE,8773
+mlrun/runtimes/base.py,sha256=qYMiFbf3mAifyCnNzrlRAWhCNuds8Eh2SMEly0vJHf8,109502
 mlrun/runtimes/constants.py,sha256=Os_6YpWE8WMhigm1h3RKvn9Uo4LXNaEbA2xNSEDDEao,6689
 mlrun/runtimes/daskjob.py,sha256=Hol5_kzTfJBP4WN6orPlGSlSRDPec1HcBD8id_jSdPA,30403
 mlrun/runtimes/funcdoc.py,sha256=U5Hrkeo7i7CUnZFQ5g7-CyWMvXiQXPGrfsNdgsDAraQ,9190
-mlrun/runtimes/function.py,sha256=vKTRqXtHAm__vVvaUJVozzFWqCdLFzulYr0Yy8_ZiKM,69047
+mlrun/runtimes/function.py,sha256=Px4WakU2rYBDrgl8_GtoeABZIoCjn7HIKNYiv-YEDf4,70153
 mlrun/runtimes/function_reference.py,sha256=ikxeXeFPAn7PpfYR1DPBRFc9MSXXFM34uH5z6iRTz9M,4911
 mlrun/runtimes/generators.py,sha256=WvXzsfZZWqxLV1ncZ-lzFHZH2ITK6ICROgDv2vfQUs4,6530
-mlrun/runtimes/kubejob.py,sha256=nNxXZWKgLLgDdOSPaetZUngnSIp-yK9YUmBtCMUUQPU,17034
+mlrun/runtimes/kubejob.py,sha256=1ct6ox1cyWYSDk73VQctAEssArOPWpgr5-7Wnvm2R5o,16895
 mlrun/runtimes/local.py,sha256=gsLkfQSbvt1DjDuO6_9tjZC4AxOGBAypR3qJQomPqIY,18005
 mlrun/runtimes/nuclio.py,sha256=ih2iwrJieeEom3iW0yU99QG4_u_Bdv7VWgs1CK4Q1P8,2885
 mlrun/runtimes/pod.py,sha256=sWtw5wkme43VrQVEYDiU67uSC4zBbSyzqXZFYwWYewM,60648
 mlrun/runtimes/remotesparkjob.py,sha256=Lp2EN32GQ1rY_34MEepdUzRTg-AP-OKyUM7xcy0Npb8,7695
-mlrun/runtimes/serving.py,sha256=ylR-qQJe-EvMrHc3C9g7oYRi3WVDSnTAvFXezus3dtE,29852
+mlrun/runtimes/serving.py,sha256=bgqd97gJmfeLqtKKyPphWg6XeptIJACEZRnPpRx2Bt4,29852
 mlrun/runtimes/utils.py,sha256=r1FGNRETsZykdsy_DNpbyvqxsVcTetYRdHg-aelFt1k,21249
 mlrun/runtimes/mpijob/__init__.py,sha256=E0oiOsSYsRBBR0HV6e_EGaL_j2_dYEJwAcICnF8mOYU,790
 mlrun/runtimes/mpijob/abstract.py,sha256=_6CCDl4FXNpDzg1qTiEQGaqXyblx4gXYquAdKIG0fk0,14782
@@ -347,9 +349,8 @@ mlrun/runtimes/mpijob/v1.py,sha256=o-ZDw3O0HOzTMZ3lI3gMjGG0NYY0Ux3RqL0Kz9K3eV4,1
 mlrun/runtimes/mpijob/v1alpha1.py,sha256=4rMpJk1C_gL6QtOmEFcgcmaCkHcor9mdKvIL46fEKFw,8460
 mlrun/runtimes/package/__init__.py,sha256=EHFvxDWnRkXqjmdv9Mj3O-nCYAFLbRsNgX5zR0ElwAk,673
 mlrun/runtimes/package/context_handler.py,sha256=5br30fRHHhWQJgSWF2zqf0M8XOxVr27ZkOFWVDgf4f0,27044
-mlrun/runtimes/sparkjob/__init__.py,sha256=cDzK8jEJvNsSSujDmEt03h1Mnlj7MUvdCEUXDNAb1wc,788
-mlrun/runtimes/sparkjob/abstract.py,sha256=o5ykIx4GkwLU2QoQNQS9oNm5qebB17sOVblbh0ZDjG8,35976
-mlrun/runtimes/sparkjob/spark2job.py,sha256=StWv5hk99pHslkIpOzz5T-szCYN2kkC20ixEdlD3yIA,2033
+mlrun/runtimes/sparkjob/__init__.py,sha256=7damn-nBbM0unppemEuVwZxCAGwXq7EzXECMJ9AaRPY,751
+mlrun/runtimes/sparkjob/abstract.py,sha256=NyHcddXo97s2S3hjHmDUoEDaGd8AjmKltd5PJTVd3BA,35869
 mlrun/runtimes/sparkjob/spark3job.py,sha256=JzwinNIiPXBapSC-gAsbhf81oouxhx41U5oQwoNeYhM,28722
 mlrun/serving/__init__.py,sha256=FOIyvEwEOH6QuJovcDrrP5VSdLeKoqummSlfhNMvFXg,1078
 mlrun/serving/merger.py,sha256=oRX6gJ3LUVeCJcTpJ1nwh4d1tk5FrSZshkXcH-y-dd4,6116
@@ -357,7 +358,7 @@ mlrun/serving/remote.py,sha256=HJS9bz5_mUuaKGsNbqRKpo-eVHKbc1fIEOEsECPmnko,18038
 mlrun/serving/routers.py,sha256=avhQ5cgogYA_TP_6kgzC_U2VQW24-yWWUXJGP449-Qg,55209
 mlrun/serving/server.py,sha256=ePfYiefH3HkNag5qHDzDyoowaNNMbjJ9YKpPS9wX9Xw,19005
 mlrun/serving/serving_wrapper.py,sha256=ivoSlFNuC92plPHRfMzHr-hdN-jGqxneGiufJdgoxVc,836
-mlrun/serving/states.py,sha256=b5pNgAQWW5o60acjgp5fLI24FaFVUBfdYwO1HOCbJTw,53922
+mlrun/serving/states.py,sha256=7ykG0Q66OQvRF32nQDQGwkmVYEyHczCvNrQxneVdUdM,53940
 mlrun/serving/utils.py,sha256=1fLfn07R4-RlUcdUgYYwUTFF47NozQmV9BnZVbu4-lU,3468
 mlrun/serving/v1_serving.py,sha256=UF6Ydgyo-DuHZ4DkrEdNMQl5oopVaZqm4EfBMBSvhDY,11814
 mlrun/serving/v2_serving.py,sha256=uIKQnTITJozqk6WrmJhO0hKfjCl8FLYc_IpCEYt5elA,21441
@@ -366,7 +367,7 @@ mlrun/utils/async_http.py,sha256=AvI4a9D0frI6XxHvqnQ4AzEXopezo90zZHvsOAJJt6g,103
 mlrun/utils/azure_vault.py,sha256=atz9fIu1vfgWAND1g6flRQDhrX-xBxZ_6MOWm34Onh0,3456
 mlrun/utils/clones.py,sha256=lKgQeibQ3xQ9LH5bZTZCoYCXoOx_2mYNGD5A-OOxwAQ,6245
 mlrun/utils/db.py,sha256=SqDrvIpGX_qHs0MNx_PWQNRcUhHaKLngoHFKLW3m5j0,1662
-mlrun/utils/helpers.py,sha256=V7mWXFogvtNGPIrx_D7AdT7wNixCpU1XVniAHHQXZkA,41426
+mlrun/utils/helpers.py,sha256=r128kqmGIHpDaPnSlnryLUmlqN81x0sPyjFvJs_SJ9o,41425
 mlrun/utils/http.py,sha256=FVgdYV8Np3TKbqszEtXrlgwqNyjScTMLAjJhlsWLifo,7093
 mlrun/utils/logger.py,sha256=O4pH45kdz0cO-e4eKeRxbZUIRLGn2X0BJcjSY9cWB04,5637
 mlrun/utils/model_monitoring.py,sha256=wS6T9KgG-XCB21MNc6edNODEqFBBqzi5bUgSvPpKt-c,7875
@@ -375,19 +376,19 @@ mlrun/utils/singleton.py,sha256=kV9HhbzdmXo8gGKBPdbHWspRDutlkKX1ZEn96xs_LRs,883
 mlrun/utils/v3io_clients.py,sha256=ekxhjfcUnjV8cejQYTQUHnhYOKHA4KekIdSaywydTdM,1319
 mlrun/utils/vault.py,sha256=1zr_3gkXoerYID_4Z5vYkBO9FeCKd43Qm-XlmGBk_Ok,9957
 mlrun/utils/notifications/__init__.py,sha256=CgWLxslSqhTH-tExi_UR4AT-ipjkGUF4gwf9mKcNua8,990
-mlrun/utils/notifications/notification_pusher.py,sha256=C5PZYwHPHmwnbES1zTVGKz4_wbX7cR3pDl1PDqjM4hA,7093
-mlrun/utils/notifications/notification/__init__.py,sha256=s7owPm6jGxuN_JBU0nFac6iD6BuLhEbnhq-NMF1TLf8,1833
-mlrun/utils/notifications/notification/base.py,sha256=dkUmwB1-o1qpFs9MCIOkYAZYsODelkU2xm1iRM3wJlU,2067
-mlrun/utils/notifications/notification/console.py,sha256=QCdulv9i8nCCYAA8ftmf22vvZwBp3SRBCKiB0v1WBQY,1901
-mlrun/utils/notifications/notification/git.py,sha256=qI5Kd4U0PX8vfIVT_yjBIrToiJYzCS-XvY96Xck2Nkc,4510
-mlrun/utils/notifications/notification/ipython.py,sha256=jsM_tTV3ZzXv5NoQN2anfZJDTYsk95NDYVQnWFc9Zow,1951
-mlrun/utils/notifications/notification/slack.py,sha256=wt6RlqJu9QC99Ki-3HNYmizYTyAnIVimFQ7bf-fYX2E,3650
+mlrun/utils/notifications/notification_pusher.py,sha256=WdkBRLg_upmcInkwYcjRJkuRexKMaZ_a_47d0yMfhSs,12297
+mlrun/utils/notifications/notification/__init__.py,sha256=iNF31hiEkenkp1VmJ4keyjixS17-gZ0oGuWNO7tlnm4,1803
+mlrun/utils/notifications/notification/base.py,sha256=0q419xnXHH6raEcpR8j1uvBUn1jHcJmIAfjFNt2XkKQ,2162
+mlrun/utils/notifications/notification/console.py,sha256=nzucPPOj3jFceJKZSV4UJLT0mu8V1yVqGUl3MbnlwZc,1962
+mlrun/utils/notifications/notification/git.py,sha256=iXjjBbJMWA741Z6aRNpGGg8FjZC78jzS2uR9UYM0yHY,4733
+mlrun/utils/notifications/notification/ipython.py,sha256=zP1jAE39Dn6rrbQIxi4R4iDhMLFBQ_GJrEtMMj8G80E,2012
+mlrun/utils/notifications/notification/slack.py,sha256=QnXQvAMOe3GUUKnlrNy_fEtUEjdG8n80FDl4hYdqXyc,3736
 mlrun/utils/version/__init__.py,sha256=hwfJgGWYGWFpepVGI1GbuCPqqEFGRbgguJg5sC0v4TU,614
-mlrun/utils/version/version.json,sha256=OHSLfR0bhakvVW7se9O4rFHj0j5Crr1BNTKoUB7gEl8,88
+mlrun/utils/version/version.json,sha256=hsn9H7rpi0XmRAbCXb0oM4t1uXJfVkQpYIiKVM_uoSA,88
 mlrun/utils/version/version.py,sha256=O4Q4kwtKlI73oK7oBPuz4SVkUI8BC11E9DJIKHT91kU,1970
-mlrun-1.3.1rc5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.3.1rc5.dist-info/METADATA,sha256=gT36rvjdaIDRiVCXykY5DEACEaeNxECuyqjoFBuOBwo,16975
-mlrun-1.3.1rc5.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-mlrun-1.3.1rc5.dist-info/entry_points.txt,sha256=ZbXmb36B9JmK7EaleP8MIAbZSOQXQV0iwKR6si0HUWk,47
-mlrun-1.3.1rc5.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.3.1rc5.dist-info/RECORD,,
+mlrun-1.4.0rc2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.4.0rc2.dist-info/METADATA,sha256=_at3cBifvsdJaad6mIL46mrg00ix_KEsCKQ8gb4b6ag,17177
+mlrun-1.4.0rc2.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+mlrun-1.4.0rc2.dist-info/entry_points.txt,sha256=ZbXmb36B9JmK7EaleP8MIAbZSOQXQV0iwKR6si0HUWk,47
+mlrun-1.4.0rc2.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.4.0rc2.dist-info/RECORD,,