mlrun 1.7.0rc22__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

mlrun/utils/notifications/notification/base.py

@@ -28,6 +28,10 @@ class NotificationBase:
         self.name = name
         self.params = params or {}
 
+    @classmethod
+    def validate_params(cls, params):
+        pass
+
     @property
     def active(self) -> bool:
         return True
@@ -69,16 +73,27 @@ class NotificationBase:
         if custom_html:
             return custom_html
 
-        if self.name:
-            message = f"{self.name}: {message}"
-
         if alert:
             if not event_data:
                 return f"[{severity}] {message}"
-            return (
-                f"[{severity}] {message} for project {alert.project} "
-                f"UID {event_data.entity.ids[0]}. Values {event_data.value_dict}"
-            )
+
+            html = f"<h3>[{severity}] {message}</h3>"
+            html += f"<br>{alert.name} alert has occurred<br>"
+            html += f"<br><h4>Project:</h4>{alert.project}<br>"
+            html += f"<br><h4>ID:</h4>{event_data.entity.ids[0]}<br>"
+            html += f"<br><h4>Summary:</h4>{mlrun.utils.helpers.format_alert_summary(alert, event_data)}<br>"
+
+            if event_data.value_dict:
+                html += "<br><h4>Event data:</h4>"
+                for key, value in event_data.value_dict.items():
+                    html += f"{key}: {value}<br>"
+
+            overview_type, url = self._get_overview_type_and_url(alert, event_data)
+            html += f"<br><h4>Overview:</h4><a href={url}>{overview_type}</a>"
+            return html
+
+        if self.name:
+            message = f"{self.name}: {message}"
 
         if not runs:
             return f"[{severity}] {message}"
@@ -90,3 +105,24 @@ class NotificationBase:
         html += "<br>click the hyper links below to see detailed results<br>"
         html += runs.show(display=False, short=True)
         return html
+
+    def _get_overview_type_and_url(
+        self,
+        alert: mlrun.common.schemas.AlertConfig,
+        event_data: mlrun.common.schemas.Event,
+    ) -> (str, str):
+        if (
+            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
+        ):  # JOB entity
+            uid = event_data.value_dict.get("uid")
+            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
+            overview_type = "Job overview"
+        else:  # MODEL entity
+            model_name = event_data.value_dict.get("model")
+            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
+            url = mlrun.utils.helpers.get_model_endpoint_url(
+                alert.project, model_name, model_endpoint_id
+            )
+            overview_type = "Model endpoint"
+
+        return overview_type, url

mlrun/utils/notifications/notification/git.py

@@ -30,6 +30,27 @@ class GitNotification(NotificationBase):
     API/Client notification for setting a rich run statuses git issue comment (github/gitlab)
     """
 
+    @classmethod
+    def validate_params(cls, params):
+        git_repo = params.get("repo", None)
+        git_issue = params.get("issue", None)
+        git_merge_request = params.get("merge_request", None)
+        token = (
+            params.get("token", None)
+            or params.get("GIT_TOKEN", None)
+            or params.get("GITHUB_TOKEN", None)
+        )
+        if not git_repo:
+            raise ValueError("Parameter 'repo' is required for GitNotification")
+
+        if not token:
+            raise ValueError("Parameter 'token' is required for GitNotification")
+
+        if not git_issue and not git_merge_request:
+            raise ValueError(
+                "At least one of 'issue' or 'merge_request' is required for GitNotification"
+            )
+
     async def push(
         self,
         message: str,

mlrun/utils/notifications/notification/slack.py

@@ -35,6 +35,14 @@ class SlackNotification(NotificationBase):
         "skipped": ":zzz:",
     }
 
+    @classmethod
+    def validate_params(cls, params):
+        webhook = params.get("webhook", None) or mlrun.get_secret_or_env(
+            "SLACK_WEBHOOK"
+        )
+        if not webhook:
+            raise ValueError("Parameter 'webhook' is required for SlackNotification")
+
     async def push(
         self,
         message: str,
@@ -153,20 +161,7 @@ class SlackNotification(NotificationBase):
             data_text = "\n".join(data_lines)
             line.append(self._get_slack_row(f"*Event data:*\n{data_text}"))
 
-        if (
-            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
-        ):  # JOB entity
-            uid = event_data.value_dict.get("uid")
-            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
-            overview_type = "Job overview"
-        else:  # MODEL entity
-            model_name = event_data.value_dict.get("model")
-            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
-            url = mlrun.utils.helpers.get_model_endpoint_url(
-                alert.project, model_name, model_endpoint_id
-            )
-            overview_type = "Model endpoint"
-
+        overview_type, url = self._get_overview_type_and_url(alert, event_data)
         line.append(self._get_slack_row(f"*Overview:*\n<{url}|*{overview_type}*>"))
 
         return line

mlrun/utils/notifications/notification/webhook.py

@@ -28,6 +28,12 @@ class WebhookNotification(NotificationBase):
     API/Client notification for sending run statuses in a http request
     """
 
+    @classmethod
+    def validate_params(cls, params):
+        url = params.get("url", None)
+        if not url:
+            raise ValueError("Parameter 'url' is required for WebhookNotification")
+
     async def push(
         self,
         message: str,
@@ -63,7 +69,7 @@ class WebhookNotification(NotificationBase):
             request_body["custom_html"] = custom_html
 
         if override_body:
-            request_body = override_body
+            request_body = self._serialize_runs_in_request_body(override_body, runs)
 
         # Specify the `verify_ssl` parameter value only for HTTPS urls.
         # The `ClientSession` allows using `ssl=None` for the default SSL check,
@@ -77,3 +83,37 @@ class WebhookNotification(NotificationBase):
                 url, headers=headers, json=request_body, ssl=verify_ssl
             )
             response.raise_for_status()
+
+    @staticmethod
+    def _serialize_runs_in_request_body(override_body, runs):
+        str_parsed_runs = ""
+        runs = runs or []
+
+        def parse_runs():
+            parsed_runs = []
+            for run in runs:
+                if hasattr(run, "to_dict"):
+                    run = run.to_dict()
+                if isinstance(run, dict):
+                    parsed_run = {
+                        "project": run["metadata"]["project"],
+                        "name": run["metadata"]["name"],
+                        "host": run["metadata"]["labels"]["host"],
+                        "status": {"state": run["status"]["state"]},
+                    }
+                    if run["status"].get("error", None):
+                        parsed_run["status"]["error"] = run["status"]["error"]
+                    elif run["status"].get("results", None):
+                        parsed_run["status"]["results"] = run["status"]["results"]
+                    parsed_runs.append(parsed_run)
+            return str(parsed_runs)
+
+        if isinstance(override_body, dict):
+            for key, value in override_body.items():
+                if "{{ runs }}" or "{{runs}}" in value:
+                    if not str_parsed_runs:
+                        str_parsed_runs = parse_runs()
+                    override_body[key] = value.replace(
+                        "{{ runs }}", str_parsed_runs
+                    ).replace("{{runs}}", str_parsed_runs)
+        return override_body

mlrun/utils/notifications/notification_pusher.py

@@ -20,9 +20,9 @@ import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
-import kfp
 import mlrun_pipelines.common.ops
 import mlrun_pipelines.models
+import mlrun_pipelines.utils
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.runtimes.constants
@@ -397,7 +397,7 @@ class NotificationPusher(_NotificationPusherBase):
             try:
                 _run = db.list_runs(
                     project=run.metadata.project,
-                    labels=f"mlrun_constants.MLRunInternalLabels.runner_pod={_step.node_name}",
+                    labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
                 )[0]
             except IndexError:
                 _run = {
@@ -484,13 +484,7 @@ class NotificationPusher(_NotificationPusherBase):
     def _get_workflow_manifest(
         workflow_id: str,
     ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_url = mlrun.mlconf.resolve_kfp_url(mlrun.mlconf.namespace)
-        if not kfp_url:
-            raise mlrun.errors.MLRunNotFoundError(
-                "KubeFlow Pipelines is not configured"
-            )
-
-        kfp_client = kfp.Client(host=kfp_url)
+        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf)
 
         # arbitrary timeout of 5 seconds, the workflow should be done by now
         kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "0cbcbb9417af7ac58072ad981672602728fee57e",
-  "version": "1.7.0-rc22"
+  "git_commit": "71818c39b83a7e8c396aa2c37f1cd4f3e192dd2f",
+  "version": "1.7.0-rc28"
 }

{mlrun-1.7.0rc22.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc22
+Version: 1.7.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -28,30 +28,30 @@ Requires-Dist: aiohttp-retry ~=2.8
 Requires-Dist: click ~=8.1
 Requires-Dist: nest-asyncio ~=1.0
 Requires-Dist: ipython ~=8.10
-Requires-Dist: nuclio-jupyter ~=0.9.17
+Requires-Dist: nuclio-jupyter ~=0.10.0
 Requires-Dist: numpy <1.27.0,>=1.16.5
 Requires-Dist: pandas <2.2,>=1.2
 Requires-Dist: pyarrow <15,>=10.0
-Requires-Dist: pyyaml ~=5.1
+Requires-Dist: pyyaml <7,>=5.4.1
 Requires-Dist: requests ~=2.31
 Requires-Dist: tabulate ~=0.8.6
 Requires-Dist: v3io ~=0.6.4
 Requires-Dist: pydantic <1.10.15,>=1.10.8
 Requires-Dist: mergedeep ~=1.3
-Requires-Dist: v3io-frames ~=0.10.12
+Requires-Dist: v3io-frames ~=0.10.14
 Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
 Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.17
+Requires-Dist: storey ~=1.7.20
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
 Requires-Dist: setuptools ~=69.1
 Requires-Dist: deprecated ~=1.2
 Requires-Dist: jinja2 >=3.1.3,~=3.1
 Requires-Dist: orjson <4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common >0.1.0,~=0.1.1
-Requires-Dist: mlrun-pipelines-kfp-v1-8 >0.1.0,~=0.1.1
+Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.2
+Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.2
 Provides-Extra: alibaba-oss
 Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
 Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
@@ -81,6 +81,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
 Requires-Dist: pyopenssl >=23 ; extra == 'all'
 Requires-Dist: redis ~=4.3 ; extra == 'all'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'all'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'all'
 Provides-Extra: api
@@ -130,6 +131,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete'
 Requires-Dist: redis ~=4.3 ; extra == 'complete'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete'
 Provides-Extra: complete-api
@@ -164,6 +166,7 @@ Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
 Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete-api'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
@@ -196,6 +199,8 @@ Provides-Extra: s3
 Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
 Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 's3'
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'snowflake'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
 Provides-Extra: tdengine