mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

mlrun/utils/notifications/notification/git.py

@@ -30,6 +30,27 @@ class GitNotification(NotificationBase):
     API/Client notification for setting a rich run statuses git issue comment (github/gitlab)
     """
 
+    @classmethod
+    def validate_params(cls, params):
+        git_repo = params.get("repo", None)
+        git_issue = params.get("issue", None)
+        git_merge_request = params.get("merge_request", None)
+        token = (
+            params.get("token", None)
+            or params.get("GIT_TOKEN", None)
+            or params.get("GITHUB_TOKEN", None)
+        )
+        if not git_repo:
+            raise ValueError("Parameter 'repo' is required for GitNotification")
+
+        if not token:
+            raise ValueError("Parameter 'token' is required for GitNotification")
+
+        if not git_issue and not git_merge_request:
+            raise ValueError(
+                "At least one of 'issue' or 'merge_request' is required for GitNotification"
+            )
+
     async def push(
         self,
         message: str,

mlrun/utils/notifications/notification/slack.py

@@ -35,6 +35,14 @@ class SlackNotification(NotificationBase):
         "skipped": ":zzz:",
     }
 
+    @classmethod
+    def validate_params(cls, params):
+        webhook = params.get("webhook", None) or mlrun.get_secret_or_env(
+            "SLACK_WEBHOOK"
+        )
+        if not webhook:
+            raise ValueError("Parameter 'webhook' is required for SlackNotification")
+
     async def push(
         self,
         message: str,
@@ -153,20 +161,7 @@ class SlackNotification(NotificationBase):
             data_text = "\n".join(data_lines)
             line.append(self._get_slack_row(f"*Event data:*\n{data_text}"))
 
-        if (
-            event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
-        ):  # JOB entity
-            uid = event_data.value_dict.get("uid")
-            url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
-            overview_type = "Job overview"
-        else:  # MODEL entity
-            model_name = event_data.value_dict.get("model")
-            model_endpoint_id = event_data.value_dict.get("model_endpoint_id")
-            url = mlrun.utils.helpers.get_model_endpoint_url(
-                alert.project, model_name, model_endpoint_id
-            )
-            overview_type = "Model endpoint"
-
+        overview_type, url = self._get_overview_type_and_url(alert, event_data)
         line.append(self._get_slack_row(f"*Overview:*\n<{url}|*{overview_type}*>"))
 
         return line

mlrun/utils/notifications/notification/webhook.py

@@ -28,6 +28,12 @@ class WebhookNotification(NotificationBase):
     API/Client notification for sending run statuses in a http request
     """
 
+    @classmethod
+    def validate_params(cls, params):
+        url = params.get("url", None)
+        if not url:
+            raise ValueError("Parameter 'url' is required for WebhookNotification")
+
     async def push(
         self,
         message: str,
@@ -63,7 +69,7 @@ class WebhookNotification(NotificationBase):
             request_body["custom_html"] = custom_html
 
         if override_body:
-            request_body = override_body
+            request_body = self._serialize_runs_in_request_body(override_body, runs)
 
         # Specify the `verify_ssl` parameter value only for HTTPS urls.
         # The `ClientSession` allows using `ssl=None` for the default SSL check,
@@ -77,3 +83,37 @@ class WebhookNotification(NotificationBase):
                 url, headers=headers, json=request_body, ssl=verify_ssl
             )
             response.raise_for_status()
+
+    @staticmethod
+    def _serialize_runs_in_request_body(override_body, runs):
+        str_parsed_runs = ""
+        runs = runs or []
+
+        def parse_runs():
+            parsed_runs = []
+            for run in runs:
+                if hasattr(run, "to_dict"):
+                    run = run.to_dict()
+                if isinstance(run, dict):
+                    parsed_run = {
+                        "project": run["metadata"]["project"],
+                        "name": run["metadata"]["name"],
+                        "host": run["metadata"]["labels"]["host"],
+                        "status": {"state": run["status"]["state"]},
+                    }
+                    if run["status"].get("error", None):
+                        parsed_run["status"]["error"] = run["status"]["error"]
+                    elif run["status"].get("results", None):
+                        parsed_run["status"]["results"] = run["status"]["results"]
+                    parsed_runs.append(parsed_run)
+            return str(parsed_runs)
+
+        if isinstance(override_body, dict):
+            for key, value in override_body.items():
+                if "{{ runs }}" or "{{runs}}" in value:
+                    if not str_parsed_runs:
+                        str_parsed_runs = parse_runs()
+                    override_body[key] = value.replace(
+                        "{{ runs }}", str_parsed_runs
+                    ).replace("{{runs}}", str_parsed_runs)
+        return override_body

mlrun/utils/notifications/notification_pusher.py

@@ -20,9 +20,9 @@ import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
-import kfp
 import mlrun_pipelines.common.ops
 import mlrun_pipelines.models
+import mlrun_pipelines.utils
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.runtimes.constants
@@ -397,7 +397,7 @@ class NotificationPusher(_NotificationPusherBase):
             try:
                 _run = db.list_runs(
                     project=run.metadata.project,
-                    labels=f"mlrun_constants.MLRunInternalLabels.runner_pod={_step.node_name}",
+                    labels=f"{mlrun_constants.MLRunInternalLabels.runner_pod}={_step.node_name}",
                 )[0]
             except IndexError:
                 _run = {
@@ -484,13 +484,7 @@ class NotificationPusher(_NotificationPusherBase):
     def _get_workflow_manifest(
         workflow_id: str,
     ) -> typing.Optional[mlrun_pipelines.models.PipelineManifest]:
-        kfp_url = mlrun.mlconf.resolve_kfp_url(mlrun.mlconf.namespace)
-        if not kfp_url:
-            raise mlrun.errors.MLRunNotFoundError(
-                "KubeFlow Pipelines is not configured"
-            )
-
-        kfp_client = kfp.Client(host=kfp_url)
+        kfp_client = mlrun_pipelines.utils.get_client(mlrun.mlconf)
 
         # arbitrary timeout of 5 seconds, the workflow should be done by now
         kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)

mlrun/utils/regex.py

@@ -92,3 +92,12 @@ artifact_key = [r"[^\/\\]+$"]
 # must be alphanumeric or _
 # max 256 length
 v3io_stream_consumer_group = [r"^(?!_)[a-zA-Z0-9_]{1,256}$"]
+
+# URI patterns
+run_uri_pattern = r"^(?P<project>.*)@(?P<uid>.*)\#(?P<iteration>.*?)(:(?P<tag>.*))?$"
+
+artifact_uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?(:(?P<tag>.*?))?(@(?P<tree>.*))?$"
+
+artifact_producer_uri_pattern = (
+    r"^((?P<project>.*)/)?(?P<uid>.*?)(\-(?P<iteration>.*?))?$"
+)

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "f869a5513ea7c9f4ccdaddad6589274eec39f0a4",
-  "version": "1.7.0-rc20"
+  "git_commit": "71818c39b83a7e8c396aa2c37f1cd4f3e192dd2f",
+  "version": "1.7.0-rc28"
 }

{mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc20
+Version: 1.7.0rc28
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -28,30 +28,30 @@ Requires-Dist: aiohttp-retry ~=2.8
 Requires-Dist: click ~=8.1
 Requires-Dist: nest-asyncio ~=1.0
 Requires-Dist: ipython ~=8.10
-Requires-Dist: nuclio-jupyter ~=0.9.17
+Requires-Dist: nuclio-jupyter ~=0.10.0
 Requires-Dist: numpy <1.27.0,>=1.16.5
 Requires-Dist: pandas <2.2,>=1.2
 Requires-Dist: pyarrow <15,>=10.0
-Requires-Dist: pyyaml ~=5.1
+Requires-Dist: pyyaml <7,>=5.4.1
 Requires-Dist: requests ~=2.31
 Requires-Dist: tabulate ~=0.8.6
 Requires-Dist: v3io ~=0.6.4
 Requires-Dist: pydantic <1.10.15,>=1.10.8
 Requires-Dist: mergedeep ~=1.3
-Requires-Dist: v3io-frames ~=0.10.12
+Requires-Dist: v3io-frames ~=0.10.14
 Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
 Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.17
+Requires-Dist: storey ~=1.7.20
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
 Requires-Dist: setuptools ~=69.1
 Requires-Dist: deprecated ~=1.2
 Requires-Dist: jinja2 >=3.1.3,~=3.1
 Requires-Dist: orjson <4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.0
-Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.0
+Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.2
+Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.2
 Provides-Extra: alibaba-oss
 Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
 Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
@@ -81,6 +81,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
 Requires-Dist: pyopenssl >=23 ; extra == 'all'
 Requires-Dist: redis ~=4.3 ; extra == 'all'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'all'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'all'
 Provides-Extra: api
@@ -88,13 +89,14 @@ Requires-Dist: uvicorn ~=0.27.1 ; extra == 'api'
 Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'api'
 Requires-Dist: apscheduler <4,>=3.10.3 ; extra == 'api'
 Requires-Dist: objgraph ~=3.6 ; extra == 'api'
-Requires-Dist: igz-mgmt ~=0.1.3 ; extra == 'api'
+Requires-Dist: igz-mgmt ~=0.2.0 ; extra == 'api'
 Requires-Dist: humanfriendly ~=10.0 ; extra == 'api'
 Requires-Dist: fastapi ~=0.110.0 ; extra == 'api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'api'
 Requires-Dist: pymysql ~=1.0 ; extra == 'api'
 Requires-Dist: alembic ~=1.9 ; extra == 'api'
 Requires-Dist: timelength ~=1.1 ; extra == 'api'
+Requires-Dist: memray ~=1.12 ; extra == 'api'
 Provides-Extra: azure-blob-storage
 Requires-Dist: msrest ~=0.6.21 ; extra == 'azure-blob-storage'
 Requires-Dist: azure-core ~=1.24 ; extra == 'azure-blob-storage'
@@ -129,6 +131,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete'
 Requires-Dist: redis ~=4.3 ; extra == 'complete'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete'
 Provides-Extra: complete-api
@@ -150,8 +153,9 @@ Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'complete-api'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete-api'
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
 Requires-Dist: humanfriendly ~=10.0 ; extra == 'complete-api'
-Requires-Dist: igz-mgmt ~=0.1.3 ; extra == 'complete-api'
+Requires-Dist: igz-mgmt ~=0.2.0 ; extra == 'complete-api'
 Requires-Dist: kafka-python ~=2.0 ; extra == 'complete-api'
+Requires-Dist: memray ~=1.12 ; extra == 'complete-api'
 Requires-Dist: mlflow ~=2.8 ; extra == 'complete-api'
 Requires-Dist: msrest ~=0.6.21 ; extra == 'complete-api'
 Requires-Dist: objgraph ~=3.6 ; extra == 'complete-api'
@@ -162,6 +166,7 @@ Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
 Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete-api'
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
@@ -194,6 +199,8 @@ Provides-Extra: s3
 Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
 Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
 Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 's3'
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'snowflake'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
 Provides-Extra: tdengine