mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

mlrun/utils/helpers.py

@@ -39,7 +39,6 @@ import pandas
 import semver
 import yaml
 from dateutil import parser
-from deprecated import deprecated
 from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -76,19 +75,6 @@ class OverwriteBuildParamsWarning(FutureWarning):
     pass
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'parse_versioned_object_uri' will be removed from this file in 1.7.0, use "
-    "'mlrun.common.helpers.parse_versioned_object_uri' instead",
-    category=FutureWarning,
-)
-def parse_versioned_object_uri(uri: str, default_project: str = ""):
-    return mlrun.common.helpers.parse_versioned_object_uri(
-        uri=uri, default_project=default_project
-    )
-
-
 class StorePrefix:
     """map mlrun store objects to prefixes"""
 
@@ -119,14 +105,9 @@ class StorePrefix:
 
 
 def get_artifact_target(item: dict, project=None):
-    if is_legacy_artifact(item):
-        db_key = item.get("db_key")
-        project_str = project or item.get("project")
-        tree = item.get("tree")
-    else:
-        db_key = item["spec"].get("db_key")
-        project_str = project or item["metadata"].get("project")
-        tree = item["metadata"].get("tree")
+    db_key = item["spec"].get("db_key")
+    project_str = project or item["metadata"].get("project")
+    tree = item["metadata"].get("tree")
 
     kind = item.get("kind")
     if kind in ["dataset", "model", "artifact"] and db_key:
@@ -135,11 +116,15 @@ def get_artifact_target(item: dict, project=None):
             target = f"{target}@{tree}"
         return target
 
-    return (
-        item.get("target_path")
-        if is_legacy_artifact(item)
-        else item["spec"].get("target_path")
-    )
+    return item["spec"].get("target_path")
+
+
+# TODO: left for migrations testing purposes. Remove in 1.8.0.
+def is_legacy_artifact(artifact):
+    if isinstance(artifact, dict):
+        return "metadata" not in artifact
+    else:
+        return not hasattr(artifact, "metadata")
 
 
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
@@ -1018,8 +1003,9 @@ def get_ui_url(project, uid=None):
 def get_workflow_url(project, id=None):
     url = ""
     if mlrun.mlconf.resolve_ui_url():
-        url = "{}/{}/{}/jobs/monitor-workflows/workflow/{}".format(
-            mlrun.mlconf.resolve_ui_url(), mlrun.mlconf.ui.projects_prefix, project, id
+        url = (
+            f"{mlrun.mlconf.resolve_ui_url()}/{mlrun.mlconf.ui.projects_prefix}"
+            f"/{project}/jobs/monitor-workflows/workflow/{id}"
         )
     return url
 
@@ -1291,13 +1277,6 @@ def str_to_timestamp(time_str: str, now_time: Timestamp = None):
     return Timestamp(time_str)
 
 
-def is_legacy_artifact(artifact):
-    if isinstance(artifact, dict):
-        return "metadata" not in artifact
-    else:
-        return not hasattr(artifact, "metadata")
-
-
 def is_link_artifact(artifact):
     if isinstance(artifact, dict):
         return (
@@ -1345,16 +1324,16 @@ def format_run(run: dict, with_project=False) -> dict:
     # as observed https://jira.iguazeng.com/browse/ML-5195
     # set to unknown to ensure a status is returned
     if run["status"] is None:
-        run["status"] = inflection.titleize(mlrun.runtimes.constants.RunStates.unknown)
+        run["status"] = inflection.titleize(
+            mlrun.common.runtimes.constants.RunStates.unknown
+        )
 
     return run
 
 
 def get_in_artifact(artifact: dict, key, default=None, raise_on_missing=False):
     """artifact can be dict or Artifact object"""
-    if is_legacy_artifact(artifact):
-        return artifact.get(key, default)
-    elif key == "kind":
+    if key == "kind":
         return artifact.get(key, default)
     else:
         for block in ["metadata", "spec", "status"]:
@@ -1596,3 +1575,11 @@ def get_serving_spec():
             )
     spec = json.loads(data)
     return spec
+
+
+def additional_filters_warning(additional_filters, class_name):
+    if additional_filters and any(additional_filters):
+        mlrun.utils.logger.warn(
+            f"additional_filters parameter is not supported in {class_name},"
+            f" parameter has been ignored."
+        )

mlrun/utils/notifications/notification/slack.py

@@ -72,12 +72,7 @@ class SlackNotification(NotificationBase):
         event_data: mlrun.common.schemas.Event = None,
     ) -> dict:
         data = {
-            "blocks": [
-                {
-                    "type": "header",
-                    "text": {"type": "plain_text", "text": f"[{severity}] {message}"},
-                },
-            ]
+            "blocks": self._generate_slack_header_blocks(severity, message),
         }
         if self.name:
             data["blocks"].append(
@@ -106,6 +101,32 @@ class SlackNotification(NotificationBase):
 
         return data
 
+    def _generate_slack_header_blocks(self, severity: str, message: str):
+        header_text = block_text = f"[{severity}] {message}"
+        section_text = None
+
+        # Slack doesn't allow headers to be longer than 150 characters
+        # If there's a comma in the message, split the message at the comma
+        # Otherwise, split the message at 150 characters
+        if len(block_text) > 150:
+            if ", " in block_text and block_text.index(", ") < 149:
+                header_text = block_text.split(",")[0]
+                section_text = block_text[len(header_text) + 2 :]
+            else:
+                header_text = block_text[:150]
+                section_text = block_text[150:]
+        blocks = [
+            {"type": "header", "text": {"type": "plain_text", "text": header_text}}
+        ]
+        if section_text:
+            blocks.append(
+                {
+                    "type": "section",
+                    "text": self._get_slack_row(section_text),
+                }
+            )
+        return blocks
+
     def _get_alert_fields(
         self,
         alert: mlrun.common.schemas.AlertConfig,
@@ -131,7 +152,9 @@ class SlackNotification(NotificationBase):
     def _get_run_line(self, run: dict) -> dict:
         meta = run["metadata"]
         url = mlrun.utils.helpers.get_ui_url(meta.get("project"), meta.get("uid"))
-        if url:
+
+        # Only show the URL if the run is not a function (serving or mlrun function)
+        if run.get("kind") not in ["serving", None] and url:
             line = f'<{url}|*{meta.get("name")}*>'
         else:
             line = meta.get("name")
@@ -148,7 +171,7 @@ class SlackNotification(NotificationBase):
             result = mlrun.utils.helpers.dict_to_str(
                 run["status"].get("results", {}), ", "
             )
-        return self._get_slack_row(result or "None")
+        return self._get_slack_row(result or state)
 
     @staticmethod
     def _get_slack_row(text: str) -> dict:

mlrun/utils/notifications/notification_pusher.py

@@ -14,15 +14,21 @@
 
 import asyncio
 import datetime
+import json
 import os
+import re
 import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
+import kfp
+
+import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.config
 import mlrun.db.base
 import mlrun.errors
+import mlrun.kfpops
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
@@ -238,20 +244,7 @@ class NotificationPusher(_NotificationPusherBase):
             custom_message = (
                 f" (workflow: {run.metadata.labels['workflow']}){custom_message}"
             )
-            db = mlrun.get_run_db()
-
-            workflow_id = run.status.results.get("workflow_id", None)
-            if workflow_id:
-                workflow_runs = db.list_runs(
-                    project=run.metadata.project,
-                    labels=f"workflow={workflow_id}",
-                )
-                logger.debug(
-                    "Found workflow runs, extending notification runs",
-                    workflow_id=workflow_id,
-                    workflow_runs_amount=len(workflow_runs),
-                )
-                runs.extend(workflow_runs)
+            runs.extend(self.get_workflow_steps(run))
 
         message = (
             self.messages.get(run.state(), "").format(resource=resource)
@@ -395,6 +388,132 @@ class NotificationPusher(_NotificationPusherBase):
             mask_params=False,
         )
 
+    def get_workflow_steps(self, run: mlrun.model.RunObject) -> list:
+        steps = []
+        db = mlrun.get_run_db()
+
+        def _add_run_step(_node_name, _):
+            steps.append(
+                db.list_runs(
+                    project=run.metadata.project,
+                    labels=f"mlrun/runner-pod={_node_name}",
+                )[0]
+            )
+
+        def _add_deploy_function_step(_, _node_template):
+            project, name, hash_key = self._extract_function_uri(
+                _node_template["metadata"]["annotations"]["mlrun/function-uri"]
+            )
+            if name:
+                try:
+                    function = db.get_function(
+                        project=project, name=name, hash_key=hash_key
+                    )
+                except mlrun.errors.MLRunNotFoundError:
+                    # If the function is not found (if build failed for example), we will create a dummy
+                    # function object for the notification to display the function name
+                    function = {
+                        "metadata": {
+                            "name": name,
+                            "project": project,
+                            "hash_key": hash_key,
+                        },
+                    }
+                function["status"] = {
+                    "state": mlrun.common.runtimes.constants.PodPhases.pod_phase_to_run_state(
+                        node["phase"]
+                    ),
+                }
+                if isinstance(function["metadata"].get("updated"), datetime.datetime):
+                    function["metadata"]["updated"] = function["metadata"][
+                        "updated"
+                    ].isoformat()
+                steps.append(function)
+
+        step_methods = {
+            mlrun.kfpops.PipelineRunType.run: _add_run_step,
+            mlrun.kfpops.PipelineRunType.build: _add_deploy_function_step,
+            mlrun.kfpops.PipelineRunType.deploy: _add_deploy_function_step,
+        }
+
+        workflow_id = run.status.results.get("workflow_id", None)
+        if not workflow_id:
+            return steps
+
+        workflow_manifest = self._get_workflow_manifest(workflow_id)
+        if not workflow_manifest:
+            return steps
+
+        try:
+            workflow_nodes = sorted(
+                workflow_manifest["status"]["nodes"].items(),
+                key=lambda _node: _node[1]["finishedAt"],
+            )
+            for node_name, node in workflow_nodes:
+                if node["type"] != "Pod":
+                    # Skip the parent DAG node
+                    continue
+
+                node_template = next(
+                    template
+                    for template in workflow_manifest["spec"]["templates"]
+                    if template["name"] == node["templateName"]
+                )
+                step_type = node_template["metadata"]["annotations"].get(
+                    "mlrun/pipeline-step-type"
+                )
+                step_method = step_methods.get(step_type)
+                if step_method:
+                    step_method(node_name, node_template)
+            return steps
+        except Exception:
+            # If we fail to read the pipeline steps, we will return the list of runs that have the same workflow id
+            logger.warning(
+                "Failed to extract workflow steps from workflow manifest, "
+                "returning all runs with the workflow id label",
+                workflow_id=workflow_id,
+                traceback=traceback.format_exc(),
+            )
+            return db.list_runs(
+                project=run.metadata.project,
+                labels=f"workflow={workflow_id}",
+            )
+
+    @staticmethod
+    def _get_workflow_manifest(workflow_id: str) -> typing.Optional[dict]:
+        kfp_client = kfp.Client(namespace=mlrun.config.config.namespace)
+
+        # arbitrary timeout of 5 seconds, the workflow should be done by now
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        if not kfp_run:
+            return None
+
+        kfp_run = kfp_run.to_dict()
+        try:
+            return json.loads(kfp_run["pipeline_runtime"]["workflow_manifest"])
+        except Exception:
+            return None
+
+    def _extract_function_uri(self, function_uri: str) -> tuple[str, str, str]:
+        """
+        Extract the project, name, and hash key from a function uri.
+        Examples:
+            - "project/name@hash_key" returns project, name, hash_key
+            - "project/name returns" project, name, ""
+        """
+        project, name, hash_key = None, None, None
+        hashed_pattern = r"^(.+)/(.+)@(.+)$"
+        pattern = r"^(.+)/(.+)$"
+        match = re.match(hashed_pattern, function_uri)
+        if match:
+            project, name, hash_key = match.groups()
+        else:
+            match = re.match(pattern, function_uri)
+            if match:
+                project, name = match.groups()
+                hash_key = ""
+        return project, name, hash_key
+
 
 class CustomNotificationPusher(_NotificationPusherBase):
     def __init__(self, notification_types: list[str] = None):

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "3bba8f5664d7a3ff4639a3437811fa5972883b2f",
-  "version": "1.7.0-rc13"
+  "git_commit": "dd3ddb2c7544ea578f0445adb74711f0e8b5cbc9",
+  "version": "1.7.0-rc15"
 }

{mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc13
+Version: 1.7.0rc15
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
 Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.7
+Requires-Dist: storey ~=1.7.11
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
 Requires-Dist: setuptools ~=69.1