mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

mlrun/run.py

@@ -47,7 +47,6 @@ from .runtimes import (
     KubejobRuntime,
     LocalRuntime,
     MpiRuntimeV1,
-    MpiRuntimeV1Alpha1,
     RemoteRuntime,
     RemoteSparkRuntime,
     RuntimeKinds,
@@ -606,7 +605,6 @@ def code_to_function(
     ignored_tags: Optional[str] = None,
     requirements_file: Optional[str] = "",
 ) -> Union[
-    MpiRuntimeV1Alpha1,
     MpiRuntimeV1,
     RemoteRuntime,
     ServingRuntime,
@@ -1150,7 +1148,7 @@ def wait_for_runs_completion(
         running = []
         for run in runs:
             state = run.state()
-            if state in mlrun.runtimes.constants.RunStates.terminal_states():
+            if state in mlrun.common.runtimes.constants.RunStates.terminal_states():
                 completed.append(run)
             else:
                 running.append(run)

mlrun/runtimes/__init__.py

@@ -30,13 +30,13 @@ __all__ = [
 
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
+from ..common.runtimes.constants import MPIJobCRDVersions
 from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
-from .constants import MPIJobCRDVersions
 from .daskjob import DaskCluster  # noqa
 from .databricks_job.databricks_runtime import DatabricksRuntime
 from .kubejob import KubejobRuntime, KubeResource  # noqa
 from .local import HandlerRuntime, LocalRuntime  # noqa
-from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1  # noqa
+from .mpijob import MpiRuntimeV1  # noqa
 from .nuclio import (
     RemoteRuntime,
     ServingRuntime,
@@ -264,7 +264,7 @@ class RuntimeKinds:
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:
-        return MpiRuntimeContainer.selector()
+        return MpiRuntimeV1
 
     if kind == RuntimeKinds.spark:
         return Spark3Runtime

mlrun/runtimes/base.py

@@ -469,7 +469,7 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(
@@ -580,9 +580,9 @@ class BaseRuntime(ModelObj):
 
         elif (
             not was_none
-            and last_state != mlrun.runtimes.constants.RunStates.completed
+            and last_state != mlrun.common.runtimes.constants.RunStates.completed
             and last_state
-            not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
+            not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
         ):
             try:
                 runtime_cls = mlrun.runtimes.get_runtime_class(kind)

mlrun/runtimes/funcdoc.py

@@ -16,8 +16,6 @@ import ast
 import inspect
 import re
 
-from deprecated import deprecated
-
 from mlrun.model import FunctionEntrypoint
 
 
@@ -73,32 +71,6 @@ def func_dict(
     }
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
-    category=FutureWarning,
-)
-def func_info(fn) -> dict:
-    sig = inspect.signature(fn)
-    doc = inspect.getdoc(fn) or ""
-
-    out = func_dict(
-        name=fn.__name__,
-        doc=doc,
-        params=[inspect_param(p) for p in sig.parameters.values()],
-        returns=param_dict(
-            type=type_name(sig.return_annotation, empty_is_none=True), default=None
-        ),
-        lineno=func_lineno(fn),
-    )
-
-    if not fn.__doc__ or not fn.__doc__.strip():
-        return out
-
-    return merge_doc(out, doc)
-
-
 def func_lineno(fn):
     try:
         return inspect.getsourcelines(fn)[1]

mlrun/runtimes/local.py

@@ -493,7 +493,7 @@ def exec_from_params(handler, runobj: RunObject, context: MLClientCtx, cwd=None)
                 logger.warning("Run was aborted", err=err_to_str(exc))
                 # Run was aborted, the state run state is updated by the abort job, no need to commit again
                 context.set_state(
-                    mlrun.runtimes.constants.RunStates.aborted, commit=False
+                    mlrun.common.runtimes.constants.RunStates.aborted, commit=False
                 )
                 commit = False
             except Exception as exc:

mlrun/runtimes/mpijob/__init__.py

@@ -21,28 +21,8 @@ from mlrun.config import config
 from .. import MPIJobCRDVersions
 from .abstract import AbstractMPIJobRuntime
 from .v1 import MpiRuntimeV1
-from .v1alpha1 import MpiRuntimeV1Alpha1
 
 
 def _resolve_mpijob_crd_version():
     # config is expected to get enriched from the API through the client-spec
     return config.mpijob_crd_version or MPIJobCRDVersions.default()
-
-
-class MpiRuntimeContainer(containers.DeclarativeContainer):
-    resolver = providers.Callable(
-        _resolve_mpijob_crd_version,
-    )
-
-    selector = providers.Selector(
-        resolver,
-        v1=providers.Object(MpiRuntimeV1),
-        v1alpha1=providers.Object(MpiRuntimeV1Alpha1),
-    )
-
-    # An empty selector to be overriden by the API
-    handler_selector = providers.Selector(
-        resolver,
-        v1=providers.Object(None),
-        v1alpha1=providers.Object(None),
-    )

mlrun/runtimes/mpijob/v1.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mlrun.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
+from mlrun.common.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
 from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime, MPIResourceSpec
 
 

mlrun/runtimes/nuclio/function.py

@@ -778,7 +778,7 @@ class RemoteRuntime(KubeResource):
             runtime_env["MLRUN_NAMESPACE"] = mlconf.namespace
         if self.metadata.credentials.access_key:
             runtime_env[
-                mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session
+                mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session
             ] = self.metadata.credentials.access_key
         return runtime_env
 

mlrun/runtimes/utils.py

@@ -27,10 +27,10 @@ import mlrun.common.constants
 import mlrun.common.schemas
 import mlrun.utils.regex
 from mlrun.artifacts import TableArtifact
+from mlrun.common.runtimes.constants import RunLabels
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.frameworks.parallel_coordinates import gen_pcp_plot
-from mlrun.runtimes.constants import RunLabels
 from mlrun.runtimes.generators import selector
 from mlrun.utils import get_in, helpers, logger, verify_field_regex
 

mlrun/utils/helpers.py

@@ -39,7 +39,6 @@ import pandas
 import semver
 import yaml
 from dateutil import parser
-from deprecated import deprecated
 from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
@@ -76,19 +75,6 @@ class OverwriteBuildParamsWarning(FutureWarning):
     pass
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'parse_versioned_object_uri' will be removed from this file in 1.7.0, use "
-    "'mlrun.common.helpers.parse_versioned_object_uri' instead",
-    category=FutureWarning,
-)
-def parse_versioned_object_uri(uri: str, default_project: str = ""):
-    return mlrun.common.helpers.parse_versioned_object_uri(
-        uri=uri, default_project=default_project
-    )
-
-
 class StorePrefix:
     """map mlrun store objects to prefixes"""
 
@@ -119,14 +105,9 @@ class StorePrefix:
 
 
 def get_artifact_target(item: dict, project=None):
-    if is_legacy_artifact(item):
-        db_key = item.get("db_key")
-        project_str = project or item.get("project")
-        tree = item.get("tree")
-    else:
-        db_key = item["spec"].get("db_key")
-        project_str = project or item["metadata"].get("project")
-        tree = item["metadata"].get("tree")
+    db_key = item["spec"].get("db_key")
+    project_str = project or item["metadata"].get("project")
+    tree = item["metadata"].get("tree")
 
     kind = item.get("kind")
     if kind in ["dataset", "model", "artifact"] and db_key:
@@ -135,11 +116,15 @@ def get_artifact_target(item: dict, project=None):
             target = f"{target}@{tree}"
         return target
 
-    return (
-        item.get("target_path")
-        if is_legacy_artifact(item)
-        else item["spec"].get("target_path")
-    )
+    return item["spec"].get("target_path")
+
+
+# TODO: left for migrations testing purposes. Remove in 1.8.0.
+def is_legacy_artifact(artifact):
+    if isinstance(artifact, dict):
+        return "metadata" not in artifact
+    else:
+        return not hasattr(artifact, "metadata")
 
 
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
@@ -1018,8 +1003,9 @@ def get_ui_url(project, uid=None):
 def get_workflow_url(project, id=None):
     url = ""
     if mlrun.mlconf.resolve_ui_url():
-        url = "{}/{}/{}/jobs/monitor-workflows/workflow/{}".format(
-            mlrun.mlconf.resolve_ui_url(), mlrun.mlconf.ui.projects_prefix, project, id
+        url = (
+            f"{mlrun.mlconf.resolve_ui_url()}/{mlrun.mlconf.ui.projects_prefix}"
+            f"/{project}/jobs/monitor-workflows/workflow/{id}"
         )
     return url
 
@@ -1291,13 +1277,6 @@ def str_to_timestamp(time_str: str, now_time: Timestamp = None):
     return Timestamp(time_str)
 
 
-def is_legacy_artifact(artifact):
-    if isinstance(artifact, dict):
-        return "metadata" not in artifact
-    else:
-        return not hasattr(artifact, "metadata")
-
-
 def is_link_artifact(artifact):
     if isinstance(artifact, dict):
         return (
@@ -1345,16 +1324,16 @@ def format_run(run: dict, with_project=False) -> dict:
     # as observed https://jira.iguazeng.com/browse/ML-5195
     # set to unknown to ensure a status is returned
     if run["status"] is None:
-        run["status"] = inflection.titleize(mlrun.runtimes.constants.RunStates.unknown)
+        run["status"] = inflection.titleize(
+            mlrun.common.runtimes.constants.RunStates.unknown
+        )
 
     return run
 
 
 def get_in_artifact(artifact: dict, key, default=None, raise_on_missing=False):
     """artifact can be dict or Artifact object"""
-    if is_legacy_artifact(artifact):
-        return artifact.get(key, default)
-    elif key == "kind":
+    if key == "kind":
         return artifact.get(key, default)
     else:
         for block in ["metadata", "spec", "status"]:
@@ -1596,3 +1575,11 @@ def get_serving_spec():
             )
     spec = json.loads(data)
     return spec
+
+
+def additional_filters_warning(additional_filters, class_name):
+    if additional_filters and any(additional_filters):
+        mlrun.utils.logger.warn(
+            f"additional_filters parameter is not supported in {class_name},"
+            f" parameter has been ignored."
+        )

mlrun/utils/notifications/notification/slack.py

@@ -152,7 +152,9 @@ class SlackNotification(NotificationBase):
     def _get_run_line(self, run: dict) -> dict:
         meta = run["metadata"]
         url = mlrun.utils.helpers.get_ui_url(meta.get("project"), meta.get("uid"))
-        if url:
+
+        # Only show the URL if the run is not a function (serving or mlrun function)
+        if run.get("kind") not in ["serving", None] and url:
             line = f'<{url}|*{meta.get("name")}*>'
         else:
             line = meta.get("name")
@@ -169,7 +171,7 @@ class SlackNotification(NotificationBase):
             result = mlrun.utils.helpers.dict_to_str(
                 run["status"].get("results", {}), ", "
             )
-        return self._get_slack_row(result or "None")
+        return self._get_slack_row(result or state)
 
     @staticmethod
     def _get_slack_row(text: str) -> dict:

mlrun/utils/notifications/notification_pusher.py

@@ -14,15 +14,21 @@
 
 import asyncio
 import datetime
+import json
 import os
+import re
 import traceback
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
+import kfp
+
+import mlrun.common.runtimes.constants
 import mlrun.common.schemas
 import mlrun.config
 import mlrun.db.base
 import mlrun.errors
+import mlrun.kfpops
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
@@ -238,20 +244,7 @@ class NotificationPusher(_NotificationPusherBase):
             custom_message = (
                 f" (workflow: {run.metadata.labels['workflow']}){custom_message}"
             )
-            db = mlrun.get_run_db()
-
-            workflow_id = run.status.results.get("workflow_id", None)
-            if workflow_id:
-                workflow_runs = db.list_runs(
-                    project=run.metadata.project,
-                    labels=f"workflow={workflow_id}",
-                )
-                logger.debug(
-                    "Found workflow runs, extending notification runs",
-                    workflow_id=workflow_id,
-                    workflow_runs_amount=len(workflow_runs),
-                )
-                runs.extend(workflow_runs)
+            runs.extend(self.get_workflow_steps(run))
 
         message = (
             self.messages.get(run.state(), "").format(resource=resource)
@@ -395,6 +388,132 @@ class NotificationPusher(_NotificationPusherBase):
             mask_params=False,
         )
 
+    def get_workflow_steps(self, run: mlrun.model.RunObject) -> list:
+        steps = []
+        db = mlrun.get_run_db()
+
+        def _add_run_step(_node_name, _):
+            steps.append(
+                db.list_runs(
+                    project=run.metadata.project,
+                    labels=f"mlrun/runner-pod={_node_name}",
+                )[0]
+            )
+
+        def _add_deploy_function_step(_, _node_template):
+            project, name, hash_key = self._extract_function_uri(
+                _node_template["metadata"]["annotations"]["mlrun/function-uri"]
+            )
+            if name:
+                try:
+                    function = db.get_function(
+                        project=project, name=name, hash_key=hash_key
+                    )
+                except mlrun.errors.MLRunNotFoundError:
+                    # If the function is not found (if build failed for example), we will create a dummy
+                    # function object for the notification to display the function name
+                    function = {
+                        "metadata": {
+                            "name": name,
+                            "project": project,
+                            "hash_key": hash_key,
+                        },
+                    }
+                function["status"] = {
+                    "state": mlrun.common.runtimes.constants.PodPhases.pod_phase_to_run_state(
+                        node["phase"]
+                    ),
+                }
+                if isinstance(function["metadata"].get("updated"), datetime.datetime):
+                    function["metadata"]["updated"] = function["metadata"][
+                        "updated"
+                    ].isoformat()
+                steps.append(function)
+
+        step_methods = {
+            mlrun.kfpops.PipelineRunType.run: _add_run_step,
+            mlrun.kfpops.PipelineRunType.build: _add_deploy_function_step,
+            mlrun.kfpops.PipelineRunType.deploy: _add_deploy_function_step,
+        }
+
+        workflow_id = run.status.results.get("workflow_id", None)
+        if not workflow_id:
+            return steps
+
+        workflow_manifest = self._get_workflow_manifest(workflow_id)
+        if not workflow_manifest:
+            return steps
+
+        try:
+            workflow_nodes = sorted(
+                workflow_manifest["status"]["nodes"].items(),
+                key=lambda _node: _node[1]["finishedAt"],
+            )
+            for node_name, node in workflow_nodes:
+                if node["type"] != "Pod":
+                    # Skip the parent DAG node
+                    continue
+
+                node_template = next(
+                    template
+                    for template in workflow_manifest["spec"]["templates"]
+                    if template["name"] == node["templateName"]
+                )
+                step_type = node_template["metadata"]["annotations"].get(
+                    "mlrun/pipeline-step-type"
+                )
+                step_method = step_methods.get(step_type)
+                if step_method:
+                    step_method(node_name, node_template)
+            return steps
+        except Exception:
+            # If we fail to read the pipeline steps, we will return the list of runs that have the same workflow id
+            logger.warning(
+                "Failed to extract workflow steps from workflow manifest, "
+                "returning all runs with the workflow id label",
+                workflow_id=workflow_id,
+                traceback=traceback.format_exc(),
+            )
+            return db.list_runs(
+                project=run.metadata.project,
+                labels=f"workflow={workflow_id}",
+            )
+
+    @staticmethod
+    def _get_workflow_manifest(workflow_id: str) -> typing.Optional[dict]:
+        kfp_client = kfp.Client(namespace=mlrun.config.config.namespace)
+
+        # arbitrary timeout of 5 seconds, the workflow should be done by now
+        kfp_run = kfp_client.wait_for_run_completion(workflow_id, 5)
+        if not kfp_run:
+            return None
+
+        kfp_run = kfp_run.to_dict()
+        try:
+            return json.loads(kfp_run["pipeline_runtime"]["workflow_manifest"])
+        except Exception:
+            return None
+
+    def _extract_function_uri(self, function_uri: str) -> tuple[str, str, str]:
+        """
+        Extract the project, name, and hash key from a function uri.
+        Examples:
+            - "project/name@hash_key" returns project, name, hash_key
+            - "project/name returns" project, name, ""
+        """
+        project, name, hash_key = None, None, None
+        hashed_pattern = r"^(.+)/(.+)@(.+)$"
+        pattern = r"^(.+)/(.+)$"
+        match = re.match(hashed_pattern, function_uri)
+        if match:
+            project, name, hash_key = match.groups()
+        else:
+            match = re.match(pattern, function_uri)
+            if match:
+                project, name = match.groups()
+                hash_key = ""
+        return project, name, hash_key
+
 
 class CustomNotificationPusher(_NotificationPusherBase):
     def __init__(self, notification_types: list[str] = None):

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "0967b2a28f5a75cc377483f5267159eb75925822",
-  "version": "1.7.0-rc14"
+  "git_commit": "dd3ddb2c7544ea578f0445adb74711f0e8b5cbc9",
+  "version": "1.7.0-rc15"
 }

{mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc14
+Version: 1.7.0rc15
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
 Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.7
+Requires-Dist: storey ~=1.7.11
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
 Requires-Dist: setuptools ~=69.1