mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/platforms/iguazio.py

@@ -15,12 +15,9 @@
 import json
 import os
 import urllib
-from collections import namedtuple
 from urllib.parse import urlparse
 
-import kfp.dsl
 import requests
-import semver
 import v3io
 
 import mlrun.errors
@@ -29,203 +26,6 @@ from mlrun.utils import dict_to_json
 
 _cached_control_session = None
 
-VolumeMount = namedtuple("Mount", ["path", "sub_path"])
-
-
-def mount_v3io(
-    name="v3io",
-    remote="",
-    access_key="",
-    user="",
-    secret=None,
-    volume_mounts=None,
-):
-    """Modifier function to apply to a Container Op to volume mount a v3io path
-
-    :param name:            the volume name
-    :param remote:          the v3io path to use for the volume. ~/ prefix will be replaced with /users/<username>/
-    :param access_key:      the access key used to auth against v3io. if not given V3IO_ACCESS_KEY env var will be used
-    :param user:            the username used to auth against v3io. if not given V3IO_USERNAME env var will be used
-    :param secret:          k8s secret name which would be used to get the username and access key to auth against v3io.
-    :param volume_mounts:   list of VolumeMount. empty volume mounts & remote will default to mount /v3io & /User.
-    """
-    volume_mounts, user = _enrich_and_validate_v3io_mounts(
-        remote=remote,
-        volume_mounts=volume_mounts,
-        user=user,
-    )
-
-    def _attach_volume_mounts_and_creds(container_op: kfp.dsl.ContainerOp):
-        from kubernetes import client as k8s_client
-
-        vol = v3io_to_vol(name, remote, access_key, user, secret=secret)
-        container_op.add_volume(vol)
-        for volume_mount in volume_mounts:
-            container_op.container.add_volume_mount(
-                k8s_client.V1VolumeMount(
-                    mount_path=volume_mount.path,
-                    sub_path=volume_mount.sub_path,
-                    name=name,
-                )
-            )
-
-        if not secret:
-            container_op = v3io_cred(access_key=access_key, user=user)(container_op)
-        return container_op
-
-    return _attach_volume_mounts_and_creds
-
-
-def _enrich_and_validate_v3io_mounts(remote="", volume_mounts=None, user=""):
-    if remote and not volume_mounts:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            "volume_mounts must be specified when remote is given"
-        )
-
-    # Empty remote & volume_mounts defaults are volume mounts of /v3io and /User
-    if not remote and not volume_mounts:
-        user = _resolve_mount_user(user)
-        if not user:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "user name/env must be specified when using empty remote and volume_mounts"
-            )
-        volume_mounts = [
-            VolumeMount(path="/v3io", sub_path=""),
-            VolumeMount(path="/User", sub_path="users/" + user),
-        ]
-
-    if not isinstance(volume_mounts, list) and any(
-        [not isinstance(x, VolumeMount) for x in volume_mounts]
-    ):
-        raise TypeError("mounts should be a list of Mount")
-
-    return volume_mounts, user
-
-
-def _resolve_mount_user(user=None):
-    return user or os.environ.get("V3IO_USERNAME")
-
-
-def mount_spark_conf():
-    def _mount_spark(container_op: kfp.dsl.ContainerOp):
-        from kubernetes import client as k8s_client
-
-        container_op.container.add_volume_mount(
-            k8s_client.V1VolumeMount(
-                name="spark-master-config", mount_path="/etc/config/spark"
-            )
-        )
-        return container_op
-
-    return _mount_spark
-
-
-def mount_v3iod(namespace, v3io_config_configmap):
-    def _mount_v3iod(container_op: kfp.dsl.ContainerOp):
-        from kubernetes import client as k8s_client
-
-        def add_vol(name, mount_path, host_path):
-            vol = k8s_client.V1Volume(
-                name=name,
-                host_path=k8s_client.V1HostPathVolumeSource(path=host_path, type=""),
-            )
-            container_op.add_volume(vol)
-            container_op.container.add_volume_mount(
-                k8s_client.V1VolumeMount(mount_path=mount_path, name=name)
-            )
-
-        # this is a legacy path for the daemon shared memory
-        host_path = "/dev/shm/"
-
-        # path to shared memory for daemon was changed in Iguazio 3.2.3-b1
-        igz_version = mlrun.mlconf.get_parsed_igz_version()
-        if igz_version and igz_version >= semver.VersionInfo.parse("3.2.3-b1"):
-            host_path = "/var/run/iguazio/dayman-shm/"
-        add_vol(name="shm", mount_path="/dev/shm", host_path=host_path + namespace)
-
-        add_vol(
-            name="v3iod-comm",
-            mount_path="/var/run/iguazio/dayman",
-            host_path="/var/run/iguazio/dayman/" + namespace,
-        )
-
-        vol = k8s_client.V1Volume(
-            name="daemon-health", empty_dir=k8s_client.V1EmptyDirVolumeSource()
-        )
-        container_op.add_volume(vol)
-        container_op.container.add_volume_mount(
-            k8s_client.V1VolumeMount(
-                mount_path="/var/run/iguazio/daemon_health", name="daemon-health"
-            )
-        )
-
-        vol = k8s_client.V1Volume(
-            name="v3io-config",
-            config_map=k8s_client.V1ConfigMapVolumeSource(
-                name=v3io_config_configmap, default_mode=420
-            ),
-        )
-        container_op.add_volume(vol)
-        container_op.container.add_volume_mount(
-            k8s_client.V1VolumeMount(mount_path="/etc/config/v3io", name="v3io-config")
-        )
-
-        container_op.container.add_env_variable(
-            k8s_client.V1EnvVar(
-                name="CURRENT_NODE_IP",
-                value_from=k8s_client.V1EnvVarSource(
-                    field_ref=k8s_client.V1ObjectFieldSelector(
-                        api_version="v1", field_path="status.hostIP"
-                    )
-                ),
-            )
-        )
-        container_op.container.add_env_variable(
-            k8s_client.V1EnvVar(
-                name="IGZ_DATA_CONFIG_FILE", value="/igz/java/conf/v3io.conf"
-            )
-        )
-
-        return container_op
-
-    return _mount_v3iod
-
-
-def v3io_cred(api="", user="", access_key=""):
-    """
-    Modifier function to copy local v3io env vars to container
-
-    Usage::
-
-        train = train_op(...)
-        train.apply(use_v3io_cred())
-    """
-
-    def _use_v3io_cred(container_op: kfp.dsl.ContainerOp):
-        from os import environ
-
-        from kubernetes import client as k8s_client
-
-        web_api = api or environ.get("V3IO_API") or mlconf.v3io_api
-        _user = user or environ.get("V3IO_USERNAME")
-        _access_key = access_key or environ.get("V3IO_ACCESS_KEY")
-        v3io_framesd = mlconf.v3io_framesd or environ.get("V3IO_FRAMESD")
-
-        return (
-            container_op.container.add_env_variable(
-                k8s_client.V1EnvVar(name="V3IO_API", value=web_api)
-            )
-            .add_env_variable(k8s_client.V1EnvVar(name="V3IO_USERNAME", value=_user))
-            .add_env_variable(
-                k8s_client.V1EnvVar(name="V3IO_ACCESS_KEY", value=_access_key)
-            )
-            .add_env_variable(
-                k8s_client.V1EnvVar(name="V3IO_FRAMESD", value=v3io_framesd)
-            )
-        )
-
-    return _use_v3io_cred
-
 
 def split_path(mntpath=""):
     if mntpath[0] == "/":
@@ -525,8 +325,8 @@ def add_or_refresh_credentials(
     # different access keys for the 2 usages
     token = (
         token
-        # can't use mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session cause this is running in the
-        # import execution path (when we're initializing the run db) and therefore we can't import mlrun.runtimes
+        # can't use mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session cause this is running
+        # in the import execution path (when we're initializing the run db) and therefore we can't import mlrun.runtimes
         or os.environ.get("MLRUN_AUTH_SESSION")
         or os.environ.get("V3IO_ACCESS_KEY")
     )
@@ -552,7 +352,9 @@ def parse_path(url, suffix="/"):
     parsed_url = urlparse(url)
     if parsed_url.netloc:
         scheme = parsed_url.scheme.lower()
-        if scheme == "v3ios":
+        if scheme == "s3":
+            prefix = "s3"
+        elif scheme == "v3ios":
             prefix = "https"
         elif scheme == "v3io":
             prefix = "http"
@@ -580,3 +382,22 @@ def sanitize_username(username: str):
     So simply replace it with dash
     """
     return username.replace("_", "-")
+
+
+def min_iguazio_versions(*versions):
+    def decorator(function):
+        def wrapper(*args, **kwargs):
+            if mlrun.utils.helpers.validate_component_version_compatibility(
+                "iguazio", *versions
+            ):
+                return function(*args, **kwargs)
+
+            message = (
+                f"{function.__name__} is supported since Iguazio {' or '.join(versions)}, currently using "
+                f"Iguazio {mlconf.igz_version}."
+            )
+            raise mlrun.errors.MLRunIncompatibleVersionError(message)
+
+        return wrapper
+
+    return decorator

mlrun/projects/operations.py

@@ -13,11 +13,15 @@
 # limitations under the License.
 #
 import warnings
-from typing import Dict, List, Optional, Union
+from typing import Optional, Union
 
-import kfp
+import mlrun_pipelines.common.models
+import mlrun_pipelines.models
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
+import mlrun.common.schemas.function
+import mlrun.common.schemas.workflow
 from mlrun.utils import hub_prefix
 
 from .pipelines import enrich_function_object, pipeline_context
@@ -48,7 +52,7 @@ def _get_engine_and_function(function, project=None):
         function = enrich_function_object(project, function, copy_function=False)
 
     if not pipeline_context.workflow:
-        return "local", function
+        return mlrun.common.schemas.workflow.EngineType.LOCAL, function
 
     return pipeline_context.workflow.engine, function
 
@@ -61,7 +65,7 @@ def run_function(
     hyperparams: dict = None,
     hyper_param_options: mlrun.model.HyperParamOptions = None,
     inputs: dict = None,
-    outputs: List[str] = None,
+    outputs: list[str] = None,
     workdir: str = "",
     labels: dict = None,
     base_task: mlrun.model.RunTemplate = None,
@@ -73,10 +77,11 @@ def run_function(
     auto_build: bool = None,
     schedule: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
     artifact_path: str = None,
-    notifications: List[mlrun.model.Notification] = None,
-    returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+    notifications: list[mlrun.model.Notification] = None,
+    returns: Optional[list[Union[str, dict[str, str]]]] = None,
     builder_env: Optional[list] = None,
-) -> Union[mlrun.model.RunObject, kfp.dsl.ContainerOp]:
+    reset_on_run: Optional[bool] = None,
+) -> Union[mlrun.model.RunObject, mlrun_pipelines.models.PipelineNodeWrapper]:
     """Run a local or remote task as part of a local/kubeflow pipeline
 
     run_function() allow you to execute a function locally, on a remote cluster, or as part of an automated workflow
@@ -86,7 +91,7 @@ def run_function(
     when functions run as part of a workflow/pipeline (project.run()) some attributes can be set at the run level,
     e.g. local=True will run all the functions locally, setting artifact_path will direct all outputs to the same path.
     project runs provide additional notifications/reporting and exception handling.
-    inside a Kubeflow pipeline (KFP) run_function() generates KFP "ContainerOps" which are used to form a DAG
+    inside a Kubeflow pipeline (KFP) run_function() generates KFP node (see PipelineNodeWrapper) which forms a DAG
     some behavior may differ between regular runs and deferred KFP runs.
 
     example (use with function object)::
@@ -95,8 +100,11 @@ def run_function(
         MODEL_CLASS = "sklearn.ensemble.RandomForestClassifier"
         DATA_PATH = "s3://bigdata/data.parquet"
         function = mlrun.import_function("hub://auto-trainer")
-        run1 = run_function(function, params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                      inputs={"dataset": DATA_PATH})
+        run1 = run_function(
+            function,
+            params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+            inputs={"dataset": DATA_PATH},
+        )
 
     example (use with project)::
 
@@ -115,8 +123,12 @@ def run_function(
         @dsl.pipeline(name="test pipeline", description="test")
         def my_pipe(url=""):
             run1 = run_function("loaddata", params={"url": url}, outputs=["data"])
-            run2 = run_function("train", params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                         inputs={"dataset": run1.outputs["data"]})
+            run2 = run_function(
+                "train",
+                params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+                inputs={"dataset": run1.outputs["data"]},
+            )
+
 
         project.run(workflow_handler=my_pipe, arguments={"param1": 7})
 
@@ -159,7 +171,10 @@ def run_function(
                               artifact type can be given there. The artifact key must appear in the dictionary as
                               "key": "the_key".
     :param builder_env:     env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
-    :return: MLRun RunObject or KubeFlow containerOp
+    :param reset_on_run:    When True, function python modules would reload prior to code execution.
+                            This ensures latest code changes are executed. This argument must be used in
+                            conjunction with the local=True argument.
+    :return: MLRun RunObject or PipelineNodeWrapper
     """
     engine, function = _get_engine_and_function(function, project_object)
     task = mlrun.new_task(
@@ -174,7 +189,11 @@ def run_function(
     )
     task.spec.verbose = task.spec.verbose or verbose
 
-    if engine == "kfp":
+    if engine == mlrun.common.schemas.workflow.EngineType.KFP:
+        if schedule:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Scheduling jobs is not supported when running a workflow with the kfp engine."
+            )
         return function.as_step(
             name=name, runspec=task, workdir=workdir, outputs=outputs, labels=labels
         )
@@ -183,7 +202,9 @@ def run_function(
         local = pipeline_context.is_run_local(local)
         task.metadata.labels = task.metadata.labels or labels or {}
         if pipeline_context.workflow_id:
-            task.metadata.labels["workflow"] = pipeline_context.workflow_id
+            task.metadata.labels[mlrun_constants.MLRunInternalLabels.workflow] = (
+                pipeline_context.workflow_id
+            )
         if function.kind == "local":
             command, function = mlrun.run.load_func_code(function)
             function.spec.command = command
@@ -205,6 +226,7 @@ def run_function(
             schedule=schedule,
             notifications=notifications,
             builder_env=builder_env,
+            reset_on_run=reset_on_run,
         )
         if run_result:
             run_result._notified = False
@@ -218,9 +240,9 @@ def run_function(
 class BuildStatus:
     """returned status from build operation"""
 
-    def __init__(self, ready, outputs={}, function=None):
+    def __init__(self, ready, outputs=None, function=None):
         self.ready = ready
-        self.outputs = outputs
+        self.outputs = outputs or {}
         self.function = function
 
     def after(self, step):
@@ -239,7 +261,7 @@ def build_function(
     base_image=None,
     commands: list = None,
     secret_name=None,
-    requirements: Union[str, List[str]] = None,
+    requirements: Union[str, list[str]] = None,
     requirements_file: str = None,
     mlrun_version_specifier=None,
     builder_env: dict = None,
@@ -247,7 +269,7 @@ def build_function(
     overwrite_build_params: bool = False,
     extra_args: str = None,
     force_build: bool = False,
-) -> Union[BuildStatus, kfp.dsl.ContainerOp]:
+) -> Union[BuildStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy ML function, build container with its dependencies
 
     :param function:        Name of the function (in the project) or function object
@@ -283,11 +305,15 @@ def build_function(
         raise mlrun.errors.MLRunInvalidArgumentError(
             "Cannot build use deploy_function()"
         )
-    if engine == "kfp":
+    if engine == mlrun.common.schemas.workflow.EngineType.KFP:
         if overwrite_build_params:
             function.spec.build.commands = None
         if requirements or requirements_file:
-            function.with_requirements(requirements, requirements_file, overwrite=True)
+            function.with_requirements(
+                requirements=requirements,
+                requirements_file=requirements_file,
+                overwrite=True,
+            )
         if commands:
             function.with_commands(commands)
         return function.deploy_step(
@@ -311,6 +337,7 @@ def build_function(
                 commands=commands,
                 secret=secret_name,
                 requirements=requirements,
+                requirements_file=requirements_file,
                 overwrite=overwrite_build_params,
                 extra_args=extra_args,
             )
@@ -329,9 +356,9 @@ def build_function(
 class DeployStatus:
     """returned status from deploy operation"""
 
-    def __init__(self, state, outputs={}, function=None):
+    def __init__(self, state, outputs=None, function=None):
         self.state = state
-        self.outputs = outputs
+        self.outputs = outputs or {}
         self.function = function
 
     def after(self, step):
@@ -351,7 +378,7 @@ def deploy_function(
     builder_env: dict = None,
     project_object=None,
     mock: bool = None,
-) -> Union[DeployStatus, kfp.dsl.ContainerOp]:
+) -> Union[DeployStatus, mlrun_pipelines.models.PipelineNodeWrapper]:
     """deploy real-time (nuclio based) functions
 
     :param function:   name of the function (in the project) or function object
@@ -368,7 +395,7 @@ def deploy_function(
         raise mlrun.errors.MLRunInvalidArgumentError(
             "deploy is used with real-time functions, for other kinds use build_function()"
         )
-    if engine == "kfp":
+    if engine == mlrun.common.schemas.workflow.EngineType.KFP:
         return function.deploy_step(models=models, env=env, tag=tag, verbose=verbose)
     else:
         if env: