mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/runtimes/__init__.py

@@ -26,23 +26,30 @@ __all__ = [
     "Spark3Runtime",
     "DatabricksRuntime",
     "KubeResource",
+    "ApplicationRuntime",
+    "MpiRuntimeV1",
 ]
 
+import typing
+
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
+from ..common.runtimes.constants import MPIJobCRDVersions
 from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
-from .constants import MPIJobCRDVersions
 from .daskjob import DaskCluster  # noqa
 from .databricks_job.databricks_runtime import DatabricksRuntime
 from .kubejob import KubejobRuntime, KubeResource  # noqa
 from .local import HandlerRuntime, LocalRuntime  # noqa
-from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1  # noqa
+from .mpijob import MpiRuntimeV1  # noqa
 from .nuclio import (
     RemoteRuntime,
     ServingRuntime,
     new_v2_model_server,
     nuclio_init_hook,
 )
+from .nuclio.api_gateway import APIGateway
+from .nuclio.application import ApplicationRuntime
+from .nuclio.serving import serving_subkind
 from .remotesparkjob import RemoteSparkRuntime
 from .sparkjob import Spark3Runtime
 
@@ -101,6 +108,7 @@ class RuntimeKinds:
     local = "local"
     handler = "handler"
     databricks = "databricks"
+    application = "application"
 
     @staticmethod
     def all():
@@ -115,6 +123,7 @@ class RuntimeKinds:
             RuntimeKinds.mpijob,
             RuntimeKinds.local,
             RuntimeKinds.databricks,
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -147,6 +156,23 @@ class RuntimeKinds:
             RuntimeKinds.remote,
             RuntimeKinds.nuclio,
             RuntimeKinds.serving,
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def pure_nuclio_deployed_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+        ]
+
+    @staticmethod
+    def handlerless_runtimes():
+        return [
+            RuntimeKinds.serving,
+            # Application runtime handler is internal reverse proxy
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -157,7 +183,7 @@ class RuntimeKinds:
         ]
 
     @staticmethod
-    def is_log_collectable_runtime(kind: str):
+    def is_log_collectable_runtime(kind: typing.Optional[str]):
         """
         whether log collector can collect logs for that runtime
         :param kind: kind name
@@ -168,13 +194,18 @@ class RuntimeKinds:
         if RuntimeKinds.is_local_runtime(kind):
             return False
 
-        if kind not in [
-            # dask implementation is different than other runtimes, because few runs can be run against the same runtime
-            # resource, so collecting logs on that runtime resource won't be correct, the way we collect logs for dask
-            # is by using `log_std` on client side after we execute the code against the cluster, as submitting the
-            # run with the dask client will return the run stdout. for more information head to `DaskCluster._run`
-            RuntimeKinds.dask
-        ]:
+        if (
+            kind
+            not in [
+                # dask implementation is different from other runtimes, because few runs can be run against the same
+                # runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
+                # logs for dask is by using `log_std` on client side after we execute the code against the cluster,
+                # as submitting the run with the dask client will return the run stdout.
+                # For more information head to `DaskCluster._run`.
+                RuntimeKinds.dask
+            ]
+            + RuntimeKinds.nuclio_runtimes()
+        ):
             return True
 
         return False
@@ -211,10 +242,43 @@ class RuntimeKinds:
         # both spark and remote spark uses different mechanism for assigning images
         return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
 
+    @staticmethod
+    def supports_from_notebook(kind):
+        return kind not in [RuntimeKinds.application]
+
+    @staticmethod
+    def resolve_nuclio_runtime(kind: str, sub_kind: str):
+        kind = kind.split(":")[0]
+        if kind not in RuntimeKinds.nuclio_runtimes():
+            raise ValueError(
+                f"Kind {kind} is not a nuclio runtime, available runtimes are {RuntimeKinds.nuclio_runtimes()}"
+            )
+
+        if sub_kind == serving_subkind:
+            return ServingRuntime()
+
+        if kind == RuntimeKinds.application:
+            return ApplicationRuntime()
+
+        runtime = RemoteRuntime()
+        runtime.spec.function_kind = sub_kind
+        return runtime
+
+    @staticmethod
+    def resolve_nuclio_sub_kind(kind):
+        is_nuclio = kind.startswith("nuclio")
+        sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
+        if kind == RuntimeKinds.serving:
+            is_nuclio = True
+            sub_kind = serving_subkind
+        elif kind == RuntimeKinds.application:
+            is_nuclio = True
+        return is_nuclio, sub_kind
+
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:
-        return MpiRuntimeContainer.selector()
+        return MpiRuntimeV1
 
     if kind == RuntimeKinds.spark:
         return Spark3Runtime
@@ -228,6 +292,7 @@ def get_runtime_class(kind: str):
         RuntimeKinds.local: LocalRuntime,
         RuntimeKinds.remotespark: RemoteSparkRuntime,
         RuntimeKinds.databricks: DatabricksRuntime,
+        RuntimeKinds.application: ApplicationRuntime,
     }
 
     return kind_runtime_map[kind]

mlrun/runtimes/base.py

@@ -21,8 +21,11 @@ from os import environ
 from typing import Callable, Optional, Union
 
 import requests.exceptions
+from mlrun_pipelines.common.ops import mlrun_op
 from nuclio.build import mlrun_footer
 
+import mlrun.common.constants
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -36,7 +39,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
 from ..config import config
 from ..datastore import store_manager
 from ..errors import err_to_str
-from ..kfpops import mlrun_op
 from ..lists import RunList
 from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
 from ..utils import (
@@ -66,6 +68,7 @@ spec_fields = [
     "disable_auto_mount",
     "allow_empty_resources",
     "clone_target_dir",
+    "reset_on_run",
 ]
 
 
@@ -138,20 +141,24 @@ class FunctionSpec(ModelObj):
 
     @property
     def clone_target_dir(self):
-        warnings.warn(
-            "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.8.0. "
-            "Use spec.build.source_code_target_dir instead.",
-            FutureWarning,
-        )
+        # TODO: remove this property in 1.9.0
+        if self.build.source_code_target_dir:
+            warnings.warn(
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "Use spec.build.source_code_target_dir instead.",
+                FutureWarning,
+            )
         return self.build.source_code_target_dir
 
     @clone_target_dir.setter
     def clone_target_dir(self, clone_target_dir):
-        warnings.warn(
-            "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.8.0. "
-            "Use spec.build.source_code_target_dir instead.",
-            FutureWarning,
-        )
+        # TODO: remove this property in 1.9.0
+        if clone_target_dir:
+            warnings.warn(
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "Use spec.build.source_code_target_dir instead.",
+                FutureWarning,
+            )
         self.build.source_code_target_dir = clone_target_dir
 
     def enrich_function_preemption_spec(self):
@@ -330,6 +337,7 @@ class BaseRuntime(ModelObj):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         """
@@ -384,6 +392,9 @@ class BaseRuntime(ModelObj):
                 standards and is at least 1 minute (-1 for infinite).
                 If the phase is active for longer than the threshold, the run will be aborted.
                 See mlconf.function.spec.state_thresholds for the state options and default values.
+        :param reset_on_run: When True, function python modules would reload prior to code execution.
+                             This ensures latest code changes are executed. This argument must be used in
+                             conjunction with the local=True argument.
         :return: Run context object (RunObject) with run metadata, results and status
         """
         launcher = mlrun.launcher.factory.LauncherFactory().create_launcher(
@@ -412,15 +423,22 @@ class BaseRuntime(ModelObj):
             notifications=notifications,
             returns=returns,
             state_thresholds=state_thresholds,
+            reset_on_run=reset_on_run,
         )
 
-    def _get_db_run(self, task: RunObject = None):
+    def _get_db_run(
+        self,
+        task: RunObject = None,
+        run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
+    ):
         if self._get_db() and task:
             project = task.metadata.project
             uid = task.metadata.uid
             iter = task.metadata.iteration
             try:
-                return self._get_db().read_run(uid, project, iter=iter)
+                return self._get_db().read_run(
+                    uid, project, iter=iter, format_=run_format
+                )
             except mlrun.db.RunDBError:
                 return None
         if task:
@@ -464,11 +482,11 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(
-                "{{run.user}}", meta.labels["owner"]
+                "{{run.user}}", meta.labels[mlrun_constants.MLRunInternalLabels.owner]
             )
 
         if db and self.kind != "handler":
@@ -537,13 +555,14 @@ class BaseRuntime(ModelObj):
         self,
         resp: dict = None,
         task: RunObject = None,
-        err=None,
+        err: Union[Exception, str] = None,
+        run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
     ) -> typing.Optional[dict]:
         """update the task state in the DB"""
         was_none = False
         if resp is None and task:
             was_none = True
-            resp = self._get_db_run(task)
+            resp = self._get_db_run(task, run_format)
 
             if not resp:
                 self.store_run(task)
@@ -575,9 +594,9 @@ class BaseRuntime(ModelObj):
 
         elif (
             not was_none
-            and last_state != mlrun.runtimes.constants.RunStates.completed
+            and last_state != mlrun.common.runtimes.constants.RunStates.completed
             and last_state
-            not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
+            not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
         ):
             try:
                 runtime_cls = mlrun.runtimes.get_runtime_class(kind)
@@ -630,7 +649,9 @@ class BaseRuntime(ModelObj):
         image = image or self.spec.image or ""
 
         image = enrich_image_url(image, client_version, client_python_version)
-        if not image.startswith("."):
+        if not image.startswith(
+            mlrun.common.constants.IMAGE_NAME_ENRICH_REGISTRY_PREFIX
+        ):
             return image
         registry, repository = get_parsed_docker_registry()
         if registry:
@@ -653,7 +674,7 @@ class BaseRuntime(ModelObj):
         selector="",
         hyper_param_options: HyperParamOptions = None,
         inputs: dict = None,
-        outputs: dict = None,
+        outputs: list = None,
         workdir: str = "",
         artifact_path: str = "",
         image: str = "",
@@ -700,11 +721,11 @@ class BaseRuntime(ModelObj):
                                   "key": "the_key".
         :param auto_build:      when set to True and the function require build it will be built on the first
                                 function run, use only if you dont plan on changing the build config between runs
-        :return: KubeFlow containerOp
+        :return: mlrun_pipelines.models.PipelineNodeWrapper
         """
 
         # if the function contain KFP PipelineParams (futures) pass the full spec to the
-        # ContainerOp this way KFP will substitute the params with previous step outputs
+        # PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
         if use_db and not self._has_pipeline_param():
             # if the same function is built as part of the pipeline we do not use the versioned function
             # rather the latest function w the same tag so we can pick up the updated image/status
@@ -782,7 +803,7 @@ class BaseRuntime(ModelObj):
         requirements: Optional[list[str]] = None,
         overwrite: bool = False,
         prepare_image_for_deploy: bool = True,
-        requirements_file: str = "",
+        requirements_file: Optional[str] = "",
     ):
         """add package requirements from file or list to build spec.
 
@@ -836,6 +857,12 @@ class BaseRuntime(ModelObj):
             or (build.source and not build.load_source_on_run)
         )
 
+    def enrich_runtime_spec(
+        self,
+        project_node_selector: dict[str, str],
+    ):
+        pass
+
     def prepare_image_for_deploy(self):
         """
         if a function has a 'spec.image' it is considered to be deployed,
@@ -902,3 +929,6 @@ class BaseRuntime(ModelObj):
                         if "default" in p:
                             line += f", default={p['default']}"
                         print("    " + line)
+
+    def skip_image_enrichment(self):
+        return False

mlrun/runtimes/daskjob.py

@@ -379,7 +379,7 @@ class DaskCluster(KubejobRuntime):
         :param show_on_failure:         show logs only in case of build failure
         :param force_build:             force building the image, even when no changes were made
 
-        :return True if the function is ready (deployed)
+        :return:                        True if the function is ready (deployed)
         """
         return super().deploy(
             watch,
@@ -494,6 +494,7 @@ class DaskCluster(KubejobRuntime):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if state_thresholds:
@@ -547,7 +548,13 @@ class DaskCluster(KubejobRuntime):
                     "specified handler (string) without command "
                     "(py file path), specify command or use handler pointer"
                 )
-            handler = load_module(self.spec.command, handler, context=context)
+            # Do not embed the module in system as it is not persistent with the dask cluster
+            handler = load_module(
+                self.spec.command,
+                handler,
+                context=context,
+                embed_in_sys=False,
+            )
         client = self.client
         setattr(context, "dask_client", client)
         sout, serr = exec_from_params(handler, runobj, context)

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -232,6 +232,7 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
         notifications: Optional[list[mlrun.model.Notification]] = None,
         returns: Optional[list[Union[str, dict[str, str]]]] = None,
         state_thresholds: Optional[dict[str, int]] = None,
+        reset_on_run: Optional[bool] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if local:

mlrun/runtimes/databricks_job/databricks_wrapper.py

@@ -99,7 +99,7 @@ def save_credentials(
         credentials["DATABRICKS_CLUSTER_ID"] = cluster_id
 
     with open(credentials_path, "w") as yaml_file:
-        yaml.dump(credentials, yaml_file, default_flow_style=False)
+        yaml.safe_dump(credentials, yaml_file, default_flow_style=False)
 
 
 def run_mlrun_databricks_job(

mlrun/runtimes/funcdoc.py

@@ -16,8 +16,6 @@ import ast
 import inspect
 import re
 
-from deprecated import deprecated
-
 from mlrun.model import FunctionEntrypoint
 
 
@@ -73,32 +71,6 @@ def func_dict(
     }
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
-    category=FutureWarning,
-)
-def func_info(fn) -> dict:
-    sig = inspect.signature(fn)
-    doc = inspect.getdoc(fn) or ""
-
-    out = func_dict(
-        name=fn.__name__,
-        doc=doc,
-        params=[inspect_param(p) for p in sig.parameters.values()],
-        returns=param_dict(
-            type=type_name(sig.return_annotation, empty_is_none=True), default=None
-        ),
-        lineno=func_lineno(fn),
-    )
-
-    if not fn.__doc__ or not fn.__doc__.strip():
-        return out
-
-    return merge_doc(out, doc)
-
-
 def func_lineno(fn):
     try:
         return inspect.getsourcelines(fn)[1]
@@ -275,7 +247,7 @@ class ASTVisitor(ast.NodeVisitor):
         self.exprs.append(node)
         super().generic_visit(node)
 
-    def visit_FunctionDef(self, node):
+    def visit_FunctionDef(self, node):  # noqa: N802
         self.funcs.append(node)
         self.generic_visit(node)
 

mlrun/runtimes/kubejob.py

@@ -11,17 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import time
+import typing
 import warnings
 
+from mlrun_pipelines.common.ops import build_op
+
 import mlrun.common.schemas
 import mlrun.db
 import mlrun.errors
 
-from ..kfpops import build_op
 from ..model import RunObject
-from ..utils import get_in, logger
 from .pod import KubeResource
 
 
@@ -65,29 +64,13 @@ class KubejobRuntime(KubeResource):
         :param pull_at_runtime: load the archive into the container at job runtime vs on build/deploy
         :param target_dir:      target dir on runtime pod or repo clone / archive extraction
         """
-        mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
-
-        self.spec.build.source = source
-        if handler:
-            self.spec.default_handler = handler
-        if workdir:
-            self.spec.workdir = workdir
-        if target_dir:
-            self.spec.build.source_code_target_dir = target_dir
-
-        self.spec.build.load_source_on_run = pull_at_runtime
-        if (
-            self.spec.build.base_image
-            and not self.spec.build.commands
-            and pull_at_runtime
-            and not self.spec.image
-        ):
-            # if we load source from repo and don't need a full build use the base_image as the image
-            self.spec.image = self.spec.build.base_image
-        elif not pull_at_runtime:
-            # clear the image so build will not be skipped
-            self.spec.build.base_image = self.spec.build.base_image or self.spec.image
-            self.spec.image = ""
+        self._configure_mlrun_build_with_source(
+            source=source,
+            workdir=workdir,
+            handler=handler,
+            pull_at_runtime=pull_at_runtime,
+            target_dir=target_dir,
+        )
 
     def build_config(
         self,
@@ -160,125 +143,48 @@ class KubejobRuntime(KubeResource):
 
     def deploy(
         self,
-        watch=True,
-        with_mlrun=None,
-        skip_deployed=False,
-        is_kfp=False,
-        mlrun_version_specifier=None,
+        watch: bool = True,
+        with_mlrun: typing.Optional[bool] = None,
+        skip_deployed: bool = False,
+        is_kfp: bool = False,
+        mlrun_version_specifier: typing.Optional[bool] = None,
         builder_env: dict = None,
         show_on_failure: bool = False,
         force_build: bool = False,
     ) -> bool:
-        """deploy function, build container with dependencies
+        """Deploy function, build container with dependencies
 
-        :param watch:                   wait for the deploy to complete (and print build logs)
-        :param with_mlrun:              add the current mlrun package to the container build
-        :param skip_deployed:           skip the build if we already have an image for the function
-        :param is_kfp:                  deploy as part of a kfp pipeline
-        :param mlrun_version_specifier: which mlrun package version to include (if not current)
+        :param watch:                   Wait for the deploy to complete (and print build logs)
+        :param with_mlrun:              Add the current mlrun package to the container build
+        :param skip_deployed:           Skip the build if we already have an image for the function
+        :param is_kfp:                  Deploy as part of a kfp pipeline
+        :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param builder_env:             Kaniko builder pod env vars dict (for config/credentials)
                                         e.g. builder_env={"GIT_TOKEN": token}
-        :param show_on_failure:         show logs only in case of build failure
-        :param force_build:             set True for force building the image, even when no changes were made
+        :param show_on_failure:         Show logs only in case of build failure
+        :param force_build:             Set True for force building the image, even when no changes were made
 
         :return: True if the function is ready (deployed)
         """
 
         build = self.spec.build
+        with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
 
-        if with_mlrun is None:
-            if build.with_mlrun is not None:
-                with_mlrun = build.with_mlrun
-            else:
-                with_mlrun = build.base_image and not (
-                    build.base_image.startswith("mlrun/")
-                    or "/mlrun/" in build.base_image
-                )
-
-        if (
-            not build.source
-            and not build.commands
-            and not build.requirements
-            and not build.extra
-            and with_mlrun
-        ):
-            logger.info(
-                "Running build to add mlrun package, set "
-                "with_mlrun=False to skip if its already in the image"
-            )
         self.status.state = ""
         if build.base_image:
             # clear the image so build will not be skipped
             self.spec.image = ""
 
-        # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
-        # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
-        if is_kfp:
-            watch = True
-
-        ready = False
-        if self._is_remote_api():
-            db = self._get_db()
-            data = db.remote_builder(
-                self,
-                with_mlrun,
-                mlrun_version_specifier,
-                skip_deployed,
-                builder_env=builder_env,
-                force_build=force_build,
-            )
-            self.status = data["data"].get("status", None)
-            self.spec.image = get_in(data, "data.spec.image")
-            self.spec.build.base_image = self.spec.build.base_image or get_in(
-                data, "data.spec.build.base_image"
-            )
-            # Get the source target dir in case it was enriched due to loading source
-            self.spec.build.source_code_target_dir = get_in(
-                data, "data.spec.build.source_code_target_dir"
-            ) or get_in(data, "data.spec.clone_target_dir")
-            ready = data.get("ready", False)
-            if not ready:
-                logger.info(
-                    f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
-                )
-            if watch and not ready:
-                state = self._build_watch(watch, show_on_failure=show_on_failure)
-                ready = state == "ready"
-                self.status.state = state
-
-        if watch and not ready:
-            raise mlrun.errors.MLRunRuntimeError("Deploy failed")
-        return ready
-
-    def _build_watch(self, watch=True, logs=True, show_on_failure=False):
-        db = self._get_db()
-        offset = 0
-        try:
-            text, _ = db.get_builder_status(self, 0, logs=logs)
-        except mlrun.db.RunDBError:
-            raise ValueError("function or build process not found")
-
-        def print_log(text):
-            if text and (not show_on_failure or self.status.state == "error"):
-                print(text, end="")
-
-        print_log(text)
-        offset += len(text)
-        if watch:
-            while self.status.state in ["pending", "running"]:
-                time.sleep(2)
-                if show_on_failure:
-                    text = ""
-                    db.get_builder_status(self, 0, logs=False)
-                    if self.status.state == "error":
-                        # re-read the full log on failure
-                        text, _ = db.get_builder_status(self, offset, logs=logs)
-                else:
-                    text, _ = db.get_builder_status(self, offset, logs=logs)
-                print_log(text)
-                offset += len(text)
-
-        return self.status.state
+        return self._build_image(
+            builder_env=builder_env,
+            force_build=force_build,
+            mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
+            skip_deployed=skip_deployed,
+            watch=watch,
+            is_kfp=is_kfp,
+            with_mlrun=with_mlrun,
+        )
 
     def deploy_step(
         self,