mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/runtimes/base.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,69 +14,54 @@
 import enum
 import getpass
 import http
-import os.path
-import shlex
+import re
 import traceback
-import typing
-import uuid
+import warnings
 from abc import ABC, abstractmethod
-from ast import literal_eval
 from base64 import b64encode
-from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from os import environ
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
-import IPython
 import requests.exceptions
+from deprecated import deprecated
 from kubernetes.client.rest import ApiException
 from nuclio.build import mlrun_footer
 from sqlalchemy.orm import Session
 
+import mlrun.api.db.sqldb.session
+import mlrun.api.utils.singletons.db
+import mlrun.common.schemas
 import mlrun.errors
+import mlrun.launcher.factory
 import mlrun.utils.helpers
+import mlrun.utils.notifications
 import mlrun.utils.regex
-from mlrun.api import schemas
 from mlrun.api.constants import LogSources
 from mlrun.api.db.base import DBInterface
 from mlrun.utils.helpers import generate_object_uri, verify_field_regex
 
-from ..config import config, is_running_as_api
+from ..config import config
 from ..datastore import store_manager
 from ..db import RunDBError, get_or_set_dburl, get_run_db
 from ..errors import err_to_str
-from ..execution import MLClientCtx
-from ..k8s_utils import get_k8s_helper
-from ..kfpops import mlrun_op, write_kfpmeta
+from ..kfpops import mlrun_op
 from ..lists import RunList
-from ..model import (
-    BaseMetadata,
-    HyperParamOptions,
-    ImageBuilder,
-    ModelObj,
-    RunObject,
-    RunTemplate,
-)
-from ..secrets import SecretsStore
+from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
 from ..utils import (
     dict_to_json,
     dict_to_yaml,
     enrich_image_url,
     get_in,
     get_parsed_docker_registry,
-    get_ui_url,
-    is_ipython,
     logger,
-    normalize_name,
     now_date,
     update_in,
 )
 from .constants import PodPhases, RunStates
 from .funcdoc import update_function_entry_points
-from .generators import get_generator
-from .utils import RunError, calc_hash, results_to_iter
+from .utils import RunError, calc_hash, get_k8s
 
-run_modes = ["pass"]
 spec_fields = [
     "command",
     "args",
@@ -90,6 +75,7 @@ spec_fields = [
     "pythonpath",
     "disable_auto_mount",
     "allow_empty_resources",
+    "clone_target_dir",
 ]
 
 
@@ -130,6 +116,7 @@ class FunctionSpec(ModelObj):
         default_handler=None,
         pythonpath=None,
         disable_auto_mount=False,
+        clone_target_dir=None,
     ):
 
         self.command = command or ""
@@ -148,6 +135,9 @@ class FunctionSpec(ModelObj):
         self.entry_points = entry_points or {}
         self.disable_auto_mount = disable_auto_mount
         self.allow_empty_resources = None
+        # the build.source is cloned/extracted to the specified clone_target_dir
+        # if a relative path is specified, it will be enriched with a temp dir path
+        self.clone_target_dir = clone_target_dir or ""
 
     @property
     def build(self) -> ImageBuilder:
@@ -183,14 +173,12 @@ class BaseRuntime(ModelObj):
         self.is_child = False
         self._status = None
         self.status = None
-        self._is_api_server = False
         self.verbose = False
         self._enriched_image = False
 
     def set_db_connection(self, conn):
         if not self._db_conn:
             self._db_conn = conn
-        self._is_api_server = mlrun.config.is_running_as_api()
 
     @property
     def metadata(self) -> BaseMetadata:
@@ -216,9 +204,6 @@ class BaseRuntime(ModelObj):
     def status(self, status):
         self._status = self._verify_dict(status, "status", FunctionStatus)
 
-    def _get_k8s(self):
-        return get_k8s_helper()
-
     def set_label(self, key, value):
         self.metadata.labels[key] = str(value)
         return self
@@ -236,39 +221,6 @@ class BaseRuntime(ModelObj):
             return True
         return False
 
-    def _use_remote_api(self):
-        if (
-            self._is_remote
-            and not self._is_api_server
-            and self._get_db()
-            and self._get_db().kind == "http"
-        ):
-            return True
-        return False
-
-    def _enrich_on_client_side(self):
-        self.try_auto_mount_based_on_config()
-        self._fill_credentials()
-
-    def _enrich_on_server_side(self):
-        pass
-
-    def _enrich_on_server_and_client_sides(self):
-        """
-        enrich function also in client side and also on server side
-        """
-        pass
-
-    def _enrich_function(self):
-        """
-        enriches the function based on the flow state we run in (sdk or server)
-        """
-        if self._use_remote_api():
-            self._enrich_on_client_side()
-        else:
-            self._enrich_on_server_side()
-        self._enrich_on_server_and_client_sides()
-
     def _function_uri(self, tag=None, hash_key=None):
         return generate_object_uri(
             self.metadata.project,
@@ -281,11 +233,11 @@ class BaseRuntime(ModelObj):
         self.spec.rundb = self.spec.rundb or get_or_set_dburl()
 
     def _get_db(self):
+        # TODO: remove this function and use the launcher db instead
         self._ensure_run_db()
         if not self._db_conn:
             if self.spec.rundb:
                 self._db_conn = get_run_db(self.spec.rundb, secrets=self._secrets)
-                self._is_api_server = mlrun.config.is_running_as_api()
         return self._db_conn
 
     # This function is different than the auto_mount function, as it mounts to runtimes based on the configuration.
@@ -321,59 +273,62 @@ class BaseRuntime(ModelObj):
 
     def run(
         self,
-        runspec: RunObject = None,
-        handler=None,
-        name: str = "",
-        project: str = "",
-        params: dict = None,
-        inputs: Dict[str, str] = None,
-        out_path: str = "",
-        workdir: str = "",
-        artifact_path: str = "",
-        watch: bool = True,
-        schedule: Union[str, schemas.ScheduleCronTrigger] = None,
-        hyperparams: Dict[str, list] = None,
-        hyper_param_options: HyperParamOptions = None,
-        verbose=None,
-        scrape_metrics: bool = None,
-        local=False,
-        local_code_path=None,
-        auto_build=None,
-        param_file_secrets: Dict[str, str] = None,
+        runspec: Optional[
+            Union["mlrun.run.RunTemplate", "mlrun.run.RunObject", dict]
+        ] = None,
+        handler: Optional[Union[str, Callable]] = None,
+        name: Optional[str] = "",
+        project: Optional[str] = "",
+        params: Optional[dict] = None,
+        inputs: Optional[Dict[str, str]] = None,
+        out_path: Optional[str] = "",
+        workdir: Optional[str] = "",
+        artifact_path: Optional[str] = "",
+        watch: Optional[bool] = True,
+        schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
+        hyperparams: Optional[Dict[str, list]] = None,
+        hyper_param_options: Optional[HyperParamOptions] = None,
+        verbose: Optional[bool] = None,
+        scrape_metrics: Optional[bool] = None,
+        local: Optional[bool] = False,
+        local_code_path: Optional[str] = None,
+        auto_build: Optional[bool] = None,
+        param_file_secrets: Optional[Dict[str, str]] = None,
+        notifications: Optional[List[mlrun.model.Notification]] = None,
         returns: Optional[List[Union[str, Dict[str, str]]]] = None,
     ) -> RunObject:
         """
         Run a local or remote task.
 
-        :param runspec:        run template object or dict (see RunTemplate)
-        :param handler:        pointer or name of a function handler
-        :param name:           execution name
-        :param project:        project name
-        :param params:         input parameters (dict)
+        :param runspec:        The run spec to generate the RunObject from. Can be RunTemplate | RunObject | dict.
+        :param handler:        Pointer or name of a function handler.
+        :param name:           Execution name.
+        :param project:        Project name.
+        :param params:         Input parameters (dict).
         :param inputs:         Input objects to pass to the handler. Type hints can be given so the input will be parsed
                                during runtime from `mlrun.DataItem` to the given type hint. The type hint can be given
                                in the key field of the dictionary after a colon, e.g: "<key> : <type_hint>".
-        :param out_path:       default artifact output path
-        :param artifact_path:  default artifact output path (will replace out_path)
-        :param workdir:        default input artifacts path
-        :param watch:          watch/follow run log
+        :param out_path:       Default artifact output path.
+        :param artifact_path:  Default artifact output path (will replace out_path).
+        :param workdir:        Default input artifacts path.
+        :param watch:          Watch/follow run log.
         :param schedule:       ScheduleCronTrigger class instance or a standard crontab expression string
                                (which will be converted to the class using its `from_crontab` constructor),
                                see this link for help:
                                https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
-        :param hyperparams:    dict of param name and list of values to be enumerated e.g. {"p1": [1,2,3]}
+        :param hyperparams:    Dict of param name and list of values to be enumerated e.g. {"p1": [1,2,3]}
                                the default strategy is grid search, can specify strategy (grid, list, random)
-                               and other options in the hyper_param_options parameter
-        :param hyper_param_options:  dict or :py:class:`~mlrun.model.HyperParamOptions` struct of
-                                     hyper parameter options
-        :param verbose:        add verbose prints/logs
-        :param scrape_metrics: whether to add the `mlrun/scrape-metrics` label to this run's resources
-        :param local:      run the function locally vs on the runtime/cluster
-        :param local_code_path: path of the code for local runs & debug
-        :param auto_build: when set to True and the function require build it will be built on the first
-                           function run, use only if you dont plan on changing the build config between runs
-        :param param_file_secrets: dictionary of secrets to be used only for accessing the hyper-param parameter file.
-                            These secrets are only used locally and will not be stored anywhere
+                               and other options in the hyper_param_options parameter.
+        :param hyper_param_options: Dict or :py:class:`~mlrun.model.HyperParamOptions` struct of hyperparameter options.
+        :param verbose:             Add verbose prints/logs.
+        :param scrape_metrics:      Whether to add the `mlrun/scrape-metrics` label to this run's resources.
+        :param local:               Run the function locally vs on the runtime/cluster.
+        :param local_code_path:     Path of the code for local runs & debug.
+        :param auto_build:          When set to True and the function require build it will be built on the first
+                                    function run, use only if you don't plan on changing the build config between runs.
+        :param param_file_secrets:  Dictionary of secrets to be used only for accessing the hyper-param parameter file.
+                                    These secrets are only used locally and will not be stored anywhere
+        :param notifications:       List of notifications to push when the run is completed
         :param returns: List of log hints - configurations for how to log the returning values from the handler's run
                         (as artifacts or results). The list's length must be equal to the amount of returning objects. A
                         log hint may be given as:
@@ -385,206 +340,34 @@ class BaseRuntime(ModelObj):
                         * A dictionary of configurations to use when logging. Further info per object type and artifact
                           type can be given there. The artifact key must appear in the dictionary as "key": "the_key".
 
-        :return: run context object (RunObject) with run metadata, results and status
+        :return: Run context object (RunObject) with run metadata, results and status
         """
-        mlrun.utils.helpers.verify_dict_items_type("Inputs", inputs, [str], [str])
-
-        if self.spec.mode and self.spec.mode not in run_modes:
-            raise ValueError(f'run mode can only be {",".join(run_modes)}')
-
-        self._enrich_function()
-
-        run = self._create_run_object(runspec)
-
-        if local:
-
-            # do not allow local function to be scheduled
-            if schedule is not None:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "local and schedule cannot be used together"
-                )
-            return self._run_local(
-                run,
-                local_code_path,
-                project,
-                name,
-                workdir,
-                handler,
-                params,
-                inputs,
-                returns,
-                artifact_path,
-            )
-
-        run = self._enrich_run(
-            run,
-            handler,
-            project,
-            name,
-            params,
-            inputs,
-            returns,
-            hyperparams,
-            hyper_param_options,
-            verbose,
-            scrape_metrics,
-            out_path,
-            artifact_path,
-            workdir,
+        launcher = mlrun.launcher.factory.LauncherFactory.create_launcher(
+            self._is_remote, local
         )
-        self._validate_output_path(run)
-        db = self._get_db()
-
-        if not self.is_deployed():
-            if self.spec.build.auto_build or auto_build:
-                logger.info(
-                    "Function is not deployed and auto_build flag is set, starting deploy..."
-                )
-                self.deploy(skip_deployed=True, show_on_failure=True)
-            else:
-                raise RunError(
-                    "function image is not built/ready, set auto_build=True or use .deploy() method first"
-                )
-
-        if self.verbose:
-            logger.info(f"runspec:\n{run.to_yaml()}")
-
-        if "V3IO_USERNAME" in environ and "v3io_user" not in run.metadata.labels:
-            run.metadata.labels["v3io_user"] = environ.get("V3IO_USERNAME")
-
-        if not self.is_child:
-            db_str = "self" if self._is_api_server else self.spec.rundb
-            logger.info(
-                "Storing function",
-                name=run.metadata.name,
-                uid=run.metadata.uid,
-                db=db_str,
-            )
-            self._store_function(run, run.metadata, db)
-
-        # execute the job remotely (to a k8s cluster via the API service)
-        if self._use_remote_api():
-            return self._submit_job(run, schedule, db, watch)
-
-        elif self._is_remote and not self._is_api_server and not self.kfp:
-            logger.warning(
-                "warning!, Api url not set, " "trying to exec remote runtime locally"
-            )
-
-        execution = MLClientCtx.from_dict(
-            run.to_dict(),
-            db,
-            autocommit=False,
-            is_api=self._is_api_server,
-            store_run=False,
-        )
-
-        self._verify_run_params(run.spec.parameters)
-
-        # create task generator (for child runs) from spec
-        task_generator = get_generator(
-            run.spec, execution, param_file_secrets=param_file_secrets
+        return launcher.launch(
+            runtime=self,
+            task=runspec,
+            handler=handler,
+            name=name,
+            project=project,
+            params=params,
+            inputs=inputs,
+            out_path=out_path,
+            workdir=workdir,
+            artifact_path=artifact_path,
+            watch=watch,
+            schedule=schedule,
+            hyperparams=hyperparams,
+            hyper_param_options=hyper_param_options,
+            verbose=verbose,
+            scrape_metrics=scrape_metrics,
+            local_code_path=local_code_path,
+            auto_build=auto_build,
+            param_file_secrets=param_file_secrets,
+            notifications=notifications,
+            returns=returns,
         )
-        if task_generator:
-            # verify valid task parameters
-            tasks = task_generator.generate(run)
-            for task in tasks:
-                self._verify_run_params(task.spec.parameters)
-
-        # post verifications, store execution in db and run pre run hooks
-        execution.store_run()
-        self._pre_run(run, execution)  # hook for runtime specific prep
-
-        last_err = None
-        # If the runtime is nested, it means the hyper-run will run within a single instance of the run.
-        # So while in the API, we consider the hyper-run as a single run, and then in the runtime itself when the
-        # runtime is now a local runtime and therefore `self._is_nested == False`, we run each task as a separate run by
-        # using the task generator
-        if task_generator and not self._is_nested:
-            # multiple runs (based on hyper params or params file)
-            runner = self._run_many
-            if hasattr(self, "_parallel_run_many") and task_generator.use_parallel():
-                runner = self._parallel_run_many
-            results = runner(task_generator, execution, run)
-            results_to_iter(results, run, execution)
-            result = execution.to_dict()
-            result = self._update_run_state(result, task=run)
-
-        else:
-            # single run
-            try:
-                resp = self._run(run, execution)
-                if (
-                    watch
-                    and mlrun.runtimes.RuntimeKinds.is_watchable(self.kind)
-                    # API shouldn't watch logs, its the client job to query the run logs
-                    and not mlrun.config.is_running_as_api()
-                ):
-                    state, _ = run.logs(True, self._get_db())
-                    if state not in ["succeeded", "completed"]:
-                        logger.warning(f"run ended with state {state}")
-                result = self._update_run_state(resp, task=run)
-            except RunError as err:
-                last_err = err
-                result = self._update_run_state(task=run, err=err)
-
-        self._post_run(result, execution)  # hook for runtime specific cleanup
-
-        return self._wrap_run_result(result, run, schedule=schedule, err=last_err)
-
-    def _wrap_run_result(
-        self, result: dict, runspec: RunObject, schedule=None, err=None
-    ):
-        # if the purpose was to schedule (and not to run) nothing to wrap
-        if schedule:
-            return
-
-        if result and self.kfp and err is None:
-            write_kfpmeta(result)
-
-        # show ipython/jupyter result table widget
-        results_tbl = RunList()
-        if result:
-            results_tbl.append(result)
-        else:
-            logger.info("no returned result (job may still be in progress)")
-            results_tbl.append(runspec.to_dict())
-
-        uid = runspec.metadata.uid
-        project = runspec.metadata.project
-        if is_ipython and config.ipython_widget:
-            results_tbl.show()
-            print()
-            ui_url = get_ui_url(project, uid)
-            if ui_url:
-                ui_url = f' or <a href="{ui_url}" target="_blank">click here</a> to open in UI'
-            IPython.display.display(
-                IPython.display.HTML(
-                    f"<b> > to track results use the .show() or .logs() methods {ui_url}</b>"
-                )
-            )
-        elif not (self.is_child and is_running_as_api()):
-            project_flag = f"-p {project}" if project else ""
-            info_cmd = f"mlrun get run {uid} {project_flag}"
-            logs_cmd = f"mlrun logs {uid} {project_flag}"
-            logger.info(
-                "To track results use the CLI", info_cmd=info_cmd, logs_cmd=logs_cmd
-            )
-            ui_url = get_ui_url(project, uid)
-            if ui_url:
-                logger.info("Or click for UI", ui_url=ui_url)
-        if result:
-            run = RunObject.from_dict(result)
-            logger.info(
-                f"run executed, status={run.status.state}", name=run.metadata.name
-            )
-            if run.status.state == "error":
-                if self._is_remote and not self.is_child:
-                    logger.error(f"runtime error: {run.status.error}")
-                raise RunError(run.status.error)
-            return run
-
-        return None
 
     def _get_db_run(self, task: RunObject = None):
         if self._get_db() and task:
@@ -613,237 +396,6 @@ class BaseRuntime(ModelObj):
             runtime_env["MLRUN_NAMESPACE"] = self.metadata.namespace or config.namespace
         return runtime_env
 
-    def _run_local(
-        self,
-        runspec,
-        local_code_path,
-        project,
-        name,
-        workdir,
-        handler,
-        params,
-        inputs,
-        returns,
-        artifact_path,
-    ):
-        # allow local run simulation with a flip of a flag
-        command = self
-        if local_code_path:
-            project = project or self.metadata.project
-            name = name or self.metadata.name
-            command = local_code_path
-        return mlrun.run_local(
-            runspec,
-            command,
-            name,
-            self.spec.args,
-            workdir=workdir,
-            project=project,
-            handler=handler,
-            params=params,
-            inputs=inputs,
-            artifact_path=artifact_path,
-            mode=self.spec.mode,
-            allow_empty_resources=self.spec.allow_empty_resources,
-            returns=returns,
-        )
-
-    def _create_run_object(self, runspec):
-        # TODO: Once implemented the `Runtime` handlers configurations (doc strings, params type hints and returning
-        #       log hints, possible parameter values, etc), the configured type hints and log hints should be set into
-        #       the `RunObject` from the `Runtime`.
-        if runspec:
-            runspec = deepcopy(runspec)
-            if isinstance(runspec, str):
-                runspec = literal_eval(runspec)
-            if not isinstance(runspec, (dict, RunTemplate, RunObject)):
-                raise ValueError(
-                    "task/runspec is not a valid task object," f" type={type(runspec)}"
-                )
-
-        if isinstance(runspec, RunTemplate):
-            runspec = RunObject.from_template(runspec)
-        if isinstance(runspec, dict) or runspec is None:
-            runspec = RunObject.from_dict(runspec)
-        return runspec
-
-    def _enrich_run(
-        self,
-        runspec,
-        handler,
-        project_name,
-        name,
-        params,
-        inputs,
-        returns,
-        hyperparams,
-        hyper_param_options,
-        verbose,
-        scrape_metrics,
-        out_path,
-        artifact_path,
-        workdir,
-    ):
-        runspec.spec.handler = (
-            handler or runspec.spec.handler or self.spec.default_handler or ""
-        )
-        if runspec.spec.handler and self.kind not in ["handler", "dask"]:
-            runspec.spec.handler = runspec.spec.handler_name
-
-        def_name = self.metadata.name
-        if runspec.spec.handler_name:
-            short_name = runspec.spec.handler_name
-            for separator in ["#", "::", "."]:
-                # drop paths, module or class name from short name
-                if separator in short_name:
-                    short_name = short_name.split(separator)[-1]
-            def_name += "-" + short_name
-
-        runspec.metadata.name = normalize_name(
-            name=name or runspec.metadata.name or def_name,
-            # if name or runspec.metadata.name are set then it means that is user defined name and we want to warn the
-            # user that the passed name needs to be set without underscore, if its not user defined but rather enriched
-            # from the handler(function) name then we replace the underscore without warning the user.
-            # most of the time handlers will have `_` in the handler name (python convention is to separate function
-            # words with `_`), therefore we don't want to be noisy when normalizing the run name
-            verbose=bool(name or runspec.metadata.name),
-        )
-        verify_field_regex(
-            "run.metadata.name", runspec.metadata.name, mlrun.utils.regex.run_name
-        )
-        runspec.metadata.project = (
-            project_name
-            or runspec.metadata.project
-            or self.metadata.project
-            or config.default_project
-        )
-        runspec.spec.parameters = params or runspec.spec.parameters
-        runspec.spec.inputs = inputs or runspec.spec.inputs
-        runspec.spec.returns = returns or runspec.spec.returns
-        runspec.spec.hyperparams = hyperparams or runspec.spec.hyperparams
-        runspec.spec.hyper_param_options = (
-            hyper_param_options or runspec.spec.hyper_param_options
-        )
-        runspec.spec.verbose = verbose or runspec.spec.verbose
-        if scrape_metrics is None:
-            if runspec.spec.scrape_metrics is None:
-                scrape_metrics = config.scrape_metrics
-            else:
-                scrape_metrics = runspec.spec.scrape_metrics
-        runspec.spec.scrape_metrics = scrape_metrics
-        runspec.spec.input_path = (
-            workdir or runspec.spec.input_path or self.spec.workdir
-        )
-        if self.spec.allow_empty_resources:
-            runspec.spec.allow_empty_resources = self.spec.allow_empty_resources
-
-        spec = runspec.spec
-        if spec.secret_sources:
-            self._secrets = SecretsStore.from_list(spec.secret_sources)
-
-        # update run metadata (uid, labels) and store in DB
-        meta = runspec.metadata
-        meta.uid = meta.uid or uuid.uuid4().hex
-
-        runspec.spec.output_path = out_path or artifact_path or runspec.spec.output_path
-
-        if not runspec.spec.output_path:
-            if runspec.metadata.project:
-                if (
-                    mlrun.pipeline_context.project
-                    and runspec.metadata.project
-                    == mlrun.pipeline_context.project.metadata.name
-                ):
-                    runspec.spec.output_path = (
-                        mlrun.pipeline_context.project.spec.artifact_path
-                        or mlrun.pipeline_context.workflow_artifact_path
-                    )
-
-                if not runspec.spec.output_path and self._get_db():
-                    try:
-                        # not passing or loading the DB before the enrichment on purpose, because we want to enrich the
-                        # spec first as get_db() depends on it
-                        project = self._get_db().get_project(runspec.metadata.project)
-                        # this is mainly for tests, so we won't need to mock get_project for so many tests
-                        # in normal use cases if no project is found we will get an error
-                        if project:
-                            runspec.spec.output_path = project.spec.artifact_path
-                    except mlrun.errors.MLRunNotFoundError:
-                        logger.warning(
-                            f"project {project_name} is not saved in DB yet, "
-                            f"enriching output path with default artifact path: {config.artifact_path}"
-                        )
-
-            if not runspec.spec.output_path:
-                runspec.spec.output_path = config.artifact_path
-
-        if runspec.spec.output_path:
-            runspec.spec.output_path = runspec.spec.output_path.replace(
-                "{{run.uid}}", meta.uid
-            )
-            runspec.spec.output_path = mlrun.utils.helpers.fill_artifact_path_template(
-                runspec.spec.output_path, runspec.metadata.project
-            )
-        return runspec
-
-    def _submit_job(self, run: RunObject, schedule, db, watch):
-        if self._secrets:
-            run.spec.secret_sources = self._secrets.to_serial()
-        try:
-            resp = db.submit_job(run, schedule=schedule)
-            if schedule:
-                action = resp.pop("action", "created")
-                logger.info(f"task schedule {action}", **resp)
-                return
-
-        except (requests.HTTPError, Exception) as err:
-            logger.error(f"got remote run err, {err_to_str(err)}")
-
-            if isinstance(err, requests.HTTPError):
-                self._handle_submit_job_http_error(err)
-
-            result = None
-            # if we got a schedule no reason to do post_run stuff (it purposed to update the run status with error,
-            # but there's no run in case of schedule)
-            if not schedule:
-                result = self._update_run_state(task=run, err=err_to_str(err))
-            return self._wrap_run_result(result, run, schedule=schedule, err=err)
-
-        if resp:
-            txt = get_in(resp, "status.status_text")
-            if txt:
-                logger.info(txt)
-        # watch is None only in scenario where we run from pipeline step, in this case we don't want to watch the run
-        # logs too frequently but rather just pull the state of the run from the DB and pull the logs every x seconds
-        # which ideally greater than the pull state interval, this reduces unnecessary load on the API server, as
-        # running a pipeline is mostly not an interactive process which means the logs pulling doesn't need to be pulled
-        # in real time
-        if (
-            watch is None
-            and self.kfp
-            and config.httpdb.logs.pipelines.pull_state.mode == "enabled"
-        ):
-            state_interval = int(
-                config.httpdb.logs.pipelines.pull_state.pull_state_interval
-            )
-            logs_interval = int(
-                config.httpdb.logs.pipelines.pull_state.pull_logs_interval
-            )
-
-            run.wait_for_completion(
-                show_logs=True,
-                sleep=state_interval,
-                logs_interval=logs_interval,
-                raise_on_failure=False,
-            )
-            resp = self._get_db_run(run)
-
-        elif watch or self.kfp:
-            run.logs(True, self._get_db())
-            resp = self._get_db_run(run)
-
-        return self._wrap_run_result(resp, run, schedule=schedule)
-
     @staticmethod
     def _handle_submit_job_http_error(error: requests.HTTPError):
         # if we receive a 400 status code, this means the request was invalid and the run wasn't created in the DB.
@@ -1048,6 +600,12 @@ class BaseRuntime(ModelObj):
         if not handler:
             raise RunError(f"handler must be provided for {self.kind} runtime")
 
+    def _has_pipeline_param(self) -> bool:
+        # check if the runtime has pipeline parameters
+        # https://www.kubeflow.org/docs/components/pipelines/v1/sdk/parameters/
+        matches = re.findall(mlrun.utils.regex.pipeline_param[0], self.to_json())
+        return bool(matches)
+
     def full_image_path(
         self, image=None, client_version: str = None, client_python_version: str = None
     ):
@@ -1127,10 +685,9 @@ class BaseRuntime(ModelObj):
         :return: KubeFlow containerOp
         """
 
-        # if self.spec.image and not image:
-        #     image = self.full_image_path()
-
-        if use_db:
+        # if the function contain KFP PipelineParams (futures) pass the full spec to the
+        # ContainerOp this way KFP will substitute the params with previous step outputs
+        if use_db and not self._has_pipeline_param():
             # if the same function is built as part of the pipeline we do not use the versioned function
             # rather the latest function w the same tag so we can pick up the updated image/status
             versioned = False if hasattr(self, "_build_in_pipeline") else True
@@ -1206,104 +763,93 @@ class BaseRuntime(ModelObj):
         self,
         requirements: Union[str, List[str]],
         overwrite: bool = False,
-        verify_base_image: bool = True,
+        verify_base_image: bool = False,
+        prepare_image_for_deploy: bool = True,
+        requirements_file: str = "",
     ):
         """add package requirements from file or list to build spec.
 
-        :param requirements:  python requirements file path or list of packages
-        :param overwrite:     overwrite existing requirements
-        :param verify_base_image:  verify that the base image is configured
+        :param requirements:                a list of python packages
+        :param requirements_file:           a local python requirements file path
+        :param overwrite:                   overwrite existing requirements
+        :param verify_base_image:           verify that the base image is configured
+                                            (deprecated, use prepare_image_for_deploy)
+        :param prepare_image_for_deploy:    prepare the image/base_image spec for deployment
         :return: function object
         """
-        encoded_requirements = self._encode_requirements(requirements)
-        commands = self.spec.build.commands or [] if not overwrite else []
-        new_command = f"python -m pip install {encoded_requirements}"
-        # make sure we dont append the same line twice
-        if new_command not in commands:
-            commands.append(new_command)
-        self.spec.build.commands = commands
-        if verify_base_image:
-            self.verify_base_image()
+        self.spec.build.with_requirements(requirements, requirements_file, overwrite)
+
+        if verify_base_image or prepare_image_for_deploy:
+            # TODO: remove verify_base_image in 1.6.0
+            if verify_base_image:
+                warnings.warn(
+                    "verify_base_image is deprecated in 1.4.0 and will be removed in 1.6.0, "
+                    "use prepare_image_for_deploy",
+                    category=FutureWarning,
+                )
+            self.prepare_image_for_deploy()
+
         return self
 
     def with_commands(
         self,
         commands: List[str],
         overwrite: bool = False,
-        verify_base_image: bool = True,
+        verify_base_image: bool = False,
+        prepare_image_for_deploy: bool = True,
     ):
         """add commands to build spec.
 
-        :param commands:  list of commands to run during build
+        :param commands:                    list of commands to run during build
+        :param overwrite:                   overwrite existing commands
+        :param verify_base_image:           verify that the base image is configured
+                                            (deprecated, use prepare_image_for_deploy)
+        :param prepare_image_for_deploy:    prepare the image/base_image spec for deployment
 
         :return: function object
         """
-        if not isinstance(commands, list):
-            raise ValueError("commands must be a string list")
-        if not self.spec.build.commands or overwrite:
-            self.spec.build.commands = commands
-        else:
-            # add commands to existing build commands
-            for command in commands:
-                if command not in self.spec.build.commands:
-                    self.spec.build.commands.append(command)
-            # using list(set(x)) won't retain order,
-            # solution inspired from https://stackoverflow.com/a/17016257/8116661
-            self.spec.build.commands = list(dict.fromkeys(self.spec.build.commands))
-        if verify_base_image:
-            self.verify_base_image()
+        self.spec.build.with_commands(commands, overwrite)
+
+        if verify_base_image or prepare_image_for_deploy:
+            # TODO: remove verify_base_image in 1.6.0
+            if verify_base_image:
+                warnings.warn(
+                    "verify_base_image is deprecated in 1.4.0 and will be removed in 1.6.0, "
+                    "use prepare_image_for_deploy",
+                    category=FutureWarning,
+                )
+
+            self.prepare_image_for_deploy()
         return self
 
     def clean_build_params(self):
-        # when using `with_requirements` we also execute `verify_base_image` which adds the base image and cleans the
-        # spec.image, so we need to restore the image back
+        # when using `with_requirements` we also execute `prepare_image_for_deploy` which adds the base image
+        # and cleans the spec.image, so we need to restore the image back
         if self.spec.build.base_image and not self.spec.image:
             self.spec.image = self.spec.build.base_image
 
         self.spec.build = {}
         return self
 
+    # TODO: remove in 1.6.0
+    @deprecated(
+        version="1.4.0",
+        reason="'verify_base_image' will be removed in 1.6.0, use 'prepare_image_for_deploy' instead",
+        category=FutureWarning,
+    )
     def verify_base_image(self):
-        build = self.spec.build
-        require_build = build.commands or (
-            build.source and not build.load_source_on_run
+        self.prepare_image_for_deploy()
+
+    def prepare_image_for_deploy(self):
+        """
+        if a function has a 'spec.image' it is considered to be deployed,
+        but because we allow the user to set 'spec.image' for usability purposes,
+        we need to check whether this is a built image or it requires to be built on top.
+        """
+        launcher = mlrun.launcher.factory.LauncherFactory.create_launcher(
+            is_remote=self._is_remote
         )
-        image = self.spec.image
-        # we allow users to not set an image, in that case we'll use the default
-        if (
-            not image
-            and self.kind in mlrun.mlconf.function_defaults.image_by_kind.to_dict()
-        ):
-            image = mlrun.mlconf.function_defaults.image_by_kind.to_dict()[self.kind]
-
-        if (
-            self.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes()
-            # TODO: need a better way to decide whether a function requires a build
-            and require_build
-            and image
-            and not self.spec.build.base_image
-            # when submitting a run we are loading the function from the db, and using new_function for it,
-            # this results reaching here, but we are already after deploy of the image, meaning we don't need to prepare
-            # the base image for deployment
-            and self._is_remote_api()
-        ):
-            # when the function require build use the image as the base_image for the build
-            self.spec.build.base_image = image
-            self.spec.image = ""
-
-    def _verify_run_params(self, parameters: typing.Dict[str, typing.Any]):
-        for param_name, param_value in parameters.items():
-
-            if isinstance(param_value, dict):
-                # if the parameter is a dict, we might have some nested parameters,
-                # in this case we need to verify them as well recursively
-                self._verify_run_params(param_value)
-
-            # verify that integer parameters don't exceed a int64
-            if isinstance(param_value, int) and abs(param_value) >= 2**63:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    f"parameter {param_name} value {param_value} exceeds int64"
-                )
+        launcher.prepare_image_for_deploy(self)
 
     def export(self, target="", format=".yaml", secrets=None, strip=True):
         """save function spec to a local/remote path (default to./function.yaml)
@@ -1334,35 +880,12 @@ class BaseRuntime(ModelObj):
         return self
 
     def save(self, tag="", versioned=False, refresh=False) -> str:
-        db = self._get_db()
-        if not db:
-            logger.error("database connection is not configured")
-            return ""
-
-        if refresh and self._is_remote_api():
-            try:
-                meta = self.metadata
-                db_func = db.get_function(meta.name, meta.project, meta.tag)
-                if db_func and "status" in db_func:
-                    self.status = db_func["status"]
-                    if (
-                        self.status.state
-                        and self.status.state == "ready"
-                        and not hasattr(self.status, "nuclio_name")
-                    ):
-                        self.spec.image = get_in(db_func, "spec.image", self.spec.image)
-            except mlrun.errors.MLRunNotFoundError:
-                pass
-
-        tag = tag or self.metadata.tag
-
-        obj = self.to_dict()
-        logger.debug(f"saving function: {self.metadata.name}, tag: {tag}")
-        hash_key = db.store_function(
-            obj, self.metadata.name, self.metadata.project, tag, versioned
+        launcher = mlrun.launcher.factory.LauncherFactory.create_launcher(
+            is_remote=self._is_remote
+        )
+        return launcher.save_function(
+            self, tag=tag, versioned=versioned, refresh=refresh
         )
-        hash_key = hash_key if versioned else None
-        return "db://" + self._function_uri(hash_key=hash_key, tag=tag)
 
     def to_dict(self, fields=None, exclude=None, strip=False):
         struct = super().to_dict(fields, exclude=exclude)
@@ -1391,76 +914,11 @@ class BaseRuntime(ModelObj):
                             line += f", default={p['default']}"
                         print("    " + line)
 
-    def _encode_requirements(self, requirements_to_encode):
-
-        # if a string, read the file then encode
-        if isinstance(requirements_to_encode, str):
-            with open(requirements_to_encode, "r") as fp:
-                requirements_to_encode = fp.read().splitlines()
-
-        requirements = []
-        for requirement in requirements_to_encode:
-            requirement = requirement.strip()
-
-            # ignore empty lines
-            # ignore comments
-            if not requirement or requirement.startswith("#"):
-                continue
-
-            # ignore inline comments as well
-            inline_comment = requirement.split(" #")
-            if len(inline_comment) > 1:
-                requirement = inline_comment[0].strip()
-
-            # -r / --requirement are flags and should not be escaped
-            # we allow such flags (could be passed within the requirements.txt file) and do not
-            # try to open the file and include its content since it might be a remote file
-            # given on the base image.
-            for req_flag in ["-r", "--requirement"]:
-                if requirement.startswith(req_flag):
-                    requirement = requirement[len(req_flag) :].strip()
-                    requirements.append(req_flag)
-                    break
-
-            # wrap in single quote to ensure that the requirement is treated as a single string
-            # quote the requirement to avoid issues with special characters, double quotes, etc.
-            requirements.append(shlex.quote(requirement))
-
-        return " ".join(requirements)
-
-    def _validate_output_path(self, run):
-        if is_local(run.spec.output_path):
-            message = ""
-            if not os.path.isabs(run.spec.output_path):
-                message = (
-                    "artifact/output path is not defined or is local and relative,"
-                    " artifacts will not be visible in the UI"
-                )
-                if mlrun.runtimes.RuntimeKinds.requires_absolute_artifacts_path(
-                    self.kind
-                ):
-                    raise mlrun.errors.MLRunPreconditionFailedError(
-                        "artifact path (`artifact_path`) must be absolute for remote tasks"
-                    )
-            elif hasattr(self.spec, "volume_mounts") and not self.spec.volume_mounts:
-                message = (
-                    "artifact output path is local while no volume mount is specified. "
-                    "artifacts would not be visible via UI."
-                )
-            if message:
-                logger.warning(message, output_path=run.spec.output_path)
-
-
-def is_local(url):
-    if not url:
-        return True
-    return "://" not in url
-
 
 class BaseRuntimeHandler(ABC):
     # setting here to allow tests to override
     kind = "base"
-    class_modes: typing.Dict[RuntimeClassMode, str] = {}
+    class_modes: Dict[RuntimeClassMode, str] = {}
     wait_for_deletion_interval = 10
 
     @staticmethod
@@ -1474,12 +932,12 @@ class BaseRuntimeHandler(ABC):
     def _should_collect_logs(self) -> bool:
         """
         There are some runtimes which we don't collect logs for using the log collector
-        :return: whether should collect log for it
+        :return: whether it should collect log for it
         """
         return True
 
     def _get_possible_mlrun_class_label_values(
-        self, class_mode: typing.Union[RuntimeClassMode, str] = None
+        self, class_mode: Union[RuntimeClassMode, str] = None
     ) -> List[str]:
         """
         Should return the possible values of the mlrun/class label for runtime resources that are of this runtime
@@ -1493,21 +951,20 @@ class BaseRuntimeHandler(ABC):
     def list_resources(
         self,
         project: str,
-        object_id: typing.Optional[str] = None,
+        object_id: Optional[str] = None,
         label_selector: str = None,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> Union[
-        mlrun.api.schemas.RuntimeResources,
-        mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+        mlrun.common.schemas.RuntimeResources,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
     ]:
         # We currently don't support removing runtime resources in non k8s env
-        if not mlrun.k8s_utils.get_k8s_helper(
-            silent=True
-        ).is_running_inside_kubernetes_cluster():
+        if not get_k8s().is_running_inside_kubernetes_cluster():
             return {}
-        k8s_helper = get_k8s_helper()
-        namespace = k8s_helper.resolve_namespace()
+        namespace = get_k8s().resolve_namespace()
         label_selector = self.resolve_label_selector(project, object_id, label_selector)
         pods = self._list_pods(namespace, label_selector)
         pod_resources = self._build_pod_resources(pods)
@@ -1523,8 +980,10 @@ class BaseRuntimeHandler(ABC):
 
     def build_output_from_runtime_resources(
         self,
-        runtime_resources_list: List[mlrun.api.schemas.RuntimeResources],
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        runtime_resources_list: List[mlrun.common.schemas.RuntimeResources],
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ):
         pod_resources = []
         crd_resources = []
@@ -1550,12 +1009,9 @@ class BaseRuntimeHandler(ABC):
         if grace_period is None:
             grace_period = config.runtime_resources_deletion_grace_period
         # We currently don't support removing runtime resources in non k8s env
-        if not mlrun.k8s_utils.get_k8s_helper(
-            silent=True
-        ).is_running_inside_kubernetes_cluster():
+        if not get_k8s().is_running_inside_kubernetes_cluster():
             return
-        k8s_helper = get_k8s_helper()
-        namespace = k8s_helper.resolve_namespace()
+        namespace = get_k8s().resolve_namespace()
         label_selector = self.resolve_label_selector("*", label_selector=label_selector)
         crd_group, crd_version, crd_plural = self._get_crd_info()
         if crd_group and crd_version and crd_plural:
@@ -1603,8 +1059,7 @@ class BaseRuntimeHandler(ABC):
         self.delete_resources(db, db_session, label_selector, force, grace_period)
 
     def monitor_runs(self, db: DBInterface, db_session: Session):
-        k8s_helper = get_k8s_helper()
-        namespace = k8s_helper.resolve_namespace()
+        namespace = get_k8s().resolve_namespace()
         label_selector = self._get_default_label_selector()
         crd_group, crd_version, crd_plural = self._get_crd_info()
         runtime_resource_is_crd = False
@@ -1753,8 +1208,8 @@ class BaseRuntimeHandler(ABC):
 
     def _add_object_label_selector_if_needed(
         self,
-        object_id: typing.Optional[str] = None,
-        label_selector: typing.Optional[str] = None,
+        object_id: Optional[str] = None,
+        label_selector: Optional[str] = None,
     ):
         if object_id:
             object_label_selector = self._get_object_label_selector(object_id)
@@ -1776,17 +1231,19 @@ class BaseRuntimeHandler(ABC):
     def _enrich_list_resources_response(
         self,
         response: Union[
-            mlrun.api.schemas.RuntimeResources,
-            mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-            mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+            mlrun.common.schemas.RuntimeResources,
+            mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+            mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
         ],
         namespace: str,
         label_selector: str = None,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> Union[
-        mlrun.api.schemas.RuntimeResources,
-        mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+        mlrun.common.schemas.RuntimeResources,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
     ]:
         """
         Override this to list resources other then pods or CRDs (which are handled by the base class)
@@ -1796,12 +1253,14 @@ class BaseRuntimeHandler(ABC):
     def _build_output_from_runtime_resources(
         self,
         response: Union[
-            mlrun.api.schemas.RuntimeResources,
-            mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-            mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+            mlrun.common.schemas.RuntimeResources,
+            mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+            mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
         ],
-        runtime_resources_list: List[mlrun.api.schemas.RuntimeResources],
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        runtime_resources_list: List[mlrun.common.schemas.RuntimeResources],
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ):
         """
         Override this to add runtime resources other than pods or CRDs (which are handled by the base class) to the
@@ -1883,7 +1342,7 @@ class BaseRuntimeHandler(ABC):
         return in_terminal_state, last_container_completion_time, run_state
 
     def _get_default_label_selector(
-        self, class_mode: typing.Union[RuntimeClassMode, str] = None
+        self, class_mode: Union[RuntimeClassMode, str] = None
     ) -> str:
         """
         Override this to add a default label selector
@@ -1932,20 +1391,18 @@ class BaseRuntimeHandler(ABC):
         return False
 
     def _list_pods(self, namespace: str, label_selector: str = None) -> List:
-        k8s_helper = get_k8s_helper()
-        pods = k8s_helper.list_pods(namespace, selector=label_selector)
+        pods = get_k8s().list_pods(namespace, selector=label_selector)
         # when we work with custom objects (list_namespaced_custom_object) it's always a dict, to be able to generalize
         # code working on runtime resource (either a custom object or a pod) we're transforming to dicts
         pods = [pod.to_dict() for pod in pods]
         return pods
 
     def _list_crd_objects(self, namespace: str, label_selector: str = None) -> List:
-        k8s_helper = get_k8s_helper()
         crd_group, crd_version, crd_plural = self._get_crd_info()
         crd_objects = []
         if crd_group and crd_version and crd_plural:
             try:
-                crd_objects = k8s_helper.crdapi.list_namespaced_custom_object(
+                crd_objects = get_k8s().crdapi.list_namespaced_custom_object(
                     crd_group,
                     crd_version,
                     namespace,
@@ -1963,9 +1420,9 @@ class BaseRuntimeHandler(ABC):
     def resolve_label_selector(
         self,
         project: str,
-        object_id: typing.Optional[str] = None,
-        label_selector: typing.Optional[str] = None,
-        class_mode: typing.Union[RuntimeClassMode, str] = None,
+        object_id: Optional[str] = None,
+        label_selector: Optional[str] = None,
+        class_mode: Union[RuntimeClassMode, str] = None,
         with_main_runtime_resource_label_selector: bool = False,
     ) -> str:
         default_label_selector = self._get_default_label_selector(class_mode=class_mode)
@@ -1996,7 +1453,7 @@ class BaseRuntimeHandler(ABC):
     @staticmethod
     def resolve_object_id(
         run: dict,
-    ) -> typing.Optional[str]:
+    ) -> Optional[str]:
         """
         Get the object id from the run object
         Override this if the object id is not the run uid
@@ -2011,11 +1468,10 @@ class BaseRuntimeHandler(ABC):
         deleted_pods: List[Dict],
         label_selector: str = None,
     ):
-        k8s_helper = get_k8s_helper()
         deleted_pod_names = [pod_dict["metadata"]["name"] for pod_dict in deleted_pods]
 
         def _verify_pods_removed():
-            pods = k8s_helper.v1api.list_namespaced_pod(
+            pods = get_k8s().v1api.list_namespaced_pod(
                 namespace, label_selector=label_selector
             )
             existing_pod_names = [pod.metadata.name for pod in pods.items]
@@ -2068,10 +1524,10 @@ class BaseRuntimeHandler(ABC):
                     "name"
                 ]
             still_in_deletion_crds_to_pod_names = {}
-            jobs_runtime_resources: mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput = self.list_resources(
+            jobs_runtime_resources: mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput = self.list_resources(
                 "*",
                 label_selector=label_selector,
-                group_by=mlrun.api.schemas.ListRuntimeResourcesGroupByField.job,
+                group_by=mlrun.common.schemas.ListRuntimeResourcesGroupByField.job,
             )
             for project, project_jobs in jobs_runtime_resources.items():
                 if project not in project_uid_crd_map:
@@ -2119,8 +1575,7 @@ class BaseRuntimeHandler(ABC):
     ) -> List[Dict]:
         if grace_period is None:
             grace_period = config.runtime_resources_deletion_grace_period
-        k8s_helper = get_k8s_helper()
-        pods = k8s_helper.v1api.list_namespaced_pod(
+        pods = get_k8s().v1api.list_namespaced_pod(
             namespace, label_selector=label_selector
         )
         deleted_pods = []
@@ -2161,7 +1616,7 @@ class BaseRuntimeHandler(ABC):
                             pod_name=pod.metadata.name,
                         )
 
-                get_k8s_helper().delete_pod(pod.metadata.name, namespace)
+                get_k8s().delete_pod(pod.metadata.name, namespace)
                 deleted_pods.append(pod_dict)
             except Exception as exc:
                 logger.warning(
@@ -2182,11 +1637,10 @@ class BaseRuntimeHandler(ABC):
     ) -> List[Dict]:
         if grace_period is None:
             grace_period = config.runtime_resources_deletion_grace_period
-        k8s_helper = get_k8s_helper()
         crd_group, crd_version, crd_plural = self._get_crd_info()
         deleted_crds = []
         try:
-            crd_objects = k8s_helper.crdapi.list_namespaced_custom_object(
+            crd_objects = get_k8s().crdapi.list_namespaced_custom_object(
                 crd_group,
                 crd_version,
                 namespace,
@@ -2238,7 +1692,7 @@ class BaseRuntimeHandler(ABC):
                                 crd_object_name=crd_object["metadata"]["name"],
                             )
 
-                    get_k8s_helper().delete_crd(
+                    get_k8s().delete_crd(
                         crd_object["metadata"]["name"],
                         crd_group,
                         crd_version,
@@ -2414,13 +1868,15 @@ class BaseRuntimeHandler(ABC):
 
     def _build_list_resources_response(
         self,
-        pod_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-        crd_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-        group_by: Optional[mlrun.api.schemas.ListRuntimeResourcesGroupByField] = None,
+        pod_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+        crd_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+        group_by: Optional[
+            mlrun.common.schemas.ListRuntimeResourcesGroupByField
+        ] = None,
     ) -> Union[
-        mlrun.api.schemas.RuntimeResources,
-        mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
-        mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput,
+        mlrun.common.schemas.RuntimeResources,
+        mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
+        mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
     ]:
         if crd_resources is None:
             crd_resources = []
@@ -2428,15 +1884,18 @@ class BaseRuntimeHandler(ABC):
             pod_resources = []
 
         if group_by is None:
-            return mlrun.api.schemas.RuntimeResources(
+            return mlrun.common.schemas.RuntimeResources(
                 crd_resources=crd_resources, pod_resources=pod_resources
             )
         else:
-            if group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.job:
+            if group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.job:
                 return self._build_grouped_by_job_list_resources_response(
                     pod_resources, crd_resources
                 )
-            elif group_by == mlrun.api.schemas.ListRuntimeResourcesGroupByField.project:
+            elif (
+                group_by
+                == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project
+            ):
                 return self._build_grouped_by_project_list_resources_response(
                     pod_resources, crd_resources
                 )
@@ -2447,9 +1906,9 @@ class BaseRuntimeHandler(ABC):
 
     def _build_grouped_by_project_list_resources_response(
         self,
-        pod_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-        crd_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-    ) -> mlrun.api.schemas.GroupedByProjectRuntimeResourcesOutput:
+        pod_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+        crd_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+    ) -> mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput:
         resources = {}
         for pod_resource in pod_resources:
             self._add_resource_to_grouped_by_project_resources_response(
@@ -2463,9 +1922,9 @@ class BaseRuntimeHandler(ABC):
 
     def _build_grouped_by_job_list_resources_response(
         self,
-        pod_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-        crd_resources: List[mlrun.api.schemas.RuntimeResource] = None,
-    ) -> mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput:
+        pod_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+        crd_resources: List[mlrun.common.schemas.RuntimeResource] = None,
+    ) -> mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput:
         resources = {}
         for pod_resource in pod_resources:
             self._add_resource_to_grouped_by_job_resources_response(
@@ -2479,9 +1938,9 @@ class BaseRuntimeHandler(ABC):
 
     def _add_resource_to_grouped_by_project_resources_response(
         self,
-        resources: mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
+        resources: mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
         resource_field_name: str,
-        resource: mlrun.api.schemas.RuntimeResource,
+        resource: mlrun.common.schemas.RuntimeResource,
     ):
         if "mlrun/class" in resource.labels:
             project = resource.labels.get("mlrun/project", "")
@@ -2493,9 +1952,9 @@ class BaseRuntimeHandler(ABC):
 
     def _add_resource_to_grouped_by_job_resources_response(
         self,
-        resources: mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
+        resources: mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
         resource_field_name: str,
-        resource: mlrun.api.schemas.RuntimeResource,
+        resource: mlrun.common.schemas.RuntimeResource,
     ):
         if "mlrun/uid" in resource.labels:
             project = resource.labels.get("mlrun/project", config.default_project)
@@ -2508,16 +1967,18 @@ class BaseRuntimeHandler(ABC):
     def _add_resource_to_grouped_by_field_resources_response(
         first_field_value: str,
         second_field_value: str,
-        resources: mlrun.api.schemas.GroupedByJobRuntimeResourcesOutput,
+        resources: mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
         resource_field_name: str,
-        resource: mlrun.api.schemas.RuntimeResource,
+        resource: mlrun.common.schemas.RuntimeResource,
     ):
         if first_field_value not in resources:
             resources[first_field_value] = {}
         if second_field_value not in resources[first_field_value]:
             resources[first_field_value][
                 second_field_value
-            ] = mlrun.api.schemas.RuntimeResources(pod_resources=[], crd_resources=[])
+            ] = mlrun.common.schemas.RuntimeResources(
+                pod_resources=[], crd_resources=[]
+            )
         if not getattr(
             resources[first_field_value][second_field_value], resource_field_name
         ):
@@ -2651,11 +2112,11 @@ class BaseRuntimeHandler(ABC):
         return project, uid, name
 
     @staticmethod
-    def _build_pod_resources(pods) -> List[mlrun.api.schemas.RuntimeResource]:
+    def _build_pod_resources(pods) -> List[mlrun.common.schemas.RuntimeResource]:
         pod_resources = []
         for pod in pods:
             pod_resources.append(
-                mlrun.api.schemas.RuntimeResource(
+                mlrun.common.schemas.RuntimeResource(
                     name=pod["metadata"]["name"],
                     labels=pod["metadata"]["labels"],
                     status=pod["status"],
@@ -2664,11 +2125,13 @@ class BaseRuntimeHandler(ABC):
         return pod_resources
 
     @staticmethod
-    def _build_crd_resources(custom_objects) -> List[mlrun.api.schemas.RuntimeResource]:
+    def _build_crd_resources(
+        custom_objects,
+    ) -> List[mlrun.common.schemas.RuntimeResource]:
         crd_resources = []
         for custom_object in custom_objects:
             crd_resources.append(
-                mlrun.api.schemas.RuntimeResource(
+                mlrun.common.schemas.RuntimeResource(
                     name=custom_object["metadata"]["name"],
                     labels=custom_object["metadata"]["labels"],
                     status=custom_object.get("status", {}),