mlrun 1.7.0rc4__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/runtimes/__init__.py

@@ -30,19 +30,22 @@ __all__ = [
 
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
+from ..common.runtimes.constants import MPIJobCRDVersions
 from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
-from .constants import MPIJobCRDVersions
 from .daskjob import DaskCluster  # noqa
 from .databricks_job.databricks_runtime import DatabricksRuntime
 from .kubejob import KubejobRuntime, KubeResource  # noqa
 from .local import HandlerRuntime, LocalRuntime  # noqa
-from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1  # noqa
+from .mpijob import MpiRuntimeV1  # noqa
 from .nuclio import (
     RemoteRuntime,
     ServingRuntime,
     new_v2_model_server,
     nuclio_init_hook,
 )
+from .nuclio.api_gateway import APIGateway
+from .nuclio.application import ApplicationRuntime
+from .nuclio.serving import serving_subkind
 from .remotesparkjob import RemoteSparkRuntime
 from .sparkjob import Spark3Runtime
 
@@ -101,6 +104,7 @@ class RuntimeKinds:
     local = "local"
     handler = "handler"
     databricks = "databricks"
+    application = "application"
 
     @staticmethod
     def all():
@@ -115,6 +119,7 @@ class RuntimeKinds:
             RuntimeKinds.mpijob,
             RuntimeKinds.local,
             RuntimeKinds.databricks,
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -147,6 +152,23 @@ class RuntimeKinds:
             RuntimeKinds.remote,
             RuntimeKinds.nuclio,
             RuntimeKinds.serving,
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def pure_nuclio_deployed_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+        ]
+
+    @staticmethod
+    def handlerless_runtimes():
+        return [
+            RuntimeKinds.serving,
+            # Application runtime handler is internal reverse proxy
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -211,10 +233,39 @@ class RuntimeKinds:
         # both spark and remote spark uses different mechanism for assigning images
         return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
 
+    @staticmethod
+    def resolve_nuclio_runtime(kind: str, sub_kind: str):
+        kind = kind.split(":")[0]
+        if kind not in RuntimeKinds.nuclio_runtimes():
+            raise ValueError(
+                f"Kind {kind} is not a nuclio runtime, available runtimes are {RuntimeKinds.nuclio_runtimes()}"
+            )
+
+        if sub_kind == serving_subkind:
+            return ServingRuntime()
+
+        if kind == RuntimeKinds.application:
+            return ApplicationRuntime()
+
+        runtime = RemoteRuntime()
+        runtime.spec.function_kind = sub_kind
+        return runtime
+
+    @staticmethod
+    def resolve_nuclio_sub_kind(kind):
+        is_nuclio = kind.startswith("nuclio")
+        sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
+        if kind == RuntimeKinds.serving:
+            is_nuclio = True
+            sub_kind = serving_subkind
+        elif kind == RuntimeKinds.application:
+            is_nuclio = True
+        return is_nuclio, sub_kind
+
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:
-        return MpiRuntimeContainer.selector()
+        return MpiRuntimeV1
 
     if kind == RuntimeKinds.spark:
         return Spark3Runtime
@@ -228,6 +279,7 @@ def get_runtime_class(kind: str):
         RuntimeKinds.local: LocalRuntime,
         RuntimeKinds.remotespark: RemoteSparkRuntime,
         RuntimeKinds.databricks: DatabricksRuntime,
+        RuntimeKinds.application: ApplicationRuntime,
     }
 
     return kind_runtime_map[kind]

mlrun/runtimes/base.py

@@ -21,8 +21,11 @@ from os import environ
 from typing import Callable, Optional, Union
 
 import requests.exceptions
+from mlrun_pipelines.common.ops import mlrun_op
 from nuclio.build import mlrun_footer
 
+import mlrun.common.constants
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -36,7 +39,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
 from ..config import config
 from ..datastore import store_manager
 from ..errors import err_to_str
-from ..kfpops import mlrun_op
 from ..lists import RunList
 from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
 from ..utils import (
@@ -138,20 +140,24 @@ class FunctionSpec(ModelObj):
 
     @property
     def clone_target_dir(self):
-        warnings.warn(
-            "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.8.0. "
-            "Use spec.build.source_code_target_dir instead.",
-            FutureWarning,
-        )
+        # TODO: remove this property in 1.9.0
+        if self.build.source_code_target_dir:
+            warnings.warn(
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "Use spec.build.source_code_target_dir instead.",
+                FutureWarning,
+            )
         return self.build.source_code_target_dir
 
     @clone_target_dir.setter
     def clone_target_dir(self, clone_target_dir):
-        warnings.warn(
-            "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.8.0. "
-            "Use spec.build.source_code_target_dir instead.",
-            FutureWarning,
-        )
+        # TODO: remove this property in 1.9.0
+        if clone_target_dir:
+            warnings.warn(
+                "The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.9.0. "
+                "Use spec.build.source_code_target_dir instead.",
+                FutureWarning,
+            )
         self.build.source_code_target_dir = clone_target_dir
 
     def enrich_function_preemption_spec(self):
@@ -464,11 +470,11 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(
-                "{{run.user}}", meta.labels["owner"]
+                "{{run.user}}", meta.labels[mlrun_constants.MLRunInternalLabels.owner]
             )
 
         if db and self.kind != "handler":
@@ -575,9 +581,9 @@ class BaseRuntime(ModelObj):
 
         elif (
             not was_none
-            and last_state != mlrun.runtimes.constants.RunStates.completed
+            and last_state != mlrun.common.runtimes.constants.RunStates.completed
             and last_state
-            not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
+            not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
         ):
             try:
                 runtime_cls = mlrun.runtimes.get_runtime_class(kind)
@@ -630,7 +636,9 @@ class BaseRuntime(ModelObj):
         image = image or self.spec.image or ""
 
         image = enrich_image_url(image, client_version, client_python_version)
-        if not image.startswith("."):
+        if not image.startswith(
+            mlrun.common.constants.IMAGE_NAME_ENRICH_REGISTRY_PREFIX
+        ):
             return image
         registry, repository = get_parsed_docker_registry()
         if registry:
@@ -700,11 +708,11 @@ class BaseRuntime(ModelObj):
                                   "key": "the_key".
         :param auto_build:      when set to True and the function require build it will be built on the first
                                 function run, use only if you dont plan on changing the build config between runs
-        :return: KubeFlow containerOp
+        :return: mlrun_pipelines.models.PipelineNodeWrapper
         """
 
         # if the function contain KFP PipelineParams (futures) pass the full spec to the
-        # ContainerOp this way KFP will substitute the params with previous step outputs
+        # PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
         if use_db and not self._has_pipeline_param():
             # if the same function is built as part of the pipeline we do not use the versioned function
             # rather the latest function w the same tag so we can pick up the updated image/status
@@ -782,7 +790,7 @@ class BaseRuntime(ModelObj):
         requirements: Optional[list[str]] = None,
         overwrite: bool = False,
         prepare_image_for_deploy: bool = True,
-        requirements_file: str = "",
+        requirements_file: Optional[str] = "",
     ):
         """add package requirements from file or list to build spec.
 
@@ -836,6 +844,12 @@ class BaseRuntime(ModelObj):
             or (build.source and not build.load_source_on_run)
         )
 
+    def enrich_runtime_spec(
+        self,
+        project_node_selector: dict[str, str],
+    ):
+        pass
+
     def prepare_image_for_deploy(self):
         """
         if a function has a 'spec.image' it is considered to be deployed,

mlrun/runtimes/databricks_job/databricks_wrapper.py

@@ -99,7 +99,7 @@ def save_credentials(
         credentials["DATABRICKS_CLUSTER_ID"] = cluster_id
 
     with open(credentials_path, "w") as yaml_file:
-        yaml.dump(credentials, yaml_file, default_flow_style=False)
+        yaml.safe_dump(credentials, yaml_file, default_flow_style=False)
 
 
 def run_mlrun_databricks_job(

mlrun/runtimes/funcdoc.py

@@ -16,8 +16,6 @@ import ast
 import inspect
 import re
 
-from deprecated import deprecated
-
 from mlrun.model import FunctionEntrypoint
 
 
@@ -73,32 +71,6 @@ def func_dict(
     }
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
-    category=FutureWarning,
-)
-def func_info(fn) -> dict:
-    sig = inspect.signature(fn)
-    doc = inspect.getdoc(fn) or ""
-
-    out = func_dict(
-        name=fn.__name__,
-        doc=doc,
-        params=[inspect_param(p) for p in sig.parameters.values()],
-        returns=param_dict(
-            type=type_name(sig.return_annotation, empty_is_none=True), default=None
-        ),
-        lineno=func_lineno(fn),
-    )
-
-    if not fn.__doc__ or not fn.__doc__.strip():
-        return out
-
-    return merge_doc(out, doc)
-
-
 def func_lineno(fn):
     try:
         return inspect.getsourcelines(fn)[1]

mlrun/runtimes/kubejob.py

@@ -12,16 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
 import warnings
 
+from mlrun_pipelines.common.ops import build_op
+
 import mlrun.common.schemas
 import mlrun.db
 import mlrun.errors
 
-from ..kfpops import build_op
 from ..model import RunObject
-from ..utils import get_in, logger
 from .pod import KubeResource
 
 
@@ -65,29 +64,13 @@ class KubejobRuntime(KubeResource):
         :param pull_at_runtime: load the archive into the container at job runtime vs on build/deploy
         :param target_dir:      target dir on runtime pod or repo clone / archive extraction
         """
-        mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
-
-        self.spec.build.source = source
-        if handler:
-            self.spec.default_handler = handler
-        if workdir:
-            self.spec.workdir = workdir
-        if target_dir:
-            self.spec.build.source_code_target_dir = target_dir
-
-        self.spec.build.load_source_on_run = pull_at_runtime
-        if (
-            self.spec.build.base_image
-            and not self.spec.build.commands
-            and pull_at_runtime
-            and not self.spec.image
-        ):
-            # if we load source from repo and don't need a full build use the base_image as the image
-            self.spec.image = self.spec.build.base_image
-        elif not pull_at_runtime:
-            # clear the image so build will not be skipped
-            self.spec.build.base_image = self.spec.build.base_image or self.spec.image
-            self.spec.image = ""
+        self._configure_mlrun_build_with_source(
+            source=source,
+            workdir=workdir,
+            handler=handler,
+            pull_at_runtime=pull_at_runtime,
+            target_dir=target_dir,
+        )
 
     def build_config(
         self,
@@ -169,116 +152,39 @@ class KubejobRuntime(KubeResource):
         show_on_failure: bool = False,
         force_build: bool = False,
     ) -> bool:
-        """deploy function, build container with dependencies
+        """Deploy function, build container with dependencies
 
-        :param watch:                   wait for the deploy to complete (and print build logs)
-        :param with_mlrun:              add the current mlrun package to the container build
-        :param skip_deployed:           skip the build if we already have an image for the function
-        :param is_kfp:                  deploy as part of a kfp pipeline
-        :param mlrun_version_specifier: which mlrun package version to include (if not current)
+        :param watch:                   Wait for the deploy to complete (and print build logs)
+        :param with_mlrun:              Add the current mlrun package to the container build
+        :param skip_deployed:           Skip the build if we already have an image for the function
+        :param is_kfp:                  Deploy as part of a kfp pipeline
+        :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param builder_env:             Kaniko builder pod env vars dict (for config/credentials)
                                         e.g. builder_env={"GIT_TOKEN": token}
-        :param show_on_failure:         show logs only in case of build failure
-        :param force_build:             set True for force building the image, even when no changes were made
+        :param show_on_failure:         Show logs only in case of build failure
+        :param force_build:             Set True for force building the image, even when no changes were made
 
         :return: True if the function is ready (deployed)
         """
 
         build = self.spec.build
+        with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
 
-        if with_mlrun is None:
-            if build.with_mlrun is not None:
-                with_mlrun = build.with_mlrun
-            else:
-                with_mlrun = build.base_image and not (
-                    build.base_image.startswith("mlrun/")
-                    or "/mlrun/" in build.base_image
-                )
-
-        if (
-            not build.source
-            and not build.commands
-            and not build.requirements
-            and not build.extra
-            and with_mlrun
-        ):
-            logger.info(
-                "Running build to add mlrun package, set "
-                "with_mlrun=False to skip if its already in the image"
-            )
         self.status.state = ""
         if build.base_image:
             # clear the image so build will not be skipped
             self.spec.image = ""
 
-        # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
-        # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
-        if is_kfp:
-            watch = True
-
-        ready = False
-        if self._is_remote_api():
-            db = self._get_db()
-            data = db.remote_builder(
-                self,
-                with_mlrun,
-                mlrun_version_specifier,
-                skip_deployed,
-                builder_env=builder_env,
-                force_build=force_build,
-            )
-            self.status = data["data"].get("status", None)
-            self.spec.image = get_in(data, "data.spec.image")
-            self.spec.build.base_image = self.spec.build.base_image or get_in(
-                data, "data.spec.build.base_image"
-            )
-            # Get the source target dir in case it was enriched due to loading source
-            self.spec.build.source_code_target_dir = get_in(
-                data, "data.spec.build.source_code_target_dir"
-            ) or get_in(data, "data.spec.clone_target_dir")
-            ready = data.get("ready", False)
-            if not ready:
-                logger.info(
-                    f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
-                )
-            if watch and not ready:
-                state = self._build_watch(watch, show_on_failure=show_on_failure)
-                ready = state == "ready"
-                self.status.state = state
-
-        if watch and not ready:
-            raise mlrun.errors.MLRunRuntimeError("Deploy failed")
-        return ready
-
-    def _build_watch(self, watch=True, logs=True, show_on_failure=False):
-        db = self._get_db()
-        offset = 0
-        try:
-            text, _ = db.get_builder_status(self, 0, logs=logs)
-        except mlrun.db.RunDBError:
-            raise ValueError("function or build process not found")
-
-        def print_log(text):
-            if text and (not show_on_failure or self.status.state == "error"):
-                print(text, end="")
-
-        print_log(text)
-        offset += len(text)
-        if watch:
-            while self.status.state in ["pending", "running"]:
-                time.sleep(2)
-                if show_on_failure:
-                    text = ""
-                    db.get_builder_status(self, 0, logs=False)
-                    if self.status.state == "error":
-                        # re-read the full log on failure
-                        text, _ = db.get_builder_status(self, offset, logs=logs)
-                else:
-                    text, _ = db.get_builder_status(self, offset, logs=logs)
-                print_log(text)
-                offset += len(text)
-
-        return self.status.state
+        return self._build_image(
+            builder_env=builder_env,
+            force_build=force_build,
+            mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
+            skip_deployed=skip_deployed,
+            watch=watch,
+            is_kfp=is_kfp,
+            with_mlrun=with_mlrun,
+        )
 
     def deploy_step(
         self,

mlrun/runtimes/local.py

@@ -33,6 +33,7 @@ from sys import executable
 from nuclio import Event
 
 import mlrun
+import mlrun.common.constants as mlrun_constants
 from mlrun.lists import RunList
 
 from ..errors import err_to_str
@@ -257,7 +258,8 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
             set_paths(os.path.realpath("."))
 
         if (
-            runobj.metadata.labels.get("kind") == RemoteSparkRuntime.kind
+            runobj.metadata.labels.get(mlrun_constants.MLRunInternalLabels.kind)
+            == RemoteSparkRuntime.kind
             and environ["MLRUN_SPARK_CLIENT_IGZ_SPARK"] == "true"
         ):
             from mlrun.runtimes.remotesparkjob import igz_spark_pre_hook
@@ -382,6 +384,7 @@ def load_module(file_name, handler, context):
         if spec is None:
             raise RunError(f"Cannot import from {file_name!r}")
         module = imputil.module_from_spec(spec)
+        sys.modules[mod_name] = module
         spec.loader.exec_module(module)
 
     class_args = {}
@@ -493,7 +496,7 @@ def exec_from_params(handler, runobj: RunObject, context: MLClientCtx, cwd=None)
                 logger.warning("Run was aborted", err=err_to_str(exc))
                 # Run was aborted, the state run state is updated by the abort job, no need to commit again
                 context.set_state(
-                    mlrun.runtimes.constants.RunStates.aborted, commit=False
+                    mlrun.common.runtimes.constants.RunStates.aborted, commit=False
                 )
                 commit = False
             except Exception as exc:

mlrun/runtimes/mpijob/__init__.py

@@ -21,28 +21,8 @@ from mlrun.config import config
 from .. import MPIJobCRDVersions
 from .abstract import AbstractMPIJobRuntime
 from .v1 import MpiRuntimeV1
-from .v1alpha1 import MpiRuntimeV1Alpha1
 
 
 def _resolve_mpijob_crd_version():
     # config is expected to get enriched from the API through the client-spec
     return config.mpijob_crd_version or MPIJobCRDVersions.default()
-
-
-class MpiRuntimeContainer(containers.DeclarativeContainer):
-    resolver = providers.Callable(
-        _resolve_mpijob_crd_version,
-    )
-
-    selector = providers.Selector(
-        resolver,
-        v1=providers.Object(MpiRuntimeV1),
-        v1alpha1=providers.Object(MpiRuntimeV1Alpha1),
-    )
-
-    # An empty selector to be overriden by the API
-    handler_selector = providers.Selector(
-        resolver,
-        v1=providers.Object(None),
-        v1alpha1=providers.Object(None),
-    )

mlrun/runtimes/mpijob/abstract.py

@@ -223,14 +223,14 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
         ```
         # Define the wanted MPI arguments
         mpi_args = []
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_DEBUG=INFO')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_SOCKET_NTHREADS=2')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_NSOCKS_PERTHREAD=8')
-        mpi_args.append('-x')
-        mpi_args.append('NCCL_MIN_NCHANNELS=4')
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_DEBUG=INFO")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_SOCKET_NTHREADS=2")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_NSOCKS_PERTHREAD=8")
+        mpi_args.append("-x")
+        mpi_args.append("NCCL_MIN_NCHANNELS=4")
 
         # Set the MPI arguments in the function
         fn.set_mpi_args(mpi_args)

mlrun/runtimes/mpijob/v1.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mlrun.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
+from mlrun.common.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
 from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime, MPIResourceSpec
 
 

mlrun/runtimes/nuclio/__init__.py

@@ -18,3 +18,4 @@ from .function import (
     min_nuclio_versions,
     RemoteRuntime,
 )  # noqa
+from .api_gateway import APIGateway