mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/runtimes/__init__.py

@@ -30,17 +30,23 @@ __all__ = [
 
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
+from ..common.runtimes.constants import MPIJobCRDVersions
 from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
-from .constants import MPIJobCRDVersions
 from .daskjob import DaskCluster  # noqa
 from .databricks_job.databricks_runtime import DatabricksRuntime
-from .function import RemoteRuntime
 from .kubejob import KubejobRuntime, KubeResource  # noqa
 from .local import HandlerRuntime, LocalRuntime  # noqa
-from .mpijob import MpiRuntimeContainer, MpiRuntimeV1, MpiRuntimeV1Alpha1  # noqa
-from .nuclio import nuclio_init_hook
+from .mpijob import MpiRuntimeV1  # noqa
+from .nuclio import (
+    RemoteRuntime,
+    ServingRuntime,
+    new_v2_model_server,
+    nuclio_init_hook,
+)
+from .nuclio.api_gateway import APIGateway
+from .nuclio.application import ApplicationRuntime
+from .nuclio.serving import serving_subkind
 from .remotesparkjob import RemoteSparkRuntime
-from .serving import ServingRuntime, new_v2_model_server
 from .sparkjob import Spark3Runtime
 
 # for legacy imports (MLModelServer moved from here to /serving)
@@ -86,7 +92,7 @@ def new_model_server(
         )
 
 
-class RuntimeKinds(object):
+class RuntimeKinds:
     remote = "remote"
     nuclio = "nuclio"
     dask = "dask"
@@ -98,6 +104,7 @@ class RuntimeKinds(object):
     local = "local"
     handler = "handler"
     databricks = "databricks"
+    application = "application"
 
     @staticmethod
     def all():
@@ -112,6 +119,7 @@ class RuntimeKinds(object):
             RuntimeKinds.mpijob,
             RuntimeKinds.local,
             RuntimeKinds.databricks,
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -144,6 +152,23 @@ class RuntimeKinds(object):
             RuntimeKinds.remote,
             RuntimeKinds.nuclio,
             RuntimeKinds.serving,
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def pure_nuclio_deployed_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+        ]
+
+    @staticmethod
+    def handlerless_runtimes():
+        return [
+            RuntimeKinds.serving,
+            # Application runtime handler is internal reverse proxy
+            RuntimeKinds.application,
         ]
 
     @staticmethod
@@ -208,10 +233,39 @@ class RuntimeKinds(object):
         # both spark and remote spark uses different mechanism for assigning images
         return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
 
+    @staticmethod
+    def resolve_nuclio_runtime(kind: str, sub_kind: str):
+        kind = kind.split(":")[0]
+        if kind not in RuntimeKinds.nuclio_runtimes():
+            raise ValueError(
+                f"Kind {kind} is not a nuclio runtime, available runtimes are {RuntimeKinds.nuclio_runtimes()}"
+            )
+
+        if sub_kind == serving_subkind:
+            return ServingRuntime()
+
+        if kind == RuntimeKinds.application:
+            return ApplicationRuntime()
+
+        runtime = RemoteRuntime()
+        runtime.spec.function_kind = sub_kind
+        return runtime
+
+    @staticmethod
+    def resolve_nuclio_sub_kind(kind):
+        is_nuclio = kind.startswith("nuclio")
+        sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
+        if kind == RuntimeKinds.serving:
+            is_nuclio = True
+            sub_kind = serving_subkind
+        elif kind == RuntimeKinds.application:
+            is_nuclio = True
+        return is_nuclio, sub_kind
+
 
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:
-        return MpiRuntimeContainer.selector()
+        return MpiRuntimeV1
 
     if kind == RuntimeKinds.spark:
         return Spark3Runtime
@@ -225,6 +279,7 @@ def get_runtime_class(kind: str):
         RuntimeKinds.local: LocalRuntime,
         RuntimeKinds.remotespark: RemoteSparkRuntime,
         RuntimeKinds.databricks: DatabricksRuntime,
+        RuntimeKinds.application: ApplicationRuntime,
     }
 
     return kind_runtime_map[kind]

mlrun/runtimes/base.py

@@ -18,11 +18,14 @@ import typing
 import warnings
 from base64 import b64encode
 from os import environ
-from typing import Callable, Dict, List, Optional, Union
+from typing import Callable, Optional, Union
 
 import requests.exceptions
+from mlrun_pipelines.common.ops import mlrun_op
 from nuclio.build import mlrun_footer
 
+import mlrun.common.constants
+import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -36,7 +39,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
 from ..config import config
 from ..datastore import store_manager
 from ..errors import err_to_str
-from ..kfpops import mlrun_op
 from ..lists import RunList
 from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
 from ..utils import (
@@ -92,6 +94,7 @@ class FunctionStatus(ModelObj):
 
 class FunctionSpec(ModelObj):
     _dict_fields = spec_fields
+    _default_fields_to_strip = []
 
     def __init__(
         self,
@@ -123,7 +126,7 @@ class FunctionSpec(ModelObj):
         self.entry_points = entry_points or {}
         self.disable_auto_mount = disable_auto_mount
         self.allow_empty_resources = None
-        # the build.source is cloned/extracted to the specified clone_target_dir
+        # The build.source is cloned/extracted to the specified clone_target_dir
         # if a relative path is specified, it will be enriched with a temp dir path
         self._clone_target_dir = clone_target_dir or None
 
@@ -169,6 +172,9 @@ class BaseRuntime(ModelObj):
     _is_nested = False
     _is_remote = False
     _dict_fields = ["kind", "metadata", "spec", "status", "verbose"]
+    _default_fields_to_strip = ModelObj._default_fields_to_strip + [
+        "status",  # Function status describes the state rather than configuration
+    ]
 
     def __init__(self, metadata=None, spec=None):
         self._metadata = None
@@ -218,7 +224,7 @@ class BaseRuntime(ModelObj):
         self.metadata.labels[key] = str(value)
         return self
 
-    def set_categories(self, categories: List[str]):
+    def set_categories(self, categories: list[str]):
         self.metadata.categories = mlrun.utils.helpers.as_list(categories)
 
     @property
@@ -292,7 +298,7 @@ class BaseRuntime(ModelObj):
             mlrun.model.Credentials.generate_access_key
         )
 
-    def generate_runtime_k8s_env(self, runobj: RunObject = None) -> List[Dict]:
+    def generate_runtime_k8s_env(self, runobj: RunObject = None) -> list[dict]:
         """
         Prepares a runtime environment as it's expected by kubernetes.models.V1Container
 
@@ -313,23 +319,23 @@ class BaseRuntime(ModelObj):
         name: Optional[str] = "",
         project: Optional[str] = "",
         params: Optional[dict] = None,
-        inputs: Optional[Dict[str, str]] = None,
+        inputs: Optional[dict[str, str]] = None,
         out_path: Optional[str] = "",
         workdir: Optional[str] = "",
         artifact_path: Optional[str] = "",
         watch: Optional[bool] = True,
         schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
-        hyperparams: Optional[Dict[str, list]] = None,
+        hyperparams: Optional[dict[str, list]] = None,
         hyper_param_options: Optional[HyperParamOptions] = None,
         verbose: Optional[bool] = None,
         scrape_metrics: Optional[bool] = None,
         local: Optional[bool] = False,
         local_code_path: Optional[str] = None,
         auto_build: Optional[bool] = None,
-        param_file_secrets: Optional[Dict[str, str]] = None,
-        notifications: Optional[List[mlrun.model.Notification]] = None,
-        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
-        state_thresholds: Optional[Dict[str, int]] = None,
+        param_file_secrets: Optional[dict[str, str]] = None,
+        notifications: Optional[list[mlrun.model.Notification]] = None,
+        returns: Optional[list[Union[str, dict[str, str]]]] = None,
+        state_thresholds: Optional[dict[str, int]] = None,
         **launcher_kwargs,
     ) -> RunObject:
         """
@@ -426,7 +432,7 @@ class BaseRuntime(ModelObj):
         if task:
             return task.to_dict()
 
-    def _generate_runtime_env(self, runobj: RunObject = None) -> Dict:
+    def _generate_runtime_env(self, runobj: RunObject = None) -> dict:
         """
         Prepares all available environment variables for usage on a runtime
         Data will be extracted from several sources and most of them are not guaranteed to be available
@@ -464,11 +470,11 @@ class BaseRuntime(ModelObj):
     def _store_function(self, runspec, meta, db):
         meta.labels["kind"] = self.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
+            meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
         )
         if runspec.spec.output_path:
             runspec.spec.output_path = runspec.spec.output_path.replace(
-                "{{run.user}}", meta.labels["owner"]
+                "{{run.user}}", meta.labels[mlrun_constants.MLRunInternalLabels.owner]
             )
 
         if db and self.kind != "handler":
@@ -575,9 +581,9 @@ class BaseRuntime(ModelObj):
 
         elif (
             not was_none
-            and last_state != mlrun.runtimes.constants.RunStates.completed
+            and last_state != mlrun.common.runtimes.constants.RunStates.completed
             and last_state
-            not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
+            not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
         ):
             try:
                 runtime_cls = mlrun.runtimes.get_runtime_class(kind)
@@ -630,7 +636,9 @@ class BaseRuntime(ModelObj):
         image = image or self.spec.image or ""
 
         image = enrich_image_url(image, client_version, client_python_version)
-        if not image.startswith("."):
+        if not image.startswith(
+            mlrun.common.constants.IMAGE_NAME_ENRICH_REGISTRY_PREFIX
+        ):
             return image
         registry, repository = get_parsed_docker_registry()
         if registry:
@@ -661,7 +669,7 @@ class BaseRuntime(ModelObj):
         use_db=True,
         verbose=None,
         scrape_metrics=False,
-        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
+        returns: Optional[list[Union[str, dict[str, str]]]] = None,
         auto_build: bool = False,
     ):
         """Run a local or remote task.
@@ -700,11 +708,11 @@ class BaseRuntime(ModelObj):
                                   "key": "the_key".
         :param auto_build:      when set to True and the function require build it will be built on the first
                                 function run, use only if you dont plan on changing the build config between runs
-        :return: KubeFlow containerOp
+        :return: mlrun_pipelines.models.PipelineNodeWrapper
         """
 
         # if the function contain KFP PipelineParams (futures) pass the full spec to the
-        # ContainerOp this way KFP will substitute the params with previous step outputs
+        # PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
         if use_db and not self._has_pipeline_param():
             # if the same function is built as part of the pipeline we do not use the versioned function
             # rather the latest function w the same tag so we can pick up the updated image/status
@@ -767,7 +775,7 @@ class BaseRuntime(ModelObj):
                     body = fp.read()
             if self.kind == mlrun.runtimes.RuntimeKinds.serving:
                 body = body + mlrun_footer.format(
-                    mlrun.runtimes.serving.serving_subkind
+                    mlrun.runtimes.nuclio.serving.serving_subkind
                 )
 
         self.spec.build.functionSourceCode = b64encode(body.encode("utf-8")).decode(
@@ -779,10 +787,10 @@ class BaseRuntime(ModelObj):
 
     def with_requirements(
         self,
-        requirements: Optional[List[str]] = None,
+        requirements: Optional[list[str]] = None,
         overwrite: bool = False,
         prepare_image_for_deploy: bool = True,
-        requirements_file: str = "",
+        requirements_file: Optional[str] = "",
     ):
         """add package requirements from file or list to build spec.
 
@@ -801,7 +809,7 @@ class BaseRuntime(ModelObj):
 
     def with_commands(
         self,
-        commands: List[str],
+        commands: list[str],
         overwrite: bool = False,
         prepare_image_for_deploy: bool = True,
     ):
@@ -836,6 +844,12 @@ class BaseRuntime(ModelObj):
             or (build.source and not build.load_source_on_run)
         )
 
+    def enrich_runtime_spec(
+        self,
+        project_node_selector: dict[str, str],
+    ):
+        pass
+
     def prepare_image_for_deploy(self):
         """
         if a function has a 'spec.image' it is considered to be deployed,
@@ -870,7 +884,7 @@ class BaseRuntime(ModelObj):
             data = dict_to_json(struct)
         stores = store_manager.set(secrets)
         target = target or "function.yaml"
-        datastore, subpath = stores.get_or_create_store(target)
+        datastore, subpath, url = stores.get_or_create_store(target)
         datastore.put(subpath, data)
         logger.info(f"function spec saved to path: {target}")
         return self
@@ -883,13 +897,6 @@ class BaseRuntime(ModelObj):
             self, tag=tag, versioned=versioned, refresh=refresh
         )
 
-    def to_dict(self, fields=None, exclude=None, strip=False):
-        struct = super().to_dict(fields, exclude=exclude)
-        if strip:
-            if "status" in struct:
-                del struct["status"]
-        return struct
-
     def doc(self):
         print("function:", self.metadata.name)
         print(self.spec.description)

mlrun/runtimes/daskjob.py

@@ -16,7 +16,7 @@ import inspect
 import socket
 import time
 from os import environ
-from typing import Callable, Dict, List, Optional, Union
+from typing import Callable, Optional, Union
 
 import mlrun.common.schemas
 import mlrun.errors
@@ -461,7 +461,7 @@ class DaskCluster(KubejobRuntime):
 
     def set_state_thresholds(
         self,
-        state_thresholds: Dict[str, str],
+        state_thresholds: dict[str, str],
         patch: bool = True,
     ):
         raise NotImplementedError(
@@ -477,23 +477,23 @@ class DaskCluster(KubejobRuntime):
         name: Optional[str] = "",
         project: Optional[str] = "",
         params: Optional[dict] = None,
-        inputs: Optional[Dict[str, str]] = None,
+        inputs: Optional[dict[str, str]] = None,
         out_path: Optional[str] = "",
         workdir: Optional[str] = "",
         artifact_path: Optional[str] = "",
         watch: Optional[bool] = True,
         schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
-        hyperparams: Optional[Dict[str, list]] = None,
+        hyperparams: Optional[dict[str, list]] = None,
         hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
         verbose: Optional[bool] = None,
         scrape_metrics: Optional[bool] = None,
         local: Optional[bool] = False,
         local_code_path: Optional[str] = None,
         auto_build: Optional[bool] = None,
-        param_file_secrets: Optional[Dict[str, str]] = None,
-        notifications: Optional[List[mlrun.model.Notification]] = None,
-        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
-        state_thresholds: Optional[Dict[str, int]] = None,
+        param_file_secrets: Optional[dict[str, str]] = None,
+        notifications: Optional[list[mlrun.model.Notification]] = None,
+        returns: Optional[list[Union[str, dict[str, str]]]] = None,
+        state_thresholds: Optional[dict[str, int]] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if state_thresholds:

mlrun/runtimes/databricks_job/databricks_cancel_task.py

@@ -33,7 +33,7 @@ def main():
                 "The Databricks credentials path does not exist."
                 " Please manually cancel the job from the Databricks environment."
             )
-    with open(credentials_path, "r") as yaml_file:
+    with open(credentials_path) as yaml_file:
         loaded_data = yaml.safe_load(yaml_file)
     # use for flat yaml only
     for key, value in loaded_data.items():

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -14,7 +14,7 @@
 
 from ast import FunctionDef, parse, unparse
 from base64 import b64decode, b64encode
-from typing import Callable, Dict, List, Optional, Union
+from typing import Callable, Optional, Union
 
 import mlrun
 import mlrun.runtimes.kubejob as kubejob
@@ -215,23 +215,23 @@ def run_mlrun_databricks_job(context,task_parameters: dict, **kwargs):
         name: Optional[str] = "",
         project: Optional[str] = "",
         params: Optional[dict] = None,
-        inputs: Optional[Dict[str, str]] = None,
+        inputs: Optional[dict[str, str]] = None,
         out_path: Optional[str] = "",
         workdir: Optional[str] = "",
         artifact_path: Optional[str] = "",
         watch: Optional[bool] = True,
         schedule: Optional[Union[str, mlrun.common.schemas.ScheduleCronTrigger]] = None,
-        hyperparams: Optional[Dict[str, list]] = None,
+        hyperparams: Optional[dict[str, list]] = None,
         hyper_param_options: Optional[HyperParamOptions] = None,
         verbose: Optional[bool] = None,
         scrape_metrics: Optional[bool] = None,
         local: Optional[bool] = False,
         local_code_path: Optional[str] = None,
         auto_build: Optional[bool] = None,
-        param_file_secrets: Optional[Dict[str, str]] = None,
-        notifications: Optional[List[mlrun.model.Notification]] = None,
-        returns: Optional[List[Union[str, Dict[str, str]]]] = None,
-        state_thresholds: Optional[Dict[str, int]] = None,
+        param_file_secrets: Optional[dict[str, str]] = None,
+        notifications: Optional[list[mlrun.model.Notification]] = None,
+        returns: Optional[list[Union[str, dict[str, str]]]] = None,
+        state_thresholds: Optional[dict[str, int]] = None,
         **launcher_kwargs,
     ) -> RunObject:
         if local:

mlrun/runtimes/databricks_job/databricks_wrapper.py

@@ -99,7 +99,7 @@ def save_credentials(
         credentials["DATABRICKS_CLUSTER_ID"] = cluster_id
 
     with open(credentials_path, "w") as yaml_file:
-        yaml.dump(credentials, yaml_file, default_flow_style=False)
+        yaml.safe_dump(credentials, yaml_file, default_flow_style=False)
 
 
 def run_mlrun_databricks_job(

mlrun/runtimes/funcdoc.py

@@ -16,8 +16,6 @@ import ast
 import inspect
 import re
 
-from deprecated import deprecated
-
 from mlrun.model import FunctionEntrypoint
 
 
@@ -73,32 +71,6 @@ def func_dict(
     }
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
-    category=FutureWarning,
-)
-def func_info(fn) -> dict:
-    sig = inspect.signature(fn)
-    doc = inspect.getdoc(fn) or ""
-
-    out = func_dict(
-        name=fn.__name__,
-        doc=doc,
-        params=[inspect_param(p) for p in sig.parameters.values()],
-        returns=param_dict(
-            type=type_name(sig.return_annotation, empty_is_none=True), default=None
-        ),
-        lineno=func_lineno(fn),
-    )
-
-    if not fn.__doc__ or not fn.__doc__.strip():
-        return out
-
-    return merge_doc(out, doc)
-
-
 def func_lineno(fn):
     try:
         return inspect.getsourcelines(fn)[1]

mlrun/runtimes/function_reference.py

@@ -111,7 +111,7 @@ class FunctionReference(ModelObj):
             code = self.code
             if kind == mlrun.runtimes.RuntimeKinds.serving:
                 code = code + mlrun_footer.format(
-                    mlrun.runtimes.serving.serving_subkind
+                    mlrun.runtimes.nuclio.serving.serving_subkind
                 )
             func = mlrun.new_function(
                 self.name, kind=kind, image=self.image or default_image