mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

mlrun/runtimes/nuclio/application/application.py

@@ -12,14 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import pathlib
+import typing
 
 import nuclio
 
+import mlrun.common.schemas as schemas
 import mlrun.errors
-from mlrun.common.schemas import AuthInfo
+from mlrun.common.runtimes.constants import NuclioIngressAddTemplatedIngressModes
 from mlrun.runtimes import RemoteRuntime
 from mlrun.runtimes.nuclio import min_nuclio_versions
+from mlrun.runtimes.nuclio.api_gateway import (
+    APIGateway,
+    APIGatewayMetadata,
+    APIGatewaySpec,
+)
 from mlrun.runtimes.nuclio.function import NuclioSpec, NuclioStatus
+from mlrun.utils import logger
 
 
 class ApplicationSpec(NuclioSpec):
@@ -113,7 +121,10 @@ class ApplicationSpec(NuclioSpec):
             state_thresholds=state_thresholds,
             disable_default_http_trigger=disable_default_http_trigger,
         )
-        self.internal_application_port = internal_application_port or 8080
+        self.internal_application_port = (
+            internal_application_port
+            or mlrun.mlconf.function.application.default_sidecar_internal_port
+        )
 
     @property
     def internal_application_port(self):
@@ -139,6 +150,9 @@ class ApplicationStatus(NuclioStatus):
         container_image=None,
         application_image=None,
         sidecar_name=None,
+        api_gateway_name=None,
+        api_gateway=None,
+        url=None,
     ):
         super().__init__(
             state=state,
@@ -151,12 +165,15 @@ class ApplicationStatus(NuclioStatus):
         )
         self.application_image = application_image or None
         self.sidecar_name = sidecar_name or None
+        self.api_gateway_name = api_gateway_name or None
+        self.api_gateway = api_gateway or None
+        self.url = url or None
 
 
 class ApplicationRuntime(RemoteRuntime):
     kind = "application"
 
-    @min_nuclio_versions("1.12.7")
+    @min_nuclio_versions("1.13.1")
     def __init__(self, spec=None, metadata=None):
         super().__init__(spec=spec, metadata=metadata)
 
@@ -176,6 +193,24 @@ class ApplicationRuntime(RemoteRuntime):
     def status(self, status):
         self._status = self._verify_dict(status, "status", ApplicationStatus)
 
+    @property
+    def api_gateway(self):
+        return self.status.api_gateway
+
+    @api_gateway.setter
+    def api_gateway(self, api_gateway: APIGateway):
+        self.status.api_gateway = api_gateway
+
+    @property
+    def url(self):
+        if not self.status.api_gateway:
+            self._sync_api_gateway()
+        return self.status.api_gateway.invoke_url
+
+    @url.setter
+    def url(self, url):
+        self.status.url = url
+
     def set_internal_application_port(self, port: int):
         self.spec.internal_application_port = port
 
@@ -220,7 +255,7 @@ class ApplicationRuntime(RemoteRuntime):
         project="",
         tag="",
         verbose=False,
-        auth_info: AuthInfo = None,
+        auth_info: schemas.AuthInfo = None,
         builder_env: dict = None,
         force_build: bool = False,
         with_mlrun=None,
@@ -228,6 +263,10 @@ class ApplicationRuntime(RemoteRuntime):
         is_kfp=False,
         mlrun_version_specifier=None,
         show_on_failure: bool = False,
+        skip_access_key_auth: bool = False,
+        direct_port_access: bool = False,
+        authentication_mode: schemas.APIGatewayAuthenticationMode = None,
+        authentication_creds: tuple[str] = None,
     ):
         """
         Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
@@ -244,6 +283,10 @@ class ApplicationRuntime(RemoteRuntime):
         :param is_kfp:                  Deploy as part of a kfp pipeline
         :param mlrun_version_specifier: Which mlrun package version to include (if not current)
         :param show_on_failure:         Show logs only in case of build failure
+        :param skip_access_key_auth:    Skip adding access key auth to the API Gateway
+        :param direct_port_access:      Set True to allow direct port access to the application sidecar
+        :param authentication_mode:     API Gateway authentication mode
+        :param authentication_creds:    API Gateway authentication credentials as a tuple (username, password)
         :return: True if the function is ready (deployed)
         """
         if self.requires_build() or force_build:
@@ -261,6 +304,16 @@ class ApplicationRuntime(RemoteRuntime):
 
         self._ensure_reverse_proxy_configurations()
         self._configure_application_sidecar()
+
+        # we only allow accessing the application via the API Gateway
+        name_tag = tag or self.metadata.tag
+        self.status.api_gateway_name = (
+            f"{self.metadata.name}-{name_tag}" if name_tag else self.metadata.name
+        )
+        self.spec.add_templated_ingress_host_mode = (
+            NuclioIngressAddTemplatedIngressModes.never
+        )
+
         super().deploy(
             project,
             tag,
@@ -269,6 +322,14 @@ class ApplicationRuntime(RemoteRuntime):
             builder_env,
         )
 
+        ports = self.spec.internal_application_port if direct_port_access else []
+        self.create_api_gateway(
+            name=self.status.api_gateway_name,
+            ports=ports,
+            authentication_mode=authentication_mode,
+            authentication_creds=authentication_creds,
+        )
+
     def with_source_archive(
         self, source, workdir=None, pull_at_runtime=True, target_dir=None
     ):
@@ -290,6 +351,92 @@ class ApplicationRuntime(RemoteRuntime):
             target_dir=target_dir,
         )
 
+    @classmethod
+    def get_filename_and_handler(cls) -> (str, str):
+        reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
+        return str(reverse_proxy_file_path), "Handler"
+
+    def create_api_gateway(
+        self,
+        name: str = None,
+        path: str = None,
+        ports: list[int] = None,
+        authentication_mode: schemas.APIGatewayAuthenticationMode = None,
+        authentication_creds: tuple[str] = None,
+    ):
+        api_gateway = APIGateway(
+            APIGatewayMetadata(
+                name=name,
+                namespace=self.metadata.namespace,
+                labels=self.metadata.labels,
+                annotations=self.metadata.annotations,
+            ),
+            APIGatewaySpec(
+                functions=[self],
+                project=self.metadata.project,
+                path=path,
+                ports=mlrun.utils.helpers.as_list(ports) if ports else None,
+            ),
+        )
+
+        authentication_mode = (
+            authentication_mode
+            or mlrun.mlconf.function.application.default_authentication_mode
+        )
+        if authentication_mode == schemas.APIGatewayAuthenticationMode.access_key:
+            api_gateway.with_access_key_auth()
+        elif authentication_mode == schemas.APIGatewayAuthenticationMode.basic:
+            api_gateway.with_basic_auth(*authentication_creds)
+
+        db = self._get_db()
+        api_gateway_scheme = db.store_api_gateway(
+            api_gateway=api_gateway.to_scheme(), project=self.metadata.project
+        )
+        if not self.status.api_gateway_name:
+            self.status.api_gateway_name = api_gateway_scheme.metadata.name
+        self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
+        self.status.api_gateway.wait_for_readiness()
+        self.url = self.status.api_gateway.invoke_url
+
+    def invoke(
+        self,
+        path: str,
+        body: typing.Union[str, bytes, dict] = None,
+        method: str = None,
+        headers: dict = None,
+        dashboard: str = "",
+        force_external_address: bool = False,
+        auth_info: schemas.AuthInfo = None,
+        mock: bool = None,
+        **http_client_kwargs,
+    ):
+        self._sync_api_gateway()
+        # If the API Gateway is not ready or not set, try to invoke the function directly (without the API Gateway)
+        if not self.status.api_gateway:
+            super().invoke(
+                path,
+                body,
+                method,
+                headers,
+                dashboard,
+                force_external_address,
+                auth_info,
+                mock,
+                **http_client_kwargs,
+            )
+
+        credentials = (auth_info.username, auth_info.password) if auth_info else None
+
+        if not method:
+            method = "POST" if body else "GET"
+        return self.status.api_gateway.invoke(
+            method=method,
+            headers=headers,
+            credentials=credentials,
+            path=path,
+            **http_client_kwargs,
+        )
+
     def _build_application_image(
         self,
         builder_env: dict = None,
@@ -301,6 +448,14 @@ class ApplicationRuntime(RemoteRuntime):
         mlrun_version_specifier=None,
         show_on_failure: bool = False,
     ):
+        if not self.spec.command:
+            logger.warning(
+                "Building the application image without a command. "
+                "Use spec.command and spec.args to specify the application entrypoint",
+                command=self.spec.command,
+                args=self.spec.args,
+            )
+
         with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
         return self._build_image(
             builder_env=builder_env,
@@ -355,7 +510,14 @@ class ApplicationRuntime(RemoteRuntime):
         self.set_env("SIDECAR_PORT", self.spec.internal_application_port)
         self.set_env("SIDECAR_HOST", "http://localhost")
 
-    @classmethod
-    def get_filename_and_handler(cls) -> (str, str):
-        reverse_proxy_file_path = pathlib.Path(__file__).parent / "reverse_proxy.go"
-        return str(reverse_proxy_file_path), "Handler"
+    def _sync_api_gateway(self):
+        if not self.status.api_gateway_name:
+            return
+
+        db = self._get_db()
+        api_gateway_scheme = db.get_api_gateway(
+            name=self.status.api_gateway_name, project=self.metadata.project
+        )
+        self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
+        self.status.api_gateway.wait_for_readiness()
+        self.url = self.status.api_gateway.invoke_url

mlrun/runtimes/nuclio/function.py

@@ -19,12 +19,15 @@ import warnings
 from datetime import datetime
 from time import sleep
 
+import inflection
 import nuclio
 import nuclio.utils
 import requests
-import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
+from mlrun_pipelines.common.mounts import VolumeMount
+from mlrun_pipelines.common.ops import deploy_op
+from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 from nuclio.deploy import find_dashboard_url, get_deploy_status
 from nuclio.triggers import V3IOStreamTrigger
 
@@ -36,15 +39,11 @@ import mlrun.utils.helpers
 from mlrun.common.schemas import AuthInfo
 from mlrun.config import config as mlconf
 from mlrun.errors import err_to_str
-from mlrun.kfpops import deploy_op
 from mlrun.lists import RunList
 from mlrun.model import RunObject
 from mlrun.platforms.iguazio import (
-    VolumeMount,
-    mount_v3io,
     parse_path,
     split_path,
-    v3io_cred,
 )
 from mlrun.runtimes.base import FunctionStatus, RunError
 from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
@@ -56,33 +55,9 @@ def validate_nuclio_version_compatibility(*min_versions):
     """
     :param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
     """
-    parsed_min_versions = [
-        semver.VersionInfo.parse(min_version) for min_version in min_versions
-    ]
-    try:
-        parsed_current_version = semver.VersionInfo.parse(mlconf.nuclio_version)
-    except ValueError:
-        # only log when version is set but invalid
-        if mlconf.nuclio_version:
-            logger.warning(
-                "Unable to parse nuclio version, assuming compatibility",
-                nuclio_version=mlconf.nuclio_version,
-                min_versions=min_versions,
-            )
-        return True
-
-    parsed_min_versions.sort(reverse=True)
-    for parsed_min_version in parsed_min_versions:
-        if (
-            parsed_current_version.major == parsed_min_version.major
-            and parsed_current_version.minor == parsed_min_version.minor
-            and parsed_current_version.patch < parsed_min_version.patch
-        ):
-            return False
-
-        if parsed_current_version >= parsed_min_version:
-            return True
-    return False
+    return mlrun.utils.helpers.validate_component_version_compatibility(
+        "nuclio", *min_versions
+    )
 
 
 def min_nuclio_versions(*versions):
@@ -91,9 +66,13 @@ def min_nuclio_versions(*versions):
             if validate_nuclio_version_compatibility(*versions):
                 return function(*args, **kwargs)
 
+            if function.__name__ == "__init__":
+                name = inflection.titleize(function.__qualname__.split(".")[0])
+            else:
+                name = function.__qualname__
+
             message = (
-                f"{function.__name__} is supported since nuclio {' or '.join(versions)}, currently using "
-                f"nuclio {mlconf.nuclio_version}, please upgrade."
+                f"'{name}' function requires Nuclio v{' or v'.join(versions)} or higher"
             )
             raise mlrun.errors.MLRunIncompatibleVersionError(message)
 
@@ -292,7 +271,8 @@ class RemoteRuntime(KubeResource):
         self._status = self._verify_dict(status, "status", NuclioStatus)
 
     def pre_deploy_validation(self):
-        pass
+        if self.metadata.tag:
+            mlrun.utils.validate_tag_name(self.metadata.tag, "function.metadata.tag")
 
     def set_config(self, key, value):
         self.spec.config[key] = value
@@ -573,7 +553,6 @@ class RemoteRuntime(KubeResource):
         if tag:
             self.metadata.tag = tag
 
-        save_record = False
         # Attempt auto-mounting, before sending to remote build
         self.try_auto_mount_based_on_config()
         self._fill_credentials()
@@ -591,15 +570,18 @@ class RemoteRuntime(KubeResource):
         #       now, functions can be not exposed (using service type ClusterIP) and hence
         #       for BC we first try to populate the external invocation url, and then
         #       if not exists, take the internal invocation url
-        if self.status.external_invocation_urls:
+        if (
+            self.status.external_invocation_urls
+            and self.status.external_invocation_urls[0] != ""
+        ):
             self.spec.command = f"http://{self.status.external_invocation_urls[0]}"
-            save_record = True
-        elif self.status.internal_invocation_urls:
+        elif (
+            self.status.internal_invocation_urls
+            and self.status.internal_invocation_urls[0] != ""
+        ):
             self.spec.command = f"http://{self.status.internal_invocation_urls[0]}"
-            save_record = True
-        elif self.status.address:
+        elif self.status.address and self.status.address != "":
             self.spec.command = f"http://{self.status.address}"
-            save_record = True
 
         logger.info(
             "Successfully deployed function",
@@ -607,8 +589,7 @@ class RemoteRuntime(KubeResource):
             external_invocation_urls=self.status.external_invocation_urls,
         )
 
-        if save_record:
-            self.save(versioned=False)
+        self.save(versioned=False)
 
         return self.spec.command
 
@@ -778,7 +759,7 @@ class RemoteRuntime(KubeResource):
             runtime_env["MLRUN_NAMESPACE"] = mlconf.namespace
         if self.metadata.credentials.access_key:
             runtime_env[
-                mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session
+                mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session
             ] = self.metadata.credentials.access_key
         return runtime_env
 
@@ -992,21 +973,30 @@ class RemoteRuntime(KubeResource):
             sidecar["image"] = image
 
         ports = mlrun.utils.helpers.as_list(ports)
+        # according to RFC-6335, port name should be less than 15 characters,
+        # so we truncate it if needed and leave room for the index
+        port_name = name[:13].rstrip("-_") if len(name) > 13 else name
         sidecar["ports"] = [
             {
-                "name": "http",
+                "name": f"{port_name}-{i}",
                 "containerPort": port,
                 "protocol": "TCP",
             }
-            for port in ports
+            for i, port in enumerate(ports)
         ]
 
-        if command:
+        # if it is a redeploy, 'command' might be set with the previous invocation url.
+        # in this case, we don't want to use it as the sidecar command
+        if command and not command.startswith("http"):
             sidecar["command"] = mlrun.utils.helpers.as_list(command)
 
-        if args:
+        if args and sidecar["command"]:
             sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
+        # populate the sidecar resources from the function spec
+        if self.spec.resources:
+            sidecar["resources"] = self.spec.resources
+
     def _set_sidecar(self, name: str) -> dict:
         self.spec.config.setdefault("spec.sidecars", [])
         sidecars = self.spec.config["spec.sidecars"]

mlrun/runtimes/pod.py

@@ -20,8 +20,9 @@ import typing
 from enum import Enum
 
 import dotenv
-import kfp.dsl
 import kubernetes.client as k8s_client
+import mlrun_pipelines.mounts
+from mlrun_pipelines.mixins import KfpAdapterMixin
 
 import mlrun.errors
 import mlrun.utils.regex
@@ -41,7 +42,6 @@ from ..k8s_utils import (
 from ..utils import logger, update_in
 from .base import BaseRuntime, FunctionSpec, spec_fields
 from .utils import (
-    apply_kfp,
     get_gpu_from_resource_requirement,
     get_item_name,
     set_named_item,
@@ -935,12 +935,12 @@ class AutoMountType(str, Enum):
     @classmethod
     def all_mount_modifiers(cls):
         return [
-            mlrun.v3io_cred.__name__,
-            mlrun.mount_v3io.__name__,
-            mlrun.platforms.other.mount_pvc.__name__,
-            mlrun.auto_mount.__name__,
-            mlrun.platforms.mount_s3.__name__,
-            mlrun.platforms.set_env_variables.__name__,
+            mlrun_pipelines.mounts.v3io_cred.__name__,
+            mlrun_pipelines.mounts.mount_v3io.__name__,
+            mlrun_pipelines.mounts.mount_pvc.__name__,
+            mlrun_pipelines.mounts.auto_mount.__name__,
+            mlrun_pipelines.mounts.mount_s3.__name__,
+            mlrun_pipelines.mounts.set_env_variables.__name__,
         ]
 
     @classmethod
@@ -957,27 +957,27 @@ class AutoMountType(str, Enum):
     def _get_auto_modifier():
         # If we're running on Iguazio - use v3io_cred
         if mlconf.igz_version != "":
-            return mlrun.v3io_cred
+            return mlrun_pipelines.mounts.v3io_cred
         # Else, either pvc mount if it's configured or do nothing otherwise
         pvc_configured = (
             "MLRUN_PVC_MOUNT" in os.environ
             or "pvc_name" in mlconf.get_storage_auto_mount_params()
         )
-        return mlrun.platforms.other.mount_pvc if pvc_configured else None
+        return mlrun_pipelines.mounts.mount_pvc if pvc_configured else None
 
     def get_modifier(self):
         return {
             AutoMountType.none: None,
-            AutoMountType.v3io_credentials: mlrun.v3io_cred,
-            AutoMountType.v3io_fuse: mlrun.mount_v3io,
-            AutoMountType.pvc: mlrun.platforms.other.mount_pvc,
+            AutoMountType.v3io_credentials: mlrun_pipelines.mounts.v3io_cred,
+            AutoMountType.v3io_fuse: mlrun_pipelines.mounts.mount_v3io,
+            AutoMountType.pvc: mlrun_pipelines.mounts.mount_pvc,
             AutoMountType.auto: self._get_auto_modifier(),
-            AutoMountType.s3: mlrun.platforms.mount_s3,
-            AutoMountType.env: mlrun.platforms.set_env_variables,
+            AutoMountType.s3: mlrun_pipelines.mounts.mount_s3,
+            AutoMountType.env: mlrun_pipelines.mounts.set_env_variables,
         }[self]
 
 
-class KubeResource(BaseRuntime):
+class KubeResource(BaseRuntime, KfpAdapterMixin):
     """
     A parent class for runtimes that generate k8s resources when executing.
     """
@@ -997,26 +997,6 @@ class KubeResource(BaseRuntime):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", KubeResourceSpec)
 
-    def apply(self, modify):
-        """
-        Apply a modifier to the runtime which is used to change the runtimes k8s object's spec.
-        Modifiers can be either KFP modifiers or MLRun modifiers (which are compatible with KFP). All modifiers accept
-        a `kfp.dsl.ContainerOp` object, apply some changes on its spec and return it so modifiers can be chained
-        one after the other.
-
-        :param modify: a modifier runnable object
-        :return: the runtime (self) after the modifications
-        """
-
-        # Kubeflow pipeline have a hook to add the component to the DAG on ContainerOp init
-        # we remove the hook to suppress kubeflow op registration and return it after the apply()
-        old_op_handler = kfp.dsl._container_op._register_op_handler
-        kfp.dsl._container_op._register_op_handler = lambda x: self.metadata.name
-        cop = kfp.dsl.ContainerOp("name", "image")
-        kfp.dsl._container_op._register_op_handler = old_op_handler
-
-        return apply_kfp(modify, cop, self)
-
     def set_env_from_secret(self, name, secret=None, secret_key=None):
         """set pod environment var from secret"""
         secret_key = secret_key or name
@@ -1086,12 +1066,12 @@ class KubeResource(BaseRuntime):
 
     def _set_env(self, name, value=None, value_from=None):
         new_var = k8s_client.V1EnvVar(name=name, value=value, value_from=value_from)
-        i = 0
-        for v in self.spec.env:
-            if get_item_name(v) == name:
-                self.spec.env[i] = new_var
+
+        # ensure we don't have duplicate env vars with the same name
+        for env_index, value_item in enumerate(self.spec.env):
+            if get_item_name(value_item) == name:
+                self.spec.env[env_index] = new_var
                 return self
-            i += 1
         self.spec.env.append(new_var)
         return self
 

mlrun/runtimes/remotesparkjob.py

@@ -15,11 +15,11 @@ import re
 from subprocess import run
 
 import kubernetes.client
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.errors
 from mlrun.config import config
 
-from ..platforms.iguazio import mount_v3io, mount_v3iod
 from .kubejob import KubejobRuntime
 from .pod import KubeResourceSpec
 
@@ -130,14 +130,20 @@ class RemoteSparkRuntime(KubejobRuntime):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", RemoteSparkSpec)
 
-    def with_spark_service(self, spark_service, provider=RemoteSparkProviders.iguazio):
+    def with_spark_service(
+        self,
+        spark_service,
+        provider=RemoteSparkProviders.iguazio,
+        with_v3io_mount=True,
+    ):
         """Attach spark service to function"""
         self.spec.provider = provider
         if provider == RemoteSparkProviders.iguazio:
             self.spec.env.append(
                 {"name": "MLRUN_SPARK_CLIENT_IGZ_SPARK", "value": "true"}
             )
-            self.apply(mount_v3io())
+            if with_v3io_mount:
+                self.apply(mount_v3io())
             self.apply(
                 mount_v3iod(
                     namespace=config.namespace,

mlrun/runtimes/sparkjob/spark3job.py

@@ -14,6 +14,7 @@
 import typing
 
 import kubernetes.client
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.common.schemas.function
 import mlrun.errors
@@ -22,7 +23,6 @@ from mlrun.config import config
 
 from ...execution import MLClientCtx
 from ...model import RunObject
-from ...platforms.iguazio import mount_v3io, mount_v3iod
 from ...utils import update_in, verify_field_regex
 from ..kubejob import KubejobRuntime
 from ..pod import KubeResourceSpec