mlrun 1.10.0rc18__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/runtimes/constants.py

@@ -0,0 +1,225 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+import mlrun.runtimes.nuclio as nuclio_runtime
+import mlrun.runtimes.nuclio.application as nuclio_application
+import mlrun.runtimes.nuclio.serving as nuclio_serving
+
+
+class RuntimeKinds:
+    remote = "remote"
+    nuclio = "nuclio"
+    dask = "dask"
+    job = "job"
+    spark = "spark"
+    remotespark = "remote-spark"
+    mpijob = "mpijob"
+    serving = "serving"
+    local = "local"
+    handler = "handler"
+    databricks = "databricks"
+    application = "application"
+
+    @staticmethod
+    def all():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+            RuntimeKinds.dask,
+            RuntimeKinds.job,
+            RuntimeKinds.spark,
+            RuntimeKinds.remotespark,
+            RuntimeKinds.mpijob,
+            RuntimeKinds.local,
+            RuntimeKinds.databricks,
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def runtime_with_handlers():
+        return [
+            RuntimeKinds.dask,
+            RuntimeKinds.job,
+            RuntimeKinds.spark,
+            RuntimeKinds.remotespark,
+            RuntimeKinds.mpijob,
+            RuntimeKinds.databricks,
+        ]
+
+    @staticmethod
+    def abortable_runtimes():
+        return [
+            RuntimeKinds.job,
+            RuntimeKinds.spark,
+            RuntimeKinds.remotespark,
+            RuntimeKinds.mpijob,
+            RuntimeKinds.databricks,
+            RuntimeKinds.local,
+            RuntimeKinds.handler,
+            "",
+        ]
+
+    @staticmethod
+    def retriable_runtimes():
+        return [
+            RuntimeKinds.job,
+        ]
+
+    @staticmethod
+    def nuclio_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def pure_nuclio_deployed_runtimes():
+        return [
+            RuntimeKinds.remote,
+            RuntimeKinds.nuclio,
+            RuntimeKinds.serving,
+        ]
+
+    @staticmethod
+    def handlerless_runtimes():
+        return [
+            RuntimeKinds.serving,
+            # Application runtime handler is internal reverse proxy
+            RuntimeKinds.application,
+        ]
+
+    @staticmethod
+    def local_runtimes():
+        return [
+            RuntimeKinds.local,
+            RuntimeKinds.handler,
+        ]
+
+    @staticmethod
+    def is_log_collectable_runtime(kind: typing.Optional[str]):
+        """
+        whether log collector can collect logs for that runtime
+        :param kind: kind name
+        :return: whether log collector can collect logs for that runtime
+        """
+        # if local run, the log collector doesn't support it as it is only supports k8s resources
+        # when runtime is local the client is responsible for logging the stdout of the run by using `log_std`
+        if RuntimeKinds.is_local_runtime(kind):
+            return False
+
+        if (
+            kind
+            not in [
+                # dask implementation is different from other runtimes, because few runs can be run against the same
+                # runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
+                # logs for dask is by using `log_std` on client side after we execute the code against the cluster,
+                # as submitting the run with the dask client will return the run stdout.
+                # For more information head to `DaskCluster._run`.
+                RuntimeKinds.dask
+            ]
+            + RuntimeKinds.nuclio_runtimes()
+        ):
+            return True
+
+        return False
+
+    @staticmethod
+    def is_local_runtime(kind):
+        # "" or None counted as local
+        if not kind or kind in RuntimeKinds.local_runtimes():
+            return True
+        return False
+
+    @staticmethod
+    def requires_k8s_name_validation(kind: str) -> bool:
+        """
+        Returns True if the runtime kind creates Kubernetes resources that use the function name.
+
+        Function names for k8s-deployed runtimes must conform to DNS-1123 label requirements:
+        - Lowercase alphanumeric characters or '-'
+        - Start and end with an alphanumeric character
+        - Maximum 63 characters
+
+        Local runtimes (local, handler) run on the local machine and don't create k8s resources,
+        so they don't require k8s naming validation.
+
+        :param kind: Runtime kind string (job, spark, serving, local, etc.)
+        :return: True if function name needs k8s DNS-1123 validation, False otherwise
+        """
+        return not RuntimeKinds.is_local_runtime(kind)
+
+    @staticmethod
+    def requires_absolute_artifacts_path(kind):
+        """
+        Returns True if the runtime kind requires absolute artifacts' path (i.e. is local), False otherwise.
+        """
+        if RuntimeKinds.is_local_runtime(kind):
+            return False
+
+        if kind not in [
+            # logging artifacts is done externally to the dask cluster by a client that can either run locally (in which
+            # case the path can be relative) or remotely (in which case the path must be absolute and will be passed
+            # to another run)
+            RuntimeKinds.dask
+        ]:
+            return True
+        return False
+
+    @staticmethod
+    def requires_image_name_for_execution(kind):
+        if RuntimeKinds.is_local_runtime(kind):
+            return False
+
+        # both spark and remote spark uses different mechanism for assigning images
+        return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
+
+    @staticmethod
+    def supports_from_notebook(kind):
+        return kind not in [RuntimeKinds.application]
+
+    @staticmethod
+    def resolve_nuclio_runtime(kind: str, sub_kind: str):
+        kind = kind.split(":")[0]
+        if kind not in RuntimeKinds.nuclio_runtimes():
+            raise ValueError(
+                f"Kind {kind} is not a nuclio runtime, "
+                f"available runtimes are {RuntimeKinds.nuclio_runtimes()}"
+            )
+
+        # These names are imported at module level below; referenced at call-time (no imports here).
+        if sub_kind == nuclio_serving.serving_subkind:
+            return nuclio_runtime.ServingRuntime()
+
+        if kind == RuntimeKinds.application:
+            return nuclio_application.ApplicationRuntime()
+
+        runtime = nuclio_runtime.RemoteRuntime()
+        runtime.spec.function_kind = sub_kind
+        return runtime
+
+    @staticmethod
+    def resolve_nuclio_sub_kind(kind: str):
+        is_nuclio = kind.startswith("nuclio")
+        sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
+        if kind == RuntimeKinds.serving:
+            is_nuclio = True
+            sub_kind = nuclio_serving.serving_subkind
+        elif kind == RuntimeKinds.application:
+            is_nuclio = True
+        return is_nuclio, sub_kind

mlrun/runtimes/daskjob.py

@@ -15,8 +15,9 @@ import datetime
 import inspect
 import socket
 import time
+from collections.abc import Callable
 from os import environ
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import mlrun.common.schemas
 import mlrun.errors
@@ -541,6 +542,7 @@ class DaskCluster(KubejobRuntime):
             notifications=notifications,
             returns=returns,
             state_thresholds=state_thresholds,
+            retry=retry,
             **launcher_kwargs,
         )
 
@@ -550,7 +552,8 @@ class DaskCluster(KubejobRuntime):
 
         # TODO: investigate if the following instructions could overwrite the environment on any MLRun API Pod
         # Such action could result on race conditions against other runtimes and MLRun itself
-        extra_env = self._generate_runtime_env(runobj)
+        extra_env, _ = self._generate_runtime_env(runobj)
+        # Since it runs locally, we don't need the external sources env vars
         environ.update(extra_env)
 
         context = MLClientCtx.from_dict(

mlrun/runtimes/databricks_job/databricks_runtime.py

@@ -14,7 +14,8 @@
 import typing
 from ast import FunctionDef, parse, unparse
 from base64 import b64decode
-from typing import Callable, Optional, Union
+from collections.abc import Callable
+from typing import Optional, Union
 
 import mlrun
 import mlrun.runtimes.kubejob as kubejob

mlrun/runtimes/local.py

@@ -29,12 +29,12 @@ from os import environ, remove
 from pathlib import Path
 from subprocess import PIPE, Popen
 from sys import executable
+from typing import Optional
 
 from nuclio import Event
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
-import mlrun.common.runtimes.constants
 from mlrun.lists import RunList
 
 from ..errors import err_to_str
@@ -201,9 +201,12 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
     kind = "local"
     _is_remote = False
 
-    def to_job(self, image=""):
+    def to_job(self, image="", func_name: Optional[str] = None):
         struct = self.to_dict()
         obj = KubejobRuntime.from_dict(struct)
+        obj.kind = "job"  # Ensure kind is set to 'job' for KubejobRuntime
+        if func_name:
+            obj.metadata.name = func_name
         if image:
             obj.spec.image = image
         return obj

mlrun/runtimes/mounts.py

@@ -14,8 +14,11 @@
 
 import os
 import typing
+import warnings
 from collections import namedtuple
 
+import mlrun.common.secrets
+import mlrun.errors
 from mlrun.config import config
 from mlrun.config import config as mlconf
 from mlrun.errors import MLRunInvalidArgumentError
@@ -247,10 +250,22 @@ def mount_s3(
     def _use_s3_cred(runtime: "KubeResource"):
         _access_key = aws_access_key or os.environ.get(prefix + "AWS_ACCESS_KEY_ID")
         _secret_key = aws_secret_key or os.environ.get(prefix + "AWS_SECRET_ACCESS_KEY")
-        _endpoint_url = endpoint_url or os.environ.get(prefix + "S3_ENDPOINT_URL")
+
+        # Check for endpoint URL with backward compatibility
+        _endpoint_url = endpoint_url or os.environ.get(prefix + "AWS_ENDPOINT_URL_S3")
+        if not _endpoint_url:
+            # Check for deprecated environment variable
+            _endpoint_url = os.environ.get(prefix + "S3_ENDPOINT_URL")
+            if _endpoint_url:
+                warnings.warn(
+                    "S3_ENDPOINT_URL is deprecated in 1.10.0 and will be removed in 1.12.0, "
+                    "use AWS_ENDPOINT_URL_S3 instead.",
+                    # TODO: Remove this in 1.12.0
+                    FutureWarning,
+                )
 
         if _endpoint_url:
-            runtime.set_env(prefix + "S3_ENDPOINT_URL", _endpoint_url)
+            runtime.set_env(prefix + "AWS_ENDPOINT_URL_S3", _endpoint_url)
         if aws_region:
             runtime.set_env(prefix + "AWS_REGION", aws_region)
         if non_anonymous:
@@ -399,6 +414,9 @@ def mount_secret(
                          the specified paths, and unlisted keys will not be
                          present."""
 
+    if secret_name:
+        mlrun.common.secrets.validate_not_forbidden_secret(secret_name.strip())
+
     def _mount_secret(runtime: "KubeResource"):
         # Define the secret volume source
         secret_volume_source = {

mlrun/runtimes/nuclio/__init__.py

@@ -12,10 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .serving import ServingRuntime, new_v2_model_server  # noqa
-from .nuclio import nuclio_init_hook  # noqa
-from .function import (
-    min_nuclio_versions,
-    RemoteRuntime,
-)  # noqa
-from .api_gateway import APIGateway
+import mlrun.runtimes.nuclio.serving as nuclio_serving  # noqa
+import mlrun.runtimes.nuclio.nuclio as nuclio_nuclio  # noqa
+import mlrun.runtimes.nuclio.function as nuclio_function  # noqa
+import mlrun.runtimes.nuclio.api_gateway as nuclio_api_gateway  # noqa
+
+ServingRuntime = nuclio_serving.ServingRuntime
+new_v2_model_server = nuclio_serving.new_v2_model_server
+nuclio_init_hook = nuclio_nuclio.nuclio_init_hook
+min_nuclio_versions = nuclio_function.min_nuclio_versions
+multiple_port_sidecar_is_supported = nuclio_function.multiple_port_sidecar_is_supported
+RemoteRuntime = nuclio_function.RemoteRuntime
+APIGateway = nuclio_api_gateway.APIGateway

mlrun/runtimes/nuclio/api_gateway.py

@@ -17,13 +17,14 @@ from typing import Optional, Union
 from urllib.parse import urljoin
 
 import requests
-from nuclio.auth import AuthInfo as NuclioAuthInfo
 from nuclio.auth import AuthKinds as NuclioAuthKinds
 
 import mlrun
+import mlrun.auth.nuclio
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.helpers
 import mlrun.common.schemas as schemas
+import mlrun.common.schemas.auth
 import mlrun.common.types
 from mlrun.model import ModelObj
 from mlrun.platforms.iguazio import min_iguazio_versions
@@ -55,6 +56,11 @@ class Authenticator(typing.Protocol):
             == schemas.APIGatewayAuthenticationMode.access_key.value
         ):
             return AccessKeyAuth()
+        elif (
+            api_gateway_spec.authenticationMode
+            == schemas.APIGatewayAuthenticationMode.iguazio.value
+        ):
+            return IguazioAuth()
         else:
             return NoneAuth()
 
@@ -112,6 +118,16 @@ class AccessKeyAuth(APIGatewayAuthenticator):
         return schemas.APIGatewayAuthenticationMode.access_key.value
 
 
+class IguazioAuth(APIGatewayAuthenticator):
+    """
+    An API gateway authenticator with Iguazio authentication.
+    """
+
+    @property
+    def authentication_mode(self) -> str:
+        return schemas.APIGatewayAuthenticationMode.iguazio.value
+
+
 class APIGatewayMetadata(ModelObj):
     _dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
 
@@ -430,7 +446,7 @@ class APIGateway(ModelObj):
                 raise mlrun.errors.MLRunInvalidArgumentError(
                     "API Gateway invocation requires authentication. Please pass credentials"
                 )
-            auth = NuclioAuthInfo(
+            auth = mlrun.auth.nuclio.NuclioAuthInfo(
                 username=credentials[0], password=credentials[1]
             ).to_requests_auth()
 
@@ -440,23 +456,30 @@ class APIGateway(ModelObj):
         ):
             # inject access key from env
             if credentials:
-                auth = NuclioAuthInfo(
+                auth = mlrun.auth.nuclio.NuclioAuthInfo(
                     username=credentials[0],
                     password=credentials[1],
                     mode=NuclioAuthKinds.iguazio,
                 ).to_requests_auth()
             else:
-                auth = NuclioAuthInfo().from_envvar().to_requests_auth()
+                auth = (
+                    mlrun.auth.nuclio.NuclioAuthInfo().from_envvar().to_requests_auth()
+                )
             if not auth:
                 raise mlrun.errors.MLRunInvalidArgumentError(
                     "API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
                 )
+        if (
+            self.spec.authentication.authentication_mode
+            == schemas.APIGatewayAuthenticationMode.iguazio.value
+        ):
+            auth = mlrun.auth.nuclio.NuclioAuthInfo.from_envvar().to_requests_auth()
         url = urljoin(self.invoke_url, path or "")
 
         # Determine the correct keyword argument for the body
         if isinstance(body, dict):
             kwargs["json"] = body
-        elif isinstance(body, (str, bytes)):
+        elif isinstance(body, str | bytes):
             kwargs["data"] = body
 
         return requests.request(
@@ -527,6 +550,13 @@ class APIGateway(ModelObj):
         """
         self.spec.authentication = AccessKeyAuth()
 
+    @min_nuclio_versions("1.15.10")
+    def with_iguazio_auth(self):
+        """
+        Set iguazio authentication for the API gateway.
+        """
+        self.spec.authentication = IguazioAuth()
+
     def with_canary(
         self,
         functions: Union[
@@ -692,7 +722,7 @@ class APIGateway(ModelObj):
     @staticmethod
     def _generate_basic_auth(username: str, password: str):
         token = base64.b64encode(f"{username}:{password}".encode()).decode()
-        return f"Basic {token}"
+        return f"{mlrun.common.schemas.AuthorizationHeaderPrefixes.basic}{token}"
 
     @staticmethod
     def _resolve_canary(