mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/runtimes/nuclio/application/reverse_proxy.go

@@ -0,0 +1,95 @@
+// Copyright 2024 Iguazio
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+    "bytes"
+    "fmt"
+    "net/http"
+    "net/http/httptest"
+    "net/http/httputil"
+    "net/url"
+    "os"
+    "strings"
+
+    nuclio "github.com/nuclio/nuclio-sdk-go"
+)
+
+func Handler(context *nuclio.Context, event nuclio.Event) (interface{}, error) {
+    reverseProxy := context.UserData.(map[string]interface{})["reverseProxy"].(*httputil.ReverseProxy)
+    sidecarUrl := context.UserData.(map[string]interface{})["server"].(string)
+
+    // populate reverse proxy http request
+    httpRequest, err := http.NewRequest(event.GetMethod(), event.GetPath(), bytes.NewReader(event.GetBody()))
+    if err != nil {
+        context.Logger.ErrorWith("Failed to create a reverse proxy request")
+        return nil, err
+    }
+    for k, v := range event.GetHeaders() {
+        httpRequest.Header[k] = []string{v.(string)}
+    }
+
+    // populate query params
+    query := httpRequest.URL.Query()
+    for k, v := range event.GetFields() {
+        query.Set(k, v.(string))
+    }
+    httpRequest.URL.RawQuery = query.Encode()
+
+    recorder := httptest.NewRecorder()
+    reverseProxy.ServeHTTP(recorder, httpRequest)
+
+    // send request to sidecar
+    context.Logger.DebugWith("Forwarding request to sidecar", "sidecarUrl", sidecarUrl, "query", httpRequest.URL.Query())
+    response := recorder.Result()
+
+    headers := make(map[string]interface{})
+    for key, value := range response.Header {
+        headers[key] = value[0]
+    }
+
+    // let the processor calculate the content length
+    delete(headers, "Content-Length")
+    return nuclio.Response{
+        StatusCode:  response.StatusCode,
+        Body:        recorder.Body.Bytes(),
+        ContentType: response.Header.Get("Content-Type"),
+        Headers:     headers,
+    }, nil
+}
+
+func InitContext(context *nuclio.Context) error {
+    sidecarHost := os.Getenv("SIDECAR_HOST")
+    sidecarPort := os.Getenv("SIDECAR_PORT")
+    if sidecarHost == "" {
+        sidecarHost = "http://localhost"
+    } else if !strings.Contains(sidecarHost, "://") {
+        sidecarHost = fmt.Sprintf("http://%s", sidecarHost)
+    }
+
+    // url for request forwarding
+    sidecarUrl := fmt.Sprintf("%s:%s", sidecarHost, sidecarPort)
+    parsedURL, err := url.Parse(sidecarUrl)
+    if err != nil {
+        context.Logger.ErrorWith("Failed to parse sidecar url", "sidecarUrl", sidecarUrl)
+        return err
+    }
+    reverseProxy := httputil.NewSingleHostReverseProxy(parsedURL)
+
+    context.UserData = map[string]interface{}{
+        "server":       sidecarUrl,
+        "reverseProxy": reverseProxy,
+    }
+    return nil
+}

mlrun/runtimes/{function.py → nuclio/function.py}

@@ -19,12 +19,16 @@ import warnings
 from datetime import datetime
 from time import sleep
 
+import inflection
 import nuclio
 import nuclio.utils
 import requests
 import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
+from mlrun_pipelines.common.mounts import VolumeMount
+from mlrun_pipelines.common.ops import deploy_op
+from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 from nuclio.deploy import find_dashboard_url, get_deploy_status
 from nuclio.triggers import V3IOStreamTrigger
 
@@ -34,56 +38,27 @@ import mlrun.k8s_utils
 import mlrun.utils
 import mlrun.utils.helpers
 from mlrun.common.schemas import AuthInfo
-
-from ..config import config as mlconf
-from ..errors import err_to_str
-from ..kfpops import deploy_op
-from ..lists import RunList
-from ..model import RunObject
-from ..platforms.iguazio import (
-    VolumeMount,
-    mount_v3io,
+from mlrun.config import config as mlconf
+from mlrun.errors import err_to_str
+from mlrun.lists import RunList
+from mlrun.model import RunObject
+from mlrun.platforms.iguazio import (
     parse_path,
     split_path,
-    v3io_cred,
 )
-from ..utils import get_in, logger, update_in
-from .base import FunctionStatus, RunError
-from .pod import KubeResource, KubeResourceSpec
-from .utils import get_item_name, log_std
+from mlrun.runtimes.base import FunctionStatus, RunError
+from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
+from mlrun.runtimes.utils import get_item_name, log_std
+from mlrun.utils import get_in, logger, update_in
 
 
 def validate_nuclio_version_compatibility(*min_versions):
     """
     :param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
     """
-    parsed_min_versions = [
-        semver.VersionInfo.parse(min_version) for min_version in min_versions
-    ]
-    try:
-        parsed_current_version = semver.VersionInfo.parse(mlconf.nuclio_version)
-    except ValueError:
-        # only log when version is set but invalid
-        if mlconf.nuclio_version:
-            logger.warning(
-                "Unable to parse nuclio version, assuming compatibility",
-                nuclio_version=mlconf.nuclio_version,
-                min_versions=min_versions,
-            )
-        return True
-
-    parsed_min_versions.sort(reverse=True)
-    for parsed_min_version in parsed_min_versions:
-        if (
-            parsed_current_version.major == parsed_min_version.major
-            and parsed_current_version.minor == parsed_min_version.minor
-            and parsed_current_version.patch < parsed_min_version.patch
-        ):
-            return False
-
-        if parsed_current_version >= parsed_min_version:
-            return True
-    return False
+    return mlrun.utils.helpers.validate_component_version_compatibility(
+        "nuclio", *min_versions
+    )
 
 
 def min_nuclio_versions(*versions):
@@ -92,9 +67,13 @@ def min_nuclio_versions(*versions):
             if validate_nuclio_version_compatibility(*versions):
                 return function(*args, **kwargs)
 
+            if function.__name__ == "__init__":
+                name = inflection.titleize(function.__qualname__.split(".")[0])
+            else:
+                name = function.__qualname__
+
             message = (
-                f"{function.__name__} is supported since nuclio {' or '.join(versions)}, currently using "
-                f"nuclio {mlconf.nuclio_version}, please upgrade."
+                f"'{name}' function requires Nuclio v{' or v'.join(versions)} or higher"
             )
             raise mlrun.errors.MLRunIncompatibleVersionError(message)
 
@@ -292,6 +271,10 @@ class RemoteRuntime(KubeResource):
     def status(self, status):
         self._status = self._verify_dict(status, "status", NuclioStatus)
 
+    def pre_deploy_validation(self):
+        if self.metadata.tag:
+            mlrun.utils.validate_tag_name(self.metadata.tag, "function.metadata.tag")
+
     def set_config(self, key, value):
         self.spec.config[key] = value
         return self
@@ -314,10 +297,37 @@ class RemoteRuntime(KubeResource):
         """
         if hasattr(spec, "to_dict"):
             spec = spec.to_dict()
+
+        self._validate_triggers(spec)
+
         spec["name"] = name
         self.spec.config[f"spec.triggers.{name}"] = spec
         return self
 
+    def _validate_triggers(self, spec):
+        # ML-7763 / NUC-233
+        min_nuclio_version = "1.13.12"
+        if mlconf.nuclio_version and semver.VersionInfo.parse(
+            mlconf.nuclio_version
+        ) < semver.VersionInfo.parse(min_nuclio_version):
+            explicit_ack_enabled = False
+            num_triggers = 0
+            trigger_name = spec.get("name", "UNKNOWN")
+            for key, config in [(f"spec.triggers.{trigger_name}", spec)] + list(
+                self.spec.config.items()
+            ):
+                if key.startswith("spec.triggers."):
+                    num_triggers += 1
+                    explicit_ack_enabled = (
+                        config.get("explicitAckMode", "disable") != "disable"
+                    )
+
+            if num_triggers > 1 and explicit_ack_enabled:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Multiple triggers cannot be used in conjunction with explicit ack. "
+                    f"Please upgrade to nuclio {min_nuclio_version} or newer."
+                )
+
     def with_source_archive(
         self,
         source,
@@ -343,17 +353,21 @@ class RemoteRuntime(KubeResource):
 
             git::
 
-                fn.with_source_archive("git://github.com/org/repo#my-branch",
-                        handler="main:handler",
-                        workdir="path/inside/repo")
+                fn.with_source_archive(
+                    "git://github.com/org/repo#my-branch",
+                    handler="main:handler",
+                    workdir="path/inside/repo",
+                )
 
             s3::
 
                 fn.spec.nuclio_runtime = "golang"
-                fn.with_source_archive("s3://my-bucket/path/in/bucket/my-functions-archive",
+                fn.with_source_archive(
+                    "s3://my-bucket/path/in/bucket/my-functions-archive",
                     handler="my_func:Handler",
                     workdir="path/inside/functions/archive",
-                    runtime="golang")
+                    runtime="golang",
+                )
         """
         self.spec.build.source = source
         # update handler in function_handler
@@ -386,7 +400,7 @@ class RemoteRuntime(KubeResource):
         workers: typing.Optional[int] = 8,
         port: typing.Optional[int] = None,
         host: typing.Optional[str] = None,
-        paths: typing.Optional[typing.List[str]] = None,
+        paths: typing.Optional[list[str]] = None,
         canary: typing.Optional[float] = None,
         secret: typing.Optional[str] = None,
         worker_timeout: typing.Optional[int] = None,
@@ -432,14 +446,8 @@ class RemoteRuntime(KubeResource):
                 raise ValueError(
                     "gateway timeout must be greater than the worker timeout"
                 )
-            annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = (
-                f"{gateway_timeout}"
-            )
-            annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = (
-                f"{gateway_timeout}"
-            )
-            annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = (
-                f"{gateway_timeout}"
+            mlrun.runtimes.utils.enrich_gateway_timeout_annotations(
+                annotations, gateway_timeout
             )
 
         trigger = nuclio.HttpTrigger(
@@ -460,6 +468,11 @@ class RemoteRuntime(KubeResource):
         return self
 
     def from_image(self, image):
+        """
+        Deploy the function with an existing nuclio processor image.
+
+        :param image: image name
+        """
         config = nuclio.config.new_config()
         update_in(
             config,
@@ -510,6 +523,11 @@ class RemoteRuntime(KubeResource):
         extra_attributes = extra_attributes or {}
         if ack_window_size:
             extra_attributes["ackWindowSize"] = ack_window_size
+
+        access_key = kwargs.pop("access_key", None)
+        if not access_key:
+            access_key = self._resolve_v3io_access_key()
+
         self.add_trigger(
             name,
             V3IOStreamTrigger(
@@ -521,11 +539,14 @@ class RemoteRuntime(KubeResource):
                 webapi=endpoint or "http://v3io-webapi:8081",
                 extra_attributes=extra_attributes,
                 read_batch_size=256,
+                access_key=access_key,
                 **kwargs,
             ),
         )
-        self.spec.min_replicas = shards
-        self.spec.max_replicas = shards
+        if self.spec.min_replicas != shards or self.spec.max_replicas != shards:
+            logger.warning(f"Setting function replicas to {shards}")
+            self.spec.min_replicas = shards
+            self.spec.max_replicas = shards
 
     def deploy(
         self,
@@ -541,11 +562,16 @@ class RemoteRuntime(KubeResource):
         :param project:    project name
         :param tag:        function tag
         :param verbose:    set True for verbose logging
-        :param auth_info:  service AuthInfo
+        :param auth_info:  service AuthInfo (deprecated and ignored)
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
         :param force_build: set True for force building the image
         """
-        # todo: verify that the function name is normalized
+        if auth_info:
+            # TODO: remove in 1.9.0
+            warnings.warn(
+                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.9.0",
+                FutureWarning,
+            )
 
         old_http_session = getattr(self, "_http_session", None)
         if old_http_session:
@@ -562,15 +588,12 @@ class RemoteRuntime(KubeResource):
         if tag:
             self.metadata.tag = tag
 
-        save_record = False
         # Attempt auto-mounting, before sending to remote build
         self.try_auto_mount_based_on_config()
         self._fill_credentials()
         db = self._get_db()
         logger.info("Starting remote function deploy")
-        data = db.remote_builder(
-            self, False, builder_env=builder_env, force_build=force_build
-        )
+        data = db.deploy_nuclio_function(func=self, builder_env=builder_env)
         self.status = data["data"].get("status")
         self._update_credentials_from_remote_build(data["data"])
 
@@ -578,19 +601,25 @@ class RemoteRuntime(KubeResource):
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
 
+        return self._enrich_command_from_status()
+
+    def _enrich_command_from_status(self):
         # NOTE: on older mlrun versions & nuclio versions, function are exposed via NodePort
         #       now, functions can be not exposed (using service type ClusterIP) and hence
         #       for BC we first try to populate the external invocation url, and then
         #       if not exists, take the internal invocation url
-        if self.status.external_invocation_urls:
+        if (
+            self.status.external_invocation_urls
+            and self.status.external_invocation_urls[0] != ""
+        ):
             self.spec.command = f"http://{self.status.external_invocation_urls[0]}"
-            save_record = True
-        elif self.status.internal_invocation_urls:
+        elif (
+            self.status.internal_invocation_urls
+            and self.status.internal_invocation_urls[0] != ""
+        ):
             self.spec.command = f"http://{self.status.internal_invocation_urls[0]}"
-            save_record = True
-        elif self.status.address:
+        elif self.status.address and self.status.address != "":
             self.spec.command = f"http://{self.status.address}"
-            save_record = True
 
         logger.info(
             "Successfully deployed function",
@@ -598,13 +627,11 @@ class RemoteRuntime(KubeResource):
             external_invocation_urls=self.status.external_invocation_urls,
         )
 
-        if save_record:
-            self.save(versioned=False)
+        self.save(versioned=False)
 
         return self.spec.command
 
     def _wait_for_function_deployment(self, db, verbose=False):
-        text = ""
         state = ""
         last_log_timestamp = 1
         while state not in ["ready", "error", "unhealthy"]:
@@ -612,7 +639,7 @@ class RemoteRuntime(KubeResource):
                 int(mlrun.mlconf.httpdb.logs.nuclio.pull_deploy_status_default_interval)
             )
             try:
-                text, last_log_timestamp = db.get_builder_status(
+                text, last_log_timestamp = db.get_nuclio_deploy_status(
                     self, last_log_timestamp=last_log_timestamp, verbose=verbose
                 )
             except mlrun.db.RunDBError:
@@ -629,9 +656,9 @@ class RemoteRuntime(KubeResource):
     def with_node_selection(
         self,
         node_name: typing.Optional[str] = None,
-        node_selector: typing.Optional[typing.Dict[str, str]] = None,
+        node_selector: typing.Optional[dict[str, str]] = None,
         affinity: typing.Optional[client.V1Affinity] = None,
-        tolerations: typing.Optional[typing.List[client.V1Toleration]] = None,
+        tolerations: typing.Optional[list[client.V1Toleration]] = None,
     ):
         """k8s node selection attributes"""
         if tolerations and not validate_nuclio_version_compatibility("1.7.5"):
@@ -683,14 +710,14 @@ class RemoteRuntime(KubeResource):
 
     def set_state_thresholds(
         self,
-        state_thresholds: typing.Dict[str, int],
+        state_thresholds: dict[str, int],
         patch: bool = True,
     ):
         raise NotImplementedError(
             "State thresholds do not apply for nuclio as it has its own function pods healthiness monitoring"
         )
 
-    @min_nuclio_versions("1.12.8")
+    @min_nuclio_versions("1.13.1")
     def disable_default_http_trigger(
         self,
     ):
@@ -699,7 +726,7 @@ class RemoteRuntime(KubeResource):
         """
         self.spec.disable_default_http_trigger = True
 
-    @min_nuclio_versions("1.12.8")
+    @min_nuclio_versions("1.13.1")
     def enable_default_http_trigger(
         self,
     ):
@@ -708,6 +735,10 @@ class RemoteRuntime(KubeResource):
         """
         self.spec.disable_default_http_trigger = False
 
+    def skip_image_enrichment(self):
+        # make sure the API does not enrich the base image if the function is not a python function
+        return self.spec.nuclio_runtime and "python" not in self.spec.nuclio_runtime
+
     def _get_state(
         self,
         dashboard="",
@@ -716,7 +747,7 @@ class RemoteRuntime(KubeResource):
         raise_on_exception=True,
         resolve_address=True,
         auth_info: AuthInfo = None,
-    ) -> typing.Tuple[str, str, typing.Optional[float]]:
+    ) -> tuple[str, str, typing.Optional[float]]:
         if dashboard:
             (
                 state,
@@ -750,7 +781,7 @@ class RemoteRuntime(KubeResource):
             return state, text, last_log_timestamp
 
         try:
-            text, last_log_timestamp = self._get_db().get_builder_status(
+            text, last_log_timestamp = self._get_db().get_nuclio_deploy_status(
                 self, last_log_timestamp=last_log_timestamp, verbose=verbose
             )
         except mlrun.db.RunDBError:
@@ -770,10 +801,13 @@ class RemoteRuntime(KubeResource):
             runtime_env["MLRUN_NAMESPACE"] = mlconf.namespace
         if self.metadata.credentials.access_key:
             runtime_env[
-                mlrun.runtimes.constants.FunctionEnvironmentVariables.auth_session
+                mlrun.common.runtimes.constants.FunctionEnvironmentVariables.auth_session
             ] = self.metadata.credentials.access_key
         return runtime_env
 
+    def _get_serving_spec(self):
+        return None
+
     def _get_nuclio_config_spec_env(self):
         env_dict = {}
         external_source_env_dict = {}
@@ -959,6 +993,62 @@ class RemoteRuntime(KubeResource):
             data = json.loads(data)
         return data
 
+    def with_sidecar(
+        self,
+        name: str = None,
+        image: str = None,
+        ports: typing.Optional[typing.Union[int, list[int]]] = None,
+        command: typing.Optional[str] = None,
+        args: typing.Optional[list[str]] = None,
+    ):
+        """
+        Add a sidecar container to the function pod
+        :param name:    Sidecar container name.
+        :param image:   Sidecar container image.
+        :param ports:   Sidecar container ports to expose. Can be a single port or a list of ports.
+        :param command: Sidecar container command instead of the image entrypoint.
+        :param args:    Sidecar container command args (requires command to be set).
+        """
+        name = name or f"{self.metadata.name}-sidecar"
+        sidecar = self._set_sidecar(name)
+        if image:
+            sidecar["image"] = image
+
+        ports = mlrun.utils.helpers.as_list(ports)
+        # according to RFC-6335, port name should be less than 15 characters,
+        # so we truncate it if needed and leave room for the index
+        port_name = name[:13].rstrip("-_") if len(name) > 13 else name
+        sidecar["ports"] = [
+            {
+                "name": f"{port_name}-{i}",
+                "containerPort": port,
+                "protocol": "TCP",
+            }
+            for i, port in enumerate(ports)
+        ]
+
+        # if it is a redeploy, 'command' might be set with the previous invocation url.
+        # in this case, we don't want to use it as the sidecar command
+        if command and not command.startswith("http"):
+            sidecar["command"] = mlrun.utils.helpers.as_list(command)
+
+        if args and sidecar.get("command"):
+            sidecar["args"] = mlrun.utils.helpers.as_list(args)
+
+        # populate the sidecar resources from the function spec
+        if self.spec.resources:
+            sidecar["resources"] = self.spec.resources
+
+    def _set_sidecar(self, name: str) -> dict:
+        self.spec.config.setdefault("spec.sidecars", [])
+        sidecars = self.spec.config["spec.sidecars"]
+        for sidecar in sidecars:
+            if sidecar["name"] == name:
+                return sidecar
+
+        sidecars.append({"name": name})
+        return sidecars[-1]
+
     def _trigger_of_kind_exists(self, kind: str) -> bool:
         if not self.spec.config:
             return False
@@ -1185,6 +1275,13 @@ class RemoteRuntime(KubeResource):
 
         return self._resolve_invocation_url("", force_external_address)
 
+    @staticmethod
+    def _resolve_v3io_access_key():
+        # Nuclio supports generating access key for v3io stream trigger only from version 1.13.11
+        if validate_nuclio_version_compatibility("1.13.11"):
+            return mlrun.model.Credentials.generate_access_key
+        return None
+
 
 def parse_logs(logs):
     logs = json.loads(logs)
@@ -1279,3 +1376,23 @@ def get_nuclio_deploy_status(
     else:
         text = "\n".join(outputs) if outputs else ""
         return state, address, name, last_log_timestamp, text, function_status
+
+
+def enrich_nuclio_function_from_headers(
+    func: RemoteRuntime,
+    headers: dict,
+):
+    func.status.state = headers.get("x-mlrun-function-status", "")
+    func.status.address = headers.get("x-mlrun-address", "")
+    func.status.nuclio_name = headers.get("x-mlrun-name", "")
+    func.status.internal_invocation_urls = (
+        headers.get("x-mlrun-internal-invocation-urls", "").split(",")
+        if headers.get("x-mlrun-internal-invocation-urls")
+        else []
+    )
+    func.status.external_invocation_urls = (
+        headers.get("x-mlrun-external-invocation-urls", "").split(",")
+        if headers.get("x-mlrun-external-invocation-urls")
+        else []
+    )
+    func.status.container_image = headers.get("x-mlrun-container-image", "")

mlrun/runtimes/{nuclio.py → nuclio/nuclio.py}

@@ -17,13 +17,13 @@ import os
 import socket
 
 import mlrun.db
+from mlrun.errors import err_to_str
+from mlrun.execution import MLClientCtx
+from mlrun.model import RunTemplate
+from mlrun.runtimes.local import get_func_arg
+from mlrun.serving.server import v2_serving_init
+from mlrun.serving.v1_serving import nuclio_serving_init
 
-from ..errors import err_to_str
-from ..execution import MLClientCtx
-from ..model import RunTemplate
-from ..serving.server import v2_serving_init
-from ..serving.v1_serving import nuclio_serving_init
-from .local import get_func_arg
 from .serving import serving_subkind