mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/runtimes/nuclio/application/application.py

@@ -275,7 +275,7 @@ class ApplicationRuntime(RemoteRuntime):
         tag="",
         verbose=False,
         auth_info: schemas.AuthInfo = None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
         with_mlrun=None,
         skip_deployed=False,
@@ -367,7 +367,7 @@ class ApplicationRuntime(RemoteRuntime):
         source,
         workdir=None,
         pull_at_runtime: bool = False,
-        target_dir: str = None,
+        target_dir: typing.Optional[str] = None,
     ):
         """load the code from git/tar/zip archive at build
 
@@ -426,12 +426,12 @@ class ApplicationRuntime(RemoteRuntime):
 
     def create_api_gateway(
         self,
-        name: str = None,
-        path: str = None,
+        name: typing.Optional[str] = None,
+        path: typing.Optional[str] = None,
         direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
-        authentication_creds: tuple[str, str] = None,
-        ssl_redirect: bool = None,
+        authentication_creds: typing.Optional[tuple[str, str]] = None,
+        ssl_redirect: typing.Optional[bool] = None,
         set_as_default: bool = False,
         gateway_timeout: typing.Optional[int] = None,
     ):
@@ -540,13 +540,12 @@ class ApplicationRuntime(RemoteRuntime):
         self,
         path: str = "",
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
-        method: str = None,
-        headers: dict = None,
-        dashboard: str = "",
+        method: typing.Optional[str] = None,
+        headers: typing.Optional[dict] = None,
         force_external_address: bool = False,
         auth_info: schemas.AuthInfo = None,
-        mock: bool = None,
-        credentials: tuple[str, str] = None,
+        mock: typing.Optional[bool] = None,
+        credentials: typing.Optional[tuple[str, str]] = None,
         **http_client_kwargs,
     ):
         self._sync_api_gateway()
@@ -569,7 +568,6 @@ class ApplicationRuntime(RemoteRuntime):
                 body,
                 method,
                 headers,
-                dashboard,
                 force_external_address,
                 auth_info,
                 mock,
@@ -653,7 +651,7 @@ class ApplicationRuntime(RemoteRuntime):
 
     def _build_application_image(
         self,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
         watch=True,
         with_mlrun=None,

mlrun/runtimes/nuclio/application/reverse_proxy.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//   http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,82 +14,84 @@
 package main
 
 import (
-    "bytes"
-    "fmt"
-    "net/http"
-    "net/http/httptest"
-    "net/http/httputil"
-    "net/url"
-    "os"
-    "strings"
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"strings"
 
-    nuclio "github.com/nuclio/nuclio-sdk-go"
+	nuclio "github.com/nuclio/nuclio-sdk-go"
 )
 
 func Handler(context *nuclio.Context, event nuclio.Event) (interface{}, error) {
-    reverseProxy := context.UserData.(map[string]interface{})["reverseProxy"].(*httputil.ReverseProxy)
-    sidecarUrl := context.UserData.(map[string]interface{})["server"].(string)
+	reverseProxy := context.UserData.(map[string]interface{})["reverseProxy"].(*httputil.ReverseProxy)
+	sidecarUrl := context.UserData.(map[string]interface{})["server"].(string)
 
-    // populate reverse proxy http request
-    httpRequest, err := http.NewRequest(event.GetMethod(), event.GetPath(), bytes.NewReader(event.GetBody()))
-    if err != nil {
-        context.Logger.ErrorWith("Failed to create a reverse proxy request")
-        return nil, err
-    }
-    for k, v := range event.GetHeaders() {
-        httpRequest.Header[k] = []string{v.(string)}
-    }
+	// populate reverse proxy http request
+	httpRequest, err := http.NewRequest(event.GetMethod(), event.GetPath(), bytes.NewReader(event.GetBody()))
+	if err != nil {
+		context.Logger.ErrorWith("Failed to create a reverse proxy request")
+		return nil, err
+	}
+	for k, v := range event.GetHeaders() {
+		httpRequest.Header[k] = []string{v.(string)}
+	}
 
-    // populate query params
-    query := httpRequest.URL.Query()
-    for k, v := range event.GetFields() {
-        query.Set(k, v.(string))
-    }
-    httpRequest.URL.RawQuery = query.Encode()
+	// populate query params
+	query := httpRequest.URL.Query()
+	for k, v := range event.GetFields() {
+		query.Set(k, v.(string))
+	}
+	httpRequest.URL.RawQuery = query.Encode()
 
-    recorder := httptest.NewRecorder()
-    reverseProxy.ServeHTTP(recorder, httpRequest)
+	recorder := httptest.NewRecorder()
+	reverseProxy.ServeHTTP(recorder, httpRequest)
 
-    // send request to sidecar
-    context.Logger.DebugWith("Forwarding request to sidecar", "sidecarUrl", sidecarUrl, "query", httpRequest.URL.Query())
-    response := recorder.Result()
+	// send request to sidecar
+	context.Logger.DebugWith("Forwarding request to sidecar",
+		"sidecarUrl", sidecarUrl,
+		"method", event.GetMethod())
+	response := recorder.Result()
 
-    headers := make(map[string]interface{})
-    for key, value := range response.Header {
-        headers[key] = value[0]
-    }
+	headers := make(map[string]interface{})
+	for key, value := range response.Header {
+		headers[key] = value[0]
+	}
 
-    // let the processor calculate the content length
-    delete(headers, "Content-Length")
-    return nuclio.Response{
-        StatusCode:  response.StatusCode,
-        Body:        recorder.Body.Bytes(),
-        ContentType: response.Header.Get("Content-Type"),
-        Headers:     headers,
-    }, nil
+	// let the processor calculate the content length
+	delete(headers, "Content-Length")
+	return nuclio.Response{
+		StatusCode:  response.StatusCode,
+		Body:        recorder.Body.Bytes(),
+		ContentType: response.Header.Get("Content-Type"),
+		Headers:     headers,
+	}, nil
 }
 
 func InitContext(context *nuclio.Context) error {
-    sidecarHost := os.Getenv("SIDECAR_HOST")
-    sidecarPort := os.Getenv("SIDECAR_PORT")
-    if sidecarHost == "" {
-        sidecarHost = "http://localhost"
-    } else if !strings.Contains(sidecarHost, "://") {
-        sidecarHost = fmt.Sprintf("http://%s", sidecarHost)
-    }
+	sidecarHost := os.Getenv("SIDECAR_HOST")
+	sidecarPort := os.Getenv("SIDECAR_PORT")
+	if sidecarHost == "" {
+		sidecarHost = "http://localhost"
+	} else if !strings.Contains(sidecarHost, "://") {
+		sidecarHost = fmt.Sprintf("http://%s", sidecarHost)
+	}
 
-    // url for request forwarding
-    sidecarUrl := fmt.Sprintf("%s:%s", sidecarHost, sidecarPort)
-    parsedURL, err := url.Parse(sidecarUrl)
-    if err != nil {
-        context.Logger.ErrorWith("Failed to parse sidecar url", "sidecarUrl", sidecarUrl)
-        return err
-    }
-    reverseProxy := httputil.NewSingleHostReverseProxy(parsedURL)
+	// url for request forwarding
+	sidecarUrl := fmt.Sprintf("%s:%s", sidecarHost, sidecarPort)
+	parsedURL, err := url.Parse(sidecarUrl)
+	if err != nil {
+		context.Logger.ErrorWith("Failed to parse sidecar url", "sidecarUrl", sidecarUrl)
+		return err
+	}
+	reverseProxy := httputil.NewSingleHostReverseProxy(parsedURL)
 
-    context.UserData = map[string]interface{}{
-        "server":       sidecarUrl,
-        "reverseProxy": reverseProxy,
-    }
-    return nil
+	context.UserData = map[string]interface{}{
+		"server":       sidecarUrl,
+		"reverseProxy": reverseProxy,
+	}
+	return nil
 }

mlrun/runtimes/nuclio/function.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import copy
 import json
 import typing
 import warnings
@@ -26,9 +27,6 @@ import requests
 import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.common.ops import deploy_op
-from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 from nuclio.deploy import find_dashboard_url, get_deploy_status
 from nuclio.triggers import V3IOStreamTrigger
 
@@ -47,9 +45,24 @@ from mlrun.platforms.iguazio import (
     split_path,
 )
 from mlrun.runtimes.base import FunctionStatus, RunError
+from mlrun.runtimes.mounts import VolumeMount, mount_v3io, v3io_cred
 from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
 from mlrun.runtimes.utils import get_item_name, log_std
 from mlrun.utils import get_in, logger, update_in
+from mlrun_pipelines.common.ops import deploy_op
+
+SENSITIVE_PATHS_IN_TRIGGER_CONFIG = {
+    "password",
+    "secret",
+    "attributes/password",
+    "attributes/accesskeyid",
+    "attributes/secretaccesskey",
+    "attributes/cacert",
+    "attributes/accesskey",
+    "attributes/accesscertificate",
+    "attributes/sasl/password",
+    "attributes/sasl/oauth/clientsecret",
+}
 
 
 def validate_nuclio_version_compatibility(*min_versions):
@@ -275,6 +288,37 @@ class RemoteRuntime(KubeResource):
         if self.metadata.tag:
             mlrun.utils.validate_tag_name(self.metadata.tag, "function.metadata.tag")
 
+    def mask_sensitive_data_in_config(self):
+        if not self.spec.config:
+            return {}
+
+        raw_config = copy.deepcopy(self.spec.config)
+
+        for key, value in self.spec.config.items():
+            if key.startswith("spec.triggers"):
+                trigger_name = key.split(".")[-1]
+
+                for path in SENSITIVE_PATHS_IN_TRIGGER_CONFIG:
+                    # Handle nested keys
+                    nested_keys = path.split("/")
+                    target = value
+                    for sub_key in nested_keys[:-1]:
+                        target = target.get(sub_key, {})
+
+                    last_key = nested_keys[-1]
+                    if last_key in target:
+                        sensitive_field = target[last_key]
+                        if sensitive_field.startswith(
+                            mlrun.model.Credentials.secret_reference_prefix
+                        ):
+                            # already masked
+                            continue
+                        target[last_key] = (
+                            f"{mlrun.model.Credentials.secret_reference_prefix}/spec/triggers/{trigger_name}/{path}"
+                        )
+
+        return raw_config
+
     def set_config(self, key, value):
         self.spec.config[key] = value
         return self
@@ -307,9 +351,12 @@ class RemoteRuntime(KubeResource):
     def _validate_triggers(self, spec):
         # ML-7763 / NUC-233
         min_nuclio_version = "1.13.12"
-        if mlconf.nuclio_version and semver.VersionInfo.parse(
+        if (
             mlconf.nuclio_version
-        ) < semver.VersionInfo.parse(min_nuclio_version):
+            and mlconf.nuclio_version != "unstable"
+            and semver.VersionInfo.parse(mlconf.nuclio_version)
+            < semver.VersionInfo.parse(min_nuclio_version)
+        ):
             explicit_ack_enabled = False
             num_triggers = 0
             trigger_name = spec.get("name", "UNKNOWN")
@@ -370,8 +417,9 @@ class RemoteRuntime(KubeResource):
                 )
         """
         self.spec.build.source = source
-        # update handler in function_handler
-        self.spec.function_handler = handler
+        # update handler in function_handler if needed
+        if handler:
+            self.spec.function_handler = handler
         if workdir:
             self.spec.workdir = workdir
         if runtime:
@@ -527,6 +575,13 @@ class RemoteRuntime(KubeResource):
         access_key = kwargs.pop("access_key", None)
         if not access_key:
             access_key = self._resolve_v3io_access_key()
+        engine = "sync"
+        explicit_ack_mode = kwargs.pop("explicit_ack_mode", None)
+        if self.spec and hasattr(self.spec, "graph"):
+            engine = getattr(self.spec.graph, "engine", None) or engine
+
+        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
+            explicit_ack_mode = explicit_ack_mode or "explicitOnly"
 
         self.add_trigger(
             name,
@@ -540,6 +595,7 @@ class RemoteRuntime(KubeResource):
                 extra_attributes=extra_attributes,
                 read_batch_size=256,
                 access_key=access_key,
+                explicit_ack_mode=explicit_ack_mode,
                 **kwargs,
             ),
         )
@@ -554,7 +610,7 @@ class RemoteRuntime(KubeResource):
         tag="",
         verbose=False,
         auth_info: AuthInfo = None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
     ):
         """Deploy the nuclio function to the cluster
@@ -567,9 +623,9 @@ class RemoteRuntime(KubeResource):
         :param force_build: set True for force building the image
         """
         if auth_info:
-            # TODO: remove in 1.9.0
+            # TODO: remove in 1.10.0
             warnings.warn(
-                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.9.0",
+                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.10.0",
                 FutureWarning,
             )
 
@@ -600,6 +656,18 @@ class RemoteRuntime(KubeResource):
         # when a function is deployed, we wait for it to be ready by default
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
+        # check if there are any background tasks related to creating model endpoints
+        model_endpoints_creation_background_tasks = (
+            mlrun.common.schemas.BackgroundTaskList(
+                **data.pop("background_tasks", {"background_tasks": []})
+            ).background_tasks
+        )
+        if model_endpoints_creation_background_tasks:
+            self._check_model_endpoint_task_state(
+                db=db,
+                background_task=model_endpoints_creation_background_tasks[0],
+                wait_for_completion=False,
+            )
 
         return self._enrich_command_from_status()
 
@@ -693,7 +761,9 @@ class RemoteRuntime(KubeResource):
         super().with_priority_class(name)
 
     def with_service_type(
-        self, service_type: str, add_templated_ingress_host_mode: str = None
+        self,
+        service_type: str,
+        add_templated_ingress_host_mode: typing.Optional[str] = None,
     ):
         """
         Enables to control the service type of the pod and the addition of templated ingress host
@@ -741,45 +811,10 @@ class RemoteRuntime(KubeResource):
 
     def _get_state(
         self,
-        dashboard="",
         last_log_timestamp=0,
         verbose=False,
         raise_on_exception=True,
-        resolve_address=True,
-        auth_info: AuthInfo = None,
     ) -> tuple[str, str, typing.Optional[float]]:
-        if dashboard:
-            (
-                state,
-                address,
-                name,
-                last_log_timestamp,
-                text,
-                function_status,
-            ) = get_nuclio_deploy_status(
-                self.metadata.name,
-                self.metadata.project,
-                self.metadata.tag,
-                dashboard,
-                last_log_timestamp=last_log_timestamp,
-                verbose=verbose,
-                resolve_address=resolve_address,
-                auth_info=auth_info,
-            )
-            self.status.internal_invocation_urls = function_status.get(
-                "internalInvocationUrls", []
-            )
-            self.status.external_invocation_urls = function_status.get(
-                "externalInvocationUrls", []
-            )
-            self.status.state = state
-            self.status.nuclio_name = name
-            self.status.container_image = function_status.get("containerImage", "")
-            if address:
-                self.status.address = address
-                self.spec.command = f"http://{address}"
-            return state, text, last_log_timestamp
-
         try:
             text, last_log_timestamp = self._get_db().get_nuclio_deploy_status(
                 self, last_log_timestamp=last_log_timestamp, verbose=verbose
@@ -887,13 +922,12 @@ class RemoteRuntime(KubeResource):
     def invoke(
         self,
         path: str,
-        body: typing.Union[str, bytes, dict] = None,
-        method: str = None,
-        headers: dict = None,
-        dashboard: str = "",
+        body: typing.Optional[typing.Union[str, bytes, dict]] = None,
+        method: typing.Optional[str] = None,
+        headers: typing.Optional[dict] = None,
         force_external_address: bool = False,
         auth_info: AuthInfo = None,
-        mock: bool = None,
+        mock: typing.Optional[bool] = None,
         **http_client_kwargs,
     ):
         """Invoke the remote (live) function and return the results
@@ -906,7 +940,6 @@ class RemoteRuntime(KubeResource):
         :param body:     request body (str, bytes or a dict for json requests)
         :param method:   HTTP method (GET, PUT, ..)
         :param headers:  key/value dict with http headers
-        :param dashboard: nuclio dashboard address (deprecated)
         :param force_external_address:   use the external ingress URL
         :param auth_info: service AuthInfo
         :param mock:     use mock server vs a real Nuclio function (for local simulations)
@@ -914,14 +947,6 @@ class RemoteRuntime(KubeResource):
                                      see this link for more information:
                                      https://requests.readthedocs.io/en/latest/api/#requests.request
         """
-        if dashboard:
-            # TODO: remove in 1.8.0
-            warnings.warn(
-                "'dashboard' parameter is no longer supported on client side, "
-                "it is being configured through the MLRun API. It will be removed in 1.8.0.",
-                FutureWarning,
-            )
-
         if not method:
             method = "POST" if body else "GET"
 
@@ -951,7 +976,7 @@ class RemoteRuntime(KubeResource):
                         "so function can not be invoked via http. Either enable default http trigger creation or "
                         "create custom http trigger"
                     )
-                state, _, _ = self._get_state(dashboard, auth_info=auth_info)
+                state, _, _ = self._get_state()
                 if state not in ["ready", "scaledToZero"]:
                     logger.warning(f"Function is in the {state} state")
                 if not self.status.address:
@@ -975,7 +1000,7 @@ class RemoteRuntime(KubeResource):
             else:
                 http_client_kwargs["json"] = body
         try:
-            logger.info("Invoking function", method=method, path=path)
+            logger.debug("Invoking function", method=method, path=path)
             if not getattr(self, "_http_session", None):
                 self._http_session = requests.Session()
             resp = self._http_session.request(
@@ -995,14 +1020,15 @@ class RemoteRuntime(KubeResource):
 
     def with_sidecar(
         self,
-        name: str = None,
-        image: str = None,
+        name: typing.Optional[str] = None,
+        image: typing.Optional[str] = None,
         ports: typing.Optional[typing.Union[int, list[int]]] = None,
         command: typing.Optional[str] = None,
         args: typing.Optional[list[str]] = None,
     ):
         """
         Add a sidecar container to the function pod
+
         :param name:    Sidecar container name.
         :param image:   Sidecar container image.
         :param ports:   Sidecar container ports to expose. Can be a single port or a list of ports.
@@ -1192,9 +1218,6 @@ class RemoteRuntime(KubeResource):
         return results
 
     def _resolve_invocation_url(self, path, force_external_address):
-        if not path.startswith("/") and path != "":
-            path = f"/{path}"
-
         # internal / external invocation urls is a nuclio >= 1.6.x feature
         # try to infer the invocation url from the internal and if not exists, use external.
         # $$$$ we do not want to use the external invocation url (e.g.: ingress, nodePort, etc.)
@@ -1203,12 +1226,16 @@ class RemoteRuntime(KubeResource):
             and self.status.internal_invocation_urls
             and mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
         ):
-            return f"http://{self.status.internal_invocation_urls[0]}{path}"
+            return mlrun.utils.helpers.join_urls(
+                f"http://{self.status.internal_invocation_urls[0]}", path
+            )
 
         if self.status.external_invocation_urls:
-            return f"http://{self.status.external_invocation_urls[0]}{path}"
+            return mlrun.utils.helpers.join_urls(
+                f"http://{self.status.external_invocation_urls[0]}", path
+            )
         else:
-            return f"http://{self.status.address}{path}"
+            return mlrun.utils.helpers.join_urls(f"http://{self.status.address}", path)
 
     def _update_credentials_from_remote_build(self, remote_data):
         self.metadata.credentials = remote_data.get("metadata", {}).get(
@@ -1244,6 +1271,9 @@ class RemoteRuntime(KubeResource):
             if remote_env.get("name") in credentials_env_var_names:
                 new_env.append(remote_env)
 
+        # update nuclio-specific credentials
+        self.mask_sensitive_data_in_config()
+
         self.spec.env = new_env
 
     def _set_as_mock(self, enable):
@@ -1283,6 +1313,33 @@ class RemoteRuntime(KubeResource):
             return mlrun.model.Credentials.generate_access_key
         return None
 
+    def _check_model_endpoint_task_state(
+        self,
+        db: mlrun.db.RunDBInterface,
+        background_task: mlrun.common.schemas.BackgroundTask,
+        wait_for_completion: bool,
+    ):
+        if wait_for_completion:
+            background_task = db._wait_for_background_task_to_reach_terminal_state(
+                name=background_task.metadata.name, project=self.metadata.project
+            )
+        else:
+            background_task = db.get_project_background_task(
+                project=self.metadata.project, name=background_task.metadata.name
+            )
+        if (
+            background_task.status.state
+            in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+        ):
+            logger.info(
+                f"Model endpoint creation task completed with state {background_task.status.state}"
+            )
+        else:
+            logger.warning(
+                f"Model endpoint creation task is still running with state {background_task.status.state}"
+                f"You can use the serving function, but it won't be monitored for the next few minutes"
+            )
+
 
 def parse_logs(logs):
     logs = json.loads(logs)