mlrun 1.7.0rc38__py3-none-any.whl → 1.7.0rc41__py3-none-any.whl

mlrun/runtimes/nuclio/application/application.py

@@ -15,6 +15,7 @@ import pathlib
 import typing
 
 import nuclio
+import nuclio.auth
 
 import mlrun.common.schemas as schemas
 import mlrun.errors
@@ -281,34 +282,29 @@ class ApplicationRuntime(RemoteRuntime):
         is_kfp=False,
         mlrun_version_specifier=None,
         show_on_failure: bool = False,
-        direct_port_access: bool = False,
-        authentication_mode: schemas.APIGatewayAuthenticationMode = None,
-        authentication_creds: tuple[str] = None,
-        ssl_redirect: bool = None,
+        create_default_api_gateway: bool = True,
     ):
         """
         Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
         Once the image is built, the function is deployed.
 
-        :param project:                 Project name
-        :param tag:                     Function tag
-        :param verbose:                 Set True for verbose logging
-        :param auth_info:               Service AuthInfo (deprecated and ignored)
-        :param builder_env:             Env vars dict for source archive config/credentials
-                                        e.g. builder_env={"GIT_TOKEN": token}
-        :param force_build:             Set True for force building the application image
-        :param with_mlrun:              Add the current mlrun package to the container build
-        :param skip_deployed:           Skip the build if we already have an image for the function
-        :param is_kfp:                  Deploy as part of a kfp pipeline
-        :param mlrun_version_specifier: Which mlrun package version to include (if not current)
-        :param show_on_failure:         Show logs only in case of build failure
-        :param direct_port_access:      Set True to allow direct port access to the application sidecar
-        :param authentication_mode:     API Gateway authentication mode
-        :param authentication_creds:    API Gateway authentication credentials as a tuple (username, password)
-        :param ssl_redirect:            Set True to force SSL redirect, False to disable. Defaults to
-                                        mlrun.mlconf.force_api_gateway_ssl_redirect()
-
-        :return: True if the function is ready (deployed)
+        :param project:                     Project name
+        :param tag:                         Function tag
+        :param verbose:                     Set True for verbose logging
+        :param auth_info:                   Service AuthInfo (deprecated and ignored)
+        :param builder_env:                 Env vars dict for source archive config/credentials
+                                            e.g. builder_env={"GIT_TOKEN": token}
+        :param force_build:                 Set True for force building the application image
+        :param with_mlrun:                  Add the current mlrun package to the container build
+        :param skip_deployed:               Skip the build if we already have an image for the function
+        :param is_kfp:                      Deploy as part of a kfp pipeline
+        :param mlrun_version_specifier:     Which mlrun package version to include (if not current)
+        :param show_on_failure:             Show logs only in case of build failure
+        :param create_default_api_gateway:  When deploy finishes the default API gateway will be created for the
+                                            application. Disabling this flag means that the application will not be
+                                            accessible until an API gateway is created for it.
+
+        :return: The default API gateway URL if created or True if the function is ready (deployed)
         """
         if (self.requires_build() and not self.spec.image) or force_build:
             self._fill_credentials()
@@ -328,10 +324,6 @@ class ApplicationRuntime(RemoteRuntime):
         self._configure_application_sidecar()
 
         # We only allow accessing the application via the API Gateway
-        name_tag = tag or self.metadata.tag
-        self.status.api_gateway_name = (
-            f"{self.metadata.name}-{name_tag}" if name_tag else self.metadata.name
-        )
         self.spec.add_templated_ingress_host_mode = (
             NuclioIngressAddTemplatedIngressModes.never
         )
@@ -344,9 +336,7 @@ class ApplicationRuntime(RemoteRuntime):
             builder_env=builder_env,
         )
         logger.info(
-            "Successfully deployed function, creating API gateway",
-            api_gateway_name=self.status.api_gateway_name,
-            authentication_mode=authentication_mode,
+            "Successfully deployed function.",
         )
 
         # Restore the source in case it was removed to make nuclio not consider it when building
@@ -354,14 +344,23 @@ class ApplicationRuntime(RemoteRuntime):
             self.spec.build.source = self.status.application_source
         self.save(versioned=False)
 
-        ports = self.spec.internal_application_port if direct_port_access else []
-        return self.create_api_gateway(
-            name=self.status.api_gateway_name,
-            ports=ports,
-            authentication_mode=authentication_mode,
-            authentication_creds=authentication_creds,
-            ssl_redirect=ssl_redirect,
-        )
+        if create_default_api_gateway:
+            try:
+                api_gateway_name = self.resolve_default_api_gateway_name()
+                return self.create_api_gateway(api_gateway_name, set_as_default=True)
+            except Exception as exc:
+                logger.warning(
+                    "Failed to create default API gateway, application may not be accessible. "
+                    "Use the `create_api_gateway` method to make it accessible",
+                    exc=mlrun.errors.err_to_str(exc),
+                )
+        elif not self.status.api_gateway:
+            logger.warning(
+                "Application is online but may not be accessible since default gateway creation was not requested."
+                "Use the `create_api_gateway` method to make it accessible."
+            )
+
+        return True
 
     def with_source_archive(
         self,
@@ -429,17 +428,54 @@ class ApplicationRuntime(RemoteRuntime):
         self,
         name: str = None,
         path: str = None,
-        ports: list[int] = None,
+        direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
-        authentication_creds: tuple[str] = None,
+        authentication_creds: tuple[str, str] = None,
         ssl_redirect: bool = None,
+        set_as_default: bool = False,
+        gateway_timeout: typing.Optional[int] = None,
     ):
+        """
+        Create the application API gateway. Once the application is deployed, the API gateway can be created.
+        An application without an API gateway is not accessible.
+        :param name:                    The name of the API gateway, defaults to <function-name>-<function-tag>
+        :param path:                    Optional path of the API gateway, default value is "/"
+        :param direct_port_access:      Set True to allow direct port access to the application sidecar
+        :param authentication_mode:     API Gateway authentication mode
+        :param authentication_creds:    API Gateway basic authentication credentials as a tuple (username, password)
+        :param ssl_redirect:            Set True to force SSL redirect, False to disable. Defaults to
+                                        mlrun.mlconf.force_api_gateway_ssl_redirect()
+        :param set_as_default:          Set the API gateway as the default for the application (`status.api_gateway`)
+        :param gateway_timeout:         nginx ingress timeout in sec (request timeout, when will the gateway return an
+                                        error)
+
+        :return:    The API gateway URL
+        """
+        if not name:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "API gateway name must be specified."
+            )
+
+        if not set_as_default and name == self.resolve_default_api_gateway_name():
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Non-default API gateway cannot use the default gateway name, {name=}."
+            )
+
+        if (
+            authentication_mode == schemas.APIGatewayAuthenticationMode.basic
+            and not authentication_creds
+        ):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Authentication credentials not provided"
+            )
+
+        ports = self.spec.internal_application_port if direct_port_access else []
+
         api_gateway = APIGateway(
             APIGatewayMetadata(
                 name=name,
                 namespace=self.metadata.namespace,
-                labels=self.metadata.labels,
-                annotations=self.metadata.annotations,
+                labels=self.metadata.labels.copy(),
             ),
             APIGatewaySpec(
                 functions=[self],
@@ -449,13 +485,14 @@ class ApplicationRuntime(RemoteRuntime):
             ),
         )
 
+        api_gateway.with_gateway_timeout(gateway_timeout)
         if ssl_redirect is None:
             ssl_redirect = mlrun.mlconf.force_api_gateway_ssl_redirect()
         if ssl_redirect:
-            # force ssl redirect so that the application is only accessible via https
+            # Force ssl redirect so that the application is only accessible via https
             api_gateway.with_force_ssl_redirect()
 
-        # add authentication if required
+        # Add authentication if required
         authentication_mode = (
             authentication_mode
             or mlrun.mlconf.function.application.default_authentication_mode
@@ -469,31 +506,64 @@ class ApplicationRuntime(RemoteRuntime):
         api_gateway_scheme = db.store_api_gateway(
             api_gateway=api_gateway.to_scheme(), project=self.metadata.project
         )
-        if not self.status.api_gateway_name:
-            self.status.api_gateway_name = api_gateway_scheme.metadata.name
-        self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
-        self.status.api_gateway.wait_for_readiness()
-        self.url = self.status.api_gateway.invoke_url
 
-        logger.info("Successfully created API gateway", url=self.url)
-        return self.url
+        if set_as_default:
+            self.status.api_gateway_name = api_gateway_scheme.metadata.name
+            self.status.api_gateway = APIGateway.from_scheme(api_gateway_scheme)
+            self.status.api_gateway.wait_for_readiness()
+            self.url = self.status.api_gateway.invoke_url
+            url = self.url
+        else:
+            api_gateway = APIGateway.from_scheme(api_gateway_scheme)
+            api_gateway.wait_for_readiness()
+            url = api_gateway.invoke_url
+            # Update application status (enriches invocation url)
+            self._get_state(raise_on_exception=False)
+
+        logger.info("Successfully created API gateway", url=url)
+        return url
+
+    def delete_api_gateway(self, name: str):
+        """
+        Delete API gateway by name.
+        Refreshes the application status to update api gateway and invocation URLs.
+        :param name:    The API gateway name
+        """
+        self._get_db().delete_api_gateway(name=name, project=self.metadata.project)
+        if name == self.status.api_gateway_name:
+            self.status.api_gateway_name = None
+            self.status.api_gateway = None
+        self._get_state()
 
     def invoke(
         self,
-        path: str,
-        body: typing.Union[str, bytes, dict] = None,
+        path: str = "",
+        body: typing.Optional[typing.Union[str, bytes, dict]] = None,
         method: str = None,
         headers: dict = None,
         dashboard: str = "",
         force_external_address: bool = False,
         auth_info: schemas.AuthInfo = None,
         mock: bool = None,
+        credentials: tuple[str, str] = None,
         **http_client_kwargs,
     ):
         self._sync_api_gateway()
+
         # If the API Gateway is not ready or not set, try to invoke the function directly (without the API Gateway)
         if not self.status.api_gateway:
-            super().invoke(
+            logger.warning(
+                "Default API gateway is not configured, invoking function invocation URL."
+            )
+            # create a requests auth object if credentials are provided and not already set in the http client kwargs
+            auth = http_client_kwargs.pop("auth", None) or (
+                nuclio.auth.AuthInfo(
+                    username=credentials[0], password=credentials[1]
+                ).to_requests_auth()
+                if credentials
+                else None
+            )
+            return super().invoke(
                 path,
                 body,
                 method,
@@ -502,18 +572,19 @@ class ApplicationRuntime(RemoteRuntime):
                 force_external_address,
                 auth_info,
                 mock,
+                auth=auth,
                 **http_client_kwargs,
             )
 
-        credentials = (auth_info.username, auth_info.password) if auth_info else None
-
         if not method:
             method = "POST" if body else "GET"
+
         return self.status.api_gateway.invoke(
             method=method,
             headers=headers,
             credentials=credentials,
             path=path,
+            body=body,
             **http_client_kwargs,
         )
 
@@ -550,6 +621,27 @@ class ApplicationRuntime(RemoteRuntime):
             reverse_proxy_func.metadata.name, reverse_proxy_func.metadata.project
         )
 
+    def resolve_default_api_gateway_name(self):
+        return (
+            f"{self.metadata.name}-{self.metadata.tag}"
+            if self.metadata.tag
+            else self.metadata.name
+        )
+
+    @min_nuclio_versions("1.13.1")
+    def disable_default_http_trigger(
+        self,
+    ):
+        raise mlrun.runtimes.RunError(
+            "Application runtime does not support disabling the default HTTP trigger"
+        )
+
+    @min_nuclio_versions("1.13.1")
+    def enable_default_http_trigger(
+        self,
+    ):
+        pass
+
     def _run(self, runobj: "mlrun.RunObject", execution):
         raise mlrun.runtimes.RunError(
             "Application runtime .run() is not yet supported. Use .invoke() instead."
@@ -648,9 +740,8 @@ class ApplicationRuntime(RemoteRuntime):
         self.set_env("SIDECAR_HOST", "http://localhost")
 
         # configure the sidecar container as the default container for logging purposes
-        self.set_config(
-            "metadata.annotations",
-            {"kubectl.kubernetes.io/default-container": self.status.sidecar_name},
+        self.metadata.annotations["kubectl.kubernetes.io/default-container"] = (
+            self.status.sidecar_name
         )
 
     def _sync_api_gateway(self):

mlrun/runtimes/nuclio/function.py

@@ -418,14 +418,8 @@ class RemoteRuntime(KubeResource):
                 raise ValueError(
                     "gateway timeout must be greater than the worker timeout"
                 )
-            annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = (
-                f"{gateway_timeout}"
-            )
-            annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = (
-                f"{gateway_timeout}"
-            )
-            annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = (
-                f"{gateway_timeout}"
+            mlrun.runtimes.utils.enrich_gateway_timeout_annotations(
+                annotations, gateway_timeout
             )
 
         trigger = nuclio.HttpTrigger(
@@ -698,7 +692,7 @@ class RemoteRuntime(KubeResource):
         """
         self.spec.disable_default_http_trigger = True
 
-    @min_nuclio_versions("1.12.8")
+    @min_nuclio_versions("1.13.1")
     def enable_default_http_trigger(
         self,
     ):
@@ -753,7 +747,7 @@ class RemoteRuntime(KubeResource):
             return state, text, last_log_timestamp
 
         try:
-            text, last_log_timestamp = self._get_db().get_builder_status(
+            text, last_log_timestamp = self._get_db().get_nuclio_deploy_status(
                 self, last_log_timestamp=last_log_timestamp, verbose=verbose
             )
         except mlrun.db.RunDBError:
@@ -1004,7 +998,7 @@ class RemoteRuntime(KubeResource):
         if command and not command.startswith("http"):
             sidecar["command"] = mlrun.utils.helpers.as_list(command)
 
-        if args and sidecar["command"]:
+        if args and sidecar.get("command"):
             sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
         # populate the sidecar resources from the function spec

mlrun/runtimes/nuclio/serving.py

@@ -676,7 +676,6 @@ class ServingRuntime(RemoteRuntime):
         """create mock server object for local testing/emulation
 
         :param namespace: one or list of namespaces/modules to search the steps classes/functions in
-        :param log_level: log level (error | info | debug)
         :param current_function: specify if you want to simulate a child function, * for all functions
         :param track_models: allow model tracking (disabled by default in the mock server)
         :param workdir:   working directory to locate the source code (if not the current one)
@@ -704,7 +703,7 @@ class ServingRuntime(RemoteRuntime):
             verbose=self.verbose,
             current_function=current_function,
             graph_initializer=self.spec.graph_initializer,
-            track_models=track_models and self.spec.track_models,
+            track_models=self.spec.track_models,
             function_uri=self._function_uri(),
             secret_sources=self.spec.secret_sources,
             default_content_type=self.spec.default_content_type,
@@ -715,6 +714,7 @@ class ServingRuntime(RemoteRuntime):
             namespace=namespace,
             logger=logger,
             is_mock=True,
+            monitoring_mock=track_models,
         )
 
         if workdir:

mlrun/runtimes/utils.py

@@ -463,3 +463,19 @@ def resolve_node_selectors(
             instance_node_selector,
         )
     return instance_node_selector
+
+
+def enrich_gateway_timeout_annotations(annotations: dict, gateway_timeout: int):
+    """
+    Set gateway proxy connect/read/send timeout annotations
+    :param annotations:     The annotations to enrich
+    :param gateway_timeout: The timeout to set
+    """
+    if not gateway_timeout:
+        return
+    gateway_timeout_str = str(gateway_timeout)
+    annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = (
+        gateway_timeout_str
+    )
+    annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = gateway_timeout_str
+    annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = gateway_timeout_str

mlrun/serving/routers.py

@@ -615,7 +615,7 @@ class VotingEnsemble(ParallelRun):
             logger.warn("GraphServer not initialized for VotingEnsemble instance")
             return
 
-        if not self.context.is_mock or self.context.server.track_models:
+        if not self.context.is_mock or self.context.monitoring_mock:
             self.model_endpoint_uid = _init_endpoint_record(server, self)
 
         self._update_weights(self.weights)

mlrun/serving/server.py

@@ -22,10 +22,14 @@ import traceback
 import uuid
 from typing import Optional, Union
 
+from nuclio import Context as NuclioContext
+from nuclio.request import Logger as NuclioLogger
+
 import mlrun
 import mlrun.common.constants
 import mlrun.common.helpers
 import mlrun.model_monitoring
+import mlrun.utils
 from mlrun.config import config
 from mlrun.errors import err_to_str
 from mlrun.secrets import SecretsStore
@@ -150,6 +154,7 @@ class GraphServer(ModelObj):
         resource_cache: ResourceCache = None,
         logger=None,
         is_mock=False,
+        monitoring_mock=False,
     ):
         """for internal use, initialize all steps (recursively)"""
 
@@ -162,6 +167,7 @@ class GraphServer(ModelObj):
 
         context = GraphContext(server=self, nuclio_context=context, logger=logger)
         context.is_mock = is_mock
+        context.monitoring_mock = monitoring_mock
         context.root = self.graph
 
         context.stream = _StreamContext(
@@ -387,7 +393,9 @@ def v2_serving_handler(context, event, get_body=False):
 
     # original path is saved in stream_path so it can be used by explicit ack, but path is reset to / as a
     # workaround for NUC-178
-    event.stream_path = event.path
+    # nuclio 1.12.12 added the topic attribute, and we must use it as part of the fix for NUC-233
+    # TODO: Remove fallback on event.path once support for nuclio<1.12.12 is dropped
+    event.stream_path = getattr(event, "topic", event.path)
     if hasattr(event, "trigger") and event.trigger.kind in (
         "kafka",
         "kafka-cluster",
@@ -483,7 +491,13 @@ class Response:
 class GraphContext:
     """Graph context object"""
 
-    def __init__(self, level="info", logger=None, server=None, nuclio_context=None):
+    def __init__(
+        self,
+        level="info",  # Unused argument
+        logger=None,
+        server=None,
+        nuclio_context: Optional[NuclioContext] = None,
+    ) -> None:
         self.state = None
         self.logger = logger
         self.worker_id = 0
@@ -493,7 +507,7 @@ class GraphContext:
         self.root = None
 
         if nuclio_context:
-            self.logger = nuclio_context.logger
+            self.logger: NuclioLogger = nuclio_context.logger
             self.Response = nuclio_context.Response
             if hasattr(nuclio_context, "trigger") and hasattr(
                 nuclio_context.trigger, "kind"
@@ -503,7 +517,7 @@ class GraphContext:
             if hasattr(nuclio_context, "platform"):
                 self.platform = nuclio_context.platform
         elif not logger:
-            self.logger = mlrun.utils.helpers.logger
+            self.logger: mlrun.utils.Logger = mlrun.utils.logger
 
         self._server = server
         self.current_function = None
@@ -516,7 +530,7 @@ class GraphContext:
         return self._server
 
     @property
-    def project(self):
+    def project(self) -> str:
         """current project name (for the current function)"""
         project, _, _, _ = mlrun.common.helpers.parse_versioned_object_uri(
             self._server.function_uri

mlrun/serving/states.py

@@ -84,6 +84,9 @@ _task_step_fields = [
 ]
 
 
+MAX_ALLOWED_STEPS = 4500
+
+
 def new_model_endpoint(class_name, model_path, handler=None, **class_args):
     class_args = deepcopy(class_args)
     class_args["model_path"] = model_path
@@ -733,6 +736,11 @@ class RouterStep(TaskStep):
         if not route:
             route = TaskStep(class_name, class_args, handler=handler)
         route.function = function or route.function
+
+        if len(self._routes) >= MAX_ALLOWED_STEPS:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Cannot create the serving graph: the maximum number of steps is {MAX_ALLOWED_STEPS}"
+            )
         route = self._routes.update(key, route)
         route.set_parent(self)
         return route

mlrun/serving/v2_serving.py

@@ -18,8 +18,9 @@ import traceback
 from typing import Optional, Union
 
 import mlrun.artifacts
-import mlrun.common.model_monitoring
+import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring
+import mlrun.model_monitoring
 from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
@@ -147,7 +148,7 @@ class V2ModelServer(StepToDict):
             logger.warn("GraphServer not initialized for VotingEnsemble instance")
             return
 
-        if not self.context.is_mock or self.context.server.track_models:
+        if not self.context.is_mock or self.context.monitoring_mock:
             self.model_endpoint_uid = _init_endpoint_record(
                 graph_server=server, model=self
             )
@@ -554,13 +555,13 @@ def _init_endpoint_record(
     except mlrun.errors.MLRunNotFoundError:
         model_ep = None
     except mlrun.errors.MLRunBadRequestError as err:
-        logger.debug(
-            f"Cant reach to model endpoints store, due to  : {err}",
+        logger.info(
+            "Cannot get the model endpoints store", err=mlrun.errors.err_to_str(err)
         )
         return
 
     if model.context.server.track_models and not model_ep:
-        logger.debug("Creating a new model endpoint record", endpoint_id=uid)
+        logger.info("Creating a new model endpoint record", endpoint_id=uid)
         model_endpoint = mlrun.common.schemas.ModelEndpoint(
             metadata=mlrun.common.schemas.ModelEndpointMetadata(
                 project=project, labels=model.labels, uid=uid
@@ -586,28 +587,35 @@ def _init_endpoint_record(
             model_endpoint=model_endpoint.dict(),
         )
 
-    elif (
-        model_ep
-        and (
+    elif model_ep:
+        attributes = {}
+        old_model_uri = model_ep.spec.model_uri
+        mlrun.model_monitoring.helpers.enrich_model_endpoint_with_model_uri(
+            model_endpoint=model_ep,
+            model_obj=model.model_spec,
+        )
+        if model_ep.spec.model_uri != old_model_uri:
+            attributes["model_uri"] = model_ep.spec.model_uri
+        if (
             model_ep.spec.monitoring_mode
             == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-        )
-        != model.context.server.track_models
-    ):
-        monitoring_mode = (
-            mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
-            if model.context.server.track_models
-            else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
-        )
-        db = mlrun.get_run_db()
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=uid,
-            attributes={"monitoring_mode": monitoring_mode},
-        )
-        logger.debug(
-            f"Updating model endpoint monitoring_mode to {monitoring_mode}",
-            endpoint_id=uid,
-        )
+        ) != model.context.server.track_models:
+            attributes["monitoring_mode"] = (
+                mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
+                if model.context.server.track_models
+                else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
+            )
+        if attributes:
+            db = mlrun.get_run_db()
+            db.patch_model_endpoint(
+                project=project,
+                endpoint_id=uid,
+                attributes=attributes,
+            )
+            logger.info(
+                "Updating model endpoint attributes",
+                attributes=attributes,
+                endpoint_id=uid,
+            )
 
     return uid