mlrun 1.7.0rc6__py3-none-any.whl → 1.7.0rc9__py3-none-any.whl

mlrun/runtimes/nuclio/application/application.py

@@ -223,7 +223,42 @@ class ApplicationRuntime(RemoteRuntime):
         auth_info: AuthInfo = None,
         builder_env: dict = None,
         force_build: bool = False,
+        with_mlrun=None,
+        skip_deployed=False,
+        is_kfp=False,
+        mlrun_version_specifier=None,
+        show_on_failure: bool = False,
     ):
+        """
+        Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
+        Once the image is built, the function is deployed.
+        :param project:                 Project name
+        :param tag:                     Function tag
+        :param verbose:                 Set True for verbose logging
+        :param auth_info:               Service AuthInfo (deprecated and ignored)
+        :param builder_env:             Env vars dict for source archive config/credentials
+                                        e.g. builder_env={"GIT_TOKEN": token}
+        :param force_build:             Set True for force building the application image
+        :param with_mlrun:              Add the current mlrun package to the container build
+        :param skip_deployed:           Skip the build if we already have an image for the function
+        :param is_kfp:                  Deploy as part of a kfp pipeline
+        :param mlrun_version_specifier: Which mlrun package version to include (if not current)
+        :param show_on_failure:         Show logs only in case of build failure
+        :return: True if the function is ready (deployed)
+        """
+        if self.requires_build() or force_build:
+            self._fill_credentials()
+            self._build_application_image(
+                builder_env=builder_env,
+                force_build=force_build,
+                watch=True,
+                with_mlrun=with_mlrun,
+                skip_deployed=skip_deployed,
+                is_kfp=is_kfp,
+                mlrun_version_specifier=mlrun_version_specifier,
+                show_on_failure=show_on_failure,
+            )
+
         self._ensure_reverse_proxy_configurations()
         self._configure_application_sidecar()
         super().deploy(
@@ -232,7 +267,50 @@ class ApplicationRuntime(RemoteRuntime):
             verbose,
             auth_info,
             builder_env,
-            force_build,
+        )
+
+    def with_source_archive(
+        self, source, workdir=None, pull_at_runtime=True, target_dir=None
+    ):
+        """load the code from git/tar/zip archive at runtime or build
+
+        :param source:          valid absolute path or URL to git, zip, or tar file, e.g.
+                                git://github.com/mlrun/something.git
+                                http://some/url/file.zip
+                                note path source must exist on the image or exist locally when run is local
+                                (it is recommended to use 'workdir' when source is a filepath instead)
+        :param workdir:         working dir relative to the archive root (e.g. './subdir') or absolute to the image root
+        :param pull_at_runtime: load the archive into the container at job runtime vs on build/deploy
+        :param target_dir:      target dir on runtime pod or repo clone / archive extraction
+        """
+        self._configure_mlrun_build_with_source(
+            source=source,
+            workdir=workdir,
+            pull_at_runtime=pull_at_runtime,
+            target_dir=target_dir,
+        )
+
+    def _build_application_image(
+        self,
+        builder_env: dict = None,
+        force_build: bool = False,
+        watch=True,
+        with_mlrun=None,
+        skip_deployed=False,
+        is_kfp=False,
+        mlrun_version_specifier=None,
+        show_on_failure: bool = False,
+    ):
+        with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
+        return self._build_image(
+            builder_env=builder_env,
+            force_build=force_build,
+            mlrun_version_specifier=mlrun_version_specifier,
+            show_on_failure=show_on_failure,
+            skip_deployed=skip_deployed,
+            watch=watch,
+            is_kfp=is_kfp,
+            with_mlrun=with_mlrun,
         )
 
     def _ensure_reverse_proxy_configurations(self):

mlrun/runtimes/nuclio/application/reverse_proxy.go

@@ -39,11 +39,19 @@ func Handler(context *nuclio.Context, event nuclio.Event) (interface{}, error) {
     for k, v := range event.GetHeaders() {
         httpRequest.Header[k] = []string{v.(string)}
     }
+
+    // populate query params
+    query := httpRequest.URL.Query()
+    for k, v := range event.GetFields() {
+        query.Set(k, v.(string))
+    }
+    httpRequest.URL.RawQuery = query.Encode()
+
     recorder := httptest.NewRecorder()
     reverseProxy.ServeHTTP(recorder, httpRequest)
 
     // send request to sidecar
-    context.Logger.InfoWith("Forwarding request to sidecar", "sidecarUrl", sidecarUrl)
+    context.Logger.DebugWith("Forwarding request to sidecar", "sidecarUrl", sidecarUrl, "query", httpRequest.URL.Query())
     response := recorder.Result()
 
     headers := make(map[string]interface{})

mlrun/runtimes/nuclio/function.py

@@ -543,11 +543,16 @@ class RemoteRuntime(KubeResource):
         :param project:    project name
         :param tag:        function tag
         :param verbose:    set True for verbose logging
-        :param auth_info:  service AuthInfo
+        :param auth_info:  service AuthInfo (deprecated and ignored)
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
         :param force_build: set True for force building the image
         """
-        # todo: verify that the function name is normalized
+        if auth_info:
+            # TODO: remove in 1.9.0
+            warnings.warn(
+                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.9.0",
+                FutureWarning,
+            )
 
         old_http_session = getattr(self, "_http_session", None)
         if old_http_session:
@@ -570,9 +575,7 @@ class RemoteRuntime(KubeResource):
         self._fill_credentials()
         db = self._get_db()
         logger.info("Starting remote function deploy")
-        data = db.remote_builder(
-            self, False, builder_env=builder_env, force_build=force_build
-        )
+        data = db.deploy_nuclio_function(func=self, builder_env=builder_env)
         self.status = data["data"].get("status")
         self._update_credentials_from_remote_build(data["data"])
 
@@ -613,7 +616,7 @@ class RemoteRuntime(KubeResource):
                 int(mlrun.mlconf.httpdb.logs.nuclio.pull_deploy_status_default_interval)
             )
             try:
-                text, last_log_timestamp = db.get_builder_status(
+                text, last_log_timestamp = db.get_nuclio_deploy_status(
                     self, last_log_timestamp=last_log_timestamp, verbose=verbose
                 )
             except mlrun.db.RunDBError:
@@ -775,6 +778,9 @@ class RemoteRuntime(KubeResource):
             ] = self.metadata.credentials.access_key
         return runtime_env
 
+    def _get_serving_spec(self):
+        return None
+
     def _get_nuclio_config_spec_env(self):
         env_dict = {}
         external_source_env_dict = {}
@@ -992,10 +998,10 @@ class RemoteRuntime(KubeResource):
         ]
 
         if command:
-            sidecar["command"] = command
+            sidecar["command"] = mlrun.utils.helpers.as_list(command)
 
         if args:
-            sidecar["args"] = args
+            sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
     def _set_sidecar(self, name: str) -> dict:
         self.spec.config.setdefault("spec.sidecars", [])

mlrun/runtimes/nuclio/serving.py

@@ -14,8 +14,9 @@
 
 import json
 import os
+import warnings
 from copy import deepcopy
-from typing import Union
+from typing import TYPE_CHECKING, Optional, Union
 
 import nuclio
 from nuclio import KafkaTrigger
@@ -24,7 +25,6 @@ import mlrun
 import mlrun.common.schemas
 from mlrun.datastore import parse_kafka_url
 from mlrun.model import ObjectList
-from mlrun.model_monitoring.tracking_policy import TrackingPolicy
 from mlrun.runtimes.function_reference import FunctionReference
 from mlrun.secrets import SecretsStore
 from mlrun.serving.server import GraphServer, create_graph_server
@@ -43,6 +43,10 @@ from .function import NuclioSpec, RemoteRuntime
 
 serving_subkind = "serving_v2"
 
+if TYPE_CHECKING:
+    # remove this block in 1.9.0
+    from mlrun.model_monitoring import TrackingPolicy
+
 
 def new_v2_model_server(
     name,
@@ -291,7 +295,9 @@ class ServingRuntime(RemoteRuntime):
                         "provided class is not a router step, must provide a router class in router topology"
                     )
             else:
-                step = RouterStep(class_name=class_name, class_args=class_args)
+                step = RouterStep(
+                    class_name=class_name, class_args=class_args, engine=engine
+                )
             self.spec.graph = step
         elif topology == StepKinds.flow:
             self.spec.graph = RootFlowStep(engine=engine)
@@ -303,12 +309,12 @@ class ServingRuntime(RemoteRuntime):
 
     def set_tracking(
         self,
-        stream_path: str = None,
-        batch: int = None,
-        sample: int = None,
-        stream_args: dict = None,
-        tracking_policy: Union[TrackingPolicy, dict] = None,
-    ):
+        stream_path: Optional[str] = None,
+        batch: Optional[int] = None,
+        sample: Optional[int] = None,
+        stream_args: Optional[dict] = None,
+        tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
+    ) -> None:
         """apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
            and analyze performance.
 
@@ -317,31 +323,17 @@ class ServingRuntime(RemoteRuntime):
         :param batch:           Micro batch size (send micro batches of N records at a time).
         :param sample:          Sample size (send only one of N records).
         :param stream_args:     Stream initialization parameters, e.g. shards, retention_in_hours, ..
-        :param tracking_policy: Tracking policy object or a dictionary that will be converted into a tracking policy
-                                object. By using TrackingPolicy, the user can apply his model monitoring requirements,
-                                such as setting the scheduling policy of the model monitoring batch job or changing
-                                the image of the model monitoring stream.
 
                                 example::
 
                                     # initialize a new serving function
                                     serving_fn = mlrun.import_function("hub://v2-model-server", new_name="serving")
-                                    # apply model monitoring and set monitoring batch job to run every 3 hours
-                                    tracking_policy = {'default_batch_intervals':"0 */3 * * *"}
-                                    serving_fn.set_tracking(tracking_policy=tracking_policy)
+                                    # apply model monitoring
+                                    serving_fn.set_tracking()
 
         """
-
         # Applying model monitoring configurations
         self.spec.track_models = True
-        self.spec.tracking_policy = None
-        if tracking_policy:
-            if isinstance(tracking_policy, dict):
-                # Convert tracking policy dictionary into `model_monitoring.TrackingPolicy` object
-                self.spec.tracking_policy = TrackingPolicy.from_dict(tracking_policy)
-            else:
-                # Tracking_policy is already a `model_monitoring.TrackingPolicy` object
-                self.spec.tracking_policy = tracking_policy
 
         if stream_path:
             self.spec.parameters["log_stream"] = stream_path
@@ -351,6 +343,14 @@ class ServingRuntime(RemoteRuntime):
             self.spec.parameters["log_stream_sample"] = sample
         if stream_args:
             self.spec.parameters["stream_args"] = stream_args
+        if tracking_policy is not None:
+            warnings.warn(
+                "The `tracking_policy` argument is deprecated from version 1.7.0 "
+                "and has no effect. It will be removed in 1.9.0.\n"
+                "To set the desired model monitoring time window and schedule, use "
+                "the `base_period` argument in `project.enable_model_monitoring()`.",
+                FutureWarning,
+            )
 
     def add_model(
         self,
@@ -491,9 +491,9 @@ class ServingRuntime(RemoteRuntime):
 
                 if (
                     stream.path.startswith("kafka://")
-                    or "kafka_bootstrap_servers" in stream.options
+                    or "kafka_brokers" in stream.options
                 ):
-                    brokers = stream.options.get("kafka_bootstrap_servers")
+                    brokers = stream.options.get("kafka_brokers")
                     if brokers:
                         brokers = brokers.split(",")
                     topic, brokers = parse_kafka_url(stream.path, brokers)
@@ -644,8 +644,7 @@ class ServingRuntime(RemoteRuntime):
             force_build=force_build,
         )
 
-    def _get_runtime_env(self):
-        env = super()._get_runtime_env()
+    def _get_serving_spec(self):
         function_name_uri_map = {f.name: f.uri(self) for f in self.spec.function_refs}
 
         serving_spec = {
@@ -658,9 +657,7 @@ class ServingRuntime(RemoteRuntime):
             "graph_initializer": self.spec.graph_initializer,
             "error_stream": self.spec.error_stream,
             "track_models": self.spec.track_models,
-            "tracking_policy": self.spec.tracking_policy.to_dict()
-            if self.spec.tracking_policy
-            else None,
+            "tracking_policy": None,
             "default_content_type": self.spec.default_content_type,
         }
 
@@ -668,8 +665,7 @@ class ServingRuntime(RemoteRuntime):
             self._secrets = SecretsStore.from_list(self.spec.secret_sources)
             serving_spec["secret_sources"] = self._secrets.to_serial()
 
-        env["SERVING_SPEC_ENV"] = json.dumps(serving_spec)
-        return env
+        return json.dumps(serving_spec)
 
     def to_mock_server(
         self,

mlrun/runtimes/pod.py

@@ -15,6 +15,7 @@ import copy
 import inspect
 import os
 import re
+import time
 import typing
 from enum import Enum
 
@@ -1057,6 +1058,32 @@ class KubeResource(BaseRuntime):
                 return True
         return False
 
+    def enrich_runtime_spec(
+        self,
+        project_node_selector: dict[str, str],
+    ):
+        """
+        Enriches the runtime spec with the project-level node selector.
+
+        This method merges the project-level node selector with the existing function node_selector.
+        The merge logic used here combines the two dictionaries, giving precedence to
+        the keys in the runtime node_selector. If there are conflicting keys between the
+        two dictionaries, the values from self.spec.node_selector will overwrite the
+        values from project_node_selector.
+
+        Example:
+        Suppose self.spec.node_selector = {"type": "gpu", "zone": "us-east-1"}
+        and project_node_selector = {"type": "cpu", "environment": "production"}.
+        After the merge, the resulting node_selector will be:
+        {"type": "gpu", "zone": "us-east-1", "environment": "production"}
+
+        Note:
+        - The merge uses the ** operator, also known as the "unpacking" operator in Python,
+          combining key-value pairs from each dictionary. Later dictionaries take precedence
+          when there are conflicting keys.
+        """
+        self.spec.node_selector = {**project_node_selector, **self.spec.node_selector}
+
     def _set_env(self, name, value=None, value_from=None):
         new_var = k8s_client.V1EnvVar(name=name, value=value, value_from=value_from)
         i = 0
@@ -1312,6 +1339,150 @@ class KubeResource(BaseRuntime):
 
         self.spec.validate_service_account(allowed_service_accounts)
 
+    def _configure_mlrun_build_with_source(
+        self, source, workdir=None, handler=None, pull_at_runtime=True, target_dir=None
+    ):
+        mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
+
+        self.spec.build.source = source
+        if handler:
+            self.spec.default_handler = handler
+        if workdir:
+            self.spec.workdir = workdir
+        if target_dir:
+            self.spec.build.source_code_target_dir = target_dir
+
+        self.spec.build.load_source_on_run = pull_at_runtime
+        if (
+            self.spec.build.base_image
+            and not self.spec.build.commands
+            and pull_at_runtime
+            and not self.spec.image
+        ):
+            # if we load source from repo and don't need a full build use the base_image as the image
+            self.spec.image = self.spec.build.base_image
+        elif not pull_at_runtime:
+            # clear the image so build will not be skipped
+            self.spec.build.base_image = self.spec.build.base_image or self.spec.image
+            self.spec.image = ""
+
+    def _resolve_build_with_mlrun(self, with_mlrun: typing.Optional[bool] = None):
+        build = self.spec.build
+        if with_mlrun is None:
+            if build.with_mlrun is not None:
+                with_mlrun = build.with_mlrun
+            else:
+                with_mlrun = build.base_image and not (
+                    build.base_image.startswith("mlrun/")
+                    or "/mlrun/" in build.base_image
+                )
+        if (
+            not build.source
+            and not build.commands
+            and not build.requirements
+            and not build.extra
+            and with_mlrun
+        ):
+            logger.info(
+                "Running build to add mlrun package, set "
+                "with_mlrun=False to skip if its already in the image"
+            )
+        return with_mlrun
+
+    def _build_image(
+        self,
+        builder_env,
+        force_build,
+        mlrun_version_specifier,
+        show_on_failure,
+        skip_deployed,
+        watch,
+        is_kfp,
+        with_mlrun,
+    ):
+        # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
+        # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
+        if is_kfp:
+            watch = True
+
+        db = self._get_db()
+        data = db.remote_builder(
+            self,
+            with_mlrun,
+            mlrun_version_specifier,
+            skip_deployed,
+            builder_env=builder_env,
+            force_build=force_build,
+        )
+        self.status = data["data"].get("status", None)
+        self.spec.image = mlrun.utils.get_in(
+            data, "data.spec.image"
+        ) or mlrun.utils.get_in(data, "data.spec.build.image")
+        self.spec.build.base_image = self.spec.build.base_image or mlrun.utils.get_in(
+            data, "data.spec.build.base_image"
+        )
+        # Get the source target dir in case it was enriched due to loading source
+        self.spec.build.source_code_target_dir = mlrun.utils.get_in(
+            data, "data.spec.build.source_code_target_dir"
+        ) or mlrun.utils.get_in(data, "data.spec.clone_target_dir")
+        ready = data.get("ready", False)
+        if not ready:
+            logger.info(
+                f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
+            )
+        if watch and not ready:
+            state = self._build_watch(
+                watch=watch,
+                show_on_failure=show_on_failure,
+            )
+            ready = state == "ready"
+            self.status.state = state
+
+        if watch and not ready:
+            raise mlrun.errors.MLRunRuntimeError("Deploy failed")
+        return ready
+
+    def _build_watch(
+        self,
+        watch: bool = True,
+        logs: bool = True,
+        show_on_failure: bool = False,
+    ):
+        db = self._get_db()
+        offset = 0
+        try:
+            text, _ = db.get_builder_status(self, 0, logs=logs)
+        except mlrun.db.RunDBError:
+            raise ValueError("function or build process not found")
+
+        def print_log(text):
+            if text and (
+                not show_on_failure
+                or self.status.state == mlrun.common.schemas.FunctionState.error
+            ):
+                print(text, end="")
+
+        print_log(text)
+        offset += len(text)
+        if watch:
+            while self.status.state in [
+                mlrun.common.schemas.FunctionState.pending,
+                mlrun.common.schemas.FunctionState.running,
+            ]:
+                time.sleep(2)
+                if show_on_failure:
+                    text = ""
+                    db.get_builder_status(self, 0, logs=False)
+                    if self.status.state == mlrun.common.schemas.FunctionState.error:
+                        # re-read the full log on failure
+                        text, _ = db.get_builder_status(self, offset, logs=logs)
+                else:
+                    text, _ = db.get_builder_status(self, offset, logs=logs)
+                print_log(text)
+                offset += len(text)
+
+        return self.status.state
+
 
 def _resolve_if_type_sanitized(attribute_name, attribute):
     attribute_config = sanitized_attributes[attribute_name]

mlrun/runtimes/utils.py

@@ -417,34 +417,6 @@ def get_func_selector(project, name=None, tag=None):
     return s
 
 
-class k8s_resource:
-    kind = ""
-    per_run = False
-    per_function = False
-    k8client = None
-
-    def deploy_function(self, function):
-        pass
-
-    def release_function(self, function):
-        pass
-
-    def submit_run(self, function, runobj):
-        pass
-
-    def get_object(self, name, namespace=None):
-        return None
-
-    def get_status(self, name, namespace=None):
-        return None
-
-    def del_object(self, name, namespace=None):
-        pass
-
-    def get_pods(self, name, namespace=None, master=False):
-        return {}
-
-
 def enrich_function_from_dict(function, function_dict):
     override_function = mlrun.new_function(runtime=function_dict, kind=function.kind)
     for attribute in [

mlrun/serving/remote.py

@@ -172,8 +172,7 @@ class RemoteStep(storey.SendToHttp):
         if not self._session:
             self._session = mlrun.utils.HTTPSessionWithRetry(
                 self.retries,
-                self.backoff_factor
-                or mlrun.config.config.http_retry_defaults.backoff_factor,
+                self.backoff_factor or mlrun.mlconf.http_retry_defaults.backoff_factor,
                 retry_on_exception=False,
                 retry_on_status=self.retries > 0,
                 retry_on_post=True,
@@ -185,7 +184,7 @@ class RemoteStep(storey.SendToHttp):
             resp = self._session.request(
                 method,
                 url,
-                verify=mlrun.config.config.httpdb.http.verify,
+                verify=mlrun.mlconf.httpdb.http.verify,
                 headers=headers,
                 data=body,
                 timeout=self.timeout,

mlrun/serving/routers.py

@@ -28,6 +28,7 @@ import numpy as np
 import mlrun
 import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring
+from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
 from ..common.helpers import parse_versioned_object_uri
@@ -1013,7 +1014,7 @@ def _init_endpoint_record(
             graph_server.function_uri
         )
     except Exception as e:
-        logger.error("Failed to parse function URI", exc=e)
+        logger.error("Failed to parse function URI", exc=err_to_str(e))
         return None
 
     # Generating version model value based on the model name and model version
@@ -1089,12 +1090,12 @@ def _init_endpoint_record(
         except Exception as exc:
             logger.warning(
                 "Failed creating model endpoint record",
-                exc=exc,
+                exc=err_to_str(exc),
                 traceback=traceback.format_exc(),
             )
 
     except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=e)
+        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
 
     return endpoint_uid
 

mlrun/serving/server.py

@@ -23,6 +23,7 @@ import uuid
 from typing import Optional, Union
 
 import mlrun
+import mlrun.common.constants
 import mlrun.common.helpers
 import mlrun.model_monitoring
 from mlrun.config import config
@@ -52,7 +53,7 @@ class _StreamContext:
         Initialize _StreamContext object.
         :param enabled:      A boolean indication for applying the stream context
         :param parameters:   Dictionary of optional parameters, such as `log_stream` and `stream_args`. Note that these
-                             parameters might be relevant to the output source such as `kafka_bootstrap_servers` if
+                             parameters might be relevant to the output source such as `kafka_brokers` if
                              the output source is from type Kafka.
         :param function_uri: Full value of the function uri, usually it's <project-name>/<function-name>
         """
@@ -311,11 +312,8 @@ class GraphServer(ModelObj):
 def v2_serving_init(context, namespace=None):
     """hook for nuclio init_context()"""
 
-    data = os.environ.get("SERVING_SPEC_ENV", "")
-    if not data:
-        raise MLRunInvalidArgumentError("failed to find spec env var")
-    spec = json.loads(data)
     context.logger.info("Initializing server from spec")
+    spec = mlrun.utils.get_serving_spec()
     server = GraphServer.from_dict(spec)
     if config.log_level.lower() == "debug":
         server.verbose = True
@@ -355,7 +353,7 @@ def v2_serving_init(context, namespace=None):
 
         async def termination_callback():
             context.logger.info("Termination callback called")
-            await server.wait_for_completion()
+            server.wait_for_completion()
             context.logger.info("Termination of async flow is completed")
 
         context.platform.set_termination_callback(termination_callback)
@@ -367,7 +365,7 @@ def v2_serving_init(context, namespace=None):
 
         async def drain_callback():
             context.logger.info("Drain callback called")
-            await server.wait_for_completion()
+            server.wait_for_completion()
             context.logger.info(
                 "Termination of async flow is completed. Rerunning async flow."
             )