mlrun 1.7.0rc25__py3-none-any.whl → 1.7.0rc29__py3-none-any.whl

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -256,7 +256,6 @@ class KVStoreBase(StoreBase):
                     self.project,
                     function,
                     model,
-                    labels,
                     top_level,
                 ),
                 raise_for_status=v3io.dataplane.RaiseForStatus.never,
@@ -269,7 +268,6 @@ class KVStoreBase(StoreBase):
                 exc=mlrun.errors.err_to_str(exc),
             )
             return endpoint_list
-
         # Create a list of model endpoints unique ids
         if uids is None:
             uids = []
@@ -282,10 +280,16 @@ class KVStoreBase(StoreBase):
 
         # Add each relevant model endpoint to the model endpoints list
         for endpoint_id in uids:
-            endpoint = self.get_model_endpoint(
+            endpoint_dict = self.get_model_endpoint(
                 endpoint_id=endpoint_id,
             )
-            endpoint_list.append(endpoint)
+
+            if labels and not self._validate_labels(
+                endpoint_dict=endpoint_dict, labels=labels
+            ):
+                continue
+
+            endpoint_list.append(endpoint_dict)
 
         return endpoint_list
 
@@ -509,20 +513,16 @@ class KVStoreBase(StoreBase):
         project: str,
         function: str = None,
         model: str = None,
-        labels: list[str] = None,
         top_level: bool = False,
     ) -> str:
         """
         Convert the provided filters into a valid filter expression. The expected filter expression includes different
         conditions, divided by ' AND '.
 
-        :param project:    The name of the project.
-        :param model:      The name of the model to filter by.
-        :param function:   The name of the function to filter by.
-        :param labels:     A list of labels to filter by. Label filters work by either filtering a specific value of
-                           a label (i.e. list("key=value")) or by looking for the existence of a given
-                           key (i.e. "key").
-        :param top_level:  If True will return only routers and endpoint that are NOT children of any router.
+        :param project:         The name of the project.
+        :param model:           The name of the model to filter by.
+        :param function:        The name of the function to filter by.
+        :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
 
         :return: A valid filter expression as a string.
 
@@ -533,25 +533,17 @@ class KVStoreBase(StoreBase):
             raise mlrun.errors.MLRunInvalidArgumentError("project can't be empty")
 
         # Add project filter
-        filter_expression = [f"project=='{project}'"]
+        filter_expression = [f"{mm_schemas.EventFieldType.PROJECT}=='{project}'"]
 
         # Add function and model filters
         if function:
-            filter_expression.append(f"function=='{function}'")
+            function_uri = f"{project}/{function}" if function else None
+            filter_expression.append(
+                f"{mm_schemas.EventFieldType.FUNCTION_URI}=='{function_uri}'"
+            )
         if model:
-            filter_expression.append(f"model=='{model}'")
-
-        # Add labels filters
-        if labels:
-            for label in labels:
-                if not label.startswith("_"):
-                    label = f"_{label}"
-
-                if "=" in label:
-                    lbl, value = list(map(lambda x: x.strip(), label.split("=")))
-                    filter_expression.append(f"{lbl}=='{value}'")
-                else:
-                    filter_expression.append(f"exists({label})")
+            model = model if ":" in model else f"{model}:latest"
+            filter_expression.append(f"{mm_schemas.EventFieldType.MODEL}=='{model}'")
 
         # Apply top_level filter (remove endpoints that considered a child of a router)
         if top_level:

mlrun/model_monitoring/db/tsdb/__init__.py

@@ -57,41 +57,46 @@ class ObjectTSDBFactory(enum.Enum):
         :param value: Provided enum (invalid) value.
         """
         valid_values = list(cls.__members__.keys())
-        raise mlrun.errors.MLRunInvalidArgumentError(
+        raise mlrun.errors.MLRunInvalidMMStoreType(
             f"{value} is not a valid tsdb, please choose a valid value: %{valid_values}."
         )
 
 
 def get_tsdb_connector(
     project: str,
-    tsdb_connector_type: str = "",
     secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+    tsdb_connection_string: typing.Optional[str] = None,
     **kwargs,
 ) -> TSDBConnector:
     """
     Get TSDB connector object.
-    :param project: The name of the project.
-    :param tsdb_connector_type: The type of the TSDB connector. See mlrun.model_monitoring.db.tsdb.ObjectTSDBFactory
-                                for available options.
-    :param secret_provider: An optional secret provider to get the connection string secret.
+    :param project:                 The name of the project.
+    :param secret_provider:         An optional secret provider to get the connection string secret.
+    :param tsdb_connection_string:  An optional explicit connection string to the TSDB.
 
     :return: `TSDBConnector` object. The main goal of this object is to handle different operations on the
              TSDB connector such as updating drift metrics or write application record result.
+    :raise: `MLRunInvalidMMStoreType` if the user didn't provide TSDB connection
+            or the provided TSDB connection is invalid.
     """
 
-    tsdb_connection_string = mlrun.model_monitoring.helpers.get_tsdb_connection_string(
-        secret_provider=secret_provider
+    tsdb_connection_string = (
+        tsdb_connection_string
+        or mlrun.model_monitoring.helpers.get_tsdb_connection_string(
+            secret_provider=secret_provider
+        )
     )
 
     if tsdb_connection_string and tsdb_connection_string.startswith("taosws"):
         tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.TDEngine
         kwargs["connection_string"] = tsdb_connection_string
-
-    # Set the default TSDB connector type if no connection has been set
-    tsdb_connector_type = (
-        tsdb_connector_type
-        or mlrun.mlconf.model_endpoint_monitoring.tsdb_connector_type
-    )
+    elif tsdb_connection_string and tsdb_connection_string == "v3io":
+        tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.V3IO_TSDB
+    else:
+        raise mlrun.errors.MLRunInvalidMMStoreType(
+            "You must provide a valid tsdb store connection by using "
+            "set_model_monitoring_credentials API."
+        )
 
     # Get connector type value from ObjectTSDBFactory enum class
     tsdb_connector_factory = ObjectTSDBFactory(tsdb_connector_type)

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -38,7 +38,10 @@ def _is_no_schema_error(exc: v3io_frames.ReadError) -> bool:
     In case of a nonexistent TSDB table - a `v3io_frames.ReadError` error is raised.
     Check if the error message contains the relevant string to verify the cause.
     """
-    return "No TSDB schema file found" in str(exc)
+    msg = str(exc)
+    # https://github.com/v3io/v3io-tsdb/blob/v0.14.1/pkg/tsdb/v3iotsdb.go#L205
+    # https://github.com/v3io/v3io-tsdb/blob/v0.14.1/pkg/partmgr/partmgr.go#L238
+    return "No TSDB schema file found" in msg or "Failed to read schema at path" in msg
 
 
 class V3IOTSDBConnector(TSDBConnector):
@@ -596,7 +599,6 @@ class V3IOTSDBConnector(TSDBConnector):
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],
             filter_query=f"endpoint_id=='{endpoint_id}'",
-            interval=aggregation_window,
             agg_funcs=agg_funcs,
             sliding_window_step=aggregation_window,
         )

mlrun/model_monitoring/helpers.py

@@ -59,13 +59,17 @@ def get_stream_path(
 
     stream_uri = mlrun.get_secret_or_env(
         mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
-    ) or mlrun.mlconf.get_model_monitoring_file_target_path(
-        project=project,
-        kind=mlrun.common.schemas.model_monitoring.FileTargetKind.STREAM,
-        target="online",
-        function_name=function_name,
     )
 
+    if not stream_uri or stream_uri == "v3io":
+        # TODO : remove the first part of this condition in 1.9.0
+        stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=project,
+            kind=mlrun.common.schemas.model_monitoring.FileTargetKind.STREAM,
+            target="online",
+            function_name=function_name,
+        )
+
     if isinstance(stream_uri, list):  # ML-6043 - user side gets only the new stream uri
         stream_uri = stream_uri[1]  # get new stream path, under projects
     return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(

mlrun/model_monitoring/writer.py

@@ -153,11 +153,7 @@ class ModelMonitoringWriter(StepToDict):
         result_kind: int, result_status: int
     ) -> alert_objects.EventKind:
         """Generate the required Event Kind format for the alerting system"""
-        if result_kind == ResultKindApp.custom.value:
-            # Custom kind is represented as an anomaly detection
-            event_kind = "mm_app_anomaly"
-        else:
-            event_kind = ResultKindApp(value=result_kind).name
+        event_kind = ResultKindApp(value=result_kind).name
 
         if result_status == ResultStatusApp.detected.value:
             event_kind = f"{event_kind}_detected"

mlrun/projects/operations.py

@@ -330,6 +330,7 @@ def build_function(
                 commands=commands,
                 secret=secret_name,
                 requirements=requirements,
+                requirements_file=requirements_file,
                 overwrite=overwrite_build_params,
                 extra_args=extra_args,
             )

mlrun/projects/project.py

@@ -714,7 +714,8 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
     name_from_struct = struct.get("metadata", {}).get("name", "")
     if name and name_from_struct and name_from_struct != name:
         error_message = (
-            f"project name mismatch, {name_from_struct} != {name}, please do one of the following:\n"
+            f"Project name mismatch, {name_from_struct} != {name}, project is loaded from {name_from_struct} "
+            f"project yaml. To prevent/allow this, you can take one of the following actions:\n"
             "1. Set the `allow_cross_project=True` when loading the project.\n"
             f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
             "3. Use different project context dir."
@@ -722,14 +723,14 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
 
         if allow_cross_project is None:
             # TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
-            logger.warn(
-                "Project name is different than specified on its project yaml."
-                "You should fix it until version 1.9.0",
-                description=error_message,
+            warnings.warn(
+                f"Project {name=} is different than specified on the context's project yaml. "
+                "This behavior is deprecated and will not be supported in version 1.9.0."
             )
+            logger.warn(error_message)
         elif allow_cross_project:
-            logger.warn(
-                "Project name is different than specified on its project yaml. Overriding.",
+            logger.debug(
+                "Project name is different than specified on the context's project yaml. Overriding.",
                 existing_name=name_from_struct,
                 overriding_name=name,
             )
@@ -1007,8 +1008,13 @@ class ProjectSpec(ModelObj):
                 key = artifact.key
                 artifact = artifact.to_dict()
             else:  # artifact is a dict
-                # imported artifacts don't have metadata,spec,status fields
-                key_field = "key" if _is_imported_artifact(artifact) else "metadata.key"
+                # imported/legacy artifacts don't have metadata,spec,status fields
+                key_field = (
+                    "key"
+                    if _is_imported_artifact(artifact)
+                    or mlrun.utils.is_legacy_artifact(artifact)
+                    else "metadata.key"
+                )
                 key = mlrun.utils.get_in(artifact, key_field, "")
                 if not key:
                     raise ValueError(f'artifacts "{key_field}" must be specified')
@@ -2127,6 +2133,7 @@ class MlrunProject(ModelObj):
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
         rebuild_images: bool = False,
+        fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         """
         Deploy model monitoring application controller, writer and stream functions.
@@ -2136,17 +2143,18 @@ class MlrunProject(ModelObj):
         The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
         is detected. It processes the new events into statistics that are then written to statistics databases.
 
-        :param default_controller_image:        Deprecated.
-        :param base_period:                     The time period in minutes in which the model monitoring controller
-                                                function is triggered. By default, the base period is 10 minutes.
-        :param image:                           The image of the model monitoring controller, writer, monitoring
-                                                stream & histogram data drift functions, which are real time nuclio
-                                                functions. By default, the image is mlrun/mlrun.
-        :param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
-        :param wait_for_deployment:             If true, return only after the deployment is done on the backend.
-                                                Otherwise, deploy the model monitoring infrastructure on the
-                                                background, including the histogram data drift app if selected.
-        :param rebuild_images:                  If true, force rebuild of model monitoring infrastructure images.
+        :param default_controller_image:          Deprecated.
+        :param base_period:                       The time period in minutes in which the model monitoring controller
+                                                  function is triggered. By default, the base period is 10 minutes.
+        :param image:                             The image of the model monitoring controller, writer, monitoring
+                                                  stream & histogram data drift functions, which are real time nuclio
+                                                  functions. By default, the image is mlrun/mlrun.
+        :param deploy_histogram_data_drift_app:   If true, deploy the default histogram-based data drift application.
+        :param wait_for_deployment:               If true, return only after the deployment is done on the backend.
+                                                  Otherwise, deploy the model monitoring infrastructure on the
+                                                  background, including the histogram data drift app if selected.
+        :param rebuild_images:                    If true, force rebuild of model monitoring infrastructure images.
+        :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
         """
         if default_controller_image != "mlrun/mlrun":
             # TODO: Remove this in 1.9.0
@@ -2163,6 +2171,7 @@ class MlrunProject(ModelObj):
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
             rebuild_images=rebuild_images,
+            fetch_credentials_from_sys_config=fetch_credentials_from_sys_config,
         )
 
         if wait_for_deployment:
@@ -2337,7 +2346,8 @@ class MlrunProject(ModelObj):
                                     Default: job
         :param image:               Docker image to be used, can also be specified in the function object/yaml
         :param handler:             Default function handler to invoke (can only be set with .py/.ipynb files)
-        :param with_repo:           Add (clone) the current repo to the build source
+        :param with_repo:           Add (clone) the current repo to the build source - use when the function code is in
+                                    the project repo (project.spec.source).
         :param tag:                 Function version tag to set (none for current or 'latest')
                                     Specifying a tag as a parameter will update the project's tagged function
                                     (myfunc:v1) and the untagged function (myfunc)
@@ -2484,25 +2494,17 @@ class MlrunProject(ModelObj):
         self.spec.remove_function(name)
 
     def remove_model_monitoring_function(self, name: Union[str, list[str]]):
-        """remove the specified model-monitoring-app function/s from the project spec
+        """delete the specified model-monitoring-app function/s
 
         :param name: name of the model-monitoring-function/s (under the project)
         """
-        names = name if isinstance(name, list) else [name]
-        for func_name in names:
-            function = self.get_function(key=func_name)
-            if (
-                function.metadata.labels.get(mm_constants.ModelMonitoringAppLabel.KEY)
-                == mm_constants.ModelMonitoringAppLabel.VAL
-            ):
-                self.remove_function(name=func_name)
-                logger.info(
-                    f"{func_name} function has been removed from {self.name} project"
-                )
-            else:
-                raise logger.warn(
-                    f"There is no model monitoring function with {func_name} name"
-                )
+        # TODO: Remove this in 1.9.0
+        warnings.warn(
+            "'remove_model_monitoring_function' is deprecated and will be removed in 1.9.0. "
+            "Please use `delete_model_monitoring_function` instead.",
+            FutureWarning,
+        )
+        self.delete_model_monitoring_function(name)
 
     def delete_model_monitoring_function(self, name: Union[str, list[str]]):
         """delete the specified model-monitoring-app function/s
@@ -3204,49 +3206,44 @@ class MlrunProject(ModelObj):
         stream_path: Optional[str] = None,
         tsdb_connection: Optional[str] = None,
     ):
-        """Set the credentials that will be used by the project's model monitoring
+        """
+        Set the credentials that will be used by the project's model monitoring
         infrastructure functions. Important to note that you have to set the credentials before deploying any
         model monitoring or serving function.
 
-        :param access_key:                Model Monitoring access key for managing user permissions
-        :param endpoint_store_connection: Endpoint store connection string
-        :param stream_path:               Path to the model monitoring stream
-        :param tsdb_connection:           Connection string to the time series database
+        :param access_key:                Model Monitoring access key for managing user permissions.
+        :param endpoint_store_connection: Endpoint store connection string. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io endpoint store,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. MySQL/SQLite - for SQL endpoint store, please provide full
+                                             connection string, for example
+                                             mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
+        :param stream_path:               Path to the model monitoring stream. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io stream,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. Kafka - for Kafka stream, please provide full connection string without
+                                             custom topic, for example kafka://<some_kafka_broker>:<port>.
+        :param tsdb_connection:           Connection string to the time series database. By default, None.
+                                          Options:
+                                          1. None, will be set from the system configuration.
+                                          2. v3io - for v3io stream,
+                                             pass `v3io` and the system will generate the exact path.
+                                          3. TDEngine - for TDEngine tsdb, please provide full websocket connection URL,
+                                             for example taosws://<username>:<password>@<host>:<port>.
         """
-
-        secrets_dict = {}
-        if access_key:
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY
-            ] = access_key
-
-        if endpoint_store_connection:
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION
-            ] = endpoint_store_connection
-
-        if stream_path:
-            if stream_path.startswith("kafka://") and "?topic" in stream_path:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "Custom kafka topic is not allowed"
-                )
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.STREAM_PATH
-            ] = stream_path
-
-        if tsdb_connection:
-            if not tsdb_connection.startswith("taosws://"):
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "Currently only TDEngine websocket connection is supported for non-v3io TSDB,"
-                    "please provide a full URL (e.g. taosws://user:password@host:port)"
-                )
-            secrets_dict[
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.TSDB_CONNECTION
-            ] = tsdb_connection
-
-        self.set_secrets(
-            secrets=secrets_dict,
-            provider=mlrun.common.schemas.SecretProviderName.kubernetes,
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        db.set_model_monitoring_credentials(
+            project=self.name,
+            credentials={
+                "access_key": access_key,
+                "endpoint_store_connection": endpoint_store_connection,
+                "stream_path": stream_path,
+                "tsdb_connection": tsdb_connection,
+            },
         )
 
     def run_function(
@@ -3663,6 +3660,7 @@ class MlrunProject(ModelObj):
         kind: str = None,
         category: typing.Union[str, mlrun.common.schemas.ArtifactCategories] = None,
         tree: str = None,
+        limit: int = None,
     ) -> mlrun.lists.ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -3692,6 +3690,7 @@ class MlrunProject(ModelObj):
         :param kind: Return artifacts of the requested kind.
         :param category: Return artifacts of the requested category.
         :param tree: Return artifacts of the requested tree.
+        :param limit: Maximum number of artifacts to return.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_artifacts(
@@ -3706,6 +3705,7 @@ class MlrunProject(ModelObj):
             kind=kind,
             category=category,
             tree=tree,
+            limit=limit,
         )
 
     def list_models(

mlrun/render.py

@@ -283,9 +283,14 @@ function copyToClipboard(fld) {
 }
 function expandPanel(el) {
   const panelName = "#" + el.getAttribute('paneName');
-  console.log(el.title);
 
-  document.querySelector(panelName + "-title").innerHTML = el.title
+  // Get the base URL of the current notebook
+  var baseUrl = window.location.origin;
+
+  // Construct the full URL
+  var fullUrl = new URL(el.title, baseUrl).href;
+
+  document.querySelector(panelName + "-title").innerHTML = fullUrl
   iframe = document.querySelector(panelName + "-body");
 
   const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}
@@ -299,7 +304,7 @@ function expandPanel(el) {
   }
 
   function reqListener () {
-    if (el.title.endsWith(".csv")) {
+    if (fullUrl.endsWith(".csv")) {
       iframe.setAttribute("srcdoc", tblcss + csvToHtmlTable(this.responseText));
     } else {
       iframe.setAttribute("srcdoc", this.responseText);
@@ -309,11 +314,11 @@ function expandPanel(el) {
 
   const oReq = new XMLHttpRequest();
   oReq.addEventListener("load", reqListener);
-  oReq.open("GET", el.title);
+  oReq.open("GET", fullUrl);
   oReq.send();
 
 
-  //iframe.src = el.title;
+  //iframe.src = fullUrl;
   const resultPane = document.querySelector(panelName + "-pane");
   if (resultPane.classList.contains("hidden")) {
     resultPane.classList.remove("hidden");

mlrun/run.py

@@ -63,11 +63,11 @@ from .runtimes.funcdoc import update_function_entry_points
 from .runtimes.nuclio.application import ApplicationRuntime
 from .runtimes.utils import add_code_metadata, global_context
 from .utils import (
+    RunKeys,
     extend_hub_uri_if_needed,
     get_in,
     logger,
     retry_until_successful,
-    run_keys,
     update_in,
 )
 
@@ -280,7 +280,7 @@ def get_or_create_ctx(
             artifact_path = mlrun.utils.helpers.template_artifact_path(
                 mlconf.artifact_path, project or mlconf.default_project
             )
-            update_in(newspec, ["spec", run_keys.output_path], artifact_path)
+            update_in(newspec, ["spec", RunKeys.output_path], artifact_path)
 
     newspec.setdefault("metadata", {})
     update_in(newspec, "metadata.name", name, replace=False)

mlrun/runtimes/daskjob.py

@@ -548,7 +548,13 @@ class DaskCluster(KubejobRuntime):
                     "specified handler (string) without command "
                     "(py file path), specify command or use handler pointer"
                 )
-            handler = load_module(self.spec.command, handler, context=context)
+            # Do not embed the module in system as it is not persistent with the dask cluster
+            handler = load_module(
+                self.spec.command,
+                handler,
+                context=context,
+                embed_in_sys=False,
+            )
         client = self.client
         setattr(context, "dask_client", client)
         sout, serr = exec_from_params(handler, runobj, context)

mlrun/runtimes/local.py

@@ -58,7 +58,9 @@ class ParallelRunner:
 
         return TrackerManager()
 
-    def _get_handler(self, handler, context):
+    def _get_handler(
+        self, handler: str, context: MLClientCtx, embed_in_sys: bool = True
+    ):
         return handler
 
     def _get_dask_client(self, options):
@@ -86,7 +88,7 @@ class ParallelRunner:
         handler = runobj.spec.handler
         self._force_handler(handler)
         set_paths(self.spec.pythonpath)
-        handler = self._get_handler(handler, execution)
+        handler = self._get_handler(handler, execution, embed_in_sys=False)
 
         client, function_name = self._get_dask_client(generator.options)
         parallel_runs = generator.options.parallel_runs or 4
@@ -224,12 +226,14 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
     def is_deployed(self):
         return True
 
-    def _get_handler(self, handler, context):
+    def _get_handler(
+        self, handler: str, context: MLClientCtx, embed_in_sys: bool = True
+    ):
         command = self.spec.command
         if not command and self.spec.build.functionSourceCode:
             # if the code is embedded in the function object extract or find it
             command, _ = mlrun.run.load_func_code(self)
-        return load_module(command, handler, context)
+        return load_module(command, handler, context, embed_in_sys=embed_in_sys)
 
     def _pre_run(self, runobj: RunObject, execution: MLClientCtx):
         workdir = self.spec.workdir
@@ -372,8 +376,20 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
             return run_obj_dict
 
 
-def load_module(file_name, handler, context):
-    """Load module from file name"""
+def load_module(
+    file_name: str,
+    handler: str,
+    context: MLClientCtx,
+    embed_in_sys: bool = True,
+):
+    """
+    Load module from filename
+    :param file_name:       The module path to load
+    :param handler:         The callable to load
+    :param context:         Execution context
+    :param embed_in_sys:    Embed the file-named module in sys.modules. This is not persistent with remote
+                            environments and therefore can effect pickling.
+    """
     module = None
     if file_name:
         path = Path(file_name)
@@ -384,7 +400,8 @@ def load_module(file_name, handler, context):
         if spec is None:
             raise RunError(f"Cannot import from {file_name!r}")
         module = imputil.module_from_spec(spec)
-        sys.modules[mod_name] = module
+        if embed_in_sys:
+            sys.modules[mod_name] = module
         spec.loader.exec_module(module)
 
     class_args = {}

mlrun/runtimes/nuclio/function.py

@@ -1327,3 +1327,23 @@ def get_nuclio_deploy_status(
     else:
         text = "\n".join(outputs) if outputs else ""
         return state, address, name, last_log_timestamp, text, function_status
+
+
+def enrich_nuclio_function_from_headers(
+    func: RemoteRuntime,
+    headers: dict,
+):
+    func.status.state = headers.get("x-mlrun-function-status", "")
+    func.status.address = headers.get("x-mlrun-address", "")
+    func.status.nuclio_name = headers.get("x-mlrun-name", "")
+    func.status.internal_invocation_urls = (
+        headers.get("x-mlrun-internal-invocation-urls", "").split(",")
+        if headers.get("x-mlrun-internal-invocation-urls")
+        else []
+    )
+    func.status.external_invocation_urls = (
+        headers.get("x-mlrun-external-invocation-urls", "").split(",")
+        if headers.get("x-mlrun-external-invocation-urls")
+        else []
+    )
+    func.status.container_image = headers.get("x-mlrun-container-image", "")