mlrun 1.8.0rc44__py3-none-any.whl → 1.8.0rc46__py3-none-any.whl

mlrun/model_monitoring/db/_schedules.py

@@ -13,52 +13,38 @@
 # limitations under the License.
 
 import json
+from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from types import TracebackType
 from typing import Final, Optional
 
 import botocore.exceptions
 
-import mlrun.common.schemas
+import mlrun.common.schemas as schemas
 import mlrun.errors
 import mlrun.model_monitoring.helpers
 from mlrun.utils import logger
 
 
-class ModelMonitoringSchedulesFile(AbstractContextManager):
+class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
     DEFAULT_SCHEDULES: Final = {}
     INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
     ENCODING = "utf-8"
 
-    def __init__(self, project: str, endpoint_id: str) -> None:
-        """
-        Initialize applications monitoring schedules file object.
-        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
-        When working with the schedules data, use this class as a context manager to read and write the data.
-
-        :param project:     The project name.
-        :param endpoint_id: The endpoint ID.
-        """
-        # `self._item` is the persistent version of the monitoring schedules.
-        self._item = mlrun.model_monitoring.helpers.get_monitoring_schedules_data(
-            project=project, endpoint_id=endpoint_id
-        )
-        self._path = self._item.url
-        self._fs = self._item.store.filesystem
-        # `self._schedules` is an in-memory copy of the DB for all the applications for
-        # the same model endpoint.
-        self._schedules: dict[str, int] = self.DEFAULT_SCHEDULES.copy()
-        # Does `self._schedules` hold the content of `self._item`?
-        self._open_schedules = False
-
-    @classmethod
-    def from_model_endpoint(
-        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
-    ) -> "ModelMonitoringSchedulesFile":
-        return cls(
-            project=model_endpoint.metadata.project,
-            endpoint_id=model_endpoint.metadata.uid,
-        )
+    def __init__(self):
+        self._item = self.get_data_item_object()
+        if self._item:
+            self._path = self._item.url
+            self._fs = self._item.store.filesystem
+            # `self._schedules` is an in-memory copy of the DB for all the applications for
+            # the same model endpoint.
+            self._schedules = self.DEFAULT_SCHEDULES.copy()
+            # Does `self._schedules` hold the content of `self._item`?
+            self._open_schedules = False
+
+    @abstractmethod
+    def get_data_item_object(self) -> mlrun.DataItem:
+        pass
 
     def create(self) -> None:
         """Create a schedules file with initial content - an empty dictionary"""
@@ -114,7 +100,7 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         self._schedules = self.DEFAULT_SCHEDULES
         self._open_schedules = False
 
-    def __enter__(self) -> "ModelMonitoringSchedulesFile":
+    def __enter__(self) -> "ModelMonitoringSchedulesFileBase":
         self._open()
         return super().__enter__()
 
@@ -132,6 +118,36 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
                 "Open the schedules file as a context manager first"
             )
 
+
+class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize applications monitoring schedules file object.
+        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        :param endpoint_id: The endpoint ID.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        self._endpoint_id = endpoint_id
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_endpoint_data(
+            project=self._project, endpoint_id=self._endpoint_id
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: schemas.ModelEndpoint
+    ) -> "ModelMonitoringSchedulesFileEndpoint":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
     def get_application_time(self, application: str) -> Optional[int]:
         self._check_open_schedules()
         return self._schedules.get(application)
@@ -149,6 +165,68 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         return min(self._schedules.values(), default=None)
 
 
+class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str) -> None:
+        """
+        Initialize applications monitoring schedules chief file object.
+        The JSON file stores a dictionary of registered model endpoints uid as key and point to a dictionary of
+        "last_request" and "last_analyzed" mapped to two Unix timestamps as values.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_chief_data(
+            project=self._project
+        )
+
+    def get_endpoint_last_request(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST
+            )
+        else:
+            return None
+
+    def update_endpoint_timestamps(
+        self, endpoint_uid: str, last_request: int, last_analyzed: int
+    ) -> None:
+        self._check_open_schedules()
+        self._schedules[endpoint_uid] = {
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST: last_request,
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED: last_analyzed,
+        }
+
+    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED
+            )
+        else:
+            return None
+
+    def get_endpoint_list(self) -> set[str]:
+        self._check_open_schedules()
+        return set(self._schedules.keys())
+
+    def get_or_create(self) -> None:
+        try:
+            self._open()
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ):
+            self.create()
+
+
 def delete_model_monitoring_schedules_folder(project: str) -> None:
     """Delete the model monitoring schedules folder of the project"""
     folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -33,7 +33,12 @@ _TSDB_BE = "tsdb"
 _TSDB_RATE = "1/s"
 _CONTAINER = "users"
 
-V3IO_MEPS_LIMIT = 200
+V3IO_FRAMESD_MEPS_LIMIT = (
+    200  # Maximum number of model endpoints per single request when using V3IO Frames
+)
+V3IO_CLIENT_MEPS_LIMIT = (
+    150  # Maximum number of model endpoints per single request when using V3IO Client
+)
 
 
 def _is_no_schema_error(exc: v3io_frames.Error) -> bool:
@@ -475,8 +480,8 @@ class V3IOTSDBConnector(TSDBConnector):
         tables = mm_schemas.V3IOTSDBTables.list()
 
         # Split the endpoint ids into chunks to avoid exceeding the v3io-engine filter-expression limit
-        for i in range(0, len(endpoint_ids), V3IO_MEPS_LIMIT):
-            endpoint_id_chunk = endpoint_ids[i : i + V3IO_MEPS_LIMIT]
+        for i in range(0, len(endpoint_ids), V3IO_FRAMESD_MEPS_LIMIT):
+            endpoint_id_chunk = endpoint_ids[i : i + V3IO_FRAMESD_MEPS_LIMIT]
             filter_query = f"endpoint_id IN({str(endpoint_id_chunk)[1:-1]}) "
             for table in tables:
                 try:
@@ -684,11 +689,11 @@ class V3IOTSDBConnector(TSDBConnector):
         if isinstance(endpoint_id, str):
             return f"endpoint_id=='{endpoint_id}'"
         elif isinstance(endpoint_id, list):
-            if len(endpoint_id) > V3IO_MEPS_LIMIT:
+            if len(endpoint_id) > V3IO_FRAMESD_MEPS_LIMIT:
                 logger.info(
                     "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
                     "retrieving all the model endpoints from the db.",
-                    limit=V3IO_MEPS_LIMIT,
+                    limit=V3IO_FRAMESD_MEPS_LIMIT,
                     amount=len(endpoint_id),
                 )
                 return None
@@ -880,23 +885,41 @@ class V3IOTSDBConnector(TSDBConnector):
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
     ) -> dict[str, float]:
-        if isinstance(endpoint_ids, str):
-            filter_expression = f"__name=='{endpoint_ids}'"
-        else:
-            filter_expression = " OR ".join(
-                [f"__name=='{endpoint_id}'" for endpoint_id in endpoint_ids]
-            )
-
         # Get the last request timestamp for each endpoint from the KV table.
         # The result of the query is a list of dictionaries,
         # each dictionary contains the endpoint id and the last request timestamp.
+        last_request_timestamps = {}
+        if isinstance(endpoint_ids, str):
+            endpoint_ids = [endpoint_ids]
 
         try:
-            res = self.v3io_client.kv.new_cursor(
-                container=self.container,
-                table_path=self.last_request_table,
-                filter_expression=filter_expression,
-            ).all()
+            if len(endpoint_ids) > V3IO_CLIENT_MEPS_LIMIT:
+                logger.warning(
+                    "The number of endpoint ids exceeds the v3io-engine filter-expression limit, "
+                    "retrieving last request for all the model endpoints from the KV table.",
+                    limit=V3IO_CLIENT_MEPS_LIMIT,
+                    amount=len(endpoint_ids),
+                )
+
+                res = self.v3io_client.kv.new_cursor(
+                    container=self.container,
+                    table_path=self.last_request_table,
+                ).all()
+                last_request_timestamps.update(
+                    {d["__name"]: d["last_request_timestamp"] for d in res}
+                )
+            else:
+                filter_expression = " OR ".join(
+                    [f"__name=='{endpoint_id}'" for endpoint_id in endpoint_ids]
+                )
+                res = self.v3io_client.kv.new_cursor(
+                    container=self.container,
+                    table_path=self.last_request_table,
+                    filter_expression=filter_expression,
+                ).all()
+                last_request_timestamps.update(
+                    {d["__name"]: d["last_request_timestamp"] for d in res}
+                )
         except Exception as e:
             logger.warning(
                 "Failed to get last request timestamp from V3IO KV table.",
@@ -904,9 +927,8 @@ class V3IOTSDBConnector(TSDBConnector):
                 project=self.project,
                 table=self.last_request_table,
             )
-            return {}
 
-        return {d["__name"]: d["last_request_timestamp"] for d in res}
+        return last_request_timestamps
 
     def get_drift_status(
         self,
@@ -1131,4 +1153,8 @@ class V3IOTSDBConnector(TSDBConnector):
             endpoint_ids=list(model_endpoint_objects_by_uid.keys())
         )
         for uid, mep in model_endpoint_objects_by_uid.items():
-            mep.status.last_request = last_request_dictionary.get(uid)
+            # Set the last request timestamp to the MEP object. If not found, keep the existing value from the
+            # DB (relevant for batch EP).
+            mep.status.last_request = last_request_dictionary.get(
+                uid, mep.status.last_request
+            )

mlrun/model_monitoring/helpers.py

@@ -472,6 +472,7 @@ def update_model_endpoint_last_request(
             "Bumping model endpoint last request time (EP without serving)",
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            function_name=model_endpoint.spec.function_name,
             last_request=model_endpoint.status.last_request,
             current_request=current_request.isoformat(),
             bumped_last_request=bumped_last_request,
@@ -586,16 +587,43 @@ def _get_monitoring_schedules_folder_path(project: str) -> str:
     )
 
 
-def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> str:
+def _get_monitoring_schedules_file_endpoint_path(
+    *, project: str, endpoint_id: str
+) -> str:
     return os.path.join(
         _get_monitoring_schedules_folder_path(project), f"{endpoint_id}.json"
     )
 
 
-def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
+def get_monitoring_schedules_endpoint_data(
+    *, project: str, endpoint_id: str
+) -> "DataItem":
+    """
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_endpoint_path(
+            project=project, endpoint_id=endpoint_id
+        )
+    )
+
+
+def get_monitoring_schedules_chief_data(
+    *,
+    project: str,
+) -> "DataItem":
     """
     Get the model monitoring schedules' data item of the project's model endpoint.
     """
     return mlrun.datastore.store_manager.object(
-        _get_monitoring_schedules_file_path(project=project, endpoint_id=endpoint_id)
+        _get_monitoring_schedules_file_chief_path(project=project)
+    )
+
+
+def _get_monitoring_schedules_file_chief_path(
+    *,
+    project: str,
+) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{project}.json"
     )

mlrun/model_monitoring/writer.py

@@ -180,7 +180,7 @@ class ModelMonitoringWriter(StepToDict):
                 data, timestamp
             )
         logger.info(
-            "Updating the model endpoint statistics",
+            "Updated the model endpoint statistics",
             endpoint_id=endpoint_id,
             stats_kind=stat_kind,
         )

mlrun/projects/project.py

@@ -2451,7 +2451,22 @@ class MlrunProject(ModelObj):
         :param image:                             The image of the model monitoring controller, writer, monitoring
                                                   stream & histogram data drift functions, which are real time nuclio
                                                   functions. By default, the image is mlrun/mlrun.
-        :param deploy_histogram_data_drift_app:   If true, deploy the default histogram-based data drift application.
+        :param deploy_histogram_data_drift_app:   If true, deploy the default histogram-based data drift application:
+            :py:class:`~mlrun.model_monitoring.applications.histogram_data_drift.HistogramDataDriftApplication`.
+            If false, and you want to deploy the histogram data drift application
+            afterwards, you may use the
+            :py:func:`~set_model_monitoring_function` method::
+
+                import mlrun.model_monitoring.applications.histogram_data_drift as histogram_data_drift
+
+                hist_app = project.set_model_monitoring_function(
+                    name=histogram_data_drift.HistogramDataDriftApplicationConstants.NAME,  # keep the default name
+                    func=histogram_data_drift.__file__,
+                    application_class=histogram_data_drift.HistogramDataDriftApplication.__name__,
+                )
+
+                project.deploy_function(hist_app)
+
         :param wait_for_deployment:               If true, return only after the deployment is done on the backend.
                                                   Otherwise, deploy the model monitoring infrastructure on the
                                                   background, including the histogram data drift app if selected.
@@ -2488,30 +2503,6 @@ class MlrunProject(ModelObj):
                 )
             self._wait_for_functions_deployment(deployment_functions)
 
-    def deploy_histogram_data_drift_app(
-        self,
-        *,
-        image: str = "mlrun/mlrun",
-        db: Optional[mlrun.db.RunDBInterface] = None,
-        wait_for_deployment: bool = False,
-    ) -> None:
-        """
-        Deploy the histogram data drift application.
-
-        :param image:               The image on which the application will run.
-        :param db:                  An optional DB object.
-        :param wait_for_deployment: If true, return only after the deployment is done on the backend.
-                                    Otherwise, deploy the application on the background.
-        """
-        if db is None:
-            db = mlrun.db.get_run_db(secrets=self._secrets)
-        db.deploy_histogram_data_drift_app(project=self.name, image=image)
-
-        if wait_for_deployment:
-            self._wait_for_functions_deployment(
-                [mm_constants.HistogramDataDriftApplicationConstants.NAME]
-            )
-
     def update_model_monitoring_controller(
         self,
         base_period: int = 10,
@@ -5034,14 +5025,20 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.get_alert_config(alert_name, self.metadata.name)
 
-    def list_alerts_configs(self) -> list[AlertConfig]:
+    def list_alerts_configs(
+        self, limit: Optional[int] = None, offset: Optional[int] = None
+    ) -> list[AlertConfig]:
         """
         Retrieve list of alerts of a project.
 
+        :param limit: The maximum number of alerts to return.
+            Defaults to `mlconf.alerts.default_list_alert_configs_limit` if not provided.
+        :param offset: The number of alerts to skip before starting to collect alerts.
+
         :return: All the alerts objects of the project.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
-        return db.list_alerts_configs(self.metadata.name)
+        return db.list_alerts_configs(self.metadata.name, limit=limit, offset=offset)
 
     def delete_alert_config(
         self, alert_data: AlertConfig = None, alert_name: Optional[str] = None
@@ -5269,7 +5266,7 @@ class MlrunProject(ModelObj):
             )
 
         # if engine is remote then skip the local file validation
-        if engine and not engine.startswith("remote"):
+        if engine and engine.startswith("remote"):
             return
 
         code_path = self.spec.get_code_path()

mlrun/runtimes/function_reference.py

@@ -36,6 +36,7 @@ class FunctionReference(ModelObj):
         spec=None,
         kind=None,
         name=None,
+        track_models=None,
     ):
         self.url = url
         self.kind = kind
@@ -46,6 +47,7 @@ class FunctionReference(ModelObj):
             spec = spec.to_dict()
         self.spec = spec
         self.code = code
+        self.track_models = track_models
 
         self._function = None
         self._address = None
@@ -130,6 +132,7 @@ class FunctionReference(ModelObj):
         if self.requirements:
             func.with_requirements(self.requirements)
         self._function = func
+        func.spec.track_models = self.track_models
         return func
 
     @property

mlrun/runtimes/nuclio/application/reverse_proxy.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//   http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,82 +14,84 @@
 package main
 
 import (
-    "bytes"
-    "fmt"
-    "net/http"
-    "net/http/httptest"
-    "net/http/httputil"
-    "net/url"
-    "os"
-    "strings"
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"strings"
 
-    nuclio "github.com/nuclio/nuclio-sdk-go"
+	nuclio "github.com/nuclio/nuclio-sdk-go"
 )
 
 func Handler(context *nuclio.Context, event nuclio.Event) (interface{}, error) {
-    reverseProxy := context.UserData.(map[string]interface{})["reverseProxy"].(*httputil.ReverseProxy)
-    sidecarUrl := context.UserData.(map[string]interface{})["server"].(string)
+	reverseProxy := context.UserData.(map[string]interface{})["reverseProxy"].(*httputil.ReverseProxy)
+	sidecarUrl := context.UserData.(map[string]interface{})["server"].(string)
 
-    // populate reverse proxy http request
-    httpRequest, err := http.NewRequest(event.GetMethod(), event.GetPath(), bytes.NewReader(event.GetBody()))
-    if err != nil {
-        context.Logger.ErrorWith("Failed to create a reverse proxy request")
-        return nil, err
-    }
-    for k, v := range event.GetHeaders() {
-        httpRequest.Header[k] = []string{v.(string)}
-    }
+	// populate reverse proxy http request
+	httpRequest, err := http.NewRequest(event.GetMethod(), event.GetPath(), bytes.NewReader(event.GetBody()))
+	if err != nil {
+		context.Logger.ErrorWith("Failed to create a reverse proxy request")
+		return nil, err
+	}
+	for k, v := range event.GetHeaders() {
+		httpRequest.Header[k] = []string{v.(string)}
+	}
 
-    // populate query params
-    query := httpRequest.URL.Query()
-    for k, v := range event.GetFields() {
-        query.Set(k, v.(string))
-    }
-    httpRequest.URL.RawQuery = query.Encode()
+	// populate query params
+	query := httpRequest.URL.Query()
+	for k, v := range event.GetFields() {
+		query.Set(k, v.(string))
+	}
+	httpRequest.URL.RawQuery = query.Encode()
 
-    recorder := httptest.NewRecorder()
-    reverseProxy.ServeHTTP(recorder, httpRequest)
+	recorder := httptest.NewRecorder()
+	reverseProxy.ServeHTTP(recorder, httpRequest)
 
-    // send request to sidecar
-    context.Logger.DebugWith("Forwarding request to sidecar", "sidecarUrl", sidecarUrl, "query", httpRequest.URL.Query())
-    response := recorder.Result()
+	// send request to sidecar
+	context.Logger.DebugWith("Forwarding request to sidecar",
+		"sidecarUrl", sidecarUrl,
+		"method", event.GetMethod())
+	response := recorder.Result()
 
-    headers := make(map[string]interface{})
-    for key, value := range response.Header {
-        headers[key] = value[0]
-    }
+	headers := make(map[string]interface{})
+	for key, value := range response.Header {
+		headers[key] = value[0]
+	}
 
-    // let the processor calculate the content length
-    delete(headers, "Content-Length")
-    return nuclio.Response{
-        StatusCode:  response.StatusCode,
-        Body:        recorder.Body.Bytes(),
-        ContentType: response.Header.Get("Content-Type"),
-        Headers:     headers,
-    }, nil
+	// let the processor calculate the content length
+	delete(headers, "Content-Length")
+	return nuclio.Response{
+		StatusCode:  response.StatusCode,
+		Body:        recorder.Body.Bytes(),
+		ContentType: response.Header.Get("Content-Type"),
+		Headers:     headers,
+	}, nil
 }
 
 func InitContext(context *nuclio.Context) error {
-    sidecarHost := os.Getenv("SIDECAR_HOST")
-    sidecarPort := os.Getenv("SIDECAR_PORT")
-    if sidecarHost == "" {
-        sidecarHost = "http://localhost"
-    } else if !strings.Contains(sidecarHost, "://") {
-        sidecarHost = fmt.Sprintf("http://%s", sidecarHost)
-    }
+	sidecarHost := os.Getenv("SIDECAR_HOST")
+	sidecarPort := os.Getenv("SIDECAR_PORT")
+	if sidecarHost == "" {
+		sidecarHost = "http://localhost"
+	} else if !strings.Contains(sidecarHost, "://") {
+		sidecarHost = fmt.Sprintf("http://%s", sidecarHost)
+	}
 
-    // url for request forwarding
-    sidecarUrl := fmt.Sprintf("%s:%s", sidecarHost, sidecarPort)
-    parsedURL, err := url.Parse(sidecarUrl)
-    if err != nil {
-        context.Logger.ErrorWith("Failed to parse sidecar url", "sidecarUrl", sidecarUrl)
-        return err
-    }
-    reverseProxy := httputil.NewSingleHostReverseProxy(parsedURL)
+	// url for request forwarding
+	sidecarUrl := fmt.Sprintf("%s:%s", sidecarHost, sidecarPort)
+	parsedURL, err := url.Parse(sidecarUrl)
+	if err != nil {
+		context.Logger.ErrorWith("Failed to parse sidecar url", "sidecarUrl", sidecarUrl)
+		return err
+	}
+	reverseProxy := httputil.NewSingleHostReverseProxy(parsedURL)
 
-    context.UserData = map[string]interface{}{
-        "server":       sidecarUrl,
-        "reverseProxy": reverseProxy,
-    }
-    return nil
+	context.UserData = map[string]interface{}{
+		"server":       sidecarUrl,
+		"reverseProxy": reverseProxy,
+	}
+	return nil
 }

mlrun/runtimes/nuclio/serving.py

@@ -337,6 +337,17 @@ class ServingRuntime(RemoteRuntime):
         """
         # Applying model monitoring configurations
         self.spec.track_models = enable_tracking
+        if self._spec and self._spec.function_refs:
+            logger.debug(
+                "Set tracking for children references", enable_tracking=enable_tracking
+            )
+            for name in self._spec.function_refs.keys():
+                self._spec.function_refs[name].track_models = enable_tracking
+                # Check if function_refs _function is filled if so update track_models field:
+                if self._spec.function_refs[name]._function:
+                    self._spec.function_refs[
+                        name
+                    ]._function.spec.track_models = enable_tracking
 
         if not 0 < sampling_percentage <= 100:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -506,7 +517,11 @@ class ServingRuntime(RemoteRuntime):
         :return function object
         """
         function_reference = FunctionReference(
-            url, image, requirements=requirements, kind=kind or "serving"
+            url,
+            image,
+            requirements=requirements,
+            kind=kind or "serving",
+            track_models=self.spec.track_models,
         )
         self._spec.function_refs.update(function_reference, name)
         func = function_reference.to_function(self.kind)