mlrun 1.8.0rc45__py3-none-any.whl → 1.8.0rc46__py3-none-any.whl

mlrun/common/schemas/model_monitoring/constants.py

@@ -289,6 +289,11 @@ class ModelMonitoringMode(StrEnum):
     disabled = "disabled"
 
 
+class ScheduleChiefFields(StrEnum):
+    LAST_REQUEST = "last_request"
+    LAST_ANALYZED = "last_analyzed"
+
+
 class EndpointType(IntEnum):
     NODE_EP = 1  # end point that is not a child of a router
     ROUTER = 2  # endpoint that is router

mlrun/data_types/__init__.py

@@ -27,8 +27,12 @@ class BaseDataInfer:
     get_stats = None
 
 
+def is_spark_dataframe(df) -> bool:
+    return "rdd" in dir(df)
+
+
 def get_infer_interface(df) -> BaseDataInfer:
-    if hasattr(df, "rdd"):
+    if is_spark_dataframe(df):
         from .spark import SparkDataInfer
 
         return SparkDataInfer

mlrun/datastore/targets.py

@@ -40,7 +40,7 @@ from mlrun.utils.helpers import to_parquet
 from mlrun.utils.v3io_clients import get_frames_client
 
 from .. import errors
-from ..data_types import ValueType
+from ..data_types import ValueType, is_spark_dataframe
 from ..platforms.iguazio import parse_path, split_path
 from .datastore_profile import datastore_profile_read
 from .spark_utils import spark_session_update_hadoop_options
@@ -86,8 +86,10 @@ def generate_target_run_id():
 
 
 def write_spark_dataframe_with_options(spark_options, df, mode, write_format=None):
+    # TODO: Replace with just df.sparkSession when Spark 3.2 support is dropped
+    spark_session = getattr(df, "sparkSession") or df.sql_ctx.sparkSession
     non_hadoop_spark_options = spark_session_update_hadoop_options(
-        df.sql_ctx.sparkSession, spark_options
+        spark_session, spark_options
     )
     if write_format:
         df.write.format(write_format).mode(mode).save(**non_hadoop_spark_options)
@@ -510,7 +512,7 @@ class BaseStoreTarget(DataTargetBase):
         chunk_id=0,
         **kwargs,
     ) -> Optional[int]:
-        if hasattr(df, "rdd"):
+        if is_spark_dataframe(df):
             options = self.get_spark_options(key_column, timestamp_key)
             options.update(kwargs)
             df = self.prepare_spark_df(df, key_column, timestamp_key, options)
@@ -1376,7 +1378,7 @@ class NoSqlBaseTarget(BaseStoreTarget):
     def write_dataframe(
         self, df, key_column=None, timestamp_key=None, chunk_id=0, **kwargs
     ):
-        if hasattr(df, "rdd"):
+        if is_spark_dataframe(df):
             options = self.get_spark_options(key_column, timestamp_key)
             options.update(kwargs)
             df = self.prepare_spark_df(df)
@@ -2108,7 +2110,7 @@ class SQLTarget(BaseStoreTarget):
 
         self._create_sql_table()
 
-        if hasattr(df, "rdd"):
+        if is_spark_dataframe(df):
             raise ValueError("Spark is not supported")
         else:
             (

mlrun/model_monitoring/api.py

@@ -50,8 +50,8 @@ DatasetType = typing.Union[
 
 def get_or_create_model_endpoint(
     project: str,
+    model_endpoint_name: str,
     model_path: str = "",
-    model_endpoint_name: str = "",
     endpoint_id: str = "",
     function_name: str = "",
     function_tag: str = "latest",
@@ -59,6 +59,7 @@ def get_or_create_model_endpoint(
     sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
     monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
     db_session=None,
+    feature_analysis: bool = False,
 ) -> ModelEndpoint:
     """
     Get a single model endpoint object. If not exist, generate a new model endpoint with the provided parameters. Note
@@ -66,9 +67,9 @@ def get_or_create_model_endpoint(
     features, set `monitoring_mode=enabled`.
 
     :param project:                  Project name.
-    :param model_path:               The model store path (applicable only to new endpoint_id).
     :param model_endpoint_name:      If a new model endpoint is created, the model endpoint name will be presented
                                      under this endpoint (applicable only to new endpoint_id).
+    :param model_path:               The model store path (applicable only to new endpoint_id).
     :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record based
                                      on the provided `endpoint_id`.
     :param function_name:            If a new model endpoint is created, use this function name.
@@ -80,6 +81,7 @@ def get_or_create_model_endpoint(
     :param monitoring_mode:          If enabled, apply model monitoring features on the provided endpoint id
                                      (applicable only to new endpoint_id).
     :param db_session:               A runtime session that manages the current dialog with the database.
+    :param feature_analysis:         If True, the model endpoint will be retrieved with the feature analysis mode.
 
     :return: A ModelEndpoint object
     """
@@ -99,6 +101,7 @@ def get_or_create_model_endpoint(
             endpoint_id=endpoint_id,
             function_name=function_name,
             function_tag=function_tag or "latest",
+            feature_analysis=feature_analysis,
         )
         # If other fields provided, validate that they are correspond to the existing model endpoint data
         _model_endpoint_validations(

mlrun/model_monitoring/applications/context.py

@@ -76,7 +76,6 @@ class MonitoringApplicationContext:
         :param sample_df:               (pd.DataFrame) The new sample DataFrame.
         :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
         :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
-        :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
         :param endpoint_id:             (str) ID of the monitored model endpoint
         :param feature_set:              (FeatureSet) the model endpoint feature set
         :param endpoint_name:           (str) Name of the monitored model endpoint
@@ -208,6 +207,20 @@ class MonitoringApplicationContext:
     @property
     def sample_df(self) -> pd.DataFrame:
         if self._sample_df is None:
+            if (
+                self.endpoint_name is None
+                or self.endpoint_id is None
+                or pd.isnull(self.start_infer_time)
+                or pd.isnull(self.end_infer_time)
+            ):
+                raise mlrun.errors.MLRunValueError(
+                    "You have tried to access `monitoring_context.sample_df`, but have not provided it directly "
+                    "through `sample_data`, nor have you provided the model endpoint's name, ID, and the start and "
+                    f"end times: `endpoint_name`={self.endpoint_name}, `endpoint_uid`={self.endpoint_id}, "
+                    f"`start`={self.start_infer_time}, and `end`={self.end_infer_time}. "
+                    "You can either provide the sample dataframe directly, the model endpoint's details and times, "
+                    "or adapt the application's logic to not access the sample dataframe."
+                )
             feature_set = self.feature_set
             features = [f"{feature_set.metadata.name}.*"]
             vector = fstore.FeatureVector(

mlrun/model_monitoring/controller.py

@@ -28,6 +28,7 @@ import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store as fstore
 import mlrun.model_monitoring
+import mlrun.model_monitoring.db._schedules as schedules
 import mlrun.model_monitoring.helpers
 from mlrun.common.schemas import EndpointType
 from mlrun.common.schemas.model_monitoring.constants import (
@@ -36,7 +37,6 @@ from mlrun.common.schemas.model_monitoring.constants import (
     ControllerEventKind,
 )
 from mlrun.errors import err_to_str
-from mlrun.model_monitoring.db._schedules import ModelMonitoringSchedulesFile
 from mlrun.model_monitoring.helpers import batch_dict2timedelta
 from mlrun.utils import datetime_now, logger
 
@@ -53,7 +53,7 @@ class _BatchWindow:
     def __init__(
         self,
         *,
-        schedules_file: ModelMonitoringSchedulesFile,
+        schedules_file: schedules.ModelMonitoringSchedulesFileEndpoint,
         application: str,
         timedelta_seconds: int,
         last_updated: int,
@@ -153,7 +153,7 @@ class _BatchWindowGenerator(AbstractContextManager):
         self._project = project
         self._endpoint_id = endpoint_id
         self._timedelta = window_length
-        self._schedules_file = ModelMonitoringSchedulesFile(
+        self._schedules_file = schedules.ModelMonitoringSchedulesFileEndpoint(
             project=project, endpoint_id=endpoint_id
         )
 
@@ -273,6 +273,7 @@ class MonitoringApplicationController:
         endpoint: mlrun.common.schemas.ModelEndpoint,
         application_names: set,
         base_period_minutes: int,
+        schedules_file: schedules.ModelMonitoringSchedulesFileChief,
     ) -> bool:
         """
         checks if there is a need to monitor the given endpoint, we should monitor endpoint if it stands in the
@@ -281,11 +282,23 @@ class MonitoringApplicationController:
             2.  first request exists
             3.  last request exists
             4.  endpoint_type is not ROUTER
-        if the four above conditions apply we require one of the three conditions to monitor:
+        if the four above conditions apply we require one of the two condition monitor:
             1.  never monitored the one of the endpoint applications meaning min_last_analyzed is None
-            2.  last request has a higher timestamp than the min_last_analyzed timestamp
-            3.  We didn't analyze one of the application for over than _MAX_OPEN_WINDOWS_ALLOWED windows
+            2.  min_last_analyzed stands in the condition for sending NOP event and this the first time regular event
+            is sent with the combination of  current last_request  & current last_analyzed  per endpoint.
         """
+        last_timestamp_sent = schedules_file.get_endpoint_last_request(
+            endpoint.metadata.uid
+        )
+        last_analyzed_sent = schedules_file.get_endpoint_last_analyzed(
+            endpoint.metadata.uid
+        )
+        logger.debug(
+            "Chief should monitor endpoint check",
+            last_timestamp_sent=last_timestamp_sent,
+            last_analyzed_sent=last_analyzed_sent,
+            uid=endpoint.metadata.uid,
+        )
         if (
             # Is the model endpoint monitored?
             endpoint.status.monitoring_mode == mm_constants.ModelMonitoringMode.enabled
@@ -300,26 +313,43 @@ class MonitoringApplicationController:
                 project=endpoint.metadata.project,
                 endpoint_id=endpoint.metadata.uid,
             ) as batch_window_generator:
-                base_period_seconds = base_period_minutes * _SECONDS_IN_MINUTE
-                if application_names != batch_window_generator.get_application_list():
+                current_time = mlrun.utils.datetime_now()
+                current_min_last_analyzed = (
+                    batch_window_generator.get_min_last_analyzed()
+                )
+                if (
+                    # Different application names, or last analyzed never updated while there are application to monitor
+                    application_names
+                    and (
+                        application_names
+                        != batch_window_generator.get_application_list()
+                        or not current_min_last_analyzed
+                    )
+                ):
                     return True
                 elif (
-                    not batch_window_generator.get_min_last_analyzed()
-                    or batch_window_generator.get_min_last_analyzed()
-                    <= int(endpoint.status.last_request.timestamp())
-                    or mlrun.utils.datetime_now().timestamp()
-                    - batch_window_generator.get_min_last_analyzed()
-                    >= self._MAX_OPEN_WINDOWS_ALLOWED * base_period_seconds
+                    # Does nop event will be sent to close the relevant window
+                    self._should_send_nop_event(
+                        base_period_minutes, current_min_last_analyzed, current_time
+                    )
+                    and (
+                        int(endpoint.status.last_request.timestamp())
+                        != last_timestamp_sent
+                        or current_min_last_analyzed != last_analyzed_sent
+                    )
                 ):
+                    # Write to schedule chief file the last_request, min_last_analyzed we pushed event to stream
+                    schedules_file.update_endpoint_timestamps(
+                        endpoint_uid=endpoint.metadata.uid,
+                        last_request=int(endpoint.status.last_request.timestamp()),
+                        last_analyzed=current_min_last_analyzed,
+                    )
                     return True
                 else:
                     logger.info(
                         "All the possible intervals were already analyzed, didn't push regular event",
                         endpoint_id=endpoint.metadata.uid,
-                        last_analyzed=datetime.datetime.fromtimestamp(
-                            batch_window_generator.get_min_last_analyzed(),
-                            tz=datetime.timezone.utc,
-                        ),
+                        last_analyzed=current_min_last_analyzed,
                         last_request=endpoint.status.last_request,
                     )
         else:
@@ -334,6 +364,21 @@ class MonitoringApplicationController:
             )
         return False
 
+    @staticmethod
+    def _should_send_nop_event(
+        base_period_minutes: int,
+        min_last_analyzed: int,
+        current_time: datetime.datetime,
+    ):
+        if min_last_analyzed:
+            return (
+                current_time.timestamp() - min_last_analyzed
+                >= datetime.timedelta(minutes=base_period_minutes).total_seconds()
+                + mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
+            )
+        else:
+            return True
+
     def run(self, event: nuclio_sdk.Event) -> None:
         """
         Main method for controller chief, runs all the relevant monitoring applications for a single endpoint.
@@ -441,9 +486,11 @@ class MonitoringApplicationController:
                 ]
                 current_time = mlrun.utils.datetime_now()
                 if (
-                    current_time.timestamp()
-                    - batch_window_generator.get_min_last_analyzed()
-                    >= datetime.timedelta(minutes=base_period).total_seconds()
+                    self._should_send_nop_event(
+                        base_period,
+                        batch_window_generator.get_min_last_analyzed(),
+                        current_time,
+                    )
                     and event[ControllerEvent.KIND] != ControllerEventKind.NOP_EVENT
                 ):
                     event = {
@@ -581,29 +628,33 @@ class MonitoringApplicationController:
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=min(len(endpoints), 10)
         ) as pool:
-            futures = {
-                pool.submit(
-                    self.endpoint_to_regular_event,
-                    endpoint,
-                    policy,
-                    set(applications_names),
-                    self.v3io_access_key,
-                ): endpoint
-                for endpoint in endpoints
-            }
-            for future in concurrent.futures.as_completed(futures):
-                if future.exception():
-                    exception = future.exception()
-                    error = (
-                        f"Failed to push event. Endpoint name: {futures[future].metadata.name}, "
-                        f"endpoint uid: {futures[future].metadata.uid}, traceback:\n"
-                    )
-                    error += "".join(
-                        traceback.format_exception(
-                            None, exception, exception.__traceback__
+            with schedules.ModelMonitoringSchedulesFileChief(
+                self.project
+            ) as schedule_file:
+                futures = {
+                    pool.submit(
+                        self.endpoint_to_regular_event,
+                        endpoint,
+                        policy,
+                        set(applications_names),
+                        self.v3io_access_key,
+                        schedule_file,
+                    ): endpoint
+                    for endpoint in endpoints
+                }
+                for future in concurrent.futures.as_completed(futures):
+                    if future.exception():
+                        exception = future.exception()
+                        error = (
+                            f"Failed to push event. Endpoint name: {futures[future].metadata.name}, "
+                            f"endpoint uid: {futures[future].metadata.uid}, traceback:\n"
                         )
-                    )
-                    logger.error(error)
+                        error += "".join(
+                            traceback.format_exception(
+                                None, exception, exception.__traceback__
+                            )
+                        )
+                        logger.error(error)
         logger.info("Finishing monitoring controller chief")
 
     def endpoint_to_regular_event(
@@ -612,14 +663,16 @@ class MonitoringApplicationController:
         policy: dict,
         applications_names: set,
         v3io_access_key: str,
+        schedule_file: schedules.ModelMonitoringSchedulesFileChief,
     ) -> None:
         if self._should_monitor_endpoint(
             endpoint,
             set(applications_names),
             policy.get(ControllerEventEndpointPolicy.BASE_PERIOD, 10),
+            schedule_file,
         ):
-            logger.info(
-                "Regular event is being pushed to controller stream for model endpoint",
+            logger.debug(
+                "Endpoint data is being prepared for regular event",
                 endpoint_id=endpoint.metadata.uid,
                 endpoint_name=endpoint.metadata.name,
                 timestamp=endpoint.status.last_request.isoformat(

mlrun/model_monitoring/db/_schedules.py

@@ -13,52 +13,38 @@
 # limitations under the License.
 
 import json
+from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from types import TracebackType
 from typing import Final, Optional
 
 import botocore.exceptions
 
-import mlrun.common.schemas
+import mlrun.common.schemas as schemas
 import mlrun.errors
 import mlrun.model_monitoring.helpers
 from mlrun.utils import logger
 
 
-class ModelMonitoringSchedulesFile(AbstractContextManager):
+class ModelMonitoringSchedulesFileBase(AbstractContextManager, ABC):
     DEFAULT_SCHEDULES: Final = {}
     INITIAL_CONTENT = json.dumps(DEFAULT_SCHEDULES)
     ENCODING = "utf-8"
 
-    def __init__(self, project: str, endpoint_id: str) -> None:
-        """
-        Initialize applications monitoring schedules file object.
-        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
-        When working with the schedules data, use this class as a context manager to read and write the data.
-
-        :param project:     The project name.
-        :param endpoint_id: The endpoint ID.
-        """
-        # `self._item` is the persistent version of the monitoring schedules.
-        self._item = mlrun.model_monitoring.helpers.get_monitoring_schedules_data(
-            project=project, endpoint_id=endpoint_id
-        )
-        self._path = self._item.url
-        self._fs = self._item.store.filesystem
-        # `self._schedules` is an in-memory copy of the DB for all the applications for
-        # the same model endpoint.
-        self._schedules: dict[str, int] = self.DEFAULT_SCHEDULES.copy()
-        # Does `self._schedules` hold the content of `self._item`?
-        self._open_schedules = False
-
-    @classmethod
-    def from_model_endpoint(
-        cls, model_endpoint: mlrun.common.schemas.ModelEndpoint
-    ) -> "ModelMonitoringSchedulesFile":
-        return cls(
-            project=model_endpoint.metadata.project,
-            endpoint_id=model_endpoint.metadata.uid,
-        )
+    def __init__(self):
+        self._item = self.get_data_item_object()
+        if self._item:
+            self._path = self._item.url
+            self._fs = self._item.store.filesystem
+            # `self._schedules` is an in-memory copy of the DB for all the applications for
+            # the same model endpoint.
+            self._schedules = self.DEFAULT_SCHEDULES.copy()
+            # Does `self._schedules` hold the content of `self._item`?
+            self._open_schedules = False
+
+    @abstractmethod
+    def get_data_item_object(self) -> mlrun.DataItem:
+        pass
 
     def create(self) -> None:
         """Create a schedules file with initial content - an empty dictionary"""
@@ -114,7 +100,7 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         self._schedules = self.DEFAULT_SCHEDULES
         self._open_schedules = False
 
-    def __enter__(self) -> "ModelMonitoringSchedulesFile":
+    def __enter__(self) -> "ModelMonitoringSchedulesFileBase":
         self._open()
         return super().__enter__()
 
@@ -132,6 +118,36 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
                 "Open the schedules file as a context manager first"
             )
 
+
+class ModelMonitoringSchedulesFileEndpoint(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str, endpoint_id: str) -> None:
+        """
+        Initialize applications monitoring schedules file object.
+        The JSON file stores a dictionary of registered application name as key and Unix timestamp as value.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        :param endpoint_id: The endpoint ID.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        self._endpoint_id = endpoint_id
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_endpoint_data(
+            project=self._project, endpoint_id=self._endpoint_id
+        )
+
+    @classmethod
+    def from_model_endpoint(
+        cls, model_endpoint: schemas.ModelEndpoint
+    ) -> "ModelMonitoringSchedulesFileEndpoint":
+        return cls(
+            project=model_endpoint.metadata.project,
+            endpoint_id=model_endpoint.metadata.uid,
+        )
+
     def get_application_time(self, application: str) -> Optional[int]:
         self._check_open_schedules()
         return self._schedules.get(application)
@@ -149,6 +165,68 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         return min(self._schedules.values(), default=None)
 
 
+class ModelMonitoringSchedulesFileChief(ModelMonitoringSchedulesFileBase):
+    def __init__(self, project: str) -> None:
+        """
+        Initialize applications monitoring schedules chief file object.
+        The JSON file stores a dictionary of registered model endpoints uid as key and point to a dictionary of
+        "last_request" and "last_analyzed" mapped to two Unix timestamps as values.
+        When working with the schedules data, use this class as a context manager to read and write the data.
+
+        :param project:     The project name.
+        """
+        # `self._item` is the persistent version of the monitoring schedules.
+        self._project = project
+        super().__init__()
+
+    def get_data_item_object(self) -> mlrun.DataItem:
+        return mlrun.model_monitoring.helpers.get_monitoring_schedules_chief_data(
+            project=self._project
+        )
+
+    def get_endpoint_last_request(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST
+            )
+        else:
+            return None
+
+    def update_endpoint_timestamps(
+        self, endpoint_uid: str, last_request: int, last_analyzed: int
+    ) -> None:
+        self._check_open_schedules()
+        self._schedules[endpoint_uid] = {
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_REQUEST: last_request,
+            schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED: last_analyzed,
+        }
+
+    def get_endpoint_last_analyzed(self, endpoint_uid: str) -> Optional[int]:
+        self._check_open_schedules()
+        if endpoint_uid in self._schedules:
+            return self._schedules[endpoint_uid].get(
+                schemas.model_monitoring.constants.ScheduleChiefFields.LAST_ANALYZED
+            )
+        else:
+            return None
+
+    def get_endpoint_list(self) -> set[str]:
+        self._check_open_schedules()
+        return set(self._schedules.keys())
+
+    def get_or_create(self) -> None:
+        try:
+            self._open()
+        except (
+            mlrun.errors.MLRunNotFoundError,
+            # Different errors are raised for S3 or local storage, see ML-8042
+            botocore.exceptions.ClientError,
+            FileNotFoundError,
+        ):
+            self.create()
+
+
 def delete_model_monitoring_schedules_folder(project: str) -> None:
     """Delete the model monitoring schedules folder of the project"""
     folder = mlrun.model_monitoring.helpers._get_monitoring_schedules_folder_path(

mlrun/model_monitoring/helpers.py

@@ -472,6 +472,7 @@ def update_model_endpoint_last_request(
             "Bumping model endpoint last request time (EP without serving)",
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            function_name=model_endpoint.spec.function_name,
             last_request=model_endpoint.status.last_request,
             current_request=current_request.isoformat(),
             bumped_last_request=bumped_last_request,
@@ -586,16 +587,43 @@ def _get_monitoring_schedules_folder_path(project: str) -> str:
     )
 
 
-def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> str:
+def _get_monitoring_schedules_file_endpoint_path(
+    *, project: str, endpoint_id: str
+) -> str:
     return os.path.join(
         _get_monitoring_schedules_folder_path(project), f"{endpoint_id}.json"
     )
 
 
-def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
+def get_monitoring_schedules_endpoint_data(
+    *, project: str, endpoint_id: str
+) -> "DataItem":
+    """
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_endpoint_path(
+            project=project, endpoint_id=endpoint_id
+        )
+    )
+
+
+def get_monitoring_schedules_chief_data(
+    *,
+    project: str,
+) -> "DataItem":
     """
     Get the model monitoring schedules' data item of the project's model endpoint.
     """
     return mlrun.datastore.store_manager.object(
-        _get_monitoring_schedules_file_path(project=project, endpoint_id=endpoint_id)
+        _get_monitoring_schedules_file_chief_path(project=project)
+    )
+
+
+def _get_monitoring_schedules_file_chief_path(
+    *,
+    project: str,
+) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{project}.json"
     )

mlrun/projects/project.py

@@ -5266,7 +5266,7 @@ class MlrunProject(ModelObj):
             )
 
         # if engine is remote then skip the local file validation
-        if engine and not engine.startswith("remote"):
+        if engine and engine.startswith("remote"):
             return
 
         code_path = self.spec.get_code_path()

mlrun/runtimes/function_reference.py

@@ -36,6 +36,7 @@ class FunctionReference(ModelObj):
         spec=None,
         kind=None,
         name=None,
+        track_models=None,
     ):
         self.url = url
         self.kind = kind
@@ -46,6 +47,7 @@ class FunctionReference(ModelObj):
             spec = spec.to_dict()
         self.spec = spec
         self.code = code
+        self.track_models = track_models
 
         self._function = None
         self._address = None
@@ -130,6 +132,7 @@ class FunctionReference(ModelObj):
         if self.requirements:
             func.with_requirements(self.requirements)
         self._function = func
+        func.spec.track_models = self.track_models
         return func
 
     @property

mlrun/runtimes/nuclio/serving.py

@@ -337,6 +337,17 @@ class ServingRuntime(RemoteRuntime):
         """
         # Applying model monitoring configurations
         self.spec.track_models = enable_tracking
+        if self._spec and self._spec.function_refs:
+            logger.debug(
+                "Set tracking for children references", enable_tracking=enable_tracking
+            )
+            for name in self._spec.function_refs.keys():
+                self._spec.function_refs[name].track_models = enable_tracking
+                # Check if function_refs _function is filled if so update track_models field:
+                if self._spec.function_refs[name]._function:
+                    self._spec.function_refs[
+                        name
+                    ]._function.spec.track_models = enable_tracking
 
         if not 0 < sampling_percentage <= 100:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -506,7 +517,11 @@ class ServingRuntime(RemoteRuntime):
         :return function object
         """
         function_reference = FunctionReference(
-            url, image, requirements=requirements, kind=kind or "serving"
+            url,
+            image,
+            requirements=requirements,
+            kind=kind or "serving",
+            track_models=self.spec.track_models,
         )
         self._spec.function_refs.update(function_reference, name)
         func = function_reference.to_function(self.kind)

mlrun/serving/v2_serving.py

@@ -145,8 +145,14 @@ class V2ModelServer(StepToDict):
                 feature.name for feature in self.model_spec.outputs
             ]
 
+        if (
+            kwargs.get("endpoint_type", mlrun.common.schemas.EndpointType.LEAF_EP)
+            == mlrun.common.schemas.EndpointType.NODE_EP
+        ):
+            self._initialize_model_logger()
+
     def _lazy_init(self, event):
-        if event and isinstance(event, dict):
+        if event and isinstance(event, dict) and not self.initialized:
             background_task_state = event.get("background_task_state", None)
             if (
                 background_task_state
@@ -461,6 +467,50 @@ class V2ModelServer(StepToDict):
         request["inputs"] = new_inputs
         return request
 
+    def _initialize_model_logger(self):
+        server: mlrun.serving.GraphServer = getattr(
+            self.context, "_server", None
+        ) or getattr(self.context, "server", None)
+        if not self.context.is_mock or self.context.monitoring_mock:
+            if server.model_endpoint_creation_task_name:
+                background_task = mlrun.get_run_db().get_project_background_task(
+                    server.project, server.model_endpoint_creation_task_name
+                )
+                logger.debug(
+                    "Checking model endpoint creation task status",
+                    task_name=server.model_endpoint_creation_task_name,
+                )
+                if (
+                    background_task.status.state
+                    in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+                ):
+                    logger.debug(
+                        f"Model endpoint creation task completed with state {background_task.status.state}"
+                    )
+                    if (
+                        background_task.status.state
+                        == mlrun.common.schemas.BackgroundTaskState.succeeded
+                    ):
+                        self._model_logger = (
+                            _ModelLogPusher(self, self.context)
+                            if self.context
+                            and self.context.stream.enabled
+                            and self.model_endpoint_uid
+                            else None
+                        )
+                        self.initialized = True
+
+                else:  # in progress
+                    logger.debug(
+                        f"Model endpoint creation task is still in progress with the current state: "
+                        f"{background_task.status.state}.",
+                        name=self.name,
+                    )
+            else:
+                logger.debug(
+                    "Model endpoint creation task name not provided",
+                )
+
 
 class _ModelLogPusher:
     def __init__(self, model: V2ModelServer, context, output_stream=None):

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "b434e35c26bb66407a60bedddb7a9af71141902b",
-  "version": "1.8.0-rc45"
+  "git_commit": "78aff2128b074d4d18751c0cbd2078e14929fa17",
+  "version": "1.8.0-rc46"
 }

{mlrun-1.8.0rc45.dist-info → mlrun-1.8.0rc46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.8.0rc45
+Version: 1.8.0rc46
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.8.0rc45.dist-info → mlrun-1.8.0rc46.dist-info}/RECORD

@@ -73,10 +73,10 @@ mlrun/common/schemas/serving.py,sha256=81ZxlDHP1fm9VPmXZGkjZj2n6cVRmqEN478hsmvv5
 mlrun/common/schemas/tag.py,sha256=HRZi5QZ4vVGaCr2AMk9eJgcNiAIXmH4YDc8a4fvF770,893
 mlrun/common/schemas/workflow.py,sha256=6u9niXfXpV-_c2rZL97gFIdAnOfM5WK-OCbrM5Kk34s,2108
 mlrun/common/schemas/model_monitoring/__init__.py,sha256=SxHG-GIdcTEuFxpKzkUdT9zKaU5Xqz9qF1uCwXvZ2z8,1709
-mlrun/common/schemas/model_monitoring/constants.py,sha256=BWGNxOUoXMcR2epGck_UId1C33bbmxhLwgQYktIrPxE,12462
+mlrun/common/schemas/model_monitoring/constants.py,sha256=wbNe_n5wX98gD1XQ6jmt97Jh59S9GsE54UBPZl9Pg20,12570
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=THQlLfPBevBksta8p5OaIsBaJtsNSXexLvHrDxOaVns,2095
 mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=O0i-pvzcXJVgf9E_tNcudDTa1xLaJchzPGfZZ8MNdD4,11482
-mlrun/data_types/__init__.py,sha256=unRo9GGwCmj0hBKBRsXJ2P4BzpQaddlQTvIrVQaKluI,984
+mlrun/data_types/__init__.py,sha256=wdxGS1PTnaKXiNZ7PYGxxo86OifHH7NYoArIjDJksLA,1054
 mlrun/data_types/data_types.py,sha256=0_oKLC6-sXL2_nnaDMP_HSXB3fD1nJAG4J2Jq6sGNNw,4998
 mlrun/data_types/infer.py,sha256=Ogp3rsENVkjU0GDaGa9J1vjGrvMxgzwbSEuG51nt61E,6477
 mlrun/data_types/spark.py,sha256=4fPpqjFCYeFgK_yHhUNM4rT-1Gw9YiXazyjTK7TtbTI,9626
@@ -100,7 +100,7 @@ mlrun/datastore/spark_udf.py,sha256=NnnB3DZxZb-rqpRy7b-NC7QWXuuqFn3XkBDc86tU4mQ,
 mlrun/datastore/spark_utils.py,sha256=_AsVoU5Ix_-W7Gyq8io8V-2GTk0m8THJNDP3WGGaWJY,2865
 mlrun/datastore/store_resources.py,sha256=PFOMrZ6KH6hBOb0PiO-cHx_kv0UpHu5P2t8_mrR-lS4,6842
 mlrun/datastore/storeytargets.py,sha256=g5zAdizdFkcESoVGxbKWC11ZiXFgM77UL4642G32JaU,6459
-mlrun/datastore/targets.py,sha256=eBp2ECV1Wk6D1X7kYhaneiVfsGW_TyKkaKdBXYgvWVM,81018
+mlrun/datastore/targets.py,sha256=k6IU7XPOYm9jJi5foINfO4NH3NvuXcwYB_B7Rt7V-cg,81195
 mlrun/datastore/utils.py,sha256=CbKbDI6CdFRCqyAXe-jykVvN_GH6R0JkxIQFAogR2GA,10604
 mlrun/datastore/v3io.py,sha256=QSYBORRLcJTeM9mt0EaWzyLcdmzrPkqrF7k5uLTam5U,8209
 mlrun/datastore/vectorstore.py,sha256=k-yom5gfw20hnVG0Rg7aBEehuXwvAloZwn0cx0VGals,11708
@@ -218,23 +218,23 @@ mlrun/launcher/factory.py,sha256=RW7mfzEFi8fR0M-4W1JQg1iq3_muUU6OTqT_3l4Ubrk,233
 mlrun/launcher/local.py,sha256=775HY-8S9LFUX5ubGXrLO0N1lVh8bn-DHFmNYuNqQPA,11451
 mlrun/launcher/remote.py,sha256=rLJW4UAnUT5iUb4BsGBOAV3K4R29a0X4lFtRkVKlyYU,7709
 mlrun/model_monitoring/__init__.py,sha256=ELy7njEtZnz09Dc6PGZSFFEGtnwI15bJNWM3Pj4_YIs,753
-mlrun/model_monitoring/api.py,sha256=nkNlBq_X12tGgs4rbVutzq-ce9P49zAyg_hvffwmz7I,27544
-mlrun/model_monitoring/controller.py,sha256=kRKoI47YMAhL8YdqD5D7n8fX4vqcX4uWDl6gi5WVnqA,31698
+mlrun/model_monitoring/api.py,sha256=Dk5uEAk8HTU00vUwyDPUFqSOgkT9z4gxwaDXbtj-4-U,27737
+mlrun/model_monitoring/controller.py,sha256=mbKonkixNgeXT1BRNKQi6i-bk0pquwWgNclQHAjRvcA,33862
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
-mlrun/model_monitoring/helpers.py,sha256=Q4vcc7x41lCJdFQIE8UFPY0WIQ8a-4tSGhziMA4ib4w,22003
+mlrun/model_monitoring/helpers.py,sha256=Cc1TGsqoiuey_XDkwOeaOVV2OnXCryZnBEGGlI8Z524,22620
 mlrun/model_monitoring/stream_processing.py,sha256=4M0H4txMlsC2Q5iKTPp992KWoNPAJjPHj9rqWhXbl8w,33321
 mlrun/model_monitoring/tracking_policy.py,sha256=PBIGrUYWrwcE5gwXupBIVzOb0QRRwPJsgQm_yLGQxB4,5595
 mlrun/model_monitoring/writer.py,sha256=ibbhvfSHb8Reqlb7RGFEAUNM4iTyK1gk8-2m46mP6VM,8428
 mlrun/model_monitoring/applications/__init__.py,sha256=xDBxkBjl-whHSG_4t1mLkxiypLH-fzn8TmAW9Mjo2uI,759
 mlrun/model_monitoring/applications/_application_steps.py,sha256=PxULZznKW66Oq-fKaraOAbsTuGnV0zgXh6_91wX3KUo,8367
 mlrun/model_monitoring/applications/base.py,sha256=7XL12idItWkoE3CJ_48F6cwVx5pJH3bgfG92hb8LcN8,24872
-mlrun/model_monitoring/applications/context.py,sha256=Wou9lviSETjEqyMoIAi0Ko58luRkx0uy3ZDUVyRheNA,16144
+mlrun/model_monitoring/applications/context.py,sha256=DKUDOfN4iY5wpOMjfsarx4pVN9A1sORyu7y2EEKEvMs,16964
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=09t0tfC35W0SeJA3fzN29pJiB6G-V_8GlcvULVq6H9Q,15179
 mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
 mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
 mlrun/model_monitoring/applications/evidently/base.py,sha256=C8402vQJH7jmY-i49DnYjy6p6dETWex4Tdi8ylFLecA,5097
 mlrun/model_monitoring/db/__init__.py,sha256=r47xPGZpIfMuv8J3PQCZTSqVPMhUta4sSJCZFKcS7FM,644
-mlrun/model_monitoring/db/_schedules.py,sha256=AKyCJBAt0opNE3K3pg2TjCoD_afk1LKw5TY88rLQ2VA,6097
+mlrun/model_monitoring/db/_schedules.py,sha256=RWn4wtKsIXg668gMLpxO9I8GlkxvPSaA5y7w-wFDcgE,9048
 mlrun/model_monitoring/db/_stats.py,sha256=VVMWLMqG3Us3ozBkLaokJF22Ewv8WKmVE1-OvS_g9vA,6943
 mlrun/model_monitoring/db/tsdb/__init__.py,sha256=4S86V_Ot_skE16SLkw0WwsaAUB0ECH6SoJdp-TIu6s8,4645
 mlrun/model_monitoring/db/tsdb/base.py,sha256=55lZfKmAWPW_Zi8DJhGib6euYhRhNxEpj528_rfh9Ww,26894
@@ -270,12 +270,12 @@ mlrun/platforms/iguazio.py,sha256=6VBTq8eQ3mzT96tzjYhAtcMQ2VjF4x8LpIPW5DAcX2Q,13
 mlrun/projects/__init__.py,sha256=0Krf0WIKfnZa71WthYOg0SoaTodGg3sV_hK3f_OlTPI,1220
 mlrun/projects/operations.py,sha256=TzPbTYBgmYrjxTKP_wOtBJYFFFwDCQtaVvF1Snr0TfM,20029
 mlrun/projects/pipelines.py,sha256=wud7ezeEmhIJvfYE_wzQbA4ygEfGXHtbOtoOpan6poY,48556
-mlrun/projects/project.py,sha256=H-mjTHqV_T9PhfCfGQRNm1R-7cM-7boS1Jd1TN9SyYo,235337
+mlrun/projects/project.py,sha256=Jmr7l9VwFGBItS50DElEN6PuDE6OPPvXn1j16kb1z80,235333
 mlrun/runtimes/__init__.py,sha256=J9Sy2HiyMlztNv6VUurMzF5H2XzttNil8nRsWDsqLyg,8923
 mlrun/runtimes/base.py,sha256=EL14Kmc1vWEjnBPJwLj5hHC6CtRAQHJLmohCD3sFEHo,37855
 mlrun/runtimes/daskjob.py,sha256=JwuGvOiPsxEDHHMMUS4Oie4hLlYYIZwihAl6DjroTY0,19521
 mlrun/runtimes/funcdoc.py,sha256=zRFHrJsV8rhDLJwoUhcfZ7Cs0j-tQ76DxwUqdXV_Wyc,9810
-mlrun/runtimes/function_reference.py,sha256=CLvRY-wXX9qhI9YEzSl0VWt8piH_-5FQYQ8ObUYLLDc,4911
+mlrun/runtimes/function_reference.py,sha256=fnMKUEieKgy4JyVLhFpDtr6JvKgOaQP8F_K2H3-Pk9U,5030
 mlrun/runtimes/generators.py,sha256=X8NDlCEPveDDPOHtOGcSpbl3pAVM3DP7fuPj5xVhxEY,7290
 mlrun/runtimes/kubejob.py,sha256=K-nR3J0-S3Em6Ez-JD0BxHczobQhC4m0829HLdSwX8g,8797
 mlrun/runtimes/local.py,sha256=yedo3R1c46cB1mX7aOz8zORXswQPvX86U-_fYxXoqTY,22717
@@ -294,7 +294,7 @@ mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVY
 mlrun/runtimes/nuclio/api_gateway.py,sha256=vH9ClKVP4Mb24rvA67xPuAvAhX-gAv6vVtjVxyplhdc,26969
 mlrun/runtimes/nuclio/function.py,sha256=j_gKYhaGfJjr_mVBdUcnSgXcXOHJrKHtUMpmOu8TII8,52979
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
-mlrun/runtimes/nuclio/serving.py,sha256=1QPza0oG63bt3Bpib2VGhDcW3PNEjjsBUzIYBhiYR0s,32666
+mlrun/runtimes/nuclio/serving.py,sha256=qetAyl-nfn8SWp7KyNgRtMNUVcX_q75SY9dLZP0uH6o,33365
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
 mlrun/runtimes/nuclio/application/application.py,sha256=VPX-ruYQJ7-7yd5c2sWdF4U5JCGSS3kYjUfOgev6l_Y,29186
 mlrun/runtimes/nuclio/application/reverse_proxy.go,sha256=lEHH74vr2PridIHp1Jkc_NjkrWb5b6zawRrNxHQhwGU,2913
@@ -309,7 +309,7 @@ mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBI
 mlrun/serving/states.py,sha256=Kst2N7R5SaTKYMYB8re9wTlhQwEDgkG61-4JtROKlNI,72803
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
-mlrun/serving/v2_serving.py,sha256=nMxaFcc7vzjIYDqZMRGkMcvm2VxwnTCKocoYLByQdbw,23121
+mlrun/serving/v2_serving.py,sha256=b3C5Utv2_AOPrH_hPi3NarjNbAK3kRoeIfqMU4qNuUo,25362
 mlrun/track/__init__.py,sha256=yVXbT52fXvGKRlc_ByHqIVt7-9L3DRE634RSeQwgXtU,665
 mlrun/track/tracker.py,sha256=CyTU6Qd3_5GGEJ_hpocOj71wvV65EuFYUjaYEUKAL6Q,3575
 mlrun/track/tracker_manager.py,sha256=IYBl99I62IC6VCCmG1yt6JoHNOQXa53C4DURJ2sWgio,5726
@@ -340,11 +340,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=eQvmctTh6wIG5xVOesLLV9S1-UUCu5UEQ9JIJOor3ts,7183
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=M386FKHwnC4eHpp3xG7fFtQEQllorzsiu9pDs5U8888,89
+mlrun/utils/version/version.json,sha256=BdVuIuX64LHL8ZydQn4fu4jqZorhEvAHUQZ0YBVTWr8,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc45.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc45.dist-info/METADATA,sha256=Hrx8b3n6ywe0Ttl7L2PHZSvuIXzm9ELWceKlutJ-7jk,26008
-mlrun-1.8.0rc45.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-mlrun-1.8.0rc45.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc45.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc45.dist-info/RECORD,,
+mlrun-1.8.0rc46.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc46.dist-info/METADATA,sha256=-rHQLgCZW8i6RUoHhIzNHEJKGFNvyoI_kIgS59Ajzdk,26008
+mlrun-1.8.0rc46.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
+mlrun-1.8.0rc46.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc46.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc46.dist-info/RECORD,,

{mlrun-1.8.0rc45.dist-info → mlrun-1.8.0rc46.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.0.2)
 Root-Is-Purelib: true
 Tag: py3-none-any