mlrun 1.8.0rc27__py3-none-any.whl → 1.8.0rc29__py3-none-any.whl

mlrun/model_monitoring/controller.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import concurrent.futures
 import datetime
 import json
 import os
@@ -138,7 +139,9 @@ class _BatchWindow:
 
 
 class _BatchWindowGenerator(AbstractContextManager):
-    def __init__(self, project: str, endpoint_id: str, window_length: int) -> None:
+    def __init__(
+        self, project: str, endpoint_id: str, window_length: Optional[int] = None
+    ) -> None:
         """
         Initialize a batch window generator object that generates batch window objects
         for the monitoring functions.
@@ -165,6 +168,12 @@ class _BatchWindowGenerator(AbstractContextManager):
             exc_type=exc_type, exc_value=exc_value, traceback=traceback
         )
 
+    def get_application_list(self) -> set[str]:
+        return self._schedules_file.get_application_list()
+
+    def get_min_last_analyzed(self) -> Optional[int]:
+        return self._schedules_file.get_min_timestamp()
+
     @classmethod
     def _get_last_updated_time(
         cls, last_request: datetime.datetime, not_batch_endpoint: bool
@@ -234,8 +243,7 @@ class MonitoringApplicationController:
     def __init__(self) -> None:
         """Initialize Monitoring Application Controller"""
         self.project = cast(str, mlrun.mlconf.default_project)
-        self.project_obj = mlrun.load_project(name=self.project, url=self.project)
-
+        self.project_obj = mlrun.get_run_db().get_project(name=self.project)
         logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
 
         self._window_length = _get_window_length()
@@ -255,8 +263,10 @@ class MonitoringApplicationController:
         return access_key
 
     @staticmethod
-    def _should_monitor_endpoint(endpoint: mlrun.common.schemas.ModelEndpoint) -> bool:
-        return (
+    def _should_monitor_endpoint(
+        endpoint: mlrun.common.schemas.ModelEndpoint, application_names: set
+    ) -> bool:
+        if (
             # Is the model endpoint monitored?
             endpoint.status.monitoring_mode == mm_constants.ModelMonitoringMode.enabled
             # Was the model endpoint called? I.e., are the first and last requests nonempty?
@@ -265,7 +275,40 @@ class MonitoringApplicationController:
             # Is the model endpoint not a router endpoint? Router endpoint has no feature stats
             and endpoint.metadata.endpoint_type.value
             != mm_constants.EndpointType.ROUTER.value
-        )
+        ):
+            with _BatchWindowGenerator(
+                project=endpoint.metadata.project,
+                endpoint_id=endpoint.metadata.uid,
+            ) as batch_window_generator:
+                if application_names != batch_window_generator.get_application_list():
+                    return True
+                elif (
+                    not batch_window_generator.get_min_last_analyzed()
+                    or batch_window_generator.get_min_last_analyzed()
+                    <= int(endpoint.status.last_request.timestamp())
+                ):
+                    return True
+                else:
+                    logger.info(
+                        "All the possible intervals were already analyzed, didn't push regular event",
+                        endpoint_id=endpoint.metadata.uid,
+                        last_analyzed=datetime.datetime.fromtimestamp(
+                            batch_window_generator.get_min_last_analyzed(),
+                            tz=datetime.timezone.utc,
+                        ),
+                        last_request=endpoint.status.last_request,
+                    )
+        else:
+            logger.info(
+                "Should not monitor model endpoint, didn't push regular event",
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                last_request=endpoint.status.last_request,
+                first_request=endpoint.status.first_request,
+                endpoint_type=endpoint.metadata.endpoint_type,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+            )
+        return False
 
     def run(self, event: nuclio_sdk.Event) -> None:
         """
@@ -314,7 +357,7 @@ class MonitoringApplicationController:
             )
             m_fs = fstore.get_feature_set(event[ControllerEvent.FEATURE_SET_URI])
             logger.info(
-                "Starting analyzing for:", timestamp=event[ControllerEvent.TIMESTAMP]
+                "Starting analyzing for", timestamp=event[ControllerEvent.TIMESTAMP]
             )
             last_stream_timestamp = datetime.datetime.fromisoformat(
                 event[ControllerEvent.TIMESTAMP]
@@ -370,7 +413,7 @@ class MonitoringApplicationController:
                 current_time = mlrun.utils.datetime_now()
                 if (
                     current_time.timestamp()
-                    - batch_window_generator.batch_window._get_last_analyzed()
+                    - batch_window_generator.get_min_last_analyzed()
                     >= datetime.timedelta(minutes=base_period).total_seconds()
                     and event[ControllerEvent.KIND] != ControllerEventKind.NOP_EVENT
                 ):
@@ -399,6 +442,9 @@ class MonitoringApplicationController:
                         event=event,
                         endpoint_id=endpoint_id,
                     )
+            logger.info(
+                "Finish analyze for", timestamp=event[ControllerEvent.TIMESTAMP]
+            )
 
         except Exception:
             logger.exception(
@@ -455,17 +501,14 @@ class MonitoringApplicationController:
                 [data]
             )
 
-    def push_regular_event_to_controller_stream(self, event: nuclio_sdk.Event) -> None:
+    def push_regular_event_to_controller_stream(self) -> None:
         """
         pushes a regular event to the controller stream.
         :param event: the nuclio trigger event
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
-        db = mlrun.get_run_db()
-        endpoints = db.list_model_endpoints(
-            project=self.project, tsdb_metrics=True
-        ).endpoints
+        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=True).endpoints
         if not endpoints:
             logger.info("No model endpoints found", project=self.project)
             return
@@ -505,48 +548,59 @@ class MonitoringApplicationController:
                 // 60
             ),
         }
-        for endpoint in endpoints:
-            if self._should_monitor_endpoint(endpoint):
-                logger.info(
-                    "Regular event is being pushed to controller stream for model endpoint",
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    timestamp=endpoint.status.last_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    first_request=endpoint.status.first_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
-                    endpoint_policy=json.dumps(policy),
-                )
-                self.push_to_controller_stream(
-                    kind=mm_constants.ControllerEventKind.REGULAR_EVENT,
-                    project=self.project,
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    stream_access_key=self.v3io_access_key,
-                    timestamp=endpoint.status.last_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    first_request=endpoint.status.first_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
-                    endpoint_policy=policy,
-                )
-            else:
-                logger.info(
-                    "Should not monitor model endpoint, didn't push regular event",
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    timestamp=endpoint.status.last_request,
-                    first_request=endpoint.status.first_request,
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=min(len(endpoints), 10)
+        ) as pool:
+            for endpoint in endpoints:
+                pool.submit(
+                    MonitoringApplicationController.endpoint_to_regular_event,
+                    endpoint,
+                    policy,
+                    set(applications_names),
+                    self.v3io_access_key,
                 )
+        logger.info("Finishing monitoring controller chief")
+
+    @staticmethod
+    def endpoint_to_regular_event(
+        endpoint: mlrun.common.schemas.ModelEndpoint,
+        policy: dict,
+        applications_names: set,
+        v3io_access_key: str,
+    ) -> None:
+        if MonitoringApplicationController._should_monitor_endpoint(
+            endpoint, set(applications_names)
+        ):
+            logger.info(
+                "Regular event is being pushed to controller stream for model endpoint",
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                timestamp=endpoint.status.last_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                first_request=endpoint.status.first_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                endpoint_type=endpoint.metadata.endpoint_type,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+                endpoint_policy=json.dumps(policy),
+            )
+            MonitoringApplicationController.push_to_controller_stream(
+                kind=mm_constants.ControllerEventKind.REGULAR_EVENT,
+                project=endpoint.metadata.project,
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                stream_access_key=v3io_access_key,
+                timestamp=endpoint.status.last_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                first_request=endpoint.status.first_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                endpoint_type=endpoint.metadata.endpoint_type.value,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+                endpoint_policy=policy,
+            )
 
     @staticmethod
     def push_to_controller_stream(
@@ -557,7 +611,7 @@ class MonitoringApplicationController:
         stream_access_key: str,
         timestamp: str,
         first_request: str,
-        endpoint_type: str,
+        endpoint_type: int,
         feature_set_uri: str,
         endpoint_policy: dict[str, Any],
     ) -> None:
@@ -633,7 +687,13 @@ def handler(context: nuclio_sdk.Context, event: nuclio_sdk.Event) -> None:
 
     if event.trigger.kind == "http":
         # Runs controller chief:
-        MonitoringApplicationController().push_regular_event_to_controller_stream(event)
+        context.user_data.monitor_app_controller.push_regular_event_to_controller_stream()
     else:
         # Runs controller worker:
-        MonitoringApplicationController().run(event=event)
+        context.user_data.monitor_app_controller.run(event)
+
+
+def init_context(context):
+    monitor_app_controller = MonitoringApplicationController()
+    setattr(context.user_data, "monitor_app_controller", monitor_app_controller)
+    context.logger.info("Monitoring application controller initialized")

mlrun/model_monitoring/db/_schedules.py

@@ -140,6 +140,14 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         self._check_open_schedules()
         self._schedules[application] = timestamp
 
+    def get_application_list(self) -> set[str]:
+        self._check_open_schedules()
+        return set(self._schedules.keys())
+
+    def get_min_timestamp(self) -> Optional[int]:
+        self._check_open_schedules()
+        return min(self._schedules.values(), default=None)
+
 
 def delete_model_monitoring_schedules_folder(project: str) -> None:
     """Delete the model monitoring schedules folder of the project"""

mlrun/model_monitoring/db/tsdb/__init__.py

@@ -75,10 +75,11 @@ def get_tsdb_connector(
     :param secret_provider:         An optional secret provider to get the connection string secret.
     :param profile:                 An optional profile to initialize the TSDB connector from.
 
-    :return: `TSDBConnector` object. The main goal of this object is to handle different operations on the
+    :return: ``TSDBConnector`` object. The main goal of this object is to handle different operations on the
              TSDB connector such as updating drift metrics or write application record result.
-    :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide TSDB connection
-            or the provided TSDB connection is invalid.
+    :raise: ``MLRunNotFoundError`` if the user didn't set the TSDB datastore profile and didn't provide it through
+            the ``profile`` parameter.
+    :raise: ``MLRunInvalidMMStoreTypeError`` if the TSDB datastore profile is of an invalid type.
     """
     profile = profile or mlrun.model_monitoring.helpers._get_tsdb_profile(
         project=project, secret_provider=secret_provider
@@ -93,9 +94,15 @@ def get_tsdb_connector(
         tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.TDEngine
         kwargs["connection_string"] = profile.dsn()
     else:
+        extra_message = (
+            ""
+            if profile
+            else " by using `project.set_model_monitoring_credentials` API"
+        )
         raise mlrun.errors.MLRunInvalidMMStoreTypeError(
-            "You must provide a valid tsdb store connection by using "
-            "set_model_monitoring_credentials API."
+            "You must provide a valid TSDB datastore profile"
+            f"{extra_message}. "
+            f"Found an unexpected profile of class: {type(profile)}"
         )
 
     # Get connector type value from ObjectTSDBFactory enum class

mlrun/model_monitoring/stream_processing.py

@@ -378,7 +378,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
     def do(self, full_event):
         event = full_event.body
         if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
-            logger.info("Skipped nop event inside of ProcessEndpointEvent", event=event)
+            logger.debug(
+                "Skipped nop event inside of ProcessEndpointEvent", event=event
+            )
             return storey.Event(body=[event])
         # Getting model version and function uri from event
         # and use them for retrieving the endpoint_id
@@ -637,7 +639,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
     def do(self, event: dict):
         if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
-            logger.info("Skipped nop event inside of MapFeatureNames", event=event)
             return event
         endpoint_id = event[EventFieldType.ENDPOINT_ID]
 

mlrun/projects/project.py

@@ -31,6 +31,7 @@ from os import environ, makedirs, path
 from typing import Callable, Optional, Union, cast
 from urllib.parse import urlparse
 
+import deprecated
 import dotenv
 import git
 import git.exc
@@ -1948,7 +1949,8 @@ class MlrunProject(ModelObj):
                     kwargs={"extract_images": True}
                 )
         :param upload: Whether to upload the artifact
-        :param labels: Key-value labels
+        :param labels:  Key-value labels. A 'source' label is automatically added using either
+                        local_path or target_path to facilitate easier document searching.
         :param target_path: Target file path
         :param kwargs: Additional keyword arguments
         :return: DocumentArtifact object
@@ -1978,13 +1980,24 @@ class MlrunProject(ModelObj):
                 "The document loader is configured to not support downloads but the upload flag is set to True."
                 "Either set loader.download_object=True or set upload=False"
             )
+        original_source = local_path or target_path
         doc_artifact = DocumentArtifact(
             key=key,
-            original_source=local_path or target_path,
+            original_source=original_source,
             document_loader_spec=document_loader_spec,
             collections=kwargs.pop("collections", None),
             **kwargs,
         )
+
+        # limit label to a max of 255 characters (for db reasons)
+        max_length = 255
+        labels = labels or {}
+        labels["source"] = (
+            original_source[: max_length - 3] + "..."
+            if len(original_source) > max_length
+            else original_source
+        )
+
         return self.log_artifact(
             item=doc_artifact,
             tag=tag,
@@ -2406,7 +2419,6 @@ class MlrunProject(ModelObj):
         *,
         deploy_histogram_data_drift_app: bool = True,
         wait_for_deployment: bool = False,
-        rebuild_images: bool = False,
         fetch_credentials_from_sys_config: bool = False,
     ) -> None:
         """
@@ -2428,7 +2440,6 @@ class MlrunProject(ModelObj):
         :param wait_for_deployment:               If true, return only after the deployment is done on the backend.
                                                   Otherwise, deploy the model monitoring infrastructure on the
                                                   background, including the histogram data drift app if selected.
-        :param rebuild_images:                    If true, force rebuild of model monitoring infrastructure images.
         :param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
         """
         if default_controller_image != "mlrun/mlrun":
@@ -2451,7 +2462,6 @@ class MlrunProject(ModelObj):
             image=image,
             base_period=base_period,
             deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
-            rebuild_images=rebuild_images,
             fetch_credentials_from_sys_config=fetch_credentials_from_sys_config,
         )
 
@@ -2839,6 +2849,13 @@ class MlrunProject(ModelObj):
 
         self.spec.set_function(name, function_object, func)
 
+    # TODO: Remove this in 1.10.0
+    @deprecated.deprecated(
+        version="1.8.0",
+        reason="'remove_function' is deprecated and will be removed in 1.10.0. "
+        "Please use `delete_function` instead.",
+        category=FutureWarning,
+    )
     def remove_function(self, name):
         """remove the specified function from the project
 
@@ -2846,6 +2863,18 @@ class MlrunProject(ModelObj):
         """
         self.spec.remove_function(name)
 
+    def delete_function(self, name, delete_from_db=False):
+        """deletes the specified function from the project
+
+        :param name: name of the function (under the project)
+        :param delete_from_db: default is False. If False, the function is removed
+                               only from the project's cache and spec.
+                               If True, the function is also removed from the database.
+        """
+        if delete_from_db:
+            mlrun.db.get_run_db().delete_function(name=name, project=self.metadata.name)
+        self.spec.remove_function(name)
+
     def remove_model_monitoring_function(self, name: Union[str, list[str]]):
         """delete the specified model-monitoring-app function/s
 
@@ -3762,8 +3791,8 @@ class MlrunProject(ModelObj):
                 "Please keep in mind that if you already had model monitoring functions "
                 "/ model monitoring infra / tracked model server "
                 "deployed on your project, you will need to redeploy them. "
-                "For redeploying the model monitoring infra, please use `enable_model_monitoring` API "
-                "and set `rebuild_images=True`"
+                "For redeploying the model monitoring infra, first disable it using "
+                "`project.disable_model_monitoring()` and then enable it using `project.enable_model_monitoring()`."
             )
 
     def list_model_endpoints(
@@ -3779,6 +3808,7 @@ class MlrunProject(ModelObj):
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
+        tsdb_metrics: bool = True,
     ) -> mlrun.common.schemas.ModelEndpointList:
         """
         Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
@@ -3829,6 +3859,7 @@ class MlrunProject(ModelObj):
             top_level=top_level,
             uids=uids,
             latest_only=latest_only,
+            tsdb_metrics=tsdb_metrics,
         )
 
     def run_function(

mlrun/runtimes/base.py

@@ -855,7 +855,7 @@ class BaseRuntime(ModelObj):
 
     def requires_build(self) -> bool:
         build = self.spec.build
-        return (
+        return bool(
             build.commands
             or build.requirements
             or (build.source and not build.load_source_on_run)

mlrun/runtimes/generators.py

@@ -182,7 +182,7 @@ class ListGenerator(TaskGenerator):
             yield newrun
 
 
-def get_run_copy(run):
+def get_run_copy(run: RunObject):
     newrun = deepcopy(run)
     newrun.spec.hyperparams = None
     newrun.spec.param_file = None

mlrun/runtimes/nuclio/function.py

@@ -599,6 +599,18 @@ class RemoteRuntime(KubeResource):
         # when a function is deployed, we wait for it to be ready by default
         # this also means that the function object will be updated with the function status
         self._wait_for_function_deployment(db, verbose=verbose)
+        # check if there are any background tasks related to creating model endpoints
+        model_endpoints_creation_background_tasks = (
+            mlrun.common.schemas.BackgroundTaskList(
+                **data.pop("background_tasks", {"background_tasks": []})
+            ).background_tasks
+        )
+        if model_endpoints_creation_background_tasks:
+            self._check_model_endpoint_task_state(
+                db=db,
+                background_task=model_endpoints_creation_background_tasks[0],
+                wait_for_completion=False,
+            )
 
         return self._enrich_command_from_status()
 
@@ -1285,6 +1297,33 @@ class RemoteRuntime(KubeResource):
             return mlrun.model.Credentials.generate_access_key
         return None
 
+    def _check_model_endpoint_task_state(
+        self,
+        db: mlrun.db.RunDBInterface,
+        background_task: mlrun.common.schemas.BackgroundTask,
+        wait_for_completion: bool,
+    ):
+        if wait_for_completion:
+            background_task = db._wait_for_background_task_to_reach_terminal_state(
+                name=background_task.metadata.name, project=self.metadata.project
+            )
+        else:
+            background_task = db.get_project_background_task(
+                project=self.metadata.project, name=background_task.metadata.name
+            )
+        if (
+            background_task.status.state
+            in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+        ):
+            logger.info(
+                f"Model endpoint creation task completed with state {background_task.status.state}"
+            )
+        else:
+            logger.warning(
+                f"Model endpoint creation task is still running with state {background_task.status.state}"
+                f"You can use the serving function, but it won't be monitored for the next few minutes"
+            )
+
 
 def parse_logs(logs):
     logs = json.loads(logs)

mlrun/runtimes/nuclio/serving.py

@@ -152,6 +152,7 @@ class ServingSpec(NuclioSpec):
         clone_target_dir=None,
         state_thresholds=None,
         disable_default_http_trigger=None,
+        model_endpoint_creation_task_name=None,
     ):
         super().__init__(
             command=command,
@@ -209,6 +210,7 @@ class ServingSpec(NuclioSpec):
         self.tracking_policy = tracking_policy
         self.secret_sources = secret_sources or []
         self.default_content_type = default_content_type
+        self.model_endpoint_creation_task_name = model_endpoint_creation_task_name
 
     @property
     def graph(self) -> Union[RouterStep, RootFlowStep]:
@@ -696,6 +698,7 @@ class ServingRuntime(RemoteRuntime):
             "track_models": self.spec.track_models,
             "tracking_policy": None,
             "default_content_type": self.spec.default_content_type,
+            "model_endpoint_creation_task_name": self.spec.model_endpoint_creation_task_name,
         }
 
         if self.spec.secret_sources:

mlrun/runtimes/pod.py

@@ -214,9 +214,7 @@ class KubeResourceSpec(FunctionSpec):
         # default service account is set in mlrun.utils.process_function_service_account
         # due to project specific defaults
         self.service_account = service_account
-        self.image_pull_secret = (
-            image_pull_secret or mlrun.mlconf.function.spec.image_pull_secret.default
-        )
+        self.image_pull_secret = image_pull_secret
         self.node_name = node_name
         self.node_selector = node_selector or {}
         self._affinity = affinity