mlrun 1.8.0rc26__py3-none-any.whl → 1.8.0rc28__py3-none-any.whl

mlrun/model_monitoring/applications/base.py

@@ -215,6 +215,116 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         )
         function.deploy()
 
+    @classmethod
+    def get_job_handler(cls, handler_to_class: str) -> str:
+        """
+        A helper function to get the handler to the application job ``_handler``.
+
+        :param handler_to_class: The handler to the application class, e.g. ``my_package.sub_module1.MonitoringApp1``.
+        :returns:                The handler to the job of the application class.
+        """
+        return f"{handler_to_class}::{cls._handler.__name__}"
+
+    @classmethod
+    def to_job(
+        cls,
+        *,
+        class_handler: Optional[str] = None,
+        func_path: Optional[str] = None,
+        func_name: Optional[str] = None,
+        tag: Optional[str] = None,
+        image: Optional[str] = None,
+        with_repo: Optional[bool] = False,
+        requirements: Optional[Union[str, list[str]]] = None,
+        requirements_file: str = "",
+        project: Optional["mlrun.MlrunProject"] = None,
+    ) -> mlrun.runtimes.KubejobRuntime:
+        """
+        Get the application's :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
+        model monitoring logic as a :py:class:`~mlrun.runtimes.KubejobRuntime`.
+
+        The returned job can be run as any MLRun job with the relevant inputs and params to your application:
+
+        .. code-block:: python
+
+            job = ModelMonitoringApplicationBase.to_job(
+                class_handler="package.module.AppClass"
+            )
+            job.run(inputs={}, params={}, local=False)  # Add the relevant inputs and params
+
+        Optional inputs:
+
+        * ``sample_data``, ``pd.DataFrame``
+        * ``reference_data``, ``pd.DataFrame``
+
+        Optional params:
+
+        * ``endpoints``, ``list[tuple[str, str]]``
+        * ``start``, ``datetime``
+        * ``end``, ``datetime``
+        * ``base_period``, ``int``
+
+        For Git sources, add the source archive to the returned job and change the handler:
+
+        .. code-block:: python
+
+            handler = ModelMonitoringApplicationBase.get_job_handler("module.AppClass")
+            job.with_source_archive(
+                "git://github.com/owner/repo.git#branch-category/specific-task",
+                workdir="path/to/application/folder",
+                handler=handler,
+            )
+
+        :param class_handler:     The handler to the class, e.g. ``path.to.module::MonitoringApplication``,
+                                  useful when using Git sources or code from images.
+                                  If ``None``, the current class, deriving from
+                                  :py:class:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase`,
+                                  is used.
+        :param func_path:         The path to the function. If ``None``, the current notebook is used.
+        :param func_name:         The name of the function. If not ``None``, the class name is used.
+        :param tag:               Tag for the function.
+        :param image:             Docker image to run the job on (when running remotely).
+        :param with_repo:         Whether to clone the current repo to the build source.
+        :param requirements:      List of Python requirements to be installed in the image.
+        :param requirements_file: Path to a Python requirements file to be installed in the image.
+        :param project:           The current project to set the function to. If not set, the current project is used.
+
+        :returns: The :py:class:`~mlrun.runtimes.KubejobRuntime` job that wraps the model monitoring application's
+                  logic.
+        """
+        project = project or cast("mlrun.MlrunProject", mlrun.get_current_project())
+
+        if not class_handler and cls == ModelMonitoringApplicationBase:
+            raise ValueError(
+                "You must provide a handler to the model monitoring application class"
+            )
+
+        handler_to_class = class_handler or cls.__name__
+        handler = cls.get_job_handler(handler_to_class)
+
+        if not class_handler:
+            class_name = cls.__name__
+        else:
+            class_name = handler_to_class.split(".")[-1].split("::")[-1]
+
+        job_name = func_name if func_name else class_name
+
+        job = cast(
+            mlrun.runtimes.KubejobRuntime,
+            project.set_function(
+                func=func_path,
+                name=job_name,
+                kind=mlrun.runtimes.KubejobRuntime.kind,
+                handler=handler,
+                tag=tag,
+                image=image,
+                with_repo=with_repo,
+                requirements=requirements,
+                requirements_file=requirements_file,
+            ),
+        )
+        return job
+
     @classmethod
     def evaluate(
         cls,
@@ -223,10 +333,12 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         *,
         tag: Optional[str] = None,
         run_local: bool = True,
+        auto_build: bool = True,
         sample_data: Optional[pd.DataFrame] = None,
         reference_data: Optional[pd.DataFrame] = None,
         image: Optional[str] = None,
         with_repo: Optional[bool] = False,
+        class_handler: Optional[str] = None,
         requirements: Optional[Union[str, list[str]]] = None,
         requirements_file: str = "",
         endpoints: Optional[list[tuple[str, str]]] = None,
@@ -239,19 +351,21 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
         model monitoring logic as a :py:class:`~mlrun.runtimes.KubejobRuntime`, which is an MLRun function.
 
-        This method has default values for all of its arguments. You should be change them when you want to pass
+        This function has default values for all of its arguments. You should be change them when you want to pass
         data to the application.
 
         :param func_path:         The path to the function. If ``None``, the current notebook is used.
         :param func_name:         The name of the function. If not ``None``, the class name is used.
         :param tag:               Tag for the function.
         :param run_local:         Whether to run the function locally or remotely.
+        :param auto_build:        Whether to auto build the function.
         :param sample_data:       Pandas data-frame as the current dataset.
                                   When set, it replaces the data read from the model endpoint's offline source.
         :param reference_data:    Pandas data-frame of the reference dataset.
                                   When set, its statistics override the model endpoint's feature statistics.
-        :param image:             Docker image to run the job on.
+        :param image:             Docker image to run the job on (when running remotely).
         :param with_repo:         Whether to clone the current repo to the build source.
+        :param class_handler:     The relative path to the class, useful when using Git sources or code from images.
         :param requirements:      List of Python requirements to be installed in the image.
         :param requirements_file: Path to a Python requirements file to be installed in the image.
         :param endpoints:         A list of tuples of the model endpoint (name, uid) to get the data from.
@@ -268,23 +382,17 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                   method with the given parameters and inputs, wrapped in a :py:class:`~mlrun.model.RunObject`.
         """
         project = cast("mlrun.MlrunProject", mlrun.get_current_project())
-        class_name = cls.__name__
-        job_name = func_name if func_name is not None else class_name
-        handler = f"{class_name}::{cls._handler.__name__}"
 
-        job = cast(
-            mlrun.runtimes.KubejobRuntime,
-            project.set_function(
-                func=func_path,
-                name=job_name,
-                kind=mlrun.runtimes.KubejobRuntime.kind,
-                handler=handler,
-                tag=tag,
-                image=image,
-                with_repo=with_repo,
-                requirements=requirements,
-                requirements_file=requirements_file,
-            ),
+        job = cls.to_job(
+            func_path=func_path,
+            func_name=func_name,
+            class_handler=class_handler,
+            tag=tag,
+            image=image,
+            with_repo=with_repo,
+            requirements=requirements,
+            requirements_file=requirements_file,
+            project=project,
         )
 
         params: dict[str, Union[list[tuple[str, str]], datetime, int, None]] = {}
@@ -305,18 +413,21 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             (reference_data, "reference_data"),
         ]:
             if data is not None:
-                key = f"{job_name}_{identifier}"
+                key = f"{job.metadata.name}_{identifier}"
                 inputs[identifier] = project.log_dataset(
                     key,
                     data,
                     labels={
                         mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
                         mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-job",
-                        mlrun_constants.MLRunInternalLabels.app_name: class_name,
+                        mlrun_constants.MLRunInternalLabels.app_name: func_name
+                        or cls.__name__,
                     },
                 ).uri
 
-        run_result = job.run(local=run_local, params=params, inputs=inputs)
+        run_result = job.run(
+            local=run_local, auto_build=auto_build, params=params, inputs=inputs
+        )
         return run_result
 
     @abstractmethod

mlrun/model_monitoring/applications/context.py

@@ -169,9 +169,8 @@ class MonitoringApplicationContext:
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
     ) -> "MonitoringApplicationContext":
-        project = mlrun.load_project(url=graph_context.project)
         nuclio_logger = graph_context.logger
-        artifacts_logger = project
+        artifacts_logger = graph_context.project_obj
         logger = mlrun.utils.create_logger(
             level=mlrun.mlconf.log_level,
             formatter_kind=mlrun.mlconf.log_formatter,
@@ -180,7 +179,7 @@ class MonitoringApplicationContext:
         return cls(
             application_name=application_name,
             event=event,
-            project=project,
+            project=graph_context.project_obj,
             model_endpoint_dict=model_endpoint_dict,
             logger=logger,
             nuclio_logger=nuclio_logger,

mlrun/model_monitoring/controller.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import concurrent.futures
 import datetime
 import json
 import os
@@ -138,7 +139,9 @@ class _BatchWindow:
 
 
 class _BatchWindowGenerator(AbstractContextManager):
-    def __init__(self, project: str, endpoint_id: str, window_length: int) -> None:
+    def __init__(
+        self, project: str, endpoint_id: str, window_length: Optional[int] = None
+    ) -> None:
         """
         Initialize a batch window generator object that generates batch window objects
         for the monitoring functions.
@@ -165,6 +168,12 @@ class _BatchWindowGenerator(AbstractContextManager):
             exc_type=exc_type, exc_value=exc_value, traceback=traceback
         )
 
+    def get_application_list(self) -> set[str]:
+        return self._schedules_file.get_application_list()
+
+    def get_min_last_analyzed(self) -> Optional[int]:
+        return self._schedules_file.get_min_timestamp()
+
     @classmethod
     def _get_last_updated_time(
         cls, last_request: datetime.datetime, not_batch_endpoint: bool
@@ -234,8 +243,7 @@ class MonitoringApplicationController:
     def __init__(self) -> None:
         """Initialize Monitoring Application Controller"""
         self.project = cast(str, mlrun.mlconf.default_project)
-        self.project_obj = mlrun.load_project(name=self.project, url=self.project)
-
+        self.project_obj = mlrun.get_run_db().get_project(name=self.project)
         logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
 
         self._window_length = _get_window_length()
@@ -255,8 +263,10 @@ class MonitoringApplicationController:
         return access_key
 
     @staticmethod
-    def _should_monitor_endpoint(endpoint: mlrun.common.schemas.ModelEndpoint) -> bool:
-        return (
+    def _should_monitor_endpoint(
+        endpoint: mlrun.common.schemas.ModelEndpoint, application_names: set
+    ) -> bool:
+        if (
             # Is the model endpoint monitored?
             endpoint.status.monitoring_mode == mm_constants.ModelMonitoringMode.enabled
             # Was the model endpoint called? I.e., are the first and last requests nonempty?
@@ -265,7 +275,40 @@ class MonitoringApplicationController:
             # Is the model endpoint not a router endpoint? Router endpoint has no feature stats
             and endpoint.metadata.endpoint_type.value
             != mm_constants.EndpointType.ROUTER.value
-        )
+        ):
+            with _BatchWindowGenerator(
+                project=endpoint.metadata.project,
+                endpoint_id=endpoint.metadata.uid,
+            ) as batch_window_generator:
+                if application_names != batch_window_generator.get_application_list():
+                    return True
+                elif (
+                    not batch_window_generator.get_min_last_analyzed()
+                    or batch_window_generator.get_min_last_analyzed()
+                    <= int(endpoint.status.last_request.timestamp())
+                ):
+                    return True
+                else:
+                    logger.info(
+                        "All the possible intervals were already analyzed, didn't push regular event",
+                        endpoint_id=endpoint.metadata.uid,
+                        last_analyzed=datetime.datetime.fromtimestamp(
+                            batch_window_generator.get_min_last_analyzed(),
+                            tz=datetime.timezone.utc,
+                        ),
+                        last_request=endpoint.status.last_request,
+                    )
+        else:
+            logger.info(
+                "Should not monitor model endpoint, didn't push regular event",
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                last_request=endpoint.status.last_request,
+                first_request=endpoint.status.first_request,
+                endpoint_type=endpoint.metadata.endpoint_type,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+            )
+        return False
 
     def run(self, event: nuclio_sdk.Event) -> None:
         """
@@ -314,7 +357,7 @@ class MonitoringApplicationController:
             )
             m_fs = fstore.get_feature_set(event[ControllerEvent.FEATURE_SET_URI])
             logger.info(
-                "Starting analyzing for:", timestamp=event[ControllerEvent.TIMESTAMP]
+                "Starting analyzing for", timestamp=event[ControllerEvent.TIMESTAMP]
             )
             last_stream_timestamp = datetime.datetime.fromisoformat(
                 event[ControllerEvent.TIMESTAMP]
@@ -370,7 +413,7 @@ class MonitoringApplicationController:
                 current_time = mlrun.utils.datetime_now()
                 if (
                     current_time.timestamp()
-                    - batch_window_generator.batch_window._get_last_analyzed()
+                    - batch_window_generator.get_min_last_analyzed()
                     >= datetime.timedelta(minutes=base_period).total_seconds()
                     and event[ControllerEvent.KIND] != ControllerEventKind.NOP_EVENT
                 ):
@@ -399,6 +442,9 @@ class MonitoringApplicationController:
                         event=event,
                         endpoint_id=endpoint_id,
                     )
+            logger.info(
+                "Finish analyze for", timestamp=event[ControllerEvent.TIMESTAMP]
+            )
 
         except Exception:
             logger.exception(
@@ -455,17 +501,14 @@ class MonitoringApplicationController:
                 [data]
             )
 
-    def push_regular_event_to_controller_stream(self, event: nuclio_sdk.Event) -> None:
+    def push_regular_event_to_controller_stream(self) -> None:
         """
         pushes a regular event to the controller stream.
         :param event: the nuclio trigger event
         """
         logger.info("Starting monitoring controller chief")
         applications_names = []
-        db = mlrun.get_run_db()
-        endpoints = db.list_model_endpoints(
-            project=self.project, tsdb_metrics=True
-        ).endpoints
+        endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=True).endpoints
         if not endpoints:
             logger.info("No model endpoints found", project=self.project)
             return
@@ -505,48 +548,59 @@ class MonitoringApplicationController:
                 // 60
             ),
         }
-        for endpoint in endpoints:
-            if self._should_monitor_endpoint(endpoint):
-                logger.info(
-                    "Regular event is being pushed to controller stream for model endpoint",
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    timestamp=endpoint.status.last_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    first_request=endpoint.status.first_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
-                    endpoint_policy=json.dumps(policy),
-                )
-                self.push_to_controller_stream(
-                    kind=mm_constants.ControllerEventKind.REGULAR_EVENT,
-                    project=self.project,
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    stream_access_key=self.v3io_access_key,
-                    timestamp=endpoint.status.last_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    first_request=endpoint.status.first_request.isoformat(
-                        sep=" ", timespec="microseconds"
-                    ),
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
-                    endpoint_policy=policy,
-                )
-            else:
-                logger.info(
-                    "Should not monitor model endpoint, didn't push regular event",
-                    endpoint_id=endpoint.metadata.uid,
-                    endpoint_name=endpoint.metadata.name,
-                    timestamp=endpoint.status.last_request,
-                    first_request=endpoint.status.first_request,
-                    endpoint_type=endpoint.metadata.endpoint_type,
-                    feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=min(len(endpoints), 10)
+        ) as pool:
+            for endpoint in endpoints:
+                pool.submit(
+                    MonitoringApplicationController.endpoint_to_regular_event,
+                    endpoint,
+                    policy,
+                    set(applications_names),
+                    self.v3io_access_key,
                 )
+        logger.info("Finishing monitoring controller chief")
+
+    @staticmethod
+    def endpoint_to_regular_event(
+        endpoint: mlrun.common.schemas.ModelEndpoint,
+        policy: dict,
+        applications_names: set,
+        v3io_access_key: str,
+    ) -> None:
+        if MonitoringApplicationController._should_monitor_endpoint(
+            endpoint, set(applications_names)
+        ):
+            logger.info(
+                "Regular event is being pushed to controller stream for model endpoint",
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                timestamp=endpoint.status.last_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                first_request=endpoint.status.first_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                endpoint_type=endpoint.metadata.endpoint_type,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+                endpoint_policy=json.dumps(policy),
+            )
+            MonitoringApplicationController.push_to_controller_stream(
+                kind=mm_constants.ControllerEventKind.REGULAR_EVENT,
+                project=endpoint.metadata.project,
+                endpoint_id=endpoint.metadata.uid,
+                endpoint_name=endpoint.metadata.name,
+                stream_access_key=v3io_access_key,
+                timestamp=endpoint.status.last_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                first_request=endpoint.status.first_request.isoformat(
+                    sep=" ", timespec="microseconds"
+                ),
+                endpoint_type=endpoint.metadata.endpoint_type.value,
+                feature_set_uri=endpoint.spec.monitoring_feature_set_uri,
+                endpoint_policy=policy,
+            )
 
     @staticmethod
     def push_to_controller_stream(
@@ -557,7 +611,7 @@ class MonitoringApplicationController:
         stream_access_key: str,
         timestamp: str,
         first_request: str,
-        endpoint_type: str,
+        endpoint_type: int,
         feature_set_uri: str,
         endpoint_policy: dict[str, Any],
     ) -> None:
@@ -633,7 +687,13 @@ def handler(context: nuclio_sdk.Context, event: nuclio_sdk.Event) -> None:
 
     if event.trigger.kind == "http":
         # Runs controller chief:
-        MonitoringApplicationController().push_regular_event_to_controller_stream(event)
+        context.user_data.monitor_app_controller.push_regular_event_to_controller_stream()
     else:
         # Runs controller worker:
-        MonitoringApplicationController().run(event=event)
+        context.user_data.monitor_app_controller.run(event)
+
+
+def init_context(context):
+    monitor_app_controller = MonitoringApplicationController()
+    setattr(context.user_data, "monitor_app_controller", monitor_app_controller)
+    context.logger.info("Monitoring application controller initialized")

mlrun/model_monitoring/db/_schedules.py

@@ -140,6 +140,14 @@ class ModelMonitoringSchedulesFile(AbstractContextManager):
         self._check_open_schedules()
         self._schedules[application] = timestamp
 
+    def get_application_list(self) -> set[str]:
+        self._check_open_schedules()
+        return set(self._schedules.keys())
+
+    def get_min_timestamp(self) -> Optional[int]:
+        self._check_open_schedules()
+        return min(self._schedules.values(), default=None)
+
 
 def delete_model_monitoring_schedules_folder(project: str) -> None:
     """Delete the model monitoring schedules folder of the project"""

mlrun/model_monitoring/db/tsdb/__init__.py

@@ -75,10 +75,11 @@ def get_tsdb_connector(
     :param secret_provider:         An optional secret provider to get the connection string secret.
     :param profile:                 An optional profile to initialize the TSDB connector from.
 
-    :return: `TSDBConnector` object. The main goal of this object is to handle different operations on the
+    :return: ``TSDBConnector`` object. The main goal of this object is to handle different operations on the
              TSDB connector such as updating drift metrics or write application record result.
-    :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide TSDB connection
-            or the provided TSDB connection is invalid.
+    :raise: ``MLRunNotFoundError`` if the user didn't set the TSDB datastore profile and didn't provide it through
+            the ``profile`` parameter.
+    :raise: ``MLRunInvalidMMStoreTypeError`` if the TSDB datastore profile is of an invalid type.
     """
     profile = profile or mlrun.model_monitoring.helpers._get_tsdb_profile(
         project=project, secret_provider=secret_provider
@@ -93,9 +94,15 @@ def get_tsdb_connector(
         tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.TDEngine
         kwargs["connection_string"] = profile.dsn()
     else:
+        extra_message = (
+            ""
+            if profile
+            else " by using `project.set_model_monitoring_credentials` API"
+        )
         raise mlrun.errors.MLRunInvalidMMStoreTypeError(
-            "You must provide a valid tsdb store connection by using "
-            "set_model_monitoring_credentials API."
+            "You must provide a valid TSDB datastore profile"
+            f"{extra_message}. "
+            f"Found an unexpected profile of class: {type(profile)}"
         )
 
     # Get connector type value from ObjectTSDBFactory enum class

mlrun/model_monitoring/stream_processing.py

@@ -378,7 +378,9 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
     def do(self, full_event):
         event = full_event.body
         if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
-            logger.info("Skipped nop event inside of ProcessEndpointEvent", event=event)
+            logger.debug(
+                "Skipped nop event inside of ProcessEndpointEvent", event=event
+            )
             return storey.Event(body=[event])
         # Getting model version and function uri from event
         # and use them for retrieving the endpoint_id
@@ -637,7 +639,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
 
     def do(self, event: dict):
         if event.get(ControllerEvent.KIND, "") == ControllerEventKind.NOP_EVENT:
-            logger.info("Skipped nop event inside of MapFeatureNames", event=event)
             return event
         endpoint_id = event[EventFieldType.ENDPOINT_ID]