mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/model_monitoring/controller.py

@@ -17,26 +17,28 @@ import datetime
 import json
 import os
 import re
-from typing import Any, Iterator, NamedTuple, Optional, Union, cast
+from collections.abc import Iterator
+from typing import Any, NamedTuple, Optional, Union, cast
 
-from v3io.dataplane.response import HttpResponseError
+import nuclio
 
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
 import mlrun.feature_store as fstore
+import mlrun.model_monitoring.db.stores
 from mlrun.common.model_monitoring.helpers import FeatureStats, pad_features_hist
 from mlrun.datastore import get_stream_pusher
 from mlrun.datastore.targets import ParquetTarget
-from mlrun.model_monitoring.batch import calculate_inputs_statistics
+from mlrun.errors import err_to_str
 from mlrun.model_monitoring.helpers import (
     _BatchDict,
     batch_dict2timedelta,
+    calculate_inputs_statistics,
     get_monitoring_parquet_path,
     get_stream_path,
 )
-from mlrun.utils import create_logger, datetime_now, logger
-from mlrun.utils.v3io_clients import get_v3io_client
+from mlrun.utils import datetime_now, logger
 
 
 class _Interval(NamedTuple):
@@ -45,8 +47,6 @@ class _Interval(NamedTuple):
 
 
 class _BatchWindow:
-    V3IO_CONTAINER_FORMAT = "users/pipelines/{project}/monitoring-schedules/functions"
-
     def __init__(
         self,
         project: str,
@@ -62,27 +62,22 @@ class _BatchWindow:
         All the time values are in seconds.
         The start and stop time are in seconds since the epoch.
         """
+        self.project = project
         self._endpoint = endpoint
         self._application = application
         self._first_request = first_request
-        self._kv_storage = get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api,
-            # Avoid noisy warning logs before the KV table is created
-            logger=create_logger(name="v3io_client", level="error"),
-        ).kv
-        self._v3io_container = self.V3IO_CONTAINER_FORMAT.format(project=project)
         self._stop = last_updated
         self._step = timedelta_seconds
+        self._db = mlrun.model_monitoring.get_store_object(project=self.project)
         self._start = self._get_last_analyzed()
 
     def _get_last_analyzed(self) -> Optional[int]:
         try:
-            data = self._kv_storage.get(
-                container=self._v3io_container,
-                table_path=self._endpoint,
-                key=self._application,
+            last_analyzed = self._db.get_last_analyzed(
+                endpoint_id=self._endpoint,
+                application_name=self._application,
             )
-        except HttpResponseError as err:
+        except mlrun.errors.MLRunNotFoundError:
             logger.info(
                 "No last analyzed time was found for this endpoint and "
                 "application, as this is probably the first time this "
@@ -93,7 +88,7 @@ class _BatchWindow:
                 first_request=self._first_request,
                 last_updated=self._stop,
             )
-            logger.debug("Error while getting last analyzed time", err=err)
+
             if self._first_request and self._stop:
                 # TODO : Change the timedelta according to the policy.
                 first_period_in_seconds = max(
@@ -105,7 +100,6 @@ class _BatchWindow:
                 )
             return self._first_request
 
-        last_analyzed = data.output.item[mm_constants.SchedulingKeys.LAST_ANALYZED]
         logger.info(
             "Got the last analyzed time for this endpoint and application",
             endpoint=self._endpoint,
@@ -121,11 +115,11 @@ class _BatchWindow:
             application=self._application,
             last_analyzed=last_analyzed,
         )
-        self._kv_storage.put(
-            container=self._v3io_container,
-            table_path=self._endpoint,
-            key=self._application,
-            attributes={mm_constants.SchedulingKeys.LAST_ANALYZED: last_analyzed},
+
+        self._db.update_last_analyzed(
+            endpoint_id=self._endpoint,
+            application_name=self._application,
+            last_analyzed=last_analyzed,
         )
 
     def get_intervals(
@@ -281,33 +275,33 @@ class MonitoringApplicationController:
 
     def __init__(
         self,
-        context: mlrun.run.MLClientCtx,
+        mlrun_context: mlrun.run.MLClientCtx,
         project: str,
     ):
         """
         Initialize Monitoring Application Processor object.
 
-        :param context:                     An MLRun context.
+        :param mlrun_context:               An MLRun context.
         :param project:                     Project name.
         """
-        self.context = context
+        self.context = mlrun_context
         self.project = project
         self.project_obj = mlrun.get_or_create_project(project)
 
-        context.logger.debug(f"Initializing {self.__class__.__name__}", project=project)
+        mlrun_context.logger.debug(
+            f"Initializing {self.__class__.__name__}", project=project
+        )
 
-        self.db = mlrun.model_monitoring.get_model_endpoint_store(project=project)
+        self.db = mlrun.model_monitoring.get_store_object(project=project)
 
         self._batch_window_generator = _BatchWindowGenerator(
-            batch_dict=context.parameters[
-                mm_constants.EventFieldType.BATCH_INTERVALS_DICT
-            ]
+            batch_dict=json.loads(
+                mlrun.get_secret_or_env(
+                    mm_constants.EventFieldType.BATCH_INTERVALS_DICT
+                )
+            )
         )
 
-        # If provided, only model endpoints in that that list will be analyzed
-        self.model_endpoints = context.parameters.get(
-            mm_constants.EventFieldType.MODEL_ENDPOINTS, None
-        )
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
         self.parquet_directory = get_monitoring_parquet_path(
             self.project_obj,
@@ -334,66 +328,91 @@ class MonitoringApplicationController:
             v3io_access_key=self.model_monitoring_access_key, v3io_api=self.v3io_api
         )
 
-    def run(self):
+    def run(self, event: nuclio.Event):
         """
         Main method for run all the relevant monitoring applications on each endpoint
+
+        :param event:   trigger event
         """
+        logger.info("Start running monitoring controller")
         try:
-            endpoints = self.db.list_model_endpoints(uids=self.model_endpoints)
+            applications_names = []
+            endpoints = self.db.list_model_endpoints()
+            if not endpoints:
+                self.context.logger.info(
+                    "No model endpoints found", project=self.project
+                )
+                return
             monitoring_functions = self.project_obj.list_model_monitoring_functions()
             if monitoring_functions:
+                # Gets only application in ready state
                 applications_names = list(
-                    {app.metadata.name for app in monitoring_functions}
+                    {
+                        app.metadata.name
+                        for app in monitoring_functions
+                        if (
+                            app.status.state == "ready"
+                            # workaround for the default app, as its `status.state` is `None`
+                            or app.metadata.name
+                            == mm_constants.HistogramDataDriftApplicationConstants.NAME
+                        )
+                    }
                 )
-            else:
+            if not applications_names:
                 self.context.logger.info(
                     "No monitoring functions found", project=self.project
                 )
-                applications_names = []
+                return
+            self.context.logger.info(
+                "Starting to iterate over the applications",
+                applications=applications_names,
+            )
 
         except Exception as e:
-            self.context.logger.error("Failed to list endpoints", exc=e)
-            return
-        if endpoints and applications_names:
-            # Initialize a process pool that will be used to run each endpoint applications on a dedicated process
-            pool = concurrent.futures.ProcessPoolExecutor(
-                max_workers=min(len(endpoints), 10),
+            self.context.logger.error(
+                "Failed to list endpoints and monitoring applications",
+                exc=err_to_str(e),
             )
-            futures = []
-            for endpoint in endpoints:
+            return
+        # Initialize a process pool that will be used to run each endpoint applications on a dedicated process
+        pool = concurrent.futures.ProcessPoolExecutor(
+            max_workers=min(len(endpoints), 10),
+        )
+        futures = []
+        for endpoint in endpoints:
+            if (
+                endpoint[mm_constants.EventFieldType.ACTIVE]
+                and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
+                == mm_constants.ModelMonitoringMode.enabled.value
+            ):
+                # Skip router endpoint:
                 if (
-                    endpoint[mm_constants.EventFieldType.ACTIVE]
-                    and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
-                    == mm_constants.ModelMonitoringMode.enabled.value
+                    int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
+                    == mm_constants.EndpointType.ROUTER
                 ):
-                    # Skip router endpoint:
-                    if (
-                        int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
-                        == mm_constants.EndpointType.ROUTER
-                    ):
-                        # Router endpoint has no feature stats
-                        logger.info(
-                            f"{endpoint[mm_constants.EventFieldType.UID]} is router skipping"
-                        )
-                        continue
-                    future = pool.submit(
-                        MonitoringApplicationController.model_endpoint_process,
-                        endpoint=endpoint,
-                        applications_names=applications_names,
-                        batch_window_generator=self._batch_window_generator,
-                        project=self.project,
-                        parquet_directory=self.parquet_directory,
-                        storage_options=self.storage_options,
-                        model_monitoring_access_key=self.model_monitoring_access_key,
+                    # Router endpoint has no feature stats
+                    logger.info(
+                        f"{endpoint[mm_constants.EventFieldType.UID]} is router skipping"
                     )
-                    futures.append(future)
+                    continue
+                future = pool.submit(
+                    MonitoringApplicationController.model_endpoint_process,
+                    endpoint=endpoint,
+                    applications_names=applications_names,
+                    batch_window_generator=self._batch_window_generator,
+                    project=self.project,
+                    parquet_directory=self.parquet_directory,
+                    storage_options=self.storage_options,
+                    model_monitoring_access_key=self.model_monitoring_access_key,
+                )
+                futures.append(future)
 
-            for future in concurrent.futures.as_completed(futures):
-                result = future.result()
-                if result:
-                    self.context.log_results(result)
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result:
+                self.context.log_results(result)
 
-            self._delete_old_parquet(endpoints=endpoints)
+        self._delete_old_parquet(endpoints=endpoints)
 
     @classmethod
     def model_endpoint_process(
@@ -438,6 +457,7 @@ class MonitoringApplicationController:
                 )
 
                 for start_infer_time, end_infer_time in batch_window.get_intervals():
+                    # start - TODO : delete in 1.9.0 (V1 app deprecation)
                     try:
                         # Get application sample data
                         offline_response = cls._get_sample_df(
@@ -483,10 +503,9 @@ class MonitoringApplicationController:
 
                     # Get the current stats:
                     current_stats = calculate_inputs_statistics(
-                        sample_set_statistics=feature_stats,
-                        inputs=df,
+                        sample_set_statistics=feature_stats, inputs=df
                     )
-
+                    # end - TODO : delete in 1.9.0 (V1 app deprecation)
                     cls._push_to_applications(
                         current_stats=current_stats,
                         feature_stats=feature_stats,
@@ -517,7 +536,7 @@ class MonitoringApplicationController:
         """
         if self.parquet_directory.startswith("v3io:///"):
             # create fs with access to the user side (under projects)
-            store, _ = mlrun.store_manager.get_or_create_store(
+            store, _, _ = mlrun.store_manager.get_or_create_store(
                 self.parquet_directory,
                 {"V3IO_ACCESS_KEY": self.model_monitoring_access_key},
             )
@@ -593,12 +612,13 @@ class MonitoringApplicationController:
             mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
             mm_constants.ApplicationEvent.OUTPUT_STREAM_URI: get_stream_path(
                 project=project,
-                application_name=mm_constants.MonitoringFunctionNames.WRITER,
+                function_name=mm_constants.MonitoringFunctionNames.WRITER,
             ),
+            mm_constants.ApplicationEvent.MLRUN_CONTEXT: {},  # TODO : for future use by ad-hoc batch infer
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})
-            stream_uri = get_stream_path(project=project, application_name=app_name)
+            stream_uri = get_stream_path(project=project, function_name=app_name)
 
             logger.info(
                 f"push endpoint_id {endpoint_id} to {app_name} by stream :{stream_uri}"

mlrun/model_monitoring/controller_handler.py

@@ -11,19 +11,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import nuclio
 
 import mlrun
 from mlrun.model_monitoring.controller import MonitoringApplicationController
 
 
-def handler(context: mlrun.run.MLClientCtx) -> None:
+def handler(context: nuclio.Context, event: nuclio.Event) -> None:
     """
     Run model monitoring application processor
 
-    :param context: the MLRun context
+    :param context: the Nuclio context
+    :param event:   trigger event
     """
+    context.user_data.monitor_app_controller.run(event)
+
+
+def init_context(context):
+    mlrun_context = mlrun.get_or_create_ctx("model_monitoring_controller")
+    mlrun_context.logger.info("Initialize monitoring app controller")
     monitor_app_controller = MonitoringApplicationController(
-        context=context,
-        project=context.project,
+        mlrun_context=mlrun_context,
+        project=mlrun_context.project,
     )
-    monitor_app_controller.run()
+    setattr(context.user_data, "monitor_app_controller", monitor_app_controller)

mlrun/model_monitoring/db/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .stores import ObjectStoreFactory, get_store_object
+from .stores.base import StoreBase
+from .tsdb import get_tsdb_connector
+from .tsdb.base import TSDBConnector

mlrun/model_monitoring/{stores → db/stores}/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Iguazio
+# Copyright 2024 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,64 +12,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 import enum
 import typing
+import warnings
 
 import mlrun.common.schemas.secret
 import mlrun.errors
 
-from .model_endpoint_store import ModelEndpointStore
+from .base import StoreBase
 
 
-class ModelEndpointStoreType(enum.Enum):
-    """Enum class to handle the different store type values for saving a model endpoint record."""
+class ObjectStoreFactory(enum.Enum):
+    """Enum class to handle the different store type values for saving model monitoring records."""
 
     v3io_nosql = "v3io-nosql"
     SQL = "sql"
 
-    def to_endpoint_store(
+    def to_object_store(
         self,
         project: str,
         access_key: str = None,
-        endpoint_store_connection: str = None,
         secret_provider: typing.Callable = None,
-    ) -> ModelEndpointStore:
+    ) -> StoreBase:
         """
-        Return a ModelEndpointStore object based on the provided enum value.
-
-        :param project:                    The name of the project.
-        :param access_key:                 Access key with permission to the DB table. Note that if access key is None
-                                           and the endpoint target is from type KV then the access key will be
-                                           retrieved from the environment variable.
-        :param endpoint_store_connection: A valid connection string for model endpoint target. Contains several
-                                          key-value pairs that required for the database connection.
-                                          e.g. A root user with password 1234, tries to connect a schema called
-                                          mlrun within a local MySQL DB instance:
-                                          'mysql+pymysql://root:1234@localhost:3306/mlrun'.
+        Return a StoreBase object based on the provided enum value.
+
+        :param project:                   The name of the project.
+        :param access_key:                Access key with permission to the DB table. Note that if access key is None
+                                          and the endpoint target is from type KV then the access key will be
+                                          retrieved from the environment variable.
         :param secret_provider:           An optional secret provider to get the connection string secret.
 
-        :return: `ModelEndpointStore` object.
+        :return: `StoreBase` object.
 
         """
 
-        if self.value == ModelEndpointStoreType.v3io_nosql.value:
-            from .kv_model_endpoint_store import KVModelEndpointStore
+        if self == self.v3io_nosql:
+            from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
 
             # Get V3IO access key from env
             access_key = access_key or mlrun.mlconf.get_v3io_access_key()
 
-            return KVModelEndpointStore(project=project, access_key=access_key)
+            return KVStoreBase(project=project, access_key=access_key)
 
         # Assuming SQL store target if store type is not KV.
         # Update these lines once there are more than two store target types.
 
-        from .sql_model_endpoint_store import SQLModelEndpointStore
+        from mlrun.model_monitoring.db.stores.sqldb.sql_store import SQLStoreBase
 
-        return SQLModelEndpointStore(
+        return SQLStoreBase(
             project=project,
-            sql_connection_string=endpoint_store_connection,
             secret_provider=secret_provider,
         )
 
@@ -88,7 +80,24 @@ def get_model_endpoint_store(
     project: str,
     access_key: str = None,
     secret_provider: typing.Callable = None,
-) -> ModelEndpointStore:
+) -> StoreBase:
+    # Leaving here for backwards compatibility
+    warnings.warn(
+        "The 'get_model_endpoint_store' function is deprecated and will be removed in 1.9.0. "
+        "Please use `get_store_object` instead.",
+        # TODO: remove in 1.9.0
+        FutureWarning,
+    )
+    return get_store_object(
+        project=project, access_key=access_key, secret_provider=secret_provider
+    )
+
+
+def get_store_object(
+    project: str,
+    access_key: str = None,
+    secret_provider: typing.Callable = None,
+) -> StoreBase:
     """
     Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
 
@@ -96,16 +105,14 @@ def get_model_endpoint_store(
     :param access_key:      Access key with permission to the DB table.
     :param secret_provider: An optional secret provider to get the connection string secret.
 
-    :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
-             model endpoint record such as write, update, get and delete.
+    :return: `StoreBase` object. Using this object, the user can apply different operations on the
+             model monitoring record such as write, update, get and delete a model endpoint.
     """
 
-    # Get store type value from ModelEndpointStoreType enum class
-    model_endpoint_store_type = ModelEndpointStoreType(
-        mlrun.mlconf.model_endpoint_monitoring.store_type
-    )
+    # Get store type value from ObjectStoreFactory enum class
+    store_type = ObjectStoreFactory(mlrun.mlconf.model_endpoint_monitoring.store_type)
 
-    # Convert into model endpoint store target object
-    return model_endpoint_store_type.to_endpoint_store(
+    # Convert into store target object
+    return store_type.to_object_store(
         project=project, access_key=access_key, secret_provider=secret_provider
     )

mlrun/model_monitoring/db/stores/base/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .store import StoreBase