mlrun 1.8.0rc4__py3-none-any.whl → 1.8.0rc7__py3-none-any.whl

mlrun/model_monitoring/controller.py

@@ -19,7 +19,7 @@ import os
 from collections.abc import Iterator
 from contextlib import AbstractContextManager
 from types import TracebackType
-from typing import Any, NamedTuple, Optional, cast
+from typing import NamedTuple, Optional, cast
 
 import nuclio_sdk
 
@@ -27,6 +27,7 @@ import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store as fstore
 import mlrun.model_monitoring
+from mlrun.common.schemas import EndpointType
 from mlrun.datastore import get_stream_pusher
 from mlrun.errors import err_to_str
 from mlrun.model_monitoring.db._schedules import ModelMonitoringSchedulesFile
@@ -65,7 +66,7 @@ class _BatchWindow:
         self._start = self._get_last_analyzed()
 
     def _get_saved_last_analyzed(self) -> Optional[int]:
-        return self._db.get_application_time(self._application)
+        return cast(int, self._db.get_application_time(self._application))
 
     def _update_last_analyzed(self, last_analyzed: int) -> None:
         self._db.update_application_time(
@@ -161,18 +162,20 @@ class _BatchWindowGenerator(AbstractContextManager):
         )
 
     @classmethod
-    def _get_last_updated_time(cls, last_request: str, has_stream: bool) -> int:
+    def _get_last_updated_time(
+        cls, last_request: datetime.datetime, not_batch_endpoint: bool
+    ) -> int:
         """
         Get the last updated time of a model endpoint.
         """
         last_updated = int(
-            cls._date_string2timestamp(last_request)
+            last_request.timestamp()
             - cast(
                 float,
                 mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs,
             )
         )
-        if not has_stream:
+        if not not_batch_endpoint:
             # If the endpoint does not have a stream, `last_updated` should be
             # the minimum between the current time and the last updated time.
             # This compensates for the bumping mechanism - see
@@ -183,17 +186,13 @@ class _BatchWindowGenerator(AbstractContextManager):
             )
         return last_updated
 
-    @staticmethod
-    def _date_string2timestamp(date_string: str) -> int:
-        return int(datetime.datetime.fromisoformat(date_string).timestamp())
-
     def get_intervals(
         self,
         *,
         application: str,
-        first_request: str,
-        last_request: str,
-        has_stream: bool,
+        first_request: datetime.datetime,
+        last_request: datetime.datetime,
+        not_batch_endpoint: bool,
     ) -> Iterator[_Interval]:
         """
         Get the batch window for a specific endpoint and application.
@@ -204,8 +203,8 @@ class _BatchWindowGenerator(AbstractContextManager):
             schedules_file=self._schedules_file,
             application=application,
             timedelta_seconds=self._timedelta,
-            last_updated=self._get_last_updated_time(last_request, has_stream),
-            first_request=self._date_string2timestamp(first_request),
+            last_updated=self._get_last_updated_time(last_request, not_batch_endpoint),
+            first_request=int(first_request.timestamp()),
         )
         yield from batch_window.get_intervals()
 
@@ -235,8 +234,6 @@ class MonitoringApplicationController:
 
         logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
 
-        self.db = mlrun.model_monitoring.get_store_object(project=self.project)
-
         self._window_length = _get_window_length()
 
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
@@ -253,19 +250,16 @@ class MonitoringApplicationController:
         return access_key
 
     @staticmethod
-    def _should_monitor_endpoint(endpoint: dict[str, Any]) -> bool:
+    def _should_monitor_endpoint(endpoint: mlrun.common.schemas.ModelEndpoint) -> bool:
         return (
-            # Is the model endpoint active?
-            endpoint[mm_constants.EventFieldType.ACTIVE]
             # Is the model endpoint monitored?
-            and endpoint[mm_constants.EventFieldType.MONITORING_MODE]
-            == mm_constants.ModelMonitoringMode.enabled
+            endpoint.status.monitoring_mode == mm_constants.ModelMonitoringMode.enabled
             # Was the model endpoint called? I.e., are the first and last requests nonempty?
-            and endpoint[mm_constants.EventFieldType.FIRST_REQUEST]
-            and endpoint[mm_constants.EventFieldType.LAST_REQUEST]
+            and endpoint.status.first_request
+            and endpoint.status.last_request
             # Is the model endpoint not a router endpoint? Router endpoint has no feature stats
-            and int(endpoint[mm_constants.EventFieldType.ENDPOINT_TYPE])
-            != mm_constants.EndpointType.ROUTER
+            and endpoint.metadata.endpoint_type.value
+            != mm_constants.EndpointType.ROUTER.value
         )
 
     def run(self) -> None:
@@ -281,7 +275,10 @@ class MonitoringApplicationController:
         logger.info("Start running monitoring controller")
         try:
             applications_names = []
-            endpoints = self.db.list_model_endpoints(include_stats=True)
+            endpoints_list = mlrun.db.get_run_db().list_model_endpoints(
+                project=self.project, tsdb_metrics=True
+            )
+            endpoints = endpoints_list.endpoints
             if not endpoints:
                 logger.info("No model endpoints found", project=self.project)
                 return
@@ -333,12 +330,19 @@ class MonitoringApplicationController:
                         model_monitoring_access_key=self.model_monitoring_access_key,
                         storage_options=self.storage_options,
                     )
+                else:
+                    logger.debug(
+                        "Skipping endpoint, not ready or not suitable for monitoring",
+                        endpoint_id=endpoint.metadata.uid,
+                        endpoint_name=endpoint.metadata.name,
+                    )
+        logger.info("Finished running monitoring controller")
 
     @classmethod
     def model_endpoint_process(
         cls,
         project: str,
-        endpoint: dict,
+        endpoint: mlrun.common.schemas.ModelEndpoint,
         applications_names: list[str],
         window_length: int,
         model_monitoring_access_key: str,
@@ -356,11 +360,11 @@ class MonitoringApplicationController:
         :param model_monitoring_access_key: (str) Access key to apply the model monitoring process.
         :param storage_options:             (dict) Storage options for reading the infer parquet files.
         """
-        endpoint_id = endpoint[mm_constants.EventFieldType.UID]
-        has_stream = endpoint[mm_constants.EventFieldType.STREAM_PATH] != ""
-        m_fs = fstore.get_feature_set(
-            endpoint[mm_constants.EventFieldType.FEATURE_SET_URI]
+        endpoint_id = endpoint.metadata.uid
+        not_batch_endpoint = not (
+            endpoint.metadata.endpoint_type == EndpointType.BATCH_EP
         )
+        m_fs = fstore.get_feature_set(endpoint.spec.monitoring_feature_set_uri)
         try:
             with _BatchWindowGenerator(
                 project=project, endpoint_id=endpoint_id, window_length=window_length
@@ -371,11 +375,9 @@ class MonitoringApplicationController:
                         end_infer_time,
                     ) in batch_window_generator.get_intervals(
                         application=application,
-                        first_request=endpoint[
-                            mm_constants.EventFieldType.FIRST_REQUEST
-                        ],
-                        last_request=endpoint[mm_constants.EventFieldType.LAST_REQUEST],
-                        has_stream=has_stream,
+                        first_request=endpoint.status.first_request,
+                        last_request=endpoint.status.last_request,
+                        not_batch_endpoint=not_batch_endpoint,
                     ):
                         df = m_fs.to_dataframe(
                             start_time=start_infer_time,
@@ -401,15 +403,17 @@ class MonitoringApplicationController:
                                 start_infer_time=start_infer_time,
                                 end_infer_time=end_infer_time,
                                 endpoint_id=endpoint_id,
+                                endpoint_name=endpoint.metadata.name,
                                 project=project,
                                 applications_names=[application],
                                 model_monitoring_access_key=model_monitoring_access_key,
                             )
+                logger.info("Finished processing endpoint", endpoint_id=endpoint_id)
 
         except Exception:
             logger.exception(
                 "Encountered an exception",
-                endpoint_id=endpoint[mm_constants.EventFieldType.UID],
+                endpoint_id=endpoint.metadata.uid,
             )
 
     @staticmethod
@@ -417,6 +421,7 @@ class MonitoringApplicationController:
         start_infer_time: datetime.datetime,
         end_infer_time: datetime.datetime,
         endpoint_id: str,
+        endpoint_name: str,
         project: str,
         applications_names: list[str],
         model_monitoring_access_key: str,
@@ -440,6 +445,7 @@ class MonitoringApplicationController:
                 sep=" ", timespec="microseconds"
             ),
             mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
+            mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
             mm_constants.ApplicationEvent.OUTPUT_STREAM_URI: get_stream_path(
                 project=project,
                 function_name=mm_constants.MonitoringFunctionNames.WRITER,

mlrun/model_monitoring/db/__init__.py

@@ -12,7 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .stores import ObjectStoreFactory, get_store_object
-from .stores.base import StoreBase
 from .tsdb import get_tsdb_connector
 from .tsdb.base import TSDBConnector

mlrun/model_monitoring/db/tsdb/base.py

@@ -47,7 +47,7 @@ class TSDBConnector(ABC):
         self.project = project
 
     @abstractmethod
-    def apply_monitoring_stream_steps(self, graph) -> None:
+    def apply_monitoring_stream_steps(self, graph, **kwargs) -> None:
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
         different key metric dictionaries. This data is being used by the monitoring dashboards in
@@ -294,6 +294,7 @@ class TSDBConnector(ABC):
     ) -> pd.DataFrame:
         """
         Fetches data from the predictions TSDB table and returns the average latency for each specified endpoint
+        in the provided time range, which by default is the last 24 hours.
 
         :param endpoint_ids:    A list of model endpoint identifiers.
         :param start:           The start time for the query.

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -164,7 +164,7 @@ class TDEngineConnector(TSDBConnector):
     def _convert_to_datetime(val: typing.Union[str, datetime]) -> datetime:
         return datetime.fromisoformat(val) if isinstance(val, str) else val
 
-    def apply_monitoring_stream_steps(self, graph):
+    def apply_monitoring_stream_steps(self, graph, **kwarg):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
         different key metric dictionaries. This data is being used by the monitoring dashboards in
@@ -701,6 +701,7 @@ class TDEngineConnector(TSDBConnector):
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -168,6 +168,9 @@ class V3IOTSDBConnector(TSDBConnector):
         tsdb_batching_max_events: int = 1000,
         tsdb_batching_timeout_secs: int = 30,
         sample_window: int = 10,
+        aggregate_windows: Optional[list[str]] = None,
+        aggregate_period: str = "1m",
+        **kwarg,
     ):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -178,7 +181,40 @@ class V3IOTSDBConnector(TSDBConnector):
         - endpoint_features (Prediction and feature names and values)
         - custom_metrics (user-defined metrics)
         """
+        aggregate_windows = aggregate_windows or ["5m", "1h"]
 
+        # Calculate number of predictions and average latency
+        def apply_storey_aggregations():
+            # Calculate number of predictions for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.AggregateByKey",
+                aggregates=[
+                    {
+                        "name": EventFieldType.LATENCY,
+                        "column": EventFieldType.LATENCY,
+                        "operations": ["count", "avg"],
+                        "windows": aggregate_windows,
+                        "period": aggregate_period,
+                    }
+                ],
+                name=EventFieldType.LATENCY,
+                after="MapFeatureNames",
+                step_name="Aggregates",
+                table=".",
+                key_field=EventFieldType.ENDPOINT_ID,
+            )
+            # Calculate average latency time for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.Rename",
+                mapping={
+                    "latency_count_5m": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_5M,
+                    "latency_count_1h": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_1H,
+                },
+                name="Rename",
+                after=EventFieldType.LATENCY,
+            )
+
+        apply_storey_aggregations()
         # Write latency per prediction, labeled by endpoint ID only
         graph.add_step(
             "storey.TSDBTarget",
@@ -853,6 +889,7 @@ class V3IOTSDBConnector(TSDBConnector):
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.PREDICTIONS,
@@ -864,4 +901,10 @@ class V3IOTSDBConnector(TSDBConnector):
         )
         if not df.empty:
             df.dropna(inplace=True)
+            df.rename(
+                columns={
+                    f"avg({mm_schemas.EventFieldType.LATENCY})": f"avg_{mm_schemas.EventFieldType.LATENCY}"
+                },
+                inplace=True,
+            )
         return df.reset_index(drop=True)

mlrun/model_monitoring/helpers.py

@@ -24,6 +24,9 @@ if typing.TYPE_CHECKING:
     from mlrun.db.base import RunDBInterface
     from mlrun.projects import MlrunProject
 
+from fnmatch import fnmatchcase
+from typing import Optional
+
 import mlrun
 import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
@@ -31,11 +34,11 @@ import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
 import mlrun.model_monitoring
 import mlrun.utils.helpers
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
     _compose_full_name,
 )
-from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
 
@@ -45,6 +48,70 @@ class _BatchDict(typing.TypedDict):
     days: int
 
 
+def _is_results_regex_match(
+    existing_result_name: Optional[str],
+    result_name_filters: Optional[list[str]],
+) -> bool:
+    if existing_result_name.count(".") != 3 or any(
+        part == "" for part in existing_result_name.split(".")
+    ):
+        logger.warning(
+            f"_is_results_regex_match: existing_result_name illegal, will be ignored."
+            f" existing_result_name: {existing_result_name}"
+        )
+        return False
+    existing_result_name = ".".join(existing_result_name.split(".")[i] for i in [1, 3])
+    for result_name_filter in result_name_filters:
+        if fnmatchcase(existing_result_name, result_name_filter):
+            return True
+    return False
+
+
+def filter_results_by_regex(
+    existing_result_names: Optional[list[str]] = None,
+    result_name_filters: Optional[list[str]] = None,
+) -> list[str]:
+    """
+    Filter a list of existing result names by a list of filters.
+
+    This function returns only the results that match the filters provided. If no filters are given,
+    it returns all results. Invalid inputs are ignored.
+
+    :param existing_result_names: List of existing results' fully qualified names (FQNs)
+                                  in the format: endpoint_id.app_name.type.name.
+                                  Example: mep1.app1.result.metric1
+    :param result_name_filters:   List of filters in the format: app.result_name.
+                                  Wildcards can be used, such as app.result* or *.result
+
+    :return: List of FQNs of the matching results
+    """
+
+    if not result_name_filters:
+        return existing_result_names
+
+    if not existing_result_names:
+        return []
+
+    #  filters validations
+    validated_filters = []
+    for result_name_filter in result_name_filters:
+        if result_name_filter.count(".") != 1:
+            logger.warning(
+                f"filter_results_by_regex: result_name_filter illegal, will be ignored."
+                f"Filter: {result_name_filter}"
+            )
+        else:
+            validated_filters.append(result_name_filter)
+    filtered_metrics_names = []
+    for existing_result_name in existing_result_names:
+        if _is_results_regex_match(
+            existing_result_name=existing_result_name,
+            result_name_filters=validated_filters,
+        ):
+            filtered_metrics_names.append(existing_result_name)
+    return filtered_metrics_names
+
+
 def get_stream_path(
     project: str,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
@@ -162,24 +229,6 @@ def get_monitoring_drift_measures_data(project: str, endpoint_id: str) -> "DataI
     )
 
 
-def get_connection_string(
-    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
-) -> str:
-    """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
-    configurations.
-
-    :param secret_provider: An optional secret provider to get the connection string secret.
-
-    :return:                Valid SQL connection string.
-
-    """
-
-    return mlrun.get_secret_or_env(
-        key=mm_constants.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION,
-        secret_provider=secret_provider,
-    )
-
-
 def get_tsdb_connection_string(
     secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
 ) -> str:
@@ -252,19 +301,24 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    is_model_server_endpoint = model_endpoint.spec.stream_path != ""
-    if is_model_server_endpoint:
-        current_request = current_request.isoformat()
+    is_batch_endpoint = (
+        model_endpoint.metadata.endpoint_type == mm_constants.EndpointType.BATCH_EP
+    )
+    if not is_batch_endpoint:
         logger.info(
             "Update model endpoint last request time (EP with serving)",
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             last_request=model_endpoint.status.last_request,
             current_request=current_request,
         )
         db.patch_model_endpoint(
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
         )
     else:  # model endpoint without any serving function - close the window "manually"
@@ -283,7 +337,7 @@ def update_model_endpoint_last_request(
             + datetime.timedelta(
                 seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
             )
-        ).isoformat()
+        )
         logger.info(
             "Bumping model endpoint last request time (EP without serving)",
             project=project,
@@ -295,6 +349,8 @@ def update_model_endpoint_last_request(
         db.patch_model_endpoint(
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             attributes={mm_constants.EventFieldType.LAST_REQUEST: bumped_last_request},
         )
 
@@ -336,17 +392,6 @@ def calculate_inputs_statistics(
     return inputs_statistics
 
 
-def get_endpoint_record(
-    project: str,
-    endpoint_id: str,
-    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
-) -> dict[str, typing.Any]:
-    model_endpoint_store = mlrun.model_monitoring.get_store_object(
-        project=project, secret_provider=secret_provider
-    )
-    return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
-
-
 def get_result_instance_fqn(
     model_endpoint_id: str, app_name: str, result_name: str
 ) -> str:
@@ -386,38 +431,6 @@ def get_invocations_metric(project: str) -> ModelEndpointMonitoringMetric:
     )
 
 
-def enrich_model_endpoint_with_model_uri(
-    model_endpoint: ModelEndpoint,
-    model_obj: mlrun.artifacts.ModelArtifact,
-):
-    """
-    Enrich the model endpoint object with the model uri from the model object. We will use a unique reference
-    to the model object that includes the project, db_key, iter, and tree.
-    In addition, we verify that the model object is of type `ModelArtifact`.
-
-    :param model_endpoint:    An object representing the model endpoint that will be enriched with the model uri.
-    :param model_obj:         An object representing the model artifact.
-
-    :raise: `MLRunInvalidArgumentError` if the model object is not of type `ModelArtifact`.
-    """
-    mlrun.utils.helpers.verify_field_of_type(
-        field_name="model_endpoint.spec.model_uri",
-        field_value=model_obj,
-        expected_type=mlrun.artifacts.ModelArtifact,
-    )
-
-    # Update model_uri with a unique reference to handle future changes
-    model_artifact_uri = mlrun.utils.helpers.generate_artifact_uri(
-        project=model_endpoint.metadata.project,
-        key=model_obj.db_key,
-        iter=model_obj.iter,
-        tree=model_obj.tree,
-    )
-    model_endpoint.spec.model_uri = mlrun.datastore.get_store_uri(
-        kind=mlrun.utils.helpers.StorePrefix.Model, uri=model_artifact_uri
-    )
-
-
 def _get_monitoring_schedules_folder_path(project: str) -> str:
     return typing.cast(
         str,