mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from io import StringIO
 from typing import Literal, Optional, Union
 
@@ -168,6 +168,9 @@ class V3IOTSDBConnector(TSDBConnector):
         tsdb_batching_max_events: int = 1000,
         tsdb_batching_timeout_secs: int = 30,
         sample_window: int = 10,
+        aggregate_windows: Optional[list[str]] = None,
+        aggregate_period: str = "1m",
+        **kwarg,
     ):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -178,7 +181,40 @@ class V3IOTSDBConnector(TSDBConnector):
         - endpoint_features (Prediction and feature names and values)
         - custom_metrics (user-defined metrics)
         """
+        aggregate_windows = aggregate_windows or ["5m", "1h"]
 
+        # Calculate number of predictions and average latency
+        def apply_storey_aggregations():
+            # Calculate number of predictions for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.AggregateByKey",
+                aggregates=[
+                    {
+                        "name": EventFieldType.LATENCY,
+                        "column": EventFieldType.LATENCY,
+                        "operations": ["count", "avg"],
+                        "windows": aggregate_windows,
+                        "period": aggregate_period,
+                    }
+                ],
+                name=EventFieldType.LATENCY,
+                after="MapFeatureNames",
+                step_name="Aggregates",
+                table=".",
+                key_field=EventFieldType.ENDPOINT_ID,
+            )
+            # Calculate average latency time for each window (5 min and 1 hour by default)
+            graph.add_step(
+                class_name="storey.Rename",
+                mapping={
+                    "latency_count_5m": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_5M,
+                    "latency_count_1h": mm_schemas.EventLiveStats.PREDICTIONS_COUNT_1H,
+                },
+                name="Rename",
+                after=EventFieldType.LATENCY,
+            )
+
+        apply_storey_aggregations()
         # Write latency per prediction, labeled by endpoint ID only
         graph.add_step(
             "storey.TSDBTarget",
@@ -310,6 +346,7 @@ class V3IOTSDBConnector(TSDBConnector):
             ],
             index_cols=[
                 mm_schemas.EventFieldType.ENDPOINT_ID,
+                mm_schemas.EventFieldType.ERROR_TYPE,
             ],
             max_events=tsdb_batching_max_events,
             flush_after_seconds=tsdb_batching_timeout_secs,
@@ -338,9 +375,6 @@ class V3IOTSDBConnector(TSDBConnector):
         elif kind == mm_schemas.WriterEventKind.RESULT:
             table = self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS]
             index_cols = index_cols_base + [mm_schemas.ResultData.RESULT_NAME]
-            event.pop(mm_schemas.ResultData.CURRENT_STATS, None)
-            # TODO: remove this when extra data is supported (ML-7460)
-            event.pop(mm_schemas.ResultData.RESULT_EXTRA_DATA, None)
         else:
             raise ValueError(f"Invalid {kind = }")
 
@@ -544,6 +578,7 @@ class V3IOTSDBConnector(TSDBConnector):
         end: datetime,
         metrics: list[mm_schemas.ModelEndpointMonitoringMetric],
         type: Literal["metrics", "results"] = "results",
+        with_result_extra_data: bool = False,
     ) -> Union[
         list[
             Union[
@@ -565,6 +600,12 @@ class V3IOTSDBConnector(TSDBConnector):
         """
 
         if type == "metrics":
+            if with_result_extra_data:
+                logger.warning(
+                    "The 'with_result_extra_data' parameter is not supported for metrics, just for results",
+                    project=self.project,
+                    endpoint_id=endpoint_id,
+                )
             table_path = self.tables[mm_schemas.V3IOTSDBTables.METRICS]
             name = mm_schemas.MetricData.METRIC_NAME
             columns = [mm_schemas.MetricData.METRIC_VALUE]
@@ -577,6 +618,8 @@ class V3IOTSDBConnector(TSDBConnector):
                 mm_schemas.ResultData.RESULT_STATUS,
                 mm_schemas.ResultData.RESULT_KIND,
             ]
+            if with_result_extra_data:
+                columns.append(mm_schemas.ResultData.RESULT_EXTRA_DATA)
             df_handler = self.df_to_results_values
         else:
             raise ValueError(f"Invalid {type = }")
@@ -605,6 +648,9 @@ class V3IOTSDBConnector(TSDBConnector):
             endpoint_id=endpoint_id,
             is_empty=df.empty,
         )
+        if not with_result_extra_data and type == "results":
+            # Set the extra data to an empty string if it's not requested
+            df[mm_schemas.ResultData.RESULT_EXTRA_DATA] = ""
 
         return df_handler(df=df, metrics=metrics, project=self.project)
 
@@ -700,12 +746,13 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_last_request(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.PREDICTIONS,
             start=start,
@@ -734,12 +781,14 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_drift_status(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "now-24h",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
@@ -758,9 +807,10 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_metrics_metadata(
         self,
         endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.METRICS,
             start=start,
@@ -778,9 +828,10 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_results_metadata(
         self,
         endpoint_id: str,
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.V3IOTSDBTables.APP_RESULTS,
             start=start,
@@ -803,18 +854,20 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_error_count(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.ERRORS,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.ERROR_COUNT],
-            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]})",
+            filter_query=f"endpoint_id IN({str(endpoint_ids)[1:-1]}) "
+            f"AND {mm_schemas.EventFieldType.ERROR_TYPE} == '{mm_schemas.EventFieldType.INFER_ERROR}'",
             agg_funcs=["count"],
         )
         if not df.empty:
@@ -830,12 +883,14 @@ class V3IOTSDBConnector(TSDBConnector):
     def get_avg_latency(
         self,
         endpoint_ids: Union[str, list[str]],
-        start: Union[datetime, str] = "0",
-        end: Union[datetime, str] = "now",
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
     ) -> pd.DataFrame:
         endpoint_ids = (
             endpoint_ids if isinstance(endpoint_ids, list) else [endpoint_ids]
         )
+        start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
+        start, end = self._get_start_end(start, end)
         df = self._get_records(
             table=mm_schemas.FileTargetKind.PREDICTIONS,
             start=start,
@@ -846,4 +901,10 @@ class V3IOTSDBConnector(TSDBConnector):
         )
         if not df.empty:
             df.dropna(inplace=True)
+            df.rename(
+                columns={
+                    f"avg({mm_schemas.EventFieldType.LATENCY})": f"avg_{mm_schemas.EventFieldType.LATENCY}"
+                },
+                inplace=True,
+            )
         return df.reset_index(drop=True)

mlrun/model_monitoring/helpers.py

@@ -13,26 +13,32 @@
 # limitations under the License.
 
 import datetime
+import os
 import typing
 
 import numpy as np
 import pandas as pd
 
 if typing.TYPE_CHECKING:
+    from mlrun.datastore import DataItem
     from mlrun.db.base import RunDBInterface
     from mlrun.projects import MlrunProject
 
+from fnmatch import fnmatchcase
+from typing import Optional
+
 import mlrun
 import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
 import mlrun.model_monitoring
+import mlrun.utils.helpers
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
     _compose_full_name,
 )
-from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
 
@@ -42,6 +48,70 @@ class _BatchDict(typing.TypedDict):
     days: int
 
 
+def _is_results_regex_match(
+    existing_result_name: Optional[str],
+    result_name_filters: Optional[list[str]],
+) -> bool:
+    if existing_result_name.count(".") != 3 or any(
+        part == "" for part in existing_result_name.split(".")
+    ):
+        logger.warning(
+            f"_is_results_regex_match: existing_result_name illegal, will be ignored."
+            f" existing_result_name: {existing_result_name}"
+        )
+        return False
+    existing_result_name = ".".join(existing_result_name.split(".")[i] for i in [1, 3])
+    for result_name_filter in result_name_filters:
+        if fnmatchcase(existing_result_name, result_name_filter):
+            return True
+    return False
+
+
+def filter_results_by_regex(
+    existing_result_names: Optional[list[str]] = None,
+    result_name_filters: Optional[list[str]] = None,
+) -> list[str]:
+    """
+    Filter a list of existing result names by a list of filters.
+
+    This function returns only the results that match the filters provided. If no filters are given,
+    it returns all results. Invalid inputs are ignored.
+
+    :param existing_result_names: List of existing results' fully qualified names (FQNs)
+                                  in the format: endpoint_id.app_name.type.name.
+                                  Example: mep1.app1.result.metric1
+    :param result_name_filters:   List of filters in the format: app.result_name.
+                                  Wildcards can be used, such as app.result* or *.result
+
+    :return: List of FQNs of the matching results
+    """
+
+    if not result_name_filters:
+        return existing_result_names
+
+    if not existing_result_names:
+        return []
+
+    #  filters validations
+    validated_filters = []
+    for result_name_filter in result_name_filters:
+        if result_name_filter.count(".") != 1:
+            logger.warning(
+                f"filter_results_by_regex: result_name_filter illegal, will be ignored."
+                f"Filter: {result_name_filter}"
+            )
+        else:
+            validated_filters.append(result_name_filter)
+    filtered_metrics_names = []
+    for existing_result_name in existing_result_names:
+        if _is_results_regex_match(
+            existing_result_name=existing_result_name,
+            result_name_filters=validated_filters,
+        ):
+            filtered_metrics_names.append(existing_result_name)
+    return filtered_metrics_names
+
+
 def get_stream_path(
     project: str,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
@@ -98,19 +168,64 @@ def get_monitoring_parquet_path(
     return parquet_path
 
 
-def get_connection_string(secret_provider: typing.Callable[[str], str] = None) -> str:
-    """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
-    configurations.
+def get_monitoring_stats_directory_path(
+    project: str,
+    kind: str = mm_constants.FileTargetKind.STATS,
+) -> str:
+    """
+    Get model monitoring stats target for the current project and kind. The stats target path is based on the
+    project artifact path. If project artifact path is not defined, the stats target path will be based on MLRun
+    artifact path.
+    :param project:     Project object.
+    :param kind:        indicate the kind of the stats path
+    :return:            Monitoring stats target path.
+    """
+    stats_path = mlrun.mlconf.get_model_monitoring_file_target_path(
+        project=project,
+        kind=kind,
+    )
+    return stats_path
 
-    :param secret_provider: An optional secret provider to get the connection string secret.
 
-    :return:                Valid SQL connection string.
+def _get_monitoring_current_stats_file_path(project: str, endpoint_id: str) -> str:
+    return os.path.join(
+        get_monitoring_stats_directory_path(project),
+        f"{endpoint_id}_current_stats.json",
+    )
+
 
+def _get_monitoring_drift_measures_file_path(project: str, endpoint_id: str) -> str:
+    return os.path.join(
+        get_monitoring_stats_directory_path(project),
+        f"{endpoint_id}_drift_measures.json",
+    )
+
+
+def get_monitoring_current_stats_data(project: str, endpoint_id: str) -> "DataItem":
+    """
+    getter for data item of current stats for project and endpoint
+    :param project: project name str
+    :param endpoint_id: endpoint id str
+    :return: DataItem
     """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_current_stats_file_path(
+            project=project, endpoint_id=endpoint_id
+        )
+    )
 
-    return mlrun.get_secret_or_env(
-        key=mm_constants.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION,
-        secret_provider=secret_provider,
+
+def get_monitoring_drift_measures_data(project: str, endpoint_id: str) -> "DataItem":
+    """
+    getter for data item of drift measures for project and endpoint
+    :param project: project name str
+    :param endpoint_id: endpoint id str
+    :return: DataItem
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_drift_measures_file_path(
+            project=project, endpoint_id=endpoint_id
+        )
     )
 
 
@@ -186,19 +301,24 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    is_model_server_endpoint = model_endpoint.spec.stream_path != ""
-    if is_model_server_endpoint:
-        current_request = current_request.isoformat()
+    is_batch_endpoint = (
+        model_endpoint.metadata.endpoint_type == mm_constants.EndpointType.BATCH_EP
+    )
+    if not is_batch_endpoint:
         logger.info(
             "Update model endpoint last request time (EP with serving)",
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             last_request=model_endpoint.status.last_request,
             current_request=current_request,
         )
         db.patch_model_endpoint(
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
         )
     else:  # model endpoint without any serving function - close the window "manually"
@@ -217,7 +337,7 @@ def update_model_endpoint_last_request(
             + datetime.timedelta(
                 seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
             )
-        ).isoformat()
+        )
         logger.info(
             "Bumping model endpoint last request time (EP without serving)",
             project=project,
@@ -229,6 +349,8 @@ def update_model_endpoint_last_request(
         db.patch_model_endpoint(
             project=project,
             endpoint_id=model_endpoint.metadata.uid,
+            name=model_endpoint.metadata.name,
+            function_name=model_endpoint.spec.function_name,
             attributes={mm_constants.EventFieldType.LAST_REQUEST: bumped_last_request},
         )
 
@@ -270,17 +392,6 @@ def calculate_inputs_statistics(
     return inputs_statistics
 
 
-def get_endpoint_record(
-    project: str,
-    endpoint_id: str,
-    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
-) -> dict[str, typing.Any]:
-    model_endpoint_store = mlrun.model_monitoring.get_store_object(
-        project=project, secret_provider=secret_provider
-    )
-    return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
-
-
 def get_result_instance_fqn(
     model_endpoint_id: str, app_name: str, result_name: str
 ) -> str:
@@ -320,33 +431,25 @@ def get_invocations_metric(project: str) -> ModelEndpointMonitoringMetric:
     )
 
 
-def enrich_model_endpoint_with_model_uri(
-    model_endpoint: ModelEndpoint,
-    model_obj: mlrun.artifacts.ModelArtifact,
-):
-    """
-    Enrich the model endpoint object with the model uri from the model object. We will use a unique reference
-    to the model object that includes the project, db_key, iter, and tree.
-    In addition, we verify that the model object is of type `ModelArtifact`.
+def _get_monitoring_schedules_folder_path(project: str) -> str:
+    return typing.cast(
+        str,
+        mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=project, kind=mm_constants.FileTargetKind.MONITORING_SCHEDULES
+        ),
+    )
 
-    :param model_endpoint:    An object representing the model endpoint that will be enriched with the model uri.
-    :param model_obj:         An object representing the model artifact.
 
-    :raise: `MLRunInvalidArgumentError` if the model object is not of type `ModelArtifact`.
-    """
-    mlrun.utils.helpers.verify_field_of_type(
-        field_name="model_endpoint.spec.model_uri",
-        field_value=model_obj,
-        expected_type=mlrun.artifacts.ModelArtifact,
+def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{endpoint_id}.json"
     )
 
-    # Update model_uri with a unique reference to handle future changes
-    model_artifact_uri = mlrun.utils.helpers.generate_artifact_uri(
-        project=model_endpoint.metadata.project,
-        key=model_obj.db_key,
-        iter=model_obj.iter,
-        tree=model_obj.tree,
-    )
-    model_endpoint.spec.model_uri = mlrun.datastore.get_store_uri(
-        kind=mlrun.utils.helpers.StorePrefix.Model, uri=model_artifact_uri
+
+def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
+    """
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_path(project=project, endpoint_id=endpoint_id)
     )