mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/model_monitoring/helpers.py

@@ -13,65 +13,150 @@
 # limitations under the License.
 
 import datetime
-import typing
+import functools
+import os
+from fnmatch import fnmatchcase
+from typing import TYPE_CHECKING, Callable, Optional, TypedDict, Union, cast
 
 import numpy as np
 import pandas as pd
 
-if typing.TYPE_CHECKING:
-    from mlrun.db.base import RunDBInterface
-    from mlrun.projects import MlrunProject
-
 import mlrun
 import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
+import mlrun.datastore.datastore_profile
 import mlrun.model_monitoring
+import mlrun.platforms.iguazio
+import mlrun.utils.helpers
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
     ModelEndpointMonitoringMetric,
-    _compose_full_name,
+    compose_full_name,
 )
-from mlrun.model_monitoring.model_endpoint import ModelEndpoint
 from mlrun.utils import logger
 
+if TYPE_CHECKING:
+    from mlrun.datastore import DataItem
+    from mlrun.db.base import RunDBInterface
+    from mlrun.projects import MlrunProject
 
-class _BatchDict(typing.TypedDict):
+
+class _BatchDict(TypedDict):
     minutes: int
     hours: int
     days: int
 
 
+def _is_results_regex_match(
+    existing_result_name: Optional[str],
+    result_name_filters: Optional[list[str]],
+) -> bool:
+    if existing_result_name.count(".") != 3 or any(
+        part == "" for part in existing_result_name.split(".")
+    ):
+        logger.warning(
+            f"_is_results_regex_match: existing_result_name illegal, will be ignored."
+            f" existing_result_name: {existing_result_name}"
+        )
+        return False
+    existing_result_name = ".".join(existing_result_name.split(".")[i] for i in [1, 3])
+    for result_name_filter in result_name_filters:
+        if fnmatchcase(existing_result_name, result_name_filter):
+            return True
+    return False
+
+
+def filter_results_by_regex(
+    existing_result_names: Optional[list[str]] = None,
+    result_name_filters: Optional[list[str]] = None,
+) -> list[str]:
+    """
+    Filter a list of existing result names by a list of filters.
+
+    This function returns only the results that match the filters provided. If no filters are given,
+    it returns all results. Invalid inputs are ignored.
+
+    :param existing_result_names: List of existing results' fully qualified names (FQNs)
+                                  in the format: endpoint_id.app_name.type.name.
+                                  Example: mep1.app1.result.metric1
+    :param result_name_filters:   List of filters in the format: app.result_name.
+                                  Wildcards can be used, such as app.result* or *.result
+
+    :return: List of FQNs of the matching results
+    """
+
+    if not result_name_filters:
+        return existing_result_names
+
+    if not existing_result_names:
+        return []
+
+    #  filters validations
+    validated_filters = []
+    for result_name_filter in result_name_filters:
+        if result_name_filter.count(".") != 1:
+            logger.warning(
+                f"filter_results_by_regex: result_name_filter illegal, will be ignored."
+                f"Filter: {result_name_filter}"
+            )
+        else:
+            validated_filters.append(result_name_filter)
+    filtered_metrics_names = []
+    for existing_result_name in existing_result_names:
+        if _is_results_regex_match(
+            existing_result_name=existing_result_name,
+            result_name_filters=validated_filters,
+        ):
+            filtered_metrics_names.append(existing_result_name)
+    return list(set(filtered_metrics_names))
+
+
 def get_stream_path(
     project: str,
     function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
-    stream_uri: typing.Optional[str] = None,
+    stream_uri: Optional[str] = None,
+    secret_provider: Optional[Callable[[str], str]] = None,
+    profile: Optional[mlrun.datastore.datastore_profile.DatastoreProfile] = None,
 ) -> str:
     """
     Get stream path from the project secret. If wasn't set, take it from the system configurations
 
     :param project:             Project name.
     :param function_name:       Application name. Default is model_monitoring_stream.
-    :param stream_uri:          Stream URI. If provided, it will be used instead of the one from the project secret.
-
+    :param stream_uri:          Stream URI. If provided, it will be used instead of the one from the project's secret.
+    :param secret_provider:     Optional secret provider to get the connection string secret.
+                                If not set, the env vars are used.
+    :param profile:             Optional datastore profile of the stream (V3IO/KafkaSource profile).
     :return:                    Monitoring stream path to the relevant application.
     """
 
-    stream_uri = stream_uri or mlrun.get_secret_or_env(
-        mm_constants.ProjectSecretKeys.STREAM_PATH
+    profile = profile or _get_stream_profile(
+        project=project, secret_provider=secret_provider
     )
 
-    if not stream_uri or stream_uri == "v3io":
+    if isinstance(profile, mlrun.datastore.datastore_profile.DatastoreProfileV3io):
         stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
             project=project,
             kind=mm_constants.FileTargetKind.STREAM,
             target="online",
             function_name=function_name,
         )
+        return stream_uri.replace("v3io://", f"ds://{profile.name}")
 
-    return mlrun.common.model_monitoring.helpers.parse_monitoring_stream_path(
-        stream_uri=stream_uri, project=project, function_name=function_name
-    )
+    elif isinstance(
+        profile, mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource
+    ):
+        topic = mlrun.common.model_monitoring.helpers.get_kafka_topic(
+            project=project, function_name=function_name
+        )
+        return f"ds://{profile.name}/{topic}"
+    else:
+        raise mlrun.errors.MLRunValueError(
+            f"Received an unexpected stream profile type: {type(profile)}\n"
+            "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaSource`."
+        )
 
 
 def get_monitoring_parquet_path(
@@ -98,37 +183,198 @@ def get_monitoring_parquet_path(
     return parquet_path
 
 
-def get_connection_string(secret_provider: typing.Callable[[str], str] = None) -> str:
-    """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
-    configurations.
+def get_monitoring_stats_directory_path(
+    project: str,
+    kind: str = mm_constants.FileTargetKind.STATS,
+) -> str:
+    """
+    Get model monitoring stats target for the current project and kind. The stats target path is based on the
+    project artifact path. If project artifact path is not defined, the stats target path will be based on MLRun
+    artifact path.
+    :param project:     Project object.
+    :param kind:        indicate the kind of the stats path
+    :return:            Monitoring stats target path.
+    """
+    stats_path = mlrun.mlconf.get_model_monitoring_file_target_path(
+        project=project,
+        kind=kind,
+    )
+    return stats_path
+
+
+def _get_monitoring_current_stats_file_path(project: str, endpoint_id: str) -> str:
+    return os.path.join(
+        get_monitoring_stats_directory_path(project),
+        f"{endpoint_id}_current_stats.json",
+    )
 
-    :param secret_provider: An optional secret provider to get the connection string secret.
 
-    :return:                Valid SQL connection string.
+def _get_monitoring_drift_measures_file_path(project: str, endpoint_id: str) -> str:
+    return os.path.join(
+        get_monitoring_stats_directory_path(project),
+        f"{endpoint_id}_drift_measures.json",
+    )
+
 
+def get_monitoring_current_stats_data(project: str, endpoint_id: str) -> "DataItem":
     """
+    getter for data item of current stats for project and endpoint
+    :param project: project name str
+    :param endpoint_id: endpoint id str
+    :return: DataItem
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_current_stats_file_path(
+            project=project, endpoint_id=endpoint_id
+        )
+    )
+
 
-    return mlrun.get_secret_or_env(
-        key=mm_constants.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION,
-        secret_provider=secret_provider,
+def get_monitoring_drift_measures_data(project: str, endpoint_id: str) -> "DataItem":
+    """
+    getter for data item of drift measures for project and endpoint
+    :param project: project name str
+    :param endpoint_id: endpoint id str
+    :return: DataItem
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_drift_measures_file_path(
+            project=project, endpoint_id=endpoint_id
+        )
     )
 
 
-def get_tsdb_connection_string(
-    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
-) -> str:
-    """Get TSDB connection string from the project secret. If wasn't set, take it from the system
-    configurations.
-    :param secret_provider: An optional secret provider to get the connection string secret.
-    :return:                Valid TSDB connection string.
+def _get_profile(
+    project: str,
+    secret_provider: Optional[Callable[[str], str]],
+    profile_name_key: str,
+) -> mlrun.datastore.datastore_profile.DatastoreProfile:
     """
+    Get the datastore profile from the project name and secret provider, where the profile's name
+    is saved as a secret named `profile_name_key`.
 
-    return mlrun.get_secret_or_env(
-        key=mm_constants.ProjectSecretKeys.TSDB_CONNECTION,
-        secret_provider=secret_provider,
+    :param project:          The project name.
+    :param secret_provider:  Secret provider to get the secrets from, or `None` for env vars.
+    :param profile_name_key: The profile name key in the secret store.
+    :return:                 Datastore profile.
+    """
+    profile_name = mlrun.get_secret_or_env(
+        key=profile_name_key, secret_provider=secret_provider
+    )
+    if not profile_name:
+        raise mlrun.errors.MLRunNotFoundError(
+            f"Not found `{profile_name_key}` profile name for project '{project}'"
+        )
+    return mlrun.datastore.datastore_profile.datastore_profile_read(
+        url=f"ds://{profile_name}", project_name=project, secrets=secret_provider
     )
 
 
+_get_tsdb_profile = functools.partial(
+    _get_profile, profile_name_key=mm_constants.ProjectSecretKeys.TSDB_PROFILE_NAME
+)
+_get_stream_profile = functools.partial(
+    _get_profile, profile_name_key=mm_constants.ProjectSecretKeys.STREAM_PROFILE_NAME
+)
+
+
+def _get_v3io_output_stream(
+    *,
+    v3io_profile: mlrun.datastore.datastore_profile.DatastoreProfileV3io,
+    project: str,
+    function_name: str,
+    v3io_access_key: Optional[str],
+    mock: bool = False,
+) -> mlrun.platforms.iguazio.OutputStream:
+    stream_uri = mlrun.mlconf.get_model_monitoring_file_target_path(
+        project=project,
+        kind=mm_constants.FileTargetKind.STREAM,
+        target="online",
+        function_name=function_name,
+    )
+    endpoint, stream_path = mlrun.platforms.iguazio.parse_path(stream_uri)
+    return mlrun.platforms.iguazio.OutputStream(
+        stream_path,
+        endpoint=endpoint,
+        access_key=v3io_access_key or v3io_profile.v3io_access_key,
+        mock=mock,
+    )
+
+
+def _get_kafka_output_stream(
+    *,
+    kafka_profile: mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource,
+    project: str,
+    function_name: str,
+    mock: bool = False,
+) -> mlrun.platforms.iguazio.KafkaOutputStream:
+    topic = mlrun.common.model_monitoring.helpers.get_kafka_topic(
+        project=project, function_name=function_name
+    )
+    attributes = kafka_profile.attributes()
+    producer_options = mlrun.datastore.utils.KafkaParameters(attributes).producer()
+
+    return mlrun.platforms.iguazio.KafkaOutputStream(
+        brokers=kafka_profile.brokers,
+        topic=topic,
+        producer_options=producer_options,
+        mock=mock,
+    )
+
+
+def get_output_stream(
+    project: str,
+    function_name: str = mm_constants.MonitoringFunctionNames.STREAM,
+    secret_provider: Optional[Callable[[str], str]] = None,
+    profile: Optional[mlrun.datastore.datastore_profile.DatastoreProfile] = None,
+    v3io_access_key: Optional[str] = None,
+    mock: bool = False,
+) -> Union[
+    mlrun.platforms.iguazio.OutputStream, mlrun.platforms.iguazio.KafkaOutputStream
+]:
+    """
+    Get stream path from the project secret. If wasn't set, take it from the system configurations
+
+    :param project:             Project name.
+    :param function_name:       Application name. Default is model_monitoring_stream.
+    :param secret_provider:     Optional secret provider to get the connection string secret.
+                                If not set, the env vars are used.
+    :param profile:             Optional datastore profile of the stream (V3IO/KafkaSource profile).
+    :param v3io_access_key:     Optional V3IO access key.
+    :param mock:                Should the output stream be mocked or not.
+    :return:                    Monitoring stream path to the relevant application.
+    """
+
+    profile = profile or _get_stream_profile(
+        project=project, secret_provider=secret_provider
+    )
+
+    if isinstance(profile, mlrun.datastore.datastore_profile.DatastoreProfileV3io):
+        return _get_v3io_output_stream(
+            v3io_profile=profile,
+            project=project,
+            function_name=function_name,
+            v3io_access_key=v3io_access_key,
+            mock=mock,
+        )
+
+    elif isinstance(
+        profile, mlrun.datastore.datastore_profile.DatastoreProfileKafkaSource
+    ):
+        return _get_kafka_output_stream(
+            kafka_profile=profile,
+            project=project,
+            function_name=function_name,
+            mock=mock,
+        )
+
+    else:
+        raise mlrun.errors.MLRunValueError(
+            f"Received an unexpected stream profile type: {type(profile)}\n"
+            "Expects `DatastoreProfileV3io` or `DatastoreProfileKafkaSource`."
+        )
+
+
 def batch_dict2timedelta(batch_dict: _BatchDict) -> datetime.timedelta:
     """
     Convert a batch dictionary to timedelta.
@@ -186,51 +432,23 @@ def update_model_endpoint_last_request(
     :param current_request: current request time
     :param db:              DB interface.
     """
-    is_model_server_endpoint = model_endpoint.spec.stream_path != ""
-    if is_model_server_endpoint:
-        current_request = current_request.isoformat()
-        logger.info(
-            "Update model endpoint last request time (EP with serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
-        )
-    else:  # model endpoint without any serving function - close the window "manually"
-        try:
-            time_window = _get_monitoring_time_window_from_controller_run(project, db)
-        except mlrun.errors.MLRunNotFoundError:
-            logger.warn(
-                "Not bumping model endpoint last request time - the monitoring controller isn't deployed yet.\n"
-                "Call `project.enable_model_monitoring()` first."
-            )
-            return
 
-        bumped_last_request = (
-            current_request
-            + time_window
-            + datetime.timedelta(
-                seconds=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_timeout_secs
-            )
-        ).isoformat()
-        logger.info(
-            "Bumping model endpoint last request time (EP without serving)",
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            last_request=model_endpoint.status.last_request,
-            current_request=current_request.isoformat(),
-            bumped_last_request=bumped_last_request,
-        )
-        db.patch_model_endpoint(
-            project=project,
-            endpoint_id=model_endpoint.metadata.uid,
-            attributes={mm_constants.EventFieldType.LAST_REQUEST: bumped_last_request},
-        )
+    logger.info(
+        "Update model endpoint last request time (EP with serving)",
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        last_request=model_endpoint.status.last_request,
+        current_request=current_request,
+    )
+    db.patch_model_endpoint(
+        project=project,
+        endpoint_id=model_endpoint.metadata.uid,
+        name=model_endpoint.metadata.name,
+        function_name=model_endpoint.spec.function_name,
+        attributes={mm_constants.EventFieldType.LAST_REQUEST: current_request},
+    )
 
 
 def calculate_inputs_statistics(
@@ -270,23 +488,28 @@ def calculate_inputs_statistics(
     return inputs_statistics
 
 
-def get_endpoint_record(
-    project: str,
-    endpoint_id: str,
-    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
-) -> dict[str, typing.Any]:
-    model_endpoint_store = mlrun.model_monitoring.get_store_object(
-        project=project, secret_provider=secret_provider
-    )
-    return model_endpoint_store.get_model_endpoint(endpoint_id=endpoint_id)
-
-
 def get_result_instance_fqn(
     model_endpoint_id: str, app_name: str, result_name: str
 ) -> str:
     return f"{model_endpoint_id}.{app_name}.result.{result_name}"
 
 
+def get_alert_name_from_result_fqn(result_fqn: str):
+    """
+    :param   result_fqn: current get_result_instance_fqn format: `{model_endpoint_id}.{app_name}.result.{result_name}`
+
+    :return: shorter fqn without forbidden alert characters.
+    """
+    if result_fqn.count(".") != 3 or result_fqn.split(".")[2] != "result":
+        raise mlrun.errors.MLRunValueError(
+            f"result_fqn: {result_fqn} is not in the correct format: {{model_endpoint_id}}.{{app_name}}."
+            f"result.{{result_name}}"
+        )
+    # Name format cannot contain "."
+    # The third component is always `result`, so it is not necessary for checking uniqueness.
+    return "_".join(result_fqn.split(".")[i] for i in [0, 1, 3])
+
+
 def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
     return get_result_instance_fqn(
         model_endpoint_id,
@@ -296,7 +519,7 @@ def get_default_result_instance_fqn(model_endpoint_id: str) -> str:
 
 
 def get_invocations_fqn(project: str) -> str:
-    return _compose_full_name(
+    return compose_full_name(
         project=project,
         app=mm_constants.SpecialApps.MLRUN_INFRA,
         name=mm_constants.PredictionsQueryConstants.INVOCATIONS,
@@ -320,33 +543,52 @@ def get_invocations_metric(project: str) -> ModelEndpointMonitoringMetric:
     )
 
 
-def enrich_model_endpoint_with_model_uri(
-    model_endpoint: ModelEndpoint,
-    model_obj: mlrun.artifacts.ModelArtifact,
-):
-    """
-    Enrich the model endpoint object with the model uri from the model object. We will use a unique reference
-    to the model object that includes the project, db_key, iter, and tree.
-    In addition, we verify that the model object is of type `ModelArtifact`.
+def _get_monitoring_schedules_folder_path(project: str) -> str:
+    return cast(
+        str,
+        mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=project, kind=mm_constants.FileTargetKind.MONITORING_SCHEDULES
+        ),
+    )
+
+
+def _get_monitoring_schedules_file_endpoint_path(
+    *, project: str, endpoint_id: str
+) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{endpoint_id}.json"
+    )
 
-    :param model_endpoint:    An object representing the model endpoint that will be enriched with the model uri.
-    :param model_obj:         An object representing the model artifact.
 
-    :raise: `MLRunInvalidArgumentError` if the model object is not of type `ModelArtifact`.
+def get_monitoring_schedules_endpoint_data(
+    *, project: str, endpoint_id: str
+) -> "DataItem":
     """
-    mlrun.utils.helpers.verify_field_of_type(
-        field_name="model_endpoint.spec.model_uri",
-        field_value=model_obj,
-        expected_type=mlrun.artifacts.ModelArtifact,
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_endpoint_path(
+            project=project, endpoint_id=endpoint_id
+        )
     )
 
-    # Update model_uri with a unique reference to handle future changes
-    model_artifact_uri = mlrun.utils.helpers.generate_artifact_uri(
-        project=model_endpoint.metadata.project,
-        key=model_obj.db_key,
-        iter=model_obj.iter,
-        tree=model_obj.tree,
+
+def get_monitoring_schedules_chief_data(
+    *,
+    project: str,
+) -> "DataItem":
+    """
+    Get the model monitoring schedules' data item of the project's model endpoint.
+    """
+    return mlrun.datastore.store_manager.object(
+        _get_monitoring_schedules_file_chief_path(project=project)
     )
-    model_endpoint.spec.model_uri = mlrun.datastore.get_store_uri(
-        kind=mlrun.utils.helpers.StorePrefix.Model, uri=model_artifact_uri
+
+
+def _get_monitoring_schedules_file_chief_path(
+    *,
+    project: str,
+) -> str:
+    return os.path.join(
+        _get_monitoring_schedules_folder_path(project), f"{project}.json"
     )