mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/common/schemas/model_monitoring/constants.py

@@ -13,14 +13,22 @@
 # limitations under the License.
 
 import hashlib
-from dataclasses import dataclass
+import re
+from dataclasses import dataclass, field
 from enum import Enum, IntEnum
 from typing import Optional
 
+import mlrun.common.constants
 import mlrun.common.helpers
 from mlrun.common.types import StrEnum
 
 
+class MonitoringStrEnum(StrEnum):
+    @classmethod
+    def list(cls):
+        return list(map(lambda c: c.value, cls))
+
+
 class EventFieldType:
     FUNCTION_URI = "function_uri"
     FUNCTION = "function"
@@ -46,9 +54,11 @@ class EventFieldType:
     PREDICTIONS = "predictions"
     NAMED_PREDICTIONS = "named_predictions"
     ERROR_COUNT = "error_count"
+    MODEL_ERROR = "model_error"
     ENTITIES = "entities"
     FIRST_REQUEST = "first_request"
     LAST_REQUEST = "last_request"
+    LAST_REQUEST_TIMESTAMP = "last_request_timestamp"
     METRIC = "metric"
     METRICS = "metrics"
     BATCH_INTERVALS_DICT = "batch_intervals_dict"
@@ -72,15 +82,9 @@ class EventFieldType:
     FEATURE_SET_URI = "monitoring_feature_set_uri"
     ALGORITHM = "algorithm"
     VALUE = "value"
-    DRIFT_DETECTED_THRESHOLD = "drift_detected_threshold"
-    POSSIBLE_DRIFT_THRESHOLD = "possible_drift_threshold"
     SAMPLE_PARQUET_PATH = "sample_parquet_path"
-
-
-class MonitoringStrEnum(StrEnum):
-    @classmethod
-    def list(cls):
-        return list(map(lambda c: c.value, cls))
+    TIME = "time"
+    TABLE_COLUMN = "table_column"
 
 
 class FeatureSetFeatures(MonitoringStrEnum):
@@ -99,12 +103,8 @@ class FeatureSetFeatures(MonitoringStrEnum):
 
 class ApplicationEvent:
     APPLICATION_NAME = "application_name"
-    CURRENT_STATS = "current_stats"
-    FEATURE_STATS = "feature_stats"
-    SAMPLE_PARQUET_PATH = "sample_parquet_path"
     START_INFER_TIME = "start_infer_time"
     END_INFER_TIME = "end_infer_time"
-    LAST_REQUEST = "last_request"
     ENDPOINT_ID = "endpoint_id"
     OUTPUT_STREAM_URI = "output_stream_uri"
 
@@ -114,6 +114,21 @@ class WriterEvent(MonitoringStrEnum):
     ENDPOINT_ID = "endpoint_id"
     START_INFER_TIME = "start_infer_time"
     END_INFER_TIME = "end_infer_time"
+    EVENT_KIND = "event_kind"  # metric or result
+    DATA = "data"
+
+
+class WriterEventKind(MonitoringStrEnum):
+    METRIC = "metric"
+    RESULT = "result"
+
+
+class MetricData(MonitoringStrEnum):
+    METRIC_NAME = "metric_name"
+    METRIC_VALUE = "metric_value"
+
+
+class ResultData(MonitoringStrEnum):
     RESULT_NAME = "result_name"
     RESULT_VALUE = "result_value"
     RESULT_KIND = "result_kind"
@@ -138,21 +153,40 @@ class EventKeyMetrics:
     REAL_TIME = "real_time"
 
 
-class TimeSeriesTarget:
-    TSDB = "tsdb"
-
-
-class ModelEndpointTarget:
+class ModelEndpointTarget(MonitoringStrEnum):
     V3IO_NOSQL = "v3io-nosql"
     SQL = "sql"
 
 
+class StreamKind(MonitoringStrEnum):
+    V3IO_STREAM = "v3io_stream"
+    KAFKA = "kafka"
+
+
+class TSDBTarget(MonitoringStrEnum):
+    V3IO_TSDB = "v3io-tsdb"
+    TDEngine = "tdengine"
+
+
 class ProjectSecretKeys:
     ENDPOINT_STORE_CONNECTION = "MODEL_MONITORING_ENDPOINT_STORE_CONNECTION"
     ACCESS_KEY = "MODEL_MONITORING_ACCESS_KEY"
-    PIPELINES_ACCESS_KEY = "MODEL_MONITORING_PIPELINES_ACCESS_KEY"
-    KAFKA_BOOTSTRAP_SERVERS = "KAFKA_BOOTSTRAP_SERVERS"
     STREAM_PATH = "STREAM_PATH"
+    TSDB_CONNECTION = "TSDB_CONNECTION"
+
+    @classmethod
+    def mandatory_secrets(cls):
+        return [
+            cls.ENDPOINT_STORE_CONNECTION,
+            cls.STREAM_PATH,
+            cls.TSDB_CONNECTION,
+        ]
+
+
+class ModelEndpointTargetSchemas(MonitoringStrEnum):
+    V3IO = "v3io"
+    MYSQL = "mysql"
+    SQLITE = "sqlite"
 
 
 class ModelMonitoringStoreKinds:
@@ -162,15 +196,24 @@ class ModelMonitoringStoreKinds:
 
 class SchedulingKeys:
     LAST_ANALYZED = "last_analyzed"
+    ENDPOINT_ID = "endpoint_id"
+    APPLICATION_NAME = "application_name"
+    UID = "uid"
 
 
 class FileTargetKind:
     ENDPOINTS = "endpoints"
     EVENTS = "events"
+    PREDICTIONS = "predictions"
     STREAM = "stream"
     PARQUET = "parquet"
     APPS_PARQUET = "apps_parquet"
     LOG_STREAM = "log_stream"
+    APP_RESULTS = "app_results"
+    APP_METRICS = "app_metrics"
+    MONITORING_SCHEDULES = "monitoring_schedules"
+    MONITORING_APPLICATION = "monitoring_application"
+    ERRORS = "errors"
 
 
 class ModelMonitoringMode(str, Enum):
@@ -184,29 +227,23 @@ class EndpointType(IntEnum):
     LEAF_EP = 3  # end point that is a child of a router
 
 
-class PrometheusMetric:
-    PREDICTIONS_TOTAL = "predictions_total"
-    MODEL_LATENCY_SECONDS = "model_latency_seconds"
-    INCOME_FEATURES = "income_features"
-    ERRORS_TOTAL = "errors_total"
-    DRIFT_METRICS = "drift_metrics"
-    DRIFT_STATUS = "drift_status"
+class MonitoringFunctionNames(MonitoringStrEnum):
+    STREAM = "model-monitoring-stream"
+    APPLICATION_CONTROLLER = "model-monitoring-controller"
+    WRITER = "model-monitoring-writer"
 
 
-class MonitoringFunctionNames:
-    WRITER = "model-monitoring-writer"
-    BATCH = "model-monitoring-batch"
-    APPLICATION_CONTROLLER = "model-monitoring-controller"
-    STREAM = None
+class V3IOTSDBTables(MonitoringStrEnum):
+    APP_RESULTS = "app-results"
+    METRICS = "metrics"
+    EVENTS = "events"
+    ERRORS = "errors"
 
-    @staticmethod
-    def all():
-        return [
-            MonitoringFunctionNames.WRITER,
-            MonitoringFunctionNames.STREAM,
-            MonitoringFunctionNames.BATCH,
-            MonitoringFunctionNames.APPLICATION_CONTROLLER,
-        ]
+
+class TDEngineSuperTables(MonitoringStrEnum):
+    APP_RESULTS = "app_results"
+    METRICS = "metrics"
+    PREDICTIONS = "predictions"
 
 
 @dataclass
@@ -252,7 +289,7 @@ class EndpointUID:
     function_hash_key: str
     model: str
     model_version: str
-    uid: Optional[str] = None
+    uid: str = field(init=False)
 
     def __post_init__(self):
         function_ref = (
@@ -285,6 +322,7 @@ class ResultKindApp(Enum):
     concept_drift = 1
     model_performance = 2
     system_performance = 3
+    mm_app_anomaly = 4
 
 
 class ResultStatusApp(IntEnum):
@@ -299,9 +337,52 @@ class ResultStatusApp(IntEnum):
 
 
 class ModelMonitoringAppLabel:
-    KEY = "mlrun__type"
+    KEY = mlrun.common.constants.MLRunInternalLabels.mlrun_type
     VAL = "mlrun__model-monitoring-application"
 
+    def __str__(self) -> str:
+        return f"{self.KEY}={self.VAL}"
+
 
 class ControllerPolicy:
     BASE_PERIOD = "base_period"
+
+
+class HistogramDataDriftApplicationConstants:
+    NAME = "histogram-data-drift"
+    GENERAL_RESULT_NAME = "general_drift"
+
+
+class PredictionsQueryConstants:
+    DEFAULT_AGGREGATION_GRANULARITY = "10m"
+    INVOCATIONS = "invocations"
+
+
+class SpecialApps:
+    MLRUN_INFRA = "mlrun-infra"
+
+
+_RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
+
+
+V3IO_MODEL_MONITORING_DB = "v3io"
+
+
+class ModelEndpointMonitoringMetricType(StrEnum):
+    RESULT = "result"
+    METRIC = "metric"
+
+
+_FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
+FQN_PATTERN = (
+    rf"^(?P<project>{_FQN_PART_PATTERN})\."
+    rf"(?P<app>{_FQN_PART_PATTERN})\."
+    rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
+    rf"(?P<name>{_FQN_PART_PATTERN})$"
+)
+FQN_REGEX = re.compile(FQN_PATTERN)
+
+# refer to `mlrun.utils.regex.project_name`
+PROJECT_PATTERN = r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$"
+
+MODEL_ENDPOINT_ID_PATTERN = r"^[a-zA-Z0-9_-]+$"

mlrun/common/schemas/model_monitoring/grafana.py

@@ -11,12 +11,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
-from typing import List, Optional, Tuple, Union
+from typing import Optional, Union
 
 from pydantic import BaseModel
 
+import mlrun.common.types
+
+
+class GrafanaColumnType(mlrun.common.types.StrEnum):
+    NUMBER = "number"
+    STRING = "string"
+
 
 class GrafanaColumn(BaseModel):
     text: str
@@ -24,18 +30,16 @@ class GrafanaColumn(BaseModel):
 
 
 class GrafanaNumberColumn(GrafanaColumn):
-    text: str
-    type: str = "number"
+    type: str = GrafanaColumnType.NUMBER
 
 
 class GrafanaStringColumn(GrafanaColumn):
-    text: str
-    type: str = "string"
+    type: str = GrafanaColumnType.STRING
 
 
 class GrafanaTable(BaseModel):
-    columns: List[GrafanaColumn]
-    rows: List[List[Optional[Union[float, int, str]]]] = []
+    columns: list[GrafanaColumn]
+    rows: list[list[Optional[Union[float, int, str]]]] = []
     type: str = "table"
 
     def add_row(self, *args):
@@ -49,7 +53,7 @@ class GrafanaDataPoint(BaseModel):
 
 class GrafanaTimeSeriesTarget(BaseModel):
     target: str
-    datapoints: List[Tuple[float, int]] = []
+    datapoints: list[tuple[float, int]] = []
 
     def add_data_point(self, data_point: GrafanaDataPoint):
         self.datapoints.append((data_point.value, data_point.timestamp))

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -11,27 +11,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import enum
 import json
-import typing
-from typing import Any, Dict, List, Optional
+from datetime import datetime
+from typing import Any, NamedTuple, Optional, TypeVar
 
-from pydantic import BaseModel, Field, validator
-from pydantic.main import Extra
+from pydantic import BaseModel, Extra, Field, constr, validator
 
+# TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
+# At the moment `make lint` fails if this is removed.
 import mlrun.common.model_monitoring
 
 from ..object import ObjectKind, ObjectSpec, ObjectStatus
 from .constants import (
+    FQN_REGEX,
+    MODEL_ENDPOINT_ID_PATTERN,
+    PROJECT_PATTERN,
     EndpointType,
     EventFieldType,
     EventKeyMetrics,
     EventLiveStats,
+    ModelEndpointMonitoringMetricType,
     ModelMonitoringMode,
+    ResultKindApp,
+    ResultStatusApp,
 )
 
+Model = TypeVar("Model", bound=BaseModel)
+
 
 class ModelMonitoringStoreKinds:
     # TODO: do changes in examples & demos In 1.5.0 remove
@@ -40,27 +48,26 @@ class ModelMonitoringStoreKinds:
 
 
 class ModelEndpointMetadata(BaseModel):
-    project: Optional[str] = ""
+    project: constr(regex=PROJECT_PATTERN)
+    uid: constr(regex=MODEL_ENDPOINT_ID_PATTERN)
     labels: Optional[dict] = {}
-    uid: Optional[str] = ""
 
     class Config:
         extra = Extra.allow
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
+    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
         """Create a `ModelEndpointMetadata` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [EventFieldType.LABELS]
 
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -71,8 +78,8 @@ class ModelEndpointSpec(ObjectSpec):
     model: Optional[str] = ""  # <model_name>:<version>
     model_class: Optional[str] = ""
     model_uri: Optional[str] = ""
-    feature_names: Optional[List[str]] = []
-    label_names: Optional[List[str]] = []
+    feature_names: Optional[list[str]] = []
+    label_names: Optional[list[str]] = []
     stream_path: Optional[str] = ""
     algorithm: Optional[str] = ""
     monitor_configuration: Optional[dict] = {}
@@ -80,14 +87,13 @@ class ModelEndpointSpec(ObjectSpec):
     monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
+    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
         """Create a `ModelEndpointSpec` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_NAMES,
@@ -95,23 +101,13 @@ class ModelEndpointSpec(ObjectSpec):
                 EventFieldType.MONITOR_CONFIGURATION,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
 
-    @validator("monitor_configuration")
-    def set_name(cls, monitor_configuration):
-        return monitor_configuration or {
-            EventFieldType.DRIFT_DETECTED_THRESHOLD: (
-                mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.drift_detected
-            ),
-            EventFieldType.POSSIBLE_DRIFT_THRESHOLD: (
-                mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.possible_drift
-            ),
-        }
-
     @validator("model_uri")
+    @classmethod
     def validate_model_uri(cls, model_uri):
         """Validate that the model uri includes the required prefix"""
         prefix, uri = mlrun.datastore.parse_store_uri(model_uri)
@@ -123,8 +119,8 @@ class ModelEndpointSpec(ObjectSpec):
 
 
 class Histogram(BaseModel):
-    buckets: List[float]
-    counts: List[int]
+    buckets: list[float]
+    counts: list[int]
 
 
 class FeatureValues(BaseModel):
@@ -175,15 +171,15 @@ class ModelEndpointStatus(ObjectStatus):
     error_count: Optional[int] = 0
     drift_status: Optional[str] = ""
     drift_measures: Optional[dict] = {}
-    metrics: Optional[Dict[str, Dict[str, Any]]] = {
+    metrics: Optional[dict[str, dict[str, Any]]] = {
         EventKeyMetrics.GENERIC: {
             EventLiveStats.LATENCY_AVG_1H: 0,
             EventLiveStats.PREDICTIONS_PER_SECOND: 0,
         }
     }
-    features: Optional[List[Features]] = []
-    children: Optional[List[str]] = []
-    children_uids: Optional[List[str]] = []
+    features: Optional[list[Features]] = []
+    children: Optional[list[str]] = []
+    children_uids: Optional[list[str]] = []
     endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP
     monitoring_feature_set_uri: Optional[str] = ""
     state: Optional[str] = ""
@@ -192,14 +188,13 @@ class ModelEndpointStatus(ObjectStatus):
         extra = Extra.allow
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
+    def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
         """Create a `ModelEndpointStatus` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
         """
-        new_object = cls()
         if json_parse_values is None:
             json_parse_values = [
                 EventFieldType.FEATURE_STATS,
@@ -211,7 +206,7 @@ class ModelEndpointStatus(ObjectStatus):
                 EventFieldType.ENDPOINT_TYPE,
             ]
         return _mapping_attributes(
-            base_model=new_object,
+            model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
         )
@@ -219,22 +214,13 @@ class ModelEndpointStatus(ObjectStatus):
 
 class ModelEndpoint(BaseModel):
     kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
-    metadata: ModelEndpointMetadata = ModelEndpointMetadata()
+    metadata: ModelEndpointMetadata
     spec: ModelEndpointSpec = ModelEndpointSpec()
     status: ModelEndpointStatus = ModelEndpointStatus()
 
     class Config:
         extra = Extra.allow
 
-    def __init__(self, **data: Any):
-        super().__init__(**data)
-        if self.metadata.uid is None:
-            uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
-                function_uri=self.spec.function_uri,
-                versioned_model=self.spec.model,
-            )
-            self.metadata.uid = str(uid)
-
     def flat_dict(self):
         """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
         the model endpoint object in the database.
@@ -275,7 +261,7 @@ class ModelEndpoint(BaseModel):
         return flatten_dict
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict):
+    def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpoint":
         """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
         is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
 
@@ -290,24 +276,85 @@ class ModelEndpoint(BaseModel):
 
 
 class ModelEndpointList(BaseModel):
-    endpoints: List[ModelEndpoint] = []
+    endpoints: list[ModelEndpoint] = []
+
+
+class ModelEndpointMonitoringMetric(BaseModel):
+    project: str
+    app: str
+    type: ModelEndpointMonitoringMetricType
+    name: str
+    full_name: str
+
+
+def _compose_full_name(
+    *,
+    project: str,
+    app: str,
+    name: str,
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT,
+) -> str:
+    return ".".join([project, app, type, name])
+
+
+def _parse_metric_fqn_to_monitoring_metric(fqn: str) -> ModelEndpointMonitoringMetric:
+    match = FQN_REGEX.fullmatch(fqn)
+    if match is None:
+        raise ValueError("The fully qualified name is not in the expected format")
+    return ModelEndpointMonitoringMetric.parse_obj(
+        match.groupdict() | {"full_name": fqn}
+    )
+
+
+class _MetricPoint(NamedTuple):
+    timestamp: datetime
+    value: float
+
+
+class _ResultPoint(NamedTuple):
+    timestamp: datetime
+    value: float
+    status: ResultStatusApp
+
+
+class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
+    full_name: str
+    type: ModelEndpointMonitoringMetricType
+    data: bool
+
+
+class ModelEndpointMonitoringMetricValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.METRIC
+    values: list[_MetricPoint]
+    data: bool = True
+
+
+class ModelEndpointMonitoringResultValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT
+    result_kind: ResultKindApp
+    values: list[_ResultPoint]
+    data: bool = True
+
+
+class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBase):
+    full_name: str
+    type: ModelEndpointMonitoringMetricType
+    data: bool = False
 
 
 def _mapping_attributes(
-    base_model: BaseModel,
-    flattened_dictionary: dict,
-    json_parse_values: typing.List = None,
-):
+    model_class: type[Model], flattened_dictionary: dict, json_parse_values: list
+) -> Model:
     """Generate a `BaseModel` object with the provided dictionary attributes.
 
-    :param base_model:           `BaseModel` object (e.g. `ModelEndpointMetadata`).
+    :param model_class:          `BaseModel` class (e.g. `ModelEndpointMetadata`).
     :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
     :param json_parse_values:    List of dictionary keys with a JSON string value that will be parsed into a
                                  dictionary using json.loads().
     """
     # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
     # from the flattened dictionary.
-    wanted_keys = base_model.__fields__.keys()
+    wanted_keys = model_class.__fields__.keys()
 
     # Generate a filtered flattened dictionary that will be parsed into the BaseModel object
     dict_to_parse = {}
@@ -321,7 +368,7 @@ def _mapping_attributes(
             else:
                 dict_to_parse[field_key] = flattened_dictionary[field_key]
 
-    return base_model.parse_obj(dict_to_parse)
+    return model_class.parse_obj(dict_to_parse)
 
 
 def _json_loads_if_not_none(field: Any) -> Any: