mlrun 1.7.0rc4__py3-none-any.whl → 1.7.2__py3-none-any.whl

mlrun/model_monitoring/db/stores/__init__.py

@@ -0,0 +1,136 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+import typing
+import warnings
+
+import mlrun.common.schemas.secret
+import mlrun.errors
+
+from .base import StoreBase
+
+
+class ObjectStoreFactory(enum.Enum):
+    """Enum class to handle the different store type values for saving model monitoring records."""
+
+    v3io_nosql = "v3io-nosql"
+    SQL = "sql"
+
+    def to_object_store(
+        self,
+        project: str,
+        **kwargs,
+    ) -> StoreBase:
+        """
+        Return a StoreBase object based on the provided enum value.
+
+        :param project:                   The name of the project.
+
+        :return: `StoreBase` object.
+
+        """
+
+        if self == self.v3io_nosql:
+            from mlrun.model_monitoring.db.stores.v3io_kv.kv_store import KVStoreBase
+
+            return KVStoreBase(project=project)
+
+        # Assuming SQL store target if store type is not KV.
+        # Update these lines once there are more than two store target types.
+
+        from mlrun.model_monitoring.db.stores.sqldb.sql_store import SQLStoreBase
+
+        return SQLStoreBase(
+            project=project,
+            **kwargs,
+        )
+
+    @classmethod
+    def _missing_(cls, value: typing.Any):
+        """A lookup function to handle an invalid value.
+        :param value: Provided enum (invalid) value.
+        """
+        valid_values = list(cls.__members__.keys())
+        raise mlrun.errors.MLRunInvalidMMStoreTypeError(
+            f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
+        )
+
+
+def get_model_endpoint_store(
+    project: str,
+    access_key: str = None,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+) -> StoreBase:
+    # Leaving here for backwards compatibility
+    warnings.warn(
+        "The 'get_model_endpoint_store' function is deprecated and will be removed in 1.9.0. "
+        "Please use `get_store_object` instead.",
+        # TODO: remove in 1.9.0
+        FutureWarning,
+    )
+    return get_store_object(
+        project=project, access_key=access_key, secret_provider=secret_provider
+    )
+
+
+def get_store_object(
+    project: str,
+    secret_provider: typing.Optional[typing.Callable[[str], str]] = None,
+    store_connection_string: typing.Optional[str] = None,
+    **kwargs,
+) -> StoreBase:
+    """
+    Generate a store object. If a connection string is provided, the store type will be updated according to the
+    connection string. Currently, the supported store types are SQL and v3io-nosql.
+
+    :param project:                 The name of the project.
+    :param secret_provider:         An optional secret provider to get the connection string secret.
+    :param store_connection_string: Optional explicit connection string of the store.
+
+    :return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
+             and delete a model endpoint record.
+    :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide store connection
+             or the provided store connection is invalid.
+    """
+
+    store_connection_string = (
+        store_connection_string
+        or mlrun.model_monitoring.helpers.get_connection_string(
+            secret_provider=secret_provider
+        )
+    )
+
+    if store_connection_string and (
+        store_connection_string.startswith("mysql")
+        or store_connection_string.startswith("sqlite")
+    ):
+        store_type = mlrun.common.schemas.model_monitoring.ModelEndpointTarget.SQL
+        kwargs["store_connection_string"] = store_connection_string
+    elif store_connection_string and store_connection_string == "v3io":
+        store_type = (
+            mlrun.common.schemas.model_monitoring.ModelEndpointTarget.V3IO_NOSQL
+        )
+    else:
+        raise mlrun.errors.MLRunInvalidMMStoreTypeError(
+            "You must provide a valid store connection by using "
+            "set_model_monitoring_credentials API."
+        )
+    # Get store type value from ObjectStoreFactory enum class
+    store_type_fact = ObjectStoreFactory(store_type)
+
+    # Convert into store target object
+    return store_type_fact.to_object_store(
+        project=project, secret_provider=secret_provider, **kwargs
+    )

mlrun/model_monitoring/db/stores/base/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .store import StoreBase

mlrun/model_monitoring/db/stores/base/store.py

@@ -0,0 +1,213 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import typing
+from abc import ABC, abstractmethod
+
+import mlrun.common.schemas.model_monitoring as mm_schemas
+
+
+class StoreBase(ABC):
+    type: typing.ClassVar[str]
+    """
+    An abstract class to handle the store object in the DB target.
+    """
+
+    def __init__(self, project: str):
+        """
+        Initialize a new store target.
+
+        :param project: The name of the project.
+        """
+        self.project = project
+
+    @abstractmethod
+    def write_model_endpoint(self, endpoint: dict[str, typing.Any]):
+        """
+        Create a new endpoint record in the DB table.
+
+        :param endpoint: model endpoint dictionary that will be written into the DB.
+        """
+        pass
+
+    @abstractmethod
+    def update_model_endpoint(
+        self, endpoint_id: str, attributes: dict[str, typing.Any]
+    ):
+        """
+        Update a model endpoint record with a given attributes.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        :param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
+                           of the attributes dictionary should exist in the DB table.
+
+        """
+        pass
+
+    @abstractmethod
+    def delete_model_endpoint(self, endpoint_id: str):
+        """
+        Deletes the record of a given model endpoint id.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        """
+        pass
+
+    @abstractmethod
+    def delete_model_endpoints_resources(self):
+        """
+        Delete all model endpoints resources.
+
+        """
+        pass
+
+    @abstractmethod
+    def get_model_endpoint(
+        self,
+        endpoint_id: str,
+    ) -> dict[str, typing.Any]:
+        """
+        Get a single model endpoint record.
+
+        :param endpoint_id: The unique id of the model endpoint.
+
+        :return: A model endpoint record as a dictionary.
+        """
+        pass
+
+    @abstractmethod
+    def list_model_endpoints(
+        self,
+        model: str = None,
+        function: str = None,
+        labels: list[str] = None,
+        top_level: bool = None,
+        uids: list = None,
+        include_stats: bool = None,
+    ) -> list[dict[str, typing.Any]]:
+        """
+        Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
+        By default, when no filters are applied, all available model endpoints for the given project will
+        be listed.
+
+        :param model:           The name of the model to filter by.
+        :param function:        The name of the function to filter by.
+        :param labels:          A list of labels to filter by. Label filters work by either filtering a specific value
+                                of a label (i.e. list("key=value")) or by looking for the existence of a given
+                                key (i.e. "key").
+        :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
+        :param uids:             List of model endpoint unique ids to include in the result.
+        :param include_stats:   If True, will include model endpoint statistics in the result.
+
+        :return: A list of model endpoint dictionaries.
+        """
+        pass
+
+    @abstractmethod
+    def write_application_event(
+        self,
+        event: dict[str, typing.Any],
+        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
+    ) -> None:
+        """
+        Write a new event in the target table.
+
+        :param event: An event dictionary that represents the application result, should be corresponded to the
+                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
+                      object.
+        :param kind: The type of the event, can be either "result" or "metric".
+        """
+
+    @abstractmethod
+    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
+        """
+        Get the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+
+        :return: Timestamp as a Unix time.
+        :raise:  MLRunNotFoundError if last analyzed value is not found.
+        """
+        pass
+
+    @abstractmethod
+    def update_last_analyzed(
+        self,
+        endpoint_id: str,
+        application_name: str,
+        last_analyzed: int,
+    ):
+        """
+        Update the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
+                                 application and model endpoint.
+
+        """
+        pass
+
+    @abstractmethod
+    def get_model_endpoint_metrics(
+        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
+    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
+        """
+        Get the model monitoring results and metrics of the requested model endpoint.
+
+        :param: endpoint_id: The model endpoint identifier.
+        :param: type:        The type of the requested metrics ("result" or "metric").
+
+        :return:             A list of the available metrics.
+        """
+
+    @staticmethod
+    def _validate_labels(
+        endpoint_dict: dict,
+        labels: list,
+    ) -> bool:
+        """Validate that the model endpoint dictionary has the provided labels. There are 2 possible cases:
+        1 - Labels were provided as a list of key-values pairs (e.g. ['label_1=value_1', 'label_2=value_2']): Validate
+            that each pair exist in the endpoint dictionary.
+        2 - Labels were provided as a list of key labels (e.g. ['label_1', 'label_2']): Validate that each key exist in
+            the endpoint labels dictionary.
+
+        :param endpoint_dict: Dictionary of the model endpoint records.
+        :param labels:        List of dictionary of required labels.
+
+        :return: True if the labels exist in the endpoint labels dictionary, otherwise False.
+        """
+
+        # Convert endpoint labels into dictionary
+        endpoint_labels = json.loads(
+            endpoint_dict.get(mm_schemas.EventFieldType.LABELS)
+        )
+
+        for label in labels:
+            # Case 1 - label is a key=value pair
+            if "=" in label:
+                lbl, value = list(map(lambda x: x.strip(), label.split("=")))
+                if lbl not in endpoint_labels or str(endpoint_labels[lbl]) != value:
+                    return False
+            # Case 2 - label is just a key
+            else:
+                if label not in endpoint_labels:
+                    return False
+
+        return True
+
+    def create_tables(self):
+        pass

mlrun/model_monitoring/db/stores/sqldb/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mlrun/model_monitoring/db/stores/sqldb/models/__init__.py

@@ -0,0 +1,71 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import partial
+from typing import Optional, TypeVar, Union
+
+from .mysql import ApplicationMetricsTable as MySQLApplicationMetricsTable
+from .mysql import ApplicationResultTable as MySQLApplicationResultTable
+from .mysql import ModelEndpointsTable as MySQLModelEndpointsTable
+from .mysql import MonitoringSchedulesTable as MySQLMonitoringSchedulesTable
+from .sqlite import ApplicationMetricsTable as SQLiteApplicationMetricsTable
+from .sqlite import ApplicationResultTable as SQLiteApplicationResultTable
+from .sqlite import ModelEndpointsTable as SQLiteModelEndpointsTable
+from .sqlite import MonitoringSchedulesTable as SQLiteMonitoringSchedulesTable
+
+MySQLTableType = TypeVar("MySQLTableType")
+SQLiteTableType = TypeVar("SQLiteTableType")
+
+_MYSQL_SCHEME = "mysql:"
+
+
+def _get_sql_table(
+    *,
+    mysql_table: MySQLTableType,
+    sqlite_table: SQLiteTableType,
+    connection_string: Optional[str] = None,
+) -> Union[MySQLTableType, SQLiteTableType]:
+    """
+    Return a SQLAlchemy table for MySQL or SQLite according to the connection string.
+    Note: this function should not be directly used in other modules.
+    """
+    if connection_string and _MYSQL_SCHEME in connection_string:
+        return mysql_table
+    return sqlite_table
+
+
+_get_model_endpoints_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLModelEndpointsTable,
+    sqlite_table=SQLiteModelEndpointsTable,
+)
+
+
+_get_application_result_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationResultTable,
+    sqlite_table=SQLiteApplicationResultTable,
+)
+
+_get_application_metrics_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationMetricsTable,
+    sqlite_table=SQLiteApplicationMetricsTable,
+)
+
+_get_monitoring_schedules_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLMonitoringSchedulesTable,
+    sqlite_table=SQLiteMonitoringSchedulesTable,
+)

mlrun/model_monitoring/db/stores/sqldb/models/base.py

@@ -0,0 +1,190 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import (
+    DATETIME,
+    TIMESTAMP,  # TODO: migrate to DATETIME, see ML-6921
+    Boolean,
+    Column,
+    Float,
+    Integer,
+    String,
+    Text,
+)
+
+from mlrun.common.schemas.model_monitoring import (
+    EventFieldType,
+    FileTargetKind,
+    MetricData,
+    ResultData,
+    SchedulingKeys,
+    WriterEvent,
+)
+from mlrun.utils.db import BaseModel
+
+
+class ModelEndpointsBaseTable(BaseModel):
+    __tablename__ = EventFieldType.MODEL_ENDPOINTS
+
+    uid = Column(
+        EventFieldType.UID,
+        String(40),
+        primary_key=True,
+    )
+    state = Column(EventFieldType.STATE, String(10))
+    project = Column(EventFieldType.PROJECT, String(40))
+    function_uri = Column(
+        EventFieldType.FUNCTION_URI,
+        String(255),
+    )
+    model = Column(EventFieldType.MODEL, String(255))
+    model_class = Column(
+        EventFieldType.MODEL_CLASS,
+        String(255),
+    )
+    labels = Column(EventFieldType.LABELS, Text)
+    model_uri = Column(EventFieldType.MODEL_URI, String(255))
+    stream_path = Column(EventFieldType.STREAM_PATH, Text)
+    algorithm = Column(
+        EventFieldType.ALGORITHM,
+        String(255),
+    )
+    active = Column(EventFieldType.ACTIVE, Boolean)
+    monitoring_mode = Column(
+        EventFieldType.MONITORING_MODE,
+        String(10),
+    )
+    feature_stats = Column(EventFieldType.FEATURE_STATS, Text)
+    current_stats = Column(EventFieldType.CURRENT_STATS, Text)
+    feature_names = Column(EventFieldType.FEATURE_NAMES, Text)
+    children = Column(EventFieldType.CHILDREN, Text)
+    label_names = Column(EventFieldType.LABEL_NAMES, Text)
+    endpoint_type = Column(
+        EventFieldType.ENDPOINT_TYPE,
+        String(10),
+    )
+    children_uids = Column(EventFieldType.CHILDREN_UIDS, Text)
+    drift_measures = Column(EventFieldType.DRIFT_MEASURES, Text)
+    drift_status = Column(
+        EventFieldType.DRIFT_STATUS,
+        String(40),
+    )
+    monitor_configuration = Column(
+        EventFieldType.MONITOR_CONFIGURATION,
+        Text,
+    )
+    monitoring_feature_set_uri = Column(
+        EventFieldType.FEATURE_SET_URI,
+        String(255),
+    )
+    error_count = Column(EventFieldType.ERROR_COUNT, Integer)
+    metrics = Column(EventFieldType.METRICS, Text)
+    first_request = Column(
+        EventFieldType.FIRST_REQUEST,
+        TIMESTAMP(timezone=True),  # TODO: migrate to DATETIME, see ML-6921
+    )
+    last_request = Column(
+        EventFieldType.LAST_REQUEST,
+        TIMESTAMP(timezone=True),  # TODO: migrate to DATETIME, see ML-6921
+    )
+
+
+class ApplicationResultBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.APP_RESULTS
+
+    uid = Column(EventFieldType.UID, String(120), primary_key=True)
+
+    application_name = Column(
+        WriterEvent.APPLICATION_NAME,
+        String(40),
+        nullable=True,
+    )
+
+    endpoint_id = Column(
+        WriterEvent.ENDPOINT_ID,
+        String(40),
+        nullable=True,
+    )
+
+    result_name = Column(
+        ResultData.RESULT_NAME,
+        String(40),
+    )
+
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        DATETIME(timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        DATETIME(timezone=True),
+    )
+
+    result_status = Column(ResultData.RESULT_STATUS, String(10))
+    result_kind = Column(ResultData.RESULT_KIND, String(40))
+    result_value = Column(ResultData.RESULT_VALUE, Float)
+    result_extra_data = Column(ResultData.RESULT_EXTRA_DATA, Text)
+    current_stats = Column(ResultData.CURRENT_STATS, Text)
+
+
+class ApplicationMetricsBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.APP_METRICS
+
+    uid = Column(EventFieldType.UID, String(120), primary_key=True)
+    application_name = Column(
+        WriterEvent.APPLICATION_NAME,
+        String(40),
+        nullable=True,
+    )
+    endpoint_id = Column(
+        WriterEvent.ENDPOINT_ID,
+        String(40),
+        nullable=True,
+    )
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        DATETIME(timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        DATETIME(timezone=True),
+    )
+    metric_name = Column(
+        MetricData.METRIC_NAME,
+        String(40),
+    )
+    metric_value = Column(MetricData.METRIC_VALUE, Float)
+
+
+class MonitoringSchedulesBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.MONITORING_SCHEDULES
+
+    uid = Column(SchedulingKeys.UID, String(32), primary_key=True)
+
+    application_name = Column(
+        SchedulingKeys.APPLICATION_NAME,
+        String(40),
+        nullable=False,
+    )
+
+    endpoint_id = Column(
+        SchedulingKeys.ENDPOINT_ID,
+        String(40),
+        nullable=False,
+    )
+
+    last_analyzed = Column(
+        SchedulingKeys.LAST_ANALYZED,
+        Integer,
+    )