mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py}

@@ -1,4 +1,4 @@
-# Copyright 2023 Iguazio
+# Copyright 2024 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,27 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import typing
 from abc import ABC, abstractmethod
 
+import mlrun.common.schemas.model_monitoring as mm_schemas
+
 
-class ModelEndpointStore(ABC):
+class StoreBase(ABC):
     """
-    An abstract class to handle the model endpoint in the DB target.
+    An abstract class to handle the store object in the DB target.
     """
 
     def __init__(self, project: str):
         """
-        Initialize a new model endpoint target.
+        Initialize a new store target.
 
-        :param project:             The name of the project.
+        :param project: The name of the project.
         """
         self.project = project
 
     @abstractmethod
-    def write_model_endpoint(self, endpoint: typing.Dict[str, typing.Any]):
+    def write_model_endpoint(self, endpoint: dict[str, typing.Any]):
         """
         Create a new endpoint record in the DB table.
 
@@ -41,7 +42,7 @@ class ModelEndpointStore(ABC):
 
     @abstractmethod
     def update_model_endpoint(
-        self, endpoint_id: str, attributes: typing.Dict[str, typing.Any]
+        self, endpoint_id: str, attributes: dict[str, typing.Any]
     ):
         """
         Update a model endpoint record with a given attributes.
@@ -63,14 +64,10 @@ class ModelEndpointStore(ABC):
         pass
 
     @abstractmethod
-    def delete_model_endpoints_resources(
-        self, endpoints: typing.List[typing.Dict[str, typing.Any]]
-    ):
+    def delete_model_endpoints_resources(self):
         """
         Delete all model endpoints resources.
 
-        :param endpoints: A list of model endpoints flattened dictionaries.
-
         """
         pass
 
@@ -78,7 +75,7 @@ class ModelEndpointStore(ABC):
     def get_model_endpoint(
         self,
         endpoint_id: str,
-    ) -> typing.Dict[str, typing.Any]:
+    ) -> dict[str, typing.Any]:
         """
         Get a single model endpoint record.
 
@@ -93,10 +90,10 @@ class ModelEndpointStore(ABC):
         self,
         model: str = None,
         function: str = None,
-        labels: typing.List[str] = None,
+        labels: list[str] = None,
         top_level: bool = None,
-        uids: typing.List = None,
-    ) -> typing.List[typing.Dict[str, typing.Any]]:
+        uids: list = None,
+    ) -> list[dict[str, typing.Any]]:
         """
         Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
         By default, when no filters are applied, all available model endpoints for the given project will
@@ -115,33 +112,60 @@ class ModelEndpointStore(ABC):
         pass
 
     @abstractmethod
-    def get_endpoint_real_time_metrics(
+    def write_application_event(
         self,
-        endpoint_id: str,
-        metrics: typing.List[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> typing.Dict[str, typing.List[typing.Tuple[str, float]]]:
+        event: dict[str, typing.Any],
+        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
+    ) -> None:
+        """
+        Write a new event in the target table.
+
+        :param event: An event dictionary that represents the application result, should be corresponded to the
+                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
+                      object.
+        :param kind: The type of the event, can be either "result" or "metric".
         """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
+
+    @abstractmethod
+    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
+        """
+        Get the last analyzed time for the provided model endpoint and application.
 
         :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
+        :param application_name: Registered application name.
+
+        :return: Timestamp as a Unix time.
+        :raise:  MLRunNotFoundError if last analyzed value is not found.
         """
+        pass
 
+    @abstractmethod
+    def update_last_analyzed(
+        self,
+        endpoint_id: str,
+        application_name: str,
+        last_analyzed: int,
+    ):
+        """
+        Update the last analyzed time for the provided model endpoint and application.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param application_name: Registered application name.
+        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
+                                 application and model endpoint.
+
+        """
         pass
+
+    @abstractmethod
+    def get_model_endpoint_metrics(
+        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
+    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
+        """
+        Get the model monitoring results and metrics of the requested model endpoint.
+
+        :param: endpoint_id: The model endpoint identifier.
+        :param: type:        The type of the requested metrics ("result" or "metric").
+
+        :return:             A list of the available metrics.
+        """

mlrun/model_monitoring/db/stores/sqldb/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mlrun/model_monitoring/db/stores/sqldb/models/__init__.py

@@ -0,0 +1,71 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import partial
+from typing import Optional, TypeVar, Union
+
+from .mysql import ApplicationMetricsTable as MySQLApplicationMetricsTable
+from .mysql import ApplicationResultTable as MySQLApplicationResultTable
+from .mysql import ModelEndpointsTable as MySQLModelEndpointsTable
+from .mysql import MonitoringSchedulesTable as MySQLMonitoringSchedulesTable
+from .sqlite import ApplicationMetricsTable as SQLiteApplicationMetricsTable
+from .sqlite import ApplicationResultTable as SQLiteApplicationResultTable
+from .sqlite import ModelEndpointsTable as SQLiteModelEndpointsTable
+from .sqlite import MonitoringSchedulesTable as SQLiteMonitoringSchedulesTable
+
+MySQLTableType = TypeVar("MySQLTableType")
+SQLiteTableType = TypeVar("SQLiteTableType")
+
+_MYSQL_SCHEME = "mysql:"
+
+
+def _get_sql_table(
+    *,
+    mysql_table: MySQLTableType,
+    sqlite_table: SQLiteTableType,
+    connection_string: Optional[str] = None,
+) -> Union[MySQLTableType, SQLiteTableType]:
+    """
+    Return a SQLAlchemy table for MySQL or SQLite according to the connection string.
+    Note: this function should not be directly used in other modules.
+    """
+    if connection_string and _MYSQL_SCHEME in connection_string:
+        return mysql_table
+    return sqlite_table
+
+
+_get_model_endpoints_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLModelEndpointsTable,
+    sqlite_table=SQLiteModelEndpointsTable,
+)
+
+
+_get_application_result_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationResultTable,
+    sqlite_table=SQLiteApplicationResultTable,
+)
+
+_get_application_metrics_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationMetricsTable,
+    sqlite_table=SQLiteApplicationMetricsTable,
+)
+
+_get_monitoring_schedules_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLMonitoringSchedulesTable,
+    sqlite_table=SQLiteMonitoringSchedulesTable,
+)

mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py

@@ -11,10 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from sqlalchemy import (
+    TIMESTAMP,
+    Boolean,
+    Column,
+    Float,
+    Integer,
+    String,
+    Text,
+)
 
-from sqlalchemy import TIMESTAMP, Boolean, Column, Integer, String, Text
-
-from mlrun.common.schemas.model_monitoring import EventFieldType
+from mlrun.common.schemas.model_monitoring import (
+    EventFieldType,
+    FileTargetKind,
+    MetricData,
+    ResultData,
+    SchedulingKeys,
+    WriterEvent,
+)
 from mlrun.utils.db import BaseModel
 
 
@@ -76,9 +90,99 @@ class ModelEndpointsBaseTable(BaseModel):
     metrics = Column(EventFieldType.METRICS, Text)
     first_request = Column(
         EventFieldType.FIRST_REQUEST,
-        TIMESTAMP,
+        TIMESTAMP(timezone=True),
     )
     last_request = Column(
         EventFieldType.LAST_REQUEST,
-        TIMESTAMP,
+        TIMESTAMP(timezone=True),
+    )
+
+
+class ApplicationResultBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.APP_RESULTS
+
+    uid = Column(EventFieldType.UID, String(120), primary_key=True)
+
+    application_name = Column(
+        WriterEvent.APPLICATION_NAME,
+        String(40),
+        nullable=True,
+    )
+
+    endpoint_id = Column(
+        WriterEvent.ENDPOINT_ID,
+        String(40),
+        nullable=True,
+    )
+
+    result_name = Column(
+        ResultData.RESULT_NAME,
+        String(40),
+    )
+
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+
+    result_status = Column(ResultData.RESULT_STATUS, String(10))
+    result_kind = Column(ResultData.RESULT_KIND, String(40))
+    result_value = Column(ResultData.RESULT_VALUE, Float)
+    result_extra_data = Column(ResultData.RESULT_EXTRA_DATA, Text)
+    current_stats = Column(ResultData.CURRENT_STATS, Text)
+
+
+class ApplicationMetricsBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.APP_METRICS
+
+    uid = Column(EventFieldType.UID, String(120), primary_key=True)
+    application_name = Column(
+        WriterEvent.APPLICATION_NAME,
+        String(40),
+        nullable=True,
+    )
+    endpoint_id = Column(
+        WriterEvent.ENDPOINT_ID,
+        String(40),
+        nullable=True,
+    )
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+    metric_name = Column(
+        MetricData.METRIC_NAME,
+        String(40),
+    )
+    metric_value = Column(MetricData.METRIC_VALUE, Float)
+
+
+class MonitoringSchedulesBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.MONITORING_SCHEDULES
+
+    uid = Column(SchedulingKeys.UID, String(32), primary_key=True)
+
+    application_name = Column(
+        SchedulingKeys.APPLICATION_NAME,
+        String(40),
+        nullable=False,
+    )
+
+    endpoint_id = Column(
+        SchedulingKeys.ENDPOINT_ID,
+        String(40),
+        nullable=False,
+    )
+
+    last_analyzed = Column(
+        SchedulingKeys.LAST_ANALYZED,
+        Integer,
     )

mlrun/model_monitoring/db/stores/sqldb/models/mysql.py

@@ -0,0 +1,88 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sqlalchemy.dialects.mysql
+from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy.ext.declarative import declarative_base, declared_attr
+
+from mlrun.common.schemas.model_monitoring import (
+    EventFieldType,
+    WriterEvent,
+)
+
+from .base import (
+    ApplicationMetricsBaseTable,
+    ApplicationResultBaseTable,
+    ModelEndpointsBaseTable,
+    MonitoringSchedulesBaseTable,
+)
+
+Base = declarative_base()
+
+
+class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
+    first_request = Column(
+        EventFieldType.FIRST_REQUEST,
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
+    )
+    last_request = Column(
+        EventFieldType.LAST_REQUEST,
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
+    )
+
+
+class _ApplicationResultOrMetric:
+    """
+    This class sets common columns of `ApplicationResultTable` and `ApplicationMetricsTable`
+    to the correct values in MySQL.
+    Note: This class must come before the base tables in the inheritance order to override
+    the relevant columns.
+    """
+
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
+    )
+
+    @declared_attr
+    def endpoint_id(self):
+        return Column(
+            String(40),
+            ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),
+        )
+
+
+class ApplicationResultTable(
+    Base, _ApplicationResultOrMetric, ApplicationResultBaseTable
+):
+    pass
+
+
+class ApplicationMetricsTable(
+    Base, _ApplicationResultOrMetric, ApplicationMetricsBaseTable
+):
+    pass
+
+
+class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
+    @declared_attr
+    def endpoint_id(self):
+        return Column(
+            String(40),
+            ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),
+        )

mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py}

@@ -12,23 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sqlalchemy.dialects.mysql
-from sqlalchemy import Column
 from sqlalchemy.ext.declarative import declarative_base
 
-from mlrun.common.schemas.model_monitoring import EventFieldType
-
-from .base import ModelEndpointsBaseTable
+from .base import (
+    ApplicationMetricsBaseTable,
+    ApplicationResultBaseTable,
+    ModelEndpointsBaseTable,
+    MonitoringSchedulesBaseTable,
+)
 
 Base = declarative_base()
 
 
 class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
-    first_request = Column(
-        EventFieldType.FIRST_REQUEST,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
-    )
-    last_request = Column(
-        EventFieldType.LAST_REQUEST,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
-    )
+    pass
+
+
+class ApplicationResultTable(Base, ApplicationResultBaseTable):
+    pass
+
+
+class ApplicationMetricsTable(Base, ApplicationMetricsBaseTable):
+    pass
+
+
+class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
+    pass