mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc22__py3-none-any.whl

mlrun/model_monitoring/controller.py

@@ -457,6 +457,7 @@ class MonitoringApplicationController:
                 )
 
                 for start_infer_time, end_infer_time in batch_window.get_intervals():
+                    # start - TODO : delete in 1.9.0 (V1 app deprecation)
                     try:
                         # Get application sample data
                         offline_response = cls._get_sample_df(
@@ -504,7 +505,7 @@ class MonitoringApplicationController:
                     current_stats = calculate_inputs_statistics(
                         sample_set_statistics=feature_stats, inputs=df
                     )
-
+                    # end - TODO : delete in 1.9.0 (V1 app deprecation)
                     cls._push_to_applications(
                         current_stats=current_stats,
                         feature_stats=feature_stats,
@@ -613,6 +614,7 @@ class MonitoringApplicationController:
                 project=project,
                 function_name=mm_constants.MonitoringFunctionNames.WRITER,
             ),
+            mm_constants.ApplicationEvent.MLRUN_CONTEXT: {},  # TODO : for future use by ad-hoc batch infer
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})

mlrun/model_monitoring/db/__init__.py

@@ -14,3 +14,5 @@
 
 from .stores import ObjectStoreFactory, get_store_object
 from .stores.base import StoreBase
+from .tsdb import get_tsdb_connector
+from .tsdb.base import TSDBConnector

mlrun/model_monitoring/db/stores/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 import enum
 import typing
 import warnings

mlrun/model_monitoring/db/stores/base/store.py

@@ -15,6 +15,8 @@
 import typing
 from abc import ABC, abstractmethod
 
+import mlrun.common.schemas.model_monitoring as mm_schemas
+
 
 class StoreBase(ABC):
     """
@@ -62,12 +64,10 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def delete_model_endpoints_resources(self, endpoints: list[dict[str, typing.Any]]):
+    def delete_model_endpoints_resources(self):
         """
         Delete all model endpoints resources.
 
-        :param endpoints: A list of model endpoints flattened dictionaries.
-
         """
         pass
 
@@ -112,47 +112,19 @@ class StoreBase(ABC):
         pass
 
     @abstractmethod
-    def get_endpoint_real_time_metrics(
+    def write_application_event(
         self,
-        endpoint_id: str,
-        metrics: list[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> dict[str, list[tuple[str, float]]]:
-        """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
-        """
-
-        pass
-
-    @abstractmethod
-    def write_application_result(self, event: dict[str, typing.Any]):
+        event: dict[str, typing.Any],
+        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
+    ) -> None:
         """
-        Write a new application result event in the target table.
+        Write a new event in the target table.
 
         :param event: An event dictionary that represents the application result, should be corresponded to the
                       schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
                       object.
+        :param kind: The type of the event, can be either "result" or "metric".
         """
-        pass
 
     @abstractmethod
     def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
@@ -184,3 +156,16 @@ class StoreBase(ABC):
 
         """
         pass
+
+    @abstractmethod
+    def get_model_endpoint_metrics(
+        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
+    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
+        """
+        Get the model monitoring results and metrics of the requested model endpoint.
+
+        :param: endpoint_id: The model endpoint identifier.
+        :param: type:        The type of the requested metrics ("result" or "metric").
+
+        :return:             A list of the available metrics.
+        """

mlrun/model_monitoring/db/stores/sqldb/models/__init__.py

@@ -12,38 +12,60 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional, Union
+from functools import partial
+from typing import Optional, TypeVar, Union
 
+from .mysql import ApplicationMetricsTable as MySQLApplicationMetricsTable
 from .mysql import ApplicationResultTable as MySQLApplicationResultTable
 from .mysql import ModelEndpointsTable as MySQLModelEndpointsTable
 from .mysql import MonitoringSchedulesTable as MySQLMonitoringSchedulesTable
+from .sqlite import ApplicationMetricsTable as SQLiteApplicationMetricsTable
 from .sqlite import ApplicationResultTable as SQLiteApplicationResultTable
 from .sqlite import ModelEndpointsTable as SQLiteModelEndpointsTable
 from .sqlite import MonitoringSchedulesTable as SQLiteMonitoringSchedulesTable
 
+MySQLTableType = TypeVar("MySQLTableType")
+SQLiteTableType = TypeVar("SQLiteTableType")
 
-def _get_model_endpoints_table(
-    connection_string: Optional[str] = None,
-) -> Union[type[MySQLModelEndpointsTable], type[SQLiteModelEndpointsTable]]:
-    """Return ModelEndpointsTable based on the provided connection string"""
-    if connection_string and "mysql:" in connection_string:
-        return MySQLModelEndpointsTable
-    return SQLiteModelEndpointsTable
+_MYSQL_SCHEME = "mysql:"
 
 
-def _get_application_result_table(
+def _get_sql_table(
+    *,
+    mysql_table: MySQLTableType,
+    sqlite_table: SQLiteTableType,
     connection_string: Optional[str] = None,
-) -> Union[type[MySQLApplicationResultTable], type[SQLiteApplicationResultTable]]:
-    """Return ModelEndpointsTable based on the provided connection string"""
-    if connection_string and "mysql:" in connection_string:
-        return MySQLApplicationResultTable
-    return SQLiteApplicationResultTable
+) -> Union[MySQLTableType, SQLiteTableType]:
+    """
+    Return a SQLAlchemy table for MySQL or SQLite according to the connection string.
+    Note: this function should not be directly used in other modules.
+    """
+    if connection_string and _MYSQL_SCHEME in connection_string:
+        return mysql_table
+    return sqlite_table
 
 
-def _get_monitoring_schedules_table(
-    connection_string: Optional[str] = None,
-) -> Union[type[MySQLMonitoringSchedulesTable], type[SQLiteMonitoringSchedulesTable]]:
-    """Return ModelEndpointsTable based on the provided connection string"""
-    if connection_string and "mysql:" in connection_string:
-        return MySQLMonitoringSchedulesTable
-    return SQLiteMonitoringSchedulesTable
+_get_model_endpoints_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLModelEndpointsTable,
+    sqlite_table=SQLiteModelEndpointsTable,
+)
+
+
+_get_application_result_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationResultTable,
+    sqlite_table=SQLiteApplicationResultTable,
+)
+
+_get_application_metrics_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLApplicationMetricsTable,
+    sqlite_table=SQLiteApplicationMetricsTable,
+)
+
+_get_monitoring_schedules_table = partial(
+    _get_sql_table,
+    mysql_table=MySQLMonitoringSchedulesTable,
+    sqlite_table=SQLiteMonitoringSchedulesTable,
+)

mlrun/model_monitoring/db/stores/sqldb/models/base.py

@@ -24,6 +24,8 @@ from sqlalchemy import (
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
     FileTargetKind,
+    MetricData,
+    ResultData,
     SchedulingKeys,
     WriterEvent,
 )
@@ -88,11 +90,11 @@ class ModelEndpointsBaseTable(BaseModel):
     metrics = Column(EventFieldType.METRICS, Text)
     first_request = Column(
         EventFieldType.FIRST_REQUEST,
-        TIMESTAMP,
+        TIMESTAMP(timezone=True),
     )
     last_request = Column(
         EventFieldType.LAST_REQUEST,
-        TIMESTAMP,
+        TIMESTAMP(timezone=True),
     )
 
 
@@ -114,7 +116,7 @@ class ApplicationResultBaseTable(BaseModel):
     )
 
     result_name = Column(
-        WriterEvent.RESULT_NAME,
+        ResultData.RESULT_NAME,
         String(40),
     )
 
@@ -127,11 +129,40 @@ class ApplicationResultBaseTable(BaseModel):
         TIMESTAMP(timezone=True),
     )
 
-    result_status = Column(WriterEvent.RESULT_STATUS, String(10))
-    result_kind = Column(WriterEvent.RESULT_KIND, String(40))
-    result_value = Column(WriterEvent.RESULT_VALUE, Float)
-    result_extra_data = Column(WriterEvent.RESULT_EXTRA_DATA, Text)
-    current_stats = Column(WriterEvent.CURRENT_STATS, Text)
+    result_status = Column(ResultData.RESULT_STATUS, String(10))
+    result_kind = Column(ResultData.RESULT_KIND, String(40))
+    result_value = Column(ResultData.RESULT_VALUE, Float)
+    result_extra_data = Column(ResultData.RESULT_EXTRA_DATA, Text)
+    current_stats = Column(ResultData.CURRENT_STATS, Text)
+
+
+class ApplicationMetricsBaseTable(BaseModel):
+    __tablename__ = FileTargetKind.APP_METRICS
+
+    uid = Column(EventFieldType.UID, String(120), primary_key=True)
+    application_name = Column(
+        WriterEvent.APPLICATION_NAME,
+        String(40),
+        nullable=True,
+    )
+    endpoint_id = Column(
+        WriterEvent.ENDPOINT_ID,
+        String(40),
+        nullable=True,
+    )
+    start_infer_time = Column(
+        WriterEvent.START_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+    end_infer_time = Column(
+        WriterEvent.END_INFER_TIME,
+        TIMESTAMP(timezone=True),
+    )
+    metric_name = Column(
+        MetricData.METRIC_NAME,
+        String(40),
+    )
+    metric_value = Column(MetricData.METRIC_VALUE, Float)
 
 
 class MonitoringSchedulesBaseTable(BaseModel):

mlrun/model_monitoring/db/stores/sqldb/models/mysql.py

@@ -22,6 +22,7 @@ from mlrun.common.schemas.model_monitoring import (
 )
 
 from .base import (
+    ApplicationMetricsBaseTable,
     ApplicationResultBaseTable,
     ModelEndpointsBaseTable,
     MonitoringSchedulesBaseTable,
@@ -33,35 +34,54 @@ Base = declarative_base()
 class ModelEndpointsTable(Base, ModelEndpointsBaseTable):
     first_request = Column(
         EventFieldType.FIRST_REQUEST,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
     )
     last_request = Column(
         EventFieldType.LAST_REQUEST,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
     )
 
 
-class ApplicationResultTable(Base, ApplicationResultBaseTable):
+class _ApplicationResultOrMetric:
+    """
+    This class sets common columns of `ApplicationResultTable` and `ApplicationMetricsTable`
+    to the correct values in MySQL.
+    Note: This class must come before the base tables in the inheritance order to override
+    the relevant columns.
+    """
+
     start_infer_time = Column(
         WriterEvent.START_INFER_TIME,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
     )
     end_infer_time = Column(
         WriterEvent.END_INFER_TIME,
-        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
+        sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3, timezone=True),
     )
 
     @declared_attr
-    def endpoint_id(cls):
+    def endpoint_id(self):
         return Column(
             String(40),
             ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),
         )
 
 
+class ApplicationResultTable(
+    Base, _ApplicationResultOrMetric, ApplicationResultBaseTable
+):
+    pass
+
+
+class ApplicationMetricsTable(
+    Base, _ApplicationResultOrMetric, ApplicationMetricsBaseTable
+):
+    pass
+
+
 class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
     @declared_attr
-    def endpoint_id(cls):
+    def endpoint_id(self):
         return Column(
             String(40),
             ForeignKey(f"{EventFieldType.MODEL_ENDPOINTS}.{EventFieldType.UID}"),

mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py

@@ -15,6 +15,7 @@
 from sqlalchemy.ext.declarative import declarative_base
 
 from .base import (
+    ApplicationMetricsBaseTable,
     ApplicationResultBaseTable,
     ModelEndpointsBaseTable,
     MonitoringSchedulesBaseTable,
@@ -31,5 +32,9 @@ class ApplicationResultTable(Base, ApplicationResultBaseTable):
     pass
 
 
+class ApplicationMetricsTable(Base, ApplicationMetricsBaseTable):
+    pass
+
+
 class MonitoringSchedulesTable(Base, MonitoringSchedulesBaseTable):
     pass