mlrun 1.7.2__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -14,7 +14,6 @@
 
 import datetime
 import typing
-import uuid
 
 import pandas as pd
 import sqlalchemy
@@ -75,9 +74,6 @@ class SQLStoreBase(StoreBase):
 
     def _init_tables(self):
         self._init_model_endpoints_table()
-        self._init_application_results_table()
-        self._init_application_metrics_table()
-        self._init_monitoring_schedules_table()
 
     def _init_model_endpoints_table(self):
         self.model_endpoints_table = (
@@ -89,32 +85,6 @@ class SQLStoreBase(StoreBase):
             self.model_endpoints_table
         )
 
-    def _init_application_results_table(self):
-        self.application_results_table = (
-            mlrun.model_monitoring.db.stores.sqldb.models._get_application_result_table(
-                connection_string=self._sql_connection_string
-            )
-        )
-        self._tables[mm_schemas.FileTargetKind.APP_RESULTS] = (
-            self.application_results_table
-        )
-
-    def _init_application_metrics_table(self) -> None:
-        self.application_metrics_table = mlrun.model_monitoring.db.stores.sqldb.models._get_application_metrics_table(
-            connection_string=self._sql_connection_string
-        )
-        self._tables[mm_schemas.FileTargetKind.APP_METRICS] = (
-            self.application_metrics_table
-        )
-
-    def _init_monitoring_schedules_table(self):
-        self.MonitoringSchedulesTable = mlrun.model_monitoring.db.stores.sqldb.models._get_monitoring_schedules_table(
-            connection_string=self._sql_connection_string
-        )
-        self._tables[mm_schemas.FileTargetKind.MONITORING_SCHEDULES] = (
-            self.MonitoringSchedulesTable
-        )
-
     def _write(self, table_name: str, event: dict[str, typing.Any]) -> None:
         """
         Create a new record in the SQL table.
@@ -272,12 +242,12 @@ class SQLStoreBase(StoreBase):
 
     def list_model_endpoints(
         self,
-        model: str = None,
-        function: str = None,
-        labels: list[str] = None,
-        top_level: bool = None,
-        uids: list = None,
-        include_stats: bool = None,
+        model: typing.Optional[str] = None,
+        function: typing.Optional[str] = None,
+        labels: typing.Optional[list[str]] = None,
+        top_level: typing.Optional[bool] = None,
+        uids: typing.Optional[list] = None,
+        include_stats: typing.Optional[bool] = None,
     ) -> list[dict[str, typing.Any]]:
         # Generate an empty model endpoints that will be filled afterwards with model endpoint dictionaries
         endpoint_list = []
@@ -348,182 +318,12 @@ class SQLStoreBase(StoreBase):
 
         return endpoint_list
 
-    def write_application_event(
-        self,
-        event: dict[str, typing.Any],
-        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
-    ) -> None:
-        """
-        Write a new application event in the target table.
-
-        :param event: An event dictionary that represents the application result or metric,
-                      should be corresponded to the schema defined in the
-                      :py:class:`~mm_constants.constants.WriterEvent` object.
-        :param kind: The type of the event, can be either "result" or "metric".
-        """
-
-        if kind == mm_schemas.WriterEventKind.METRIC:
-            table = self.application_metrics_table
-            table_name = mm_schemas.FileTargetKind.APP_METRICS
-        elif kind == mm_schemas.WriterEventKind.RESULT:
-            table = self.application_results_table
-            table_name = mm_schemas.FileTargetKind.APP_RESULTS
-        else:
-            raise ValueError(f"Invalid {kind = }")
-
-        application_result_uid = self._generate_application_result_uid(event, kind=kind)
-        criteria = [table.uid == application_result_uid]
-
-        application_record = self._get(table=table, criteria=criteria)
-        if application_record:
-            self._convert_to_datetime(
-                event=event, key=mm_schemas.WriterEvent.START_INFER_TIME
-            )
-            self._convert_to_datetime(
-                event=event, key=mm_schemas.WriterEvent.END_INFER_TIME
-            )
-            # Update an existing application result
-            self._update(attributes=event, table=table, criteria=criteria)
-        else:
-            # Write a new application result
-            event[mm_schemas.EventFieldType.UID] = application_result_uid
-            self._write(table_name=table_name, event=event)
-
     @staticmethod
     def _convert_to_datetime(event: dict[str, typing.Any], key: str) -> None:
         if isinstance(event[key], str):
             event[key] = datetime.datetime.fromisoformat(event[key])
         event[key] = event[key].astimezone(tz=datetime.timezone.utc)
 
-    @staticmethod
-    def _generate_application_result_uid(
-        event: dict[str, typing.Any],
-        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
-    ) -> str:
-        if kind == mm_schemas.WriterEventKind.RESULT:
-            name = event[mm_schemas.ResultData.RESULT_NAME]
-        else:
-            name = event[mm_schemas.MetricData.METRIC_NAME]
-        return "_".join(
-            [
-                event[mm_schemas.WriterEvent.ENDPOINT_ID],
-                event[mm_schemas.WriterEvent.APPLICATION_NAME],
-                name,
-            ]
-        )
-
-    @staticmethod
-    def _get_filter_criteria(
-        *,
-        table: sqlalchemy.orm.decl_api.DeclarativeMeta,
-        endpoint_id: str,
-        application_name: typing.Optional[str] = None,
-    ) -> list[BinaryExpression]:
-        """
-        Return the filter criteria for the given endpoint_id and application_name.
-        Note: the table object must include the relevant columns:
-        `endpoint_id` and `application_name`.
-        """
-        criteria = [table.endpoint_id == endpoint_id]
-        if application_name is not None:
-            criteria.append(table.application_name == application_name)
-        return criteria
-
-    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
-        """
-        Get the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-
-        :return: Timestamp as a Unix time.
-        :raise:  MLRunNotFoundError if last analyzed value is not found.
-        """
-        monitoring_schedule_record = self._get(
-            table=self.MonitoringSchedulesTable,
-            criteria=self._get_filter_criteria(
-                table=self.MonitoringSchedulesTable,
-                endpoint_id=endpoint_id,
-                application_name=application_name,
-            ),
-        )
-        if not monitoring_schedule_record:
-            raise mlrun.errors.MLRunNotFoundError(
-                f"No last analyzed value has been found for {application_name} "
-                f"that processes model endpoint {endpoint_id}"
-            )
-        return monitoring_schedule_record.last_analyzed
-
-    def update_last_analyzed(
-        self, endpoint_id: str, application_name: str, last_analyzed: int
-    ):
-        """
-        Update the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
-                                 application and model endpoint.
-        """
-        criteria = self._get_filter_criteria(
-            table=self.MonitoringSchedulesTable,
-            endpoint_id=endpoint_id,
-            application_name=application_name,
-        )
-        monitoring_schedule_record = self._get(
-            table=self.MonitoringSchedulesTable, criteria=criteria
-        )
-        if not monitoring_schedule_record:
-            # Add a new record with last analyzed value
-            self._write(
-                table_name=mm_schemas.FileTargetKind.MONITORING_SCHEDULES,
-                event={
-                    mm_schemas.SchedulingKeys.UID: uuid.uuid4().hex,
-                    mm_schemas.SchedulingKeys.APPLICATION_NAME: application_name,
-                    mm_schemas.SchedulingKeys.ENDPOINT_ID: endpoint_id,
-                    mm_schemas.SchedulingKeys.LAST_ANALYZED: last_analyzed,
-                },
-            )
-
-        self._update(
-            attributes={mm_schemas.SchedulingKeys.LAST_ANALYZED: last_analyzed},
-            table=self.MonitoringSchedulesTable,
-            criteria=criteria,
-        )
-
-    def _delete_last_analyzed(
-        self, endpoint_id: str, application_name: typing.Optional[str] = None
-    ) -> None:
-        criteria = self._get_filter_criteria(
-            table=self.MonitoringSchedulesTable,
-            endpoint_id=endpoint_id,
-            application_name=application_name,
-        )
-        # Delete the model endpoint record using sqlalchemy ORM
-        self._delete(table=self.MonitoringSchedulesTable, criteria=criteria)
-
-    def _delete_application_result(
-        self, endpoint_id: str, application_name: typing.Optional[str] = None
-    ) -> None:
-        criteria = self._get_filter_criteria(
-            table=self.application_results_table,
-            endpoint_id=endpoint_id,
-            application_name=application_name,
-        )
-        # Delete the relevant records from the results table
-        self._delete(table=self.application_results_table, criteria=criteria)
-
-    def _delete_application_metrics(
-        self, endpoint_id: str, application_name: typing.Optional[str] = None
-    ) -> None:
-        criteria = self._get_filter_criteria(
-            table=self.application_metrics_table,
-            endpoint_id=endpoint_id,
-            application_name=application_name,
-        )
-        # Delete the relevant records from the metrics table
-        self._delete(table=self.application_metrics_table, criteria=criteria)
-
     def _create_tables_if_not_exist(self):
         self._init_tables()
 
@@ -593,12 +393,6 @@ class SQLStoreBase(StoreBase):
                 endpoint_id=endpoint_id,
                 project=self.project,
             )
-            # Delete last analyzed records
-            self._delete_last_analyzed(endpoint_id=endpoint_id)
-
-            # Delete application results and metrics records
-            self._delete_application_result(endpoint_id=endpoint_id)
-            self._delete_application_metrics(endpoint_id=endpoint_id)
 
             # Delete model endpoint record
             self.delete_model_endpoint(endpoint_id=endpoint_id)
@@ -612,48 +406,3 @@ class SQLStoreBase(StoreBase):
             "Successfully deleted model monitoring endpoints resources from the SQL tables",
             project=self.project,
         )
-
-    def get_model_endpoint_metrics(
-        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
-    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
-        """
-        Fetch the model endpoint metrics or results (according to `type`) for the
-        requested endpoint.
-        """
-        logger.debug(
-            "Fetching metrics for model endpoint",
-            project=self.project,
-            endpoint_id=endpoint_id,
-            type=type,
-        )
-        if type == mm_schemas.ModelEndpointMonitoringMetricType.METRIC:
-            table = self.application_metrics_table
-            name_col = mm_schemas.MetricData.METRIC_NAME
-        else:
-            table = self.application_results_table
-            name_col = mm_schemas.ResultData.RESULT_NAME
-
-        # Note: the block below does not use self._get, as we need here all the
-        # results, not only `one_or_none`.
-        with sqlalchemy.orm.Session(self.engine) as session:
-            metric_rows = (
-                session.query(table)  # pyright: ignore[reportOptionalCall]
-                .filter(table.endpoint_id == endpoint_id)
-                .all()
-            )
-
-        return [
-            mm_schemas.ModelEndpointMonitoringMetric(
-                project=self.project,
-                app=metric_row.application_name,
-                type=type,
-                name=getattr(metric_row, name_col),
-                full_name=mlrun.model_monitoring.helpers._compose_full_name(
-                    project=self.project,
-                    app=metric_row.application_name,
-                    type=type,
-                    name=getattr(metric_row, name_col),
-                ),
-            )
-            for metric_row in metric_rows
-        ]

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -11,11 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import http
+
 import json
 import typing
 from dataclasses import dataclass
-from http import HTTPStatus
 
 import v3io.dataplane
 import v3io.dataplane.output
@@ -34,14 +33,6 @@ fields_to_encode_decode = [
     mm_schemas.EventFieldType.CURRENT_STATS,
 ]
 
-_METRIC_FIELDS: list[str] = [
-    mm_schemas.WriterEvent.APPLICATION_NAME.value,
-    mm_schemas.MetricData.METRIC_NAME.value,
-    mm_schemas.MetricData.METRIC_VALUE.value,
-    mm_schemas.WriterEvent.START_INFER_TIME.value,
-    mm_schemas.WriterEvent.END_INFER_TIME.value,
-]
-
 
 class SchemaField(typing.TypedDict):
     name: str
@@ -55,37 +46,6 @@ class SchemaParams:
     fields: list[SchemaField]
 
 
-_RESULT_SCHEMA: list[SchemaField] = [
-    SchemaField(
-        name=mm_schemas.ResultData.RESULT_NAME,
-        type=mm_schemas.GrafanaColumnType.STRING,
-        nullable=False,
-    )
-]
-
-_METRIC_SCHEMA: list[SchemaField] = [
-    SchemaField(
-        name=mm_schemas.WriterEvent.APPLICATION_NAME,
-        type=mm_schemas.GrafanaColumnType.STRING,
-        nullable=False,
-    ),
-    SchemaField(
-        name=mm_schemas.MetricData.METRIC_NAME,
-        type=mm_schemas.GrafanaColumnType.STRING,
-        nullable=False,
-    ),
-]
-
-
-_KIND_TO_SCHEMA_PARAMS: dict[mm_schemas.WriterEventKind, SchemaParams] = {
-    mm_schemas.WriterEventKind.RESULT: SchemaParams(
-        key=mm_schemas.WriterEvent.APPLICATION_NAME, fields=_RESULT_SCHEMA
-    ),
-    mm_schemas.WriterEventKind.METRIC: SchemaParams(
-        key="metric_id", fields=_METRIC_SCHEMA
-    ),
-}
-
 _EXCLUDE_SCHEMA_FILTER_EXPRESSION = '__name!=".#schema"'
 
 
@@ -227,12 +187,12 @@ class KVStoreBase(StoreBase):
 
     def list_model_endpoints(
         self,
-        model: str = None,
-        function: str = None,
-        labels: list[str] = None,
-        top_level: bool = None,
-        uids: list = None,
-        include_stats: bool = None,
+        model: typing.Optional[str] = None,
+        function: typing.Optional[str] = None,
+        labels: typing.Optional[list[str]] = None,
+        top_level: typing.Optional[bool] = None,
+        uids: typing.Optional[list] = None,
+        include_stats: typing.Optional[bool] = None,
     ) -> list[dict[str, typing.Any]]:
         # # Initialize an empty model endpoints list
         endpoint_list = []
@@ -339,135 +299,6 @@ class KVStoreBase(StoreBase):
                 raise_for_status=v3io.dataplane.RaiseForStatus.never,
             )
 
-    @staticmethod
-    def _get_results_table_path(endpoint_id: str) -> str:
-        return endpoint_id
-
-    @staticmethod
-    def _get_metrics_table_path(endpoint_id: str) -> str:
-        return f"{endpoint_id}_metrics"
-
-    def write_application_event(
-        self,
-        event: dict[str, typing.Any],
-        kind: mm_schemas.WriterEventKind = mm_schemas.WriterEventKind.RESULT,
-    ) -> None:
-        """
-        Write a new application event in the target table.
-
-        :param event: An event dictionary that represents the application result, should be corresponded to the
-                      schema defined in the :py:class:`~mlrun.common.schemas.model_monitoring.constants.WriterEvent`
-                      object.
-        :param kind: The type of the event, can be either "result" or "metric".
-        """
-
-        container = self.get_v3io_monitoring_apps_container(project_name=self.project)
-        endpoint_id = event.pop(mm_schemas.WriterEvent.ENDPOINT_ID)
-
-        if kind == mm_schemas.WriterEventKind.METRIC:
-            table_path = self._get_metrics_table_path(endpoint_id)
-            key = f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}.{event[mm_schemas.MetricData.METRIC_NAME]}"
-            attributes = {event_key: event[event_key] for event_key in _METRIC_FIELDS}
-        elif kind == mm_schemas.WriterEventKind.RESULT:
-            table_path = self._get_results_table_path(endpoint_id)
-            key = event.pop(mm_schemas.WriterEvent.APPLICATION_NAME)
-            metric_name = event.pop(mm_schemas.ResultData.RESULT_NAME)
-            attributes = {metric_name: self._encode_field(json.dumps(event))}
-        else:
-            raise ValueError(f"Invalid {kind = }")
-
-        self.client.kv.update(
-            container=container,
-            table_path=table_path,
-            key=key,
-            attributes=attributes,
-        )
-
-        schema_file = self.client.kv.new_cursor(
-            container=container,
-            table_path=table_path,
-            filter_expression='__name==".#schema"',
-        )
-
-        if not schema_file.all():
-            logger.info(
-                "Generating a new V3IO KV schema file",
-                container=container,
-                table_path=table_path,
-            )
-            self._generate_kv_schema(
-                container=container, table_path=table_path, kind=kind
-            )
-        logger.info("Updated V3IO KV successfully", key=key)
-
-    def _generate_kv_schema(
-        self, *, container: str, table_path: str, kind: mm_schemas.WriterEventKind
-    ) -> None:
-        """Generate V3IO KV schema file which will be used by the model monitoring applications dashboard in Grafana."""
-        schema_params = _KIND_TO_SCHEMA_PARAMS[kind]
-        res = self.client.kv.create_schema(
-            container=container,
-            table_path=table_path,
-            key=schema_params.key,
-            fields=schema_params.fields,
-        )
-        if res.status_code != HTTPStatus.OK:
-            raise mlrun.errors.MLRunBadRequestError(
-                f"Couldn't infer schema for endpoint {table_path} which is required for Grafana dashboards"
-            )
-        else:
-            logger.info("Generated V3IO KV schema successfully", table_path=table_path)
-
-    def get_last_analyzed(self, endpoint_id: str, application_name: str) -> int:
-        """
-        Get the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-
-        :return: Timestamp as a Unix time.
-        :raise:  MLRunNotFoundError if last analyzed value is not found.
-
-        """
-        try:
-            response = self.client.kv.get(
-                container=self._get_monitoring_schedules_container(
-                    project_name=self.project
-                ),
-                table_path=endpoint_id,
-                key=application_name,
-            )
-            return response.output.item[mm_schemas.SchedulingKeys.LAST_ANALYZED]
-        except v3io.dataplane.response.HttpResponseError as err:
-            if err.status_code == http.HTTPStatus.NOT_FOUND:
-                logger.debug("Last analyzed time not found", err=err)
-                raise mlrun.errors.MLRunNotFoundError(
-                    f"No last analyzed value has been found for {application_name} "
-                    f"that processes model endpoint {endpoint_id}",
-                )
-            logger.error("Error while getting last analyzed time", err=err)
-            raise err
-
-    def update_last_analyzed(
-        self, endpoint_id: str, application_name: str, last_analyzed: int
-    ):
-        """
-        Update the last analyzed time for the provided model endpoint and application.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param application_name: Registered application name.
-        :param last_analyzed:    Timestamp as a Unix time that represents the last analyzed time of a certain
-                                 application and model endpoint.
-        """
-        self.client.kv.put(
-            container=self._get_monitoring_schedules_container(
-                project_name=self.project
-            ),
-            table_path=endpoint_id,
-            key=application_name,
-            attributes={mm_schemas.SchedulingKeys.LAST_ANALYZED: last_analyzed},
-        )
-
     def _generate_tsdb_paths(self) -> tuple[str, str]:
         """Generate a short path to the TSDB resources and a filtered path for the frames object
         :return: A tuple of:
@@ -524,8 +355,8 @@ class KVStoreBase(StoreBase):
     @staticmethod
     def _build_kv_cursor_filter_expression(
         project: str,
-        function: str = None,
-        model: str = None,
+        function: typing.Optional[str] = None,
+        model: typing.Optional[str] = None,
         top_level: bool = False,
     ) -> str:
         """
@@ -631,96 +462,3 @@ class KVStoreBase(StoreBase):
     @staticmethod
     def _get_monitoring_schedules_container(project_name: str) -> str:
         return f"users/pipelines/{project_name}/monitoring-schedules/functions"
-
-    def _extract_results_from_items(
-        self, app_items: list[dict[str, str]]
-    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
-        """Assuming .#schema items are filtered out"""
-        metrics: list[mm_schemas.ModelEndpointMonitoringMetric] = []
-        for app_item in app_items:
-            app_name = app_item.pop("__name")
-            for result_name in app_item:
-                metrics.append(
-                    mm_schemas.ModelEndpointMonitoringMetric(
-                        project=self.project,
-                        app=app_name,
-                        type=mm_schemas.ModelEndpointMonitoringMetricType.RESULT,
-                        name=result_name,
-                        full_name=mm_schemas.model_endpoints._compose_full_name(
-                            project=self.project, app=app_name, name=result_name
-                        ),
-                    )
-                )
-        return metrics
-
-    def _extract_metrics_from_items(
-        self, result_items: list[dict[str, str]]
-    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
-        metrics: list[mm_schemas.ModelEndpointMonitoringMetric] = []
-        logger.debug("Result items", result_items=result_items)
-        for result_item in result_items:
-            app = result_item[mm_schemas.WriterEvent.APPLICATION_NAME]
-            name = result_item[mm_schemas.MetricData.METRIC_NAME]
-            metrics.append(
-                mm_schemas.ModelEndpointMonitoringMetric(
-                    project=self.project,
-                    app=app,
-                    type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
-                    name=name,
-                    full_name=mm_schemas.model_endpoints._compose_full_name(
-                        project=self.project,
-                        app=app,
-                        name=name,
-                        type=mm_schemas.ModelEndpointMonitoringMetricType.METRIC,
-                    ),
-                )
-            )
-        return metrics
-
-    def get_model_endpoint_metrics(
-        self, endpoint_id: str, type: mm_schemas.ModelEndpointMonitoringMetricType
-    ) -> list[mm_schemas.ModelEndpointMonitoringMetric]:
-        """Get model monitoring results and metrics on the endpoint"""
-        metrics: list[mm_schemas.ModelEndpointMonitoringMetric] = []
-        container = self.get_v3io_monitoring_apps_container(self.project)
-        if type == mm_schemas.ModelEndpointMonitoringMetricType.METRIC:
-            table_path = self._get_metrics_table_path(endpoint_id)
-            items_extractor = self._extract_metrics_from_items
-        elif type == mm_schemas.ModelEndpointMonitoringMetricType.RESULT:
-            table_path = self._get_results_table_path(endpoint_id)
-            items_extractor = self._extract_results_from_items
-        else:
-            raise ValueError(f"Invalid metric {type = }")
-
-        def scan(
-            marker: typing.Optional[str] = None,
-        ) -> v3io.dataplane.response.Response:
-            # TODO: Use AIO client: `v3io.aio.dataplane.client.Client`
-            return self.client.kv.scan(
-                container=container,
-                table_path=table_path,
-                marker=marker,
-                filter_expression=_EXCLUDE_SCHEMA_FILTER_EXPRESSION,
-            )
-
-        try:
-            response = scan()
-        except v3io.dataplane.response.HttpResponseError as err:
-            if err.status_code == HTTPStatus.NOT_FOUND:
-                logger.warning(
-                    f"Attempt getting {type}s - no data. Check the "
-                    "project name, endpoint, or wait for the applications to start.",
-                    container=container,
-                    table_path=table_path,
-                )
-                return []
-            raise
-
-        while True:
-            output = typing.cast(v3io.dataplane.output.GetItemsOutput, response.output)
-            metrics.extend(items_extractor(output.items))
-            if output.last:
-                break
-            response = scan(marker=output.next_marker)
-
-        return metrics