mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/model_monitoring/stores/__init__.py

@@ -0,0 +1,106 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
+
+import enum
+import typing
+
+import mlrun
+
+from .model_endpoint_store import ModelEndpointStore
+
+
+class ModelEndpointStoreType(enum.Enum):
+    """Enum class to handle the different store type values for saving a model endpoint record."""
+
+    v3io_nosql = "v3io-nosql"
+    SQL = "sql"
+
+    def to_endpoint_store(
+        self,
+        project: str,
+        access_key: str = None,
+        endpoint_store_connection: str = None,
+    ) -> ModelEndpointStore:
+        """
+        Return a ModelEndpointStore object based on the provided enum value.
+
+        :param project:                    The name of the project.
+        :param access_key:                 Access key with permission to the DB table. Note that if access key is None
+                                           and the endpoint target is from type KV then the access key will be
+                                           retrieved from the environment variable.
+        :param endpoint_store_connection: A valid connection string for model endpoint target. Contains several
+                                          key-value pairs that required for the database connection.
+                                          e.g. A root user with password 1234, tries to connect a schema called
+                                          mlrun within a local MySQL DB instance:
+                                          'mysql+pymysql://root:1234@localhost:3306/mlrun'.
+
+        :return: `ModelEndpointStore` object.
+
+        """
+
+        if self.value == ModelEndpointStoreType.v3io_nosql.value:
+
+            from .kv_model_endpoint_store import KVModelEndpointStore
+
+            # Get V3IO access key from env
+            access_key = access_key or mlrun.mlconf.get_v3io_access_key()
+
+            return KVModelEndpointStore(project=project, access_key=access_key)
+
+        # Assuming SQL store target if store type is not KV.
+        # Update these lines once there are more than two store target types.
+        from mlrun.utils.model_monitoring import get_connection_string
+
+        sql_connection_string = endpoint_store_connection or get_connection_string(
+            project=project
+        )
+        from .sql_model_endpoint_store import SQLModelEndpointStore
+
+        return SQLModelEndpointStore(
+            project=project, sql_connection_string=sql_connection_string
+        )
+
+    @classmethod
+    def _missing_(cls, value: typing.Any):
+        """A lookup function to handle an invalid value.
+        :param value: Provided enum (invalid) value.
+        """
+        valid_values = list(cls.__members__.keys())
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
+        )
+
+
+def get_model_endpoint_store(
+    project: str, access_key: str = None
+) -> ModelEndpointStore:
+    """
+    Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
+
+    :param project:    The name of the project.
+    :param access_key: Access key with permission to the DB table.
+
+    :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
+             model endpoint record such as write, update, get and delete.
+    """
+
+    # Get store type value from ModelEndpointStoreType enum class
+    model_endpoint_store_type = ModelEndpointStoreType(
+        mlrun.mlconf.model_endpoint_monitoring.store_type
+    )
+
+    # Convert into model endpoint store target object
+    return model_endpoint_store_type.to_endpoint_store(project, access_key)

mlrun/model_monitoring/stores/kv_model_endpoint_store.py

@@ -0,0 +1,448 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import typing
+
+import v3io.dataplane
+import v3io_frames
+
+import mlrun
+import mlrun.common.model_monitoring as model_monitoring_constants
+import mlrun.utils.model_monitoring
+import mlrun.utils.v3io_clients
+from mlrun.utils import logger
+
+from .model_endpoint_store import ModelEndpointStore
+
+
+class KVModelEndpointStore(ModelEndpointStore):
+    """
+    Handles the DB operations when the DB target is from type KV. For the KV operations, we use an instance of V3IO
+    client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
+    """
+
+    def __init__(self, project: str, access_key: str):
+        super().__init__(project=project)
+        # Initialize a V3IO client instance
+        self.access_key = access_key or os.environ.get("V3IO_ACCESS_KEY")
+        self.client = mlrun.utils.v3io_clients.get_v3io_client(
+            endpoint=mlrun.mlconf.v3io_api, access_key=self.access_key
+        )
+        # Get the KV table path and container
+        self.path, self.container = self._get_path_and_container()
+
+    def write_model_endpoint(self, endpoint: typing.Dict[str, typing.Any]):
+        """
+        Create a new endpoint record in the KV table.
+
+        :param endpoint: model endpoint dictionary that will be written into the DB.
+        """
+
+        self.client.kv.put(
+            container=self.container,
+            table_path=self.path,
+            key=endpoint[model_monitoring_constants.EventFieldType.UID],
+            attributes=endpoint,
+        )
+
+    def update_model_endpoint(
+        self, endpoint_id: str, attributes: typing.Dict[str, typing.Any]
+    ):
+        """
+        Update a model endpoint record with a given attributes.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        :param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
+                           of the attributes dictionary should exist in the KV table.
+
+        """
+
+        self.client.kv.update(
+            container=self.container,
+            table_path=self.path,
+            key=endpoint_id,
+            attributes=attributes,
+        )
+
+    def delete_model_endpoint(
+        self,
+        endpoint_id: str,
+    ):
+        """
+        Deletes the KV record of a given model endpoint id.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        """
+
+        self.client.kv.delete(
+            container=self.container,
+            table_path=self.path,
+            key=endpoint_id,
+        )
+
+    def get_model_endpoint(
+        self,
+        endpoint_id: str,
+    ) -> typing.Dict[str, typing.Any]:
+        """
+        Get a single model endpoint record.
+
+        :param endpoint_id: The unique id of the model endpoint.
+
+        :return: A model endpoint record as a dictionary.
+
+        :raise MLRunNotFoundError: If the endpoint was not found.
+        """
+
+        # Getting the raw data from the KV table
+        endpoint = self.client.kv.get(
+            container=self.container,
+            table_path=self.path,
+            key=endpoint_id,
+            raise_for_status=v3io.dataplane.RaiseForStatus.never,
+            access_key=self.access_key,
+        )
+        endpoint = endpoint.output.item
+
+        if not endpoint:
+            raise mlrun.errors.MLRunNotFoundError(f"Endpoint {endpoint_id} not found")
+
+        # For backwards compatability: replace null values for `error_count` and `metrics`
+        mlrun.utils.model_monitoring.validate_old_schema_fields(endpoint=endpoint)
+
+        return endpoint
+
+    def _get_path_and_container(self):
+        """Getting path and container based on the model monitoring configurations"""
+        path = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
+            project=self.project,
+            kind=model_monitoring_constants.ModelMonitoringStoreKinds.ENDPOINTS,
+        )
+        (
+            _,
+            container,
+            path,
+        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(path)
+        return path, container
+
+    def list_model_endpoints(
+        self,
+        model: str = None,
+        function: str = None,
+        labels: typing.List[str] = None,
+        top_level: bool = None,
+        uids: typing.List = None,
+    ) -> typing.List[typing.Dict[str, typing.Any]]:
+        """
+        Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
+        By default, when no filters are applied, all available model endpoints for the given project will
+        be listed.
+
+        :param model:           The name of the model to filter by.
+        :param function:        The name of the function to filter by.
+        :param labels:          A list of labels to filter by. Label filters work by either filtering a specific value
+                                of a label (i.e. list("key=value")) or by looking for the existence of a given
+                                key (i.e. "key").
+        :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
+        :param uids:            List of model endpoint unique ids to include in the result.
+
+
+        :return: A list of model endpoint dictionaries.
+        """
+
+        # # Initialize an empty model endpoints list
+        endpoint_list = []
+
+        # Retrieve the raw data from the KV table and get the endpoint ids
+        try:
+            cursor = self.client.kv.new_cursor(
+                container=self.container,
+                table_path=self.path,
+                filter_expression=self._build_kv_cursor_filter_expression(
+                    self.project,
+                    function,
+                    model,
+                    labels,
+                    top_level,
+                ),
+                raise_for_status=v3io.dataplane.RaiseForStatus.never,
+            )
+            items = cursor.all()
+
+        except Exception as exc:
+            logger.warning("Failed retrieving raw data from kv table", exc=exc)
+            return endpoint_list
+
+        # Create a list of model endpoints unique ids
+        if uids is None:
+            uids = []
+            for item in items:
+                if model_monitoring_constants.EventFieldType.UID not in item:
+                    # This is kept for backwards compatibility - in old versions the key column named endpoint_id
+                    uids.append(
+                        item[model_monitoring_constants.EventFieldType.ENDPOINT_ID]
+                    )
+                else:
+                    uids.append(item[model_monitoring_constants.EventFieldType.UID])
+
+        # Add each relevant model endpoint to the model endpoints list
+        for endpoint_id in uids:
+            endpoint = self.get_model_endpoint(
+                endpoint_id=endpoint_id,
+            )
+            endpoint_list.append(endpoint)
+
+        return endpoint_list
+
+    def delete_model_endpoints_resources(
+        self, endpoints: typing.List[typing.Dict[str, typing.Any]]
+    ):
+        """
+        Delete all model endpoints resources in both KV and the time series DB.
+
+        :param endpoints: A list of model endpoints flattened dictionaries.
+        """
+
+        # Delete model endpoint record from KV table
+        for endpoint_dict in endpoints:
+            if model_monitoring_constants.EventFieldType.UID not in endpoint_dict:
+                # This is kept for backwards compatibility - in old versions the key column named endpoint_id
+                endpoint_id = endpoint_dict[
+                    model_monitoring_constants.EventFieldType.ENDPOINT_ID
+                ]
+            else:
+                endpoint_id = endpoint_dict[
+                    model_monitoring_constants.EventFieldType.UID
+                ]
+            self.delete_model_endpoint(
+                endpoint_id,
+            )
+
+        # Delete remain records in the KV
+        all_records = self.client.kv.new_cursor(
+            container=self.container,
+            table_path=self.path,
+            raise_for_status=v3io.dataplane.RaiseForStatus.never,
+        ).all()
+
+        all_records = [r["__name"] for r in all_records]
+
+        # Cleanup KV
+        for record in all_records:
+            self.client.kv.delete(
+                container=self.container,
+                table_path=self.path,
+                key=record,
+                raise_for_status=v3io.dataplane.RaiseForStatus.never,
+            )
+
+        # Cleanup TSDB
+        frames = mlrun.utils.v3io_clients.get_frames_client(
+            token=self.access_key,
+            address=mlrun.mlconf.v3io_framesd,
+            container=self.container,
+        )
+
+        # Generate the required tsdb paths
+        tsdb_path, filtered_path = self._generate_tsdb_paths()
+
+        # Delete time series DB resources
+        try:
+            frames.delete(
+                backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
+                table=filtered_path,
+            )
+        except (v3io_frames.errors.DeleteError, v3io_frames.errors.CreateError) as e:
+            # Frames might raise an exception if schema file does not exist.
+            logger.warning("Failed to delete TSDB schema file:", err=e)
+            pass
+
+        # Final cleanup of tsdb path
+        tsdb_path.replace("://u", ":///u")
+        store, _ = mlrun.store_manager.get_or_create_store(tsdb_path)
+        store.rm(tsdb_path, recursive=True)
+
+    def get_endpoint_real_time_metrics(
+        self,
+        endpoint_id: str,
+        metrics: typing.List[str],
+        start: str = "now-1h",
+        end: str = "now",
+        access_key: str = None,
+    ) -> typing.Dict[str, typing.List[typing.Tuple[str, float]]]:
+        """
+        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
+        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param metrics:          A list of real-time metrics to return for the model endpoint.
+        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
+                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
+                                 earliest time.
+        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
+                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
+                                 earliest time.
+        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
+                                 provided, the access key will be retrieved from the environment variables.
+
+        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
+                 includes timestamps and the values.
+        """
+
+        # Initialize access key
+        access_key = access_key or mlrun.mlconf.get_v3io_access_key()
+
+        if not metrics:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Metric names must be provided"
+            )
+
+        # Initialize metrics mapping dictionary
+        metrics_mapping = {}
+
+        # Getting the path for the time series DB
+        events_path = (
+            mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
+                project=self.project,
+                kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
+            )
+        )
+        (
+            _,
+            container,
+            events_path,
+        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(events_path)
+
+        # Retrieve the raw data from the time series DB based on the provided metrics and time ranges
+        frames_client = mlrun.utils.v3io_clients.get_frames_client(
+            token=access_key,
+            address=mlrun.mlconf.v3io_framesd,
+            container=container,
+        )
+
+        try:
+            data = frames_client.read(
+                backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
+                table=events_path,
+                columns=["endpoint_id", *metrics],
+                filter=f"endpoint_id=='{endpoint_id}'",
+                start=start,
+                end=end,
+            )
+
+            # Fill the metrics mapping dictionary with the metric name and values
+            data_dict = data.to_dict()
+            for metric in metrics:
+                metric_data = data_dict.get(metric)
+                if metric_data is None:
+                    continue
+
+                values = [
+                    (str(timestamp), value) for timestamp, value in metric_data.items()
+                ]
+                metrics_mapping[metric] = values
+
+        except v3io_frames.errors.ReadError:
+            logger.warn("Failed to read tsdb", endpoint=endpoint_id)
+
+        return metrics_mapping
+
+    def _generate_tsdb_paths(self) -> typing.Tuple[str, str]:
+        """Generate a short path to the TSDB resources and a filtered path for the frames object
+        :return: A tuple of:
+             [0] = Short path to the TSDB resources
+             [1] = Filtered path to TSDB events without schema and container
+        """
+        # Full path for the time series DB events
+        full_path = (
+            mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
+                project=self.project,
+                kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
+            )
+        )
+
+        # Generate the main directory with the TSDB resources
+        tsdb_path = mlrun.utils.model_monitoring.parse_model_endpoint_project_prefix(
+            full_path, self.project
+        )
+
+        # Generate filtered path without schema and container as required by the frames object
+        (
+            _,
+            _,
+            filtered_path,
+        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(full_path)
+        return tsdb_path, filtered_path
+
+    @staticmethod
+    def _build_kv_cursor_filter_expression(
+        project: str,
+        function: str = None,
+        model: str = None,
+        labels: typing.List[str] = None,
+        top_level: bool = False,
+    ) -> str:
+        """
+        Convert the provided filters into a valid filter expression. The expected filter expression includes different
+        conditions, divided by ' AND '.
+
+        :param project:    The name of the project.
+        :param model:      The name of the model to filter by.
+        :param function:   The name of the function to filter by.
+        :param labels:     A list of labels to filter by. Label filters work by either filtering a specific value of
+                           a label (i.e. list("key=value")) or by looking for the existence of a given
+                           key (i.e. "key").
+        :param top_level:  If True will return only routers and endpoint that are NOT children of any router.
+
+        :return: A valid filter expression as a string.
+
+        :raise MLRunInvalidArgumentError: If project value is None.
+        """
+
+        if not project:
+            raise mlrun.errors.MLRunInvalidArgumentError("project can't be empty")
+
+        # Add project filter
+        filter_expression = [f"project=='{project}'"]
+
+        # Add function and model filters
+        if function:
+            filter_expression.append(f"function=='{function}'")
+        if model:
+            filter_expression.append(f"model=='{model}'")
+
+        # Add labels filters
+        if labels:
+            for label in labels:
+                if not label.startswith("_"):
+                    label = f"_{label}"
+
+                if "=" in label:
+                    lbl, value = list(map(lambda x: x.strip(), label.split("=")))
+                    filter_expression.append(f"{lbl}=='{value}'")
+                else:
+                    filter_expression.append(f"exists({label})")
+
+        # Apply top_level filter (remove endpoints that considered a child of a router)
+        if top_level:
+            filter_expression.append(
+                f"(endpoint_type=='{str(model_monitoring_constants.EndpointType.NODE_EP.value)}' "
+                f"OR  endpoint_type=='{str(model_monitoring_constants.EndpointType.ROUTER.value)}')"
+            )
+
+        return " AND ".join(filter_expression)

mlrun/model_monitoring/stores/model_endpoint_store.py

@@ -0,0 +1,147 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+from abc import ABC, abstractmethod
+
+
+class ModelEndpointStore(ABC):
+    """
+    An abstract class to handle the model endpoint in the DB target.
+    """
+
+    def __init__(self, project: str):
+        """
+        Initialize a new model endpoint target.
+
+        :param project:             The name of the project.
+        """
+        self.project = project
+
+    @abstractmethod
+    def write_model_endpoint(self, endpoint: typing.Dict[str, typing.Any]):
+        """
+        Create a new endpoint record in the DB table.
+
+        :param endpoint: model endpoint dictionary that will be written into the DB.
+        """
+        pass
+
+    @abstractmethod
+    def update_model_endpoint(
+        self, endpoint_id: str, attributes: typing.Dict[str, typing.Any]
+    ):
+        """
+        Update a model endpoint record with a given attributes.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        :param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
+                           of the attributes dictionary should exist in the DB table.
+
+        """
+        pass
+
+    @abstractmethod
+    def delete_model_endpoint(self, endpoint_id: str):
+        """
+        Deletes the record of a given model endpoint id.
+
+        :param endpoint_id: The unique id of the model endpoint.
+        """
+        pass
+
+    @abstractmethod
+    def delete_model_endpoints_resources(
+        self, endpoints: typing.List[typing.Dict[str, typing.Any]]
+    ):
+        """
+        Delete all model endpoints resources.
+
+        :param endpoints: A list of model endpoints flattened dictionaries.
+
+        """
+        pass
+
+    @abstractmethod
+    def get_model_endpoint(
+        self,
+        endpoint_id: str,
+    ) -> typing.Dict[str, typing.Any]:
+        """
+        Get a single model endpoint record.
+
+        :param endpoint_id: The unique id of the model endpoint.
+
+        :return: A model endpoint record as a dictionary.
+        """
+        pass
+
+    @abstractmethod
+    def list_model_endpoints(
+        self,
+        model: str = None,
+        function: str = None,
+        labels: typing.List[str] = None,
+        top_level: bool = None,
+        uids: typing.List = None,
+    ) -> typing.List[typing.Dict[str, typing.Any]]:
+        """
+        Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
+        By default, when no filters are applied, all available model endpoints for the given project will
+        be listed.
+
+        :param model:           The name of the model to filter by.
+        :param function:        The name of the function to filter by.
+        :param labels:          A list of labels to filter by. Label filters work by either filtering a specific value
+                                of a label (i.e. list("key=value")) or by looking for the existence of a given
+                                key (i.e. "key").
+        :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
+        :param uids:             List of model endpoint unique ids to include in the result.
+
+        :return: A list of model endpoint dictionaries.
+        """
+        pass
+
+    @abstractmethod
+    def get_endpoint_real_time_metrics(
+        self,
+        endpoint_id: str,
+        metrics: typing.List[str],
+        start: str = "now-1h",
+        end: str = "now",
+        access_key: str = None,
+    ) -> typing.Dict[str, typing.List[typing.Tuple[str, float]]]:
+        """
+        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
+        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
+
+        :param endpoint_id:      The unique id of the model endpoint.
+        :param metrics:          A list of real-time metrics to return for the model endpoint.
+        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
+                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
+                                 earliest time.
+        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
+                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
+                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
+                                 earliest time.
+        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
+                                 provided, the access key will be retrieved from the environment variables.
+
+        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
+                 includes timestamps and the values.
+        """
+
+        pass