mlrun 1.3.2rc1__py3-none-any.whl → 1.3.2rc2__py3-none-any.whl

mlrun/model_monitoring/model_endpoint.py

@@ -1,141 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from typing import Any, Dict, List, Optional
-
-import mlrun.model
-
-from .common import EndpointType, ModelMonitoringMode
-from .constants import EventKeyMetrics, EventLiveStats
-
-
-class ModelEndpointSpec(mlrun.model.ModelObj):
-    def __init__(
-        self,
-        function_uri: Optional[str] = "",
-        model: Optional[str] = "",
-        model_class: Optional[str] = "",
-        model_uri: Optional[str] = "",
-        feature_names: Optional[List[str]] = None,
-        label_names: Optional[List[str]] = None,
-        stream_path: Optional[str] = "",
-        algorithm: Optional[str] = "",
-        monitor_configuration: Optional[dict] = None,
-        active: Optional[bool] = True,
-        monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled,
-    ):
-        self.function_uri = function_uri  # <project_name>/<function_name>:<tag>
-        self.model = model  # <model_name>:<version>
-        self.model_class = model_class
-        self.model_uri = model_uri
-        self.feature_names = feature_names or []
-        self.label_names = label_names or []
-        self.stream_path = stream_path
-        self.algorithm = algorithm
-        self.monitor_configuration = monitor_configuration or {}
-        self.active = active
-        self.monitoring_mode = monitoring_mode
-
-
-class ModelEndpointStatus(mlrun.model.ModelObj):
-    def __init__(
-        self,
-        feature_stats: Optional[dict] = None,
-        current_stats: Optional[dict] = None,
-        first_request: Optional[str] = "",
-        last_request: Optional[str] = "",
-        error_count: Optional[int] = 0,
-        drift_status: Optional[str] = "",
-        drift_measures: Optional[dict] = None,
-        metrics: Optional[Dict[str, Dict[str, Any]]] = None,
-        features: Optional[List[Dict[str, Any]]] = None,
-        children: Optional[List[str]] = None,
-        children_uids: Optional[List[str]] = None,
-        endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP.value,
-        monitoring_feature_set_uri: Optional[str] = "",
-        state: Optional[str] = "",
-    ):
-        self.feature_stats = feature_stats or {}
-        self.current_stats = current_stats or {}
-        self.first_request = first_request
-        self.last_request = last_request
-        self.error_count = error_count
-        self.drift_status = drift_status
-        self.drift_measures = drift_measures or {}
-        self.features = features or []
-        self.children = children or []
-        self.children_uids = children_uids or []
-        self.endpoint_type = endpoint_type
-        self.monitoring_feature_set_uri = monitoring_feature_set_uri
-        if metrics is None:
-            self.metrics = {
-                EventKeyMetrics.GENERIC: {
-                    EventLiveStats.LATENCY_AVG_1H: 0,
-                    EventLiveStats.PREDICTIONS_PER_SECOND: 0,
-                }
-            }
-        self.state = state
-
-
-class ModelEndpoint(mlrun.model.ModelObj):
-    kind = "model-endpoint"
-    _dict_fields = ["kind", "metadata", "spec", "status"]
-
-    def __init__(self):
-        self._status: ModelEndpointStatus = ModelEndpointStatus()
-        self._spec: ModelEndpointSpec = ModelEndpointSpec()
-        self._metadata: mlrun.model.VersionedObjMetadata = (
-            mlrun.model.VersionedObjMetadata()
-        )
-
-    @property
-    def status(self) -> ModelEndpointStatus:
-        return self._status
-
-    @status.setter
-    def status(self, status):
-        self._status = self._verify_dict(status, "status", ModelEndpointStatus)
-
-    @property
-    def spec(self) -> ModelEndpointSpec:
-        return self._spec
-
-    @spec.setter
-    def spec(self, spec):
-        self._spec = self._verify_dict(spec, "spec", ModelEndpointSpec)
-
-    @property
-    def metadata(self) -> mlrun.model.VersionedObjMetadata:
-        return self._metadata
-
-    @metadata.setter
-    def metadata(self, metadata):
-        self._metadata = self._verify_dict(
-            metadata, "metadata", mlrun.model.VersionedObjMetadata
-        )
-
-    @classmethod
-    def from_flat_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
-        new_obj = cls()
-        new_obj._metadata = mlrun.model.VersionedObjMetadata().from_dict(
-            struct=struct, fields=fields, deprecated_fields=deprecated_fields
-        )
-        new_obj._status = ModelEndpointStatus().from_dict(
-            struct=struct, fields=fields, deprecated_fields=deprecated_fields
-        )
-        new_obj._spec = ModelEndpointSpec().from_dict(
-            struct=struct, fields=fields, deprecated_fields=deprecated_fields
-        )
-        return new_obj

mlrun/model_monitoring/stores/__init__.py

@@ -1,106 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
-import enum
-import typing
-
-import mlrun
-
-from .model_endpoint_store import ModelEndpointStore
-
-
-class ModelEndpointStoreType(enum.Enum):
-    """Enum class to handle the different store type values for saving a model endpoint record."""
-
-    v3io_nosql = "v3io-nosql"
-    SQL = "sql"
-
-    def to_endpoint_store(
-        self,
-        project: str,
-        access_key: str = None,
-        endpoint_store_connection: str = None,
-    ) -> ModelEndpointStore:
-        """
-        Return a ModelEndpointStore object based on the provided enum value.
-
-        :param project:                    The name of the project.
-        :param access_key:                 Access key with permission to the DB table. Note that if access key is None
-                                           and the endpoint target is from type KV then the access key will be
-                                           retrieved from the environment variable.
-        :param endpoint_store_connection: A valid connection string for model endpoint target. Contains several
-                                          key-value pairs that required for the database connection.
-                                          e.g. A root user with password 1234, tries to connect a schema called
-                                          mlrun within a local MySQL DB instance:
-                                          'mysql+pymysql://root:1234@localhost:3306/mlrun'.
-
-        :return: `ModelEndpointStore` object.
-
-        """
-
-        if self.value == ModelEndpointStoreType.v3io_nosql.value:
-
-            from .kv_model_endpoint_store import KVModelEndpointStore
-
-            # Get V3IO access key from env
-            access_key = access_key or mlrun.mlconf.get_v3io_access_key()
-
-            return KVModelEndpointStore(project=project, access_key=access_key)
-
-        # Assuming SQL store target if store type is not KV.
-        # Update these lines once there are more than two store target types.
-        from mlrun.utils.model_monitoring import get_connection_string
-
-        sql_connection_string = endpoint_store_connection or get_connection_string(
-            project=project
-        )
-        from .sql_model_endpoint_store import SQLModelEndpointStore
-
-        return SQLModelEndpointStore(
-            project=project, sql_connection_string=sql_connection_string
-        )
-
-    @classmethod
-    def _missing_(cls, value: typing.Any):
-        """A lookup function to handle an invalid value.
-        :param value: Provided enum (invalid) value.
-        """
-        valid_values = list(cls.__members__.keys())
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
-        )
-
-
-def get_model_endpoint_store(
-    project: str, access_key: str = None
-) -> ModelEndpointStore:
-    """
-    Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
-
-    :param project:    The name of the project.
-    :param access_key: Access key with permission to the DB table.
-
-    :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
-             model endpoint record such as write, update, get and delete.
-    """
-
-    # Get store type value from ModelEndpointStoreType enum class
-    model_endpoint_store_type = ModelEndpointStoreType(
-        mlrun.mlconf.model_endpoint_monitoring.store_type
-    )
-
-    # Convert into model endpoint store target object
-    return model_endpoint_store_type.to_endpoint_store(project, access_key)

mlrun/model_monitoring/stores/kv_model_endpoint_store.py

@@ -1,448 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import typing
-
-import v3io.dataplane
-import v3io_frames
-
-import mlrun
-import mlrun.model_monitoring.constants as model_monitoring_constants
-import mlrun.utils.model_monitoring
-import mlrun.utils.v3io_clients
-from mlrun.utils import logger
-
-from .model_endpoint_store import ModelEndpointStore
-
-
-class KVModelEndpointStore(ModelEndpointStore):
-    """
-    Handles the DB operations when the DB target is from type KV. For the KV operations, we use an instance of V3IO
-    client and usually the KV table can be found under v3io:///users/pipelines/project-name/model-endpoints/endpoints/.
-    """
-
-    def __init__(self, project: str, access_key: str):
-        super().__init__(project=project)
-        # Initialize a V3IO client instance
-        self.access_key = access_key or os.environ.get("V3IO_ACCESS_KEY")
-        self.client = mlrun.utils.v3io_clients.get_v3io_client(
-            endpoint=mlrun.mlconf.v3io_api, access_key=self.access_key
-        )
-        # Get the KV table path and container
-        self.path, self.container = self._get_path_and_container()
-
-    def write_model_endpoint(self, endpoint: typing.Dict[str, typing.Any]):
-        """
-        Create a new endpoint record in the KV table.
-
-        :param endpoint: model endpoint dictionary that will be written into the DB.
-        """
-
-        self.client.kv.put(
-            container=self.container,
-            table_path=self.path,
-            key=endpoint[model_monitoring_constants.EventFieldType.UID],
-            attributes=endpoint,
-        )
-
-    def update_model_endpoint(
-        self, endpoint_id: str, attributes: typing.Dict[str, typing.Any]
-    ):
-        """
-        Update a model endpoint record with a given attributes.
-
-        :param endpoint_id: The unique id of the model endpoint.
-        :param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
-                           of the attributes dictionary should exist in the KV table.
-
-        """
-
-        self.client.kv.update(
-            container=self.container,
-            table_path=self.path,
-            key=endpoint_id,
-            attributes=attributes,
-        )
-
-    def delete_model_endpoint(
-        self,
-        endpoint_id: str,
-    ):
-        """
-        Deletes the KV record of a given model endpoint id.
-
-        :param endpoint_id: The unique id of the model endpoint.
-        """
-
-        self.client.kv.delete(
-            container=self.container,
-            table_path=self.path,
-            key=endpoint_id,
-        )
-
-    def get_model_endpoint(
-        self,
-        endpoint_id: str,
-    ) -> typing.Dict[str, typing.Any]:
-        """
-        Get a single model endpoint record.
-
-        :param endpoint_id: The unique id of the model endpoint.
-
-        :return: A model endpoint record as a dictionary.
-
-        :raise MLRunNotFoundError: If the endpoint was not found.
-        """
-
-        # Getting the raw data from the KV table
-        endpoint = self.client.kv.get(
-            container=self.container,
-            table_path=self.path,
-            key=endpoint_id,
-            raise_for_status=v3io.dataplane.RaiseForStatus.never,
-            access_key=self.access_key,
-        )
-        endpoint = endpoint.output.item
-
-        if not endpoint:
-            raise mlrun.errors.MLRunNotFoundError(f"Endpoint {endpoint_id} not found")
-
-        # For backwards compatability: replace null values for `error_count` and `metrics`
-        mlrun.utils.model_monitoring.validate_errors_and_metrics(endpoint=endpoint)
-
-        return endpoint
-
-    def _get_path_and_container(self):
-        """Getting path and container based on the model monitoring configurations"""
-        path = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
-            project=self.project,
-            kind=model_monitoring_constants.ModelMonitoringStoreKinds.ENDPOINTS,
-        )
-        (
-            _,
-            container,
-            path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(path)
-        return path, container
-
-    def list_model_endpoints(
-        self,
-        model: str = None,
-        function: str = None,
-        labels: typing.List[str] = None,
-        top_level: bool = None,
-        uids: typing.List = None,
-    ) -> typing.List[typing.Dict[str, typing.Any]]:
-        """
-        Returns a list of model endpoint dictionaries, supports filtering by model, function, labels or top level.
-        By default, when no filters are applied, all available model endpoints for the given project will
-        be listed.
-
-        :param model:           The name of the model to filter by.
-        :param function:        The name of the function to filter by.
-        :param labels:          A list of labels to filter by. Label filters work by either filtering a specific value
-                                of a label (i.e. list("key=value")) or by looking for the existence of a given
-                                key (i.e. "key").
-        :param top_level:       If True will return only routers and endpoint that are NOT children of any router.
-        :param uids:            List of model endpoint unique ids to include in the result.
-
-
-        :return: A list of model endpoint dictionaries.
-        """
-
-        # # Initialize an empty model endpoints list
-        endpoint_list = []
-
-        # Retrieve the raw data from the KV table and get the endpoint ids
-        try:
-            cursor = self.client.kv.new_cursor(
-                container=self.container,
-                table_path=self.path,
-                filter_expression=self._build_kv_cursor_filter_expression(
-                    self.project,
-                    function,
-                    model,
-                    labels,
-                    top_level,
-                ),
-                raise_for_status=v3io.dataplane.RaiseForStatus.never,
-            )
-            items = cursor.all()
-
-        except Exception as exc:
-            logger.warning("Failed retrieving raw data from kv table", exc=exc)
-            return endpoint_list
-
-        # Create a list of model endpoints unique ids
-        if uids is None:
-            uids = []
-            for item in items:
-                if model_monitoring_constants.EventFieldType.UID not in item:
-                    # This is kept for backwards compatibility - in old versions the key column named endpoint_id
-                    uids.append(
-                        item[model_monitoring_constants.EventFieldType.ENDPOINT_ID]
-                    )
-                else:
-                    uids.append(item[model_monitoring_constants.EventFieldType.UID])
-
-        # Add each relevant model endpoint to the model endpoints list
-        for endpoint_id in uids:
-            endpoint = self.get_model_endpoint(
-                endpoint_id=endpoint_id,
-            )
-            endpoint_list.append(endpoint)
-
-        return endpoint_list
-
-    def delete_model_endpoints_resources(
-        self, endpoints: typing.List[typing.Dict[str, typing.Any]]
-    ):
-        """
-        Delete all model endpoints resources in both KV and the time series DB.
-
-        :param endpoints: A list of model endpoints flattened dictionaries.
-        """
-
-        # Delete model endpoint record from KV table
-        for endpoint_dict in endpoints:
-            if model_monitoring_constants.EventFieldType.UID not in endpoint_dict:
-                # This is kept for backwards compatibility - in old versions the key column named endpoint_id
-                endpoint_id = endpoint_dict[
-                    model_monitoring_constants.EventFieldType.ENDPOINT_ID
-                ]
-            else:
-                endpoint_id = endpoint_dict[
-                    model_monitoring_constants.EventFieldType.UID
-                ]
-            self.delete_model_endpoint(
-                endpoint_id,
-            )
-
-        # Delete remain records in the KV
-        all_records = self.client.kv.new_cursor(
-            container=self.container,
-            table_path=self.path,
-            raise_for_status=v3io.dataplane.RaiseForStatus.never,
-        ).all()
-
-        all_records = [r["__name"] for r in all_records]
-
-        # Cleanup KV
-        for record in all_records:
-            self.client.kv.delete(
-                container=self.container,
-                table_path=self.path,
-                key=record,
-                raise_for_status=v3io.dataplane.RaiseForStatus.never,
-            )
-
-        # Cleanup TSDB
-        frames = mlrun.utils.v3io_clients.get_frames_client(
-            token=self.access_key,
-            address=mlrun.mlconf.v3io_framesd,
-            container=self.container,
-        )
-
-        # Generate the required tsdb paths
-        tsdb_path, filtered_path = self._generate_tsdb_paths()
-
-        # Delete time series DB resources
-        try:
-            frames.delete(
-                backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
-                table=filtered_path,
-            )
-        except (v3io_frames.errors.DeleteError, v3io_frames.errors.CreateError) as e:
-            # Frames might raise an exception if schema file does not exist.
-            logger.warning("Failed to delete TSDB schema file:", err=e)
-            pass
-
-        # Final cleanup of tsdb path
-        tsdb_path.replace("://u", ":///u")
-        store, _ = mlrun.store_manager.get_or_create_store(tsdb_path)
-        store.rm(tsdb_path, recursive=True)
-
-    def get_endpoint_real_time_metrics(
-        self,
-        endpoint_id: str,
-        metrics: typing.List[str],
-        start: str = "now-1h",
-        end: str = "now",
-        access_key: str = None,
-    ) -> typing.Dict[str, typing.List[typing.Tuple[str, float]]]:
-        """
-        Getting metrics from the time series DB. There are pre-defined metrics for model endpoints such as
-        `predictions_per_second` and `latency_avg_5m` but also custom metrics defined by the user.
-
-        :param endpoint_id:      The unique id of the model endpoint.
-        :param metrics:          A list of real-time metrics to return for the model endpoint.
-        :param start:            The start time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param end:              The end time of the metrics. Can be represented by a string containing an RFC 3339
-                                 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
-                                 `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or 0 for the
-                                 earliest time.
-        :param access_key:       V3IO access key that will be used for generating Frames client object. If not
-                                 provided, the access key will be retrieved from the environment variables.
-
-        :return: A dictionary of metrics in which the key is a metric name and the value is a list of tuples that
-                 includes timestamps and the values.
-        """
-
-        # Initialize access key
-        access_key = access_key or mlrun.mlconf.get_v3io_access_key()
-
-        if not metrics:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Metric names must be provided"
-            )
-
-        # Initialize metrics mapping dictionary
-        metrics_mapping = {}
-
-        # Getting the path for the time series DB
-        events_path = (
-            mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
-                project=self.project,
-                kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
-            )
-        )
-        (
-            _,
-            container,
-            events_path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(events_path)
-
-        # Retrieve the raw data from the time series DB based on the provided metrics and time ranges
-        frames_client = mlrun.utils.v3io_clients.get_frames_client(
-            token=access_key,
-            address=mlrun.mlconf.v3io_framesd,
-            container=container,
-        )
-
-        try:
-            data = frames_client.read(
-                backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
-                table=events_path,
-                columns=["endpoint_id", *metrics],
-                filter=f"endpoint_id=='{endpoint_id}'",
-                start=start,
-                end=end,
-            )
-
-            # Fill the metrics mapping dictionary with the metric name and values
-            data_dict = data.to_dict()
-            for metric in metrics:
-                metric_data = data_dict.get(metric)
-                if metric_data is None:
-                    continue
-
-                values = [
-                    (str(timestamp), value) for timestamp, value in metric_data.items()
-                ]
-                metrics_mapping[metric] = values
-
-        except v3io_frames.errors.ReadError:
-            logger.warn("Failed to read tsdb", endpoint=endpoint_id)
-
-        return metrics_mapping
-
-    def _generate_tsdb_paths(self) -> typing.Tuple[str, str]:
-        """Generate a short path to the TSDB resources and a filtered path for the frames object
-        :return: A tuple of:
-             [0] = Short path to the TSDB resources
-             [1] = Filtered path to TSDB events without schema and container
-        """
-        # Full path for the time series DB events
-        full_path = (
-            mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
-                project=self.project,
-                kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
-            )
-        )
-
-        # Generate the main directory with the TSDB resources
-        tsdb_path = mlrun.utils.model_monitoring.parse_model_endpoint_project_prefix(
-            full_path, self.project
-        )
-
-        # Generate filtered path without schema and container as required by the frames object
-        (
-            _,
-            _,
-            filtered_path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(full_path)
-        return tsdb_path, filtered_path
-
-    @staticmethod
-    def _build_kv_cursor_filter_expression(
-        project: str,
-        function: str = None,
-        model: str = None,
-        labels: typing.List[str] = None,
-        top_level: bool = False,
-    ) -> str:
-        """
-        Convert the provided filters into a valid filter expression. The expected filter expression includes different
-        conditions, divided by ' AND '.
-
-        :param project:    The name of the project.
-        :param model:      The name of the model to filter by.
-        :param function:   The name of the function to filter by.
-        :param labels:     A list of labels to filter by. Label filters work by either filtering a specific value of
-                           a label (i.e. list("key=value")) or by looking for the existence of a given
-                           key (i.e. "key").
-        :param top_level:  If True will return only routers and endpoint that are NOT children of any router.
-
-        :return: A valid filter expression as a string.
-
-        :raise MLRunInvalidArgumentError: If project value is None.
-        """
-
-        if not project:
-            raise mlrun.errors.MLRunInvalidArgumentError("project can't be empty")
-
-        # Add project filter
-        filter_expression = [f"project=='{project}'"]
-
-        # Add function and model filters
-        if function:
-            filter_expression.append(f"function=='{function}'")
-        if model:
-            filter_expression.append(f"model=='{model}'")
-
-        # Add labels filters
-        if labels:
-            for label in labels:
-                if not label.startswith("_"):
-                    label = f"_{label}"
-
-                if "=" in label:
-                    lbl, value = list(map(lambda x: x.strip(), label.split("=")))
-                    filter_expression.append(f"{lbl}=='{value}'")
-                else:
-                    filter_expression.append(f"exists({label})")
-
-        # Apply top_level filter (remove endpoints that considered a child of a router)
-        if top_level:
-            filter_expression.append(
-                f"(endpoint_type=='{str(mlrun.model_monitoring.EndpointType.NODE_EP.value)}' "
-                f"OR  endpoint_type=='{str(mlrun.model_monitoring.EndpointType.ROUTER.value)}')"
-            )
-
-        return " AND ".join(filter_expression)