mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/model_monitoring/prometheus.py

@@ -1,219 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import typing
-
-import prometheus_client
-
-from mlrun.common.schemas.model_monitoring import EventFieldType, PrometheusMetric
-
-# Memory path for Prometheus registry file
-_registry_path = "/tmp/prom-reg.txt"
-
-# Initializing Promethues metric collector registry
-_registry: prometheus_client.CollectorRegistry = prometheus_client.CollectorRegistry()
-
-# The following real-time metrics are being updated through the monitoring stream graph steps
-_prediction_counter: prometheus_client.Counter = prometheus_client.Counter(
-    name=PrometheusMetric.PREDICTIONS_TOTAL,
-    documentation="Counter for total predictions",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-        EventFieldType.ENDPOINT_TYPE,
-    ],
-)
-_model_latency: prometheus_client.Summary = prometheus_client.Summary(
-    name=PrometheusMetric.MODEL_LATENCY_SECONDS,
-    documentation="Summary for for model latency",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-        EventFieldType.ENDPOINT_TYPE,
-    ],
-)
-_income_features: prometheus_client.Gauge = prometheus_client.Gauge(
-    name=PrometheusMetric.INCOME_FEATURES,
-    documentation="Samples of features and predictions",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.METRIC,
-    ],
-)
-_error_counter: prometheus_client.Counter = prometheus_client.Counter(
-    name=PrometheusMetric.ERRORS_TOTAL,
-    documentation="Counter for total errors",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.MODEL,
-    ],
-)
-
-# The following metrics are being updated through the model monitoring batch job
-_batch_metrics: prometheus_client.Gauge = prometheus_client.Gauge(
-    name=PrometheusMetric.DRIFT_METRICS,
-    documentation="Results from the batch drift analysis",
-    registry=_registry,
-    labelnames=[
-        EventFieldType.PROJECT,
-        EventFieldType.ENDPOINT_ID,
-        EventFieldType.METRIC,
-    ],
-)
-_drift_status: prometheus_client.Enum = prometheus_client.Enum(
-    name=PrometheusMetric.DRIFT_STATUS,
-    documentation="Drift status of the model endpoint",
-    registry=_registry,
-    states=["NO_DRIFT", "DRIFT_DETECTED", "POSSIBLE_DRIFT"],
-    labelnames=[EventFieldType.PROJECT, EventFieldType.ENDPOINT_ID],
-)
-
-
-def _write_registry(func):
-    def wrapper(*args, **kwargs):
-        global _registry
-        """A wrapper function to update the registry file each time a metric has been updated"""
-        func(*args, **kwargs)
-        prometheus_client.write_to_textfile(path=_registry_path, registry=_registry)
-
-    return wrapper
-
-
-@_write_registry
-def write_predictions_and_latency_metrics(
-    project: str, endpoint_id: str, latency: int, model_name: str, endpoint_type: int
-):
-    """
-    Update the prediction counter and the latency value of the provided model endpoint within Prometheus registry.
-    Please note that while the prediction counter is ALWAYS increasing by 1,the latency summary metric is being
-    increased by the event latency time. Grafana dashboard will query the average latency time by dividing the total
-    latency value by the total amount of predictions.
-
-    :param project:       Project name.
-    :param endpoint_id:   Model endpoint unique id.
-    :param latency:       Latency time (microsecond) in which the event has been processed through the model server.
-    :param model_name:    Model name which will be used by Grafana for displaying the results by model.
-    :param endpoint_type: Endpoint type that is represented by an int (possible values: 1,2,3) corresponding to the
-                          Enum class :py:class:`~mlrun.common.schemas.model_monitoring.EndpointType`.
-    """
-
-    # Increase the prediction counter by 1
-    _prediction_counter.labels(
-        project=project,
-        endpoint_id=endpoint_id,
-        model=model_name,
-        endpoint_type=endpoint_type,
-    ).inc(1)
-
-    # Increase the latency value according to the provided latency of the current event
-    _model_latency.labels(
-        project=project,
-        endpoint_id=endpoint_id,
-        model=model_name,
-        endpoint_type=endpoint_type,
-    ).observe(latency)
-
-
-@_write_registry
-def write_income_features(
-    project: str, endpoint_id: str, features: typing.Dict[str, float]
-):
-    """Update a sample of features.
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param features:    Dictionary in which the key is a feature name and the value is a float number.
-
-
-    """
-
-    for metric in features:
-        _income_features.labels(
-            project=project, endpoint_id=endpoint_id, metric=metric
-        ).set(value=features[metric])
-
-
-@_write_registry
-def write_drift_metrics(project: str, endpoint_id: str, metric: str, value: float):
-    """Update drift metrics that have been calculated through the monitoring batch job
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param metric:      Metric name (e.g. TVD, Hellinger).
-    :param value:       Metric value as a float.
-
-    """
-
-    _batch_metrics.labels(project=project, endpoint_id=endpoint_id, metric=metric).set(
-        value=value
-    )
-
-
-@_write_registry
-def write_drift_status(project: str, endpoint_id: str, drift_status: str):
-    """
-    Update the drift status enum for a specific model endpoint.
-
-    :param project:      Project name.
-    :param endpoint_id:  Model endpoint unique id.
-    :param drift_status: Drift status value, can be one of the following: 'NO_DRIFT', 'DRIFT_DETECTED', or
-                         'POSSIBLE_DRIFT'.
-    """
-
-    _drift_status.labels(project=project, endpoint_id=endpoint_id).state(drift_status)
-
-
-@_write_registry
-def write_errors(project: str, endpoint_id: str, model_name: str):
-    """
-    Update the error counter for a specific model endpoint.
-
-    :param project:     Project name.
-    :param endpoint_id: Model endpoint unique id.
-    :param model_name:  Model name. Will be used by Grafana to show the amount of errors per model by time.
-    """
-
-    _error_counter.labels(
-        project=project, endpoint_id=endpoint_id, model=model_name
-    ).inc(1)
-
-
-def get_registry() -> str:
-    """Returns the parsed registry file according to the exposition format of Prometheus."""
-
-    # Read the registry file (note that the text is stored in UTF-8 format)
-    f = open(_registry_path)
-    lines = f.read()
-    f.close()
-
-    # Reset part of the metrics to avoid a repeating scraping of the same value
-    clean_metrics()
-
-    return lines
-
-
-@_write_registry
-def clean_metrics():
-    """Clean the income features values. As these results are relevant only for a certain timestamp, we will remove
-    them from the global registry after they have been scraped by Prometheus."""
-
-    _income_features.clear()

mlrun/model_monitoring/stores/__init__.py

@@ -1,111 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
-import enum
-import typing
-
-import mlrun.common.schemas.secret
-import mlrun.errors
-
-from .model_endpoint_store import ModelEndpointStore
-
-
-class ModelEndpointStoreType(enum.Enum):
-    """Enum class to handle the different store type values for saving a model endpoint record."""
-
-    v3io_nosql = "v3io-nosql"
-    SQL = "sql"
-
-    def to_endpoint_store(
-        self,
-        project: str,
-        access_key: str = None,
-        endpoint_store_connection: str = None,
-        secret_provider: typing.Callable = None,
-    ) -> ModelEndpointStore:
-        """
-        Return a ModelEndpointStore object based on the provided enum value.
-
-        :param project:                    The name of the project.
-        :param access_key:                 Access key with permission to the DB table. Note that if access key is None
-                                           and the endpoint target is from type KV then the access key will be
-                                           retrieved from the environment variable.
-        :param endpoint_store_connection: A valid connection string for model endpoint target. Contains several
-                                          key-value pairs that required for the database connection.
-                                          e.g. A root user with password 1234, tries to connect a schema called
-                                          mlrun within a local MySQL DB instance:
-                                          'mysql+pymysql://root:1234@localhost:3306/mlrun'.
-        :param secret_provider:           An optional secret provider to get the connection string secret.
-
-        :return: `ModelEndpointStore` object.
-
-        """
-
-        if self.value == ModelEndpointStoreType.v3io_nosql.value:
-            from .kv_model_endpoint_store import KVModelEndpointStore
-
-            # Get V3IO access key from env
-            access_key = access_key or mlrun.mlconf.get_v3io_access_key()
-
-            return KVModelEndpointStore(project=project, access_key=access_key)
-
-        # Assuming SQL store target if store type is not KV.
-        # Update these lines once there are more than two store target types.
-
-        from .sql_model_endpoint_store import SQLModelEndpointStore
-
-        return SQLModelEndpointStore(
-            project=project,
-            sql_connection_string=endpoint_store_connection,
-            secret_provider=secret_provider,
-        )
-
-    @classmethod
-    def _missing_(cls, value: typing.Any):
-        """A lookup function to handle an invalid value.
-        :param value: Provided enum (invalid) value.
-        """
-        valid_values = list(cls.__members__.keys())
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
-        )
-
-
-def get_model_endpoint_store(
-    project: str,
-    access_key: str = None,
-    secret_provider: typing.Callable = None,
-) -> ModelEndpointStore:
-    """
-    Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
-
-    :param project:         The name of the project.
-    :param access_key:      Access key with permission to the DB table.
-    :param secret_provider: An optional secret provider to get the connection string secret.
-
-    :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
-             model endpoint record such as write, update, get and delete.
-    """
-
-    # Get store type value from ModelEndpointStoreType enum class
-    model_endpoint_store_type = ModelEndpointStoreType(
-        mlrun.mlconf.model_endpoint_monitoring.store_type
-    )
-
-    # Convert into model endpoint store target object
-    return model_endpoint_store_type.to_endpoint_store(
-        project=project, access_key=access_key, secret_provider=secret_provider
-    )