mlrun 1.3.2rc1__py3-none-any.whl → 1.3.2rc2__py3-none-any.whl

mlrun/api/crud/model_monitoring/grafana.py

@@ -1,427 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from typing import Any, Dict, List, Optional, Set
-
-import numpy as np
-import pandas as pd
-from fastapi.concurrency import run_in_threadpool
-from sqlalchemy.orm import Session
-
-import mlrun.api.crud
-import mlrun.api.schemas
-import mlrun.api.utils.auth.verifier
-import mlrun.model_monitoring
-from mlrun.api.schemas import (
-    GrafanaColumn,
-    GrafanaDataPoint,
-    GrafanaNumberColumn,
-    GrafanaTable,
-    GrafanaTimeSeriesTarget,
-    ProjectsFormat,
-)
-from mlrun.api.utils.singletons.project_member import get_project_member
-from mlrun.errors import MLRunBadRequestError
-from mlrun.utils import config, logger
-from mlrun.utils.model_monitoring import parse_model_endpoint_store_prefix
-from mlrun.utils.v3io_clients import get_frames_client
-
-
-def grafana_list_projects(
-    db_session: Session,
-    auth_info: mlrun.api.schemas.AuthInfo,
-    query_parameters: Dict[str, str],
-) -> List[str]:
-    """
-    List available project names. Will be used as a filter in each grafana dashboard.
-
-    :param db_session:        A session that manages the current dialog with the database.
-    :param auth_info:         The auth info of the request.
-    :param query_parameters:  Dictionary of query parameters attached to the request. Note that this parameter is
-                              required by the API even though it is not being used in this function.
-
-    :return: List of available project names.
-    """
-
-    projects_output = get_project_member().list_projects(
-        db_session, format_=ProjectsFormat.name_only, leader_session=auth_info.session
-    )
-    return projects_output.projects
-
-
-# TODO: remove in 1.5.0 the following functions: grafana_list_endpoints, grafana_individual_feature_analysis,
-#  grafana_overall_feature_analysis, grafana_income_features, parse_query_parameters, drop_grafana_escape_chars,
-
-
-async def grafana_list_endpoints(
-    body: Dict[str, Any],
-    query_parameters: Dict[str, str],
-    auth_info: mlrun.api.schemas.AuthInfo,
-) -> List[GrafanaTable]:
-    project = query_parameters.get("project")
-
-    # Filters
-    model = query_parameters.get("model", None)
-    function = query_parameters.get("function", None)
-    labels = query_parameters.get("labels", "")
-    labels = labels.split(",") if labels else []
-
-    # Metrics to include
-    metrics = query_parameters.get("metrics", "")
-    metrics = metrics.split(",") if metrics else []
-
-    # Time range for metrics
-    start = body.get("rangeRaw", {}).get("start", "now-1h")
-    end = body.get("rangeRaw", {}).get("end", "now")
-
-    if project:
-        await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
-            project,
-            mlrun.api.schemas.AuthorizationAction.read,
-            auth_info,
-        )
-    endpoint_list = await run_in_threadpool(
-        mlrun.api.crud.ModelEndpoints().list_model_endpoints,
-        auth_info=auth_info,
-        project=project,
-        model=model,
-        function=function,
-        labels=labels,
-        metrics=metrics,
-        start=start,
-        end=end,
-    )
-    allowed_endpoints = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
-        endpoint_list.endpoints,
-        lambda _endpoint: (
-            _endpoint.metadata.project,
-            _endpoint.metadata.uid,
-        ),
-        auth_info,
-    )
-    endpoint_list.endpoints = allowed_endpoints
-
-    columns = [
-        GrafanaColumn(text="endpoint_id", type="string"),
-        GrafanaColumn(text="endpoint_function", type="string"),
-        GrafanaColumn(text="endpoint_model", type="string"),
-        GrafanaColumn(text="endpoint_model_class", type="string"),
-        GrafanaColumn(text="first_request", type="time"),
-        GrafanaColumn(text="last_request", type="time"),
-        GrafanaColumn(text="accuracy", type="number"),
-        GrafanaColumn(text="error_count", type="number"),
-        GrafanaColumn(text="drift_status", type="number"),
-        GrafanaColumn(text="predictions_per_second", type="number"),
-        GrafanaColumn(text="latency_avg_1h", type="number"),
-    ]
-
-    table = GrafanaTable(columns=columns)
-    for endpoint in endpoint_list.endpoints:
-        row = [
-            endpoint.metadata.uid,
-            endpoint.spec.function_uri,
-            endpoint.spec.model,
-            endpoint.spec.model_class,
-            endpoint.status.first_request,
-            endpoint.status.last_request,
-            "N/A",  # Leaving here for backwards compatibility
-            endpoint.status.error_count,
-            endpoint.status.drift_status,
-        ]
-
-        if (
-            endpoint.status.metrics
-            and mlrun.model_monitoring.EventKeyMetrics.GENERIC
-            in endpoint.status.metrics
-        ):
-            row.extend(
-                [
-                    endpoint.status.metrics[
-                        mlrun.model_monitoring.EventKeyMetrics.GENERIC
-                    ][mlrun.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND],
-                    endpoint.status.metrics[
-                        mlrun.model_monitoring.EventKeyMetrics.GENERIC
-                    ][mlrun.model_monitoring.EventLiveStats.LATENCY_AVG_1H],
-                ]
-            )
-
-        table.add_row(*row)
-
-    return [table]
-
-
-async def grafana_individual_feature_analysis(
-    body: Dict[str, Any],
-    query_parameters: Dict[str, str],
-    auth_info: mlrun.api.schemas.AuthInfo,
-):
-    endpoint_id = query_parameters.get("endpoint_id")
-    project = query_parameters.get("project")
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
-        project,
-        endpoint_id,
-        mlrun.api.schemas.AuthorizationAction.read,
-        auth_info,
-    )
-
-    endpoint = await run_in_threadpool(
-        mlrun.api.crud.ModelEndpoints().get_model_endpoint,
-        auth_info=auth_info,
-        project=project,
-        endpoint_id=endpoint_id,
-        feature_analysis=True,
-    )
-
-    # Load JSON data from KV, make sure not to fail if a field is missing
-    feature_stats = endpoint.status.feature_stats or {}
-    current_stats = endpoint.status.current_stats or {}
-    drift_measures = endpoint.status.drift_measures or {}
-
-    table = GrafanaTable(
-        columns=[
-            GrafanaColumn(text="feature_name", type="string"),
-            GrafanaColumn(text="actual_min", type="number"),
-            GrafanaColumn(text="actual_mean", type="number"),
-            GrafanaColumn(text="actual_max", type="number"),
-            GrafanaColumn(text="expected_min", type="number"),
-            GrafanaColumn(text="expected_mean", type="number"),
-            GrafanaColumn(text="expected_max", type="number"),
-            GrafanaColumn(text="tvd", type="number"),
-            GrafanaColumn(text="hellinger", type="number"),
-            GrafanaColumn(text="kld", type="number"),
-        ]
-    )
-
-    for feature, base_stat in feature_stats.items():
-        current_stat = current_stats.get(feature, {})
-        drift_measure = drift_measures.get(feature, {})
-
-        table.add_row(
-            feature,
-            current_stat.get("min"),
-            current_stat.get("mean"),
-            current_stat.get("max"),
-            base_stat.get("min"),
-            base_stat.get("mean"),
-            base_stat.get("max"),
-            drift_measure.get("tvd"),
-            drift_measure.get("hellinger"),
-            drift_measure.get("kld"),
-        )
-
-    return [table]
-
-
-async def grafana_overall_feature_analysis(
-    body: Dict[str, Any],
-    query_parameters: Dict[str, str],
-    auth_info: mlrun.api.schemas.AuthInfo,
-):
-    endpoint_id = query_parameters.get("endpoint_id")
-    project = query_parameters.get("project")
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
-        project,
-        endpoint_id,
-        mlrun.api.schemas.AuthorizationAction.read,
-        auth_info,
-    )
-    endpoint = await run_in_threadpool(
-        mlrun.api.crud.ModelEndpoints().get_model_endpoint,
-        auth_info=auth_info,
-        project=project,
-        endpoint_id=endpoint_id,
-        feature_analysis=True,
-    )
-
-    table = GrafanaTable(
-        columns=[
-            GrafanaNumberColumn(text="tvd_sum"),
-            GrafanaNumberColumn(text="tvd_mean"),
-            GrafanaNumberColumn(text="hellinger_sum"),
-            GrafanaNumberColumn(text="hellinger_mean"),
-            GrafanaNumberColumn(text="kld_sum"),
-            GrafanaNumberColumn(text="kld_mean"),
-        ]
-    )
-
-    if endpoint.status.drift_measures:
-        table.add_row(
-            endpoint.status.drift_measures.get("tvd_sum"),
-            endpoint.status.drift_measures.get("tvd_mean"),
-            endpoint.status.drift_measures.get("hellinger_sum"),
-            endpoint.status.drift_measures.get("hellinger_mean"),
-            endpoint.status.drift_measures.get("kld_sum"),
-            endpoint.status.drift_measures.get("kld_mean"),
-        )
-
-    return [table]
-
-
-async def grafana_incoming_features(
-    body: Dict[str, Any],
-    query_parameters: Dict[str, str],
-    auth_info: mlrun.api.schemas.AuthInfo,
-):
-    endpoint_id = query_parameters.get("endpoint_id")
-    project = query_parameters.get("project")
-    start = body.get("rangeRaw", {}).get("from", "now-1h")
-    end = body.get("rangeRaw", {}).get("to", "now")
-
-    await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
-        mlrun.api.schemas.AuthorizationResourceTypes.model_endpoint,
-        project,
-        endpoint_id,
-        mlrun.api.schemas.AuthorizationAction.read,
-        auth_info,
-    )
-
-    endpoint = await run_in_threadpool(
-        mlrun.api.crud.ModelEndpoints().get_model_endpoint,
-        auth_info=auth_info,
-        project=project,
-        endpoint_id=endpoint_id,
-    )
-
-    time_series = []
-
-    feature_names = endpoint.spec.feature_names
-
-    if not feature_names:
-        logger.warn(
-            "'feature_names' is either missing or not initialized in endpoint record",
-            endpoint_id=endpoint.metadata.uid,
-        )
-        return time_series
-
-    path = config.model_endpoint_monitoring.store_prefixes.default.format(
-        project=project, kind=mlrun.api.schemas.ModelMonitoringStoreKinds.EVENTS
-    )
-    _, container, path = parse_model_endpoint_store_prefix(path)
-
-    client = get_frames_client(
-        token=auth_info.data_session,
-        address=config.v3io_framesd,
-        container=container,
-    )
-
-    data: pd.DataFrame = await run_in_threadpool(
-        client.read,
-        backend="tsdb",
-        table=path,
-        columns=feature_names,
-        filter=f"endpoint_id=='{endpoint_id}'",
-        start=start,
-        end=end,
-    )
-
-    data.drop(["endpoint_id"], axis=1, inplace=True, errors="ignore")
-    data.index = data.index.astype(np.int64) // 10**6
-
-    for feature, indexed_values in data.to_dict().items():
-        target = GrafanaTimeSeriesTarget(target=feature)
-        for index, value in indexed_values.items():
-            data_point = GrafanaDataPoint(value=float(value), timestamp=index)
-            target.add_data_point(data_point)
-        time_series.append(target)
-
-    return time_series
-
-
-def parse_query_parameters(request_body: Dict[str, Any]) -> Dict[str, str]:
-    """
-    This function searches for the target field in Grafana's SimpleJson json. Once located, the target string is
-    parsed by splitting on semi-colons (;). Each part in the resulting list is then split by an equal sign (=) to be
-    read as key-value pairs.
-    """
-
-    # Try to get the target
-    targets = request_body.get("targets", [])
-
-    if len(targets) > 1:
-        logger.warn(
-            f"The 'targets' list contains more then one element ({len(targets)}), all targets except the first one are "
-            f"ignored."
-        )
-
-    target_obj = targets[0] if targets else {}
-    target_query = target_obj.get("target") if target_obj else ""
-
-    if not target_query:
-        raise MLRunBadRequestError(f"Target missing in request body:\n {request_body}")
-
-    parameters = _parse_parameters(target_query)
-
-    return parameters
-
-
-def parse_search_parameters(request_body: Dict[str, Any]) -> Dict[str, str]:
-    """
-    This function searches for the target field in Grafana's SimpleJson json. Once located, the target string is
-    parsed by splitting on semi-colons (;). Each part in the resulting list is then split by an equal sign (=) to be
-    read as key-value pairs.
-    """
-
-    # Try to get the target
-    target = request_body.get("target")
-
-    if not target:
-        raise MLRunBadRequestError(f"Target missing in request body:\n {request_body}")
-
-    parameters = _parse_parameters(target)
-
-    return parameters
-
-
-def _parse_parameters(target_query):
-    parameters = {}
-    for query in filter(lambda q: q, target_query.split(";")):
-        query_parts = query.split("=")
-        if len(query_parts) < 2:
-            raise MLRunBadRequestError(
-                f"Query must contain both query key and query value. Expected query_key=query_value, found {query} "
-                f"instead."
-            )
-        parameters[query_parts[0]] = query_parts[1]
-    return parameters
-
-
-def drop_grafana_escape_chars(query_parameters: Dict[str, str]):
-    query_parameters = dict(query_parameters)
-    endpoint_id = query_parameters.get("endpoint_id")
-    if endpoint_id is not None:
-        query_parameters["endpoint_id"] = endpoint_id.replace("\\", "")
-    return query_parameters
-
-
-def validate_query_parameters(
-    query_parameters: Dict[str, str], supported_endpoints: Optional[Set[str]] = None
-):
-    """Validates the parameters sent via Grafana's SimpleJson query"""
-    if "target_endpoint" not in query_parameters:
-        raise MLRunBadRequestError(
-            f"Expected 'target_endpoint' field in query, found {query_parameters} instead"
-        )
-
-    if (
-        supported_endpoints is not None
-        and query_parameters["target_endpoint"] not in supported_endpoints
-    ):
-        raise MLRunBadRequestError(
-            f"{query_parameters['target_endpoint']} unsupported in query parameters: {query_parameters}. "
-            f"Currently supports: {','.join(supported_endpoints)}"
-        )

mlrun/datastore/spark_udf.py

@@ -1,40 +0,0 @@
-# Copyright 2023 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import hashlib
-
-from pyspark.sql.functions import udf
-from pyspark.sql.types import StringType
-
-
-def _hash_list(*list_to_hash):
-    list_to_hash = [str(element) for element in list_to_hash]
-    str_concatted = "".join(list_to_hash)
-    sha1 = hashlib.sha1()
-    sha1.update(str_concatted.encode("utf8"))
-    return sha1.hexdigest()
-
-
-def _redis_stringify_key(key_list):
-    suffix = "}:static"
-    if isinstance(key_list, list):
-        if len(key_list) >= 2:
-            return str(key_list[0]) + "." + _hash_list(key_list[1:]) + suffix
-        if len(key_list) == 2:
-            return str(key_list[0]) + "." + str(key_list[1]) + suffix
-        return str(key_list[0]) + suffix
-    return str(key_list) + suffix
-
-
-hash_and_concat_v3io_udf = udf(_hash_list, StringType())
-hash_and_concat_redis_udf = udf(_redis_stringify_key, StringType())

mlrun/model_monitoring/__init__.py

@@ -1,44 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-# for backwards compatibility
-
-__all__ = [
-    "ModelEndpoint",
-    "ModelMonitoringMode",
-    "EndpointType",
-    "create_model_endpoint_uid",
-    "EventFieldType",
-    "EventLiveStats",
-    "EventKeyMetrics",
-    "TimeSeriesTarget",
-    "ModelEndpointTarget",
-    "FileTargetKind",
-    "ProjectSecretKeys",
-    "ModelMonitoringStoreKinds",
-]
-
-from .common import EndpointType, ModelMonitoringMode, create_model_endpoint_uid
-from .constants import (
-    EventFieldType,
-    EventKeyMetrics,
-    EventLiveStats,
-    FileTargetKind,
-    ModelEndpointTarget,
-    ModelMonitoringStoreKinds,
-    ProjectSecretKeys,
-    TimeSeriesTarget,
-)
-from .model_endpoint import ModelEndpoint

mlrun/model_monitoring/common.py

@@ -1,112 +0,0 @@
-# Copyright 2018 Iguazio
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import enum
-import hashlib
-from dataclasses import dataclass
-from typing import Optional
-
-import mlrun.utils
-
-
-class ModelMonitoringMode(str, enum.Enum):
-    enabled = "enabled"
-    disabled = "disabled"
-
-
-class EndpointType(enum.IntEnum):
-    NODE_EP = 1  # end point that is not a child of a router
-    ROUTER = 2  # endpoint that is router
-    LEAF_EP = 3  # end point that is a child of a router
-
-
-def create_model_endpoint_uid(function_uri: str, versioned_model: str):
-    function_uri = FunctionURI.from_string(function_uri)
-    versioned_model = VersionedModel.from_string(versioned_model)
-
-    if (
-        not function_uri.project
-        or not function_uri.function
-        or not versioned_model.model
-    ):
-        raise ValueError("Both function_uri and versioned_model have to be initialized")
-
-    uid = EndpointUID(
-        function_uri.project,
-        function_uri.function,
-        function_uri.tag,
-        function_uri.hash_key,
-        versioned_model.model,
-        versioned_model.version,
-    )
-
-    return uid
-
-
-@dataclass
-class FunctionURI:
-    project: str
-    function: str
-    tag: Optional[str] = None
-    hash_key: Optional[str] = None
-
-    @classmethod
-    def from_string(cls, function_uri):
-        project, uri, tag, hash_key = mlrun.utils.parse_versioned_object_uri(
-            function_uri
-        )
-        return cls(
-            project=project,
-            function=uri,
-            tag=tag or None,
-            hash_key=hash_key or None,
-        )
-
-
-@dataclass
-class VersionedModel:
-    model: str
-    version: Optional[str]
-
-    @classmethod
-    def from_string(cls, model):
-        try:
-            model, version = model.split(":")
-        except ValueError:
-            model, version = model, None
-
-        return cls(model, version)
-
-
-@dataclass
-class EndpointUID:
-    project: str
-    function: str
-    function_tag: str
-    function_hash_key: str
-    model: str
-    model_version: str
-    uid: Optional[str] = None
-
-    def __post_init__(self):
-        function_ref = (
-            f"{self.function}_{self.function_tag or self.function_hash_key or 'N/A'}"
-        )
-        versioned_model = f"{self.model}_{self.model_version or 'N/A'}"
-        unique_string = f"{self.project}_{function_ref}_{versioned_model}"
-        self.uid = hashlib.sha1(unique_string.encode("utf-8")).hexdigest()
-
-    def __str__(self):
-        return self.uid