veadk-python 0.2.27__py3-none-any.whl

veadk/evaluation/utils/prometheus.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+from prometheus_client.exposition import basic_auth_handler
+from pydantic import Field
+
+from veadk.config import getenv
+from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
+
+
+class PrometheusPushgatewayConfig:
+    """Configures connection to Prometheus Pushgateway for metrics export.
+
+    This class holds settings for pushing evaluation metrics to Prometheus.
+    It uses environment variables for default values.
+
+    Attributes:
+        url (str): URL of the Prometheus Pushgateway endpoint.
+            Defaults to OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL environment variable.
+        username (str): Username for authentication.
+            Defaults to OBSERVABILITY_PROMETHEUS_USERNAME environment variable.
+        password (str): Password for authentication.
+            Defaults to OBSERVABILITY_PROMETHEUS_PASSWORD environment variable.
+
+    Note:
+        All fields are optional and use environment variables if not provided.
+    """
+
+    url: str = Field(
+        default_factory=lambda: getenv(
+            "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL",
+        ),
+    )
+    username: str = Field(
+        default_factory=lambda: getenv(
+            "OBSERVABILITY_PROMETHEUS_USERNAME",
+        ),
+    )
+    password: str = Field(
+        default_factory=lambda: getenv(
+            "OBSERVABILITY_PROMETHEUS_PASSWORD",
+        ),
+    )
+
+
+registry = CollectorRegistry()
+
+test_cases_total_metric = Gauge(
+    "test_cases_total",
+    "Total number of test cases in this evaluation",
+    registry=registry,
+)
+
+test_cases_success_metric = Gauge(
+    "test_cases_success", "Success number of test cases", registry=registry
+)
+
+test_cases_pass_metric = Gauge(
+    "test_cases_pass", "Passed number of test cases", registry=registry
+)
+
+test_cases_failure_metric = Gauge(
+    "test_cases_failure", "Failuer number of test cases", registry=registry
+)
+
+case_threshold_metric = Gauge("threshold", "Threshold of test cases", registry=registry)
+diff_threshold_metric = Gauge(
+    "diff_threshold", "Diff threshold of test cases", registry=registry
+)
+
+test_cases_data_metric = Gauge(
+    "test_cases_data",
+    "Specific data of test cases",
+    registry=registry,
+    labelnames=["data"],
+)
+
+eval_data_metric = Gauge(
+    "eval_data",
+    "Specific data of evaluation",
+    registry=registry,
+    labelnames=["data"],
+)
+
+
+def post_pushgateway(
+    pushgateway_url: str,
+    username: str,
+    password: str,
+    job_name: str,
+    registry: CollectorRegistry,
+    grouping_key: dict[str, str] | None = None,
+):
+    """Pushes metrics to Prometheus Pushgateway with authentication.
+
+    This function sends collected metrics to the specified Pushgateway URL.
+    It uses basic authentication and optional grouping keys.
+
+    Args:
+        pushgateway_url (str): URL of the Pushgateway endpoint.
+        username (str): Authentication username.
+        password (str): Authentication password.
+        job_name (str): Name of the job for metrics labeling.
+        registry (CollectorRegistry): Registry containing metrics to push.
+        grouping_key (dict[str, str] | None): Optional key-value pairs for grouping.
+
+    Raises:
+        Exception: If push operation fails due to network or auth issues.
+
+    Note:
+        Authentication handler is created internally using provided credentials.
+    """
+
+    def auth_handler(url, method, timeout, headers, data):
+        return basic_auth_handler(
+            url, method, timeout, headers, data, username, password
+        )
+
+    push_to_gateway(
+        gateway=pushgateway_url,
+        job=job_name,
+        registry=registry,
+        grouping_key=grouping_key,
+        handler=auth_handler,
+    )
+
+
+def push_to_prometheus(
+    test_name: str,
+    test_cases_total: int,
+    test_cases_failure: int,
+    test_cases_pass: int,
+    test_data_list: list[EvalResultCaseData],
+    eval_data: EvalResultMetadata,
+    case_threshold: float = 0.5,
+    diff_threshold: float = 0.2,
+    url: str = "",
+    username: str = "",
+    password: str = "",
+):
+    """Sets and pushes evaluation metrics to Prometheus.
+
+    This function updates gauge metrics with evaluation results and pushes them.
+    It handles counts, thresholds, and specific data labels.
+
+    Args:
+        test_name (str): Name of the test for grouping.
+        test_cases_total (int): Total number of test cases.
+        test_cases_failure (int): Number of failed test cases.
+        test_cases_pass (int): Number of passed test cases.
+        test_data_list (list[EvalResultCaseData]): List of case data for labeling.
+        eval_data (EvalResultMetadata): Metadata for evaluation.
+        case_threshold (float): Threshold value for cases. Defaults to 0.5.
+        diff_threshold (float): Diff threshold value. Defaults to 0.2.
+        url (str): Pushgateway URL. Defaults to empty.
+        username (str): Auth username. Defaults to empty.
+        password (str): Auth password. Defaults to empty.
+
+    Returns:
+        None: Metrics are set and pushed directly.
+
+    Raises:
+        ValueError: If required data is invalid.
+    """
+    test_cases_total_metric.set(test_cases_total)
+    test_cases_failure_metric.set(test_cases_failure)
+    test_cases_pass_metric.set(test_cases_pass)
+
+    for test_data in test_data_list:
+        test_cases_data_metric.labels(data=str(test_data.__dict__)).set(1)
+
+    eval_data_metric.labels(data=str(eval_data.__dict__)).set(1)
+    case_threshold_metric.set(case_threshold)
+    diff_threshold_metric.set(diff_threshold)
+
+    post_pushgateway(
+        pushgateway_url=url,
+        username=username,
+        password=password,
+        job_name="veadk_eval_job",
+        registry=registry,
+        grouping_key={"test_name": test_name},
+    )

veadk/integrations/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

veadk/integrations/ve_apig/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

veadk/integrations/ve_apig/ve_apig.py

@@ -0,0 +1,349 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import volcenginesdkcore
+from volcenginesdkapig import APIGApi
+from volcenginesdkapig20221112 import APIG20221112Api, UpstreamListForCreateRouteInput
+
+from veadk.utils.volcengine_sign import ve_request
+
+
+class APIGateway:
+    def __init__(self, access_key: str, secret_key: str, region: str = "cn-beijing"):
+        self.ak = access_key
+        self.sk = secret_key
+        self.region = region
+        configuration = volcenginesdkcore.Configuration()
+        configuration.ak = self.ak
+        configuration.sk = self.sk
+        configuration.region = region
+
+        self.api_client = volcenginesdkcore.ApiClient(configuration=configuration)
+        self.apig_20221112_client = APIG20221112Api(api_client=self.api_client)
+        self.apig_client = APIGApi(api_client=self.api_client)
+
+    def list_gateways(self):
+        from volcenginesdkapig import ListGatewaysRequest
+
+        request = ListGatewaysRequest()
+        thread = self.apig_client.list_gateways(request, async_req=True)
+        result = thread.get()
+        return result
+
+    def create_serverless_gateway(self, instance_name: str) -> str:  # instance
+        from volcenginesdkapig import (
+            CreateGatewayRequest,
+            ResourceSpecForCreateGatewayInput,
+            ListGatewaysRequest,
+        )
+
+        request = CreateGatewayRequest(
+            name=instance_name,
+            region=self.region,
+            type="serverless",
+            resource_spec=ResourceSpecForCreateGatewayInput(
+                replicas=2,
+                instance_spec_code="1c2g",
+                clb_spec_code="small_1",
+                public_network_billing_type="traffic",
+                network_type={
+                    "EnablePublicNetwork": True,
+                    "EnablePrivateNetwork": False,
+                },
+            ),
+        )
+        thread = self.apig_client.create_gateway(request, async_req=True)
+        result = thread.get()
+        gateway_id = result.to_dict()["id"]
+
+        found = False
+        while not found:
+            request = ListGatewaysRequest()
+            thread = self.apig_client.list_gateways(request, async_req=True)
+            result = thread.get()
+            for item in result.items:
+                if (
+                    item.to_dict()["id"] == gateway_id
+                    and item.to_dict()["status"] == "Running"
+                ):
+                    found = True
+                    break
+            if not found:
+                time.sleep(5)
+        return gateway_id
+
+    def create_gateway_service(self, gateway_id: str, service_name: str) -> str:
+        """
+        Create a gateway service. (Domain name)
+        Args:
+            gateway_id (str): The ID of the gateway to which the service belongs.
+            service_name (str): The name of the service to be created.
+        Returns:
+            str: The ID of the created service.
+        """
+        from volcenginesdkapig import (
+            AuthSpecForCreateGatewayServiceInput,
+            CreateGatewayServiceRequest,
+        )
+
+        request = CreateGatewayServiceRequest(
+            gateway_id=gateway_id,
+            service_name=service_name,
+            protocol=["HTTP", "HTTPS"],
+            auth_spec=AuthSpecForCreateGatewayServiceInput(enable=False),
+        )
+        thread = self.apig_client.create_gateway_service(request, async_req=True)
+        result = thread.get()
+        return result.to_dict()["id"]
+
+    def create_vefaas_upstream(
+        self, function_id: str, gateway_id: str, upstream_name: str
+    ):
+        from volcenginesdkapig import (
+            CreateUpstreamRequest,
+            UpstreamSpecForCreateUpstreamInput,
+            VeFaasForCreateUpstreamInput,
+        )
+
+        request = CreateUpstreamRequest(
+            gateway_id=gateway_id,
+            name=upstream_name,
+            source_type="VeFaas",
+            upstream_spec=UpstreamSpecForCreateUpstreamInput(
+                ve_faas=VeFaasForCreateUpstreamInput(function_id=function_id)
+            ),
+        )
+        thread = self.apig_client.create_upstream(request, async_req=True)
+        result = thread.get()
+        return result.to_dict()["id"]
+
+    def create_domain_upstream(
+        self,
+        domain: str,
+        port: int,
+        is_https: bool,
+        gateway_id: str,
+        upstream_name: str,
+    ) -> str:
+        """
+        Create a domain upstream.
+        Args:
+            domain (str): The domain of the upstream.
+            port (int): The port of the upstream.
+            is_https (bool): Whether the upstream works on HTTPS.
+            gateway_id (str): The ID of the gateway to which the upstream belongs.
+            upstream_name (str): The name of the upstream.
+        Returns:
+            str: The ID of the created upstream.
+        """
+
+        request_body = {
+            "Name": upstream_name,
+            "GatewayId": gateway_id,
+            "SourceType": "Domain",
+            "UpstreamSpec": {
+                "Domain": {"DomainList": [{"Domain": domain, "Port": port}]}
+            },
+        }
+        if is_https:
+            request_body["TlsSettings"] = {"TlsMode": "SIMPLE", "Sni": domain}
+        else:
+            request_body["TlsSettings"] = {"TlsMode": "DISABLE"}
+
+        response = ve_request(
+            request_body=request_body,
+            action="CreateUpstream",
+            ak=self.ak,
+            sk=self.sk,
+            service="apig",
+            version="2021-03-03",
+            region=self.region,
+            host="open.volcengineapi.com",
+        )
+
+        try:
+            return response["Result"]["Id"]
+        except Exception as _:
+            raise ValueError(f"Create domain upstream failed: {response}")
+
+    def check_domain_upstream_exist(
+        self, domain: str, port: int, gateway_id: str
+    ) -> str | None:
+        """
+        Check whether the domain upstream exists.
+        Args:
+            domain (str): The domain of the upstream.
+            port (int): The port of the upstream.
+            gateway_id (str): The ID of the gateway to which the upstream belongs.
+        Returns:
+            str | None: The ID of the existed upstream or None if no upstream exists.
+        """
+
+        request_body = {
+            "GatewayId": gateway_id,
+            "UpstreamSpec": {
+                "Domain": {"DomainList": [{"Domain": domain, "Port": port}]}
+            },
+        }
+
+        response = ve_request(
+            request_body=request_body,
+            action="CheckUpstreamSpecExist",
+            ak=self.ak,
+            sk=self.sk,
+            service="apig",
+            version="2021-03-03",
+            region=self.region,
+            host="open.volcengineapi.com",
+        )
+
+        try:
+            exist = response["Result"]["Exist"]
+            if exist:
+                return response["Result"]["Id"]
+            else:
+                return None
+        except Exception as _:
+            raise ValueError(f"Check domain upstream spec exist failed: {response}")
+
+    def create_gateway_service_routes(
+        self, service_id: str, upstream_id: str, route_name: str, match_rule: dict
+    ):
+        """
+        Create gateway service routes.
+
+        Args:
+            service_id (str): The ID of the gateway service, used to specify the target service for which the route is to be created.
+            upstream_id (str): The ID of the upstream service, to which the route will point.
+            route_name (str): The name of the route to be created.
+            match_rule (dict): The route matching rule, containing the following key - value pairs:
+                - match_content (str): The path matching content, a string like "/abc", used to specify the path to be matched.
+                - match_type (str): The path matching type, with optional values "Exact", "Regex", "Prefix".
+                - match_method (list[str]): The list of HTTP request methods, possible values include "GET", "POST", etc.
+        Returns:
+            str: The ID of the created route.
+        """
+        from volcenginesdkapig20221112 import (
+            CreateRouteRequest,
+            MatchRuleForCreateRouteInput,
+            PathForCreateRouteInput,
+        )
+
+        match_content: str = match_rule["match_content"]
+        match_type: str = match_rule["match_type"]
+        match_method: list[str] = match_rule["match_method"]
+
+        request = CreateRouteRequest(
+            service_id=service_id,
+            enable=True,
+            match_rule=MatchRuleForCreateRouteInput(
+                path=PathForCreateRouteInput(
+                    match_content=match_content, match_type=match_type
+                ),
+                method=match_method,
+            ),
+            name=route_name,
+            priority=1,
+            upstream_list=[
+                UpstreamListForCreateRouteInput(
+                    upstream_id=upstream_id,
+                    weight=1,
+                )
+            ],
+        )
+
+        thread = self.apig_20221112_client.create_route(request, async_req=True)
+        result = thread.get()
+        return result.to_dict()["id"]
+
+    def create_plugin_binding(
+        self, scope: str, target: str, plugin_name: str, plugin_config: str
+    ) -> str:
+        """
+        Create a plugin binding.
+        Args:
+            scope (str): The type of the target.
+                Choices are 'GATEWAY', 'SERVICE' or 'ROUTE'.
+            target (str): The ID of the gateway, service or route.
+            plugin_name (str): The name of the plugin.
+            plugin_config (str): The config of the plugin.
+        Returns:
+            str: The ID of the created service.
+        """
+
+        from volcenginesdkapig import CreatePluginBindingRequest
+
+        request = CreatePluginBindingRequest(
+            scope=scope,
+            target=target,
+            plugin_name=plugin_name,
+            plugin_config=plugin_config,
+            enable=True,
+        )
+        thread = self.apig_client.create_plugin_binding(request, async_req=True)
+        result = thread.get()
+        return result.to_dict()["id"]
+
+    def create(
+        self,
+        function_id: str,
+        apig_instance_name: str,
+        service_name: str,
+        upstream_name: str,
+        routes: list[dict],
+    ):
+        """
+        Create an API gateway instance, service, and multiple routes.
+
+        Args:
+            function_id (str): The ID of the function to be associated with the routes.
+            apig_instance_name (str): The name of the API gateway instance.
+            service_name (str): The name of the service to be created.
+            upstream_name (str): The name of the upstream service to be created.
+            routes (list[dict]): A list of route configurations. Each dictionary in the list contains the following key - value pairs:
+                - route_name (str): The name of the route to be created.
+                - match_content (str): The path matching content, a string like "/abc", used to specify the path to be matched.
+                - match_type (str): The path matching type, with optional values "Exact", "Regex", "Prefix".
+                - match_method (list[str]): The list of HTTP request methods, possible values include "GET", "POST", etc.
+
+        Returns:
+            dict: A dictionary containing the IDs of the created gateway, service, upstream, and routes.
+        """
+        gateway_id = self.create_serverless_gateway(apig_instance_name)
+        service_id = self.create_gateway_service(gateway_id, service_name)
+        upstream_id = self.create_vefaas_upstream(
+            function_id, gateway_id, upstream_name
+        )
+
+        route_ids = []
+        for route in routes:
+            route_name = route["route_name"]
+            match_rule = {
+                "match_content": route["match_content"],
+                "match_type": route["match_type"],
+                "match_method": route["match_method"],
+            }
+            route_id = self.create_gateway_service_routes(
+                service_id, upstream_id, route_name, match_rule
+            )
+            route_ids.append(route_id)
+
+        return {
+            "gateway_id": gateway_id,
+            "service_id": service_id,
+            "upstream_id": upstream_id,
+            "route_ids": route_ids,
+        }