veadk-python 0.1.0__py3-none-any.whl

veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from deepeval import evaluate
+from deepeval.evaluate.types import EvaluationResult
+from deepeval.key_handler import KEY_FILE_HANDLER, KeyValues
+from deepeval.metrics import BaseMetric
+from deepeval.models import LocalModel
+from deepeval.test_case import LLMTestCase
+from deepeval.test_case.llm_test_case import ToolCall
+from typing_extensions import override
+
+from veadk.config import getenv
+from veadk.utils.logger import get_logger
+
+from ..base_evaluator import BaseEvaluator, EvalResultData, MetricResult
+from ..utils.prometheus import (
+    EvalResultCaseData,
+    EvalResultMetadata,
+    PrometheusPushgatewayConfig,
+    push_to_prometheus,
+)
+
+logger = get_logger(__name__)
+
+
+def formatted_timestamp():
+    # YYYYMMDDHHMMSS
+    return time.strftime("%Y%m%d%H%M%S", time.localtime())
+
+
+class DeepevalEvaluator(BaseEvaluator):
+    def __init__(
+        self,
+        agent,
+        judge_model_api_key: str = getenv("MODEL_JUDGE_API_KEY"),
+        judge_model_name: str = getenv(
+            "MODEL_JUDGE_NAME",
+            "doubao-seed-1-6-250615",
+        ),
+        judge_model_api_base: str = getenv(
+            "MODEL_JUDGE_API_BASE",
+            "https://ark.cn-beijing.volces.com/api/v3/",
+        ),
+        name: str = "veadk_deepeval_evaluator",
+        prometheus_config: PrometheusPushgatewayConfig = None,
+    ):
+        super().__init__(agent=agent, name=name)
+
+        self.judge_model_name = judge_model_name
+        self.judge_model = self.create_judge_model(
+            model_name=judge_model_name,
+            api_key=judge_model_api_key,
+            api_base=judge_model_api_base,
+        )
+
+        self.prometheus_config = prometheus_config
+
+    def create_judge_model(
+        self,
+        model_name: str,
+        api_key: str,
+        api_base: str,
+    ):
+        KEY_FILE_HANDLER.write_key(KeyValues.LOCAL_MODEL_NAME, model_name)
+        KEY_FILE_HANDLER.write_key(KeyValues.LOCAL_MODEL_BASE_URL, api_base)
+        KEY_FILE_HANDLER.write_key(KeyValues.LOCAL_MODEL_API_KEY, api_key)
+        KEY_FILE_HANDLER.write_key(KeyValues.USE_LOCAL_MODEL, "YES")
+        KEY_FILE_HANDLER.write_key(KeyValues.USE_AZURE_OPENAI, "NO")
+        return LocalModel()
+
+    @override
+    async def eval(
+        self,
+        eval_set_file_path: str,
+        metrics: list[BaseMetric],
+        eval_id: str = f"test_{formatted_timestamp()}",
+    ):
+        """Target to Google ADK, we will use the same evaluation case format as Google ADK."""
+
+        # Get evaluation data by parsing eval set file
+        self.generate_eval_data(eval_set_file_path)
+        # Get actual data by running agent
+        logger.info("Start to run agent for actual data.")
+        await self._run_agent_for_actual_data()
+        eval_case_data_list = self.invocation_list
+
+        # Build test cases in Deepeval format
+        logger.info("Start to build test cases in Deepeval format.")
+        test_cases = []
+        for eval_case_data in eval_case_data_list:
+            for invocation in eval_case_data.invocations:
+                invocations_context_actual: str = (
+                    ""  # {"role": "user", "content": "xxxxx"}
+                )
+                invocations_context_expect: str = ""
+
+                test_case = LLMTestCase(
+                    input=invocation.input,
+                    actual_output=invocation.actual_output,
+                    expected_output=invocation.expected_output,
+                    tools_called=[
+                        ToolCall(name=tool["name"], input_parameters=tool["args"])
+                        for tool in invocation.actual_tool
+                    ],
+                    expected_tools=[
+                        ToolCall(name=tool["name"], input_parameters=tool["args"])
+                        for tool in invocation.expected_tool
+                    ],
+                    additional_metadata={"latency": invocation.latency},
+                    context=[
+                        "actual_conversation_history: "
+                        + (invocations_context_actual or "Empty"),
+                        "expect_conversation_history: "
+                        + (invocations_context_expect or "Empty"),
+                    ],
+                )
+                invocations_context_actual += (
+                    f'{{"role": "user", "content": "{invocation.input}"}}\n'
+                )
+                invocations_context_actual += f'{{"role": "assistant", "content": "{invocation.actual_output}"}}\n'
+                invocations_context_expect += (
+                    f'{{"role": "user", "content": "{invocation.input}"}}\n'
+                )
+                invocations_context_expect += f'{{"role": "assistant", "content": "{invocation.expected_output}"}}\n'
+
+                test_cases.append(test_case)
+
+        # Run Deepeval evaluation according to metrics
+        logger.info("Start to run Deepeval evaluation according to metrics.")
+        test_results = evaluate(test_cases=test_cases, metrics=metrics)
+        for test_result in test_results.test_results:
+            eval_result_data = EvalResultData(metric_results=[])
+            for metrics_data_item in test_result.metrics_data:
+                metric_result = MetricResult(
+                    metric_type=metrics_data_item.name,
+                    success=metrics_data_item.success,
+                    score=metrics_data_item.score,
+                    reason=metrics_data_item.reason,
+                )
+                eval_result_data.metric_results.append(metric_result)
+
+            eval_result_data.call_before_append()  # calculate average score and generate total reason
+            self.result_list.append(eval_result_data)
+            self.result_list.reverse()  # deepeval test_results is in reverse order
+
+        # export to Prometheus if needed
+        if self.prometheus_config is not None:
+            self.export_results(eval_id, test_results)
+
+        return test_results
+
+    def export_results(self, eval_id: str, test_results: EvaluationResult):
+        # fixed attributions
+        test_name = eval_id
+        test_cases_total = len(test_results.test_results)
+        eval_data = EvalResultMetadata(
+            tested_model=self.agent.model_name,
+            judge_model=self.judge_model_name,
+        )
+        # parsed attributions
+        test_cases_failure = 0
+        test_cases_pass = 0
+        test_data_list = []
+        # NOTE: we hard-coding the following two attributions for development
+        case_threshold = 0.5
+        diff_threshold = 0.2
+
+        for idx, test_result in enumerate(test_results.test_results):
+            pass_flag = "PASSED"
+            if test_result.success:
+                test_cases_pass += 1
+            else:
+                pass_flag = "FAILURE"
+                test_cases_failure += 1
+
+            test_data_list.append(
+                EvalResultCaseData(
+                    id=str(idx),
+                    input=test_result.input,
+                    actual_output=test_result.actual_output,
+                    expected_output=test_result.expected_output,
+                    # [temporary] score: This method is not generally applicable now and is currently only available in the GEval mode.
+                    score=str(test_result.metrics_data[0].score),
+                    reason=test_result.metrics_data[0].reason,
+                    status=pass_flag,
+                    latency=test_result.additional_metadata["latency"],
+                )
+            )
+
+        exported_data = {
+            "test_name": test_name,
+            "test_cases_total": test_cases_total,
+            "test_cases_failure": test_cases_failure,
+            "test_cases_pass": test_cases_pass,
+            "test_data_list": test_data_list,
+            "eval_data": eval_data,
+            "case_threshold": case_threshold,
+            "diff_threshold": diff_threshold,
+        }
+
+        push_to_prometheus(
+            **exported_data,
+            url=self.prometheus_config.url,
+            username=self.prometheus_config.username,
+            password=self.prometheus_config.password,
+        )
+        logger.info(
+            f"Upload to Prometheus Pushgateway ({self.prometheus_config.url}) successfully! Test name: {eval_id}"
+        )

veadk/evaluation/eval_set_file_loader.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.local_eval_sets_manager import (
+    load_eval_set_from_file as adk_load_eval_set_from_file,
+)
+
+
+def load_eval_set_from_file(eval_set_file_path: str) -> EvalSet:
+    try:
+        eval_set = adk_load_eval_set_from_file(eval_set_file_path, eval_set_file_path)
+    except Exception as e:
+        raise Exception(
+            f"Failed to load eval set from file {eval_set_file_path}"
+        ) from e
+    return eval_set

veadk/evaluation/eval_set_recorder.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from pathlib import Path
+
+from google.adk.cli.utils import evals
+from google.adk.evaluation.eval_case import EvalCase, SessionInput
+from google.adk.evaluation.local_eval_sets_manager import LocalEvalSetsManager
+from google.adk.sessions import BaseSessionService
+
+from veadk.utils.logger import get_logger
+from veadk.utils.misc import formatted_timestamp
+
+logger = get_logger(__name__)
+
+
+class EvalSetRecorder(LocalEvalSetsManager):
+    def __init__(
+        self, session_service: BaseSessionService, eval_set_id: str = "default"
+    ):
+        super().__init__(agents_dir="/tmp/")
+        self.eval_set_id = eval_set_id if eval_set_id != "" else "default"
+        self.session_service: BaseSessionService = session_service
+
+    # adapted from google.adk.cli.fast_api
+    async def add_session_to_eval_set(
+        self,
+        app_name: str,
+        eval_set_id: str,
+        session_id: str,
+        user_id: str,
+    ):
+        eval_id = f"veadk_eval_{formatted_timestamp()}"
+
+        # Get the session
+        session = await self.session_service.get_session(
+            app_name=app_name, user_id=user_id, session_id=session_id
+        )
+        assert session, "Session not found."
+
+        # Convert the session data to eval invocations
+        invocations = evals.convert_session_to_eval_invocations(session)
+
+        # Populate the session with initial session state.
+        # initial_session_state = create_empty_state(agent_loader.load_agent(app_name))
+
+        new_eval_case = EvalCase(
+            eval_id=eval_id,
+            conversation=invocations,
+            session_input=SessionInput(app_name=app_name, user_id=user_id),
+            creation_timestamp=time.time(),
+        )
+
+        try:
+            self.add_eval_case(app_name, eval_set_id, new_eval_case)
+        except ValueError as ve:
+            raise ValueError(f"Add eval case to eval set error: {ve}")
+
+    async def dump(
+        self,
+        app_name: str,
+        user_id: str,
+        session_id: str,
+    ) -> str:
+        dump_path = self._get_eval_set_file_path(app_name, self.eval_set_id)
+        Path(dump_path).parent.mkdir(parents=True, exist_ok=True)
+
+        self.create_eval_set(app_name=app_name, eval_set_id=self.eval_set_id)
+
+        await self.add_session_to_eval_set(
+            app_name=app_name,
+            eval_set_id=self.eval_set_id,
+            session_id=session_id,
+            user_id=user_id,
+        )
+
+        logger.info(f"Dump eval set to {dump_path}")
+
+        return dump_path

veadk/evaluation/utils/prometheus.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+from prometheus_client.exposition import basic_auth_handler
+
+from veadk.config import getenv
+
+
+@dataclass
+class EvalResultCaseData:
+    id: str
+    input: str
+    actual_output: str
+    expected_output: str
+    score: str
+    reason: str
+    status: str  # `PASSED` or `FAILURE`
+    latency: str
+
+
+@dataclass
+class EvalResultMetadata:
+    tested_model: str
+    judge_model: str
+
+
+class PrometheusPushgatewayConfig:
+    url: str = getenv(
+        "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL",
+    )
+    username: str = getenv("OBSERVABILITY_PROMETHEUS_USERNAME")
+    password: str = getenv("OBSERVABILITY_PROMETHEUS_PASSWORD")
+
+
+registry = CollectorRegistry()
+
+test_cases_total_metric = Gauge(
+    "test_cases_total",
+    "Total number of test cases in this evaluation",
+    registry=registry,
+)
+
+test_cases_success_metric = Gauge(
+    "test_cases_success", "Success number of test cases", registry=registry
+)
+
+test_cases_pass_metric = Gauge(
+    "test_cases_pass", "Passed number of test cases", registry=registry
+)
+
+test_cases_failure_metric = Gauge(
+    "test_cases_failure", "Failuer number of test cases", registry=registry
+)
+
+case_threshold_metric = Gauge("threshold", "Threshold of test cases", registry=registry)
+diff_threshold_metric = Gauge(
+    "diff_threshold", "Diff threshold of test cases", registry=registry
+)
+
+test_cases_data_metric = Gauge(
+    "test_cases_data",
+    "Specific data of test cases",
+    registry=registry,
+    labelnames=["data"],
+)
+
+eval_data_metric = Gauge(
+    "eval_data",
+    "Specific data of evaluation",
+    registry=registry,
+    labelnames=["data"],
+)
+
+
+def post_pushgateway(
+    pushgateway_url: str,
+    username: str,
+    password: str,
+    job_name: str,
+    registry: CollectorRegistry,
+    grouping_key: dict[str, str] = None,
+):
+    def auth_handler(url, method, timeout, headers, data):
+        return basic_auth_handler(
+            url, method, timeout, headers, data, username, password
+        )
+
+    push_to_gateway(
+        gateway=pushgateway_url,
+        job=job_name,
+        registry=registry,
+        grouping_key=grouping_key,
+        handler=auth_handler,
+    )
+
+
+def push_to_prometheus(
+    test_name: str,
+    test_cases_total: int,
+    test_cases_failure: int,
+    test_cases_pass: int,
+    test_data_list: list[EvalResultCaseData],
+    eval_data: EvalResultMetadata,
+    case_threshold: float = 0.5,
+    diff_threshold: float = 0.2,
+    url: str = "",
+    username: str = "",
+    password: str = "",
+):
+    test_cases_total_metric.set(test_cases_total)
+    test_cases_failure_metric.set(test_cases_failure)
+    test_cases_pass_metric.set(test_cases_pass)
+
+    for test_data in test_data_list:
+        test_cases_data_metric.labels(data=str(test_data.__dict__)).set(1)
+
+    eval_data_metric.labels(data=str(eval_data.__dict__)).set(1)
+    case_threshold_metric.set(case_threshold)
+    diff_threshold_metric.set(diff_threshold)
+
+    post_pushgateway(
+        pushgateway_url=url,
+        username=username,
+        password=password,
+        job_name="veadk_eval_job",
+        registry=registry,
+        grouping_key={"test_name": test_name},
+    )

veadk/knowledgebase/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .knowledgebase import KnowledgeBase
+
+__all__ = ["KnowledgeBase"]

veadk/knowledgebase/knowledgebase.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import BinaryIO, Literal, TextIO
+
+from veadk.database.database_factory import DatabaseFactory
+from veadk.utils.logger import get_logger
+
+from .knowledgebase_database_adapter import get_knowledgebase_adapter
+
+logger = get_logger(__name__)
+
+
+class KnowledgeBase:
+    def __init__(
+        self,
+        backend: Literal["local", "opensearch", "viking", "redis", "mysql"] = "local",
+        top_k: int = 5,
+        db_config=None,
+    ):
+        logger.debug(f"Create knowledgebase, backend is {backend}")
+        self.backend = backend
+        self.top_k = top_k
+
+        self.db_client = DatabaseFactory.create(backend=backend, config=db_config)
+        self.adapter = get_knowledgebase_adapter(backend)(
+            database_client=self.db_client
+        )
+
+    def add(
+        self,
+        data: str | list[str] | TextIO | BinaryIO | bytes,
+        app_name: str,
+        **kwargs,
+    ):
+        """
+        Add documents to the vector database.
+        You can only upload files or file characters when the adapter type you use is vikingdb.
+        In addition, if you upload data of the bytes type,
+            for example, if you read the file stream of a pdf, then you need to pass an additional parameter file_ext = '.pdf'.
+        """
+        kwargs.pop("session_id", None)  # remove session_id
+        self.adapter.add(
+            data, app_name, user_id="user_id", session_id="session_id", **kwargs
+        )
+
+    def search(self, query: str, app_name: str, top_k: int = None) -> list[str]:
+        """Retrieve documents similar to the query text in the vector database.
+
+        Args:
+            query (str): The query text to be retrieved (e.g., "Who proposed the Turing machine model?")
+
+        Returns:
+            list[str]: A list of the top most similar document contents retrieved (sorted by vector similarity)
+        """
+        top_k = self.top_k if top_k is None else top_k
+
+        result = self.adapter.query(
+            query=query, app_name=app_name, user_id="user_id", top_k=top_k
+        )
+        if len(result) == 0:
+            logger.warning(f"No documents found in knowledgebase. Query: {query}")
+        return result
+
+    def delete(self, app_name: str, user_id: str, session_id: str):
+        """Delete documents in the vector database.
+        Args:
+            app_name (str): The name of the application
+            user_id (str): The user ID
+            session_id (str): The session ID
+        """
+        self.adapter.delete(app_name=app_name, user_id=user_id, session_id=session_id)