veadk-python 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

veadk/evaluation/base_evaluator.py

@@ -17,7 +17,7 @@ import json
 import time
 import uuid
 from abc import abstractmethod
-from typing import Any
+from typing import Any, Optional
 
 from google.adk import Runner
 from google.adk.evaluation.eval_set import EvalSet
@@ -120,56 +120,72 @@ class BaseEvaluator:
             # Extract tool_uses from spans with name starting with "execute_tool"
             for span in spans:
                 if span["name"].startswith("execute_tool"):
+                    # Extract tool parameters from gen_ai.tool.input
+                    tool_input_str = span["attributes"].get("gen_ai.tool.input", "{}")
+                    try:
+                        tool_input = json.loads(tool_input_str)
+                        tool_args = tool_input.get("parameters", {})
+                    except json.JSONDecodeError:
+                        tool_args = {}
+
+                    # Extract the tool call ID from gen_ai.tool.output
+                    tool_output_str = span["attributes"].get("gen_ai.tool.output", "{}")
+                    tool_call_id = None
+                    try:
+                        tool_output = json.loads(tool_output_str)
+                        tool_call_id = tool_output.get("id", None)
+                    except json.JSONDecodeError:
+                        tool_call_id = None
+
                     tool_uses.append(
                         {
-                            "id": span["attributes"].get("gen_ai.tool.call.id", None),
-                            "args": json.loads(
-                                span["attributes"].get(
-                                    "gcp.vertex.agent.tool_call_args", "{}"
-                                )
-                            ),
+                            "id": tool_call_id,
+                            "args": tool_args,
                             "name": span["attributes"].get("gen_ai.tool.name", None),
                         }
                     )
 
-            # Extract conversation data from spans with name starting with "invocation"
-            for span in spans:
-                if span["name"].startswith("invocation"):
-                    # Parse input.value and output.value as JSON
-                    input_value = json.loads(
-                        span["attributes"].get("input.value", "{}")
-                    )
-                    output_value = json.loads(
-                        span["attributes"].get("output.value", "{}")
-                    )
+            # Extract conversation data from call_llm spans
+            user_input = ""
+            final_output = ""
 
-                    user_content = json.loads(input_value.get("new_message", {}))
-                    final_response = json.loads(json.dumps(user_content))
-                    final_response["parts"][0]["text"] = (
-                        output_value.get("content", {})
-                        .get("parts", [{}])[0]
-                        .get("text", None)
-                    )
-                    final_response["role"] = None
-                    conversation.append(
-                        {
-                            "invocation_id": output_value.get(
-                                "invocation_id", str(uuid.uuid4())
-                            ),
-                            "user_content": user_content,
-                            "final_response": final_response,
-                            "intermediate_data": {
-                                "tool_uses": tool_uses,
-                                "intermediate_responses": [],
-                            },
-                            "creation_timestamp": span["start_time"] / 1e9,
-                        }
-                    )
-                    user_id = input_value.get("user_id", None)
-                    app_name = (
-                        span["name"].replace("invocation", "").strip().strip("[]")
-                    )
-                    creation_timestamp = span["start_time"] / 1e9
+            # Find the first call_llm span for user input and the last one for final output
+            call_llm_spans = [span for span in spans if span["name"] == "call_llm"]
+
+            if call_llm_spans:
+                # Get user input from the first call_llm span
+                first_span = call_llm_spans[0]
+                user_input = first_span["attributes"].get("gen_ai.prompt.0.content", "")
+
+                # Get final output from the last call_llm span
+                last_span = call_llm_spans[-1]
+                final_output = last_span["attributes"].get(
+                    "gen_ai.completion.0.content", ""
+                )
+
+                # Get metadata from any span
+                app_name = first_span["attributes"].get("gen_ai.app.name", "")
+                user_id = first_span["attributes"].get("gen_ai.user.id", "")
+                creation_timestamp = first_span["start_time"] / 1e9
+
+            if user_input and final_output:
+                # Create user_content and final_response in the expected format
+                user_content = {"role": "user", "parts": [{"text": user_input}]}
+
+                final_response = {"role": "model", "parts": [{"text": final_output}]}
+
+                conversation.append(
+                    {
+                        "invocation_id": str(uuid.uuid4()),
+                        "user_content": user_content,
+                        "final_response": final_response,
+                        "intermediate_data": {
+                            "tool_uses": tool_uses,
+                            "intermediate_responses": [],
+                        },
+                        "creation_timestamp": creation_timestamp,
+                    }
+                )
 
         eval_cases.append(
             {
@@ -194,33 +210,43 @@ class BaseEvaluator:
 
         return evalset
 
-    def build_eval_set(self, file_path: str):
+    def build_eval_set(
+        self, eval_set: Optional[EvalSet] = None, file_path: Optional[str] = None
+    ):
         """Generate evaluation data from a given file and assign it to the class attribute `invocation_list`."""
-        eval_case_data_list: list[EvalTestCase] = []
 
-        try:
-            with open(file_path, "r") as f:
-                file_content = json.load(f)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON format in file {file_path}: {e}")
-        except Exception as e:
-            raise ValueError(f"Error reading file {file_path}: {e}")
-
-        if isinstance(file_content, dict) and "eval_cases" in file_content:
-            eval_cases = self._build_eval_set_from_eval_json(file_path).eval_cases
-        elif (
-            isinstance(file_content, list)
-            and len(file_content) > 0
-            and all(
-                isinstance(span, dict) and "trace_id" in span for span in file_content
-            )
-        ):
-            eval_cases = self._build_eval_set_from_tracing_json(file_path).eval_cases
+        if eval_set is None and file_path is None:
+            raise ValueError("eval_set or file_path is required")
+        if eval_set:
+            eval_cases = eval_set.eval_cases
         else:
-            raise ValueError(
-                f"Unsupported file format in {file_path}. Please provide a valid file."
-            )
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    file_content = json.load(f)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON format in file {file_path}: {e}")
+            except Exception as e:
+                raise ValueError(f"Error reading file {file_path}: {e}")
+
+            if isinstance(file_content, dict) and "eval_cases" in file_content:
+                eval_cases = self._build_eval_set_from_eval_json(file_path).eval_cases
+            elif (
+                isinstance(file_content, list)
+                and len(file_content) > 0
+                and all(
+                    isinstance(span, dict) and "trace_id" in span
+                    for span in file_content
+                )
+            ):
+                eval_cases = self._build_eval_set_from_tracing_json(
+                    file_path
+                ).eval_cases
+            else:
+                raise ValueError(
+                    f"Unsupported file format in {file_path}. Please provide a valid file."
+                )
 
+        eval_case_data_list: list[EvalTestCase] = []
         for eval_case in eval_cases:
             eval_case_data = EvalTestCase(invocations=[])
             if eval_case.session_input:
@@ -368,8 +394,9 @@ class BaseEvaluator:
     @abstractmethod
     async def evaluate(
         self,
-        eval_set_file_path: str,
         metrics: list[Any],
+        eval_set: Optional[EvalSet],
+        eval_set_file_path: Optional[str],
         eval_id: str,
     ):
         """An abstract method for evaluation based on metrics。"""

veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py

@@ -22,16 +22,16 @@ from deepeval.models import LocalModel
 from deepeval.test_case import LLMTestCase
 from deepeval.test_case.llm_test_case import ToolCall
 from typing_extensions import override
-
+from typing import Optional
+from google.adk.evaluation.eval_set import EvalSet
 from veadk.config import getenv
-from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
-from veadk.utils.logger import get_logger
-
 from veadk.evaluation.base_evaluator import BaseEvaluator, EvalResultData, MetricResult
+from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
 from veadk.evaluation.utils.prometheus import (
     PrometheusPushgatewayConfig,
     push_to_prometheus,
 )
+from veadk.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
@@ -45,20 +45,27 @@ class DeepevalEvaluator(BaseEvaluator):
     def __init__(
         self,
         agent,
-        judge_model_api_key: str = getenv("MODEL_JUDGE_API_KEY"),
-        judge_model_name: str = getenv(
-            "MODEL_JUDGE_NAME",
-            "doubao-seed-1-6-250615",
-        ),
-        judge_model_api_base: str = getenv(
-            "MODEL_JUDGE_API_BASE",
-            "https://ark.cn-beijing.volces.com/api/v3/",
-        ),
+        judge_model_api_key: str = "",
+        judge_model_name: str = "",
+        judge_model_api_base: str = "",
         name: str = "veadk_deepeval_evaluator",
         prometheus_config: PrometheusPushgatewayConfig | None = None,
     ):
         super().__init__(agent=agent, name=name)
 
+        if not judge_model_api_key:
+            judge_model_api_key = getenv("MODEL_JUDGE_API_KEY")
+        if not judge_model_name:
+            judge_model_name = getenv(
+                "MODEL_JUDGE_NAME",
+                "doubao-seed-1-6-250615",
+            )
+        if not judge_model_api_base:
+            judge_model_api_base = getenv(
+                "MODEL_JUDGE_API_BASE",
+                "https://ark.cn-beijing.volces.com/api/v3/",
+            )
+
         self.judge_model_name = judge_model_name
         self.judge_model = LocalModel(
             model=judge_model_name,
@@ -71,13 +78,14 @@ class DeepevalEvaluator(BaseEvaluator):
     @override
     async def evaluate(
         self,
-        eval_set_file_path: str,
         metrics: list[BaseMetric],
+        eval_set: Optional[EvalSet] = None,
+        eval_set_file_path: Optional[str] = None,
         eval_id: str = f"test_{formatted_timestamp()}",
     ):
         """Target to Google ADK, we will use the same evaluation case format as Google ADK."""
         # Get evaluation data by parsing eval set file
-        self.build_eval_set(eval_set_file_path)
+        self.build_eval_set(eval_set, eval_set_file_path)
 
         # Get actual data by running agent
         logger.info("Start to run agent for actual data.")

veadk/evaluation/eval_set_recorder.py

@@ -21,7 +21,7 @@ from google.adk.evaluation.local_eval_sets_manager import LocalEvalSetsManager
 from google.adk.sessions import BaseSessionService
 
 from veadk.utils.logger import get_logger
-from veadk.utils.misc import formatted_timestamp
+from veadk.utils.misc import formatted_timestamp, get_temp_dir
 
 logger = get_logger(__name__)
 
@@ -30,7 +30,7 @@ class EvalSetRecorder(LocalEvalSetsManager):
     def __init__(
         self, session_service: BaseSessionService, eval_set_id: str = "default"
     ):
-        super().__init__(agents_dir="/tmp/")
+        super().__init__(agents_dir=get_temp_dir())
         self.eval_set_id = eval_set_id if eval_set_id != "" else "default"
         self.session_service: BaseSessionService = session_service
 

veadk/integrations/ve_prompt_pilot/ve_prompt_pilot.py

@@ -16,7 +16,6 @@ from __future__ import annotations
 
 import agent_pilot as ap
 from agent_pilot.models import TaskType
-
 from veadk import Agent
 from veadk.prompts import prompt_optimization
 from veadk.utils.logger import get_logger
@@ -26,9 +25,15 @@ logger = get_logger(__name__)
 
 class VePromptPilot:
     def __init__(
-        self, api_key: str, path: str = "", task_id: str | None = None
+        self,
+        api_key: str,
+        workspace_id: str,
+        path: str = "",
+        task_id: str | None = None,
     ) -> None:
         self.api_key = api_key
+        self.workspace_id = workspace_id
+
         self.path = path
 
     def optimize(
@@ -57,12 +62,13 @@ class VePromptPilot:
             usage = None
             for chunk in ap.generate_prompt_stream(
                 task_description=task_description,
-                current_prompt=agent.instruction,
+                current_prompt=str(agent.instruction),
                 model_name=model_name,
                 task_type=TaskType.DIALOG,
                 temperature=1.0,
                 top_p=0.7,
                 api_key=self.api_key,
+                workspace_id=self.workspace_id,
             ):  # stream chunks of optimized prompt
                 # Process each chunk as it arrives
                 optimized_prompt += chunk.data.content if chunk.data else ""

veadk/integrations/ve_tls/utils.py

@@ -17,6 +17,7 @@ import json
 from typing import Any, Literal
 
 import httpx
+from veadk.utils.logger import get_logger
 from volcengine.ApiInfo import ApiInfo
 from volcengine.auth.SignerV4 import SignerV4
 from volcengine.tls.const import (
@@ -28,8 +29,6 @@ from volcengine.tls.const import (
 )
 from volcengine.tls.TLSService import TLSService
 
-from veadk.utils.logger import get_logger
-
 logger = get_logger(__name__)
 
 HEADER_API_VERSION = "x-tls-apiversion"

veadk/integrations/ve_tls/ve_tls.py

@@ -28,9 +28,8 @@ class VeTLS:
         region: str = "cn-beijing",
     ):
         try:
-            from volcengine.tls.TLSService import TLSService
-
             from veadk.integrations.ve_tls.utils import ve_tls_request
+            from volcengine.tls.TLSService import TLSService
         except ImportError:
             raise ImportError(
                 "Please install volcengine SDK before init VeTLS: pip install volcengine"
@@ -162,11 +161,12 @@ class VeTLS:
                 client=self._client,
                 api="CreateTraceInstance",
                 body=request_body,
+                request_headers={"TraceTag": "veadk"},
             )
 
-            if res["ErrorCode"] == "TopicAlreadyExist":
+            if "ErrorCode" in res and res["ErrorCode"] == "TopicAlreadyExist":
                 logger.debug(
-                    f"Log project '{trace_instance_name}' already exists. Check its ID."
+                    f"Tracing instance '{trace_instance_name}' already exists. Check its ID."
                 )
                 return self.get_trace_instance_by_name(
                     log_project_id, trace_instance_name
@@ -176,10 +176,14 @@ class VeTLS:
             res = self._ve_tls_request(
                 client=self._client,
                 api="DescribeTraceInstance",
-                body={"TraceInstanceID": res["TraceInstanceID"]},
+                params={"TraceInstanceId": res["TraceInstanceId"]},
                 method="GET",
             )
 
+            logger.info(
+                f"Create tracing instance finished, tracing instance ID: {res['TraceInstanceId']}"
+            )
+
             return res
         except KeyError:
             raise ValueError(f"Failed to create tracing instance: {res}")