veadk-python 0.2.16__py3-none-any.whl → 0.2.17__py3-none-any.whl

veadk/evaluation/base_evaluator.py

@@ -29,12 +29,44 @@ from veadk.utils.misc import formatted_timestamp
 
 
 class ToolInvocation(BaseModel):
+    """Represents a single tool invocation in agent execution.
+
+    This model holds tool name, arguments, and result.
+    Used in tracking tool usage during evaluation.
+
+    Attributes:
+        tool_name (str): Name of the tool called.
+        tool_args (dict[str, Any]): Arguments passed to the tool. Defaults to empty dict.
+        tool_result (Any): Result from tool execution. Defaults to None.
+
+    Note:
+        Flexible for various tool types and results.
+    """
+
     tool_name: str
     tool_args: dict[str, Any] = {}
     tool_result: Any = None
 
 
 class Invocation(BaseModel):
+    """Models a single invocation in the evaluation process.
+
+    This class stores input, expected and actual outputs, tools, and latency.
+    Essential for comparing agent behavior.
+
+    Attributes:
+        invocation_id (str): Unique ID for the invocation. Defaults to empty.
+        input (str): User input prompt.
+        actual_output (str): Actual response from agent.
+        expected_output (str): Expected response.
+        actual_tool (list[dict]): List of actual tools called with details.
+        expected_tool (list[dict]): List of expected tools.
+        latency (str): Execution time in ms. Defaults to empty.
+
+    Note:
+        Tools are dicts with 'name' and 'args'.
+    """
+
     invocation_id: str = ""
     input: str
     actual_output: str
@@ -45,10 +77,37 @@ class Invocation(BaseModel):
 
 
 class EvalTestCase(BaseModel):
+    """Groups invocations for a single test case.
+
+    This model contains a list of invocations for one evaluation scenario.
+    Used to structure test data.
+
+    Attributes:
+        invocations (list[Invocation]): List of invocation objects in the case.
+
+    Note:
+        Each case corresponds to one session or conversation.
+    """
+
     invocations: list[Invocation]
 
 
 class MetricResult(BaseModel):
+    """Stores result of a single metric evaluation.
+
+    This model holds the outcome of one metric application.
+    Includes success, score, and reason.
+
+    Attributes:
+        metric_type (str): Type or name of the metric.
+        success (bool): If the metric passed.
+        score (float): Numerical score from evaluation.
+        reason (str): Explanation for the score.
+
+    Note:
+        Score is float between 0 and 1 typically.
+    """
+
     metric_type: str
     success: bool
     score: float
@@ -56,32 +115,102 @@ class MetricResult(BaseModel):
 
 
 class EvalResultData(BaseModel):
+    """Aggregates metric results for an evaluation.
+
+    This class collects multiple metric results and computes averages.
+    Used for overall case scoring.
+
+    Attributes:
+        metric_results (list[MetricResult]): List of individual metric outcomes.
+        average_score (float): Computed average score. Defaults to 0.0.
+        total_reason (str): Combined reasons. Defaults to empty.
+
+    Note:
+        Call call_before_append to compute averages and reasons.
+    """
+
     metric_results: list[MetricResult]
     average_score: float = 0.0
     total_reason: str = ""
 
     def calculate_average_score(self):
+        """Calculates the average score from metric results.
+
+        This method sums scores and divides by count.
+        Updates average_score attribute.
+
+        Returns:
+            None: Updates internal state.
+
+        Raises:
+            ZeroDivisionError: If no metrics.
+        """
         total_score = sum(result.score for result in self.metric_results)
         self.average_score = (
             total_score / len(self.metric_results) if self.metric_results else 0.0
         )
 
     def generate_total_reason(self):
+        """Generates a combined reason string from all metrics.
+
+        This method joins reasons with metric types.
+        Updates total_reason attribute.
+
+        Returns:
+            None: Updates internal state.
+
+        Note:
+            Format: 'metric_type: reason'
+        """
         self.total_reason = "\n".join(
             f"{result.metric_type:}:{result.reason}" for result in self.metric_results
         )
 
     def call_before_append(self):
+        """Computes average score and total reason before adding to list.
+
+        This method calls calculate_average_score and generate_total_reason.
+        Ensures data is ready for storage.
+
+        Returns:
+            None: Updates internal state.
+        """
         self.calculate_average_score()
         self.generate_total_reason()
 
 
 class BaseEvaluator:
+    """Base class for all evaluators in the system.
+
+    This abstract class provides common functionality for evaluation.
+    Handles building eval sets, generating outputs, and abstract evaluate.
+
+    Attributes:
+        name (str): Name of the evaluator.
+        agent: The agent being evaluated.
+        invocation_list (list[EvalTestCase]): List of test cases.
+        result_list (list[EvalResultData]): List of evaluation results.
+        agent_information_list (list[dict]): List of agent config info.
+
+    Note:
+        Subclasses must implement evaluate method.
+        Supports JSON and tracing formats for input.
+    """
+
     def __init__(
         self,
         agent,
         name: str,
     ):
+        """Initializes the base evaluator with agent and name.
+
+        Args:
+            agent: Agent instance to evaluate.
+            name (str): Identifier for the evaluator.
+
+        Raises:
+            ValueError: If agent or name invalid.
+        """
         self.name = name
         self.agent = agent
         self.invocation_list: list[EvalTestCase] = []
@@ -89,11 +218,41 @@ class BaseEvaluator:
         self.agent_information_list: list[dict] = []
 
     def _build_eval_set_from_eval_json(self, eval_json_path: str) -> EvalSet:
+        """Builds eval set from standard eval JSON file.
+
+        This private method loads using file loader.
+
+        Args:
+            eval_json_path (str): Path to JSON file.
+
+        Returns:
+            EvalSet: Loaded set.
+
+        Raises:
+            ValueError: If loading fails.
+        """
         from veadk.evaluation.eval_set_file_loader import load_eval_set_from_file
 
         return load_eval_set_from_file(eval_json_path)
 
     def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
+        """Builds eval set from tracing JSON spans.
+
+        This private method parses spans, groups by trace, extracts tools and conversation.
+
+        Args:
+            tracing_json_path (str): Path to tracing JSON.
+
+        Returns:
+            EvalSet: Constructed set from traces.
+
+        Raises:
+            ValueError: If JSON invalid or parsing fails.
+            json.JSONDecodeError: For malformed JSON.
+
+        Note:
+            Assumes spans have gen_ai attributes for tools and content.
+        """
         try:
             with open(tracing_json_path, "r") as f:
                 tracing_data = json.load(f)
@@ -213,7 +372,21 @@ class BaseEvaluator:
     def build_eval_set(
         self, eval_set: Optional[EvalSet] = None, file_path: Optional[str] = None
     ):
-        """Generate evaluation data from a given file and assign it to the class attribute `invocation_list`."""
+        """Builds invocation list from eval set or file.
+
+        This method parses input, extracts invocations with expected data.
+        Supports eval JSON and tracing JSON formats.
+
+        Args:
+            eval_set (Optional[EvalSet]): Direct eval set object.
+            file_path (Optional[str]): Path to file for loading.
+
+        Raises:
+            ValueError: If neither provided or format unsupported.
+
+        Note:
+        Generates random session IDs for isolation.
+        """
 
         if eval_set is None and file_path is None:
             raise ValueError("eval_set or file_path is required")
@@ -294,6 +467,21 @@ class BaseEvaluator:
         self.invocation_list = eval_case_data_list
 
     async def generate_actual_outputs(self):
+        """Generates actual outputs by running the agent on inputs.
+
+        This method uses Runner to execute agent for each invocation.
+        Captures outputs, tools, and latency.
+
+        Returns:
+            None: Updates invocation actual fields.
+
+        Raises:
+            Exception: If runner or execution fails.
+
+        Note:
+        Uses InMemorySessionService for isolation.
+        Supports long-term memory if present.
+        """
         for eval_case_data, agent_information in zip(
             self.invocation_list, self.agent_information_list
         ):
@@ -366,7 +554,17 @@ class BaseEvaluator:
                 invocation.latency = _latency
 
     def get_eval_set_information(self) -> list[list[dict[str, Any]]]:
-        """Merge the evaluation data and return it in the format of list[list[dict]]"""
+        """Retrieves combined evaluation information.
+
+        This method merges invocations and results into dict lists.
+        Useful for reporting.
+
+        Returns:
+            list[list[dict[str, Any]]]: Nested list of case data dicts.
+
+        Note:
+        Defaults to empty results if not evaluated yet.
+        """
         result = []
         for i, eval_case in enumerate(self.invocation_list):
             case_data = []
@@ -399,5 +597,23 @@ class BaseEvaluator:
         eval_set_file_path: Optional[str],
         eval_id: str,
     ):
-        """An abstract method for evaluation based on metrics。"""
+        """Abstract method for performing the evaluation.
+
+        Subclasses implement specific metric evaluation logic.
+
+        Args:
+            metrics (list[Any]): Metrics to apply.
+            eval_set (Optional[EvalSet]): Eval set.
+            eval_set_file_path (Optional[str]): File path.
+            eval_id (str): Evaluation ID.
+
+        Returns:
+            Any: Evaluation results specific to subclass.
+
+        Raises:
+            NotImplementedError: If not overridden.
+
+        Note:
+        Must populate result_list after evaluation.
+        """
         pass

veadk/evaluation/deepeval_evaluator/deepeval_evaluator.py

@@ -37,11 +37,44 @@ logger = get_logger(__name__)
 
 
 def formatted_timestamp():
+    """Generates a formatted timestamp string in YYYYMMDDHHMMSS format.
+
+    This function creates a string representation of the current time.
+    It uses local time for formatting.
+
+    Returns:
+        str: Timestamp string like '20251028123045'.
+    """
     # YYYYMMDDHHMMSS
     return time.strftime("%Y%m%d%H%M%S", time.localtime())
 
 
 class DeepevalEvaluator(BaseEvaluator):
+    """Evaluates agents using DeepEval metrics with Prometheus export.
+
+    This class uses DeepEval to test agent performance.
+    It runs agents on test cases and scores them.
+    Results can be sent to Prometheus for monitoring.
+
+    Attributes:
+        judge_model_name (str): Name of the model that judges the agent.
+        judge_model (LocalModel): The judge model instance.
+        prometheus_config (PrometheusPushgatewayConfig | None): Settings for
+            Prometheus export. If None, no export happens.
+
+    Note:
+        Needs judge model credentials from environment if not given.
+        Turns off cache to get fresh results each time.
+
+    Examples:
+        ```python
+        agent = Agent(tools=[get_city_weather])
+        evaluator = DeepevalEvaluator(agent=agent)
+        metrics = [GEval(threshold=0.8)]
+        results = await evaluator.evaluate(metrics, eval_set_file_path="test.json")
+        ```
+    """
+
     def __init__(
         self,
         agent,
@@ -51,6 +84,32 @@ class DeepevalEvaluator(BaseEvaluator):
         name: str = "veadk_deepeval_evaluator",
         prometheus_config: PrometheusPushgatewayConfig | None = None,
     ):
+        """Sets up the DeepEval evaluator with agent and judge model.
+
+        Args:
+            agent: The agent to test.
+            judge_model_api_key: API key for the judge model. If empty,
+                gets from MODEL_JUDGE_API_KEY environment variable.
+            judge_model_name: Name of the judge model. If empty,
+                gets from MODEL_JUDGE_NAME environment variable.
+            judge_model_api_base: Base URL for judge model API. If empty,
+                gets from MODEL_JUDGE_API_BASE environment variable.
+            name: Name for this evaluator. Defaults to 'veadk_deepeval_evaluator'.
+            prometheus_config: Settings for Prometheus export. If None,
+                no export happens.
+
+        Raises:
+            ValueError: If model settings are wrong.
+            EnvironmentError: If environment variables are missing.
+
+        Examples:
+            ```python
+            evaluator = DeepevalEvaluator(
+                agent=my_agent,
+                judge_model_api_key="sk-...",
+                prometheus_config=prometheus_config)
+            ```
+        """
         super().__init__(agent=agent, name=name)
 
         if not judge_model_api_key:
@@ -83,7 +142,38 @@ class DeepevalEvaluator(BaseEvaluator):
         eval_set_file_path: Optional[str] = None,
         eval_id: str = f"test_{formatted_timestamp()}",
     ):
-        """Target to Google ADK, we will use the same evaluation case format as Google ADK."""
+        """Tests agent using DeepEval on given test cases.
+
+        This method does these steps:
+        1. Loads test cases from memory or file
+        2. Runs agent to get actual responses
+        3. Converts to DeepEval test format
+        4. Runs metrics evaluation
+        5. Sends results to Prometheus if needed
+
+        Args:
+            metrics: List of DeepEval metrics to use for scoring.
+            eval_set: Test cases in memory. If given, used first.
+            eval_set_file_path: Path to test case file. Used if no eval_set.
+            eval_id: Unique name for this test run. Used for tracking.
+
+        Returns:
+            EvaluationResult: Results from DeepEval with scores and details.
+
+        Raises:
+            ValueError: If no test cases found.
+            FileNotFoundError: If test file not found.
+            EvaluationError: If agent fails or metrics fail.
+
+        Examples:
+            ```python
+            metrics = [GEval(threshold=0.8), ToolCorrectnessMetric(threshold=0.5)]
+            results = await evaluator.evaluate(
+                metrics=metrics,
+                eval_set_file_path="test_cases.json")
+            print(f"Test cases run: {len(results.test_results)}")
+            ```
+        """
         # Get evaluation data by parsing eval set file
         self.build_eval_set(eval_set, eval_set_file_path)
 
@@ -162,6 +252,31 @@ class DeepevalEvaluator(BaseEvaluator):
         return test_results
 
     def export_results(self, eval_id: str, test_results: EvaluationResult):
+        """Sends evaluation results to Prometheus for monitoring.
+
+        This method takes test results, counts passes and failures,
+        and sends metrics to Prometheus.
+
+        Args:
+            eval_id: Unique name for this test. Used as label in Prometheus.
+            test_results: Results from DeepEval evaluation.
+
+        Returns:
+            None: Results are sent directly to Prometheus.
+
+        Raises:
+            PrometheusConnectionError: If cannot connect to Prometheus.
+            PrometheusPushError: If sending data fails.
+
+        Note:
+            Uses fixed thresholds for now: case_threshold=0.5, diff_threshold=0.2.
+            These may change later.
+
+        Examples:
+            ```python
+            evaluator.export_results("test_20240101", test_results)
+            ```
+        """
         # fixed attributions
         test_name = eval_id
         test_cases_total = len(test_results.test_results)

veadk/evaluation/eval_set_file_loader.py

@@ -19,6 +19,26 @@ from google.adk.evaluation.local_eval_sets_manager import (
 
 
 def load_eval_set_from_file(eval_set_file_path: str) -> EvalSet:
+    """Loads an evaluation set from a JSON file.
+
+    This function uses ADK's loader to parse the file into an EvalSet object.
+    It handles errors in file reading or parsing.
+
+    Args:
+        eval_set_file_path (str): Path to the JSON eval set file.
+
+    Returns:
+        EvalSet: Loaded evaluation set object.
+
+    Raises:
+        Exception: If file loading or parsing fails, with details.
+
+    Examples:
+        ```python
+        eval_set = load_eval_set_from_file("my_eval.json")
+        print(len(eval_set.eval_cases))
+        ```
+    """
     try:
         eval_set = adk_load_eval_set_from_file(eval_set_file_path, eval_set_file_path)
     except Exception as e:

veadk/evaluation/eval_set_recorder.py

@@ -27,9 +27,32 @@ logger = get_logger(__name__)
 
 
 class EvalSetRecorder(LocalEvalSetsManager):
+    """Records evaluation sets from sessions for later use in testing.
+
+    This class extends LocalEvalSetsManager to add sessions to eval sets.
+    It handles dumping eval sets to files from session data.
+
+    Attributes:
+        eval_set_id (str): ID of the eval set. Defaults to 'default'.
+        session_service (BaseSessionService): Service for session management.
+
+    Note:
+        Uses temporary directory for storing eval sets.
+        Creates eval cases from session invocations.
+    """
+
     def __init__(
         self, session_service: BaseSessionService, eval_set_id: str = "default"
     ):
+        """Initializes the eval set recorder with session service and ID.
+
+        Args:
+            session_service (BaseSessionService): Service to retrieve sessions.
+            eval_set_id (str): ID for the eval set. Defaults to 'default'.
+
+        Raises:
+            ValueError: If eval_set_id is invalid.
+        """
         super().__init__(agents_dir=get_temp_dir())
         self.eval_set_id = eval_set_id if eval_set_id != "" else "default"
         self.session_service: BaseSessionService = session_service
@@ -42,6 +65,21 @@ class EvalSetRecorder(LocalEvalSetsManager):
         session_id: str,
         user_id: str,
     ):
+        """Adds a session to the evaluation set as an eval case.
+
+        This method retrieves a session and converts it to eval invocations.
+        It creates a new eval case with timestamp.
+
+        Args:
+            app_name (str): Name of the app for the session.
+            eval_set_id (str): ID of the eval set to add to.
+            session_id (str): ID of the session to add.
+            user_id (str): ID of the user owning the session.
+
+        Raises:
+            AssertionError: If session not found.
+            ValueError: If adding eval case fails.
+        """
         eval_id = f"veadk_eval_{formatted_timestamp()}"
 
         # Get the session
@@ -74,6 +112,22 @@ class EvalSetRecorder(LocalEvalSetsManager):
         user_id: str,
         session_id: str,
     ) -> str:
+        """Dumps the current eval set to a file path.
+
+        This method creates the eval set if needed and adds the session.
+        It ensures directory exists and logs the dump path.
+
+        Args:
+            app_name (str): Name of the app.
+            user_id (str): ID of the user.
+            session_id (str): ID of the session to dump.
+
+        Returns:
+            str: Path where the eval set was dumped.
+
+        Raises:
+            ValueError: If dump operation fails.
+        """
         dump_path = self._get_eval_set_file_path(app_name, self.eval_set_id)
         Path(dump_path).parent.mkdir(parents=True, exist_ok=True)
 

veadk/evaluation/types.py

@@ -17,6 +17,25 @@ from dataclasses import dataclass
 
 @dataclass
 class EvalResultCaseData:
+    """Holds data for a single evaluation case result.
+
+    This dataclass stores input, outputs, score, and status for one test case.
+    Used in evaluation reporting and metrics export.
+
+    Attributes:
+        id (str): Unique ID of the case.
+        input (str): User input for the case.
+        actual_output (str): Actual agent response.
+        expected_output (str): Expected agent response.
+        score (str): Score as string from evaluation.
+        reason (str): Reason for the score.
+        status (str): Status like 'PASSED' or 'FAILURE'.
+        latency (str): Latency in milliseconds as string.
+
+    Note:
+        Score and latency are strings for compatibility with external systems.
+    """
+
     id: str
     input: str
     actual_output: str
@@ -29,5 +48,18 @@ class EvalResultCaseData:
 
 @dataclass
 class EvalResultMetadata:
+    """Stores metadata about the evaluation run.
+
+    This dataclass captures model information for the evaluation.
+    Used in reporting and tracing.
+
+    Attributes:
+        tested_model (str): Name of the model being tested.
+        judge_model (str): Name of the judge model used.
+
+    Note:
+        Simple structure for quick metadata access.
+    """
+
     tested_model: str
     judge_model: str

veadk/evaluation/utils/prometheus.py

@@ -22,6 +22,23 @@ from veadk.evaluation.types import EvalResultCaseData, EvalResultMetadata
 
 
 class PrometheusPushgatewayConfig:
+    """Configures connection to Prometheus Pushgateway for metrics export.
+
+    This class holds settings for pushing evaluation metrics to Prometheus.
+    It uses environment variables for default values.
+
+    Attributes:
+        url (str): URL of the Prometheus Pushgateway endpoint.
+            Defaults to OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL environment variable.
+        username (str): Username for authentication.
+            Defaults to OBSERVABILITY_PROMETHEUS_USERNAME environment variable.
+        password (str): Password for authentication.
+            Defaults to OBSERVABILITY_PROMETHEUS_PASSWORD environment variable.
+
+    Note:
+        All fields are optional and use environment variables if not provided.
+    """
+
     url: str = Field(
         default_factory=lambda: getenv(
             "OBSERVABILITY_PROMETHEUS_PUSHGATEWAY_URL",
@@ -87,6 +104,26 @@ def post_pushgateway(
     registry: CollectorRegistry,
     grouping_key: dict[str, str] | None = None,
 ):
+    """Pushes metrics to Prometheus Pushgateway with authentication.
+
+    This function sends collected metrics to the specified Pushgateway URL.
+    It uses basic authentication and optional grouping keys.
+
+    Args:
+        pushgateway_url (str): URL of the Pushgateway endpoint.
+        username (str): Authentication username.
+        password (str): Authentication password.
+        job_name (str): Name of the job for metrics labeling.
+        registry (CollectorRegistry): Registry containing metrics to push.
+        grouping_key (dict[str, str] | None): Optional key-value pairs for grouping.
+
+    Raises:
+        Exception: If push operation fails due to network or auth issues.
+
+    Note:
+        Authentication handler is created internally using provided credentials.
+    """
+
     def auth_handler(url, method, timeout, headers, data):
         return basic_auth_handler(
             url, method, timeout, headers, data, username, password
@@ -114,6 +151,30 @@ def push_to_prometheus(
     username: str = "",
     password: str = "",
 ):
+    """Sets and pushes evaluation metrics to Prometheus.
+
+    This function updates gauge metrics with evaluation results and pushes them.
+    It handles counts, thresholds, and specific data labels.
+
+    Args:
+        test_name (str): Name of the test for grouping.
+        test_cases_total (int): Total number of test cases.
+        test_cases_failure (int): Number of failed test cases.
+        test_cases_pass (int): Number of passed test cases.
+        test_data_list (list[EvalResultCaseData]): List of case data for labeling.
+        eval_data (EvalResultMetadata): Metadata for evaluation.
+        case_threshold (float): Threshold value for cases. Defaults to 0.5.
+        diff_threshold (float): Diff threshold value. Defaults to 0.2.
+        url (str): Pushgateway URL. Defaults to empty.
+        username (str): Auth username. Defaults to empty.
+        password (str): Auth password. Defaults to empty.
+
+    Returns:
+        None: Metrics are set and pushed directly.
+
+    Raises:
+        ValueError: If required data is invalid.
+    """
     test_cases_total_metric.set(test_cases_total)
     test_cases_failure_metric.set(test_cases_failure)
     test_cases_pass_metric.set(test_cases_pass)