ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/annotate.py

@@ -1,10 +1,12 @@
-from wxo_agentic_evaluation.type import Message, EvaluationData
-from wxo_agentic_evaluation.arg_configs import TestCaseGenerationConfig
-from wxo_agentic_evaluation.data_annotator import DataAnnotator
 import json
+import os
 from pprint import pprint
+
 from jsonargparse import CLI
-import os
+
+from wxo_agentic_evaluation.arg_configs import TestCaseGenerationConfig
+from wxo_agentic_evaluation.data_annotator import DataAnnotator
+from wxo_agentic_evaluation.type import Message, OrchestrateDataset
 
 
 def main(config: TestCaseGenerationConfig):
@@ -15,7 +17,7 @@ def main(config: TestCaseGenerationConfig):
             messages.append(Message.model_validate(entry))
 
     with open(config.seed_data_path, "r") as f:
-        evaluation_data = EvaluationData(**json.load(f))
+        evaluation_data = OrchestrateDataset(**json.load(f))
 
     # Generate annonated dataset
     annotator = DataAnnotator(

wxo_agentic_evaluation/arg_configs.py

@@ -1,16 +1,22 @@
 import os
 from dataclasses import dataclass, field
-from typing import List
+from enum import StrEnum
+from typing import List, Optional, Union
+
 from wxo_agentic_evaluation import __file__
 
 root_dir = os.path.dirname(__file__)
-LLAMA_USER_PROMPT_PATH = os.path.join(root_dir, "prompt", "llama_user_prompt.jinja2")
-KEYWORDS_GENERATION_PROMPT_PATH = os.path.join(root_dir, "prompt", "keywords_generation_prompt.jinja2")
+LLAMA_USER_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "llama_user_prompt.jinja2"
+)
+KEYWORDS_GENERATION_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "keywords_generation_prompt.jinja2"
+)
 
 
 @dataclass
 class AuthConfig:
-    url: str
+    url: Optional[str] = None
     tenant_name: str = "local"
     token: str = None
 
@@ -25,7 +31,33 @@ class LLMUserConfig:
 @dataclass
 class ProviderConfig:
     model_id: str = field(default="meta-llama/llama-3-405b-instruct")
-    provider: str = field(default="watsonx")
+    provider: str = field(
+        default_factory=lambda: (
+            "gateway"
+            if os.getenv("USE_GATEWAY_MODEL_PROVIDER", "").lower() == "true"
+            else "watsonx"
+        )
+    )
+    embedding_model_id: str = field(
+        default="sentence-transformers/all-minilm-l6-v2"
+    )
+
+
+@dataclass
+class CustomMetricsConfig:
+    paths: Optional[list[str]] = field(default=None)
+    llmaaj_config: ProviderConfig = field(default_factory=ProviderConfig)
+
+
+@dataclass
+class ExtractorsConfig:
+    paths: Optional[list[str]] = field(default=None)
+
+
+
+class ControllerConfig:
+    enable_verbose_logging: bool = True
+    enable_manual_user_input: bool = False
 
 
 @dataclass
@@ -36,16 +68,59 @@ class TestConfig:
     wxo_lite_version: str
     provider_config: ProviderConfig = field(default_factory=ProviderConfig)
     llm_user_config: LLMUserConfig = field(default_factory=LLMUserConfig)
+    custom_metrics_config: CustomMetricsConfig = field(
+        default_factory=CustomMetricsConfig
+    )
+    extractors_config: ExtractorsConfig = field(default_factory=ExtractorsConfig)
     enable_verbose_logging: bool = True
     enable_manual_user_input: bool = False
     skip_available_results: bool = False
     data_annotation_run: bool = False
     num_workers: int = 2
+    n_runs: int = 1
+    similarity_threshold: float = 0.8
+    enable_fuzzy_matching: bool = False
+    strict_topological_matching: bool = True
+    enable_recursive_search: bool = False
+    skip_legacy_evaluation: bool = False # Skip legacy evaluation and only run user/agent simulations
+
+
+@dataclass
+class AttackConfig:
+    attack_paths: List[str]
+    output_dir: str
+    auth_config: AuthConfig
+    provider_config: ProviderConfig = field(default_factory=ProviderConfig)
+    llm_user_config: LLMUserConfig = field(default_factory=LLMUserConfig)
+    enable_verbose_logging: bool = True
+    enable_manual_user_input: bool = False
+    num_workers: int = 2
+    skip_available_results: bool = True
+
+
+@dataclass
+class AttackGeneratorConfig:
+    attacks_list: Union[List[str], str]
+    datasets_path: Union[List[str], str]
+    agents_list_or_path: Union[List[str], str]
+    target_agent_name: str
+    auth_config: AuthConfig
+    output_dir: str = None
+    max_variants: int = None
+
+
+class AnalyzeMode(StrEnum):
+    default = "default"
+    enhanced = "enhanced"
 
 
 @dataclass
 class AnalyzeConfig:
     data_path: str
+    tool_definition_path: Optional[str] = None
+    mode: str = AnalyzeMode.default
+    num_workers: int = 10
+    run: int = -1
 
 
 @dataclass
@@ -74,6 +149,12 @@ class ChatRecordingConfig:
     service_url: str = "http://localhost:4321"
     tenant_name: str = "local"
     token: str = None
+    max_retries: int = 5
+
+
+@dataclass
+class QuickEvalConfig(TestConfig):
+    tools_path: str = None
 
 
 @dataclass
@@ -83,3 +164,14 @@ class BatchAnnotateConfig:
     stories_path: str
     output_dir: str
     num_variants: int = 2
+
+
+@dataclass
+class CompareRunsConfig:
+    reference_file_location: str
+    experiment_file_location: str
+    csv_output: Optional[str] = None
+    column_stats_csv: Optional[str] = (
+        "column_by_column_summary_stats_comparison.csv"
+    )
+    verbose: bool = False

wxo_agentic_evaluation/base_user.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from wxo_agentic_evaluation.type import Message
+
+
+class BaseUserSimulator(ABC):
+    """Abstract base class for user simulators."""
+
+    @abstractmethod
+    def generate_user_input(
+        self, user_story: str, conversation_history: List[Message], **kwargs
+    ) -> Message:
+        """
+        Generate user input based on the user story and conversation history.
+
+        Args:
+            user_story: The user's story or goal
+            conversation_history: List of previous messages in the conversation
+            **kwargs: Additional parameters specific to the simulator implementation
+
+        Returns:
+            Message: The generated user input message
+        """
+        pass

wxo_agentic_evaluation/batch_annotate.py

@@ -1,22 +1,28 @@
-import json
 import ast
 import csv
+import json
 import os
 from pathlib import Path
+
 from jsonargparse import CLI
 
-from wxo_agentic_evaluation.service_provider import get_provider
-from wxo_agentic_evaluation.prompt.template_render import BatchTestCaseGeneratorTemplateRenderer
-from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
 from wxo_agentic_evaluation import __file__
+from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
+from wxo_agentic_evaluation.prompt.template_render import (
+    BatchTestCaseGeneratorTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
 
 root_dir = os.path.dirname(__file__)
-BATCH_TEST_CASE_GENERATOR_PROMPT_PATH = os.path.join(root_dir, "prompt", "batch_testcase_prompt.jinja2")
+BATCH_TEST_CASE_GENERATOR_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "batch_testcase_prompt.jinja2"
+)
 EXAMPLE_PATH = os.path.join(root_dir, "prompt", "examples", "data_simple.json")
 
 
-def parse_tools_with_filter(agent_name: str, tools_path: Path, allowed_tool_names: list[str]) -> tuple[
-    dict, list[dict]]:
+def parse_tools_with_filter(
+    agent_name: str, tools_path: Path, allowed_tool_names: list[str]
+) -> tuple[dict, list[dict]]:
     if not allowed_tool_names:
         raise ValueError("Allowed tool list cannot be empty.")
 
@@ -29,7 +35,9 @@ def parse_tools_with_filter(agent_name: str, tools_path: Path, allowed_tool_name
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
@@ -41,21 +49,29 @@ def parse_tools_with_filter(agent_name: str, tools_path: Path, allowed_tool_name
             # Process only module-level functions
             for node in parsed_code.body:
                 if isinstance(node, ast.FunctionDef):
-                    tool_data.append({
-                        "Function Name": node.name,
-                        "Arguments": [arg.arg for arg in node.args.args],
-                        "Docstring": ast.get_docstring(node)
-                    })
+                    tool_data.append(
+                        {
+                            "Function Name": node.name,
+                            "Arguments": [arg.arg for arg in node.args.args],
+                            "Docstring": ast.get_docstring(node),
+                        }
+                    )
 
         except Exception as e:
             print(f"Warning: Failed to parse {file_path}: {str(e)}")
             continue
 
     # Filter tools based on allowed names
-    filtered_tools = [tool for tool in tool_data if tool["Function Name"] in allowed_tool_names]
+    filtered_tools = [
+        tool
+        for tool in tool_data
+        if tool["Function Name"] in allowed_tool_names
+    ]
 
     if not filtered_tools:
-        print(f"Warning: No matching tools found. Available tools: {[t['Function Name'] for t in tool_data]}")
+        print(
+            f"Warning: No matching tools found. Available tools: {[t['Function Name'] for t in tool_data]}"
+        )
 
     return {"name": agent_name}, filtered_tools
 
@@ -75,8 +91,17 @@ def load_example(example_path: Path):
 
 
 # Step 4: Prompt builder for N test cases from a given story
-def build_prompt_for_story(agent, tools, tool_inputs, example_case: dict, story: str, num_variants: int = 2):
-    renderer = BatchTestCaseGeneratorTemplateRenderer(BATCH_TEST_CASE_GENERATOR_PROMPT_PATH)
+def build_prompt_for_story(
+    agent,
+    tools,
+    tool_inputs,
+    example_case: dict,
+    story: str,
+    num_variants: int = 2,
+):
+    renderer = BatchTestCaseGeneratorTemplateRenderer(
+        BATCH_TEST_CASE_GENERATOR_PROMPT_PATH
+    )
 
     tool_blocks = "\n".join(
         f"- Tool: {t['Function Name']}\n  Description: {t['Docstring']}\n  Args: {', '.join(t['Arguments']) or 'None'}"
@@ -93,16 +118,43 @@ def build_prompt_for_story(agent, tools, tool_inputs, example_case: dict, story:
     )
     return prompt
 
+
 # Step 5: Send prompt to LLM and save test cases
-def generate_multiple_in_one(prompt, output_dir, starting_index, model_id="meta-llama/llama-3-405b-instruct", ):
+def generate_multiple_in_one(
+    prompt,
+    output_dir,
+    starting_index,
+    model_id="meta-llama/llama-3-405b-instruct",
+    # model_id="gpt-4o",
+):
     output_dir.mkdir(parents=True, exist_ok=True)
 
+    # Legacy provider (e.g., watsonx)
     provider = get_provider(
         model_id=model_id,
-        params={"min_new_tokens": 50, "decoding_method": "greedy", "max_new_tokens": 3000},
+        params={
+            "min_new_tokens": 50,
+            "decoding_method": "greedy",
+            "max_new_tokens": 3000,
+        },
+        use_portkey_provider=False,
     )
-
-    response = provider.query(prompt)
+    response = provider.chat(prompt).text
+
+    # # OpenAI provider
+    # provider = get_provider(provider="openai", model_id=model_id, api_key=os.getenv("OPENAI_API_KEY"))
+    # response = provider.chat(prompt).choices[0].message.content
+
+    # # Azure OpenAI provider
+    # provider = get_provider(
+    #     provider = "azure-openai",
+    #     azure_model_name = model_id,
+    #     azure_deployment_id = os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"),
+    #     azure_resource_name = os.getenv("AZURE_OPENAI_RESOURCE_NAME"),
+    #     azure_api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
+    #     api_key = f"Bearer {os.getenv('AZURE_OPENAI_API_KEY')}"
+    # )
+    # response = provider.chat(prompt).choices[0].message.content
 
     try:
         raw_text = response
@@ -124,8 +176,19 @@ def generate_multiple_in_one(prompt, output_dir, starting_index, model_id="meta-
         print("Raw text:\n", raw_text)
         print("Error:", str(e))
 
-def generate_test_cases_from_stories(agent_name: str, stories: list[str], tools_path: Path, snapshot_path: Path, output_dir: Path, allowed_tools: list[str], num_variants: int = 2):
-    agent, tools = parse_tools_with_filter(agent_name, tools_path, allowed_tools)
+
+def generate_test_cases_from_stories(
+    agent_name: str,
+    stories: list[str],
+    tools_path: Path,
+    snapshot_path: Path,
+    output_dir: Path,
+    allowed_tools: list[str],
+    num_variants: int = 2,
+):
+    agent, tools = parse_tools_with_filter(
+        agent_name, tools_path, allowed_tools
+    )
     tool_inputs = extract_inputs_from_snapshot(snapshot_path)
     example_json = load_example(Path(EXAMPLE_PATH))
 
@@ -134,23 +197,29 @@ def generate_test_cases_from_stories(agent_name: str, stories: list[str], tools_
         print(f"\n Generating test cases for story {idx}: {story}")
 
         prompt = build_prompt_for_story(
-            agent, tools, tool_inputs, example_json, story, num_variants=num_variants
+            agent,
+            tools,
+            tool_inputs,
+            example_json,
+            story,
+            num_variants=num_variants,
         )
 
         generate_multiple_in_one(
             prompt=prompt,
             output_dir=output_dir,
-            starting_index=test_case_counter
+            starting_index=test_case_counter,
         )
 
         test_case_counter += num_variants
 
+
 def main(config: BatchAnnotateConfig):
     stories_path = Path(config.stories_path)
 
     stories = []
     agent_name = None
-    with stories_path.open("r", encoding="utf-8", newline='') as f:
+    with stories_path.open("r", encoding="utf-8", newline="") as f:
         csv_reader = csv.DictReader(f)
         for row in csv_reader:
             stories.append(row["story"])
@@ -168,8 +237,9 @@ def main(config: BatchAnnotateConfig):
         snapshot_path,
         output_dir,
         config.allowed_tools,
-        num_variants=config.num_variants
+        num_variants=config.num_variants,
     )
 
+
 if __name__ == "__main__":
     main(CLI(BatchAnnotateConfig, as_positional=False))

wxo_agentic_evaluation/clients.py

@@ -0,0 +1,103 @@
+import copy
+from dataclasses import asdict, dataclass
+
+from wxo_agentic_evaluation.arg_configs import ProviderConfig, TestConfig
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.prompt.template_render import (
+    LlamaUserTemplateRenderer,
+)
+from wxo_agentic_evaluation.resource_map import ResourceMap
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import (
+    WXORuntimeAdapter,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
+from wxo_agentic_evaluation.service_provider.provider import Provider
+from wxo_agentic_evaluation.wxo_client import WXOClient, get_wxo_client
+
+
+@dataclass
+class Clients:
+    wxo_client: WXOClient
+    llmaaj_provider: Provider
+    resource_map: ResourceMap
+    inference_backend: WXORuntimeAdapter
+    llm_user: LLMUser
+
+
+def bootstrap_clients(config: TestConfig) -> Clients:
+    """
+    Bootstrap all clients needed for the evaluation.
+
+    Args:
+        config: The test configuration
+
+    Returns:
+        A tuple containing:
+        - wxo_client: The WXO client
+        - llmaaj_provider: The provider for custom metrics
+        - resource_map: The resource map
+        - inference_backend: The inference backend
+        - llm_user: The LLM user
+    """
+    # Initialize WXO client
+    wxo_client = get_wxo_client(
+        config.auth_config.url,
+        config.auth_config.tenant_name,
+        config.auth_config.token,
+    )
+
+    # Initialize provider for custom metrics
+    original_provider_config = config.provider_config
+    provider_config_dict = asdict(original_provider_config)
+
+    provider_kwargs = {
+        "config": ProviderConfig(**provider_config_dict),
+        "model_id": config.llm_user_config.model_id,
+    }
+
+    if provider_config_dict.get("provider", "gateway") == "gateway":
+        provider_kwargs.update(
+            token=config.auth_config.token or wxo_client.api_key,
+            instance_url=wxo_client.service_url,
+        )
+        config.auth_config.token = (
+            config.auth_config.token or wxo_client.api_key
+        )
+        config.auth_config.url = (
+            config.auth_config.url or wxo_client.service_url
+        )
+
+    # Initialize resource map
+    resource_map = ResourceMap(wxo_client)
+
+    # Initialize inference backend
+    inference_backend = WXORuntimeAdapter(wxo_client=wxo_client)
+
+    # Initialize LLM user
+    llm_user = LLMUser(
+        wai_client=get_provider(**provider_kwargs),
+        template=LlamaUserTemplateRenderer(
+            config.llm_user_config.prompt_config
+        ),
+        user_response_style=config.llm_user_config.user_response_style,
+    )
+
+    llamaj_provider_kwargs = copy.deepcopy(provider_kwargs)
+    llamaj_config_dict = asdict(llamaj_provider_kwargs["config"])
+
+    llamaj_config_dict["model_id"] = (
+        config.custom_metrics_config.llmaaj_config.model_id
+    )
+    llamaj_config_dict["embedding_model_id"] = (
+        config.custom_metrics_config.llmaaj_config.embedding_model_id
+    )
+    llamaj_provider_kwargs["config"] = ProviderConfig(**llamaj_config_dict)
+    llmaaj_provider = get_provider(**llamaj_provider_kwargs)
+
+    return Clients(
+        wxo_client=wxo_client,
+        llmaaj_provider=llmaaj_provider,
+        resource_map=resource_map,
+        inference_backend=inference_backend,
+        llm_user=llm_user,
+    )

wxo_agentic_evaluation/compare_runs/compare_2_runs.py

@@ -0,0 +1,74 @@
+import csv
+import os
+import statistics
+import sys
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Union
+
+from jsonargparse import CLI
+
+from wxo_agentic_evaluation.arg_configs import CompareRunsConfig
+from wxo_agentic_evaluation.compare_runs.diff import DiffResults
+from wxo_agentic_evaluation.compare_runs.model import EvaluationResult
+from wxo_agentic_evaluation.utils.utils import create_table, read_file
+
+
+def main(config: CompareRunsConfig):
+    """Main function to compare two run result files."""
+    # Extract values from config
+    reference_file = config.reference_file_location
+    experiment_file = config.experiment_file_location
+    csv_output = config.csv_output
+    column_stats_csv = config.column_stats_csv
+    verbose = config.verbose
+
+    try:
+        # Read the files
+        obj1 = read_file(reference_file)
+        obj2 = read_file(experiment_file)
+
+        # Create evaluation results
+        result1 = EvaluationResult.from_csv(obj1)
+        result2 = EvaluationResult.from_csv(obj2)
+
+        # Create diff results
+        diff_results = DiffResults(result1, result2)
+
+        # Display summary statistics
+        summary_stats = diff_results.summary_statistics()
+        summary_table = create_table(summary_stats, title="Summary Statistics")
+        print(
+            "\nALL metrics are computed on OVERLAPPING test cases, ie cases that exist in both the Reference and Experiment runs\n"
+        )
+        print(
+            "If Experiment - Reference is Positive, that's an increase in the metric. If Experiment - Reference is Negative, that's a decrease in the metric.\n"
+        )
+        summary_table.print()
+
+        # Display exclusive tests
+        if verbose:
+            diff_results.display_exclusive_tests()
+
+            # Display test cases with differing summary match and success status
+            diff_results.display_differing_summary_matches()
+
+        # Display tabular diff
+        diff_results.compute_tabular_diff(verbose=verbose)
+
+        # Write results to CSV if specified
+        if csv_output:
+            diff_results.to_csv(csv_output)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    args = CLI(CompareRunsConfig, as_positional=False)
+    sys.exit(main(args))
+
+# Made with Bob