ibm-watsonx-orchestrate-evaluation-framework 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

wxo_agentic_evaluation/service_provider/referenceless_provider_wrapper.py

@@ -1,11 +1,15 @@
-import requests
-from typing import List, Mapping, Union, Optional, Any
 from abc import ABC, abstractmethod
+from typing import Any, List, Mapping, Optional, Union
 
+import requests
 import rich
 
-from wxo_agentic_evaluation.service_provider.model_proxy_provider import ModelProxyProvider
-from wxo_agentic_evaluation.service_provider.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
+    ModelProxyProvider,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import (
+    WatsonXProvider,
+)
 
 
 class LLMResponse:
@@ -14,7 +18,9 @@ class LLMResponse:
     Response object that can contain both content and tool calls
     """
 
-    def __init__(self, content: str, tool_calls: Optional[List[Mapping[str, Any]]] = None):
+    def __init__(
+        self, content: str, tool_calls: Optional[List[Mapping[str, Any]]] = None
+    ):
         self.content = content
         self.tool_calls = tool_calls or []
 
@@ -26,25 +32,26 @@ class LLMResponse:
         """Return a string representation of the LLMResponse object."""
         return f"LLMResponse(content='{self.content}', tool_calls={self.tool_calls})"
 
+
 class LLMKitWrapper(ABC):
-    """ In the future this wrapper won't be neccesary.
+    """In the future this wrapper won't be neccesary.
     Right now the referenceless code requires a `generate()` function for the metrics client.
     In refactor, rewrite referenceless code so this wrapper is not needed.
     """
+
     @abstractmethod
     def chat():
         pass
 
     def generate(
-            self,
-            prompt: Union[str, List[Mapping[str, str]]],
-            *,
-            schema,
-            retries: int = 3,
-            generation_args: Optional[Any] = None,
-            **kwargs: Any
-        ):
-
+        self,
+        prompt: Union[str, List[Mapping[str, str]]],
+        *,
+        schema,
+        retries: int = 3,
+        generation_args: Optional[Any] = None,
+        **kwargs: Any,
+    ):
         """
         In future, implement validation of response like in llmevalkit
         """
@@ -55,7 +62,9 @@ class LLMKitWrapper(ABC):
                 response = self._parse_llm_response(raw_response)
                 return response
             except Exception as e:
-                rich.print(f"[b][r] Generation failed with error '{str(e)}' during `quick-eval` ... Attempt ({attempt} / {retries}))")
+                rich.print(
+                    f"[b][r] Generation failed with error '{str(e)}' during `quick-eval` ... Attempt ({attempt} / {retries}))"
+                )
 
     def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]:
         """
@@ -82,10 +91,12 @@ class LLMKitWrapper(ABC):
                                 "id": tool_call.get("id"),
                                 "type": tool_call.get("type", "function"),
                                 "function": {
-                                    "name": tool_call.get("function", {}).get("name"),
-                                    "arguments": tool_call.get("function", {}).get(
-                                        "arguments"
+                                    "name": tool_call.get("function", {}).get(
+                                        "name"
                                     ),
+                                    "arguments": tool_call.get(
+                                        "function", {}
+                                    ).get("arguments"),
                                 },
                             }
                             tool_calls.append(tool_call_dict)
@@ -101,6 +112,7 @@ class LLMKitWrapper(ABC):
 
         return content
 
+
 class ModelProxyProviderLLMKitWrapper(ModelProxyProvider, LLMKitWrapper):
     def chat(self, sentence: List[str]):
         if self.model_id is None:
@@ -113,7 +125,7 @@ class ModelProxyProviderLLMKitWrapper(ModelProxyProvider, LLMKitWrapper):
             "messages": sentence,
             "parameters": self.params,
             "space_id": "1",
-            "timeout": self.timeout
+            "timeout": self.timeout,
         }
         resp = requests.post(url=chat_url, headers=headers, json=data)
         if resp.status_code == 200:
@@ -121,6 +133,7 @@ class ModelProxyProviderLLMKitWrapper(ModelProxyProvider, LLMKitWrapper):
         else:
             resp.raise_for_status()
 
+
 class WatsonXLLMKitWrapper(WatsonXProvider, LLMKitWrapper):
     def chat(self, sentence: list):
         chat_url = f"{self.api_endpoint}/ml/v1/text/chat?version=2023-05-02"
@@ -129,7 +142,7 @@ class WatsonXLLMKitWrapper(WatsonXProvider, LLMKitWrapper):
             "model_id": self.model_id,
             "messages": sentence,
             "parameters": self.params,
-            "space_id": self.space_id
+            "space_id": self.space_id,
         }
         resp = requests.post(url=chat_url, headers=headers, json=data)
         if resp.status_code == 200:

wxo_agentic_evaluation/service_provider/watsonx_provider.py

@@ -1,11 +1,13 @@
-import os
-import requests
+import dataclasses
 import json
+import os
+import time
+from threading import Lock
 from types import MappingProxyType
 from typing import List, Mapping, Union
-import dataclasses
-from threading import Lock
-import time
+
+import requests
+
 from wxo_agentic_evaluation.service_provider.provider import Provider
 
 ACCESS_URL = "https://iam.cloud.ibm.com/identity/token"
@@ -36,7 +38,9 @@ class WatsonXProvider(Provider):
         super().__init__()
         self.url = url
         if (embedding_model_id is None) and (model_id is None):
-            raise Exception("either model_id or embedding_model_id must be specified")
+            raise Exception(
+                "either model_id or embedding_model_id must be specified"
+            )
         self.model_id = model_id
         api_key = os.environ.get("WATSONX_APIKEY", api_key)
         if not api_key:
@@ -56,7 +60,7 @@ class WatsonXProvider(Provider):
         self.lock = Lock()
 
         self.params = params if params else DEFAULT_PARAM
-        
+
         if isinstance(self.params, MappingProxyType):
             self.params = dict(self.params)
         if dataclasses.is_dataclass(self.params):
@@ -68,7 +72,10 @@ class WatsonXProvider(Provider):
 
     def _get_access_token(self):
         response = requests.post(
-            self.url, headers=ACCESS_HEADER, data=self.access_data, timeout=self.timeout
+            self.url,
+            headers=ACCESS_HEADER,
+            data=self.access_data,
+            timeout=self.timeout,
         )
         if response.status_code == 200:
             token_data = json.loads(response.text)
@@ -84,16 +91,24 @@ class WatsonXProvider(Provider):
         )
 
     def prepare_header(self):
-        headers = {"Authorization": f"Bearer {self.access_token}",
-                  "Content-Type": "application/json"}
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json",
+        }
         return headers
 
     def _query(self, sentence: str):
         headers = self.prepare_header()
 
-        data = {"model_id": self.model_id, "input": sentence,
-                "parameters": self.params, "space_id": self.space_id}
-        generation_url = f"{self.api_endpoint}/ml/v1/text/generation?version=2023-05-02"
+        data = {
+            "model_id": self.model_id,
+            "input": sentence,
+            "parameters": self.params,
+            "space_id": self.space_id,
+        }
+        generation_url = (
+            f"{self.api_endpoint}/ml/v1/text/generation?version=2023-05-02"
+        )
         resp = requests.post(url=generation_url, headers=headers, json=data)
         if resp.status_code == 200:
             return resp.json()["results"][0]
@@ -105,20 +120,25 @@ class WatsonXProvider(Provider):
         if not self.access_token or time.time() > self.refresh_time:
             with self.lock:
                 if not self.access_token or time.time() > self.refresh_time:
-                    self.access_token, self.refresh_time = self._get_access_token()
+                    (
+                        self.access_token,
+                        self.refresh_time,
+                    ) = self._get_access_token()
 
     def query(self, sentence: Union[str, Mapping[str, str]]) -> str:
         if self.model_id is None:
             raise Exception("model id must be specified for text generation")
         try:
             response = self._query(sentence)
-            if (generated_text := response.get("generated_text")):
+            if generated_text := response.get("generated_text"):
                 return generated_text
-            elif (message := response.get("message")):
+            elif message := response.get("message"):
                 return message
             else:
-                raise ValueError(f"Unexpected response from WatsonX: {response}")
-            
+                raise ValueError(
+                    f"Unexpected response from WatsonX: {response}"
+                )
+
         except Exception as e:
             with self.lock:
                 if "authentication_token_expired" in str(e):
@@ -130,12 +150,18 @@ class WatsonXProvider(Provider):
 
     def encode(self, sentences: List[str]) -> List[list]:
         if self.embedding_model_id is None:
-            raise Exception("embedding model id must be specified for text encoding")
+            raise Exception(
+                "embedding model id must be specified for text encoding"
+            )
 
         headers = self.prepare_header()
         url = f"{self.api_endpoint}/ml/v1/text/embeddings?version=2023-10-25"
 
-        data = {"inputs": sentences, "model_id": self.model_id, "space_id": self.space_id}
+        data = {
+            "inputs": sentences,
+            "model_id": self.model_id,
+            "space_id": self.space_id,
+        }
         resp = requests.post(url=url, headers=headers, json=data)
         if resp.status_code == 200:
             return [entry["embedding"] for entry in resp.json()["results"]]
@@ -144,7 +170,9 @@ class WatsonXProvider(Provider):
 
 
 if __name__ == "__main__":
-    provider = WatsonXProvider(model_id="meta-llama/llama-3-2-90b-vision-instruct")
+    provider = WatsonXProvider(
+        model_id="meta-llama/llama-3-2-90b-vision-instruct"
+    )
 
     prompt = """
 <|begin_of_text|><|start_header_id|>system<|end_header_id|>
@@ -176,4 +204,4 @@ Usernwaters did not take anytime off during the period<|eot_id|>
 <|eot_id|><|start_header_id|>user<|end_header_id|>
 """
 
-    print(provider.query(prompt))
+    print(provider.query(prompt))

wxo_agentic_evaluation/tool_planner.py

@@ -1,26 +1,35 @@
-import json
 import ast
 import csv
-from pathlib import Path
 import importlib.util
-import re
-from jsonargparse import CLI
+import json
 import os
+import re
 import sys
 import textwrap
-from dataclasses import is_dataclass, asdict
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+
+from jsonargparse import CLI
 
-from wxo_agentic_evaluation.service_provider import get_provider
-from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
-from wxo_agentic_evaluation.prompt.template_render import ToolPlannerTemplateRenderer, ArgsExtractorTemplateRenderer
 from wxo_agentic_evaluation import __file__
+from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
+from wxo_agentic_evaluation.prompt.template_render import (
+    ArgsExtractorTemplateRenderer,
+    ToolPlannerTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
 
 root_dir = os.path.dirname(__file__)
-TOOL_PLANNER_PROMPT_PATH = os.path.join(root_dir, "prompt", "tool_planner.jinja2")
-ARGS_EXTRACTOR_PROMPT_PATH = os.path.join(root_dir, "prompt", "args_extractor_prompt.jinja2")
+TOOL_PLANNER_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "tool_planner.jinja2"
+)
+ARGS_EXTRACTOR_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "args_extractor_prompt.jinja2"
+)
 
 MISSING_DOCSTRING_PROMPT = "No description available"
 
+
 class UniversalEncoder(json.JSONEncoder):
     def default(self, obj):
         if is_dataclass(obj):
@@ -29,12 +38,15 @@ class UniversalEncoder(json.JSONEncoder):
             return obj.__dict__
         return super().default(obj)
 
+
 def extract_first_json_list(raw: str) -> list:
     matches = re.findall(r"\[\s*{.*?}\s*]", raw, re.DOTALL)
     for match in matches:
         try:
             parsed = json.loads(match)
-            if isinstance(parsed, list) and all("tool_name" in step for step in parsed):
+            if isinstance(parsed, list) and all(
+                "tool_name" in step for step in parsed
+            ):
                 return parsed
         except Exception:
             continue
@@ -42,6 +54,7 @@ def extract_first_json_list(raw: str) -> list:
     print(raw)
     return []
 
+
 def parse_json_string(input_string):
     json_char_count = 0
     json_objects = []
@@ -79,12 +92,16 @@ def load_tools_module(tools_path: Path) -> dict:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
             module_name = file_path.stem
-            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            spec = importlib.util.spec_from_file_location(
+                module_name, file_path
+            )
             module = importlib.util.module_from_spec(spec)
             parent_dir = str(file_path.parent)
             sys_path_modified = False
@@ -99,7 +116,7 @@ def load_tools_module(tools_path: Path) -> dict:
             # Add all module's non-private functions to tools_dict
             for attr_name in dir(module):
                 attr = getattr(module, attr_name)
-                if callable(attr) and not attr_name.startswith('_'):
+                if callable(attr) and not attr_name.startswith("_"):
                     tools_dict[attr_name] = attr
         except Exception as e:
             print(f"Warning: Failed to load {file_path}: {str(e)}")
@@ -117,7 +134,9 @@ def extract_tool_signatures(tools_path: Path) -> list:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
@@ -128,19 +147,24 @@ def extract_tool_signatures(tools_path: Path) -> list:
             for node in parsed_code.body:
                 if isinstance(node, ast.FunctionDef):
                     name = node.name
-                    args = [arg.arg for arg in node.args.args if arg.arg != "self"]
+                    args = [
+                        arg.arg for arg in node.args.args if arg.arg != "self"
+                    ]
                     docstring = ast.get_docstring(node)
-                    tool_data.append({
-                        "Function Name": name,
-                        "Arguments": args,
-                        "Docstring": docstring or MISSING_DOCSTRING_PROMPT
-                    })
+                    tool_data.append(
+                        {
+                            "Function Name": name,
+                            "Arguments": args,
+                            "Docstring": docstring or MISSING_DOCSTRING_PROMPT,
+                        }
+                    )
         except Exception as e:
             print(f"Warning: Failed to parse {file_path}: {str(e)}")
             continue
 
     return tool_data
 
+
 def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
     functions = {}
     files_to_parse = []
@@ -151,7 +175,9 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
@@ -168,23 +194,35 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
                     for arg in node.args.args:
                         if arg.arg == "self":
                             continue
-                        annotation = ast.unparse(arg.annotation) if arg.annotation else "Any"
+                        annotation = (
+                            ast.unparse(arg.annotation)
+                            if arg.annotation
+                            else "Any"
+                        )
                         args.append((arg.arg, annotation))
 
                     # Get return type
-                    returns = ast.unparse(node.returns) if node.returns else "None"
+                    returns = (
+                        ast.unparse(node.returns) if node.returns else "None"
+                    )
 
                     # Get docstring
                     docstring = ast.get_docstring(node)
-                    docstring = textwrap.dedent(docstring).strip() if docstring else ""
+                    docstring = (
+                        textwrap.dedent(docstring).strip() if docstring else ""
+                    )
 
                     # Format parameter descriptions if available in docstring
                     doc_lines = docstring.splitlines()
                     doc_summary = doc_lines[0] if doc_lines else ""
-                    param_descriptions = "\n".join([line for line in doc_lines[1:] if ":param" in line])
+                    param_descriptions = "\n".join(
+                        [line for line in doc_lines[1:] if ":param" in line]
+                    )
 
                     # Compose the final string
-                    args_str = ", ".join(f"{arg}: {type_}" for arg, type_ in args)
+                    args_str = ", ".join(
+                        f"{arg}: {type_}" for arg, type_ in args
+                    )
                     function_str = f"""def {name}({args_str}) -> {returns}:
     {doc_summary}"""
                     if param_descriptions:
@@ -197,9 +235,18 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
 
     return functions
 
-def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module: dict, tool_signatures_for_prompt) -> dict:
+
+def ensure_data_available(
+    step: dict,
+    inputs: dict,
+    snapshot: dict,
+    tools_module: dict,
+    tool_signatures_for_prompt,
+) -> dict:
     tool_name = step["tool_name"]
-    cache = snapshot.setdefault("input_output_examples", {}).setdefault(tool_name, [])
+    cache = snapshot.setdefault("input_output_examples", {}).setdefault(
+        tool_name, []
+    )
     for entry in cache:
         if entry["inputs"] == inputs:
             return entry["output"]
@@ -212,7 +259,11 @@ def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module
     except:
         provider = get_provider(
             model_id="meta-llama/llama-3-405b-instruct",
-            params={"min_new_tokens": 0, "decoding_method": "greedy", "max_new_tokens": 500},
+            params={
+                "min_new_tokens": 0,
+                "decoding_method": "greedy",
+                "max_new_tokens": 500,
+            },
         )
         renderer = ArgsExtractorTemplateRenderer(ARGS_EXTRACTOR_PROMPT_PATH)
 
@@ -226,14 +277,19 @@ def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module
         try:
             output = tools_module[json_obj["tool_name"]](**json_obj["inputs"])
         except:
-            raise ValueError(f"Failed to execute tool '{tool_name}' with inputs {inputs}")
+            raise ValueError(
+                f"Failed to execute tool '{tool_name}' with inputs {inputs}"
+            )
 
     cache.append({"inputs": inputs, "output": output})
     if not isinstance(output, dict):
         print(f" Tool {tool_name} returned non-dict output: {output}")
     return output
 
-def plan_tool_calls_with_llm(story: str, agent_name: str, tool_signatures_str: str, provider) -> list:
+
+def plan_tool_calls_with_llm(
+    story: str, agent_name: str, tool_signatures_str: str, provider
+) -> list:
 
     renderer = ToolPlannerTemplateRenderer(TOOL_PLANNER_PROMPT_PATH)
 
@@ -250,7 +306,9 @@ def plan_tool_calls_with_llm(story: str, agent_name: str, tool_signatures_str: s
 
 
 # --- Tool Execution Logic ---
-def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signatures_for_prompt) -> None:
+def run_tool_chain(
+    tool_plan: list, snapshot: dict, tools_module, tool_signatures_for_prompt
+) -> None:
     memory = {}
 
     for step in tool_plan:
@@ -280,7 +338,9 @@ def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signature
 
         if list_keys:
             if len(list_keys) > 1:
-                raise ValueError(f"Tool '{name}' received multiple list inputs. Only one supported for now.")
+                raise ValueError(
+                    f"Tool '{name}' received multiple list inputs. Only one supported for now."
+                )
             list_key = list_keys[0]
             value_list = resolved_inputs[list_key]
 
@@ -289,20 +349,36 @@ def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signature
                 item_inputs = resolved_inputs.copy()
                 item_inputs[list_key] = val
                 print(f" ⚙️ Running {name} with {list_key} = {val}")
-                output = ensure_data_available(step, item_inputs, snapshot, tools_module, tool_signatures_for_prompt)
+                output = ensure_data_available(
+                    step,
+                    item_inputs,
+                    snapshot,
+                    tools_module,
+                    tool_signatures_for_prompt,
+                )
                 results.append(output)
                 memory[f"{name}_{idx}"] = output
 
             memory[name] = results
-            print(f"Stored {len(results)} outputs under '{name}' and indexed as '{name}_i'")
+            print(
+                f"Stored {len(results)} outputs under '{name}' and indexed as '{name}_i'"
+            )
         else:
-            output = ensure_data_available(step, resolved_inputs, snapshot, tools_module, tool_signatures_for_prompt)
+            output = ensure_data_available(
+                step,
+                resolved_inputs,
+                snapshot,
+                tools_module,
+                tool_signatures_for_prompt,
+            )
             memory[name] = output
             print(f"Stored output under tool name: {name} = {output}")
 
 
 # --- Main Snapshot Builder ---
-def build_snapshot(agent_name: str, tools_path: Path, stories: list, output_path: Path):
+def build_snapshot(
+    agent_name: str, tools_path: Path, stories: list, output_path: Path
+):
     agent = {"name": agent_name}
     tools_module = load_tools_module(tools_path)
     tool_signatures = extract_tool_signatures(tools_path)
@@ -310,20 +386,28 @@ def build_snapshot(agent_name: str, tools_path: Path, stories: list, output_path
 
     provider = get_provider(
         model_id="meta-llama/llama-3-405b-instruct",
-        params={"min_new_tokens": 1, "decoding_method": "greedy", "max_new_tokens": 2048},
+        params={
+            "min_new_tokens": 1,
+            "decoding_method": "greedy",
+            "max_new_tokens": 2048,
+        },
     )
 
     snapshot = {
         "agent": agent,
         "tools": tool_signatures,
-        "input_output_examples": {}
+        "input_output_examples": {},
     }
 
     for story in stories:
         print(f"\n📘 Planning tool calls for story: {story}")
-        tool_plan = plan_tool_calls_with_llm(story, agent["name"], tool_signatures, provider)
+        tool_plan = plan_tool_calls_with_llm(
+            story, agent["name"], tool_signatures, provider
+        )
         try:
-            run_tool_chain(tool_plan, snapshot, tools_module, tool_signatures_for_prompt)
+            run_tool_chain(
+                tool_plan, snapshot, tools_module, tool_signatures_for_prompt
+            )
         except ValueError as e:
             print(f"❌ Error running tool chain for story '{story}': {e}")
             continue
@@ -340,7 +424,7 @@ if __name__ == "__main__":
 
     stories = []
     agent_name = None
-    with stories_path.open("r", encoding="utf-8", newline='') as f:
+    with stories_path.open("r", encoding="utf-8", newline="") as f:
         csv_reader = csv.DictReader(f)
         for row in csv_reader:
             stories.append(row["story"])
@@ -349,4 +433,4 @@ if __name__ == "__main__":
 
     snapshot_path = stories_path.parent / f"{agent_name}_snapshot_llm.json"
 
-    build_snapshot(agent_name, tools_path, stories, snapshot_path)
+    build_snapshot(agent_name, tools_path, stories, snapshot_path)

wxo_agentic_evaluation/type.py

@@ -1,10 +1,7 @@
-from typing import Dict, List, Union, Any, Optional
-from pydantic import (
-    BaseModel, 
-    ConfigDict, 
-    Field
-    )
 from enum import StrEnum
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
 from rich.text import Text
 
 
@@ -61,9 +58,13 @@ class ConversationalConfidenceThresholdScore(BaseModel):
     def table(self):
         return {
             "response_confidence": str(self.response_confidence),
-            "response_confidence_threshold": str(self.response_confidence_threshold),
+            "response_confidence_threshold": str(
+                self.response_confidence_threshold
+            ),
             "retrieval_confidence": str(self.retrieval_confidence),
-            "retrieval_confidence_threshold": str(self.retrieval_confidence_threshold),
+            "retrieval_confidence_threshold": str(
+                self.retrieval_confidence_threshold
+            ),
         }
 
 
@@ -120,12 +121,14 @@ class GoalDetail(BaseModel):
     keywords: List = None
     knowledge_base: KnowledgeBaseGoalDetail = KnowledgeBaseGoalDetail()
 
+
 class AttackData(BaseModel):
     attack_category: AttackCategory
     attack_type: str
     attack_name: str
     attack_instructions: str
 
+
 class AttackData(BaseModel):
     agent: str
     agents_path: str
@@ -143,8 +146,8 @@ class EvaluationData(BaseModel):
     goal_details: List[GoalDetail]
     starting_sentence: str = None
 
+
 class ToolDefinition(BaseModel):
     tool_description: Optional[str]
     tool_name: str
     tool_params: List[str]
-