ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/simluation_runner.py

@@ -0,0 +1,125 @@
+from wxo_agentic_evaluation.evaluation_controller.evaluation_controller import EvaluationController
+from langfuse import get_client
+
+from wxo_agentic_evaluation.runtime_adapter.runtime_adapter import RuntimeAdapter
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import WXORuntimeAdapter
+from wxo_agentic_evaluation.type import Message, RuntimeResponse
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.llm_user_v2 import LLMUserV2
+from wxo_agentic_evaluation.arg_configs import ControllerConfig
+from wxo_agentic_evaluation.hr_agent_langgraph import agent
+
+from dotenv import load_dotenv
+load_dotenv()
+import os
+import base64
+
+os.environ["USE_PORTKEY_PROVIDER"] = "true"
+
+lf_public = os.getenv("LANGFUSE_PUBLIC_KEY")
+lf_secret = os.getenv("LANGFUSE_SECRET_KEY")
+auth_bytes = f"{lf_public}:{lf_secret}".encode("utf-8")
+auth_b64 = base64.b64encode(auth_bytes).decode("ascii")
+HEADERS = {"Authorization": f"Basic {auth_b64}"}
+
+lf_base_url = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com").rstrip("/")
+OTEL_ENDPOINT = f"{lf_base_url}/api/public/otel/v1/traces"
+
+from phoenix.otel import register
+register(endpoint=OTEL_ENDPOINT, headers=HEADERS, auto_instrument=True)
+
+
+context = {"session_id": "1", "chat_history": []}
+
+
+class MyAgentWrapper(RuntimeAdapter):
+    def run(
+        self,
+        user_message: Message,
+        context: dict,
+        thread_id=None,
+    ) -> RuntimeResponse:
+
+        message_json = user_message.model_dump()
+        messages = {"messages": [ message_json ]}
+        result = agent.invoke(messages)
+        # print(result)
+        message = Message(role="assistant", content=result["messages"][-1].content)
+        # messages = [Message(role="assistant", content=msg.content, type="tool_call") for msg in result["messages"]]
+        return RuntimeResponse(messages=[message])
+
+
+
+agent_wrapper = MyAgentWrapper()
+from openinference.instrumentation import using_session
+
+
+class SimulationRunner:
+    def __init__(self, user_agent: LLMUser,
+                 agent: RuntimeAdapter,
+                 config: ControllerConfig):
+        self.evaluation_controller = EvaluationController(
+            runtime=agent,
+            llm_user=user_agent,
+            config=config,
+        )
+        self.counter = 0
+
+
+    def run_wrapper(self, session_id = 'session-id-test-00'):
+        def run_task(*, item, **kwargs):
+            """
+            Task function for Langfuse experiment.
+            Item input should be: {"persona": "...", "scenario": "..."}
+            """
+            # print(item)
+            with using_session(session_id + "-" + self.counter.__str__()):
+                input = item.input
+                user_story = input.get("story")
+                starting_sentence = input.get("starting_sentence")
+                agent_name = input.get("agent")
+                _, _, _, thread_id = self.evaluation_controller.run(self.counter, agent_name=agent_name, story=user_story, starting_user_input=starting_sentence)
+                self.counter += 1
+                if isinstance(self.evaluation_controller.runtime, WXORuntimeAdapter):
+                    return thread_id
+            return session_id
+        
+
+        return run_task
+
+if __name__ == "__main__":
+    import json
+    with open("benchmarks/hr_sample/data_simple.json") as f:
+        data = json.load(f)
+    langfuse = get_client()
+    langfuse.create_dataset(name="dataset-test-00")
+    # Upload to Langfuse
+
+    langfuse.create_dataset_item(
+        dataset_name="dataset-test-00",
+        # any python object or value
+        input={"story": data["story"], "starting_sentence": data["starting_sentence"]},
+        # any python object or value, optional
+        expected_output={"goals": data["goals"], "goal_details": data["goal_details"]},
+    )
+    from wxo_agentic_evaluation.service_provider import get_provider
+
+    model_id = "gpt-4o-mini"
+    provider = get_provider(provider="openai", model_id=model_id, api_key=os.getenv("OPENAI_API_KEY"),
+                            use_portkey_provider=True)
+    llm_user = LLMUserV2(llm_client=provider, user_prompt_path="src/wxo_agentic_evaluation/prompt/universal_user_template.jinja2")
+    config = ControllerConfig()
+    simluation_runner = SimulationRunner(agent = agent_wrapper, user_agent=llm_user, config=config)
+    dataset = langfuse.get_dataset("dataset-test-00")
+
+    result = dataset.run_experiment(
+        name="experiment-test-00",
+        description="Synthetic conversations from persona/scenario pairs",
+        task=simluation_runner.run_wrapper()
+    )
+    
+    get_client().flush()
+    session_id = "dummy-1"
+    with using_session(session_id):
+        result = agent_wrapper.run(Message(role="user", content="hi"), context={})
+    print(result)

wxo_agentic_evaluation/test_prompt.py

@@ -1,7 +1,6 @@
 from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
 
 
-
 def parse_json_string(input_string):
     json_char_count = 0
     json_objects = []
@@ -31,9 +30,10 @@ def parse_json_string(input_string):
     is_thinking_step = len(input_string) - json_char_count > 10
     return json_objects
 
+
 wai_client = WatsonXProvider(model_id="meta-llama/llama-3-405b-instruct")
 
-prompt =  """
+prompt = """
 <|begin_of_text|><|start_header_id|>system<|end_header_id|>
 You are trying to make tool calls. Given a raw input and tool output. Try to extract the information to make the tool call
 
@@ -83,12 +83,12 @@ test_sample2 = """
  <|start_header_id|>ipython<|end_header_id|>"""
 
 
-
 outputs = wai_client.query(prompt + test_sample1)
 
 import json
+
 print(outputs["generated_text"])
 
 json_obj = parse_json_string(outputs["generated_text"])[0]
 
-print(json_obj)
+print(json_obj)

wxo_agentic_evaluation/tool_planner.py

@@ -1,22 +1,34 @@
-import json
 import ast
 import csv
-from pathlib import Path
 import importlib.util
-import re
-from jsonargparse import CLI
+import json
 import os
+import re
+import sys
 import textwrap
-from dataclasses import is_dataclass, asdict
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+
+from jsonargparse import CLI
 
-from wxo_agentic_evaluation.service_provider import get_provider
-from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
-from wxo_agentic_evaluation.prompt.template_render import ToolPlannerTemplateRenderer, ArgsExtractorTemplateRenderer
 from wxo_agentic_evaluation import __file__
+from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
+from wxo_agentic_evaluation.prompt.template_render import (
+    ArgsExtractorTemplateRenderer,
+    ToolPlannerTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
 
 root_dir = os.path.dirname(__file__)
-TOOL_PLANNER_PROMPT_PATH = os.path.join(root_dir, "prompt", "tool_planner.jinja2")
-ARGS_EXTRACTOR_PROMPT_PATH = os.path.join(root_dir, "prompt", "args_extractor_prompt.jinja2")
+TOOL_PLANNER_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "tool_planner.jinja2"
+)
+ARGS_EXTRACTOR_PROMPT_PATH = os.path.join(
+    root_dir, "prompt", "args_extractor_prompt.jinja2"
+)
+
+MISSING_DOCSTRING_PROMPT = "No description available"
+
 
 class UniversalEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -26,12 +38,15 @@ class UniversalEncoder(json.JSONEncoder):
             return obj.__dict__
         return super().default(obj)
 
+
 def extract_first_json_list(raw: str) -> list:
     matches = re.findall(r"\[\s*{.*?}\s*]", raw, re.DOTALL)
     for match in matches:
         try:
             parsed = json.loads(match)
-            if isinstance(parsed, list) and all("tool_name" in step for step in parsed):
+            if isinstance(parsed, list) and all(
+                "tool_name" in step for step in parsed
+            ):
                 return parsed
         except Exception:
             continue
@@ -39,6 +54,7 @@ def extract_first_json_list(raw: str) -> list:
     print(raw)
     return []
 
+
 def parse_json_string(input_string):
     json_char_count = 0
     json_objects = []
@@ -76,19 +92,31 @@ def load_tools_module(tools_path: Path) -> dict:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
             module_name = file_path.stem
-            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            spec = importlib.util.spec_from_file_location(
+                module_name, file_path
+            )
             module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-
+            parent_dir = str(file_path.parent)
+            sys_path_modified = False
+            if parent_dir not in sys.path:
+                sys.path.append(parent_dir)
+                sys_path_modified = True
+            try:
+                spec.loader.exec_module(module)
+            finally:
+                if sys_path_modified:
+                    sys.path.pop()
             # Add all module's non-private functions to tools_dict
             for attr_name in dir(module):
                 attr = getattr(module, attr_name)
-                if callable(attr) and not attr_name.startswith('_'):
+                if callable(attr) and not attr_name.startswith("_"):
                     tools_dict[attr_name] = attr
         except Exception as e:
             print(f"Warning: Failed to load {file_path}: {str(e)}")
@@ -106,7 +134,9 @@ def extract_tool_signatures(tools_path: Path) -> list:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
@@ -117,19 +147,24 @@ def extract_tool_signatures(tools_path: Path) -> list:
             for node in parsed_code.body:
                 if isinstance(node, ast.FunctionDef):
                     name = node.name
-                    args = [arg.arg for arg in node.args.args if arg.arg != "self"]
+                    args = [
+                        arg.arg for arg in node.args.args if arg.arg != "self"
+                    ]
                     docstring = ast.get_docstring(node)
-                    tool_data.append({
-                        "Function Name": name,
-                        "Arguments": args,
-                        "Docstring": docstring or "No description available"
-                    })
+                    tool_data.append(
+                        {
+                            "Function Name": name,
+                            "Arguments": args,
+                            "Docstring": docstring or MISSING_DOCSTRING_PROMPT,
+                        }
+                    )
         except Exception as e:
             print(f"Warning: Failed to parse {file_path}: {str(e)}")
             continue
 
     return tool_data
 
+
 def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
     functions = {}
     files_to_parse = []
@@ -140,7 +175,9 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
     elif tools_path.is_dir():
         files_to_parse.extend(tools_path.glob("**/*.py"))
     else:
-        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+        raise ValueError(
+            f"Tools path {tools_path} is neither a file nor directory"
+        )
 
     for file_path in files_to_parse:
         try:
@@ -157,23 +194,35 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
                     for arg in node.args.args:
                         if arg.arg == "self":
                             continue
-                        annotation = ast.unparse(arg.annotation) if arg.annotation else "Any"
+                        annotation = (
+                            ast.unparse(arg.annotation)
+                            if arg.annotation
+                            else "Any"
+                        )
                         args.append((arg.arg, annotation))
 
                     # Get return type
-                    returns = ast.unparse(node.returns) if node.returns else "None"
+                    returns = (
+                        ast.unparse(node.returns) if node.returns else "None"
+                    )
 
                     # Get docstring
                     docstring = ast.get_docstring(node)
-                    docstring = textwrap.dedent(docstring).strip() if docstring else ""
+                    docstring = (
+                        textwrap.dedent(docstring).strip() if docstring else ""
+                    )
 
                     # Format parameter descriptions if available in docstring
                     doc_lines = docstring.splitlines()
                     doc_summary = doc_lines[0] if doc_lines else ""
-                    param_descriptions = "\n".join([line for line in doc_lines[1:] if ":param" in line])
+                    param_descriptions = "\n".join(
+                        [line for line in doc_lines[1:] if ":param" in line]
+                    )
 
                     # Compose the final string
-                    args_str = ", ".join(f"{arg}: {type_}" for arg, type_ in args)
+                    args_str = ", ".join(
+                        f"{arg}: {type_}" for arg, type_ in args
+                    )
                     function_str = f"""def {name}({args_str}) -> {returns}:
     {doc_summary}"""
                     if param_descriptions:
@@ -186,9 +235,18 @@ def extract_tool_signatures_for_prompt(tools_path: Path) -> dict[str, str]:
 
     return functions
 
-def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module: dict, tool_signatures_for_prompt) -> dict:
+
+def ensure_data_available(
+    step: dict,
+    inputs: dict,
+    snapshot: dict,
+    tools_module: dict,
+    tool_signatures_for_prompt,
+) -> dict:
     tool_name = step["tool_name"]
-    cache = snapshot.setdefault("input_output_examples", {}).setdefault(tool_name, [])
+    cache = snapshot.setdefault("input_output_examples", {}).setdefault(
+        tool_name, []
+    )
     for entry in cache:
         if entry["inputs"] == inputs:
             return entry["output"]
@@ -201,7 +259,11 @@ def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module
     except:
         provider = get_provider(
             model_id="meta-llama/llama-3-405b-instruct",
-            params={"min_new_tokens": 0, "decoding_method": "greedy", "max_new_tokens": 500},
+            params={
+                "min_new_tokens": 0,
+                "decoding_method": "greedy",
+                "max_new_tokens": 500,
+            },
         )
         renderer = ArgsExtractorTemplateRenderer(ARGS_EXTRACTOR_PROMPT_PATH)
 
@@ -215,14 +277,19 @@ def ensure_data_available(step: dict, inputs: dict, snapshot: dict, tools_module
         try:
             output = tools_module[json_obj["tool_name"]](**json_obj["inputs"])
         except:
-            raise ValueError(f"Failed to execute tool '{tool_name}' with inputs {inputs}")
+            raise ValueError(
+                f"Failed to execute tool '{tool_name}' with inputs {inputs}"
+            )
 
     cache.append({"inputs": inputs, "output": output})
     if not isinstance(output, dict):
         print(f" Tool {tool_name} returned non-dict output: {output}")
     return output
 
-def plan_tool_calls_with_llm(story: str, agent_name: str, tool_signatures_str: str, provider) -> list:
+
+def plan_tool_calls_with_llm(
+    story: str, agent_name: str, tool_signatures_str: str, provider
+) -> list:
 
     renderer = ToolPlannerTemplateRenderer(TOOL_PLANNER_PROMPT_PATH)
 
@@ -239,7 +306,9 @@ def plan_tool_calls_with_llm(story: str, agent_name: str, tool_signatures_str: s
 
 
 # --- Tool Execution Logic ---
-def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signatures_for_prompt) -> None:
+def run_tool_chain(
+    tool_plan: list, snapshot: dict, tools_module, tool_signatures_for_prompt
+) -> None:
     memory = {}
 
     for step in tool_plan:
@@ -269,7 +338,9 @@ def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signature
 
         if list_keys:
             if len(list_keys) > 1:
-                raise ValueError(f"Tool '{name}' received multiple list inputs. Only one supported for now.")
+                raise ValueError(
+                    f"Tool '{name}' received multiple list inputs. Only one supported for now."
+                )
             list_key = list_keys[0]
             value_list = resolved_inputs[list_key]
 
@@ -278,20 +349,36 @@ def run_tool_chain(tool_plan: list, snapshot: dict, tools_module, tool_signature
                 item_inputs = resolved_inputs.copy()
                 item_inputs[list_key] = val
                 print(f" ⚙️ Running {name} with {list_key} = {val}")
-                output = ensure_data_available(step, item_inputs, snapshot, tools_module, tool_signatures_for_prompt)
+                output = ensure_data_available(
+                    step,
+                    item_inputs,
+                    snapshot,
+                    tools_module,
+                    tool_signatures_for_prompt,
+                )
                 results.append(output)
                 memory[f"{name}_{idx}"] = output
 
             memory[name] = results
-            print(f"Stored {len(results)} outputs under '{name}' and indexed as '{name}_i'")
+            print(
+                f"Stored {len(results)} outputs under '{name}' and indexed as '{name}_i'"
+            )
         else:
-            output = ensure_data_available(step, resolved_inputs, snapshot, tools_module, tool_signatures_for_prompt)
+            output = ensure_data_available(
+                step,
+                resolved_inputs,
+                snapshot,
+                tools_module,
+                tool_signatures_for_prompt,
+            )
             memory[name] = output
             print(f"Stored output under tool name: {name} = {output}")
 
 
 # --- Main Snapshot Builder ---
-def build_snapshot(agent_name: str, tools_path: Path, stories: list, output_path: Path):
+def build_snapshot(
+    agent_name: str, tools_path: Path, stories: list, output_path: Path
+):
     agent = {"name": agent_name}
     tools_module = load_tools_module(tools_path)
     tool_signatures = extract_tool_signatures(tools_path)
@@ -299,20 +386,28 @@ def build_snapshot(agent_name: str, tools_path: Path, stories: list, output_path
 
     provider = get_provider(
         model_id="meta-llama/llama-3-405b-instruct",
-        params={"min_new_tokens": 1, "decoding_method": "greedy", "max_new_tokens": 2048},
+        params={
+            "min_new_tokens": 1,
+            "decoding_method": "greedy",
+            "max_new_tokens": 2048,
+        },
     )
 
     snapshot = {
         "agent": agent,
         "tools": tool_signatures,
-        "input_output_examples": {}
+        "input_output_examples": {},
     }
 
     for story in stories:
         print(f"\n📘 Planning tool calls for story: {story}")
-        tool_plan = plan_tool_calls_with_llm(story, agent["name"], tool_signatures, provider)
+        tool_plan = plan_tool_calls_with_llm(
+            story, agent["name"], tool_signatures, provider
+        )
         try:
-            run_tool_chain(tool_plan, snapshot, tools_module, tool_signatures_for_prompt)
+            run_tool_chain(
+                tool_plan, snapshot, tools_module, tool_signatures_for_prompt
+            )
         except ValueError as e:
             print(f"❌ Error running tool chain for story '{story}': {e}")
             continue
@@ -329,7 +424,7 @@ if __name__ == "__main__":
 
     stories = []
     agent_name = None
-    with stories_path.open("r", encoding="utf-8", newline='') as f:
+    with stories_path.open("r", encoding="utf-8", newline="") as f:
         csv_reader = csv.DictReader(f)
         for row in csv_reader:
             stories.append(row["story"])
@@ -338,4 +433,4 @@ if __name__ == "__main__":
 
     snapshot_path = stories_path.parent / f"{agent_name}_snapshot_llm.json"
 
-    build_snapshot(agent_name, tools_path, stories, snapshot_path)
+    build_snapshot(agent_name, tools_path, stories, snapshot_path)