ibm-watsonx-orchestrate 1.8.0b0__py3-none-any.whl → 1.8.1__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/copilot/copilot_controller.py

@@ -11,8 +11,9 @@ from requests import ConnectionError
 from typing import List
 from ibm_watsonx_orchestrate.client.base_api_client import ClientAPIException
 from ibm_watsonx_orchestrate.agent_builder.tools import ToolSpec, ToolPermission, ToolRequestBody, ToolResponseBody
-from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController, AgentKind
-from ibm_watsonx_orchestrate.agent_builder.agents.types import DEFAULT_LLM
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController, AgentKind, SpecVersion
+from ibm_watsonx_orchestrate.agent_builder.agents.types import DEFAULT_LLM, BaseAgentSpec
+from ibm_watsonx_orchestrate.client.agents.agent_client import AgentClient
 from ibm_watsonx_orchestrate.client.tools.tool_client import ToolClient
 from ibm_watsonx_orchestrate.client.copilot.cpe.copilot_cpe_client import CPEClient
 from ibm_watsonx_orchestrate.client.utils import instantiate_client
@@ -20,21 +21,24 @@ from ibm_watsonx_orchestrate.utils.exceptions import BadRequest
 
 logger = logging.getLogger(__name__)
 
+
 def _validate_output_file(output_file: str, dry_run_flag: bool) -> None:
     if not output_file and not dry_run_flag:
-        logger.error("Please provide a valid yaml output file. Or use the `--dry-run` flag to output generated agent content to terminal")
+        logger.error(
+            "Please provide a valid yaml output file. Or use the `--dry-run` flag to output generated agent content to terminal")
         sys.exit(1)
-    
+
     if output_file and dry_run_flag:
         logger.error("Cannot set output file when performing a dry run")
         sys.exit(1)
-    
+
     if output_file:
         _, file_extension = os.path.splitext(output_file)
-        if file_extension not in  {".yaml", ".yml", ".json"}:
+        if file_extension not in {".yaml", ".yml", ".json"}:
             logger.error("Output file must be of type '.yaml', '.yml' or '.json'")
             sys.exit(1)
 
+
 def _get_progress_spinner() -> Progress:
     console = Console()
     return Progress(
@@ -44,16 +48,22 @@ def _get_progress_spinner() -> Progress:
         console=console,
     )
 
+
 def _get_incomplete_tool_from_name(tool_name: str) -> dict:
     input_schema = ToolRequestBody(**{"type": "object", "properties": {}})
     output_schema = ToolResponseBody(**{"description": "None"})
-    spec = ToolSpec(**{"name": tool_name, "description": tool_name, "permission": ToolPermission.ADMIN, "input_schema": input_schema, "output_schema": output_schema})
+    spec = ToolSpec(**{"name": tool_name, "description": tool_name, "permission": ToolPermission.ADMIN,
+                       "input_schema": input_schema, "output_schema": output_schema})
+    return spec.model_dump()
+
+def _get_incomplete_agent_from_name(agent_name: str) -> dict:
+    spec = BaseAgentSpec(**{"name": agent_name, "description": agent_name, "kind": AgentKind.NATIVE})
     return spec.model_dump()
 
 def _get_tools_from_names(tool_names: List[str]) -> List[dict]:
     if not len(tool_names):
         return []
-    
+
     tool_client = get_tool_client()
 
     try:
@@ -61,25 +71,63 @@ def _get_tools_from_names(tool_names: List[str]) -> List[dict]:
             task = progress.add_task(description="Fetching tools", total=None)
             tools = tool_client.get_drafts_by_names(tool_names)
             found_tools = {tool.get("name") for tool in tools}
-            rich.print("\n")
+            progress.remove_task(task)
+            progress.refresh()
             for tool_name in tool_names:
                 if tool_name not in found_tools:
-                    logger.warning(f"Failed to find tool named '{tool_name}'. Falling back to incomplete tool definition. Copilot performance maybe effected.")
+                    logger.warning(
+                        f"Failed to find tool named '{tool_name}'. Falling back to incomplete tool definition. Copilot performance maybe effected.")
                     tools.append(_get_incomplete_tool_from_name(tool_name))
-            progress.remove_task(task)
     except ConnectionError:
-        logger.warning(f"Failed to fetch tools from server. For optimal results please start the server and import the relevant tools {', '.join(tool_names)}.")
+        logger.warning(
+            f"Failed to fetch tools from server. For optimal results please start the server and import the relevant tools {', '.join(tool_names)}.")
         tools = []
         for tool_name in tool_names:
             tools.append(_get_incomplete_tool_from_name(tool_name))
 
     return tools
 
+
+def _get_agents_from_names(collaborators_names: List[str]) -> List[dict]:
+    if not len(collaborators_names):
+        return []
+
+    native_agents_client = get_native_client()
+
+    try:
+        with _get_progress_spinner() as progress:
+            task = progress.add_task(description="Fetching agents", total=None)
+            agents = native_agents_client.get_drafts_by_names(collaborators_names)
+            found_agents = {tool.get("name") for tool in agents}
+            progress.remove_task(task)
+            progress.refresh()
+            for collaborator_name in collaborators_names:
+                if collaborator_name not in found_agents:
+                    logger.warning(
+                        f"Failed to find agent named '{collaborator_name}'. Falling back to incomplete agent definition. Copilot performance maybe effected.")
+                    agents.append(_get_incomplete_agent_from_name(collaborator_name))
+    except ConnectionError:
+        logger.warning(
+            f"Failed to fetch tools from server. For optimal results please start the server and import the relevant tools {', '.join(collaborators_names)}.")
+        agents = []
+        for collaborator_name in collaborators_names:
+            agents.append(_get_incomplete_agent_from_name(collaborator_name))
+
+    return agents
+
 def get_cpe_client() -> CPEClient:
     url = os.getenv('CPE_URL', "http://localhost:8081")
     return instantiate_client(client=CPEClient, url=url)
 
 
+def get_tool_client(*args, **kwargs):
+    return instantiate_client(ToolClient)
+
+
+def get_native_client(*args, **kwargs):
+    return instantiate_client(AgentClient)
+
+
 def gather_utterances(max: int) -> list[str]:
     utterances = []
     logger.info("Please provide 3 sample utterances you expect your agent to handle:")
@@ -98,7 +146,16 @@ def gather_utterances(max: int) -> list[str]:
 
     return utterances
 
-def pre_cpe_step(cpe_client, tool_client):
+
+def get_deployed_tools_agents():
+    all_tools = find_tools_by_description(tool_client=get_tool_client(), description=None)
+    # TODO: this brings only the "native" agents. Can external and assistant agents also be collaborators?
+    all_agents = find_agents(agent_client=get_native_client())
+    return {"tools": all_tools, "agents": all_agents}
+
+
+def pre_cpe_step(cpe_client):
+    tools_agents = get_deployed_tools_agents()
     user_message = ""
     with _get_progress_spinner() as progress:
         task = progress.add_task(description="Initilizing Prompt Engine", total=None)
@@ -113,31 +170,47 @@ def pre_cpe_step(cpe_client, tool_client):
             message_content = {"user_message": user_message}
         elif "description" in response and response["description"]:
             res["description"] = response["description"]
-            tools = find_tools_by_description(res["description"], tool_client)
-            message_content = {"tools": tools}
+            message_content = tools_agents
         elif "metadata" in response:
             res["agent_name"] = response["metadata"]["agent_name"]
             res["agent_style"] = response["metadata"]["style"]
-            res["tools"] = [t for t in tools if t["name"] in response["metadata"]["tools"]]
+            res["tools"] = [t for t in tools_agents["tools"] if t["name"] in response["metadata"]["tools"]]
+            res["collaborators"] = [a for a in tools_agents["agents"] if
+                                    a["name"] in response["metadata"]["collaborators"]]
             return res
         with _get_progress_spinner() as progress:
             task = progress.add_task(description="Thinking...", total=None)
             response = cpe_client.submit_pre_cpe_chat(**message_content)
             progress.remove_task(task)
 
-# TODO: Add description RAG search
+
 def find_tools_by_description(description, tool_client):
     with _get_progress_spinner() as progress:
         task = progress.add_task(description="Fetching Tools", total=None)
         try:
             tools = tool_client.get()
+            progress.remove_task(task)
         except ConnectionError:
             tools = []
-            rich.print("\n")
+            progress.remove_task(task)
+            progress.refresh()
             logger.warning("Failed to contact wxo server to fetch tools. Proceeding with empty tool list")
-        progress.remove_task(task)
     return tools
 
+def find_agents(agent_client):
+    with _get_progress_spinner() as progress:
+        task = progress.add_task(description="Fetching Agents", total=None)
+        try:
+            agents = agent_client.get()
+            progress.remove_task(task)
+        except ConnectionError:
+            agents = []
+            progress.remove_task(task)
+            progress.refresh()
+            logger.warning("Failed to contact wxo server to fetch agents. Proceeding with empty agent list")
+    return agents
+
+
 def gather_examples(samples_file=None):
     if samples_file:
         if samples_file.endswith('.txt'):
@@ -167,7 +240,7 @@ def talk_to_cpe(cpe_client, samples_file=None, context_data=None):
     examples = gather_examples(samples_file)
     # upload or gather input examples
     context_data['examples'] = examples
-    response=None
+    response = None
     with _get_progress_spinner() as progress:
         task = progress.add_task(description="Thinking...", total=None)
         response = cpe_client.init_with_context(context_data=context_data)
@@ -199,20 +272,23 @@ def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | No
 
     if not output_file and not dry_run_flag:
         output_file = agent_spec
-    
+
     _validate_output_file(output_file, dry_run_flag)
 
     client = get_cpe_client()
 
     instr = agent.instructions
-    prompt = 'My current prompt is:\n' + instr if instr else "I don't have an initial prompt."
 
     tools = _get_tools_from_names(agent.tools)
 
+    collaborators = _get_agents_from_names(agent.collaborators)
     try:
-        new_prompt = talk_to_cpe(cpe_client=client, samples_file=samples_file, context_data={"prompt": prompt, 'tools': tools, 'description': agent.description})
+        new_prompt = talk_to_cpe(cpe_client=client, samples_file=samples_file,
+                                 context_data={"initial_instruction": instr, 'tools': tools, 'description': agent.description,
+                                               "collaborators": collaborators})
     except ConnectionError:
-        logger.error("Failed to connect to Copilot server. Please ensure Copilot is running via `orchestrate copilot start`")
+        logger.error(
+            "Failed to connect to Copilot server. Please ensure Copilot is running via `orchestrate copilot start`")
         sys.exit(1)
     except ClientAPIException:
         logger.error("An unexpected server error has occur with in the Copilot server. Please check the logs via `orchestrate server logs`")
@@ -223,59 +299,58 @@ def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | No
         agent.instructions = new_prompt
 
         if dry_run_flag:
-            rich.print(agent.model_dump())
+            rich.print(agent.model_dump(exclude_none=True))
         else:
             if os.path.dirname(output_file):
                 os.makedirs(os.path.dirname(output_file), exist_ok=True)
             AgentsController.persist_record(agent, output_file=output_file)
 
 
-def get_tool_client(*args, **kwargs):
-    return instantiate_client(ToolClient)
-
 def create_agent(output_file: str, llm: str, samples_file: str | None, dry_run_flag: bool = False) -> None:
     _validate_output_file(output_file, dry_run_flag)
     # 1. prepare the clients
     cpe_client = get_cpe_client()
-    tool_client = get_tool_client()
 
     # 2. Pre-CPE stage:
     try:
-        res = pre_cpe_step(cpe_client, tool_client)
+        res = pre_cpe_step(cpe_client)
     except ConnectionError:
-        logger.error("Failed to connect to Copilot server. Please ensure Copilot is running via `orchestrate copilot start`")
+        logger.error(
+            "Failed to connect to Copilot server. Please ensure Copilot is running via `orchestrate copilot start`")
         sys.exit(1)
     except ClientAPIException:
         logger.error("An unexpected server error has occur with in the Copilot server. Please check the logs via `orchestrate server logs`")
         sys.exit(1)
         
     tools = res["tools"]
+    collaborators = res["collaborators"]
     description = res["description"]
     agent_name = res["agent_name"]
     agent_style = res["agent_style"]
 
     # 4. discuss the instructions
-    instructions = talk_to_cpe(cpe_client, samples_file, {'tools': tools, 'description': description})
+    instructions = talk_to_cpe(cpe_client, samples_file, {'description': description, 'tools': tools, 'collaborators': collaborators})
 
     # 6. create and save the agent
     llm = llm if llm else DEFAULT_LLM
     params = {
         'style': agent_style,
         'tools': [t['name'] for t in tools],
-        'llm': llm
+        'llm': llm,
+        'collaborators': [c['name'] for c in collaborators]
     }
     agent = AgentsController.generate_agent_spec(agent_name, AgentKind.NATIVE, description, **params)
     agent.instructions = instructions
+    agent.spec_version = SpecVersion.V1
 
     if dry_run_flag:
-        rich.print(agent.model_dump())
+        rich.print(agent.model_dump(exclude_none=True))
         return
 
     if os.path.dirname(output_file):
         os.makedirs(os.path.dirname(output_file), exist_ok=True)
     AgentsController.persist_record(agent, output_file=output_file)
 
-
     message_lines = [
         "Your agent building session finished successfully!",
         f"Agent YAML saved in file:",
@@ -290,4 +365,4 @@ def create_agent(output_file: str, llm: str, samples_file: str | None, dry_run_f
     rich.print("╔" + "═" * frame_width + "╗")
     for line in message_lines:
         rich.print("║  " + line.ljust(max_length) + "  ║")
-    rich.print("╚" + "═" * frame_width + "╝")
+    rich.print("╚" + "═" * frame_width + "╝")

ibm_watsonx_orchestrate/cli/commands/copilot/copilot_server_controller.py

@@ -4,6 +4,7 @@ from pathlib import Path
 import subprocess
 import time
 import requests
+from urllib.parse import urlparse
 from ibm_watsonx_orchestrate.cli.commands.server.server_command import (
     get_compose_file,
     ensure_docker_compose_installed,
@@ -16,15 +17,11 @@ from ibm_watsonx_orchestrate.cli.commands.server.server_command import (
     get_default_registry_env_vars_by_dev_edition_source,
     docker_login_by_dev_edition_source,
     write_merged_env_file,
+    apply_server_env_dict_defaults
 )
 
 logger = logging.getLogger(__name__)
 
-def _verify_env_contents(env: dict) -> None:
-    if not env.get("WATSONX_APIKEY") or not env.get("WATSONX_SPACE_ID"):
-        logger.error("The Copilot feature requires wx.ai credentials to passed through the provided env file. Please set 'WATSONX_SPACE_ID' and 'WATSONX_APIKEY'")
-        sys.exit(1)
-
 def wait_for_wxo_cpe_health_check(timeout_seconds=45, interval_seconds=2):
     url = "http://localhost:8081/version"
     logger.info("Waiting for Copilot component to be initialized...")
@@ -42,6 +39,24 @@ def wait_for_wxo_cpe_health_check(timeout_seconds=45, interval_seconds=2):
         time.sleep(interval_seconds)
     return False
 
+def _trim_authorization_urls(env_dict: dict) -> dict:
+    auth_url_key = "AUTHORIZATION_URL"
+    env_dict_copy = env_dict.copy()
+
+    auth_url = env_dict_copy.get(auth_url_key)
+    if not auth_url:
+        return env_dict_copy
+    
+    
+    parsed_url = urlparse(auth_url)
+    new_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+    env_dict_copy[auth_url_key] = new_url
+
+    return env_dict_copy
+
+    
+
+
 def run_compose_lite_cpe(user_env_file: Path) -> bool:
     compose_path = get_compose_file()
     compose_command = ensure_docker_compose_installed()
@@ -66,8 +81,9 @@ def run_compose_lite_cpe(user_env_file: Path) -> bool:
         **default_env,
         **user_env,
     }
-
-    _verify_env_contents(merged_env_dict)
+    
+    merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
+    merged_env_dict = _trim_authorization_urls(merged_env_dict)
 
     try:
         docker_login_by_dev_edition_source(merged_env_dict, dev_edition_source)

ibm_watsonx_orchestrate/cli/commands/environment/types.py

@@ -9,4 +9,4 @@ class EnvironmentAuthType(str, Enum):
     CPD = 'cpd'
 
     def __str__(self):
-        return self.value
+        return self.value

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -2,11 +2,12 @@ import json
 import logging
 import typer
 import os
-import yaml
 import csv
 import rich
 import sys
 import shutil
+import tempfile
+import random
 
 from rich.panel import Panel
 from pathlib import Path
@@ -16,23 +17,55 @@ from typing_extensions import Annotated
 
 from ibm_watsonx_orchestrate import __version__
 from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 
 logger = logging.getLogger(__name__)
 
 evaluation_app = typer.Typer(no_args_is_help=True)
 
+def _native_agent_template():
+    return {
+        "spec_version": "v1",
+        "style": "default",
+        "llm": "watsonx/meta-llama/llama-3-405b-instruct",
+        "name": "",
+        "description": "Native agent for validating external agent",
+        "instructions": "Use the tools and external agent(s) provided to answer the user's question.  If you do not have enough information to answer the question, say so.  If you need more information, ask follow up questions.",
+        "collaborators": []
+    }
+
+def _random_native_agent_name(external_agent_name):
+    """ Generate a native agent name in the following format to ensure uniqueness:
+
+    "external_agent_validation_{external_agent_name}_{random number}
+
+    So if the external agent name is, "QA_Agent", and the random number generated is, '100', the native agent name is:
+    "external_agent_validation_QA_Agent_100"
+
+    """
+    seed = 42
+    random.seed(seed)
+    
+    return f"external_agent_validation_{external_agent_name}_{random.randint(0, 100)}"
+
 def read_env_file(env_path: Path|str) -> dict:
     return dotenv_values(str(env_path))
 
 def validate_watsonx_credentials(user_env_file: str) -> bool:
-    required_keys = ["WATSONX_SPACE_ID", "WATSONX_APIKEY"]
+    required_sets = [
+        ["WATSONX_SPACE_ID", "WATSONX_APIKEY"],
+        ["WO_INSTANCE", "WO_API_KEY"]
+    ]
     
-    if all(key in os.environ for key in required_keys):
+    def has_valid_keys(env: dict) -> bool:
+        return any(all(key in env for key in key_set) for key_set in required_sets)
+
+    if has_valid_keys(os.environ):
         logger.info("WatsonX credentials validated successfully.")
         return
     
     if user_env_file is None:
-        logger.error("WatsonX credentials are not set. Please set WATSONX_SPACE_ID and WATSONX_APIKEY in your system environment variables or include them in your enviroment file and pass it with --env-file option.")
+        logger.error("WatsonX credentials are not set. Please set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables or include them in your environment file and pass it with --env-file option.")
         sys.exit(1)
 
     if not Path(user_env_file).exists():
@@ -41,11 +74,15 @@ def validate_watsonx_credentials(user_env_file: str) -> bool:
     
     user_env = read_env_file(user_env_file)
     
-    if not all(key in user_env for key in required_keys):
-        logger.error("Error: The environment file does not contain the required keys: WATSONX_SPACE_ID and WATSONX_APIKEY.")
+    if not has_valid_keys(user_env):
+        logger.error("Error: The environment file does not contain the required keys: either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY.")
         sys.exit(1)
 
-    os.environ.update({key: user_env[key] for key in required_keys})
+    # Update os.environ with whichever set is present
+    for key_set in required_sets:
+        if all(key in user_env for key in key_set):
+            os.environ.update({key: user_env[key] for key in key_set})
+            break
     logger.info("WatsonX credentials validated successfully.")
 
 def read_csv(data_path: str, delimiter="\t"):
@@ -208,7 +245,7 @@ def validate_external(
             str,
             typer.Option(
                 "--external-agent-config", "-ext",
-                help="Path to the external agent yaml",
+                help="Path to the external agent json file",
 
             )
         ],
@@ -234,33 +271,65 @@ def validate_external(
             help="Path to a .env file that overrides default.env. Then environment variables override both."
         ),
     ] = None,
-    agent_name: Annotated[
-        str,
+    perf_test: Annotated[
+        bool,
         typer.Option(
-            "--agent_name", "-a",
-            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
-            " If this parameter is pased, validation of the external agent is not run.",
-            rich_help_panel="Parameters for Input Evaluation"
+            "--perf", "-p",
+            help="Performance test your external agent against the provide user stories.",
+            rich_help_panel="Parameters for Input Evaluation",
         )
-    ] = None
+    ] = False
 ):
 
     validate_watsonx_credentials(user_env_file)
-    Path(output_dir).mkdir(exist_ok=True)
-    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
 
-    if agent_name is not None:
-        eval_dir = os.path.join(output_dir, "evaluation")
+    with open(external_agent_config, 'r') as f:
+        try:
+            external_agent_config = json.load(f)
+        except Exception:
+            rich.print(
+                f"[red]: Please provide a valid external agent spec in JSON format. See 'examples/evaluations/external_agent_validation/sample_external_agent_config.json' for an example."
+            )
+            sys.exit(1)
+
+    eval_dir = os.path.join(output_dir, "evaluations")
+    if perf_test:
         if os.path.exists(eval_dir):
             rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
-            shutil.rmtree(os.path.join(output_dir, "evaluation"))
-        Path(eval_dir).mkdir(exist_ok=True)
+            shutil.rmtree(eval_dir)
+        Path(eval_dir).mkdir(exist_ok=True, parents=True)
         # save external agent config even though its not used for evaluation
         # it can help in later debugging customer agents
-        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
-            with open(external_agent_config, "r") as cfg:
-                external_agent_config = yaml.safe_load(cfg)
-            yaml.safe_dump(external_agent_config, f, indent=4)
+        with open(os.path.join(eval_dir, f"external_agent_cfg.json"), "w+") as f:
+            json.dump(external_agent_config, f, indent=4)
+
+        logger.info("Registering External Agent")
+        agent_controller = AgentsController()
+
+        external_agent_config["title"] = external_agent_config["name"]
+        external_agent_config["auth_config"] = {"token": credential}
+        external_agent_config["spec_version"] = external_agent_config.get("spec_version", "v1")
+        external_agent_config["provider"] = "external_chat"
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(external_agent_config, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
+
+        logger.info("Registering Native Agent")
+
+        native_agent_template = _native_agent_template()
+        agent_name = _random_native_agent_name(external_agent_config["name"])
+        rich.print(f"[blue][b]Generated native agent name is: [i]{agent_name}[/i][/b]")
+        native_agent_template["name"] = agent_name
+        native_agent_template["collaborators"] = [external_agent_config["name"]]
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(native_agent_template, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
 
         rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
         performance_test(
@@ -271,8 +340,6 @@ def validate_external(
         )
     
     else:
-        with open(external_agent_config, "r") as f:
-            external_agent_config = yaml.safe_load(f)
         controller = EvaluationsController()
         test_data = []
         with open(data_path, "r") as f:
@@ -280,31 +347,29 @@ def validate_external(
             for line in csv_reader:
                 test_data.append(line[0])
 
-        # save validation results in "validation_results" sub-dir
-        validation_folder = Path(output_dir) / "validation_results"
+        # save validation results in "validate_external" sub-dir
+        validation_folder = Path(output_dir) / "validate_external"
         if os.path.exists(validation_folder):
             rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
             shutil.rmtree(validation_folder)
         validation_folder.mkdir(exist_ok=True, parents=True)
+        shutil.copy(data_path, os.path.join(validation_folder, "input_sample.tsv"))
 
         # validate the inputs in the provided csv file
         summary = controller.external_validate(external_agent_config, test_data, credential)
-        with open(validation_folder / "validation_results.json", "w") as f:
-            json.dump(summary, f, indent=4)
-        
         # validate sample block inputs
-        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        rich.print("[gold3]Validating external agent against an array of messages.")
         block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
-        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
-            json.dump(block_input_summary, f, indent=4)
-
+        
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump([summary, block_input_summary], f, indent=4)
+        
         user_validation_successful = all([item["success"] for item in summary])
         block_validation_successful = all([item["success"] for item in block_input_summary])
 
         if user_validation_successful and block_validation_successful:
             msg = (
                 f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
-                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
             )
         else:
             msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -3,12 +3,12 @@ import os.path
 from typing import List, Dict, Optional, Tuple
 import csv
 from pathlib import Path
-import rich
+import sys
 from wxo_agentic_evaluation import main as evaluate
 from wxo_agentic_evaluation.tool_planner import build_snapshot
 from wxo_agentic_evaluation.analyze_run import analyze
 from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
-from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
+from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig
 from wxo_agentic_evaluation.record_chat import record_chats
 from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
 from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
@@ -41,12 +41,26 @@ class EvaluationsController:
     def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
         url, tenant_name, token = self._get_env_config()
 
+        if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
+            provider = "watsonx"
+        elif "WO_INSTANCE" in os.environ and "WO_API_KEY" in os.environ:
+            provider = "model_proxy"
+        else:
+            logger.error(
+                "No provider found. Please either provide a config_file or set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables."
+            )
+            sys.exit(1)
+        
         config_data = {
             "wxo_lite_version": __version__,
             "auth_config": AuthConfig(
                 url=url,
                 tenant_name=tenant_name,
                 token=token
+            ),
+            "provider_config": ProviderConfig(
+                provider=provider,
+                model_id="meta-llama/llama-3-405b-instruct",
             )
         }
 
@@ -62,6 +76,10 @@ class EvaluationsController:
                 if "llm_user_config" in file_config:
                     llm_config_data = file_config.pop("llm_user_config")
                     config_data["llm_user_config"] = LLMUserConfig(**llm_config_data)
+
+                if "provider_config" in file_config:
+                    provider_config_data = file_config.pop("provider_config")
+                    config_data["provider_config"] = ProviderConfig(**provider_config_data)
                 
                 config_data.update(file_config)