ibm-watsonx-orchestrate 1.6.0b0__py3-none-any.whl → 1.6.2__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -5,10 +5,12 @@ import os
 import yaml
 import csv
 import rich
-from pathlib import Path
 import sys
-from dotenv import dotenv_values
+import shutil
 
+from rich.panel import Panel
+from pathlib import Path
+from dotenv import dotenv_values
 from typing import Optional
 from typing_extensions import Annotated
 
@@ -46,6 +48,32 @@ def validate_watsonx_credentials(user_env_file: str) -> bool:
     os.environ.update({key: user_env[key] for key in required_keys})
     logger.info("WatsonX credentials validated successfully.")
 
+def read_csv(data_path: str, delimiter="\t"):
+    data = []
+    with open(data_path, "r") as f:
+        tsv_reader = csv.reader(f, delimiter=delimiter)
+        for line in tsv_reader:
+            data.append(line)
+    
+    return data
+
+def performance_test(agent_name, data_path, output_dir = None, user_env_file = None):
+    test_data = read_csv(data_path)
+
+    controller = EvaluationsController()
+    generated_performance_tests = controller.generate_performance_test(agent_name, test_data)
+    
+    generated_perf_test_dir = Path(output_dir) / "generated_performance_tests"
+    generated_perf_test_dir.mkdir(exist_ok=True, parents=True)
+
+    for idx, test in enumerate(generated_performance_tests):
+        test_name = f"validate_external_agent_evaluation_test_{idx}.json"
+        with open(generated_perf_test_dir / test_name, encoding="utf-8", mode="w+") as f:
+            json.dump(test, f, indent=4)
+
+    rich.print(f"Performance test cases saved at path '{str(generated_perf_test_dir)}'")
+    rich.print("[gold3]Running Performance Test")
+    evaluate(output_dir=output_dir, test_paths=str(generated_perf_test_dir))
 
 @evaluation_app.command(name="evaluate", help="Evaluate an agent against a set of test cases")
 def evaluate(
@@ -115,7 +143,7 @@ def generate(
     stories_path: Annotated[
         str,
         typer.Option(
-            "--stories_path", "-s",
+            "--stories-path", "-s",
             help="Path to the CSV file containing user stories for test case generation. "
                  "The file has 'story' and 'agent' columns."
         )
@@ -123,14 +151,14 @@ def generate(
     tools_path: Annotated[
         str,
         typer.Option(
-            "--tools_path", "-t",
+            "--tools-path", "-t",
             help="Path to the directory containing tool definitions."
         )
     ],
     output_dir: Annotated[
         Optional[str],
         typer.Option(
-            "--output_dir", "-o",
+            "--output-dir", "-o",
             help="Directory to save the generated test cases."
         )
     ] = None,
@@ -151,7 +179,7 @@ def generate(
 def analyze(data_path: Annotated[
         str,
         typer.Option(
-            "--data_path", "-d",
+            "--data-path", "-d",
             help="Path to the directory that has the saved results"
         )
     ],
@@ -167,30 +195,31 @@ def analyze(data_path: Annotated[
     controller = EvaluationsController()
     controller.analyze(data_path=data_path)
 
-
-@evaluation_app.command(name="validate_external", help="Validate an external agent against a set of inputs")
+@evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
 def validate_external(
     data_path: Annotated[
         str,
         typer.Option(
-            "--csv", "-c",
-            help="Path to .csv file of inputs"
+            "--tsv", "-t",
+            help="Path to .tsv file of inputs"
         )
     ],
-    config: Annotated[
+    external_agent_config: Annotated[
             str,
             typer.Option(
-                "--config", "-cf",
-                help="Path to the external agent yaml"
+                "--external-agent-config", "-ext",
+                help="Path to the external agent yaml",
+
             )
         ],
     credential: Annotated[
         str,
         typer.Option(
             "--credential", "-crd",
-            help="credential string"
+            help="credential string",
+            rich_help_panel="Parameters for Validation"
         )
-    ],
+    ] = None,
     output_dir: Annotated[
         str,
         typer.Option(
@@ -204,21 +233,80 @@ def validate_external(
             "--env-file", "-e", 
             help="Path to a .env file that overrides default.env. Then environment variables override both."
         ),
+    ] = None,
+    agent_name: Annotated[
+        str,
+        typer.Option(
+            "--agent_name", "-a",
+            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
+            " If this parameter is pased, validation of the external agent is not run.",
+            rich_help_panel="Parameters for Input Evaluation"
+        )
     ] = None
 ):
-    
+
     validate_watsonx_credentials(user_env_file)
-    with open(config, "r") as f:
-        config = yaml.safe_load(f)
-    controller = EvaluationsController()
-    test_data = []
-    with open(data_path, "r") as f:
-        csv_reader = csv.reader(f)
-        for line in csv_reader:
-            test_data.append(line[0])
-    results = controller.external_validate(config, test_data, credential)
-    os.makedirs(output_dir, exist_ok=True)
-    with open(os.path.join(output_dir, "validation_results.json"), "w") as f:
-        json.dump(results, f)
-
-    rich.print(f"[green] validation result is saved to {output_dir} [/green]")
+    Path(output_dir).mkdir(exist_ok=True)
+    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
+
+    if agent_name is not None:
+        eval_dir = os.path.join(output_dir, "evaluation")
+        if os.path.exists(eval_dir):
+            rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
+            shutil.rmtree(os.path.join(output_dir, "evaluation"))
+        Path(eval_dir).mkdir(exist_ok=True)
+        # save external agent config even though its not used for evaluation
+        # it can help in later debugging customer agents
+        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
+            with open(external_agent_config, "r") as cfg:
+                external_agent_config = yaml.safe_load(cfg)
+            yaml.safe_dump(external_agent_config, f, indent=4)
+
+        rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
+        performance_test(
+            agent_name=agent_name,
+            data_path=data_path,
+            output_dir=eval_dir,
+            user_env_file=user_env_file
+        )
+    
+    else:
+        with open(external_agent_config, "r") as f:
+            external_agent_config = yaml.safe_load(f)
+        controller = EvaluationsController()
+        test_data = []
+        with open(data_path, "r") as f:
+            csv_reader = csv.reader(f, delimiter="\t")
+            for line in csv_reader:
+                test_data.append(line[0])
+
+        # save validation results in "validation_results" sub-dir
+        validation_folder = Path(output_dir) / "validation_results"
+        if os.path.exists(validation_folder):
+            rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
+            shutil.rmtree(validation_folder)
+        validation_folder.mkdir(exist_ok=True, parents=True)
+
+        # validate the inputs in the provided csv file
+        summary = controller.external_validate(external_agent_config, test_data, credential)
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump(summary, f, indent=4)
+        
+        # validate sample block inputs
+        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
+        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
+            json.dump(block_input_summary, f, indent=4)
+
+        user_validation_successful = all([item["success"] for item in summary])
+        block_validation_successful = all([item["success"] for item in block_input_summary])
+
+        if user_validation_successful and block_validation_successful:
+            msg = (
+                f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
+                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
+            )
+        else:
+            msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
+
+        rich.print(Panel(msg))

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -1,5 +1,6 @@
 import logging
-from typing import List, Dict, Optional
+import os.path
+from typing import List, Dict, Optional, Tuple
 import csv
 from pathlib import Path
 import rich
@@ -10,12 +11,13 @@ from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stori
 from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
 from wxo_agentic_evaluation.record_chat import record_chats
 from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
+from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
 from ibm_watsonx_orchestrate import __version__
 from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
 from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
 from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 from ibm_watsonx_orchestrate.agent_builder.agents import AgentKind
-
+import uuid
 
 logger = logging.getLogger(__name__)
 
@@ -75,9 +77,13 @@ class EvaluationsController:
         evaluate.main(config)
 
     def record(self, output_dir) -> None:
+
+
+        random_uuid = str(uuid.uuid4())
+
         url, tenant_name, token = self._get_env_config()
         config_data = {
-            "output_dir": Path.cwd() if output_dir is None else Path(output_dir),
+            "output_dir": Path(os.path.join(Path.cwd(), random_uuid)) if output_dir is None else Path(os.path.join(output_dir,random_uuid)),
             "service_url": url,
             "tenant_name": tenant_name,
             "token": token
@@ -143,16 +149,23 @@ class EvaluationsController:
     def summarize(self) -> None:
         pass
 
-    def external_validate(self, config: Dict, data: List[str], credential:str):
+    def external_validate(self, config: Dict, data: List[str], credential:str, add_context: bool = False):
         validator = ExternalAgentValidation(credential=credential,
                                 auth_scheme=config["auth_scheme"],
                                 service_url=config["api_url"])
+        
         summary = []
         for entry in data:
-            results = validator.call_validation(entry)
-            if len(results) == 0:
-                rich.print(f"[red] No events are generated for input {entry} [/red]")
-            summary.append({entry: results})
+            results = validator.call_validation(entry, add_context)
+            summary.append(results)
 
         return summary
-
+    
+    def generate_performance_test(self, agent_name: str, test_data: List[Tuple[str, str]]):
+        performance_test = ExternalAgentPerformanceTest(
+            agent_name=agent_name,
+            test_data=test_data
+        )
+        generated_performance_tests = performance_test.generate_tests()
+
+        return generated_performance_tests

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py

@@ -197,10 +197,10 @@ class KnowledgeBaseController:
             )
 
             column_args = {
-                "Name": {},
+                "Name": {"overflow": "fold"},
                 "Description": {},
                 "App ID": {},
-                "ID": {}
+                "ID": {"overflow": "fold"}
             }
             
             for column in column_args:

ibm_watsonx_orchestrate/cli/commands/server/server_command.py

@@ -240,6 +240,27 @@ def apply_llm_api_key_defaults(env_dict: dict) -> None:
         env_dict.setdefault("ASSISTANT_EMBEDDINGS_SPACE_ID", space_value)
         env_dict.setdefault("ROUTING_LLM_SPACE_ID", space_value)
 
+def _is_docker_container_running(container_name):
+    ensure_docker_installed()
+    command = [ "docker",
+                "ps",
+                "-f",
+                f"name={container_name}"
+            ]
+    result = subprocess.run(command, env=os.environ, capture_output=True)
+    if container_name in str(result.stdout):
+        return True
+    return False
+
+def _check_exclusive_observibility(langfuse_enabled: bool, ibm_tele_enabled: bool):
+    if langfuse_enabled and ibm_tele_enabled:
+        return False
+    if langfuse_enabled and _is_docker_container_running("docker-frontend-server-1"):
+        return False
+    if ibm_tele_enabled and _is_docker_container_running("docker-langfuse-web-1"):
+        return False
+    return True
+
 def write_merged_env_file(merged_env: dict) -> Path:
     tmp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env")
     with tmp:
@@ -292,7 +313,8 @@ def get_persisted_user_env() -> dict | None:
     user_env = cfg.get(USER_ENV_CACHE_HEADER) if cfg.get(USER_ENV_CACHE_HEADER) else None
     return user_env
 
-def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False) -> None:
+
+def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False, experimental_with_ibm_telemetry=False) -> None:
     compose_path = get_compose_file()
     compose_command = ensure_docker_compose_installed()
     db_tag = read_env_file(final_env_file).get('DBTAG', None)
@@ -324,9 +346,15 @@ def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False) ->
             '--profile',
             'langfuse'
         ]
+    elif experimental_with_ibm_telemetry:
+        command = compose_command + [
+            '--profile',
+            'ibm-telemetry'
+        ]
     else:
         command = compose_command
 
+
     command += [
         "-f", str(compose_path),
         "--env-file", str(final_env_file),
@@ -444,6 +472,9 @@ def run_compose_lite_ui(user_env_file: Path) -> bool:
         # do nothing, as the docker login here is not mandatory
         pass
 
+    # Auto-configure callback IP for async tools
+    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
+
     #These are to removed warning and not used in UI component
     if not 'WATSONX_SPACE_ID' in merged_env_dict:
         merged_env_dict['WATSONX_SPACE_ID']='X'
@@ -621,8 +652,80 @@ def confirm_accepts_license_agreement(accepts_by_argument: bool):
             logger.error('The terms and conditions were not accepted, exiting.')
             exit(1)
 
-
-
+def auto_configure_callback_ip(merged_env_dict: dict) -> dict:
+    """
+    Automatically detect and configure CALLBACK_HOST_URL if it's empty.
+    
+    Args:
+        merged_env_dict: The merged environment dictionary
+        
+    Returns:
+        Updated environment dictionary with CALLBACK_HOST_URL set
+    """
+    callback_url = merged_env_dict.get('CALLBACK_HOST_URL', '').strip()
+    
+    # Only auto-configure if CALLBACK_HOST_URL is empty
+    if not callback_url:
+        logger.info("Auto-detecting local IP address for async tool callbacks...")
+        
+        system = platform.system()
+        ip = None
+        
+        try:
+            if system in ("Linux", "Darwin"):
+                result = subprocess.run(["ifconfig"], capture_output=True, text=True, check=True)
+                lines = result.stdout.splitlines()
+                
+                for line in lines:
+                    line = line.strip()
+                    # Unix ifconfig output format: "inet 192.168.1.100 netmask 0xffffff00 broadcast 192.168.1.255"
+                    if line.startswith("inet ") and "127.0.0.1" not in line:
+                        candidate_ip = line.split()[1]
+                        # Validate IP is not loopback or link-local
+                        if (candidate_ip and 
+                            not candidate_ip.startswith("127.") and 
+                            not candidate_ip.startswith("169.254")):
+                            ip = candidate_ip
+                            break
+            
+            elif system == "Windows":
+                result = subprocess.run(["ipconfig"], capture_output=True, text=True, check=True)
+                lines = result.stdout.splitlines()
+                
+                for line in lines:
+                    line = line.strip()
+                    # Windows ipconfig output format: "   IPv4 Address. . . . . . . . . . . : 192.168.1.100"
+                    if "IPv4 Address" in line and ":" in line:
+                        candidate_ip = line.split(":")[-1].strip()
+                        # Validate IP is not loopback or link-local
+                        if (candidate_ip and 
+                            not candidate_ip.startswith("127.") and 
+                            not candidate_ip.startswith("169.254")):
+                            ip = candidate_ip
+                            break
+            
+            else:
+                logger.warning(f"Unsupported platform: {system}")
+                ip = None
+                
+        except Exception as e:
+            logger.debug(f"IP detection failed on {system}: {e}")
+            ip = None
+        
+        if ip:
+            callback_url = f"http://{ip}:4321"
+            merged_env_dict['CALLBACK_HOST_URL'] = callback_url
+            logger.info(f"Auto-configured CALLBACK_HOST_URL to: {callback_url}")
+        else:
+            # Fallback for localhost
+            callback_url = "http://host.docker.internal:4321"
+            merged_env_dict['CALLBACK_HOST_URL'] = callback_url
+            logger.info(f"Using Docker internal URL: {callback_url}")
+            logger.info("For external tools, consider using ngrok or similar tunneling service.")
+    else:
+        logger.info(f"Using existing CALLBACK_HOST_URL: {callback_url}")
+    
+    return merged_env_dict
 
 @server_app.command(name="start")
 def server_start(
@@ -636,6 +739,11 @@ def server_start(
         '--with-langfuse', '-l',
         help='Option to enable Langfuse support.'
     ),
+    experimental_with_ibm_telemetry: bool = typer.Option(
+        False,
+        '--with-ibm-telemetry', '-i',
+        help=''
+    ),
     persist_env_secrets: bool = typer.Option(
         False,
         '--persist-env-secrets', '-p',
@@ -675,10 +783,18 @@ def server_start(
 
     merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
 
+    # Auto-configure callback IP for async tools
+    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
+    if not _check_exclusive_observibility(experimental_with_langfuse, experimental_with_ibm_telemetry):
+        logger.error("Please select either langfuse or ibm telemetry for observability not both")
+        sys.exit(1)
+
     # Add LANGFUSE_ENABLED into the merged_env_dict, for tempus to pick up.
     if experimental_with_langfuse:
         merged_env_dict['LANGFUSE_ENABLED'] = 'true'
-
+        
+    if experimental_with_ibm_telemetry:
+        merged_env_dict['USE_IBM_TELEMETRY'] = 'true'
 
     try:
         docker_login_by_dev_edition_source(merged_env_dict, dev_edition_source)
@@ -690,7 +806,9 @@ def server_start(
 
 
     final_env_file = write_merged_env_file(merged_env_dict)
-    run_compose_lite(final_env_file=final_env_file, experimental_with_langfuse=experimental_with_langfuse)
+    run_compose_lite(final_env_file=final_env_file, 
+                     experimental_with_langfuse=experimental_with_langfuse,
+                     experimental_with_ibm_telemetry=experimental_with_ibm_telemetry)
 
     run_db_migration()
 

ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_controller.py

@@ -243,9 +243,15 @@ class ToolkitController:
             rich.print(JSON(json.dumps(tools_list, indent=4)))
         else:
             table = rich.table.Table(show_header=True, header_style="bold white", show_lines=True)
-            columns = ["Name", "Kind", "Description", "Tools", "App ID"]
-            for column in columns:
-                table.add_column(column)
+            column_args = {
+                "Name": {"overflow": "fold"},
+                "Kind": {},
+                "Description": {},
+                "Tools": {},
+                "App ID": {"overflow": "fold"}
+            }
+            for column in column_args:
+                table.add_column(column,**column_args[column])
 
             tools_client = instantiate_client(ToolClient)