ibm-watsonx-orchestrate 1.5.0b1__py3-none-any.whl → 1.6.0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -0,0 +1,312 @@
+import json
+import logging
+import typer
+import os
+import yaml
+import csv
+import rich
+import sys
+import shutil
+
+from rich.panel import Panel
+from pathlib import Path
+from dotenv import dotenv_values
+from typing import Optional
+from typing_extensions import Annotated
+
+from ibm_watsonx_orchestrate import __version__
+from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+
+logger = logging.getLogger(__name__)
+
+evaluation_app = typer.Typer(no_args_is_help=True)
+
+def read_env_file(env_path: Path|str) -> dict:
+    return dotenv_values(str(env_path))
+
+def validate_watsonx_credentials(user_env_file: str) -> bool:
+    required_keys = ["WATSONX_SPACE_ID", "WATSONX_APIKEY"]
+    
+    if all(key in os.environ for key in required_keys):
+        logger.info("WatsonX credentials validated successfully.")
+        return
+    
+    if user_env_file is None:
+        logger.error("WatsonX credentials are not set. Please set WATSONX_SPACE_ID and WATSONX_APIKEY in your system environment variables or include them in your enviroment file and pass it with --env-file option.")
+        sys.exit(1)
+
+    if not Path(user_env_file).exists():
+        logger.error(f"Error: The specified environment file '{user_env_file}' does not exist.")
+        sys.exit(1)
+    
+    user_env = read_env_file(user_env_file)
+    
+    if not all(key in user_env for key in required_keys):
+        logger.error("Error: The environment file does not contain the required keys: WATSONX_SPACE_ID and WATSONX_APIKEY.")
+        sys.exit(1)
+
+    os.environ.update({key: user_env[key] for key in required_keys})
+    logger.info("WatsonX credentials validated successfully.")
+
+def read_csv(data_path: str, delimiter="\t"):
+    data = []
+    with open(data_path, "r") as f:
+        tsv_reader = csv.reader(f, delimiter=delimiter)
+        for line in tsv_reader:
+            data.append(line)
+    
+    return data
+
+def performance_test(agent_name, data_path, output_dir = None, user_env_file = None):
+    test_data = read_csv(data_path)
+
+    controller = EvaluationsController()
+    generated_performance_tests = controller.generate_performance_test(agent_name, test_data)
+    
+    generated_perf_test_dir = Path(output_dir) / "generated_performance_tests"
+    generated_perf_test_dir.mkdir(exist_ok=True, parents=True)
+
+    for idx, test in enumerate(generated_performance_tests):
+        test_name = f"validate_external_agent_evaluation_test_{idx}.json"
+        with open(generated_perf_test_dir / test_name, encoding="utf-8", mode="w+") as f:
+            json.dump(test, f, indent=4)
+
+    rich.print(f"Performance test cases saved at path '{str(generated_perf_test_dir)}'")
+    rich.print("[gold3]Running Performance Test")
+    evaluate(output_dir=output_dir, test_paths=str(generated_perf_test_dir))
+
+@evaluation_app.command(name="evaluate", help="Evaluate an agent against a set of test cases")
+def evaluate(
+    config_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--config", "-c",
+            help="Path to YAML configuration file containing evaluation settings."
+        )
+    ] = None,
+    test_paths: Annotated[
+        Optional[str],
+        typer.Option(
+            "--test-paths", "-p", 
+            help="Paths to the test files and/or directories to evaluate, separated by commas."
+        ),
+    ] = None,
+    output_dir: Annotated[
+        Optional[str], 
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the evaluation results."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    if not config_file:
+        if not test_paths or not output_dir:
+            logger.error("Error: Both --test-paths and --output-dir must be provided when not using a config file")
+            exit(1)
+    
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.evaluate(config_file=config_file, test_paths=test_paths, output_dir=output_dir)
+
+
+@evaluation_app.command(name="record", help="Record chat sessions and create test cases")
+def record(
+    output_dir: Annotated[
+        Optional[str], 
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the recorded chats."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.record(output_dir=output_dir)
+
+
+@evaluation_app.command(name="generate", help="Generate test cases from user stories and tools")
+def generate(
+    stories_path: Annotated[
+        str,
+        typer.Option(
+            "--stories-path", "-s",
+            help="Path to the CSV file containing user stories for test case generation. "
+                 "The file has 'story' and 'agent' columns."
+        )
+    ],
+    tools_path: Annotated[
+        str,
+        typer.Option(
+            "--tools-path", "-t",
+            help="Path to the directory containing tool definitions."
+        )
+    ],
+    output_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the generated test cases."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.generate(stories_path=stories_path, tools_path=tools_path, output_dir=output_dir)
+
+
+@evaluation_app.command(name="analyze", help="Analyze the results of an evaluation run")
+def analyze(data_path: Annotated[
+        str,
+        typer.Option(
+            "--data-path", "-d",
+            help="Path to the directory that has the saved results"
+        )
+    ],
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None):
+
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.analyze(data_path=data_path)
+
+@evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
+def validate_external(
+    data_path: Annotated[
+        str,
+        typer.Option(
+            "--tsv", "-t",
+            help="Path to .tsv file of inputs"
+        )
+    ],
+    external_agent_config: Annotated[
+            str,
+            typer.Option(
+                "--external-agent-config", "-ext",
+                help="Path to the external agent yaml",
+
+            )
+        ],
+    credential: Annotated[
+        str,
+        typer.Option(
+            "--credential", "-crd",
+            help="credential string",
+            rich_help_panel="Parameters for Validation"
+        )
+    ] = None,
+    output_dir: Annotated[
+        str,
+        typer.Option(
+            "--output", "-o",
+            help="where to save the validation results"
+        )
+    ] = "./test_external_agent",
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None,
+    agent_name: Annotated[
+        str,
+        typer.Option(
+            "--agent_name", "-a",
+            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
+            " If this parameter is pased, validation of the external agent is not run.",
+            rich_help_panel="Parameters for Input Evaluation"
+        )
+    ] = None
+):
+
+    validate_watsonx_credentials(user_env_file)
+    Path(output_dir).mkdir(exist_ok=True)
+    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
+
+    if agent_name is not None:
+        eval_dir = os.path.join(output_dir, "evaluation")
+        if os.path.exists(eval_dir):
+            rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
+            shutil.rmtree(os.path.join(output_dir, "evaluation"))
+        Path(eval_dir).mkdir(exist_ok=True)
+        # save external agent config even though its not used for evaluation
+        # it can help in later debugging customer agents
+        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
+            with open(external_agent_config, "r") as cfg:
+                external_agent_config = yaml.safe_load(cfg)
+            yaml.safe_dump(external_agent_config, f, indent=4)
+
+        rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
+        performance_test(
+            agent_name=agent_name,
+            data_path=data_path,
+            output_dir=eval_dir,
+            user_env_file=user_env_file
+        )
+    
+    else:
+        with open(external_agent_config, "r") as f:
+            external_agent_config = yaml.safe_load(f)
+        controller = EvaluationsController()
+        test_data = []
+        with open(data_path, "r") as f:
+            csv_reader = csv.reader(f, delimiter="\t")
+            for line in csv_reader:
+                test_data.append(line[0])
+
+        # save validation results in "validation_results" sub-dir
+        validation_folder = Path(output_dir) / "validation_results"
+        if os.path.exists(validation_folder):
+            rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
+            shutil.rmtree(validation_folder)
+        validation_folder.mkdir(exist_ok=True, parents=True)
+
+        # validate the inputs in the provided csv file
+        summary = controller.external_validate(external_agent_config, test_data, credential)
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump(summary, f, indent=4)
+        
+        # validate sample block inputs
+        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
+        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
+            json.dump(block_input_summary, f, indent=4)
+
+        user_validation_successful = all([item["success"] for item in summary])
+        block_validation_successful = all([item["success"] for item in block_input_summary])
+
+        if user_validation_successful and block_validation_successful:
+            msg = (
+                f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
+                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
+            )
+        else:
+            msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
+
+        rich.print(Panel(msg))

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -0,0 +1,171 @@
+import logging
+import os.path
+from typing import List, Dict, Optional, Tuple
+import csv
+from pathlib import Path
+import rich
+from wxo_agentic_evaluation import main as evaluate
+from wxo_agentic_evaluation.tool_planner import build_snapshot
+from wxo_agentic_evaluation.analyze_run import analyze
+from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
+from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
+from wxo_agentic_evaluation.record_chat import record_chats
+from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
+from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
+from ibm_watsonx_orchestrate import __version__
+from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
+from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
+from ibm_watsonx_orchestrate.agent_builder.agents import AgentKind
+import uuid
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationsController:
+    def __init__(self):
+        pass
+
+    def _get_env_config(self) -> tuple[str, str, str | None]:
+        cfg = Config()
+        auth_cfg = Config(AUTH_CONFIG_FILE_FOLDER, AUTH_CONFIG_FILE)
+        
+        url = cfg.get_active_env_config(ENV_WXO_URL_OPT)
+        tenant_name = cfg.get_active_env()
+
+        existing_auth_config = auth_cfg.get(AUTH_SECTION_HEADER).get(tenant_name, {})
+        token = existing_auth_config.get(AUTH_MCSP_TOKEN_OPT) if existing_auth_config else None
+
+        return url, tenant_name, token
+
+    def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
+        url, tenant_name, token = self._get_env_config()
+
+        config_data = {
+            "wxo_lite_version": __version__,
+            "auth_config": AuthConfig(
+                url=url,
+                tenant_name=tenant_name,
+                token=token
+            )
+        }
+
+        if config_file:
+            logger.info(f"Loading configuration from {config_file}")
+            with open(config_file, 'r') as f:
+                file_config = yaml_safe_load(f) or {}
+                
+                if "auth_config" in file_config:
+                    auth_config_data = file_config.pop("auth_config")
+                    config_data["auth_config"] = AuthConfig(**auth_config_data)
+                
+                if "llm_user_config" in file_config:
+                    llm_config_data = file_config.pop("llm_user_config")
+                    config_data["llm_user_config"] = LLMUserConfig(**llm_config_data)
+                
+                config_data.update(file_config)
+
+        if test_paths:
+            config_data["test_paths"] = test_paths.split(",")
+            logger.info(f"Using test paths: {config_data['test_paths']}")
+        if output_dir:
+            config_data["output_dir"] = output_dir
+            logger.info(f"Using output directory: {config_data['output_dir']}")
+
+        config = TestConfig(**config_data)
+        
+        evaluate.main(config)
+
+    def record(self, output_dir) -> None:
+
+
+        random_uuid = str(uuid.uuid4())
+
+        url, tenant_name, token = self._get_env_config()
+        config_data = {
+            "output_dir": Path(os.path.join(Path.cwd(), random_uuid)) if output_dir is None else Path(os.path.join(output_dir,random_uuid)),
+            "service_url": url,
+            "tenant_name": tenant_name,
+            "token": token
+        }
+
+        config_data["output_dir"].mkdir(parents=True, exist_ok=True)
+        logger.info(f"Recording chat sessions to {config_data['output_dir']}")
+
+        record_chats(ChatRecordingConfig(**config_data))
+
+    def generate(self, stories_path: str, tools_path: str, output_dir: str) -> None:
+        stories_path = Path(stories_path)
+        tools_path = Path(tools_path)
+
+        if output_dir is None:
+            output_dir = stories_path.parent
+        else:
+            output_dir = Path(output_dir)
+
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        stories_by_agent = {}
+        with stories_path.open("r", encoding="utf-8", newline='') as f:
+            csv_reader = csv.DictReader(f)
+            for row in csv_reader:
+                agent_name = row["agent"]
+                if agent_name not in stories_by_agent:
+                    stories_by_agent[agent_name] = []
+                stories_by_agent[agent_name].append(row["story"])
+
+        for agent_name, stories in stories_by_agent.items():
+            logger.info(f"Found {len(stories)} stories for agent '{agent_name}'")
+            try:
+                agent_controller = AgentsController()
+                agent = agent_controller.get_agent(agent_name, AgentKind.NATIVE)
+                allowed_tools = agent_controller.get_agent_tool_names(agent.tools)
+            except Exception as e:
+                logger.warning(f"Could not get tools for agent {agent_name}: {str(e)}")
+                allowed_tools = []
+
+
+            logger.info(f"Running tool planner for agent {agent_name}")
+            agent_snapshot_path = output_dir / f"{agent_name}_snapshot_llm.json"
+            build_snapshot(agent_name, tools_path, stories, agent_snapshot_path)
+
+            logger.info(f"Running batch annotate for agent {agent_name}")
+            generate_test_cases_from_stories(
+                agent_name=agent_name,
+                stories=stories,
+                tools_path=tools_path,
+                snapshot_path=agent_snapshot_path,
+                output_dir=output_dir / f"{agent_name}_test_cases",
+                allowed_tools=allowed_tools,
+                num_variants=2
+            )
+
+        logger.info("Test cases stored at: %s", output_dir)
+
+    def analyze(self, data_path: str) -> None:
+        config = AnalyzeConfig(data_path=data_path)
+        analyze(config)
+
+    def summarize(self) -> None:
+        pass
+
+    def external_validate(self, config: Dict, data: List[str], credential:str, add_context: bool = False):
+        validator = ExternalAgentValidation(credential=credential,
+                                auth_scheme=config["auth_scheme"],
+                                service_url=config["api_url"])
+        
+        summary = []
+        for entry in data:
+            results = validator.call_validation(entry, add_context)
+            summary.append(results)
+
+        return summary
+    
+    def generate_performance_test(self, agent_name: str, test_data: List[Tuple[str, str]]):
+        performance_test = ExternalAgentPerformanceTest(
+            agent_name=agent_name,
+            test_data=test_data
+        )
+        generated_performance_tests = performance_test.generate_tests()
+
+        return generated_performance_tests

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_command.py

@@ -50,7 +50,7 @@ def list_knowledge_bases(
     controller = KnowledgeBaseController()
     controller.list_knowledge_bases(verbose=verbose)
 
-@knowledge_bases_app.command(name="remove", help="Delete a knowlege base and all ingested documents")
+@knowledge_bases_app.command(name="remove", help="Remove a knowledge base. Note that if your knowledge base was created by uploading documents (for built-in Milvus), the ingested information from your documents will also be deleted. If your knowledge base uses an external knowledge source through an index_config definition, your index will not be deleted.")
 def remove_knowledge_base(
     name: Annotated[
         str,
@@ -64,7 +64,7 @@ def remove_knowledge_base(
     controller = KnowledgeBaseController()
     controller.remove_knowledge_base(id=id, name=name)
 
-@knowledge_bases_app.command(name="status", help="Get the status of a knowlege base")
+@knowledge_bases_app.command(name="status", help="Get the status of a knowledge base")
 def knowledge_base_status(
     name: Annotated[
         str,

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py

@@ -197,10 +197,10 @@ class KnowledgeBaseController:
             )
 
             column_args = {
-                "Name": {},
+                "Name": {"overflow": "fold"},
                 "Description": {},
                 "App ID": {},
-                "ID": {}
+                "ID": {"overflow": "fold"}
             }
             
             for column in column_args:

ibm_watsonx_orchestrate/cli/commands/models/model_provider_mapper.py

@@ -23,28 +23,34 @@ PROVIDER_EXTRA_PROPERTIES_LUT = {
     #     'azure_ad_token',
     #     'azure_model_name'
     # },
-    # ModelProvider.BEDROCK: {
-    #     'aws_secret_access_key',
-    #     'aws_access_key_id',
-    #     'aws_session_token',
-    #     'aws_region',
-    #     'aws_auth_type',
-    #     'aws_role_arn',
-    #     'aws_external_id',
-    #     'aws_s3_bucket',
-    #     'aws_s3_object_key',
-    #     'aws_bedrock_model',
-    #     'aws_server_side_encryption',
-    #     'aws_server_side_encryption_kms_key_id'
-    # },
-    # ModelProvider.VERTEX_AI: {
-    #     'vertex_region',
-    #     'vertex_project_id',
-    #     'vertex_service_account_json',
-    #     'vertex_storage_bucket_name',
-    #     'vertex_model_name',
-    #     'filename'
-    # },
+    ModelProvider.AZURE_OPENAI: {
+        'azure_resource_name',
+        'azure_deployment_id',
+        'azure_api_version',
+        'azure_model_name'
+    },
+    ModelProvider.BEDROCK: {
+        'aws_secret_access_key',
+        'aws_access_key_id',
+        'aws_session_token',
+        'aws_region',
+        'aws_auth_type',
+        'aws_role_arn',
+        'aws_external_id',
+        'aws_s3_bucket',
+        'aws_s3_object_key',
+        'aws_bedrock_model',
+        'aws_server_side_encryption',
+        'aws_server_side_encryption_kms_key_id'
+    },
+    ModelProvider.VERTEX_AI: {
+        'vertex_region',
+        'vertex_project_id',
+        'vertex_service_account_json',
+        'vertex_storage_bucket_name',
+        'vertex_model_name',
+        'filename'
+    },
     # ModelProvider.HUGGINGFACE: {'huggingfaceBaseUrl'},
     ModelProvider.MISTRAL_AI: {'mistral_fim_completion'},
     # ModelProvider.STABILITY_AI: {'stability_client_id', 'stability_client_user_id', 'stability_client_version'},
@@ -93,7 +99,8 @@ PROVIDER_REQUIRED_FIELDS = {k:['api_key'] for k in ModelProvider}
 # Use sets to denote when a requirement is 'or'
 PROVIDER_REQUIRED_FIELDS.update({
     ModelProvider.WATSONX: PROVIDER_REQUIRED_FIELDS[ModelProvider.WATSONX] + [{'watsonx_space_id', 'watsonx_project_id', 'watsonx_deployment_id'}],
-    ModelProvider.OLLAMA: PROVIDER_REQUIRED_FIELDS[ModelProvider.OLLAMA] + ['custom_host']
+    ModelProvider.OLLAMA: PROVIDER_REQUIRED_FIELDS[ModelProvider.OLLAMA] + ['custom_host'],
+    ModelProvider.BEDROCK: [],
 })
 
 # def env_file_to_model_ProviderConfig(model_name: str, env_file_path: str) -> ProviderConfig | None:
@@ -163,7 +170,7 @@ def _validate_requirements(provider: ModelProvider, cfg: ProviderConfig, app_id:
         if not app_id:
             missing_credentials_string = f"Missing configuration variable(s) required for the provider {provider}:"
         else:
-            missing_credentials_string = f"The following configuration variable(s) for the provider {provider} are not in the spec provider_config:"
+            missing_credentials_string = f"Be sure to include the following required fields for provider '{provider}' in the connection '{app_id}':"
         for cred in missing_credentials:
             if isinstance(cred, set):
                 cred_str = ' or '.join(list(cred))
@@ -177,7 +184,6 @@ def _validate_requirements(provider: ModelProvider, cfg: ProviderConfig, app_id:
             sys.exit(1)
         else:
             logger.info(missing_credentials_string)
-            logger.info(f"Please ensure these values are set in the connection '{app_id}'.")
 
 
 def validate_ProviderConfig(cfg: ProviderConfig, app_id: str)-> None:

ibm_watsonx_orchestrate/cli/commands/models/models_command.py

@@ -146,18 +146,18 @@ def models_policy_add(
             ModelPolicyStrategyMode,
             typer.Option('--strategy', '-s', help='How to spread traffic across models'),
         ],
+        retry_attempts: Annotated[
+            int,
+            typer.Option('--retry-attempts', help='The number of attempts to retry'),
+        ],
         strategy_on_code: Annotated[
             List[int],
             typer.Option('--strategy-on-code', help='The http status to consider invoking the strategy'),
-        ],
+        ] = None,
         retry_on_code: Annotated[
             List[int],
             typer.Option('--retry-on-code', help='The http status to consider retrying the llm call'),
-        ],
-        retry_attempts: Annotated[
-            int,
-            typer.Option('--retry-attempts', help='The number of attempts to retry'),
-        ],
+        ] = None,
         display_name: Annotated[
             str,
             typer.Option('--display-name', help='What name should this llm appear as within the ui'),