ibm-watsonx-orchestrate 1.5.1__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -0,0 +1,224 @@
+import json
+import logging
+import typer
+import os
+import yaml
+import csv
+import rich
+from pathlib import Path
+import sys
+from dotenv import dotenv_values
+
+from typing import Optional
+from typing_extensions import Annotated
+
+from ibm_watsonx_orchestrate import __version__
+from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+
+logger = logging.getLogger(__name__)
+
+evaluation_app = typer.Typer(no_args_is_help=True)
+
+def read_env_file(env_path: Path|str) -> dict:
+    return dotenv_values(str(env_path))
+
+def validate_watsonx_credentials(user_env_file: str) -> bool:
+    required_keys = ["WATSONX_SPACE_ID", "WATSONX_APIKEY"]
+    
+    if all(key in os.environ for key in required_keys):
+        logger.info("WatsonX credentials validated successfully.")
+        return
+    
+    if user_env_file is None:
+        logger.error("WatsonX credentials are not set. Please set WATSONX_SPACE_ID and WATSONX_APIKEY in your system environment variables or include them in your enviroment file and pass it with --env-file option.")
+        sys.exit(1)
+
+    if not Path(user_env_file).exists():
+        logger.error(f"Error: The specified environment file '{user_env_file}' does not exist.")
+        sys.exit(1)
+    
+    user_env = read_env_file(user_env_file)
+    
+    if not all(key in user_env for key in required_keys):
+        logger.error("Error: The environment file does not contain the required keys: WATSONX_SPACE_ID and WATSONX_APIKEY.")
+        sys.exit(1)
+
+    os.environ.update({key: user_env[key] for key in required_keys})
+    logger.info("WatsonX credentials validated successfully.")
+
+
+@evaluation_app.command(name="evaluate", help="Evaluate an agent against a set of test cases")
+def evaluate(
+    config_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--config", "-c",
+            help="Path to YAML configuration file containing evaluation settings."
+        )
+    ] = None,
+    test_paths: Annotated[
+        Optional[str],
+        typer.Option(
+            "--test-paths", "-p", 
+            help="Paths to the test files and/or directories to evaluate, separated by commas."
+        ),
+    ] = None,
+    output_dir: Annotated[
+        Optional[str], 
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the evaluation results."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    if not config_file:
+        if not test_paths or not output_dir:
+            logger.error("Error: Both --test-paths and --output-dir must be provided when not using a config file")
+            exit(1)
+    
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.evaluate(config_file=config_file, test_paths=test_paths, output_dir=output_dir)
+
+
+@evaluation_app.command(name="record", help="Record chat sessions and create test cases")
+def record(
+    output_dir: Annotated[
+        Optional[str], 
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the recorded chats."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.record(output_dir=output_dir)
+
+
+@evaluation_app.command(name="generate", help="Generate test cases from user stories and tools")
+def generate(
+    stories_path: Annotated[
+        str,
+        typer.Option(
+            "--stories_path", "-s",
+            help="Path to the CSV file containing user stories for test case generation. "
+                 "The file has 'story' and 'agent' columns."
+        )
+    ],
+    tools_path: Annotated[
+        str,
+        typer.Option(
+            "--tools_path", "-t",
+            help="Path to the directory containing tool definitions."
+        )
+    ],
+    output_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            "--output_dir", "-o",
+            help="Directory to save the generated test cases."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.generate(stories_path=stories_path, tools_path=tools_path, output_dir=output_dir)
+
+
+@evaluation_app.command(name="analyze", help="Analyze the results of an evaluation run")
+def analyze(data_path: Annotated[
+        str,
+        typer.Option(
+            "--data_path", "-d",
+            help="Path to the directory that has the saved results"
+        )
+    ],
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None):
+
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.analyze(data_path=data_path)
+
+
+@evaluation_app.command(name="validate_external", help="Validate an external agent against a set of inputs")
+def validate_external(
+    data_path: Annotated[
+        str,
+        typer.Option(
+            "--csv", "-c",
+            help="Path to .csv file of inputs"
+        )
+    ],
+    config: Annotated[
+            str,
+            typer.Option(
+                "--config", "-cf",
+                help="Path to the external agent yaml"
+            )
+        ],
+    credential: Annotated[
+        str,
+        typer.Option(
+            "--credential", "-crd",
+            help="credential string"
+        )
+    ],
+    output_dir: Annotated[
+        str,
+        typer.Option(
+            "--output", "-o",
+            help="where to save the validation results"
+        )
+    ] = "./test_external_agent",
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    
+    validate_watsonx_credentials(user_env_file)
+    with open(config, "r") as f:
+        config = yaml.safe_load(f)
+    controller = EvaluationsController()
+    test_data = []
+    with open(data_path, "r") as f:
+        csv_reader = csv.reader(f)
+        for line in csv_reader:
+            test_data.append(line[0])
+    results = controller.external_validate(config, test_data, credential)
+    os.makedirs(output_dir, exist_ok=True)
+    with open(os.path.join(output_dir, "validation_results.json"), "w") as f:
+        json.dump(results, f)
+
+    rich.print(f"[green] validation result is saved to {output_dir} [/green]")

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -0,0 +1,158 @@
+import logging
+from typing import List, Dict, Optional
+import csv
+from pathlib import Path
+import rich
+from wxo_agentic_evaluation import main as evaluate
+from wxo_agentic_evaluation.tool_planner import build_snapshot
+from wxo_agentic_evaluation.analyze_run import analyze
+from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
+from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
+from wxo_agentic_evaluation.record_chat import record_chats
+from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
+from ibm_watsonx_orchestrate import __version__
+from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
+from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
+from ibm_watsonx_orchestrate.agent_builder.agents import AgentKind
+
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationsController:
+    def __init__(self):
+        pass
+
+    def _get_env_config(self) -> tuple[str, str, str | None]:
+        cfg = Config()
+        auth_cfg = Config(AUTH_CONFIG_FILE_FOLDER, AUTH_CONFIG_FILE)
+        
+        url = cfg.get_active_env_config(ENV_WXO_URL_OPT)
+        tenant_name = cfg.get_active_env()
+
+        existing_auth_config = auth_cfg.get(AUTH_SECTION_HEADER).get(tenant_name, {})
+        token = existing_auth_config.get(AUTH_MCSP_TOKEN_OPT) if existing_auth_config else None
+
+        return url, tenant_name, token
+
+    def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
+        url, tenant_name, token = self._get_env_config()
+
+        config_data = {
+            "wxo_lite_version": __version__,
+            "auth_config": AuthConfig(
+                url=url,
+                tenant_name=tenant_name,
+                token=token
+            )
+        }
+
+        if config_file:
+            logger.info(f"Loading configuration from {config_file}")
+            with open(config_file, 'r') as f:
+                file_config = yaml_safe_load(f) or {}
+                
+                if "auth_config" in file_config:
+                    auth_config_data = file_config.pop("auth_config")
+                    config_data["auth_config"] = AuthConfig(**auth_config_data)
+                
+                if "llm_user_config" in file_config:
+                    llm_config_data = file_config.pop("llm_user_config")
+                    config_data["llm_user_config"] = LLMUserConfig(**llm_config_data)
+                
+                config_data.update(file_config)
+
+        if test_paths:
+            config_data["test_paths"] = test_paths.split(",")
+            logger.info(f"Using test paths: {config_data['test_paths']}")
+        if output_dir:
+            config_data["output_dir"] = output_dir
+            logger.info(f"Using output directory: {config_data['output_dir']}")
+
+        config = TestConfig(**config_data)
+        
+        evaluate.main(config)
+
+    def record(self, output_dir) -> None:
+        url, tenant_name, token = self._get_env_config()
+        config_data = {
+            "output_dir": Path.cwd() if output_dir is None else Path(output_dir),
+            "service_url": url,
+            "tenant_name": tenant_name,
+            "token": token
+        }
+
+        config_data["output_dir"].mkdir(parents=True, exist_ok=True)
+        logger.info(f"Recording chat sessions to {config_data['output_dir']}")
+
+        record_chats(ChatRecordingConfig(**config_data))
+
+    def generate(self, stories_path: str, tools_path: str, output_dir: str) -> None:
+        stories_path = Path(stories_path)
+        tools_path = Path(tools_path)
+
+        if output_dir is None:
+            output_dir = stories_path.parent
+        else:
+            output_dir = Path(output_dir)
+
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        stories_by_agent = {}
+        with stories_path.open("r", encoding="utf-8", newline='') as f:
+            csv_reader = csv.DictReader(f)
+            for row in csv_reader:
+                agent_name = row["agent"]
+                if agent_name not in stories_by_agent:
+                    stories_by_agent[agent_name] = []
+                stories_by_agent[agent_name].append(row["story"])
+
+        for agent_name, stories in stories_by_agent.items():
+            logger.info(f"Found {len(stories)} stories for agent '{agent_name}'")
+            try:
+                agent_controller = AgentsController()
+                agent = agent_controller.get_agent(agent_name, AgentKind.NATIVE)
+                allowed_tools = agent_controller.get_agent_tool_names(agent.tools)
+            except Exception as e:
+                logger.warning(f"Could not get tools for agent {agent_name}: {str(e)}")
+                allowed_tools = []
+
+
+            logger.info(f"Running tool planner for agent {agent_name}")
+            agent_snapshot_path = output_dir / f"{agent_name}_snapshot_llm.json"
+            build_snapshot(agent_name, tools_path, stories, agent_snapshot_path)
+
+            logger.info(f"Running batch annotate for agent {agent_name}")
+            generate_test_cases_from_stories(
+                agent_name=agent_name,
+                stories=stories,
+                tools_path=tools_path,
+                snapshot_path=agent_snapshot_path,
+                output_dir=output_dir / f"{agent_name}_test_cases",
+                allowed_tools=allowed_tools,
+                num_variants=2
+            )
+
+        logger.info("Test cases stored at: %s", output_dir)
+
+    def analyze(self, data_path: str) -> None:
+        config = AnalyzeConfig(data_path=data_path)
+        analyze(config)
+
+    def summarize(self) -> None:
+        pass
+
+    def external_validate(self, config: Dict, data: List[str], credential:str):
+        validator = ExternalAgentValidation(credential=credential,
+                                auth_scheme=config["auth_scheme"],
+                                service_url=config["api_url"])
+        summary = []
+        for entry in data:
+            results = validator.call_validation(entry)
+            if len(results) == 0:
+                rich.print(f"[red] No events are generated for input {entry} [/red]")
+            summary.append({entry: results})
+
+        return summary
+

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_command.py

@@ -50,7 +50,7 @@ def list_knowledge_bases(
     controller = KnowledgeBaseController()
     controller.list_knowledge_bases(verbose=verbose)
 
-@knowledge_bases_app.command(name="remove", help="Delete a knowlege base and all ingested documents")
+@knowledge_bases_app.command(name="remove", help="Remove a knowledge base. Note that if your knowledge base was created by uploading documents (for built-in Milvus), the ingested information from your documents will also be deleted. If your knowledge base uses an external knowledge source through an index_config definition, your index will not be deleted.")
 def remove_knowledge_base(
     name: Annotated[
         str,
@@ -64,7 +64,7 @@ def remove_knowledge_base(
     controller = KnowledgeBaseController()
     controller.remove_knowledge_base(id=id, name=name)
 
-@knowledge_bases_app.command(name="status", help="Get the status of a knowlege base")
+@knowledge_bases_app.command(name="status", help="Get the status of a knowledge base")
 def knowledge_base_status(
     name: Annotated[
         str,

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py

@@ -197,10 +197,10 @@ class KnowledgeBaseController:
             )
 
             column_args = {
-                "Name": {},
+                "Name": {"overflow": "fold"},
                 "Description": {},
                 "App ID": {},
-                "ID": {}
+                "ID": {"overflow": "fold"}
             }
             
             for column in column_args:

ibm_watsonx_orchestrate/cli/commands/models/model_provider_mapper.py

@@ -23,28 +23,34 @@ PROVIDER_EXTRA_PROPERTIES_LUT = {
     #     'azure_ad_token',
     #     'azure_model_name'
     # },
-    # ModelProvider.BEDROCK: {
-    #     'aws_secret_access_key',
-    #     'aws_access_key_id',
-    #     'aws_session_token',
-    #     'aws_region',
-    #     'aws_auth_type',
-    #     'aws_role_arn',
-    #     'aws_external_id',
-    #     'aws_s3_bucket',
-    #     'aws_s3_object_key',
-    #     'aws_bedrock_model',
-    #     'aws_server_side_encryption',
-    #     'aws_server_side_encryption_kms_key_id'
-    # },
-    # ModelProvider.VERTEX_AI: {
-    #     'vertex_region',
-    #     'vertex_project_id',
-    #     'vertex_service_account_json',
-    #     'vertex_storage_bucket_name',
-    #     'vertex_model_name',
-    #     'filename'
-    # },
+    ModelProvider.AZURE_OPENAI: {
+        'azure_resource_name',
+        'azure_deployment_id',
+        'azure_api_version',
+        'azure_model_name'
+    },
+    ModelProvider.BEDROCK: {
+        'aws_secret_access_key',
+        'aws_access_key_id',
+        'aws_session_token',
+        'aws_region',
+        'aws_auth_type',
+        'aws_role_arn',
+        'aws_external_id',
+        'aws_s3_bucket',
+        'aws_s3_object_key',
+        'aws_bedrock_model',
+        'aws_server_side_encryption',
+        'aws_server_side_encryption_kms_key_id'
+    },
+    ModelProvider.VERTEX_AI: {
+        'vertex_region',
+        'vertex_project_id',
+        'vertex_service_account_json',
+        'vertex_storage_bucket_name',
+        'vertex_model_name',
+        'filename'
+    },
     # ModelProvider.HUGGINGFACE: {'huggingfaceBaseUrl'},
     ModelProvider.MISTRAL_AI: {'mistral_fim_completion'},
     # ModelProvider.STABILITY_AI: {'stability_client_id', 'stability_client_user_id', 'stability_client_version'},
@@ -93,7 +99,8 @@ PROVIDER_REQUIRED_FIELDS = {k:['api_key'] for k in ModelProvider}
 # Use sets to denote when a requirement is 'or'
 PROVIDER_REQUIRED_FIELDS.update({
     ModelProvider.WATSONX: PROVIDER_REQUIRED_FIELDS[ModelProvider.WATSONX] + [{'watsonx_space_id', 'watsonx_project_id', 'watsonx_deployment_id'}],
-    ModelProvider.OLLAMA: PROVIDER_REQUIRED_FIELDS[ModelProvider.OLLAMA] + ['custom_host']
+    ModelProvider.OLLAMA: PROVIDER_REQUIRED_FIELDS[ModelProvider.OLLAMA] + ['custom_host'],
+    ModelProvider.BEDROCK: [],
 })
 
 # def env_file_to_model_ProviderConfig(model_name: str, env_file_path: str) -> ProviderConfig | None:
@@ -163,7 +170,7 @@ def _validate_requirements(provider: ModelProvider, cfg: ProviderConfig, app_id:
         if not app_id:
             missing_credentials_string = f"Missing configuration variable(s) required for the provider {provider}:"
         else:
-            missing_credentials_string = f"The following configuration variable(s) for the provider {provider} are not in the spec provider_config:"
+            missing_credentials_string = f"Be sure to include the following required fields for provider '{provider}' in the connection '{app_id}':"
         for cred in missing_credentials:
             if isinstance(cred, set):
                 cred_str = ' or '.join(list(cred))
@@ -177,7 +184,6 @@ def _validate_requirements(provider: ModelProvider, cfg: ProviderConfig, app_id:
             sys.exit(1)
         else:
             logger.info(missing_credentials_string)
-            logger.info(f"Please ensure these values are set in the connection '{app_id}'.")
 
 
 def validate_ProviderConfig(cfg: ProviderConfig, app_id: str)-> None:

ibm_watsonx_orchestrate/cli/commands/models/models_command.py

@@ -146,18 +146,18 @@ def models_policy_add(
             ModelPolicyStrategyMode,
             typer.Option('--strategy', '-s', help='How to spread traffic across models'),
         ],
+        retry_attempts: Annotated[
+            int,
+            typer.Option('--retry-attempts', help='The number of attempts to retry'),
+        ],
         strategy_on_code: Annotated[
             List[int],
             typer.Option('--strategy-on-code', help='The http status to consider invoking the strategy'),
-        ],
+        ] = None,
         retry_on_code: Annotated[
             List[int],
             typer.Option('--retry-on-code', help='The http status to consider retrying the llm call'),
-        ],
-        retry_attempts: Annotated[
-            int,
-            typer.Option('--retry-attempts', help='The number of attempts to retry'),
-        ],
+        ] = None,
         display_name: Annotated[
             str,
             typer.Option('--display-name', help='What name should this llm appear as within the ui'),

ibm_watsonx_orchestrate/cli/commands/models/models_controller.py

@@ -18,7 +18,7 @@ from ibm_watsonx_orchestrate.agent_builder.model_policies.types import ModelPoli
     ModelPolicyRetry, ModelPolicyStrategy, ModelPolicyStrategyMode, ModelPolicyTarget
 from ibm_watsonx_orchestrate.client.models.models_client import ModelsClient
 from ibm_watsonx_orchestrate.agent_builder.models.types import VirtualModel, ProviderConfig, ModelType, ANTHROPIC_DEFAULT_MAX_TOKENS
-from ibm_watsonx_orchestrate.client.utils import instantiate_client
+from ibm_watsonx_orchestrate.client.utils import instantiate_client, is_cpd_env
 from ibm_watsonx_orchestrate.client.connections import get_connection_id, ConnectionType
 
 logger = logging.getLogger(__name__)
@@ -85,6 +85,11 @@ def import_python_policy(file: str) -> List[ModelPolicy]:
             models.append(obj)
     return models
 
+def validate_spec_content(content: dict) -> None:
+    if not content.get("spec_version"):
+        logger.error(f"Field 'spec_version' not provided. Please ensure provided spec conforms to a valid spec format")
+        sys.exit(1)
+
 def parse_model_file(file: str) -> List[VirtualModel]:
     if file.endswith('.yaml') or file.endswith('.yml') or file.endswith(".json"):
         with open(file, 'r') as f:
@@ -92,6 +97,7 @@ def parse_model_file(file: str) -> List[VirtualModel]:
                 content = json.load(f)
             else:
                 content = yaml.load(f, Loader=yaml.SafeLoader)
+        validate_spec_content(content)
         model = create_model_from_spec(spec=content)
         return [model]
     elif file.endswith('.py'):
@@ -107,6 +113,7 @@ def parse_policy_file(file: str) -> List[ModelPolicy]:
                 content = json.load(f)
             else:
                 content = yaml.load(f, Loader=yaml.SafeLoader)
+        validate_spec_content(content)
         policy = create_policy_from_spec(spec=content)
         return [policy]
     elif file.endswith('.py'):
@@ -160,13 +167,15 @@ class ModelsController:
             logger.error("Error: WATSONX_URL is required in the environment.")
             sys.exit(1)
 
-        logger.info("Retrieving virtual-model models list...")
-        virtual_models = models_client.list()
-
-
+        if is_cpd_env(models_client.base_url):
+            virtual_models = []
+            virtual_model_policies = []
+        else:
+            logger.info("Retrieving virtual-model models list...")
+            virtual_models = models_client.list()
 
-        logger.info("Retrieving virtual-policies models list...")
-        virtual_model_policies = model_policies_client.list()
+            logger.info("Retrieving virtual-policies models list...")
+            virtual_model_policies = model_policies_client.list()
 
         logger.info("Retrieving watsonx.ai models list...")
         found_models = _get_wxai_foundational_models()
@@ -382,7 +391,7 @@ class ModelsController:
             mode=strategy,
             on_status_codes=strategy_on_code
         )
-        inner.targets = [ModelPolicyTarget(weight=1, model_name=m) for m in models]
+        inner.targets = [ModelPolicyTarget(model_name=m) for m in models]
         if retry_on_code:
             inner.retry = ModelPolicyRetry(
                 on_status_codes=retry_on_code,