ibm-watsonx-orchestrate 1.6.3__py3-none-any.whl → 1.6.4__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -2,11 +2,12 @@ import json
 import logging
 import typer
 import os
-import yaml
 import csv
 import rich
 import sys
 import shutil
+import tempfile
+import random
 
 from rich.panel import Panel
 from pathlib import Path
@@ -16,23 +17,55 @@ from typing_extensions import Annotated
 
 from ibm_watsonx_orchestrate import __version__
 from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 
 logger = logging.getLogger(__name__)
 
 evaluation_app = typer.Typer(no_args_is_help=True)
 
+def _native_agent_template():
+    return {
+        "spec_version": "v1",
+        "style": "default",
+        "llm": "watsonx/meta-llama/llama-3-405b-instruct",
+        "name": "",
+        "description": "Native agent for validating external agent",
+        "instructions": "Use the tools and external agent(s) provided to answer the user's question.  If you do not have enough information to answer the question, say so.  If you need more information, ask follow up questions.",
+        "collaborators": []
+    }
+
+def _random_native_agent_name(external_agent_name):
+    """ Generate a native agent name in the following format to ensure uniqueness:
+
+    "external_agent_validation_{external_agent_name}_{random number}
+
+    So if the external agent name is, "QA_Agent", and the random number generated is, '100', the native agent name is:
+    "external_agent_validation_QA_Agent_100"
+
+    """
+    seed = 42
+    random.seed(seed)
+    
+    return f"external_agent_validation_{external_agent_name}_{random.randint(0, 100)}"
+
 def read_env_file(env_path: Path|str) -> dict:
     return dotenv_values(str(env_path))
 
 def validate_watsonx_credentials(user_env_file: str) -> bool:
-    required_keys = ["WATSONX_SPACE_ID", "WATSONX_APIKEY"]
+    required_sets = [
+        ["WATSONX_SPACE_ID", "WATSONX_APIKEY"],
+        ["WO_INSTANCE", "WO_API_KEY"]
+    ]
     
-    if all(key in os.environ for key in required_keys):
+    def has_valid_keys(env: dict) -> bool:
+        return any(all(key in env for key in key_set) for key_set in required_sets)
+
+    if has_valid_keys(os.environ):
         logger.info("WatsonX credentials validated successfully.")
         return
     
     if user_env_file is None:
-        logger.error("WatsonX credentials are not set. Please set WATSONX_SPACE_ID and WATSONX_APIKEY in your system environment variables or include them in your enviroment file and pass it with --env-file option.")
+        logger.error("WatsonX credentials are not set. Please set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables or include them in your environment file and pass it with --env-file option.")
         sys.exit(1)
 
     if not Path(user_env_file).exists():
@@ -41,11 +74,15 @@ def validate_watsonx_credentials(user_env_file: str) -> bool:
     
     user_env = read_env_file(user_env_file)
     
-    if not all(key in user_env for key in required_keys):
-        logger.error("Error: The environment file does not contain the required keys: WATSONX_SPACE_ID and WATSONX_APIKEY.")
+    if not has_valid_keys(user_env):
+        logger.error("Error: The environment file does not contain the required keys: either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY.")
         sys.exit(1)
 
-    os.environ.update({key: user_env[key] for key in required_keys})
+    # Update os.environ with whichever set is present
+    for key_set in required_sets:
+        if all(key in user_env for key in key_set):
+            os.environ.update({key: user_env[key] for key in key_set})
+            break
     logger.info("WatsonX credentials validated successfully.")
 
 def read_csv(data_path: str, delimiter="\t"):
@@ -208,7 +245,7 @@ def validate_external(
             str,
             typer.Option(
                 "--external-agent-config", "-ext",
-                help="Path to the external agent yaml",
+                help="Path to the external agent json file",
 
             )
         ],
@@ -234,33 +271,65 @@ def validate_external(
             help="Path to a .env file that overrides default.env. Then environment variables override both."
         ),
     ] = None,
-    agent_name: Annotated[
-        str,
+    perf_test: Annotated[
+        bool,
         typer.Option(
-            "--agent_name", "-a",
-            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
-            " If this parameter is pased, validation of the external agent is not run.",
-            rich_help_panel="Parameters for Input Evaluation"
+            "--perf", "-p",
+            help="Performance test your external agent against the provide user stories.",
+            rich_help_panel="Parameters for Input Evaluation",
         )
-    ] = None
+    ] = False
 ):
 
     validate_watsonx_credentials(user_env_file)
-    Path(output_dir).mkdir(exist_ok=True)
-    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
 
-    if agent_name is not None:
-        eval_dir = os.path.join(output_dir, "evaluation")
+    with open(external_agent_config, 'r') as f:
+        try:
+            external_agent_config = json.load(f)
+        except Exception:
+            rich.print(
+                f"[red]: Please provide a valid external agent spec in JSON format. See 'examples/evaluations/external_agent_validation/sample_external_agent_config.json' for an example."
+            )
+            sys.exit(1)
+
+    eval_dir = os.path.join(output_dir, "evaluations")
+    if perf_test:
         if os.path.exists(eval_dir):
             rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
-            shutil.rmtree(os.path.join(output_dir, "evaluation"))
-        Path(eval_dir).mkdir(exist_ok=True)
+            shutil.rmtree(eval_dir)
+        Path(eval_dir).mkdir(exist_ok=True, parents=True)
         # save external agent config even though its not used for evaluation
         # it can help in later debugging customer agents
-        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
-            with open(external_agent_config, "r") as cfg:
-                external_agent_config = yaml.safe_load(cfg)
-            yaml.safe_dump(external_agent_config, f, indent=4)
+        with open(os.path.join(eval_dir, f"external_agent_cfg.json"), "w+") as f:
+            json.dump(external_agent_config, f, indent=4)
+
+        logger.info("Registering External Agent")
+        agent_controller = AgentsController()
+
+        external_agent_config["title"] = external_agent_config["name"]
+        external_agent_config["auth_config"] = {"token": credential}
+        external_agent_config["spec_version"] = external_agent_config.get("spec_version", "v1")
+        external_agent_config["provider"] = "external_chat"
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(external_agent_config, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
+
+        logger.info("Registering Native Agent")
+
+        native_agent_template = _native_agent_template()
+        agent_name = _random_native_agent_name(external_agent_config["name"])
+        rich.print(f"[blue][b]Generated native agent name is: [i]{agent_name}[/i][/b]")
+        native_agent_template["name"] = agent_name
+        native_agent_template["collaborators"] = [external_agent_config["name"]]
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(native_agent_template, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
 
         rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
         performance_test(
@@ -271,8 +340,6 @@ def validate_external(
         )
     
     else:
-        with open(external_agent_config, "r") as f:
-            external_agent_config = yaml.safe_load(f)
         controller = EvaluationsController()
         test_data = []
         with open(data_path, "r") as f:
@@ -280,31 +347,29 @@ def validate_external(
             for line in csv_reader:
                 test_data.append(line[0])
 
-        # save validation results in "validation_results" sub-dir
-        validation_folder = Path(output_dir) / "validation_results"
+        # save validation results in "validate_external" sub-dir
+        validation_folder = Path(output_dir) / "validate_external"
         if os.path.exists(validation_folder):
             rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
             shutil.rmtree(validation_folder)
         validation_folder.mkdir(exist_ok=True, parents=True)
+        shutil.copy(data_path, os.path.join(validation_folder, "input_sample.tsv"))
 
         # validate the inputs in the provided csv file
         summary = controller.external_validate(external_agent_config, test_data, credential)
-        with open(validation_folder / "validation_results.json", "w") as f:
-            json.dump(summary, f, indent=4)
-        
         # validate sample block inputs
-        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        rich.print("[gold3]Validating external agent against an array of messages.")
         block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
-        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
-            json.dump(block_input_summary, f, indent=4)
-
+        
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump([summary, block_input_summary], f, indent=4)
+        
         user_validation_successful = all([item["success"] for item in summary])
         block_validation_successful = all([item["success"] for item in block_input_summary])
 
         if user_validation_successful and block_validation_successful:
             msg = (
                 f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
-                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
             )
         else:
             msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -3,12 +3,12 @@ import os.path
 from typing import List, Dict, Optional, Tuple
 import csv
 from pathlib import Path
-import rich
+import sys
 from wxo_agentic_evaluation import main as evaluate
 from wxo_agentic_evaluation.tool_planner import build_snapshot
 from wxo_agentic_evaluation.analyze_run import analyze
 from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
-from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
+from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig
 from wxo_agentic_evaluation.record_chat import record_chats
 from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
 from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
@@ -41,12 +41,26 @@ class EvaluationsController:
     def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
         url, tenant_name, token = self._get_env_config()
 
+        if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
+            provider = "watsonx"
+        elif "WO_INSTANCE" in os.environ and "WO_API_KEY" in os.environ:
+            provider = "model_proxy"
+        else:
+            logger.error(
+                "No provider found. Please either provide a config_file or set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables."
+            )
+            sys.exit(1)
+        
         config_data = {
             "wxo_lite_version": __version__,
             "auth_config": AuthConfig(
                 url=url,
                 tenant_name=tenant_name,
                 token=token
+            ),
+            "provider_config": ProviderConfig(
+                provider=provider,
+                model_id="meta-llama/llama-3-405b-instruct",
             )
         }
 
@@ -62,6 +76,10 @@ class EvaluationsController:
                 if "llm_user_config" in file_config:
                     llm_config_data = file_config.pop("llm_user_config")
                     config_data["llm_user_config"] = LLMUserConfig(**llm_config_data)
+
+                if "provider_config" in file_config:
+                    provider_config_data = file_config.pop("provider_config")
+                    config_data["provider_config"] = ProviderConfig(**provider_config_data)
                 
                 config_data.update(file_config)
 

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_command.py

@@ -21,24 +21,6 @@ def knowledge_base_import(
     controller = KnowledgeBaseController()
     controller.import_knowledge_base(file=file, app_id=app_id)
 
-@knowledge_bases_app.command(name="patch", help="Patch a knowledge base by uploading documents, or providing an external vector index")
-def knowledge_base_patch(
-    file: Annotated[
-        str,
-        typer.Option("--file", "-f", help="YAML or JSON file with knowledge base definition"),
-    ],
-    name: Annotated[
-        str,
-        typer.Option("--name", "-n", help="Name of the knowledge base you wish to update"),
-    ]=None,
-    id: Annotated[
-        str,
-        typer.Option("--id", "-i", help="ID of the knowledge base you wish to update"),
-    ]=None
-):
-    controller = KnowledgeBaseController()
-    controller.update_knowledge_base(id=id, name=name, file=file)
-
 
 @knowledge_bases_app.command(name="list", help="List all knowledge bases")
 def list_knowledge_bases(

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py

@@ -8,7 +8,6 @@ import inspect
 from pathlib import Path
 from typing import List
 
-from ibm_watsonx_orchestrate.agent_builder.knowledge_bases.knowledge_base_requests import KnowledgeBaseUpdateRequest
 from ibm_watsonx_orchestrate.agent_builder.knowledge_bases.knowledge_base import KnowledgeBase
 from ibm_watsonx_orchestrate.client.knowledge_bases.knowledge_base_client import KnowledgeBaseClient
 from ibm_watsonx_orchestrate.client.base_api_client import ClientAPIException
@@ -72,11 +71,21 @@ class KnowledgeBaseController:
         client = self.get_client()
 
         knowledge_bases = parse_file(file=file)
+        
+        existing_knowledge_bases = client.get_by_names([kb.name for kb in knowledge_bases])
+        
         for kb in knowledge_bases:
             try:
+                file_dir = "/".join(file.split("/")[:-1])
+
+                existing = list(filter(lambda ex: ex.get('name') == kb.name, existing_knowledge_bases))
+                if len(existing) > 0:
+                    logger.info(f"Existing knowledge base '{kb.name}' found. Updating...")
+                    self.update_knowledge_base(existing[0].get("id"), kb=kb, file_dir=file_dir)
+                    continue
+
                 kb.validate_documents_or_index_exists()
                 if kb.documents:
-                    file_dir = "/".join(file.split("/")[:-1])
                     files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in kb.documents]
                     
                     kb.prioritize_built_in_index = True
@@ -106,10 +115,7 @@ class KnowledgeBaseController:
                 
                 logger.info(f"Successfully imported knowledge base '{kb.name}'")
             except ClientAPIException as e:
-                if "duplicate key value violates unique constraint" in e.response.text:
-                    logger.error(f"A knowledge base with the name '{kb.name}' already exists. Failed to import knowledge base")
-                else:
-                    logger.error(f"Error importing knowledge base '{kb.name}\n' {e.response.text}")
+                logger.error(f"Error importing knowledge base '{kb.name}\n' {e.response.text}")
     
     def get_id(
         self, id: str, name: str
@@ -131,27 +137,37 @@ class KnowledgeBaseController:
 
 
     def update_knowledge_base(
-        self, id: str, name: str, file: str
-    ) -> None:
-        knowledge_base_id = self.get_id(id, name)
-        update_request = KnowledgeBaseUpdateRequest.from_spec(file=file)
+        self, knowledge_base_id: str, kb: KnowledgeBase, file_dir: str
+    ) -> None:        
+        if kb.documents:
+            status = self.get_client().status(knowledge_base_id)
+            existing_docs = [doc.get("metadata", {}).get("original_file_name", "") for doc in status.get("documents", [])]
+            
+            removed_docs = existing_docs[:]
+            for filepath in kb.documents:
+                filename = get_file_name(filepath)
+
+                if filename in existing_docs:
+                    logger.warning(f'Document \"{filename}\" already exists in knowledge base. Updating...')
+                    removed_docs.remove(filename)
+
+            for filename in removed_docs:
+                logger.warning(f'Document \"{filename}\" removed from knowledge base.')
 
-        if update_request.documents:
-            file_dir = "/".join(file.split("/")[:-1])
-            files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in update_request.documents]
+
+            files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in kb.documents]
             
-            update_request.prioritize_built_in_index = True
-            payload = update_request.model_dump(exclude_none=True);
+            kb.prioritize_built_in_index = True
+            payload = kb.model_dump(exclude_none=True);
             payload.pop('documents');
 
             self.get_client().update_with_documents(knowledge_base_id, payload=payload, files=files)
         else:
-            if update_request.conversational_search_tool and update_request.conversational_search_tool.index_config:
-                update_request.prioritize_built_in_index = False
-            self.get_client().update(knowledge_base_id, update_request.model_dump(exclude_none=True))
+            if kb.conversational_search_tool and kb.conversational_search_tool.index_config:
+                kb.prioritize_built_in_index = False
+            self.get_client().update(knowledge_base_id, kb.model_dump(exclude_none=True))
 
-        logEnding = f"with ID '{id}'" if id else f"'{name}'"
-        logger.info(f"Successfully updated knowledge base {logEnding}")
+        logger.info(f"Knowledge base '{kb.name}' updated successfully")
 
 
     def knowledge_base_status( self, id: str, name: str) -> None:

ibm_watsonx_orchestrate/cli/commands/models/models_command.py

@@ -149,7 +149,7 @@ def models_policy_add(
         retry_attempts: Annotated[
             int,
             typer.Option('--retry-attempts', help='The number of attempts to retry'),
-        ],
+        ] = None,
         strategy_on_code: Annotated[
             List[int],
             typer.Option('--strategy-on-code', help='The http status to consider invoking the strategy'),