ibm-watsonx-orchestrate 1.8.0b1__py3-none-any.whl → 1.9.0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -2,11 +2,12 @@ import json
 import logging
 import typer
 import os
-import yaml
 import csv
 import rich
 import sys
 import shutil
+import tempfile
+import random
 
 from rich.panel import Panel
 from pathlib import Path
@@ -16,11 +17,37 @@ from typing_extensions import Annotated
 
 from ibm_watsonx_orchestrate import __version__
 from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 
 logger = logging.getLogger(__name__)
 
 evaluation_app = typer.Typer(no_args_is_help=True)
 
+def _native_agent_template():
+    return {
+        "spec_version": "v1",
+        "style": "default",
+        "llm": "watsonx/meta-llama/llama-3-405b-instruct",
+        "name": "",
+        "description": "Native agent for validating external agent",
+        "instructions": "Use the tools and external agent(s) provided to answer the user's question.  If you do not have enough information to answer the question, say so.  If you need more information, ask follow up questions.",
+        "collaborators": []
+    }
+
+def _random_native_agent_name(external_agent_name):
+    """ Generate a native agent name in the following format to ensure uniqueness:
+
+    "external_agent_validation_{external_agent_name}_{random number}
+
+    So if the external agent name is, "QA_Agent", and the random number generated is, '100', the native agent name is:
+    "external_agent_validation_QA_Agent_100"
+
+    """
+    seed = 42
+    random.seed(seed)
+    
+    return f"external_agent_validation_{external_agent_name}_{random.randint(0, 100)}"
+
 def read_env_file(env_path: Path|str) -> dict:
     return dotenv_values(str(env_path))
 
@@ -218,7 +245,7 @@ def validate_external(
             str,
             typer.Option(
                 "--external-agent-config", "-ext",
-                help="Path to the external agent yaml",
+                help="Path to the external agent json file",
 
             )
         ],
@@ -244,33 +271,65 @@ def validate_external(
             help="Path to a .env file that overrides default.env. Then environment variables override both."
         ),
     ] = None,
-    agent_name: Annotated[
-        str,
+    perf_test: Annotated[
+        bool,
         typer.Option(
-            "--agent_name", "-a",
-            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
-            " If this parameter is pased, validation of the external agent is not run.",
-            rich_help_panel="Parameters for Input Evaluation"
+            "--perf", "-p",
+            help="Performance test your external agent against the provide user stories.",
+            rich_help_panel="Parameters for Input Evaluation",
         )
-    ] = None
+    ] = False
 ):
 
     validate_watsonx_credentials(user_env_file)
-    Path(output_dir).mkdir(exist_ok=True)
-    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
 
-    if agent_name is not None:
-        eval_dir = os.path.join(output_dir, "evaluation")
+    with open(external_agent_config, 'r') as f:
+        try:
+            external_agent_config = json.load(f)
+        except Exception:
+            rich.print(
+                f"[red]: Please provide a valid external agent spec in JSON format. See 'examples/evaluations/external_agent_validation/sample_external_agent_config.json' for an example."
+            )
+            sys.exit(1)
+
+    eval_dir = os.path.join(output_dir, "evaluations")
+    if perf_test:
         if os.path.exists(eval_dir):
             rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
-            shutil.rmtree(os.path.join(output_dir, "evaluation"))
-        Path(eval_dir).mkdir(exist_ok=True)
+            shutil.rmtree(eval_dir)
+        Path(eval_dir).mkdir(exist_ok=True, parents=True)
         # save external agent config even though its not used for evaluation
         # it can help in later debugging customer agents
-        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
-            with open(external_agent_config, "r") as cfg:
-                external_agent_config = yaml.safe_load(cfg)
-            yaml.safe_dump(external_agent_config, f, indent=4)
+        with open(os.path.join(eval_dir, f"external_agent_cfg.json"), "w+") as f:
+            json.dump(external_agent_config, f, indent=4)
+
+        logger.info("Registering External Agent")
+        agent_controller = AgentsController()
+
+        external_agent_config["title"] = external_agent_config["name"]
+        external_agent_config["auth_config"] = {"token": credential}
+        external_agent_config["spec_version"] = external_agent_config.get("spec_version", "v1")
+        external_agent_config["provider"] = "external_chat"
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(external_agent_config, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
+
+        logger.info("Registering Native Agent")
+
+        native_agent_template = _native_agent_template()
+        agent_name = _random_native_agent_name(external_agent_config["name"])
+        rich.print(f"[blue][b]Generated native agent name is: [i]{agent_name}[/i][/b]")
+        native_agent_template["name"] = agent_name
+        native_agent_template["collaborators"] = [external_agent_config["name"]]
+
+        with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", suffix=".json", delete=True) as fp:
+            json.dump(native_agent_template, fp, indent=4)
+            fp.flush()
+            agents = agent_controller.import_agent(file=os.path.abspath(fp.name), app_id=None)
+            agent_controller.publish_or_update_agents(agents)
 
         rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
         performance_test(
@@ -281,8 +340,6 @@ def validate_external(
         )
     
     else:
-        with open(external_agent_config, "r") as f:
-            external_agent_config = yaml.safe_load(f)
         controller = EvaluationsController()
         test_data = []
         with open(data_path, "r") as f:
@@ -290,31 +347,29 @@ def validate_external(
             for line in csv_reader:
                 test_data.append(line[0])
 
-        # save validation results in "validation_results" sub-dir
-        validation_folder = Path(output_dir) / "validation_results"
+        # save validation results in "validate_external" sub-dir
+        validation_folder = Path(output_dir) / "validate_external"
         if os.path.exists(validation_folder):
             rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
             shutil.rmtree(validation_folder)
         validation_folder.mkdir(exist_ok=True, parents=True)
+        shutil.copy(data_path, os.path.join(validation_folder, "input_sample.tsv"))
 
         # validate the inputs in the provided csv file
         summary = controller.external_validate(external_agent_config, test_data, credential)
-        with open(validation_folder / "validation_results.json", "w") as f:
-            json.dump(summary, f, indent=4)
-        
         # validate sample block inputs
-        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        rich.print("[gold3]Validating external agent against an array of messages.")
         block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
-        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
-            json.dump(block_input_summary, f, indent=4)
-
+        
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump([summary, block_input_summary], f, indent=4)
+        
         user_validation_successful = all([item["success"] for item in summary])
         block_validation_successful = all([item["success"] for item in block_input_summary])
 
         if user_validation_successful and block_validation_successful:
             msg = (
                 f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
-                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
             )
         else:
             msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"

ibm_watsonx_orchestrate/cli/commands/models/model_provider_mapper.py

@@ -10,19 +10,23 @@ _BASIC_PROVIDER_CONFIG_KEYS = {'provider', 'api_key', 'custom_host', 'url_to_fet
 
 PROVIDER_EXTRA_PROPERTIES_LUT = {
     ModelProvider.ANTHROPIC: {'anthropic_beta', 'anthropic_version'},
-    # ModelProvider.AZURE_AI: {
-    #     'azure_resource_name',
-    #     'azure_deployment_id',
-    #     'azure_api_version',
-    #     'ad_auth',
-    #     'azure_auth_mode',
-    #     'azure_managed_client_id',
-    #     'azure_entra_client_id',
-    #     'azure_entra_client_secret',
-    #     'azure_entra_tenant_id',
-    #     'azure_ad_token',
-    #     'azure_model_name'
-    # },
+    ModelProvider.AZURE_AI: {
+        'azure_resource_name',
+        'azure_deployment_id',
+        'azure_api_version',
+        'ad_auth',
+        'azure_auth_mode',
+        'azure_managed_client_id',
+        'azure_entra_client_id',
+        'azure_entra_client_secret',
+        'azure_entra_tenant_id',
+        'azure_ad_token',
+        'azure_model_name',
+        'azure_inference_deployment_name',
+        'azure_inference_api_version',
+        'azure_inference_extra_params',
+        'azure_inference_foundry_url'
+    },
     ModelProvider.AZURE_OPENAI: {
         'azure_resource_name',
         'azure_deployment_id',

ibm_watsonx_orchestrate/cli/commands/models/models_controller.py

@@ -167,15 +167,12 @@ class ModelsController:
             logger.error("Error: WATSONX_URL is required in the environment.")
             sys.exit(1)
 
-        if is_cpd_env(models_client.base_url):
-            virtual_models = []
-            virtual_model_policies = []
-        else:
-            logger.info("Retrieving virtual-model models list...")
-            virtual_models = models_client.list()
+    
+        logger.info("Retrieving virtual-model models list...")
+        virtual_models = models_client.list()
 
-            logger.info("Retrieving virtual-policies models list...")
-            virtual_model_policies = model_policies_client.list()
+        logger.info("Retrieving virtual-policies models list...")
+        virtual_model_policies = model_policies_client.list()
 
         logger.info("Retrieving watsonx.ai models list...")
         found_models = _get_wxai_foundational_models()

ibm_watsonx_orchestrate/cli/commands/server/server_command.py

@@ -4,11 +4,13 @@ import os
 import platform
 import subprocess
 import sys
+import shutil
 import tempfile
 import time
 from pathlib import Path
 from urllib.parse import urlparse
 
+import re
 import jwt
 import requests
 import typer
@@ -32,6 +34,13 @@ logger = logging.getLogger(__name__)
 
 server_app = typer.Typer(no_args_is_help=True)
 
+_EXPORT_FILE_TYPES: set[str] = {
+    'py',
+    'yaml',
+    'yml',
+    'json',
+    'env'
+}
 
 _ALWAYS_UNSET: set[str] = {
     "WO_API_KEY",
@@ -43,11 +52,34 @@ _ALWAYS_UNSET: set[str] = {
     "WO_USERNAME",
     "WO_PASSWORD",
 }
+
+NON_SECRET_ENV_ITEMS: set[str] = {
+    "WO_DEVELOPER_EDITION_SOURCE",
+    "WO_INSTANCE",
+    "USE_SAAS_ML_TOOLS_RUNTIME",
+    "AUTHORIZATION_URL",
+    "OPENSOURCE_REGISTRY_PROXY",
+    "SAAS_WDU_RUNTIME",
+    "LATEST_ENV_FILE",
+}
   
 def define_saas_wdu_runtime(value: str = "none") -> None:
     cfg = Config()
     cfg.write(USER_ENV_CACHE_HEADER,"SAAS_WDU_RUNTIME",value)
 
+def set_compose_file_path_in_env(path: str = None) -> None:
+    Config().save(
+        {
+            USER_ENV_CACHE_HEADER: {
+                "DOCKER_COMPOSE_FILE_PATH" : path
+            }
+        }
+    )
+
+def get_compose_file_path_from_env() -> str:
+    return Config().read(USER_ENV_CACHE_HEADER,"DOCKER_COMPOSE_FILE_PATH")
+
+
 def ensure_docker_installed() -> None:
     try:
         subprocess.run(["docker", "--version"], check=True, capture_output=True)
@@ -106,6 +138,11 @@ def docker_login_by_dev_edition_source(env_dict: dict, source: str) -> None:
 
 
 def get_compose_file() -> Path:
+    custom_compose_path = get_compose_file_path_from_env()
+    return Path(custom_compose_path) if custom_compose_path else get_default_compose_file()
+
+
+def get_default_compose_file() -> Path:
     with resources.as_file(
         resources.files("ibm_watsonx_orchestrate.docker").joinpath("compose-lite.yml")
     ) as compose_file:
@@ -283,12 +320,17 @@ def _prepare_clean_env(env_file: Path) -> None:
     for key in keys_to_unset:
         os.environ.pop(key, None)
 
-def write_merged_env_file(merged_env: dict) -> Path:
-    tmp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env")
-    with tmp:
+def write_merged_env_file(merged_env: dict, target_path: str = None) -> Path:
+
+    if target_path:
+        file = open(target_path,"w")
+    else:
+        file = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env")
+
+    with file:
         for key, val in merged_env.items():
-            tmp.write(f"{key}={val}\n")
-    return Path(tmp.name)
+            file.write(f"{key}={val}\n")
+    return Path(file.name)
 
 
 def get_dbtag_from_architecture(merged_env_dict: dict) -> str:
@@ -310,14 +352,6 @@ def refresh_local_credentials() -> None:
     clear_protected_env_credentials_token()
     _login(name=PROTECTED_ENV_NAME, apikey=None)
 
-NON_SECRET_ENV_ITEMS = {
-    "WO_DEVELOPER_EDITION_SOURCE",
-    "WO_INSTANCE",
-    "USE_SAAS_ML_TOOLS_RUNTIME",
-    "AUTHORIZATION_URL",
-    "OPENSOURCE_REGISTRY_PROXY",
-    "SAAS_WDU_RUNTIME"
-}
 def persist_user_env(env: dict, include_secrets: bool = False) -> None:
     if include_secrets:
         persistable_env = env
@@ -337,7 +371,10 @@ def get_persisted_user_env() -> dict | None:
     return user_env
 
 def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False, experimental_with_ibm_telemetry=False, with_doc_processing=False) -> None:
+    
+    
     compose_path = get_compose_file()
+
     compose_command = ensure_docker_compose_installed()
     _prepare_clean_env(final_env_file)  
     db_tag = read_env_file(final_env_file).get('DBTAG', None)
@@ -754,6 +791,32 @@ def auto_configure_callback_ip(merged_env_dict: dict) -> dict:
 
     return merged_env_dict
 
+def prepare_server_env_vars(user_env: dict = {}):
+    
+    default_env = read_env_file(get_default_env_file())
+    dev_edition_source = get_dev_edition_source(user_env)
+    default_registry_vars = get_default_registry_env_vars_by_dev_edition_source(default_env, user_env, source=dev_edition_source)
+    
+    # Update the default environment with the default registry variables only if they are not already set
+    for key in default_registry_vars:
+        if key not in default_env or not default_env[key]:
+            default_env[key] = default_registry_vars[key]
+
+    # Merge the default environment with the user environment
+    merged_env_dict = {
+        **default_env,
+        **user_env,
+    }
+
+    merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
+
+    # Auto-configure callback IP for async tools
+    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
+
+    apply_llm_api_key_defaults(merged_env_dict)
+
+    return merged_env_dict
+
 @server_app.command(name="start")
 def server_start(
     user_env_file: str = typer.Option(
@@ -787,38 +850,37 @@ def server_start(
         '--with-doc-processing', '-d',
         help='Enable IBM Document Processing to extract information from your business documents. Enabling this activates the Watson Document Understanding service.'
     ),
+    custom_compose_file: str = typer.Option(
+        None,
+        '--compose-file', '-f',
+        help='Provide the path to a custom docker-compose file to use instead of the default compose file'
+    ),  
 ):
     confirm_accepts_license_agreement(accept_terms_and_conditions)
 
     define_saas_wdu_runtime()
     
+    ensure_docker_installed()
+
     if user_env_file and not Path(user_env_file).exists():
-        logger.error(f"Error: The specified environment file '{user_env_file}' does not exist.")
+        logger.error(f"The specified environment file '{user_env_file}' does not exist.")
         sys.exit(1)
-    ensure_docker_installed()
 
-    default_env = read_env_file(get_default_env_file())
+    if custom_compose_file:
+        if Path(custom_compose_file).exists():
+            logger.warning("You are using a custom docker compose file, official support will not be available for this configuration")
+        else:
+            logger.error(f"The specified docker-compose file '{custom_compose_file}' does not exist.")
+            sys.exit(1)
+    
+    #Run regardless, to allow this to set compose as 'None' when not in use 
+    set_compose_file_path_in_env(custom_compose_file)
+
     user_env = read_env_file(user_env_file) if user_env_file else {}
     persist_user_env(user_env, include_secrets=persist_env_secrets)
     
-    dev_edition_source = get_dev_edition_source(user_env)
-    default_registry_vars = get_default_registry_env_vars_by_dev_edition_source(default_env, user_env, source=dev_edition_source)
-
-    # Update the default environment with the default registry variables only if they are not already set
-    for key in default_registry_vars:
-        if key not in default_env or not default_env[key]:
-            default_env[key] = default_registry_vars[key]
+    merged_env_dict = prepare_server_env_vars(user_env)  
 
-    # Merge the default environment with the user environment
-    merged_env_dict = {
-        **default_env,
-        **user_env,
-    }
-
-    merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
-
-    # Auto-configure callback IP for async tools
-    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
     if not _check_exclusive_observibility(experimental_with_langfuse, experimental_with_ibm_telemetry):
         logger.error("Please select either langfuse or ibm telemetry for observability not both")
         sys.exit(1)
@@ -835,14 +897,12 @@ def server_start(
         merged_env_dict['USE_IBM_TELEMETRY'] = 'true'
 
     try:
+        dev_edition_source = get_dev_edition_source(merged_env_dict)
         docker_login_by_dev_edition_source(merged_env_dict, dev_edition_source)
     except ValueError as e:
         logger.error(f"Error: {e}")
         sys.exit(1)
 
-    apply_llm_api_key_defaults(merged_env_dict)
-
-
     final_env_file = write_merged_env_file(merged_env_dict)
 
     run_compose_lite(final_env_file=final_env_file,
@@ -877,7 +937,7 @@ def server_start(
     if experimental_with_langfuse:
         logger.info(f"You can access the observability platform Langfuse at http://localhost:3010, username: orchestrate@ibm.com, password: orchestrate")
     if with_doc_processing:
-        logger.info(f"Document processing capabilities are now available for use in Flows (both ADK and runtime). Note: This option is currently available only in the Developer edition.")
+        logger.info(f"Document processing in Flows (Public Preview) has been enabled.")
 
 @server_app.command(name="stop")
 def server_stop(
@@ -1000,5 +1060,55 @@ def run_db_migration() -> None:
         )
         sys.exit(1)
 
+
+def bump_file_iteration(filename: str) -> str:
+    regex = re.compile(f"^(?P<name>[^\\(\\s\\.\\)]+)(\\((?P<num>\\d+)\\))?(?P<type>\\.(?:{'|'.join(_EXPORT_FILE_TYPES)}))?$")
+    _m = regex.match(filename)
+    iter = int(_m['num']) + 1 if (_m and _m['num']) else 1
+    return f"{_m['name']}({iter}){_m['type'] or ''}"
+
+def get_next_free_file_iteration(filename: str) -> str:
+    while Path(filename).exists():
+        filename = bump_file_iteration(filename)
+    return filename
+
+@server_app.command(name="eject", help="output the docker-compose file and associated env file used to run the server")
+def server_eject(
+    user_env_file: str = typer.Option(
+        None,
+        "--env-file",
+        "-e",
+        help="Path to a .env file that overrides default.env. Then environment variables override both."
+    )
+):
+    
+    if not user_env_file:
+        logger.error(f"To use 'server eject' you need to specify an env file with '--env-file' or '-e'")
+        sys.exit(1)
+
+    if not Path(user_env_file).exists():
+        logger.error(f"The specified environment file '{user_env_file}' does not exist.")
+        sys.exit(1)
+
+    logger.warning("Changes to your docker compose file are not supported")
+    
+    compose_file_path = get_compose_file()
+
+    compose_output_file = get_next_free_file_iteration('docker-compose.yml')
+    logger.info(f"Exporting docker compose file to '{compose_output_file}'")
+
+    shutil.copyfile(compose_file_path,compose_output_file)
+
+
+    user_env = read_env_file(user_env_file)
+    merged_env_dict = prepare_server_env_vars(user_env)
+    
+    env_output_file = get_next_free_file_iteration('server.env')
+    logger.info(f"Exporting env file to '{env_output_file}'")
+
+    write_merged_env_file(merged_env=merged_env_dict,target_path=env_output_file)
+
+    logger.info(f"To make use of the exported configuration file run \"orchestrate server start -e {env_output_file} -f {compose_output_file}\"")
+
 if __name__ == "__main__":
     server_app()

ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_command.py

@@ -64,8 +64,8 @@ def import_toolkit(
     else:
         tool_list = None
 
-    if not package and not package_root:
-        logger.error("You must provide either '--package' or '--package-root'.")
+    if not package and not package_root and not command:
+        logger.error("You must provide either '--package', '--package-root' or '--command'.")
         sys.exit(1)
 
     if package_root and not command:
@@ -85,6 +85,8 @@ def import_toolkit(
             else:
                 logger.error("Unable to infer start up command: '--language' flag must be either 'node' or 'python' when using the '--package' flag without '--command' flag.")
                 sys.exit(1)
+        else:
+            logger.warning(f"Default package installation command for package '{package}' overridden by '--command {command}'.")
 
 
     toolkit_controller = ToolkitController(

ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_controller.py

@@ -70,7 +70,7 @@ class ToolkitController:
         self.client = None
 
         self.source: ToolkitSource = (
-            ToolkitSource.PUBLIC_REGISTRY if package else ToolkitSource.FILES
+            ToolkitSource.FILES if package_root else ToolkitSource.PUBLIC_REGISTRY
         )
 
     def get_client(self) -> ToolKitClient:
@@ -105,6 +105,14 @@ class ToolkitController:
         command = command_parts[0]
         args = command_parts[1:]
 
+        if self.package_root:
+            is_folder = os.path.isdir(self.package_root)
+            is_zip_file = os.path.isfile(self.package_root) and zipfile.is_zipfile(self.package_root)
+
+            if not is_folder and not is_zip_file:
+                logger.error(f"Unable to find a valid directory or zip file at location '{self.package_root}'")
+                sys.exit(1)
+
         console = Console()
 
         with tempfile.TemporaryDirectory() as tmpdir:

ibm_watsonx_orchestrate/cli/commands/tools/tools_controller.py

@@ -590,7 +590,7 @@ class ToolsController:
             for tool in tools:
                 tools_list.append(json.loads(tool.dumps_spec()))
 
-            rich.print(JSON(json.dumps(tools_list, indent=4)))
+            rich.print_json(json.dumps(tools_list, indent=4))
         else:
             table = rich.table.Table(show_header=True, header_style="bold white", show_lines=True)
             column_args = {