ibm-watsonx-orchestrate 1.11.0b0__py3-none-any.whl → 1.12.0b0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/connections/connections_command.py

@@ -6,6 +6,7 @@ from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller imp
     remove_connection,
     list_connections,
     import_connection,
+    export_connection,
     configure_connection,
     set_credentials_connection,
     set_identity_provider_connection,
@@ -65,6 +66,23 @@ def import_connection_command(
 ):
     import_connection(file=file)
 
+@connections_app.command(name="export")
+def export_connection_command(
+    app_id: Annotated[
+        str, typer.Option(
+            '--app-id', '-a',
+            help='The app id of the connection you wish to export.'
+        )
+    ],
+    output_file: Annotated[
+        str, typer.Option(
+            '--output', '-o',
+            help='Path to where the exported connection should be saved.'
+        )
+    ] = None
+):
+    export_connection(app_id=app_id,output_file=output_file)
+
 @connections_app.command(name="configure")
 def configure_connection_command(
     app_id: Annotated[

ibm_watsonx_orchestrate/cli/commands/connections/connections_controller.py

@@ -1,4 +1,7 @@
+import io
 import logging
+from pathlib import Path
+import zipfile
 import requests
 import json
 import rich
@@ -7,6 +10,7 @@ import sys
 import typer
 
 from typing import List
+from ibm_watsonx_orchestrate.agent_builder.agents.types import SpecVersion
 from ibm_watsonx_orchestrate.client.utils import is_local_dev
 from ibm_watsonx_orchestrate.agent_builder.connections.types import (
     ConnectionEnvironment,
@@ -277,6 +281,48 @@ def _connection_credentials_parse_entry(text: str, default_location: ConnectionC
     
     return ConnectionCredentialsEntry(key=key, value=value, location=location)
     
+def _combine_connection_configs(configs: List[ConnectionConfiguration]) -> dict:
+    combined_configuration = {
+        'app_id': None,
+        'spec_version': SpecVersion.V1.value,
+        'kind': 'connection',
+        'environments': {
+
+        },
+        'catalog': {
+            'name': None,
+            'description': None,
+            'icon': None
+        }
+    }
+    for config in configs:
+        if combined_configuration.get('app_id') and config.app_id != combined_configuration.get('app_id'):
+            raise ValueError(f"Cannot combine config '{config.app_id}' with config '{combined_configuration.get('app_id')}'")
+        combined_configuration['app_id'] = config.app_id
+
+        current_env = config.environment.value.lower()
+        combined_configuration['environments'][current_env] = {}
+
+        for k,v in config.model_dump().items():
+            if not v:
+                continue
+            match(k):
+                case "app_id" | "environment" | "spec_version":
+                    continue
+                case _:
+                    try:
+                        combined_configuration['environments'][current_env][k] = str(v)
+                    except:
+                        logger.error(f"Couldn't represent {k} as a string")
+    
+    return combined_configuration
+
+def _resolve_connection_ids(connection_ids: list[str]) -> list[str]:
+    client = get_connections_client()
+    connections = client.list()
+    return list(set([c.app_id for c in connections if c.connection_id in connection_ids]))
+
+
 
 def add_configuration(config: ConnectionConfiguration) -> None:
     client = get_connections_client()
@@ -396,6 +442,21 @@ def add_connection(app_id: str) -> None:
         logger.error(response_text)
         exit(1)
 
+def get_connection_configs(app_id: str) -> List[ConnectionConfiguration]:
+    client = get_connections_client()
+    connection_configs = []
+    for env in ConnectionEnvironment:
+        try:
+            config = client.get_config(app_id=app_id,env=env)
+            if not config:
+                logger.warning(f"No {env.value.lower()} configuration found for connection '{app_id}'")
+            else:
+                connection_configs.append( config.as_config() )
+        except:
+            logger.error(f"Unable to get {env.value.lower()} configs for connection '{app_id}'")
+
+    return connection_configs
+
 def remove_connection(app_id: str) -> None:
     client = get_connections_client()
 
@@ -479,6 +540,59 @@ def list_connections(environment: ConnectionEnvironment | None, verbose: bool =
 def import_connection(file: str) -> None:
     _parse_file(file=file)
 
+def export_connection(output_file: str, app_id: str | None = None, connection_id: str | None = None) -> None:
+    if not app_id and not connection_id:
+        raise ValueError(f"Connection export requires at least one of 'app_id' or 'connection_id'")
+    
+    if app_id and connection_id:
+        logger.warning(f"Connection export recieved both 'app_id' and 'connection_id', preferring 'app_id'")
+    
+    if not app_id:
+        app_ids = _resolve_connection_ids([connection_id])
+        if len(app_ids) > 0:
+            app_id = app_ids[0]
+        else:
+            raise ValueError(f"No connections found with connection_id of '{connection_id}'")
+
+
+    # verify output folder
+    output_path = Path(output_file)
+    if output_path.exists():
+        logger.error(f"Specified output file already exists")
+        sys.exit(0)
+
+    output_type = output_path.suffix.lower()
+    if output_type not in ['.zip','.yaml','.yml']:
+        logger.error(f"Output file must end with the extension '.zip', '.yaml' or '.yml'")
+        sys.exit(0)
+
+    # get connection data
+    connections = get_connection_configs(app_id=app_id)
+    combined_connections = _combine_connection_configs(connections)
+
+    # write to folder
+    match(output_type):
+        case '.zip':
+            zip_file = zipfile.ZipFile(output_path, "w")
+                
+            connection_yaml = yaml.dump(combined_connections, sort_keys=False, default_flow_style=False)
+            connection_yaml_bytes = connection_yaml.encode("utf-8")
+            connection_yaml_file = io.BytesIO(connection_yaml_bytes)
+
+            zip_file.writestr(
+                f"{output_path.stem}/{app_id}.yaml",
+                connection_yaml_file.getvalue()
+            )
+
+            zip_file.close()
+        case '.yaml' | '.yml':
+            with open(output_path,'w') as yaml_file:
+                yaml_file.write(
+                    yaml.dump(combined_connections, sort_keys=False, default_flow_style=False)
+                )
+                
+    logger.info(f"Successfully exported connection file for {app_id}")
+
 def configure_connection(**kwargs) -> None:
     if is_local_dev() and kwargs.get("environment") != ConnectionEnvironment.DRAFT:
         logger.error(f"Cannot create configuration for environment '{kwargs.get('environment')}'. Local development does not support any environments other than 'draft'.")

ibm_watsonx_orchestrate/cli/commands/copilot/copilot_server_controller.py

@@ -1,24 +1,13 @@
 import logging
 import sys
 from pathlib import Path
-import subprocess
 import time
 import requests
 from urllib.parse import urlparse
-from ibm_watsonx_orchestrate.cli.commands.server.server_command import (
-    get_compose_file,
-    ensure_docker_compose_installed,
-    _prepare_clean_env,
-    ensure_docker_installed,
-    read_env_file,
-    get_default_env_file,
-    get_persisted_user_env,
-    get_dev_edition_source,
-    get_default_registry_env_vars_by_dev_edition_source,
-    docker_login_by_dev_edition_source,
-    write_merged_env_file,
-    apply_server_env_dict_defaults
-)
+
+from ibm_watsonx_orchestrate.cli.config import Config
+from ibm_watsonx_orchestrate.utils.docker_utils import DockerLoginService, DockerComposeCore, DockerUtils
+from ibm_watsonx_orchestrate.utils.environment import EnvService
 
 logger = logging.getLogger(__name__)
 
@@ -39,71 +28,26 @@ def wait_for_wxo_cpe_health_check(timeout_seconds=45, interval_seconds=2):
         time.sleep(interval_seconds)
     return False
 
-def _trim_authorization_urls(env_dict: dict) -> dict:
-    auth_url_key = "AUTHORIZATION_URL"
-    env_dict_copy = env_dict.copy()
-
-    auth_url = env_dict_copy.get(auth_url_key)
-    if not auth_url:
-        return env_dict_copy
-    
-    
-    parsed_url = urlparse(auth_url)
-    new_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
-    env_dict_copy[auth_url_key] = new_url
-
-    return env_dict_copy
-
-    
-
-
 def run_compose_lite_cpe(user_env_file: Path) -> bool:
-    compose_path = get_compose_file()
-    compose_command = ensure_docker_compose_installed()
-    _prepare_clean_env(user_env_file)  
-    ensure_docker_installed()
-
-    default_env = read_env_file(get_default_env_file())
-    user_env = read_env_file(user_env_file) if user_env_file else {}
-    if not user_env:
-        user_env = get_persisted_user_env() or {}
-
-    dev_edition_source = get_dev_edition_source(user_env)
-    default_registry_vars = get_default_registry_env_vars_by_dev_edition_source(default_env, user_env, source=dev_edition_source)
-
-    # Update the default environment with the default registry variables only if they are not already set
-    for key in default_registry_vars:
-        if key not in default_env or not default_env[key]:
-            default_env[key] = default_registry_vars[key]
-
-    # Merge the default environment with the user environment
-    merged_env_dict = {
-        **default_env,
-        **user_env,
-    }
-    
-    merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
-    merged_env_dict = _trim_authorization_urls(merged_env_dict)
+    DockerUtils.ensure_docker_installed()
+
+    cli_config = Config()
+    env_service = EnvService(cli_config)
+    env_service.prepare_clean_env(user_env_file)
+    user_env = env_service.get_user_env(user_env_file)
+    merged_env_dict = env_service.prepare_server_env_vars(user_env=user_env, should_drop_auth_routes=True)
 
     try:
-        docker_login_by_dev_edition_source(merged_env_dict, dev_edition_source)
+        DockerLoginService(env_service=env_service).login_by_dev_edition_source(merged_env_dict)
     except ValueError as ignored:
         # do nothing, as the docker login here is not mandatory
         pass
 
-    final_env_file = write_merged_env_file(merged_env_dict)
+    final_env_file = env_service.write_merged_env_file(merged_env_dict)
 
-    command = compose_command + [
-        "-f", str(compose_path),
-        "--env-file", str(final_env_file),
-        "up",
-        "cpe",
-        "-d",
-        "--remove-orphans"
-    ]
+    compose_core = DockerComposeCore(env_service)
 
-    logger.info(f"Starting docker-compose Copilot service...")
-    result = subprocess.run(command, capture_output=False)
+    result = compose_core.service_up(service_name="cpe", friendly_name="Copilot", final_env_file=final_env_file)
 
     if result.returncode == 0:
         logger.info("Copilot Service started successfully.")
@@ -124,27 +68,16 @@ def run_compose_lite_cpe(user_env_file: Path) -> bool:
     return True
 
 def run_compose_lite_cpe_down(is_reset: bool = False) -> None:
-    compose_path = get_compose_file()
-    compose_command = ensure_docker_compose_installed()
-    ensure_docker_installed()
-
-    default_env = read_env_file(get_default_env_file())
-    final_env_file = write_merged_env_file(default_env)
-
-    command = compose_command + [
-        "-f", str(compose_path),
-        "--env-file", final_env_file,
-        "down",
-        "cpe"
-    ]
-
-    if is_reset:
-        command.append("--volumes")
-        logger.info("Stopping docker-compose Copilot service and resetting volumes...")
-    else:
-        logger.info("Stopping docker-compose Copilot service...")
+    DockerUtils.ensure_docker_installed()
+
+    default_env = EnvService.read_env_file(EnvService.get_default_env_file())
+    final_env_file = EnvService.write_merged_env_file(default_env)
+
+    cli_config = Config()
+    env_service = EnvService(cli_config)
+    compose_core = DockerComposeCore(env_service)
 
-    result = subprocess.run(command, capture_output=False)
+    result = compose_core.service_down(service_name="cpe", friendly_name="Copilot", final_env_file=final_env_file, is_reset=is_reset)
 
     if result.returncode == 0:
         logger.info("Copilot service stopped successfully.")

ibm_watsonx_orchestrate/cli/commands/environment/environment_command.py

@@ -70,7 +70,7 @@ def add_env(
         ] = None,
         type: Annotated[
             EnvironmentAuthType,
-            typer.Option("--type", "-t", help="The type of auth you wish to use"),
+            typer.Option("--type", "-t", help="The type of auth you wish to use. This overrides the auth type that is inferred from the url"),
         ] = None,
         insecure: Annotated[
             bool,

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -16,7 +16,7 @@ from typing import Optional
 from typing_extensions import Annotated
 
 from ibm_watsonx_orchestrate import __version__
-from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
+from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController, EvaluateMode
 from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 
 logger = logging.getLogger(__name__)
@@ -220,6 +220,13 @@ def analyze(data_path: Annotated[
             help="Path to the directory that has the saved results"
         )
     ],
+    tool_definition_path: Annotated[
+        Optional[str],
+        typer.Option(
+            "--tools-path", "-t",
+            help="Path to the directory containing tool definitions."
+        )
+    ] = None,
     user_env_file: Annotated[
         Optional[str],
         typer.Option(
@@ -230,7 +237,10 @@ def analyze(data_path: Annotated[
 
     validate_watsonx_credentials(user_env_file)
     controller = EvaluationsController()
-    controller.analyze(data_path=data_path)
+    controller.analyze(
+        data_path=data_path,
+        tool_definition_path=tool_definition_path
+        )
 
 @evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
 def validate_external(
@@ -375,3 +385,214 @@ def validate_external(
             msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
 
         rich.print(Panel(msg))
+
+@evaluation_app.command(name="validate-native", help="Validate native agents against a set of inputs")
+def validate_native(
+    data_path: Annotated[
+        str,
+        typer.Option(
+            "--tsv", "-t",
+            help="Path to .tsv file of inputs. The first column of the TSV is the user story, the second column is the  expected final output from the agent, and the third column is the name of the agent"
+        )
+    ],
+    output_dir: Annotated[
+        str,
+        typer.Option(
+            "--output", "-o",
+            help="where to save the validation results"
+        )
+    ] = "./test_native_agent",
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None,
+):
+    validate_watsonx_credentials(user_env_file)
+    
+    eval_dir = os.path.join(output_dir, "native_agent_evaluations")
+    if os.path.exists(eval_dir):
+        rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
+        shutil.rmtree(eval_dir)
+    Path(eval_dir).mkdir(exist_ok=True, parents=True)
+
+    test_data_path = Path(eval_dir) / "generated_test_data"
+    test_data_path.mkdir(exist_ok=True, parents=True)
+    
+    controller = EvaluationsController()
+    test_data = read_csv(data_path) # tab seperated file containing the user story, the final outcome, and the agent name
+    
+    for idx, row in enumerate(test_data):
+        agent_name = row[2] # agent name
+        dataset = [row[0:2]] # user story, expected final outcome
+        generated_test_data = controller.generate_performance_test(agent_name=agent_name, test_data=dataset)
+        for test_data in generated_test_data:
+            test_name = f"native_agent_evaluation_test_{idx}.json"
+            with open(test_data_path / test_name, encoding="utf-8", mode="w+") as f:
+                json.dump(test_data, f, indent=4)
+    
+    evaluate(output_dir=eval_dir, test_paths=str(test_data_path))
+
+@evaluation_app.command(name="quick-eval",
+                        short_help="Evaluate agent against a suite of static metrics and LLM-as-a-judge metrics",
+                        help="""
+                        Use the quick-eval command to evaluate your agent against a suite of static metrics and LLM-as-a-judge metrics.
+                        """)
+def quick_eval(
+    config_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--config", "-c",
+            help="Path to YAML configuration file containing evaluation settings."
+        )
+    ] = None,
+    test_paths: Annotated[
+        Optional[str],
+        typer.Option(
+            "--test-paths", "-p", 
+            help="Paths to the test files and/or directories to evaluate, separated by commas."
+        ),
+    ] = None,
+    tools_path: Annotated[
+        str,
+        typer.Option(
+            "--tools-path", "-t",
+            help="Path to the directory containing tool definitions."
+        )
+    ] = None,
+    output_dir: Annotated[
+        Optional[str], 
+        typer.Option(
+            "--output-dir", "-o",
+            help="Directory to save the evaluation results."
+        )
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file", "-e", 
+            help="Path to a .env file that overrides default.env. Then environment variables override both."
+        ),
+    ] = None
+):
+    if not config_file:
+        if not test_paths or not output_dir:
+            logger.error("Error: Both --test-paths and --output-dir must be provided when not using a config file")
+            exit(1)
+    
+    validate_watsonx_credentials(user_env_file)
+
+    if tools_path is None:
+        logger.error("When running `quick-eval`, please provide the path to your tools file.")
+        sys.exit(1)
+    
+    controller = EvaluationsController()
+    controller.evaluate(
+        config_file=config_file,
+        test_paths=test_paths,
+        output_dir=output_dir,
+        tools_path=tools_path, mode=EvaluateMode.referenceless
+    )
+
+
+red_teaming_app = typer.Typer(no_args_is_help=True)
+evaluation_app.add_typer(red_teaming_app, name="red-teaming")
+
+
+@red_teaming_app.command("list", help="List available red-teaming attack plans")
+def list_plans():
+    controller = EvaluationsController()
+    controller.list_red_teaming_attacks()
+
+
+@red_teaming_app.command("plan", help="Generate red-teaming attacks")
+def plan(
+    attacks_list: Annotated[
+        str,
+        typer.Option(
+            "--attacks-list",
+            "-a",
+            help="Comma-separated list of red-teaming attacks to generate.",
+        ),
+    ],
+    datasets_path: Annotated[
+        str,
+        typer.Option(
+            "--datasets-path",
+            "-d",
+            help="Path to datasets for red-teaming. This can also be a comma-separated list of files or directories.",
+        ),
+    ],
+    agents_path: Annotated[
+        str, typer.Option("--agents-path", "-g", help="Path to the directory containing all agent definitions.")
+    ],
+    target_agent_name: Annotated[
+        str,
+        typer.Option(
+            "--target-agent-name",
+            "-t",
+            help="Name of the target agent to attack (should be present in agents-path).",
+        ),
+    ],
+    output_dir: Annotated[
+        Optional[str],
+        typer.Option("--output-dir", "-o", help="Directory to save generated attacks.")
+    ]=None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file",
+            "-e",
+            help="Path to a .env file that overrides default.env. Then environment variables override both.",
+        ),
+    ] = None,
+    max_variants: Annotated[
+        Optional[int],
+        typer.Option(
+            "--max_variants",
+            "-n",
+            help="Number of variants to generate per attack type.",
+        ),
+    ] = None,
+
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.generate_red_teaming_attacks(
+        attacks_list=attacks_list,
+        datasets_path=datasets_path,
+        agents_path=agents_path,
+        target_agent_name=target_agent_name,
+        output_dir=output_dir,
+        max_variants=max_variants,
+    )
+
+
+@red_teaming_app.command("run", help="Run red-teaming attacks")
+def run(
+    attack_paths: Annotated[
+        str,
+        typer.Option(
+            "--attack-paths",
+            "-a",
+            help="Path or list of paths (comma-separated) to directories containing attack files.",
+        ),
+    ],
+    output_dir: Annotated[
+        Optional[str],
+        typer.Option("--output-dir", "-o", help="Directory to save attack results."),
+    ] = None,
+    user_env_file: Annotated[
+        Optional[str],
+        typer.Option(
+            "--env-file",
+            "-e",
+            help="Path to a .env file that overrides default.env. Then environment variables override both.",
+        ),
+    ] = None,
+):
+    validate_watsonx_credentials(user_env_file)
+    controller = EvaluationsController()
+    controller.run_red_teaming_attacks(attack_paths=attack_paths, output_dir=output_dir)