ibm-watsonx-orchestrate 1.7.0a0__py3-none-any.whl → 1.8.0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/environment/environment_controller.py

@@ -23,7 +23,7 @@ from ibm_watsonx_orchestrate.cli.config import (
 )
 from ibm_watsonx_orchestrate.client.client import Client
 from ibm_watsonx_orchestrate.client.client_errors import ClientError
-from ibm_watsonx_orchestrate.client.agents.agent_client import AgentClient, ClientAPIException
+from ibm_watsonx_orchestrate.client.knowledge_bases.knowledge_base_client import KnowledgeBaseClient, ClientAPIException
 from ibm_watsonx_orchestrate.client.credentials import Credentials
 from threading import Lock
 from ibm_watsonx_orchestrate.client.utils import is_local_dev, check_token_validity, is_cpd_env
@@ -55,13 +55,13 @@ def _validate_token_functionality(token: str, url: str) -> None:
     '''
     is_cpd = is_cpd_env(url)
     if is_cpd is True:
-        agent_client = AgentClient(base_url=url, api_key=token, is_local=is_local_dev(url), verify=False)
+        knowledge_base_client = KnowledgeBaseClient(base_url=url, api_key=token, is_local=is_local_dev(url), verify=False)
     else:
-        agent_client = AgentClient(base_url=url, api_key=token, is_local=is_local_dev(url))
-    agent_client.api_key = token
+        knowledge_base_client = KnowledgeBaseClient(base_url=url, api_key=token, is_local=is_local_dev(url))
+    knowledge_base_client.api_key = token
 
     try:
-        agent_client.get()
+        knowledge_base_client.get()
     except ClientAPIException as e:
         if e.response.status_code >= 400:
             reason = e.response.reason
@@ -167,7 +167,7 @@ def activate(name: str, apikey: str=None, username: str=None, password: str=None
 
 def add(name: str, url: str, should_activate: bool=False, iam_url: str=None, type: EnvironmentAuthType=None, insecure: bool=None, verify: str=None) -> None:
     if name == PROTECTED_ENV_NAME:
-        logger.error(f"The name '{PROTECTED_ENV_NAME}' is a reserved environment name. Please select a diffrent name or use `orchestrate env activate {PROTECTED_ENV_NAME}` to swap to '{PROTECTED_ENV_NAME}'")
+        logger.error(f"The name '{PROTECTED_ENV_NAME}' is a reserved environment name. Please select a different name or use `orchestrate env activate {PROTECTED_ENV_NAME}` to swap to '{PROTECTED_ENV_NAME}'")
         return
 
     cfg = Config()

ibm_watsonx_orchestrate/cli/commands/environment/types.py

@@ -4,7 +4,9 @@ from enum import Enum
 class EnvironmentAuthType(str, Enum):
     IBM_CLOUD_IAM = 'ibm_iam'
     MCSP = 'mcsp'
+    MCSP_V1 = 'mcsp_v1'
+    MCSP_V2 = 'mcsp_v2'
     CPD = 'cpd'
 
     def __str__(self):
-        return self.value
+        return self.value

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py

@@ -5,10 +5,12 @@ import os
 import yaml
 import csv
 import rich
-from pathlib import Path
 import sys
-from dotenv import dotenv_values
+import shutil
 
+from rich.panel import Panel
+from pathlib import Path
+from dotenv import dotenv_values
 from typing import Optional
 from typing_extensions import Annotated
 
@@ -23,14 +25,20 @@ def read_env_file(env_path: Path|str) -> dict:
     return dotenv_values(str(env_path))
 
 def validate_watsonx_credentials(user_env_file: str) -> bool:
-    required_keys = ["WATSONX_SPACE_ID", "WATSONX_APIKEY"]
+    required_sets = [
+        ["WATSONX_SPACE_ID", "WATSONX_APIKEY"],
+        ["WO_INSTANCE", "WO_API_KEY"]
+    ]
     
-    if all(key in os.environ for key in required_keys):
+    def has_valid_keys(env: dict) -> bool:
+        return any(all(key in env for key in key_set) for key_set in required_sets)
+
+    if has_valid_keys(os.environ):
         logger.info("WatsonX credentials validated successfully.")
         return
     
     if user_env_file is None:
-        logger.error("WatsonX credentials are not set. Please set WATSONX_SPACE_ID and WATSONX_APIKEY in your system environment variables or include them in your enviroment file and pass it with --env-file option.")
+        logger.error("WatsonX credentials are not set. Please set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables or include them in your environment file and pass it with --env-file option.")
         sys.exit(1)
 
     if not Path(user_env_file).exists():
@@ -39,13 +47,43 @@ def validate_watsonx_credentials(user_env_file: str) -> bool:
     
     user_env = read_env_file(user_env_file)
     
-    if not all(key in user_env for key in required_keys):
-        logger.error("Error: The environment file does not contain the required keys: WATSONX_SPACE_ID and WATSONX_APIKEY.")
+    if not has_valid_keys(user_env):
+        logger.error("Error: The environment file does not contain the required keys: either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY.")
         sys.exit(1)
 
-    os.environ.update({key: user_env[key] for key in required_keys})
+    # Update os.environ with whichever set is present
+    for key_set in required_sets:
+        if all(key in user_env for key in key_set):
+            os.environ.update({key: user_env[key] for key in key_set})
+            break
     logger.info("WatsonX credentials validated successfully.")
 
+def read_csv(data_path: str, delimiter="\t"):
+    data = []
+    with open(data_path, "r") as f:
+        tsv_reader = csv.reader(f, delimiter=delimiter)
+        for line in tsv_reader:
+            data.append(line)
+    
+    return data
+
+def performance_test(agent_name, data_path, output_dir = None, user_env_file = None):
+    test_data = read_csv(data_path)
+
+    controller = EvaluationsController()
+    generated_performance_tests = controller.generate_performance_test(agent_name, test_data)
+    
+    generated_perf_test_dir = Path(output_dir) / "generated_performance_tests"
+    generated_perf_test_dir.mkdir(exist_ok=True, parents=True)
+
+    for idx, test in enumerate(generated_performance_tests):
+        test_name = f"validate_external_agent_evaluation_test_{idx}.json"
+        with open(generated_perf_test_dir / test_name, encoding="utf-8", mode="w+") as f:
+            json.dump(test, f, indent=4)
+
+    rich.print(f"Performance test cases saved at path '{str(generated_perf_test_dir)}'")
+    rich.print("[gold3]Running Performance Test")
+    evaluate(output_dir=output_dir, test_paths=str(generated_perf_test_dir))
 
 @evaluation_app.command(name="evaluate", help="Evaluate an agent against a set of test cases")
 def evaluate(
@@ -115,7 +153,7 @@ def generate(
     stories_path: Annotated[
         str,
         typer.Option(
-            "--stories_path", "-s",
+            "--stories-path", "-s",
             help="Path to the CSV file containing user stories for test case generation. "
                  "The file has 'story' and 'agent' columns."
         )
@@ -123,14 +161,14 @@ def generate(
     tools_path: Annotated[
         str,
         typer.Option(
-            "--tools_path", "-t",
+            "--tools-path", "-t",
             help="Path to the directory containing tool definitions."
         )
     ],
     output_dir: Annotated[
         Optional[str],
         typer.Option(
-            "--output_dir", "-o",
+            "--output-dir", "-o",
             help="Directory to save the generated test cases."
         )
     ] = None,
@@ -151,7 +189,7 @@ def generate(
 def analyze(data_path: Annotated[
         str,
         typer.Option(
-            "--data_path", "-d",
+            "--data-path", "-d",
             help="Path to the directory that has the saved results"
         )
     ],
@@ -167,30 +205,31 @@ def analyze(data_path: Annotated[
     controller = EvaluationsController()
     controller.analyze(data_path=data_path)
 
-
-@evaluation_app.command(name="validate_external", help="Validate an external agent against a set of inputs")
+@evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
 def validate_external(
     data_path: Annotated[
         str,
         typer.Option(
-            "--csv", "-c",
-            help="Path to .csv file of inputs"
+            "--tsv", "-t",
+            help="Path to .tsv file of inputs"
         )
     ],
-    config: Annotated[
+    external_agent_config: Annotated[
             str,
             typer.Option(
-                "--config", "-cf",
-                help="Path to the external agent yaml"
+                "--external-agent-config", "-ext",
+                help="Path to the external agent yaml",
+
             )
         ],
     credential: Annotated[
         str,
         typer.Option(
             "--credential", "-crd",
-            help="credential string"
+            help="credential string",
+            rich_help_panel="Parameters for Validation"
         )
-    ],
+    ] = None,
     output_dir: Annotated[
         str,
         typer.Option(
@@ -204,21 +243,80 @@ def validate_external(
             "--env-file", "-e", 
             help="Path to a .env file that overrides default.env. Then environment variables override both."
         ),
+    ] = None,
+    agent_name: Annotated[
+        str,
+        typer.Option(
+            "--agent_name", "-a",
+            help="Name of the native agent which has the external agent to test registered as a collaborater. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents)." \
+            " If this parameter is pased, validation of the external agent is not run.",
+            rich_help_panel="Parameters for Input Evaluation"
+        )
     ] = None
 ):
-    
+
     validate_watsonx_credentials(user_env_file)
-    with open(config, "r") as f:
-        config = yaml.safe_load(f)
-    controller = EvaluationsController()
-    test_data = []
-    with open(data_path, "r") as f:
-        csv_reader = csv.reader(f)
-        for line in csv_reader:
-            test_data.append(line[0])
-    results = controller.external_validate(config, test_data, credential)
-    os.makedirs(output_dir, exist_ok=True)
-    with open(os.path.join(output_dir, "validation_results.json"), "w") as f:
-        json.dump(results, f)
-
-    rich.print(f"[green] validation result is saved to {output_dir} [/green]")
+    Path(output_dir).mkdir(exist_ok=True)
+    shutil.copy(data_path, os.path.join(output_dir, "input_sample.tsv"))
+
+    if agent_name is not None:
+        eval_dir = os.path.join(output_dir, "evaluation")
+        if os.path.exists(eval_dir):
+            rich.print(f"[yellow]: found existing {eval_dir} in target directory. All content is removed.")
+            shutil.rmtree(os.path.join(output_dir, "evaluation"))
+        Path(eval_dir).mkdir(exist_ok=True)
+        # save external agent config even though its not used for evaluation
+        # it can help in later debugging customer agents
+        with open(os.path.join(eval_dir, "external_agent_cfg.yaml"), "w+") as f:
+            with open(external_agent_config, "r") as cfg:
+                external_agent_config = yaml.safe_load(cfg)
+            yaml.safe_dump(external_agent_config, f, indent=4)
+
+        rich.print(f"[gold3]Starting evaluation of inputs in '{data_path}' against '{agent_name}'[/gold3]")
+        performance_test(
+            agent_name=agent_name,
+            data_path=data_path,
+            output_dir=eval_dir,
+            user_env_file=user_env_file
+        )
+    
+    else:
+        with open(external_agent_config, "r") as f:
+            external_agent_config = yaml.safe_load(f)
+        controller = EvaluationsController()
+        test_data = []
+        with open(data_path, "r") as f:
+            csv_reader = csv.reader(f, delimiter="\t")
+            for line in csv_reader:
+                test_data.append(line[0])
+
+        # save validation results in "validation_results" sub-dir
+        validation_folder = Path(output_dir) / "validation_results"
+        if os.path.exists(validation_folder):
+            rich.print(f"[yellow]: found existing {validation_folder} in target directory. All content is removed.")
+            shutil.rmtree(validation_folder)
+        validation_folder.mkdir(exist_ok=True, parents=True)
+
+        # validate the inputs in the provided csv file
+        summary = controller.external_validate(external_agent_config, test_data, credential)
+        with open(validation_folder / "validation_results.json", "w") as f:
+            json.dump(summary, f, indent=4)
+        
+        # validate sample block inputs
+        rich.print("[gold3]Validating external agent to see if it can handle an array of messages.")
+        block_input_summary = controller.external_validate(external_agent_config, test_data, credential, add_context=True)
+        with open(validation_folder / "sample_block_validation_results.json", "w") as f:
+            json.dump(block_input_summary, f, indent=4)
+
+        user_validation_successful = all([item["success"] for item in summary])
+        block_validation_successful = all([item["success"] for item in block_input_summary])
+
+        if user_validation_successful and block_validation_successful:
+            msg = (
+                f"[green]Validation is successful. The result is saved to '{str(validation_folder)}'.[/green]\n"
+                "You can add the external agent as a collaborator agent. See: https://developer.watson-orchestrate.ibm.com/agents/build_agent#native-agents."
+            )
+        else:
+            msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
+
+        rich.print(Panel(msg))

ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py

@@ -1,21 +1,23 @@
 import logging
-from typing import List, Dict, Optional
+import os.path
+from typing import List, Dict, Optional, Tuple
 import csv
 from pathlib import Path
-import rich
+import sys
 from wxo_agentic_evaluation import main as evaluate
 from wxo_agentic_evaluation.tool_planner import build_snapshot
 from wxo_agentic_evaluation.analyze_run import analyze
 from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
-from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig
+from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig
 from wxo_agentic_evaluation.record_chat import record_chats
 from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
+from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
 from ibm_watsonx_orchestrate import __version__
 from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
 from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
 from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
 from ibm_watsonx_orchestrate.agent_builder.agents import AgentKind
-
+import uuid
 
 logger = logging.getLogger(__name__)
 
@@ -39,12 +41,26 @@ class EvaluationsController:
     def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
         url, tenant_name, token = self._get_env_config()
 
+        if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
+            provider = "watsonx"
+        elif "WO_INSTANCE" in os.environ and "WO_API_KEY" in os.environ:
+            provider = "model_proxy"
+        else:
+            logger.error(
+                "No provider found. Please either provide a config_file or set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables."
+            )
+            sys.exit(1)
+        
         config_data = {
             "wxo_lite_version": __version__,
             "auth_config": AuthConfig(
                 url=url,
                 tenant_name=tenant_name,
                 token=token
+            ),
+            "provider_config": ProviderConfig(
+                provider=provider,
+                model_id="meta-llama/llama-3-405b-instruct",
             )
         }
 
@@ -60,6 +76,10 @@ class EvaluationsController:
                 if "llm_user_config" in file_config:
                     llm_config_data = file_config.pop("llm_user_config")
                     config_data["llm_user_config"] = LLMUserConfig(**llm_config_data)
+
+                if "provider_config" in file_config:
+                    provider_config_data = file_config.pop("provider_config")
+                    config_data["provider_config"] = ProviderConfig(**provider_config_data)
                 
                 config_data.update(file_config)
 
@@ -75,9 +95,13 @@ class EvaluationsController:
         evaluate.main(config)
 
     def record(self, output_dir) -> None:
+
+
+        random_uuid = str(uuid.uuid4())
+
         url, tenant_name, token = self._get_env_config()
         config_data = {
-            "output_dir": Path.cwd() if output_dir is None else Path(output_dir),
+            "output_dir": Path(os.path.join(Path.cwd(), random_uuid)) if output_dir is None else Path(os.path.join(output_dir,random_uuid)),
             "service_url": url,
             "tenant_name": tenant_name,
             "token": token
@@ -143,16 +167,23 @@ class EvaluationsController:
     def summarize(self) -> None:
         pass
 
-    def external_validate(self, config: Dict, data: List[str], credential:str):
+    def external_validate(self, config: Dict, data: List[str], credential:str, add_context: bool = False):
         validator = ExternalAgentValidation(credential=credential,
                                 auth_scheme=config["auth_scheme"],
                                 service_url=config["api_url"])
+        
         summary = []
         for entry in data:
-            results = validator.call_validation(entry)
-            if len(results) == 0:
-                rich.print(f"[red] No events are generated for input {entry} [/red]")
-            summary.append({entry: results})
+            results = validator.call_validation(entry, add_context)
+            summary.append(results)
 
         return summary
-
+    
+    def generate_performance_test(self, agent_name: str, test_data: List[Tuple[str, str]]):
+        performance_test = ExternalAgentPerformanceTest(
+            agent_name=agent_name,
+            test_data=test_data
+        )
+        generated_performance_tests = performance_test.generate_tests()
+
+        return generated_performance_tests

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_command.py

@@ -21,24 +21,6 @@ def knowledge_base_import(
     controller = KnowledgeBaseController()
     controller.import_knowledge_base(file=file, app_id=app_id)
 
-@knowledge_bases_app.command(name="patch", help="Patch a knowledge base by uploading documents, or providing an external vector index")
-def knowledge_base_patch(
-    file: Annotated[
-        str,
-        typer.Option("--file", "-f", help="YAML or JSON file with knowledge base definition"),
-    ],
-    name: Annotated[
-        str,
-        typer.Option("--name", "-n", help="Name of the knowledge base you wish to update"),
-    ]=None,
-    id: Annotated[
-        str,
-        typer.Option("--id", "-i", help="ID of the knowledge base you wish to update"),
-    ]=None
-):
-    controller = KnowledgeBaseController()
-    controller.update_knowledge_base(id=id, name=name, file=file)
-
 
 @knowledge_bases_app.command(name="list", help="List all knowledge bases")
 def list_knowledge_bases(

ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py

@@ -8,7 +8,6 @@ import inspect
 from pathlib import Path
 from typing import List
 
-from ibm_watsonx_orchestrate.agent_builder.knowledge_bases.knowledge_base_requests import KnowledgeBaseUpdateRequest
 from ibm_watsonx_orchestrate.agent_builder.knowledge_bases.knowledge_base import KnowledgeBase
 from ibm_watsonx_orchestrate.client.knowledge_bases.knowledge_base_client import KnowledgeBaseClient
 from ibm_watsonx_orchestrate.client.base_api_client import ClientAPIException
@@ -72,11 +71,21 @@ class KnowledgeBaseController:
         client = self.get_client()
 
         knowledge_bases = parse_file(file=file)
+        
+        existing_knowledge_bases = client.get_by_names([kb.name for kb in knowledge_bases])
+        
         for kb in knowledge_bases:
             try:
+                file_dir = "/".join(file.split("/")[:-1])
+
+                existing = list(filter(lambda ex: ex.get('name') == kb.name, existing_knowledge_bases))
+                if len(existing) > 0:
+                    logger.info(f"Existing knowledge base '{kb.name}' found. Updating...")
+                    self.update_knowledge_base(existing[0].get("id"), kb=kb, file_dir=file_dir)
+                    continue
+
                 kb.validate_documents_or_index_exists()
                 if kb.documents:
-                    file_dir = "/".join(file.split("/")[:-1])
                     files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in kb.documents]
                     
                     kb.prioritize_built_in_index = True
@@ -106,10 +115,7 @@ class KnowledgeBaseController:
                 
                 logger.info(f"Successfully imported knowledge base '{kb.name}'")
             except ClientAPIException as e:
-                if "duplicate key value violates unique constraint" in e.response.text:
-                    logger.error(f"A knowledge base with the name '{kb.name}' already exists. Failed to import knowledge base")
-                else:
-                    logger.error(f"Error importing knowledge base '{kb.name}\n' {e.response.text}")
+                logger.error(f"Error importing knowledge base '{kb.name}\n' {e.response.text}")
     
     def get_id(
         self, id: str, name: str
@@ -131,27 +137,37 @@ class KnowledgeBaseController:
 
 
     def update_knowledge_base(
-        self, id: str, name: str, file: str
-    ) -> None:
-        knowledge_base_id = self.get_id(id, name)
-        update_request = KnowledgeBaseUpdateRequest.from_spec(file=file)
+        self, knowledge_base_id: str, kb: KnowledgeBase, file_dir: str
+    ) -> None:        
+        if kb.documents:
+            status = self.get_client().status(knowledge_base_id)
+            existing_docs = [doc.get("metadata", {}).get("original_file_name", "") for doc in status.get("documents", [])]
+            
+            removed_docs = existing_docs[:]
+            for filepath in kb.documents:
+                filename = get_file_name(filepath)
+
+                if filename in existing_docs:
+                    logger.warning(f'Document \"{filename}\" already exists in knowledge base. Updating...')
+                    removed_docs.remove(filename)
+
+            for filename in removed_docs:
+                logger.warning(f'Document \"{filename}\" removed from knowledge base.')
 
-        if update_request.documents:
-            file_dir = "/".join(file.split("/")[:-1])
-            files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in update_request.documents]
+
+            files = [('files', (get_file_name(file_path), open(get_relative_file_path(file_path, file_dir), 'rb'))) for file_path in kb.documents]
             
-            update_request.prioritize_built_in_index = True
-            payload = update_request.model_dump(exclude_none=True);
+            kb.prioritize_built_in_index = True
+            payload = kb.model_dump(exclude_none=True);
             payload.pop('documents');
 
             self.get_client().update_with_documents(knowledge_base_id, payload=payload, files=files)
         else:
-            if update_request.conversational_search_tool and update_request.conversational_search_tool.index_config:
-                update_request.prioritize_built_in_index = False
-            self.get_client().update(knowledge_base_id, update_request.model_dump(exclude_none=True))
+            if kb.conversational_search_tool and kb.conversational_search_tool.index_config:
+                kb.prioritize_built_in_index = False
+            self.get_client().update(knowledge_base_id, kb.model_dump(exclude_none=True))
 
-        logEnding = f"with ID '{id}'" if id else f"'{name}'"
-        logger.info(f"Successfully updated knowledge base {logEnding}")
+        logger.info(f"Knowledge base '{kb.name}' updated successfully")
 
 
     def knowledge_base_status( self, id: str, name: str) -> None:

ibm_watsonx_orchestrate/cli/commands/models/models_command.py

@@ -149,7 +149,7 @@ def models_policy_add(
         retry_attempts: Annotated[
             int,
             typer.Option('--retry-attempts', help='The number of attempts to retry'),
-        ],
+        ] = None,
         strategy_on_code: Annotated[
             List[int],
             typer.Option('--strategy-on-code', help='The http status to consider invoking the strategy'),

ibm_watsonx_orchestrate/cli/commands/models/models_controller.py

@@ -167,15 +167,12 @@ class ModelsController:
             logger.error("Error: WATSONX_URL is required in the environment.")
             sys.exit(1)
 
-        if is_cpd_env(models_client.base_url):
-            virtual_models = []
-            virtual_model_policies = []
-        else:
-            logger.info("Retrieving virtual-model models list...")
-            virtual_models = models_client.list()
+    
+        logger.info("Retrieving virtual-model models list...")
+        virtual_models = models_client.list()
 
-            logger.info("Retrieving virtual-policies models list...")
-            virtual_model_policies = model_policies_client.list()
+        logger.info("Retrieving virtual-policies models list...")
+        virtual_model_policies = model_policies_client.list()
 
         logger.info("Retrieving watsonx.ai models list...")
         found_models = _get_wxai_foundational_models()