ibm-watsonx-orchestrate 1.5.1__py3-none-any.whl → 1.7.0a0__py3-none-any.whl

ibm_watsonx_orchestrate/cli/commands/server/server_command.py

@@ -67,23 +67,28 @@ def docker_login(api_key: str, registry_url: str, username:str = "iamapikey") ->
     logger.info("Successfully logged in to Docker.")
 
 def docker_login_by_dev_edition_source(env_dict: dict, source: str) -> None:
-    if not env_dict.get("REGISTRY_URL"):
-        raise ValueError("REGISTRY_URL is not set.")
-    registry_url = env_dict["REGISTRY_URL"].split("/")[0]
-    if source == "internal":
-        iam_api_key = env_dict.get("DOCKER_IAM_KEY")
-        if not iam_api_key:
-            raise ValueError("DOCKER_IAM_KEY is required in the environment file if WO_DEVELOPER_EDITION_SOURCE is set to 'internal'.")
-        docker_login(iam_api_key, registry_url, "iamapikey")
-    elif source == "myibm":
-        wo_entitlement_key = env_dict.get("WO_ENTITLEMENT_KEY")
-        if not wo_entitlement_key:
-            raise ValueError("WO_ENTITLEMENT_KEY is required in the environment file.")
-        docker_login(wo_entitlement_key, registry_url, "cp")
-    elif source == "orchestrate":
-        wo_auth_type = env_dict.get("WO_AUTH_TYPE")
-        api_key, username = get_docker_cred_by_wo_auth_type(env_dict, wo_auth_type)
-        docker_login(api_key, registry_url, username)
+    if env_dict.get('WO_DEVELOPER_EDITION_SKIP_LOGIN', None) == 'true':
+        logger.info('WO_DEVELOPER_EDITION_SKIP_LOGIN is set to true, skipping login.')
+        logger.warning('If the developer edition images are not already pulled this call will fail without first setting WO_DEVELOPER_EDITION_SKIP_LOGIN=false')
+    else:
+        if not env_dict.get("REGISTRY_URL"):
+            raise ValueError("REGISTRY_URL is not set.")
+        registry_url = env_dict["REGISTRY_URL"].split("/")[0]
+        if source == "internal":
+            iam_api_key = env_dict.get("DOCKER_IAM_KEY")
+            if not iam_api_key:
+                raise ValueError("DOCKER_IAM_KEY is required in the environment file if WO_DEVELOPER_EDITION_SOURCE is set to 'internal'.")
+            docker_login(iam_api_key, registry_url, "iamapikey")
+        elif source == "myibm":
+            wo_entitlement_key = env_dict.get("WO_ENTITLEMENT_KEY")
+            if not wo_entitlement_key:
+                raise ValueError("WO_ENTITLEMENT_KEY is required in the environment file.")
+            docker_login(wo_entitlement_key, registry_url, "cp")
+        elif source == "orchestrate":
+            wo_auth_type = env_dict.get("WO_AUTH_TYPE")
+            api_key, username = get_docker_cred_by_wo_auth_type(env_dict, wo_auth_type)
+            docker_login(api_key, registry_url, username)
+
 
 def get_compose_file() -> Path:
     with resources.as_file(
@@ -165,6 +170,8 @@ def get_docker_cred_by_wo_auth_type(env_dict: dict, auth_type: str | None) -> tu
                 auth_type = "ibm_iam"
             elif ".ibm.com" in instance_url:
                 auth_type = "mcsp"
+            elif "https://cpd" in instance_url:
+                auth_type = "cpd"
     
     if auth_type in {"mcsp", "ibm_iam"}:
         wo_api_key = env_dict.get("WO_API_KEY")
@@ -233,6 +240,27 @@ def apply_llm_api_key_defaults(env_dict: dict) -> None:
         env_dict.setdefault("ASSISTANT_EMBEDDINGS_SPACE_ID", space_value)
         env_dict.setdefault("ROUTING_LLM_SPACE_ID", space_value)
 
+def _is_docker_container_running(container_name):
+    ensure_docker_installed()
+    command = [ "docker",
+                "ps",
+                "-f",
+                f"name={container_name}"
+            ]
+    result = subprocess.run(command, env=os.environ, capture_output=True)
+    if container_name in str(result.stdout):
+        return True
+    return False
+
+def _check_exclusive_observibility(langfuse_enabled: bool, ibm_tele_enabled: bool):
+    if langfuse_enabled and ibm_tele_enabled:
+        return False
+    if langfuse_enabled and _is_docker_container_running("docker-frontend-server-1"):
+        return False
+    if ibm_tele_enabled and _is_docker_container_running("docker-langfuse-web-1"):
+        return False
+    return True
+
 def write_merged_env_file(merged_env: dict) -> Path:
     tmp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env")
     with tmp:
@@ -285,7 +313,7 @@ def get_persisted_user_env() -> dict | None:
     user_env = cfg.get(USER_ENV_CACHE_HEADER) if cfg.get(USER_ENV_CACHE_HEADER) else None
     return user_env
 
-def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False) -> None:
+def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False, experimental_with_ibm_telemetry=False, with_docproc=False) -> None:
     compose_path = get_compose_file()
     compose_command = ensure_docker_compose_installed()
     db_tag = read_env_file(final_env_file).get('DBTAG', None)
@@ -312,13 +340,17 @@ def run_compose_lite(final_env_file: Path, experimental_with_langfuse=False) ->
 
 
     # Step 2: Start all remaining services (except DB)
+    profiles = []
     if experimental_with_langfuse:
-        command = compose_command + [
-            '--profile',
-            'langfuse'
-        ]
-    else:
-        command = compose_command
+        profiles.append("langfuse")
+    if experimental_with_ibm_telemetry:
+        profiles.append("ibm-telemetry")
+    if with_docproc:
+        profiles.append("docproc")
+
+    command = compose_command[:]
+    for profile in profiles:
+        command += ["--profile", profile]
 
     command += [
         "-f", str(compose_path),
@@ -437,6 +469,9 @@ def run_compose_lite_ui(user_env_file: Path) -> bool:
         # do nothing, as the docker login here is not mandatory
         pass
 
+    # Auto-configure callback IP for async tools
+    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
+
     #These are to removed warning and not used in UI component
     if not 'WATSONX_SPACE_ID' in merged_env_dict:
         merged_env_dict['WATSONX_SPACE_ID']='X'
@@ -614,8 +649,80 @@ def confirm_accepts_license_agreement(accepts_by_argument: bool):
             logger.error('The terms and conditions were not accepted, exiting.')
             exit(1)
 
+def auto_configure_callback_ip(merged_env_dict: dict) -> dict:
+    """
+    Automatically detect and configure CALLBACK_HOST_URL if it's empty.
+
+    Args:
+        merged_env_dict: The merged environment dictionary
+
+    Returns:
+        Updated environment dictionary with CALLBACK_HOST_URL set
+    """
+    callback_url = merged_env_dict.get('CALLBACK_HOST_URL', '').strip()
+
+    # Only auto-configure if CALLBACK_HOST_URL is empty
+    if not callback_url:
+        logger.info("Auto-detecting local IP address for async tool callbacks...")
+
+        system = platform.system()
+        ip = None
+
+        try:
+            if system in ("Linux", "Darwin"):
+                result = subprocess.run(["ifconfig"], capture_output=True, text=True, check=True)
+                lines = result.stdout.splitlines()
+
+                for line in lines:
+                    line = line.strip()
+                    # Unix ifconfig output format: "inet 192.168.1.100 netmask 0xffffff00 broadcast 192.168.1.255"
+                    if line.startswith("inet ") and "127.0.0.1" not in line:
+                        candidate_ip = line.split()[1]
+                        # Validate IP is not loopback or link-local
+                        if (candidate_ip and
+                            not candidate_ip.startswith("127.") and
+                            not candidate_ip.startswith("169.254")):
+                            ip = candidate_ip
+                            break
+
+            elif system == "Windows":
+                result = subprocess.run(["ipconfig"], capture_output=True, text=True, check=True)
+                lines = result.stdout.splitlines()
+
+                for line in lines:
+                    line = line.strip()
+                    # Windows ipconfig output format: "   IPv4 Address. . . . . . . . . . . : 192.168.1.100"
+                    if "IPv4 Address" in line and ":" in line:
+                        candidate_ip = line.split(":")[-1].strip()
+                        # Validate IP is not loopback or link-local
+                        if (candidate_ip and
+                            not candidate_ip.startswith("127.") and
+                            not candidate_ip.startswith("169.254")):
+                            ip = candidate_ip
+                            break
 
+            else:
+                logger.warning(f"Unsupported platform: {system}")
+                ip = None
+
+        except Exception as e:
+            logger.debug(f"IP detection failed on {system}: {e}")
+            ip = None
+
+        if ip:
+            callback_url = f"http://{ip}:4321"
+            merged_env_dict['CALLBACK_HOST_URL'] = callback_url
+            logger.info(f"Auto-configured CALLBACK_HOST_URL to: {callback_url}")
+        else:
+            # Fallback for localhost
+            callback_url = "http://host.docker.internal:4321"
+            merged_env_dict['CALLBACK_HOST_URL'] = callback_url
+            logger.info(f"Using Docker internal URL: {callback_url}")
+            logger.info("For external tools, consider using ngrok or similar tunneling service.")
+    else:
+        logger.info(f"Using existing CALLBACK_HOST_URL: {callback_url}")
 
+    return merged_env_dict
 
 @server_app.command(name="start")
 def server_start(
@@ -629,6 +736,11 @@ def server_start(
         '--with-langfuse', '-l',
         help='Option to enable Langfuse support.'
     ),
+    experimental_with_ibm_telemetry: bool = typer.Option(
+        False,
+        '--with-ibm-telemetry', '-i',
+        help=''
+    ),
     persist_env_secrets: bool = typer.Option(
         False,
         '--persist-env-secrets', '-p',
@@ -640,6 +752,11 @@ def server_start(
         "--accept-terms-and-conditions",
         help="By providing this flag you accept the terms and conditions outlined in the logs on server start."
     ),
+    with_docproc: bool = typer.Option(
+        False,
+        '--with-docproc', '-d',
+        help='Enable IBM Document Processing to extract information from your business documents. Enabling this activates the Watson Document Understanding service.'
+    ),
 ):
     confirm_accepts_license_agreement(accept_terms_and_conditions)
 
@@ -668,10 +785,22 @@ def server_start(
 
     merged_env_dict = apply_server_env_dict_defaults(merged_env_dict)
 
-    # Add LANGFUSE_ENABLED into the merged_env_dict, for tempus to pick up.
+    # Auto-configure callback IP for async tools
+    merged_env_dict = auto_configure_callback_ip(merged_env_dict)
+    if not _check_exclusive_observibility(experimental_with_langfuse, experimental_with_ibm_telemetry):
+        logger.error("Please select either langfuse or ibm telemetry for observability not both")
+        sys.exit(1)
+
+    # Add LANGFUSE_ENABLED and DOCPROC_ENABLED into the merged_env_dict, for tempus to pick up.
     if experimental_with_langfuse:
         merged_env_dict['LANGFUSE_ENABLED'] = 'true'
 
+    if with_docproc:
+        merged_env_dict['DOCPROC_ENABLED'] = 'true'
+
+    if experimental_with_ibm_telemetry:
+        merged_env_dict['USE_IBM_TELEMETRY'] = 'true'
+
     try:
         docker_login_by_dev_edition_source(merged_env_dict, dev_edition_source)
     except ValueError as e:
@@ -682,7 +811,11 @@ def server_start(
 
 
     final_env_file = write_merged_env_file(merged_env_dict)
-    run_compose_lite(final_env_file=final_env_file, experimental_with_langfuse=experimental_with_langfuse)
+
+    run_compose_lite(final_env_file=final_env_file,
+                     experimental_with_langfuse=experimental_with_langfuse,
+                     experimental_with_ibm_telemetry=experimental_with_ibm_telemetry,
+                     with_docproc=with_docproc)
 
     run_db_migration()
 
@@ -710,6 +843,8 @@ def server_start(
 
     if experimental_with_langfuse:
         logger.info(f"You can access the observability platform Langfuse at http://localhost:3010, username: orchestrate@ibm.com, password: orchestrate")
+    if with_docproc:
+        logger.info(f"Document processing capabilities are now available for use in Flows (both ADK and runtime). Note: This option is currently available only in the Developer edition.")
 
 @server_app.command(name="stop")
 def server_stop(
@@ -817,4 +952,4 @@ def run_db_migration() -> None:
         sys.exit(1)
 
 if __name__ == "__main__":
-    server_app()
+    server_app()

ibm_watsonx_orchestrate/cli/commands/server/types.py

@@ -26,7 +26,8 @@ def _infer_auth_type_from_instance_url(instance_url: str) -> WoAuthType:
           return WoAuthType.IBM_IAM
      if ".ibm.com" in instance_url:
           return WoAuthType.MCSP
-     return WoAuthType.CPD
+     if "https://cpd" in instance_url:
+          return WoAuthType.CPD
 
 
 class WatsonXAIEnvConfig(BaseModel):

ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_controller.py

@@ -213,9 +213,6 @@ class ToolkitController:
 
     
     def remove_toolkit(self, name: str):
-        if not is_local_dev():
-            logger.error("This functionality is only available for Local Environments")
-            sys.exit(1)
         try:
             client = self.get_client()
             draft_toolkits = client.get_draft_by_name(toolkit_name=name)
@@ -246,9 +243,15 @@ class ToolkitController:
             rich.print(JSON(json.dumps(tools_list, indent=4)))
         else:
             table = rich.table.Table(show_header=True, header_style="bold white", show_lines=True)
-            columns = ["Name", "Kind", "Description", "Tools", "App ID"]
-            for column in columns:
-                table.add_column(column)
+            column_args = {
+                "Name": {"overflow": "fold"},
+                "Kind": {},
+                "Description": {},
+                "Tools": {},
+                "App ID": {"overflow": "fold"}
+            }
+            for column in column_args:
+                table.add_column(column,**column_args[column])
 
             tools_client = instantiate_client(ToolClient)
 

ibm_watsonx_orchestrate/cli/commands/tools/tools_controller.py

@@ -24,6 +24,7 @@ from rich.console import Console
 from rich.panel import Panel
 
 from ibm_watsonx_orchestrate.agent_builder.tools import BaseTool, ToolSpec
+from ibm_watsonx_orchestrate.agent_builder.tools.flow_tool import create_flow_json_tool
 from ibm_watsonx_orchestrate.agent_builder.tools.openapi_tool import create_openapi_json_tools_from_uri,create_openapi_json_tools_from_content
 from ibm_watsonx_orchestrate.cli.commands.models.models_controller import ModelHighlighter
 from ibm_watsonx_orchestrate.cli.commands.tools.types import RegistryType
@@ -33,15 +34,14 @@ from ibm_watsonx_orchestrate.cli.config import Config, CONTEXT_SECTION_HEADER, C
     PYTHON_REGISTRY_HEADER, PYTHON_REGISTRY_TYPE_OPT, PYTHON_REGISTRY_TEST_PACKAGE_VERSION_OVERRIDE_OPT, \
     DEFAULT_CONFIG_FILE_CONTENT
 from ibm_watsonx_orchestrate.agent_builder.connections import ConnectionSecurityScheme, ExpectedCredentials
-from ibm_watsonx_orchestrate.experimental.flow_builder.flows.decorators import FlowWrapper
+from ibm_watsonx_orchestrate.flow_builder.flows.decorators import FlowWrapper
 from ibm_watsonx_orchestrate.client.tools.tool_client import ToolClient
 from ibm_watsonx_orchestrate.client.toolkit.toolkit_client import ToolKitClient
 from ibm_watsonx_orchestrate.client.connections import get_connections_client, get_connection_type
 from ibm_watsonx_orchestrate.client.utils import instantiate_client, is_local_dev
+from ibm_watsonx_orchestrate.flow_builder.utils import import_flow_support_tools
 from ibm_watsonx_orchestrate.utils.utils import sanatize_app_id
 from ibm_watsonx_orchestrate.client.utils import is_local_dev
-from ibm_watsonx_orchestrate.client.tools.tempus_client import TempusClient
-from ibm_watsonx_orchestrate.experimental.flow_builder.utils import import_flow_model
 
 from  ibm_watsonx_orchestrate import __version__
 
@@ -339,24 +339,18 @@ async def import_flow_tool(file: str) -> None:
     theme = rich.theme.Theme({"model.name": "bold cyan"})
     console = rich.console.Console(highlighter=ModelHighlighter(), theme=theme)
     
-    message = f"""[bold cyan]Flow Tools: Experimental Feature[/bold cyan]
+    message = f"""[bold cyan]Flow Tools[/bold cyan]
    
 The [bold]flow tool[/bold] is being imported from [green]`{file}`[/green].  
     
 [bold cyan]Additional information:[/bold cyan]
 
-- Ensure the flow engine is running by issuing the [bold cyan]orchestrate server start[/bold cyan] command with the [bold cyan]--with-flow-runtime[/bold cyan] option
-- The [bold green]get_flow_status[/bold green] tool is being imported to support flow tools. Ensure [bold]both this tools and the one you are importing are added to your agent[/bold] to retrieve the flow output. 
-- Include additional instructions in your agent to call the [bold green]get_flow_status[/bold green] tool to retrieve the flow output. For example: [green]"If you get an instance_id, use the tool get_flow_status to retrieve the current status of a flow."[/green]
+- The [bold green]Get flow status[/bold green] tool is being imported to support flow tools. This tool can query the status of a flow tool instance.  You can add it to your agent using the UI or including the following tool name in your agent definition: [green]i__get_flow_status_intrinsic_tool__[/green]. 
 
     """
 
     console.print(Panel(message,  title="[bold blue]Flow tool support information[/bold blue]", border_style="bright_blue"))
    
-
-    if not is_local_dev():
-        raise typer.BadParameter(f"Flow tools are only supported in local environment.")
-
     model = None
     
     # Load the Flow JSON model from the file
@@ -423,7 +417,16 @@ The [bold]flow tool[/bold] is being imported from [green]`{file}`[/green].
     except Exception as e:
         raise typer.BadParameter(f"Failed to load model from file {file}: {e}")
     
-    return await import_flow_model(model)
+    tool = create_flow_json_tool(name=model["spec"]["name"],
+                                 description=model["spec"]["description"], 
+                                 permission="read_only", 
+                                 flow_model=model)   
+    
+    tools = import_flow_support_tools()
+    
+    tools.append(tool)
+
+    return tools
 
 
 async def import_openapi_tool(file: str, connection_id: str) -> List[BaseTool]:
@@ -438,6 +441,10 @@ def _get_kind_from_spec(spec: dict) -> ToolKind:
         return ToolKind.python
     elif ToolKind.openapi in tool_binding:
         return ToolKind.openapi
+    elif ToolKind.mcp in tool_binding:
+        return ToolKind.mcp
+    elif 'wxflows' in tool_binding:
+        return ToolKind.flow
     else:
         logger.error(f"Could not determine 'kind' of tool '{name}'")
         sys.exit(1) 
@@ -511,9 +518,16 @@ class ToolsController:
             rich.print(JSON(json.dumps(tools_list, indent=4)))
         else:
             table = rich.table.Table(show_header=True, header_style="bold white", show_lines=True)
-            columns = ["Name", "Description", "Permission", "Type", "Toolkit", "App ID"]
-            for column in columns:
-                table.add_column(column)
+            column_args = {
+                "Name": {"overflow": "fold"},
+                "Description": {},
+                "Permission": {}, 
+                "Type": {}, 
+                "Toolkit": {}, 
+                "App ID": {"overflow": "fold"}
+            }
+            for column in column_args:
+                table.add_column(column,**column_args[column])
 
             connections_client = get_connections_client()
             connections = connections_client.list()
@@ -555,19 +569,20 @@ class ToolsController:
                         tool_type=ToolKind.openapi
                 elif tool_binding.mcp is not None:
                         tool_type=ToolKind.mcp
+                elif tool_binding.flow is not None:
+                        tool_type=ToolKind.flow        
                 else:
                         tool_type="Unknown"
                 
                 toolkit_name = ""
 
-                if is_local_dev():
+                if tool.__tool_spec__.toolkit_id:
                     toolkit_client = instantiate_client(ToolKitClient)
-                    if tool.__tool_spec__.toolkit_id:
-                        toolkit = toolkit_client.get_draft_by_id(tool.__tool_spec__.toolkit_id)
-                        if isinstance(toolkit, dict) and "name" in toolkit:
-                            toolkit_name = toolkit["name"]
-                        elif toolkit:
-                            toolkit_name = str(toolkit)
+                    toolkit = toolkit_client.get_draft_by_id(tool.__tool_spec__.toolkit_id)
+                    if isinstance(toolkit, dict) and "name" in toolkit:
+                        toolkit_name = toolkit["name"]
+                    elif toolkit:
+                        toolkit_name = str(toolkit)
 
                 
                 table.add_row(

ibm_watsonx_orchestrate/cli/config.py

@@ -26,6 +26,8 @@ ENV_WXO_URL_OPT = "wxo_url"
 ENV_IAM_URL_OPT = "iam_url"
 PROTECTED_ENV_NAME = "local"
 ENV_AUTH_TYPE = "auth_type"
+BYPASS_SSL = "bypass_ssl"
+VERIFY = "verify"
 ENV_ACCEPT_LICENSE = 'accepts_license_agreements'
 
 DEFAULT_LOCAL_SERVICE_URL = "http://localhost:4321"

ibm_watsonx_orchestrate/cli/main.py

@@ -12,8 +12,13 @@ from ibm_watsonx_orchestrate.cli.commands.environment.environment_command import
 from ibm_watsonx_orchestrate.cli.commands.channels.channels_command import channel_app
 from ibm_watsonx_orchestrate.cli.commands.knowledge_bases.knowledge_bases_command import knowledge_bases_app
 from ibm_watsonx_orchestrate.cli.commands.toolkit.toolkit_command import toolkits_app
+from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_command import evaluation_app
 from ibm_watsonx_orchestrate.cli.init_helper import init_callback
 
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
 app = typer.Typer(
     no_args_is_help=True,
     pretty_exceptions_enable=False,
@@ -30,6 +35,7 @@ app.add_typer(server_app, name="server", help='Manipulate your local Orchestrate
 app.add_typer(chat_app, name="chat", help='Launch the chat ui for your local Developer Edition server [requires entitlement]')
 app.add_typer(models_app, name="models", help='List the available large language models (llms) that can be used in your agent definitions')
 app.add_typer(channel_app, name="channels", help="Configure channels where your agent can exist on (such as embedded webchat)")
+app.add_typer(evaluation_app, name="evaluations", help='Evaluate the performance of your agents in your active env')
 app.add_typer(settings_app, name="settings", help='Configure the settings for your active env')
 
 if __name__ == "__main__":

ibm_watsonx_orchestrate/client/agents/agent_client.py

@@ -1,7 +1,79 @@
 from ibm_watsonx_orchestrate.client.base_api_client import BaseAPIClient, ClientAPIException
-from typing_extensions import List
+from typing_extensions import List, Optional
 from ibm_watsonx_orchestrate.client.utils import is_local_dev
+from pydantic import BaseModel
 
+def transform_agents_from_flat_agent_spec(agents: dict | list[dict] ) -> dict | list[dict]:
+    if isinstance(agents,list):
+        new_agents = []
+        for agent in agents:
+            new_agents.append(_transform_agent_from_flat_agent_spec(agent))
+        agents = new_agents
+    else:
+        agents = _transform_agent_from_flat_agent_spec(agents)
+    
+    return agents
+
+
+def _transform_agent_from_flat_agent_spec(agent_spec: dict ) -> dict:
+    transformed = {"additional_properties": {}}
+    for key,value in agent_spec.items():
+        if key == "starter_prompts":
+            if value:
+                value.pop("is_default_prompts",None)
+                value["customize"] = value.pop("prompts", [])
+
+            transformed["additional_properties"] |= { key: value }
+            
+        elif key == "welcome_content":
+            if value:
+                value.pop("is_default_message", None)
+
+            transformed["additional_properties"] |= { key: value }
+
+        else:
+            transformed |= { key: value }
+
+    return transformed
+
+def transform_agents_to_flat_agent_spec(agents: dict | list[dict] ) -> dict | list[dict]:
+    if isinstance(agents,list):
+        new_agents = []
+        for agent in agents:
+            new_agents.append(_transform_agent_to_flat_agent_spec(agent))
+        agents = new_agents
+    else:
+        agents = _transform_agent_to_flat_agent_spec(agents)
+
+    return agents
+
+def _transform_agent_to_flat_agent_spec(agent_spec: dict ) -> dict:
+    additional_properties = agent_spec.get("additional_properties", None)
+    if not additional_properties:
+        return agent_spec
+    
+    transformed = agent_spec
+    for key,value in additional_properties.items():
+        if key == "starter_prompts":
+            if value:
+                value["is_default_prompts"] = False
+                value["prompts"] = value.pop("customize", [])
+
+            transformed[key] = value
+            
+        elif key == "welcome_content":
+            if value:
+             value["is_default_message"] = False
+            
+            transformed[key] = value
+            
+    transformed.pop("additional_properties",None)
+
+    return transformed
+
+class AgentUpsertResponse(BaseModel):
+    id: Optional[str] = None
+    warning: Optional[str] = None
 
 class AgentClient(BaseAPIClient):
     """
@@ -12,14 +84,16 @@ class AgentClient(BaseAPIClient):
         self.base_endpoint = "/orchestrate/agents" if is_local_dev(self.base_url) else "/agents"
 
 
-    def create(self, payload: dict) -> dict:
-        return self._post(self.base_endpoint, data=payload)
+    def create(self, payload: dict) -> AgentUpsertResponse:
+        response = self._post(self.base_endpoint, data=transform_agents_from_flat_agent_spec(payload))
+        return AgentUpsertResponse.model_validate(response)
 
     def get(self) -> dict:
-        return self._get(self.base_endpoint)
+        return transform_agents_to_flat_agent_spec(self._get(f"{self.base_endpoint}?include_hidden=true"))
 
-    def update(self, agent_id: str, data: dict) -> dict:
-        return self._patch(f"{self.base_endpoint}/{agent_id}", data=data)
+    def update(self, agent_id: str, data: dict) -> AgentUpsertResponse:
+        response = self._patch(f"{self.base_endpoint}/{agent_id}", data=transform_agents_from_flat_agent_spec(data))
+        return AgentUpsertResponse.model_validate(response)
 
     def delete(self, agent_id: str) -> dict:
         return self._delete(f"{self.base_endpoint}/{agent_id}")
@@ -29,14 +103,14 @@ class AgentClient(BaseAPIClient):
 
     def get_drafts_by_names(self, agent_names: List[str]) -> List[dict]:
         formatted_agent_names = [f"names={x}" for x  in agent_names]
-        return self._get(f"{self.base_endpoint}?{'&'.join(formatted_agent_names)}")
+        return transform_agents_to_flat_agent_spec(self._get(f"{self.base_endpoint}?{'&'.join(formatted_agent_names)}&include_hidden=true"))
     
     def get_draft_by_id(self, agent_id: str) -> List[dict]:
         if agent_id is None:
             return ""
         else:
             try:
-                agent = self._get(f"{self.base_endpoint}/{agent_id}")
+                agent = transform_agents_to_flat_agent_spec(self._get(f"{self.base_endpoint}/{agent_id}"))
                 return agent
             except ClientAPIException as e:
                 if e.response.status_code == 404 and "not found with the given name" in e.response.text:
@@ -45,5 +119,5 @@ class AgentClient(BaseAPIClient):
     
     def get_drafts_by_ids(self, agent_ids: List[str]) -> List[dict]:
         formatted_agent_ids = [f"ids={x}" for x  in agent_ids]
-        return self._get(f"{self.base_endpoint}?{'&'.join(formatted_agent_ids)}")
+        return transform_agents_to_flat_agent_spec(self._get(f"{self.base_endpoint}?{'&'.join(formatted_agent_ids)}&include_hidden=true"))
     

ibm_watsonx_orchestrate/client/agents/assistant_agent_client.py

@@ -10,7 +10,7 @@ class AssistantAgentClient(BaseAPIClient):
         return self._post("/assistants/watsonx", data=payload)
 
     def get(self) -> dict:
-        return self._get("/assistants/watsonx")
+        return self._get("/assistants/watsonx?include_hidden=true")
 
     def update(self, agent_id: str, data: dict) -> dict:
         return self._patch(f"/assistants/watsonx/{agent_id}", data=data)
@@ -23,7 +23,7 @@ class AssistantAgentClient(BaseAPIClient):
 
     def get_drafts_by_names(self, agent_names: List[str]) -> List[dict]:
         formatted_agent_names = [f"names={x}" for x  in agent_names]
-        return self._get(f"/assistants/watsonx?{'&'.join(formatted_agent_names)}")
+        return self._get(f"/assistants/watsonx?{'&'.join(formatted_agent_names)}&include_hidden=true")
     
     def get_draft_by_id(self, agent_id: str) -> dict | str:
         if agent_id is None:
@@ -39,4 +39,4 @@ class AssistantAgentClient(BaseAPIClient):
     
     def get_drafts_by_ids(self, agent_ids: List[str]) -> List[dict]:
         formatted_agent_ids = [f"ids={x}" for x  in agent_ids]
-        return self._get(f"/assistants/watsonx?{'&'.join(formatted_agent_ids)}")
+        return self._get(f"/assistants/watsonx?{'&'.join(formatted_agent_ids)}&include_hidden=true")

ibm_watsonx_orchestrate/client/agents/external_agent_client.py

@@ -10,7 +10,7 @@ class ExternalAgentClient(BaseAPIClient):
         return self._post("/agents/external-chat", data=payload)
 
     def get(self) -> dict:
-        return self._get("/agents/external-chat")
+        return self._get("/agents/external-chat?include_hidden=true")
 
     def update(self, agent_id: str, data: dict) -> dict:
         return self._patch(f"/agents/external-chat/{agent_id}", data=data)
@@ -39,4 +39,4 @@ class ExternalAgentClient(BaseAPIClient):
 
     def get_drafts_by_ids(self, agent_ids: List[str]) -> List[dict]:
         formatted_agent_ids = [f"ids={x}" for x  in agent_ids]
-        return self._get(f"/agents/external-chat?{'&'.join(formatted_agent_ids)}")
+        return self._get(f"/agents/external-chat?{'&'.join(formatted_agent_ids)}&include_hidden=true")