ibm-watsonx-orchestrate 1.5.1__py3-none-any.whl → 1.6.0b0__py3-none-any.whl

ibm_watsonx_orchestrate/{experimental/flow_builder → flow_builder}/flows/flow.py

@@ -20,6 +20,7 @@ from typing_extensions import Self
 from pydantic import BaseModel, Field, SerializeAsAny
 import yaml
 from ibm_watsonx_orchestrate.agent_builder.tools.python_tool import PythonTool
+from ibm_watsonx_orchestrate.client.tools.tool_client import ToolClient
 from ibm_watsonx_orchestrate.client.tools.tempus_client import TempusClient
 from ibm_watsonx_orchestrate.client.utils import instantiate_client
 from ..types import (
@@ -36,7 +37,7 @@ from ..types import (
 )
 
 from ..data_map import DataMap
-from ..utils import _get_json_schema_obj, get_valid_name, import_flow_model
+from ..utils import _get_json_schema_obj, get_valid_name, import_flow_model, _get_tool_request_body, _get_tool_response_body
 
 from .events import StreamConsumer
 
@@ -70,6 +71,7 @@ class Flow(Node):
     metadata: dict[str, str] = {}
     parent: Any = None
     _sequence_id: int = 0 # internal-id
+    _tool_client: ToolClient = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -77,6 +79,9 @@ class Flow(Node):
         # extract data schemas
         self._refactor_node_to_schemaref(self)
 
+        # get Tool Client
+        self._tool_client = instantiate_client(ToolClient)
+
     def _find_topmost_flow(self) -> Self:
         if self.parent:
             return self.parent._find_topmost_flow()
@@ -104,7 +109,27 @@ class Flow(Node):
         # if there is already a schema with the same name, return it
         if title:
             if title in top_flow.schemas:
-                return top_flow.schemas[title]
+                existing_schema = top_flow.schemas[title]
+                # we need a deep compare if the incoming schema and existing_schema is the same
+                # pydantic suppport nested comparison by default
+
+                schema.title = title
+                
+                if schema == existing_schema:
+                    return existing_schema
+                # we need to do a deep compare
+                incoming_model = schema.model_dump(exclude_none=True, exclude_unset=True)
+                existing_model = existing_schema.model_dump(exclude_none=True, exclude_unset=True)
+
+                # log the model
+                # logger.info(f"incoming_model: {incoming_model}")
+                # logger.info(f"existing_model: {existing_model}")
+
+                if incoming_model == existing_model:
+                    return existing_schema
+                
+                # else we need a new name, and create a new schema
+                title = title + "_" + str(self._next_sequence_id())
 
         # otherwise, create a deep copy of the schema, add it to the dictionary and return it
         if schema:
@@ -122,7 +147,7 @@ class Flow(Node):
                 elif schema.aliasName:
                     title = get_valid_name(schema.aliasName)
                 else:
-                    title = "bo_" + uuid.uuid4().hex
+                    title = "bo_" + str(self._next_sequence_id())
             
             if new_schema.type == "object":
             # iterate the properties and add schema recursively
@@ -193,7 +218,7 @@ class Flow(Node):
                             required= spec.output_schema.required)
                     spec.output_schema = self._add_schema_ref(json_obj, f"{spec.name}_output")
                 elif spec.output_schema.type == "array":
-                    if spec.output_schema.items.type == "object":
+                    if hasattr(spec.output_schema, "items") and hasattr(spec.output_schema.items, "type") and spec.output_schema.items.type == "object":
                         schema_ref = self._add_schema_ref(spec.output_schema.items)
                         spec.output_schema.items = JsonSchemaObjectRef(ref=f"{schema_ref.ref}")
 
@@ -265,23 +290,30 @@ class Flow(Node):
         
         if isinstance(tool, str):        
             name = name if name is not None and name != "" else tool
-            input_schema_obj = _get_json_schema_obj(parameter_name = "input", type_def = input_schema)
-            output_schema_obj = _get_json_schema_obj("output", output_schema)
+
+            if input_schema is None and output_schema is None:
+                # try to retrieve the schema from server
+                tool_specs: List[dict] = self._tool_client.get_draft_by_name(name)
+                if (tool_specs is None) or (len(tool_specs) == 0):
+                    raise ValueError(f"tool '{name}' not found")
+
+                # use the first spec
+                tool_spec: ToolSpec = ToolSpec.model_validate(tool_specs[0])
+                # just pick the first one that is found
+                if hasattr(tool_spec, "input_schema"):
+                    input_schema_obj = _get_json_schema_obj("input", tool_spec.input_schema, True)
+                if hasattr(tool_spec, "output_schema"):
+                    output_schema_obj = _get_json_schema_obj("output", tool_spec.output_schema)
+            else: 
+                input_schema_obj = _get_json_schema_obj("input", input_schema)
+                output_schema_obj = _get_json_schema_obj("output", output_schema)
 
             toolnode_spec = ToolNodeSpec(type = "tool",
                                      name = name,
                                      display_name = display_name,
                                      description = description,
-                                     input_schema=ToolRequestBody(
-                                        type=input_schema_obj.type,
-                                        properties=input_schema_obj.properties,
-                                        required=input_schema_obj.required,
-                                    ) if input_schema is not None else None,
-                                    output_schema=ToolResponseBody(
-                                        type=output_schema_obj.type,
-                                        properties=output_schema_obj.properties,
-                                        required=output_schema_obj.required
-                                    ) if output_schema is not None else None,
+                                     input_schema= _get_tool_request_body(input_schema_obj),
+                                     output_schema= _get_tool_response_body(output_schema_obj),
                                      output_schema_object = output_schema_obj,
                                      tool = tool)
 
@@ -341,16 +373,8 @@ class Flow(Node):
             agent=agent,
             message=message,
             guidelines=guidelines,
-            input_schema=ToolRequestBody(
-                type=input_schema_obj.type,
-                properties=input_schema_obj.properties,
-                required=input_schema_obj.required,
-            ),
-            output_schema=ToolResponseBody(
-                type=output_schema_obj.type,
-                properties=output_schema_obj.properties,
-                required=output_schema_obj.required
-            ),
+            input_schema=_get_tool_request_body(input_schema_obj),
+            output_schema=_get_tool_response_body(output_schema_obj),
             output_schema_object = output_schema_obj
         )
 
@@ -391,16 +415,8 @@ class Flow(Node):
             user_prompt=user_prompt,
             llm=llm,
             llm_parameters=llm_parameters,
-            input_schema=ToolRequestBody(
-                type=input_schema_obj.type,
-                properties=input_schema_obj.properties,
-                required=input_schema_obj.required,
-            ),
-            output_schema=ToolResponseBody(
-                type=output_schema_obj.type,
-                properties=output_schema_obj.properties,
-                required=output_schema_obj.required
-            ),
+            input_schema=_get_tool_request_body(input_schema_obj),
+            output_schema=_get_tool_response_body(output_schema_obj),
             output_schema_object = output_schema_obj
         )
 
@@ -543,21 +559,13 @@ class Flow(Node):
                 },
                 required = ["items"])
 
+        new_foreach_item_schema = self._add_schema(foreach_item_schema)
         spec = ForeachSpec(name = "foreach_" + str(self._next_sequence_id()),
-                           input_schema=ToolRequestBody(
-                                type=input_schema_obj.type,
-                                properties=input_schema_obj.properties,
-                                required=input_schema_obj.required,
-                            ),
-                            output_schema=ToolResponseBody(
-                                type=output_schema_obj.type,
-                                properties=output_schema_obj.properties,
-                                required=output_schema_obj.required
-                            ) if output_schema_obj is not None else None,
-                           item_schema = foreach_item_schema)
+                           input_schema=_get_tool_request_body(input_schema_obj),
+                           output_schema=_get_tool_response_body(output_schema_obj),
+                           item_schema = new_foreach_item_schema)
         foreach_obj = Foreach(spec = spec, parent = self)
         foreach_node = self._add_node(foreach_obj)
-        self._add_schema(foreach_item_schema)
 
         return cast(Flow, foreach_node)
 
@@ -580,16 +588,8 @@ class Flow(Node):
 
         loop_spec = LoopSpec(name = "loop_" + str(self._next_sequence_id()), 
                              evaluator = e, 
-                             input_schema=ToolRequestBody(
-                                type=input_schema_obj.type,
-                                properties=input_schema_obj.properties,
-                                required=input_schema_obj.required,
-                             ) if input_schema_obj is not None else None,
-                             output_schema=ToolResponseBody(
-                                type=output_schema_obj.type,
-                                properties=output_schema_obj.properties,
-                                required=output_schema_obj.required
-                             ) if output_schema_obj is not None else None)
+                             input_schema=_get_tool_request_body(input_schema_obj),
+                             output_schema=_get_tool_response_body(output_schema_obj))
         while_loop = Loop(spec = loop_spec, parent = self)
         while_node = self._add_node(while_loop)
         return cast(Loop, while_node)
@@ -599,23 +599,15 @@ class Flow(Node):
                  input_schema: type[BaseModel] |None=None,
                  output_schema: type[BaseModel] |None=None) -> "UserFlow": # return a UserFlow object
 
-        logger.warning("userflow is NOT working yet.")
+        raise ValueError("userflow is NOT supported yet and it's interface will change.")
 
         output_schema_obj = _get_json_schema_obj("output", output_schema)
         input_schema_obj = _get_json_schema_obj("input", input_schema)
 
         spec = UserFlowSpec(name = "userflow_" + str(self._next_sequence_id()),
-                           input_schema=ToolRequestBody(
-                                type=input_schema_obj.type,
-                                properties=input_schema_obj.properties,
-                                required=input_schema_obj.required,
-                            ) if input_schema_obj is not None else None,
-                            output_schema=ToolResponseBody(
-                                type=output_schema_obj.type,
-                                properties=output_schema_obj.properties,
-                                required=output_schema_obj.required
-                            ) if output_schema_obj is not None else None,
-                           owners = owners)
+                            input_schema=_get_tool_request_body(input_schema_obj),
+                            output_schema=_get_tool_response_body(output_schema_obj),
+                            owners = owners)
         userflow_obj = UserFlow(spec = spec, parent = self)
         userflow_node = self._add_node(userflow_obj)
 
@@ -822,7 +814,7 @@ class FlowRun(BaseModel):
     def _on_flow_end(self, event:FlowEvent):
 
         self.status = FlowRunStatus.COMPLETED
-        self.output = event.context.data["output"] if "output" in event.context.data else None
+        self.output = event.context.data.output
 
         if self.debug:
             logger.debug(f"Flow run `{self.name}`: on_complete handler called. Output: {self.output}")
@@ -938,16 +930,8 @@ class FlowFactory(BaseModel):
             display_name=display_name,
             description=description,
             initiators=initiators,
-            input_schema=ToolRequestBody(
-                type=input_schema_obj.type,
-                properties=input_schema_obj.properties,
-                required=input_schema_obj.required,
-            ) if input_schema_obj else None,
-            output_schema=ToolResponseBody(
-                type=output_schema_obj.type,
-                properties=output_schema_obj.properties,
-                required=output_schema_obj.required
-            ) if output_schema_obj else None,
+            input_schema=_get_tool_request_body(input_schema_obj),
+            output_schema=_get_tool_response_body(output_schema_obj),
             output_schema_object = output_schema_obj
         )
 
@@ -974,6 +958,9 @@ class Branch(FlowControl):
         Returns:
         Self: The current node.
         '''
+        if kind == MatchPolicy.ANY_MATCH:
+            raise ValueError("Branch with policy ANY_MATCH is not supported yet.")
+        
         self.spec.match_policy = kind
         return self
 
@@ -1286,12 +1273,8 @@ class UserFlow(Flow):
             display_name=display_name,
             description=description,
             owners=owners,
-            input_schema=None,
-            output_schema=ToolResponseBody(
-                type=output_schema_obj.type,
-                properties=output_schema_obj.properties,
-                required=output_schema_obj.required
-            ),
+            input_schema=_get_tool_request_body(output_schema_obj),
+            output_schema=_get_tool_response_body(output_schema_obj),
             text=text,
             output_schema_object = output_schema_obj
         )

ibm_watsonx_orchestrate/{experimental/flow_builder → flow_builder}/types.py

@@ -14,7 +14,7 @@ from langchain_core.tools.base import create_schema_from_function
 from langchain_core.utils.json_schema import dereference_refs
 
 from ibm_watsonx_orchestrate.agent_builder.tools import PythonTool
-from ibm_watsonx_orchestrate.experimental.flow_builder.flows.constants import ANY_USER
+from ibm_watsonx_orchestrate.flow_builder.flows.constants import ANY_USER
 from ibm_watsonx_orchestrate.agent_builder.tools.types import (
     ToolSpec, ToolRequestBody, ToolResponseBody, JsonSchemaObject
 )
@@ -29,9 +29,9 @@ class SchemaRef(BaseModel):
  
     ref: str = Field(description="The id of the schema to be used.", serialization_alias="$ref")
 
-def _assign_attribute(obj, attr_name, schema):
+def _assign_attribute(model_spec, attr_name, schema):
     if hasattr(schema, attr_name) and (getattr(schema, attr_name) is not None):
-        obj[attr_name] = getattr(schema, attr_name)
+        model_spec[attr_name] = getattr(schema, attr_name)
 
 def _to_json_from_json_schema(schema: JsonSchemaObject) -> dict[str, Any]:
     model_spec = {}
@@ -62,12 +62,14 @@ def _to_json_from_json_schema(schema: JsonSchemaObject) -> dict[str, Any]:
         model_spec["anyOf"] = [_to_json_from_json_schema(schema) for schema in schema.anyOf]
 
     _assign_attribute(model_spec, "in_field", schema)
+    _assign_attribute(model_spec, "in", schema)
     _assign_attribute(model_spec, "aliasName", schema)
 
     if hasattr(schema, 'model_extra') and schema.model_extra:
         # for each extra fiels, add it to the model spec
         for key, value in schema.model_extra.items():
-            model_spec[key] = value
+            if value is not None:
+                model_spec[key] = value
 
     if isinstance(schema, JsonSchemaObjectRef):
         model_spec["$ref"] = schema.ref
@@ -635,6 +637,11 @@ class TaskEventType(Enum):
     ON_TASK_STREAM = "on_task_stream"
     ON_TASK_ERROR = "on_task_error"
 
+class FlowData(BaseModel):
+    '''This class represents the data that is passed between tasks in a flow.'''
+    input: dict[str, Any] = Field(default_factory=dict)
+    output: dict[str, Any] = Field(default_factory=dict)
+
 class FlowContext(BaseModel):
  
     name: str | None = None # name of the process or task
@@ -642,10 +649,12 @@ class FlowContext(BaseModel):
     flow_id: str | None = None # id of the flow, this is at the flow definition level
     instance_id: str | None = None
     thread_id: str | None = None
+    correlation_id: str | None = None
+    tenant_id: str | None = None
     parent_context: Any | None = None
     child_context: List["FlowContext"] | None = None
     metadata: dict = Field(default_factory=dict[str, Any])
-    data: dict = Field(default_factory=dict[str, Any])
+    data: Optional[FlowData] = None
 
     def get(self, key: str) -> Any:
      
@@ -655,7 +664,7 @@ class FlowContext(BaseModel):
         if self.parent_context:
             pc = cast(FlowContext, self.parent_conetxt)
             return pc.get(key)
-
+    
 class FlowEventType(Enum):
  
     ON_FLOW_START = "on_flow_start"

ibm_watsonx_orchestrate/flow_builder/utils.py

@@ -0,0 +1,185 @@
+import inspect
+import json
+from pathlib import Path
+import re
+import logging
+import importlib.resources
+import yaml
+
+from pydantic import BaseModel, TypeAdapter
+from typing import types
+
+from langchain_core.utils.json_schema import dereference_refs
+import typer
+
+from ibm_watsonx_orchestrate.agent_builder.connections.types import ConnectionEnvironment, ConnectionPreference, ConnectionSecurityScheme
+from ibm_watsonx_orchestrate.agent_builder.tools.openapi_tool import create_openapi_json_tools_from_content
+from ibm_watsonx_orchestrate.agent_builder.tools.types import JsonSchemaObject, ToolRequestBody, ToolResponseBody
+from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller import add_connection, configure_connection, set_credentials_connection
+from ibm_watsonx_orchestrate.client.connections.utils import get_connections_client
+from ibm_watsonx_orchestrate.client.tools.tempus_client import TempusClient
+from ibm_watsonx_orchestrate.client.utils import instantiate_client, is_local_dev
+
+logger = logging.getLogger(__name__)
+
+def get_valid_name(name: str) -> str:
+ 
+    return re.sub('\\W|^(?=\\d)','_', name)
+
+def _get_json_schema_obj(parameter_name: str, type_def: type[BaseModel] | ToolRequestBody | ToolResponseBody | None, openapi_decode: bool = False) -> JsonSchemaObject:
+    if not type_def or type_def is None or type_def == inspect._empty:
+        return None
+
+    if inspect.isclass(type_def) and issubclass(type_def, BaseModel):
+        schema_json = type_def.model_json_schema()
+        schema_json = dereference_refs(schema_json)
+        schema_obj = JsonSchemaObject(**schema_json)
+        if schema_obj.required is None:
+            schema_obj.required = []
+        return schema_obj
+    
+    if isinstance(type_def, ToolRequestBody) or isinstance(type_def, ToolResponseBody):
+        schema_json = type_def.model_dump()
+        schema_obj = JsonSchemaObject.model_validate(schema_json)
+
+        if openapi_decode:
+            # during tool import for openapi - we convert header, path and query parameter
+            # with a prefix "header_", "path_" and "query_".  We need to remove it.
+            if schema_obj.type == 'object':
+                # for each element in properties, we need to check the key and if it is
+                # prefixed with "header_", "path_" and "query_", we need to remove the prefix.
+                if hasattr(schema_obj, "properties"):
+                    new_properties = {}
+                    for key, value in schema_obj.properties.items():
+                        if key.startswith('header_'):
+                            new_properties[key[7:]] = value
+                        elif key.startswith('path_'):
+                            new_properties[key[5:]] = value
+                        elif key.startswith('query_'):
+                            new_properties[key[6:]] = value
+                        else:
+                            new_properties[key] = value
+                        
+                    schema_obj.properties = new_properties     
+
+                # we also need to go thru required and replace it
+                if hasattr(schema_obj, "required"):
+                    new_required = []
+                    for item in schema_obj.required:
+                        if item.startswith('header_'):
+                            new_required.append(item[7:])
+                        elif item.startswith('path_'):
+                            new_required.append(item[5:])
+                        elif item.startswith('query_'):
+                            new_required.append(item[6:])
+                        else:
+                            new_required.append(item)
+                    schema_obj.required = new_required
+
+        return schema_obj
+
+    # handle the non-obvious cases
+    schema_json = TypeAdapter(type_def).json_schema()
+    schema_json = dereference_refs(schema_json)
+    return JsonSchemaObject.model_validate(schema_json)
+
+
+def _get_tool_request_body(schema_obj: JsonSchemaObject) -> ToolRequestBody:
+    if schema_obj is None:
+        return None
+    
+    if isinstance(schema_obj, JsonSchemaObject):
+        request_obj = ToolRequestBody(type='object', properties=schema_obj.properties, required=schema_obj.required)
+        if schema_obj.model_extra:
+            request_obj.__pydantic_extra__ = schema_obj.model_extra
+
+        return request_obj
+    
+    raise ValueError(f"Invalid schema object: {schema_obj}")
+
+def _get_tool_response_body(schema_obj: JsonSchemaObject) -> ToolResponseBody:
+    if schema_obj is None:
+        return None
+    
+    if isinstance(schema_obj, JsonSchemaObject):
+        response_obj = ToolResponseBody(type=schema_obj.type)
+        if schema_obj.title:
+            response_obj.title = schema_obj.title
+        if schema_obj.description:
+            response_obj.description = schema_obj.description
+        if schema_obj.properties:
+            response_obj.properties = schema_obj.properties
+        if schema_obj.items:
+            response_obj.items = schema_obj.items
+        if schema_obj.uniqueItems:
+            response_obj.uniqueItems = schema_obj.uniqueItems
+        if schema_obj.anyOf:
+            response_obj.anyOf = schema_obj.anyOf
+        if schema_obj.required:
+            response_obj.required = schema_obj.required
+
+        if schema_obj.model_extra:
+            response_obj.__pydantic_extra__ = schema_obj.model_extra
+
+        return response_obj
+    
+    raise ValueError(f"Invalid schema object: {schema_obj}")
+
+
+async def import_flow_model(model):
+
+    if not is_local_dev():
+        raise typer.BadParameter(f"Flow tools are only supported in local environment.")
+
+    if model is None:
+        raise typer.BadParameter(f"No model provided.")
+    
+    tools = []
+    
+    flow_id = model["spec"]["name"]
+
+    tempus_client: TempusClient =  instantiate_client(TempusClient)
+
+    flow_open_api = tempus_client.create_update_flow_model(flow_id=flow_id, model=model)
+
+    logger.info(f"Flow model `{flow_id}` deployed successfully.")
+
+    connections_client = get_connections_client()
+    
+    app_id = "flow_tools_app"
+    logger.info(f"Creating connection for flow model...")
+    existing_app = connections_client.get(app_id=app_id)
+    if not existing_app:
+        # logger.info(f"Creating app `{app_id}`.")
+        add_connection(app_id=app_id)
+    # else:
+    #     logger.info(f"App `{app_id}` already exists.")
+    
+    # logger.info(f"Creating connection for app...")
+    configure_connection(
+        type=ConnectionPreference.MEMBER,
+        app_id=app_id,
+        token=connections_client.api_key,
+        environment=ConnectionEnvironment.DRAFT,
+        security_scheme=ConnectionSecurityScheme.BEARER_TOKEN,
+        shared=False
+    )
+
+    set_credentials_connection(app_id=app_id, environment=ConnectionEnvironment.DRAFT, token=connections_client.api_key)
+
+    connections = connections_client.get_draft_by_app_id(app_id=app_id)
+
+    # logger.info(f"Connection `{connections.connection_id}` created successfully.")
+    
+    tools = await create_openapi_json_tools_from_content(flow_open_api, connections.connection_id)
+
+    logger.info(f"Generating 'get_flow_status' tool spec...")    
+    # Temporary code to deploy a status tool until we have full async support
+    with importlib.resources.open_text('ibm_watsonx_orchestrate.flow_builder.resources', 'flow_status.openapi.yml', encoding='utf-8') as f:
+        get_status_openapi = f.read()
+
+    get_flow_status_spec = yaml.safe_load(get_status_openapi)
+    tools.extend(await create_openapi_json_tools_from_content(get_flow_status_spec, connections.connection_id))
+
+
+    return tools

{ibm_watsonx_orchestrate-1.5.1.dist-info → ibm_watsonx_orchestrate-1.6.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ibm-watsonx-orchestrate
-Version: 1.5.1
+Version: 1.6.0b0
 Summary: IBM watsonx.orchestrate SDK
 Author-email: IBM <support@ibm.com>
 License: MIT License
@@ -11,6 +11,7 @@ Requires-Dist: click<8.2.0,>=8.0.0
 Requires-Dist: docstring-parser<1.0,>=0.16
 Requires-Dist: httpx<1.0.0,>=0.28.1
 Requires-Dist: ibm-cloud-sdk-core>=3.22.0
+Requires-Dist: ibm-watsonx-orchestrate-evaluation-framework==1.0.2
 Requires-Dist: jsonref==1.1.0
 Requires-Dist: jsonschema<5.0.0,>=4.23.0
 Requires-Dist: langchain-community<1.0.0,>=0.3.12