langroid 0.6.7__py3-none-any.whl → 0.9.0__py3-none-any.whl

langroid/agent/chat_document.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import copy
 import json
+from collections import OrderedDict
 from enum import Enum
-from typing import Any, List, Optional, Union, cast
+from typing import Any, Dict, List, Optional, Union, cast
 
 from langroid.agent.tool_message import ToolMessage
 from langroid.language_models.base import (
@@ -11,7 +12,9 @@ from langroid.language_models.base import (
     LLMMessage,
     LLMResponse,
     LLMTokenUsage,
+    OpenAIToolCall,
     Role,
+    ToolChoiceTypes,
 )
 from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.agent_chats import parse_message
@@ -51,6 +54,8 @@ class ChatDocMetaData(DocMetaData):
     agent_id: str = ""  # ChatAgent that generated this message
     msg_idx: int = -1  # index of this message in the agent `message_history`
     sender: Entity  # sender of the message
+    # tool_id corresponding to single tool result in ChatDocument.content
+    oai_tool_id: str | None = None
     tool_ids: List[str] = []  # stack of tool_ids; used by OpenAIAssistant
     block: None | Entity = None
     sender_name: str = ""
@@ -86,8 +91,39 @@ class ChatDocLoggerFields(BaseModel):
 
 
 class ChatDocument(Document):
+    """
+    Represents a message in a conversation among agents. All responders of an agent
+    have signature ChatDocument -> ChatDocument (modulo None, str, etc),
+    and so does the Task.run() method.
+
+    Attributes:
+        oai_tool_calls (Optional[List[OpenAIToolCall]]):
+            Tool-calls from an OpenAI-compatible API
+        oai_tool_id2results (Optional[OrderedDict[str, str]]):
+            Results of tool-calls from OpenAI (dict is a map of tool_id -> result)
+        oai_tool_choice: ToolChoiceTypes | Dict[str, str]: Param controlling how the
+            LLM should choose tool-use in its response
+            (auto, none, required, or a specific tool)
+        function_call (Optional[LLMFunctionCall]):
+            Function-call from an OpenAI-compatible API
+                (deprecated by OpenAI, in favor of tool-calls)
+        tool_messages (List[ToolMessage]): Langroid ToolMessages extracted from
+            - `content` field (via JSON parsing),
+            - `oai_tool_calls`, or
+            - `function_call`
+        metadata (ChatDocMetaData): Metadata for the message, e.g. sender, recipient.
+        attachment (None | ChatDocAttachment): Any additional data attached.
+    """
+
+    oai_tool_calls: Optional[List[OpenAIToolCall]] = None
+    oai_tool_id2result: Optional[OrderedDict[str, str]] = None
+    oai_tool_choice: ToolChoiceTypes | Dict[str, Dict[str, str] | str] = "auto"
     function_call: Optional[LLMFunctionCall] = None
-    tool_messages: List[ToolMessage] = []
+    tool_messages: List[ToolMessage] = []  # only handle-able tools
+    # all known tools in the msg that are in an agent's llm_tools_known list,
+    # even if non-used/handled
+    all_tool_messages: List[ToolMessage] = []
+
     metadata: ChatDocMetaData
     attachment: None | ChatDocAttachment = None
 
@@ -107,6 +143,8 @@ class ChatDocument(Document):
     def deepcopy(doc: ChatDocument) -> ChatDocument:
         new_doc = copy.deepcopy(doc)
         new_doc.metadata.id = ObjectRegistry.new_id()
+        new_doc.metadata.child_id = ""
+        new_doc.metadata.parent_id = ""
         ObjectRegistry.register_object(new_doc)
         return new_doc
 
@@ -198,6 +236,24 @@ class ChatDocument(Document):
         if len(self.metadata.tool_ids) > 0:
             self.metadata.tool_ids.pop()
 
+    @staticmethod
+    def _clean_fn_call(fc: LLMFunctionCall | None) -> None:
+        # Sometimes an OpenAI LLM (esp gpt-4o) may generate a function-call
+        # with odditities:
+        # (a) the `name` is set, as well as `arguments.request` is set,
+        #  and in langroid we use the `request` value as the `name`.
+        #  In this case we override the `name` with the `request` value.
+        # (b) the `name` looks like "functions blah" or just "functions"
+        #   In this case we strip the "functions" part.
+        if fc is None:
+            return
+        fc.name = fc.name.replace("functions", "").strip()
+        if fc.arguments is not None:
+            request = fc.arguments.get("request")
+            if request is not None and request != "":
+                fc.name = request
+                fc.arguments.pop("request")
+
     @staticmethod
     def from_LLMResponse(
         response: LLMResponse,
@@ -216,22 +272,14 @@ class ChatDocument(Document):
         if message in ["''", '""']:
             message = ""
         if response.function_call is not None:
-            # Sometimes an OpenAI LLM (esp gpt-4o) may generate a function-call
-            # with odditities:
-            # (a) the `name` is set, as well as `arugments.request` is set,
-            #  and in langroid we use the `request` value as the `name`.
-            #  In this case we override the `name` with the `request` value.
-            # (b) the `name` looks like "functions blah" or just "functions"
-            #   In this case we strip the "functions" part.
-            fc = response.function_call
-            fc.name = fc.name.replace("functions", "").strip()
-            if fc.arguments is not None:
-                request = fc.arguments.get("request")
-                if request is not None and request != "":
-                    fc.name = request
-                    fc.arguments.pop("request")
+            ChatDocument._clean_fn_call(response.function_call)
+        if response.oai_tool_calls is not None:
+            # there must be at least one if it's not None
+            for oai_tc in response.oai_tool_calls:
+                ChatDocument._clean_fn_call(oai_tc.function)
         return ChatDocument(
             content=message,
+            oai_tool_calls=response.oai_tool_calls,
             function_call=response.function_call,
             metadata=ChatDocMetaData(
                 source=Entity.LLM,
@@ -261,24 +309,33 @@ class ChatDocument(Document):
         )
 
     @staticmethod
-    def to_LLMMessage(message: Union[str, "ChatDocument"]) -> LLMMessage:
+    def to_LLMMessage(
+        message: Union[str, "ChatDocument"],
+        oai_tools: Optional[List[OpenAIToolCall]] = None,
+    ) -> List[LLMMessage]:
         """
-        Convert to LLMMessage for use with LLM.
+        Convert to list of LLMMessage, to incorporate into msg-history sent to LLM API.
+        Usually there will be just a single LLMMessage, but when the ChatDocument
+        contains results from multiple OpenAI tool-calls, we would have a sequence
+        LLMMessages, one per tool-call result.
 
         Args:
             message (str|ChatDocument): Message to convert.
+            oai_tools (Optional[List[OpenAIToolCall]]): Tool-calls currently awaiting
+                response, from the ChatAgent's latest message.
         Returns:
-            LLMMessage: LLMMessage representation of this str or ChatDocument.
-
+            List[LLMMessage]: list of LLMMessages corresponding to this ChatDocument.
         """
         sender_name = None
         sender_role = Role.USER
         fun_call = None
-        tool_id = ""
+        oai_tool_calls = None
+        tool_id = ""  # for OpenAI Assistant
         chat_document_id: str = ""
         if isinstance(message, ChatDocument):
             content = message.content
             fun_call = message.function_call
+            oai_tool_calls = message.oai_tool_calls
             if message.metadata.sender == Entity.USER and fun_call is not None:
                 # This may happen when a (parent agent's) LLM generates a
                 # a Function-call, and it ends up being sent to the current task's
@@ -289,6 +346,10 @@ class ChatDocument(Document):
                 # in the content of the message.
                 content += " " + str(fun_call)
                 fun_call = None
+            if message.metadata.sender == Entity.USER and oai_tool_calls is not None:
+                # same reasoning as for function-call above
+                content += " " + "\n\n".join(str(tc) for tc in oai_tool_calls)
+                oai_tool_calls = None
             sender_name = message.metadata.sender_name
             tool_ids = message.metadata.tool_ids
             tool_id = tool_ids[-1] if len(tool_ids) > 0 else ""
@@ -299,22 +360,74 @@ class ChatDocument(Document):
                 message.metadata.parent is not None
                 and message.metadata.parent.function_call is not None
             ):
+                # This is a response to a function call, so set the role to FUNCTION.
                 sender_role = Role.FUNCTION
                 sender_name = message.metadata.parent.function_call.name
+            elif oai_tools is not None and len(oai_tools) > 0:
+                pending_tool_ids = [tc.id for tc in oai_tools]
+                # The ChatAgent has pending OpenAI tool-call(s),
+                # so the current ChatDocument contains
+                # results for some/all/none of them.
+
+                if len(oai_tools) == 1:
+                    # Case 1:
+                    # There was exactly 1 pending tool-call, and in this case
+                    # the result would be a plain string in `content`
+                    return [
+                        LLMMessage(
+                            role=Role.TOOL,
+                            tool_call_id=oai_tools[0].id,
+                            content=content,
+                            chat_document_id=chat_document_id,
+                        )
+                    ]
+
+                elif (
+                    message.metadata.oai_tool_id is not None
+                    and message.metadata.oai_tool_id in pending_tool_ids
+                ):
+                    # Case 2:
+                    # ChatDocument.content has result of a single tool-call
+                    return [
+                        LLMMessage(
+                            role=Role.TOOL,
+                            tool_call_id=message.metadata.oai_tool_id,
+                            content=content,
+                            chat_document_id=chat_document_id,
+                        )
+                    ]
+                elif message.oai_tool_id2result is not None:
+                    # Case 2:
+                    # There were > 1 tool-calls awaiting response,
+                    assert (
+                        len(message.oai_tool_id2result) > 1
+                    ), "oai_tool_id2result must have more than 1 item."
+                    return [
+                        LLMMessage(
+                            role=Role.TOOL,
+                            tool_call_id=tool_id,
+                            content=result,
+                            chat_document_id=chat_document_id,
+                        )
+                        for tool_id, result in message.oai_tool_id2result.items()
+                    ]
             elif message.metadata.sender == Entity.LLM:
                 sender_role = Role.ASSISTANT
         else:
             # LLM can only respond to text content, so extract it
             content = message
 
-        return LLMMessage(
-            role=sender_role,
-            tool_id=tool_id,
-            content=content,
-            function_call=fun_call,
-            name=sender_name,
-            chat_document_id=chat_document_id,
-        )
+        return [
+            LLMMessage(
+                role=sender_role,
+                tool_id=tool_id,  # for OpenAI Assistant
+                content=content,
+                function_call=fun_call,
+                tool_calls=oai_tool_calls,
+                name=sender_name,
+                chat_document_id=chat_document_id,
+            )
+        ]
 
 
 LLMMessage.update_forward_refs()

langroid/agent/openai_assistant.py

@@ -79,7 +79,7 @@ class OpenAIAssistantConfig(ChatAgentConfig):
     # set to True once we can add Assistant msgs in threads
     cache_responses: bool = True
     timeout: int = 30  # can be different from llm.timeout
-    llm = OpenAIGPTConfig(chat_model=OpenAIChatModel.GPT4_TURBO)
+    llm = OpenAIGPTConfig(chat_model=OpenAIChatModel.GPT4o)
     tools: List[AssistantTool] = []
     files: List[str] = []
 
@@ -160,7 +160,7 @@ class OpenAIAssistant(ChatAgent):
 
     def enable_message(
         self,
-        message_class: Optional[Type[ToolMessage]],
+        message_class: Optional[Type[ToolMessage] | List[Type[ToolMessage]]],
         use: bool = True,
         handle: bool = True,
         force: bool = False,
@@ -173,6 +173,17 @@ class OpenAIAssistant(ChatAgent):
         fn-calling seems to pay attn to these, and if we don't want this,
         we should set this to False.
         """
+        if message_class is not None and isinstance(message_class, list):
+            for msg_class in message_class:
+                self.enable_message(
+                    msg_class,
+                    use=use,
+                    handle=handle,
+                    force=force,
+                    require_recipient=require_recipient,
+                    include_defaults=include_defaults,
+                )
+            return
         super().enable_message(
             message_class,
             use=use,
@@ -192,7 +203,7 @@ class OpenAIAssistant(ChatAgent):
             self.set_system_message(sys_msg.content)
         if not self.config.use_functions_api:
             return
-        functions, _ = self._function_args()
+        functions, _, _, _ = self._function_args()
         if functions is None:
             return
         # add the functions to the assistant:
@@ -720,7 +731,12 @@ class OpenAIAssistant(ChatAgent):
         """
         is_tool_output = False
         if message is not None:
-            llm_msg = ChatDocument.to_LLMMessage(message)
+            # note: to_LLMMessage returns a list of LLMMessage,
+            # which is allowed to have len > 1, in case the msg
+            # represents results of multiple (non-assistant) tool-calls.
+            # But for OAI Assistant, we only assume exactly one tool-call at a time.
+            # TODO look into multi-tools
+            llm_msg = ChatDocument.to_LLMMessage(message)[0]
             tool_id = llm_msg.tool_id
             if tool_id in self.pending_tool_ids:
                 if isinstance(message, ChatDocument):

langroid/agent/special/lance_doc_chat_agent.py

@@ -17,10 +17,11 @@ from typing import Any, Dict, List, Tuple
 import pandas as pd
 
 from langroid.agent.special.doc_chat_agent import DocChatAgent, DocChatAgentConfig
-from langroid.agent.special.lance_tools import QueryPlanTool
+from langroid.agent.special.lance_tools import AnswerTool, QueryPlanTool
+from langroid.agent.tools.orchestration import AgentDoneTool
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.table_loader import describe_dataframe
-from langroid.utils.constants import DONE, NO_ANSWER
+from langroid.utils.constants import NO_ANSWER
 from langroid.utils.pydantic_utils import (
     dataframe_to_documents,
 )
@@ -106,7 +107,7 @@ class LanceDocChatAgent(DocChatAgent):
             """
         return schema
 
-    def query_plan(self, msg: QueryPlanTool) -> str:
+    def query_plan(self, msg: QueryPlanTool) -> AgentDoneTool | str:
         """
         Handle the LLM's use of the FilterTool.
         Temporarily set the config filter and either return the final answer
@@ -120,13 +121,15 @@ class LanceDocChatAgent(DocChatAgent):
         except Exception as e:
             logger.error(f"Error setting up documents: {e}")
             # say DONE with err msg so it goes back to LanceFilterAgent
-            return f"""
-            {DONE} Possible Filter Error:\n {e}
-            
-            Note that only the following fields are allowed in the filter
-            of a query plan: 
-            {", ".join(self.config.filter_fields)}
-            """
+            return AgentDoneTool(
+                content=f"""
+                Possible Filter Error:\n {e}
+                
+                Note that only the following fields are allowed in the filter
+                of a query plan: 
+                {", ".join(self.config.filter_fields)}
+                """
+            )
 
         # update the filter so it is used in the DocChatAgent
         self.config.filter = plan.filter or None
@@ -139,22 +142,25 @@ class LanceDocChatAgent(DocChatAgent):
             # The calc step can later be done with a separate Agent/Tool.
             if plan.query is None or plan.query.strip() == "":
                 if plan.filter is None or plan.filter.strip() == "":
-                    return """DONE
-                    Cannot execute Query Plan since filter as well as 
-                    rephrased query are empty.
-                    """
+                    return AgentDoneTool(
+                        content="""
+                        Cannot execute Query Plan since filter as well as 
+                        rephrased query are empty.                    
+                        """
+                    )
                 else:
                     # no query to match, so just get all docs matching filter
                     docs = self.vecdb.get_all_documents(plan.filter)
             else:
                 _, docs = self.get_relevant_extracts(plan.query)
             if len(docs) == 0:
-                return DONE + " " + NO_ANSWER
-            result = self.vecdb.compute_from_docs(docs, plan.dataframe_calc)
-            return DONE + " " + result
+                return AgentDoneTool(content=NO_ANSWER)
+            answer = self.vecdb.compute_from_docs(docs, plan.dataframe_calc)
         else:
             # pass on the query so LLM can handle it
-            return plan.query
+            response = self.llm_response(plan.query)
+            answer = NO_ANSWER if response is None else response.content
+        return AgentDoneTool(tools=[AnswerTool(answer=answer)])
 
     def ingest_docs(
         self,
@@ -242,6 +248,7 @@ class LanceDocChatAgent(DocChatAgent):
             .replace("NOT", "not")
             .replace("'", "")
             .replace('"', "")
+            .replace(":", "--")
         )
 
         tbl = self.vecdb.client.open_table(self.vecdb.config.collection_name)

langroid/agent/special/lance_rag/critic_agent.py

@@ -26,8 +26,8 @@ from langroid.agent.special.lance_tools import (
     QueryPlanAnswerTool,
     QueryPlanFeedbackTool,
 )
-from langroid.mytypes import Entity
-from langroid.utils.constants import DONE, NO_ANSWER, PASS
+from langroid.agent.tools.orchestration import AgentDoneTool
+from langroid.utils.constants import NO_ANSWER
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +50,9 @@ class QueryPlanCriticConfig(LanceQueryPlanAgentConfig):
       to create a QUERY PLAN, to be handled by an ASSISTANT.
     - PANDAS-LIKE FILTER, WHICH CAN BE EMPTY (and it's fine if results sound reasonable)
       FILTER SHOULD ONLY BE USED IF EXPLICITLY REQUIRED BY THE QUERY.
+      This filter selects the documents over which the REPHRASED QUERY will be applied,
+      thus naturally, the Re-phrased Query should NOT mention any FILTER fields,
+      since it applies to the documents AFTER FILTERING.
     - REPHRASED QUERY (CANNOT BE EMPTY) that will be used to match against the 
       CONTENT (not filterable) of the documents.
       In general the REPHRASED QUERY should be relied upon to match the CONTENT 
@@ -61,9 +64,31 @@ class QueryPlanCriticConfig(LanceQueryPlanAgentConfig):
         The assistant will answer based on documents whose CONTENTS match the QUERY, 
         possibly REPHRASED. 
         !!!!****THE REPHRASED QUERY SHOULD NEVER BE EMPTY****!!!
+        
+        
     - DATAFRAME CALCULATION, which must be a SINGLE LINE calculation (or empty),
         [NOTE ==> This calculation is applied AFTER the FILTER and REPHRASED QUERY.],
     - ANSWER received from an assistant that used this QUERY PLAN.
+      IT IS TOTALLY FINE FOR THE ANSWER TO NOT MENTION ANY FILTERING CONDITIONS,
+      or if the ANSWER STATEMENT is MISSING SOME CRITERIA in the ORIGINAL QUERY.
+
+        Here is an example of a VALID Plan + Answer:
+        
+        ORIGINAL QUERY: "Which crime novels were written by Russian authors after 1900?"
+        FILTER: "author_nationality == 'Russian' and year_written > 1900"
+        REPHRASED QUERY: "crime novel" [NOTICE NO FILTER FIELDS MENTIONED!!!]
+        DATAFRAME CALC: ""
+        ANSWER: "The Master and Margarita by Mikhail Bulgakov" 
+            [NOTICE the answer does NOT need to say "crime novel" or "russian author"]
+            
+            
+        Other examples of VALID ANSWER for a given ORIGINAL QUERY:
+        
+        ORIGINAL QUERY: "Which mountain is taller than 8000 meters?"
+        ANSWER: "Mount Everest" [NOTICE no mention of "taller than 8000 meters"]
+        
+        ORIGINAL QUERY: "Which country has hosted the most olympics?"
+        ANSWER: "United States" [NOTICE no mention of "most olympics"]
 
     In addition to the above SCHEMA fields there is a `content` field which:
     - CANNOT appear in a FILTER, 
@@ -141,21 +166,28 @@ class QueryPlanCritic(ChatAgent):
         self.config = cfg
         self.enable_message(QueryPlanAnswerTool, use=False, handle=True)
         self.enable_message(QueryPlanFeedbackTool, use=True, handle=True)
+        self.enable_message(AgentDoneTool, use=False, handle=True)
+
+    def init_state(self) -> None:
+        self.expecting_feedback_tool = False
 
     def query_plan_answer(self, msg: QueryPlanAnswerTool) -> str:
         """Present query plan + answer in plain text (not JSON)
         so LLM can give feedback"""
+        self.expecting_feedback_tool = True
         return plain_text_query_plan(msg)
 
-    def query_plan_feedback(self, msg: QueryPlanFeedbackTool) -> str:
+    def query_plan_feedback(self, msg: QueryPlanFeedbackTool) -> AgentDoneTool:
         """Format Valid so return to Query Planner"""
-        return DONE + " " + PASS  # return to Query Planner
+        self.expecting_feedback_tool = False
+        # indicate this task is Done, and return the tool as result
+        return AgentDoneTool(tools=[msg])
 
     def handle_message_fallback(
         self, msg: str | ChatDocument
     ) -> str | ChatDocument | None:
         """Remind the LLM to use QueryPlanFeedbackTool since it forgot"""
-        if isinstance(msg, ChatDocument) and msg.metadata.sender == Entity.LLM:
+        if self.expecting_feedback_tool:
             return """
             You forgot to use the `query_plan_feedback` tool/function.
             Re-try your response using the `query_plan_feedback` tool/function,