langroid 0.33.6__py3-none-any.whl → 0.33.7__py3-none-any.whl

langroid/agent/chat_agent.py

@@ -0,0 +1,1899 @@
+import copy
+import inspect
+import json
+import logging
+import textwrap
+from contextlib import ExitStack
+from inspect import isclass
+from typing import Dict, List, Optional, Self, Set, Tuple, Type, Union, cast
+
+import openai
+from rich import print
+from rich.console import Console
+from rich.markup import escape
+
+from langroid.agent.base import Agent, AgentConfig, async_noop_fn, noop_fn
+from langroid.agent.chat_document import ChatDocument
+from langroid.agent.tool_message import (
+    ToolMessage,
+    format_schema_for_strict,
+)
+from langroid.agent.xml_tool_message import XMLToolMessage
+from langroid.language_models.base import (
+    LLMFunctionCall,
+    LLMFunctionSpec,
+    LLMMessage,
+    LLMResponse,
+    OpenAIJsonSchemaSpec,
+    OpenAIToolSpec,
+    Role,
+    StreamingIfAllowed,
+    ToolChoiceTypes,
+)
+from langroid.language_models.openai_gpt import OpenAIGPT
+from langroid.pydantic_v1 import BaseModel, ValidationError
+from langroid.utils.configuration import settings
+from langroid.utils.object_registry import ObjectRegistry
+from langroid.utils.output import status
+from langroid.utils.pydantic_utils import PydanticWrapper, get_pydantic_wrapper
+
+console = Console()
+
+logger = logging.getLogger(__name__)
+
+
+class ChatAgentConfig(AgentConfig):
+    """
+    Configuration for ChatAgent
+    Attributes:
+        system_message: system message to include in message sequence
+             (typically defines role and task of agent).
+             Used only if `task` is not specified in the constructor.
+        user_message: user message to include in message sequence.
+             Used only if `task` is not specified in the constructor.
+        use_tools: whether to use our own ToolMessages mechanism
+        use_functions_api: whether to use functions/tools native to the LLM API
+                (e.g. OpenAI's `function_call` or `tool_call` mechanism)
+        use_tools_api: When `use_functions_api` is True, if this is also True,
+            the OpenAI tool-call API is used, rather than the older/deprecated
+            function-call API. However the tool-call API has some tricky aspects,
+            hence we set this to False by default.
+        strict_recovery: whether to enable strict schema recovery when there
+            is a tool-generation error.
+        enable_orchestration_tool_handling: whether to enable handling of orchestration
+            tools, e.g. ForwardTool, DoneTool, PassTool, etc.
+        output_format: When supported by the LLM (certain OpenAI LLMs
+            and local LLMs served by providers such as vLLM), ensures
+            that the output is a JSON matching the corresponding
+            schema via grammar-based decoding
+        handle_output_format: When `output_format` is a `ToolMessage` T,
+            controls whether T is "enabled for handling".
+        use_output_format: When `output_format` is a `ToolMessage` T,
+            controls whether T is "enabled for use" (by LLM) and
+            instructions on using T are added to the system message.
+        instructions_output_format: Controls whether we generate instructions for
+            `output_format` in the system message.
+        use_tools_on_output_format: Controls whether to automatically switch
+            to the Langroid-native tools mechanism when `output_format` is set.
+            Note that LLMs may generate tool calls which do not belong to
+            `output_format` even when strict JSON mode is enabled, so this should be
+            enabled when such tool calls are not desired.
+        output_format_include_defaults: Whether to include fields with default arguments
+            in the output schema
+    """
+
+    system_message: str = "You are a helpful assistant."
+    user_message: Optional[str] = None
+    use_tools: bool = False
+    use_functions_api: bool = True
+    use_tools_api: bool = False
+    strict_recovery: bool = True
+    enable_orchestration_tool_handling: bool = True
+    output_format: Optional[type] = None
+    handle_output_format: bool = True
+    use_output_format: bool = True
+    instructions_output_format: bool = True
+    output_format_include_defaults: bool = True
+    use_tools_on_output_format: bool = True
+
+    def _set_fn_or_tools(self, fn_available: bool) -> None:
+        """
+        Enable Langroid Tool or OpenAI-like fn-calling,
+        depending on config settings and availability of fn-calling.
+        """
+        if self.use_functions_api and not fn_available:
+            logger.debug(
+                """
+                You have enabled `use_functions_api` but the LLM does not support it.
+                So we will enable `use_tools` instead, so we can use 
+                Langroid's ToolMessage mechanism.
+                """
+            )
+            self.use_functions_api = False
+            self.use_tools = True
+
+        if not self.use_functions_api or not self.use_tools:
+            return
+        if self.use_functions_api and self.use_tools:
+            logger.debug(
+                """
+                You have enabled both `use_tools` and `use_functions_api`.
+                Turning off `use_tools`, since the LLM supports function-calling.
+                """
+            )
+            self.use_tools = False
+            self.use_functions_api = True
+
+
+class ChatAgent(Agent):
+    """
+    Chat Agent interacting with external env
+    (could be human, or external tools).
+    The agent (the LLM actually) is provided with an optional "Task Spec",
+    which is a sequence of `LLMMessage`s. These are used to initialize
+    the `task_messages` of the agent.
+    In most applications we will use a `ChatAgent` rather than a bare `Agent`.
+    The `Agent` class mainly exists to hold various common methods and attributes.
+    One difference between `ChatAgent` and `Agent` is that `ChatAgent`'s
+    `llm_response` method uses "chat mode" API (i.e. one that takes a
+    message sequence rather than a single message),
+    whereas the same method in the `Agent` class uses "completion mode" API (i.e. one
+    that takes a single message).
+    """
+
+    def __init__(
+        self,
+        config: ChatAgentConfig = ChatAgentConfig(),
+        task: Optional[List[LLMMessage]] = None,
+    ):
+        """
+        Chat-mode agent initialized with task spec as the initial message sequence
+        Args:
+            config: settings for the agent
+
+        """
+        super().__init__(config)
+        self.config: ChatAgentConfig = config
+        self.config._set_fn_or_tools(self._fn_call_available())
+        self.message_history: List[LLMMessage] = []
+        self.init_state()
+        # An agent's "task" is defined by a system msg and an optional user msg;
+        # These are "priming" messages that kick off the agent's conversation.
+        self.system_message: str = self.config.system_message
+        self.user_message: str | None = self.config.user_message
+
+        if task is not None:
+            # if task contains a system msg, we override the config system msg
+            if len(task) > 0 and task[0].role == Role.SYSTEM:
+                self.system_message = task[0].content
+            # if task contains a user msg, we override the config user msg
+            if len(task) > 1 and task[1].role == Role.USER:
+                self.user_message = task[1].content
+
+        # system-level instructions for using tools/functions:
+        # We maintain these as tools/functions are enabled/disabled,
+        # and whenever an LLM response is sought, these are used to
+        # recreate the system message (via `_create_system_and_tools_message`)
+        # each time, so it reflects the current set of enabled tools/functions.
+        # (a) these are general instructions on using certain tools/functions,
+        #   if they are specified in a ToolMessage class as a classmethod `instructions`
+        self.system_tool_instructions: str = ""
+        # (b) these are only for the builtin in Langroid TOOLS mechanism:
+        self.system_tool_format_instructions: str = ""
+
+        self.llm_functions_map: Dict[str, LLMFunctionSpec] = {}
+        self.llm_functions_handled: Set[str] = set()
+        self.llm_functions_usable: Set[str] = set()
+        self.llm_function_force: Optional[Dict[str, str]] = None
+
+        self.output_format: Optional[type[ToolMessage | BaseModel]] = None
+
+        self.saved_requests_and_tool_setings = self._requests_and_tool_settings()
+        # This variable is not None and equals a `ToolMessage` T, if and only if:
+        # (a) T has been set as the output_format of this agent, AND
+        # (b) T has been "enabled for use" ONLY for enforcing this output format, AND
+        # (c) T has NOT been explicitly "enabled for use" by this Agent.
+        self.enabled_use_output_format: Optional[type[ToolMessage]] = None
+        # As above but deals with "enabled for handling" instead of "enabled for use".
+        self.enabled_handling_output_format: Optional[type[ToolMessage]] = None
+        if config.output_format is not None:
+            self.set_output_format(config.output_format)
+        # instructions specifically related to enforcing `output_format`
+        self.output_format_instructions = ""
+
+        # controls whether to disable strict schemas for this agent if
+        # strict mode causes exception
+        self.disable_strict = False
+        # Tracks whether any strict tool is enabled; used to determine whether to set
+        # `self.disable_strict` on an exception
+        self.any_strict = False
+        # Tracks the set of tools on which we force-disable strict decoding
+        self.disable_strict_tools_set: set[str] = set()
+
+        if self.config.enable_orchestration_tool_handling:
+            # Only enable HANDLING by `agent_response`, NOT LLM generation of these.
+            # This is useful where tool-handlers or agent_response generate these
+            # tools, and need to be handled.
+            # We don't want enable orch tool GENERATION by default, since that
+            # might clutter-up the LLM system message unnecessarily.
+            from langroid.agent.tools.orchestration import (
+                AgentDoneTool,
+                AgentSendTool,
+                DonePassTool,
+                DoneTool,
+                ForwardTool,
+                PassTool,
+                ResultTool,
+                SendTool,
+            )
+
+            self.enable_message(ForwardTool, use=False, handle=True)
+            self.enable_message(DoneTool, use=False, handle=True)
+            self.enable_message(AgentDoneTool, use=False, handle=True)
+            self.enable_message(PassTool, use=False, handle=True)
+            self.enable_message(DonePassTool, use=False, handle=True)
+            self.enable_message(SendTool, use=False, handle=True)
+            self.enable_message(AgentSendTool, use=False, handle=True)
+            self.enable_message(ResultTool, use=False, handle=True)
+
+    def init_state(self) -> None:
+        """
+        Initialize the state of the agent. Just conversation state here,
+        but subclasses can override this to initialize other state.
+        """
+        super().init_state()
+        self.clear_history(0)
+        self.clear_dialog()
+
+    @staticmethod
+    def from_id(id: str) -> "ChatAgent":
+        """
+        Get an agent from its ID
+        Args:
+            agent_id (str): ID of the agent
+        Returns:
+            ChatAgent: The agent with the given ID
+        """
+        return cast(ChatAgent, Agent.from_id(id))
+
+    def clone(self, i: int = 0) -> "ChatAgent":
+        """Create i'th clone of this agent, ensuring tool use/handling is cloned.
+        Important: We assume all member variables are in the __init__ method here
+        and in the Agent class.
+        TODO: We are attempting to clone an agent after its state has been
+        changed in possibly many ways. Below is an imperfect solution. Caution advised.
+        Revisit later.
+        """
+        agent_cls = type(self)
+        config_copy = copy.deepcopy(self.config)
+        config_copy.name = f"{config_copy.name}-{i}"
+        new_agent = agent_cls(config_copy)
+        new_agent.system_tool_instructions = self.system_tool_instructions
+        new_agent.system_tool_format_instructions = self.system_tool_format_instructions
+        new_agent.llm_tools_map = self.llm_tools_map
+        new_agent.llm_functions_map = self.llm_functions_map
+        new_agent.llm_functions_handled = self.llm_functions_handled
+        new_agent.llm_functions_usable = self.llm_functions_usable
+        new_agent.llm_function_force = self.llm_function_force
+        # Caution - we are copying the vector-db, maybe we don't always want this?
+        new_agent.vecdb = self.vecdb
+        new_agent.id = ObjectRegistry.new_id()
+        if self.config.add_to_registry:
+            ObjectRegistry.register_object(new_agent)
+        return new_agent
+
+    def _strict_mode_for_tool(self, tool: str | type[ToolMessage]) -> bool:
+        """Should we enable strict mode for a given tool?"""
+        if isinstance(tool, str):
+            tool_class = self.llm_tools_map[tool]
+        else:
+            tool_class = tool
+        name = tool_class.default_value("request")
+        if name in self.disable_strict_tools_set or self.disable_strict:
+            return False
+        strict: Optional[bool] = tool_class.default_value("strict")
+        if strict is None:
+            strict = self._strict_tools_available()
+
+        return strict
+
+    def _fn_call_available(self) -> bool:
+        """Does this agent's LLM support function calling?"""
+        return (
+            self.llm is not None
+            and isinstance(self.llm, OpenAIGPT)
+            and self.llm.is_openai_chat_model()
+            and self.llm.supports_functions_or_tools()
+        )
+
+    def _strict_tools_available(self) -> bool:
+        """Does this agent's LLM support strict tools?"""
+        return (
+            not self.disable_strict
+            and self.llm is not None
+            and isinstance(self.llm, OpenAIGPT)
+            and self.llm.config.parallel_tool_calls is False
+            and self.llm.supports_strict_tools
+        )
+
+    def _json_schema_available(self) -> bool:
+        """Does this agent's LLM support strict JSON schema output format?"""
+        return (
+            not self.disable_strict
+            and self.llm is not None
+            and isinstance(self.llm, OpenAIGPT)
+            and self.llm.supports_json_schema
+        )
+
+    def set_system_message(self, msg: str) -> None:
+        self.system_message = msg
+        if len(self.message_history) > 0:
+            # if there is message history, update the system message in it
+            self.message_history[0].content = msg
+
+    def set_user_message(self, msg: str) -> None:
+        self.user_message = msg
+
+    @property
+    def task_messages(self) -> List[LLMMessage]:
+        """
+        The task messages are the initial messages that define the task
+        of the agent. There will be at least a system message plus possibly a user msg.
+        Returns:
+            List[LLMMessage]: the task messages
+        """
+        msgs = [self._create_system_and_tools_message()]
+        if self.user_message:
+            msgs.append(LLMMessage(role=Role.USER, content=self.user_message))
+        return msgs
+
+    def _drop_msg_update_tool_calls(self, msg: LLMMessage) -> None:
+        id2idx = {t.id: i for i, t in enumerate(self.oai_tool_calls)}
+        if msg.role == Role.TOOL:
+            # dropping tool result, so ADD the corresponding tool-call back
+            # to the list of pending calls!
+            id = msg.tool_call_id
+            if id in self.oai_tool_id2call:
+                self.oai_tool_calls.append(self.oai_tool_id2call[id])
+        elif msg.tool_calls is not None:
+            # dropping a msg with tool-calls, so DROP these from pending list
+            # as well as from id -> call map
+            for tool_call in msg.tool_calls:
+                if tool_call.id in id2idx:
+                    self.oai_tool_calls.pop(id2idx[tool_call.id])
+                if tool_call.id in self.oai_tool_id2call:
+                    del self.oai_tool_id2call[tool_call.id]
+
+    def clear_history(self, start: int = -2) -> None:
+        """
+        Clear the message history, starting at the index `start`
+
+        Args:
+            start (int): index of first message to delete; default = -2
+                    (i.e. delete last 2 messages, typically these
+                    are the last user and assistant messages)
+        """
+        if start < 0:
+            n = len(self.message_history)
+            start = max(0, n + start)
+        dropped = self.message_history[start:]
+        # consider the dropped msgs in REVERSE order, so we are
+        # carefully updating self.oai_tool_calls
+        for msg in reversed(dropped):
+            self._drop_msg_update_tool_calls(msg)
+            # clear out the chat document from the ObjectRegistry
+            ChatDocument.delete_id(msg.chat_document_id)
+        self.message_history = self.message_history[:start]
+
+    def update_history(self, message: str, response: str) -> None:
+        """
+        Update the message history with the latest user message and LLM response.
+        Args:
+            message (str): user message
+            response: (str): LLM response
+        """
+        self.message_history.extend(
+            [
+                LLMMessage(role=Role.USER, content=message),
+                LLMMessage(role=Role.ASSISTANT, content=response),
+            ]
+        )
+
+    def tool_format_rules(self) -> str:
+        """
+        Specification of tool formatting rules
+        (typically JSON-based but can be non-JSON, e.g. XMLToolMessage),
+        based on the currently enabled usable `ToolMessage`s
+
+        Returns:
+            str: formatting rules
+        """
+        # ONLY Usable tools (i.e. LLM-generation allowed),
+        usable_tool_classes: List[Type[ToolMessage]] = [
+            t
+            for t in list(self.llm_tools_map.values())
+            if t.default_value("request") in self.llm_tools_usable
+        ]
+
+        if len(usable_tool_classes) == 0:
+            return "You can ask questions in natural language."
+        format_instructions = "\n\n".join(
+            [
+                msg_cls.format_instructions(tool=self.config.use_tools)
+                for msg_cls in usable_tool_classes
+            ]
+        )
+        # if any of the enabled classes has json_group_instructions, then use that,
+        # else fall back to ToolMessage.json_group_instructions
+        for msg_cls in usable_tool_classes:
+            if hasattr(msg_cls, "json_group_instructions") and callable(
+                getattr(msg_cls, "json_group_instructions")
+            ):
+                return msg_cls.group_format_instructions().format(
+                    format_instructions=format_instructions
+                )
+        return ToolMessage.group_format_instructions().format(
+            format_instructions=format_instructions
+        )
+
+    def tool_instructions(self) -> str:
+        """
+        Instructions for tools or function-calls, for enabled and usable Tools.
+        These are inserted into system prompt regardless of whether we are using
+        our own ToolMessage mechanism or the LLM's function-call mechanism.
+
+        Returns:
+            str: concatenation of instructions for all usable tools
+        """
+        enabled_classes: List[Type[ToolMessage]] = list(self.llm_tools_map.values())
+        if len(enabled_classes) == 0:
+            return ""
+        instructions = []
+        for msg_cls in enabled_classes:
+            if msg_cls.default_value("request") in self.llm_tools_usable:
+                class_instructions = ""
+                if hasattr(msg_cls, "instructions") and inspect.ismethod(
+                    msg_cls.instructions
+                ):
+                    class_instructions = msg_cls.instructions()
+                if (
+                    self.config.use_tools
+                    and hasattr(msg_cls, "langroid_tools_instructions")
+                    and inspect.ismethod(msg_cls.langroid_tools_instructions)
+                ):
+                    class_instructions += msg_cls.langroid_tools_instructions()
+                # example will be shown in tool_format_rules() when using TOOLs,
+                # so we don't need to show it here.
+                example = "" if self.config.use_tools else (msg_cls.usage_examples())
+                if example != "":
+                    example = "EXAMPLES:\n" + example
+                guidance = (
+                    ""
+                    if class_instructions == ""
+                    else ("GUIDANCE: " + class_instructions)
+                )
+                if guidance == "" and example == "":
+                    continue
+                instructions.append(
+                    textwrap.dedent(
+                        f"""
+                        TOOL: {msg_cls.default_value("request")}:
+                        {guidance}
+                        {example}
+                        """.lstrip()
+                    )
+                )
+        if len(instructions) == 0:
+            return ""
+        instructions_str = "\n\n".join(instructions)
+        return textwrap.dedent(
+            f"""
+            === GUIDELINES ON SOME TOOLS/FUNCTIONS USAGE ===
+            {instructions_str}
+            """.lstrip()
+        )
+
+    def augment_system_message(self, message: str) -> None:
+        """
+        Augment the system message with the given message.
+        Args:
+            message (str): system message
+        """
+        self.system_message += "\n\n" + message
+
+    def last_message_with_role(self, role: Role) -> LLMMessage | None:
+        """from `message_history`, return the last message with role `role`"""
+        n_role_msgs = len([m for m in self.message_history if m.role == role])
+        if n_role_msgs == 0:
+            return None
+        idx = self.nth_message_idx_with_role(role, n_role_msgs)
+        return self.message_history[idx]
+
+    def nth_message_idx_with_role(self, role: Role, n: int) -> int:
+        """Index of `n`th message in message_history, with specified role.
+        (n is assumed to be 1-based, i.e. 1 is the first message with that role).
+        Return -1 if not found. Index = 0 is the first message in the history.
+        """
+        indices_with_role = [
+            i for i, m in enumerate(self.message_history) if m.role == role
+        ]
+
+        if len(indices_with_role) < n:
+            return -1
+        return indices_with_role[n - 1]
+
+    def update_last_message(self, message: str, role: str = Role.USER) -> None:
+        """
+        Update the last message that has role `role` in the message history.
+        Useful when we want to replace a long user prompt, that may contain context
+        documents plus a question, with just the question.
+        Args:
+            message (str): new message to replace with
+            role (str): role of message to replace
+        """
+        if len(self.message_history) == 0:
+            return
+        # find last message in self.message_history with role `role`
+        for i in range(len(self.message_history) - 1, -1, -1):
+            if self.message_history[i].role == role:
+                self.message_history[i].content = message
+                break
+
+    def _create_system_and_tools_message(self) -> LLMMessage:
+        """
+        (Re-)Create the system message for the LLM of the agent,
+        taking into account any tool instructions that have been added
+        after the agent was initialized.
+
+        The system message will consist of:
+        (a) the system message from the `task` arg in constructor, if any,
+            otherwise the default system message from the config
+        (b) the system tool instructions, if any
+        (c) the system json tool instructions, if any
+
+        Returns:
+            LLMMessage object
+        """
+        content = textwrap.dedent(
+            f"""
+            {self.system_message}
+            
+            {self.system_tool_instructions}
+            
+            {self.system_tool_format_instructions}
+
+            {self.output_format_instructions}
+            """
+        )
+        # remove leading and trailing newlines and other whitespace
+        return LLMMessage(role=Role.SYSTEM, content=content.strip())
+
+    def unhandled_tools(self) -> set[str]:
+        """The set of tools that are known but not handled.
+        Useful in task flow: an agent can refuse to accept an incoming msg
+        when it only has unhandled tools.
+        """
+        return self.llm_tools_known - self.llm_tools_handled
+
+    def enable_message(
+        self,
+        message_class: Optional[Type[ToolMessage] | List[Type[ToolMessage]]],
+        use: bool = True,
+        handle: bool = True,
+        force: bool = False,
+        require_recipient: bool = False,
+        include_defaults: bool = True,
+    ) -> None:
+        """
+        Add the tool (message class) to the agent, and enable either
+        - tool USE (i.e. the LLM can generate JSON to use this tool),
+        - tool HANDLING (i.e. the agent can handle JSON from this tool),
+
+        Args:
+            message_class: The ToolMessage class OR List of such classes to enable,
+                for USE, or HANDLING, or both.
+                If this is a list of ToolMessage classes, then the remain args are
+                applied to all classes.
+                Optional; if None, then apply the enabling to all tools in the
+                agent's toolset that have been enabled so far.
+            use: IF True, allow the agent (LLM) to use this tool (or all tools),
+                else disallow
+            handle: if True, allow the agent (LLM) to handle (i.e. respond to) this
+                tool (or all tools)
+            force: whether to FORCE the agent (LLM) to USE the specific
+                 tool represented by `message_class`.
+                 `force` is ignored if `message_class` is None.
+            require_recipient: whether to require that recipient be specified
+                when using the tool message (only applies if `use` is True).
+            include_defaults: whether to include fields that have default values,
+                in the "properties" section of the JSON format instructions.
+                (Normally the OpenAI completion API ignores these fields,
+                but the Assistant fn-calling seems to pay attn to these,
+                and if we don't want this, we should set this to False.)
+        """
+        if message_class is not None and isinstance(message_class, list):
+            for mc in message_class:
+                self.enable_message(
+                    mc,
+                    use=use,
+                    handle=handle,
+                    force=force,
+                    require_recipient=require_recipient,
+                    include_defaults=include_defaults,
+                )
+            return None
+        if require_recipient and message_class is not None:
+            message_class = message_class.require_recipient()
+        if isinstance(message_class, XMLToolMessage):
+            # XMLToolMessage is not compatible with OpenAI's Tools/functions API,
+            # so we disable use of functions API, enable langroid-native Tools,
+            # which are prompt-based.
+            self.config.use_functions_api = False
+            self.config.use_tools = True
+        super().enable_message_handling(message_class)  # enables handling only
+        tools = self._get_tool_list(message_class)
+        if message_class is not None:
+            request = message_class.default_value("request")
+            if request == "":
+                raise ValueError(
+                    f"""
+                    ToolMessage class {message_class} must have a non-empty 
+                    'request' field if it is to be enabled as a tool.
+                    """
+                )
+            llm_function = message_class.llm_function_schema(defaults=include_defaults)
+            self.llm_functions_map[request] = llm_function
+            if force:
+                self.llm_function_force = dict(name=request)
+            else:
+                self.llm_function_force = None
+
+        for t in tools:
+            self.llm_tools_known.add(t)
+
+            if handle:
+                self.llm_tools_handled.add(t)
+                self.llm_functions_handled.add(t)
+
+                if (
+                    self.enabled_handling_output_format is not None
+                    and self.enabled_handling_output_format.name() == t
+                ):
+                    # `t` was designated as "enabled for handling" ONLY for
+                    # output_format enforcement, but we are explicitly ]
+                    # enabling it for handling here, so we set the variable to None.
+                    self.enabled_handling_output_format = None
+            else:
+                self.llm_tools_handled.discard(t)
+                self.llm_functions_handled.discard(t)
+
+            if use:
+                tool_class = self.llm_tools_map[t]
+                if tool_class._allow_llm_use:
+                    self.llm_tools_usable.add(t)
+                    self.llm_functions_usable.add(t)
+                else:
+                    logger.warning(
+                        f"""
+                        ToolMessage class {tool_class} does not allow LLM use,
+                        because `_allow_llm_use=False` either in the Tool or a 
+                        parent class of this tool;
+                        so not enabling LLM use for this tool!
+                        If you intended an LLM to use this tool, 
+                        set `_allow_llm_use=True` when you define the tool.
+                        """
+                    )
+                if (
+                    self.enabled_use_output_format is not None
+                    and self.enabled_use_output_format.default_value("request") == t
+                ):
+                    # `t` was designated as "enabled for use" ONLY for output_format
+                    # enforcement, but we are explicitly enabling it for use here,
+                    # so we set the variable to None.
+                    self.enabled_use_output_format = None
+            else:
+                self.llm_tools_usable.discard(t)
+                self.llm_functions_usable.discard(t)
+
+        # Set tool instructions and JSON format instructions
+        if self.config.use_tools:
+            self.system_tool_format_instructions = self.tool_format_rules()
+        self.system_tool_instructions = self.tool_instructions()
+
+    def _requests_and_tool_settings(self) -> tuple[Optional[set[str]], bool, bool]:
+        """
+        Returns the current set of enabled requests for inference and tools configs.
+        Used for restoring setings overriden by `set_output_format`.
+        """
+        return (
+            self.enabled_requests_for_inference,
+            self.config.use_functions_api,
+            self.config.use_tools,
+        )
+
+    @property
+    def all_llm_tools_known(self) -> set[str]:
+        """All known tools; we include `output_format` if it is a `ToolMessage`."""
+        known = self.llm_tools_known
+
+        if self.output_format is not None and issubclass(
+            self.output_format, ToolMessage
+        ):
+            return known.union({self.output_format.default_value("request")})
+
+        return known
+
+    def set_output_format(
+        self,
+        output_type: Optional[type],
+        force_tools: Optional[bool] = None,
+        use: Optional[bool] = None,
+        handle: Optional[bool] = None,
+        instructions: Optional[bool] = None,
+        is_copy: bool = False,
+    ) -> None:
+        """
+        Sets `output_format` to `output_type` and, if `force_tools` is enabled,
+        switches to the native Langroid tools mechanism to ensure that no tool
+        calls not of `output_type` are generated. By default, `force_tools`
+        follows the `use_tools_on_output_format` parameter in the config.
+
+        If `output_type` is None, restores to the state prior to setting
+        `output_format`.
+
+        If `use`, we enable use of `output_type` when it is a subclass
+        of `ToolMesage`. Note that this primarily controls instruction
+        generation: the model will always generate `output_type` regardless
+        of whether `use` is set. Defaults to the `use_output_format`
+        parameter in the config. Similarly, handling of `output_type` is
+        controlled by `handle`, which defaults to the
+        `handle_output_format` parameter in the config.
+
+        `instructions` controls whether we generate instructions specifying
+        the output format schema. Defaults to the `instructions_output_format`
+        parameter in the config.
+
+        `is_copy` is set when called via `__getitem__`. In that case, we must
+        copy certain fields to ensure that we do not overwrite the main agent's
+        setings.
+        """
+        # Disable usage of an output format which was not specifically enabled
+        # by `enable_message`
+        if self.enabled_use_output_format is not None:
+            self.disable_message_use(self.enabled_use_output_format)
+            self.enabled_use_output_format = None
+
+        # Disable handling of an output format which did not specifically have
+        # handling enabled via `enable_message`
+        if self.enabled_handling_output_format is not None:
+            self.disable_message_handling(self.enabled_handling_output_format)
+            self.enabled_handling_output_format = None
+
+        # Reset any previous instructions
+        self.output_format_instructions = ""
+
+        if output_type is None:
+            self.output_format = None
+            (
+                requests_for_inference,
+                use_functions_api,
+                use_tools,
+            ) = self.saved_requests_and_tool_setings
+            self.config = self.config.copy()
+            self.enabled_requests_for_inference = requests_for_inference
+            self.config.use_functions_api = use_functions_api
+            self.config.use_tools = use_tools
+        else:
+            if force_tools is None:
+                force_tools = self.config.use_tools_on_output_format
+
+            if not any(
+                (isclass(output_type) and issubclass(output_type, t))
+                for t in [ToolMessage, BaseModel]
+            ):
+                output_type = get_pydantic_wrapper(output_type)
+
+            if self.output_format is None and force_tools:
+                self.saved_requests_and_tool_setings = (
+                    self._requests_and_tool_settings()
+                )
+
+            self.output_format = output_type
+            if issubclass(output_type, ToolMessage):
+                name = output_type.default_value("request")
+                if use is None:
+                    use = self.config.use_output_format
+
+                if handle is None:
+                    handle = self.config.handle_output_format
+
+                if use or handle:
+                    is_usable = name in self.llm_tools_usable.union(
+                        self.llm_functions_usable
+                    )
+                    is_handled = name in self.llm_tools_handled.union(
+                        self.llm_functions_handled
+                    )
+
+                    if is_copy:
+                        if use:
+                            # We must copy `llm_tools_usable` so the base agent
+                            # is unmodified
+                            self.llm_tools_usable = copy.copy(self.llm_tools_usable)
+                            self.llm_functions_usable = copy.copy(
+                                self.llm_functions_usable
+                            )
+                        if handle:
+                            # If handling the tool, do the same for `llm_tools_handled`
+                            self.llm_tools_handled = copy.copy(self.llm_tools_handled)
+                            self.llm_functions_handled = copy.copy(
+                                self.llm_functions_handled
+                            )
+                    # Enable `output_type`
+                    self.enable_message(
+                        output_type,
+                        # Do not override existing settings
+                        use=use or is_usable,
+                        handle=handle or is_handled,
+                    )
+
+                    # If the `output_type` ToilMessage was not already enabled for
+                    # use, this means we are ONLY enabling it for use specifically
+                    # for enforcing this output format, so we set the
+                    # `enabled_use_output_forma  to this output_type, to
+                    # record that it should be disabled when `output_format` is changed
+                    if not is_usable:
+                        self.enabled_use_output_format = output_type
+
+                    # (same reasoning as for use-enabling)
+                    if not is_handled:
+                        self.enabled_handling_output_format = output_type
+
+                generated_tool_instructions = name in self.llm_tools_usable.union(
+                    self.llm_functions_usable
+                )
+            else:
+                generated_tool_instructions = False
+
+            if instructions is None:
+                instructions = self.config.instructions_output_format
+            if issubclass(output_type, BaseModel) and instructions:
+                if generated_tool_instructions:
+                    # Already generated tool instructions as part of "enabling for use",
+                    # so only need to generate a reminder to use this tool.
+                    name = cast(ToolMessage, output_type).default_value("request")
+                    self.output_format_instructions = textwrap.dedent(
+                        f"""
+                        === OUTPUT FORMAT INSTRUCTIONS ===
+
+                        Please provide output using the `{name}` tool/function.
+                        """
+                    )
+                else:
+                    if issubclass(output_type, ToolMessage):
+                        output_format_schema = output_type.llm_function_schema(
+                            request=True,
+                            defaults=self.config.output_format_include_defaults,
+                        ).parameters
+                    else:
+                        output_format_schema = output_type.schema()
+
+                    format_schema_for_strict(output_format_schema)
+
+                    self.output_format_instructions = textwrap.dedent(
+                        f"""
+                        === OUTPUT FORMAT INSTRUCTIONS ===
+                        Please provide output as JSON with the following schema:
+
+                        {output_format_schema}
+                        """
+                    )
+
+            if force_tools:
+                if issubclass(output_type, ToolMessage):
+                    self.enabled_requests_for_inference = {
+                        output_type.default_value("request")
+                    }
+                if self.config.use_functions_api:
+                    self.config = self.config.copy()
+                    self.config.use_functions_api = False
+                    self.config.use_tools = True
+
+    def __getitem__(self, output_type: type) -> Self:
+        """
+        Returns a (shallow) copy of `self` with a forced output type.
+        """
+        clone = copy.copy(self)
+        clone.set_output_format(output_type, is_copy=True)
+        return clone
+
+    def disable_message_handling(
+        self,
+        message_class: Optional[Type[ToolMessage]] = None,
+    ) -> None:
+        """
+        Disable this agent from RESPONDING to a `message_class` (Tool). If
+            `message_class` is None, then disable this agent from responding to ALL.
+        Args:
+            message_class: The ToolMessage class to disable; Optional.
+        """
+        super().disable_message_handling(message_class)
+        for t in self._get_tool_list(message_class):
+            self.llm_tools_handled.discard(t)
+            self.llm_functions_handled.discard(t)
+
+    def disable_message_use(
+        self,
+        message_class: Optional[Type[ToolMessage]],
+    ) -> None:
+        """
+        Disable this agent from USING a message class (Tool).
+        If `message_class` is None, then disable this agent from USING ALL tools.
+        Args:
+            message_class: The ToolMessage class to disable.
+                If None, disable all.
+        """
+        for t in self._get_tool_list(message_class):
+            self.llm_tools_usable.discard(t)
+            self.llm_functions_usable.discard(t)
+
+    def disable_message_use_except(self, message_class: Type[ToolMessage]) -> None:
+        """
+        Disable this agent from USING ALL messages EXCEPT a message class (Tool)
+        Args:
+            message_class: The only ToolMessage class to allow
+        """
+        request = message_class.__fields__["request"].default
+        to_remove = [r for r in self.llm_tools_usable if r != request]
+        for r in to_remove:
+            self.llm_tools_usable.discard(r)
+            self.llm_functions_usable.discard(r)
+
+    def _load_output_format(self, message: ChatDocument) -> None:
+        """
+        If set, attempts to parse a value of type `self.output_format` from the message
+        contents or any tool/function call and assigns it to `content_any`.
+        """
+        if self.output_format is not None:
+            any_succeeded = False
+            attempts: list[str | LLMFunctionCall] = [
+                message.content,
+            ]
+
+            if message.function_call is not None:
+                attempts.append(message.function_call)
+
+            if message.oai_tool_calls is not None:
+                attempts.extend(
+                    [
+                        c.function
+                        for c in message.oai_tool_calls
+                        if c.function is not None
+                    ]
+                )
+
+            for attempt in attempts:
+                try:
+                    if isinstance(attempt, str):
+                        content = json.loads(attempt)
+                    else:
+                        if not (
+                            issubclass(self.output_format, ToolMessage)
+                            and attempt.name
+                            == self.output_format.default_value("request")
+                        ):
+                            continue
+
+                        content = attempt.arguments
+
+                    content_any = self.output_format.parse_obj(content)
+
+                    if issubclass(self.output_format, PydanticWrapper):
+                        message.content_any = content_any.value  # type: ignore
+                    else:
+                        message.content_any = content_any
+                    any_succeeded = True
+                    break
+                except (ValidationError, json.JSONDecodeError):
+                    continue
+
+            if not any_succeeded:
+                self.disable_strict = True
+                logging.warning(
+                    """
+                    Validation error occured with strict output format enabled.
+                    Disabling strict mode.
+                    """
+                )
+
+    def get_tool_messages(
+        self,
+        msg: str | ChatDocument | None,
+        all_tools: bool = False,
+    ) -> List[ToolMessage]:
+        """
+        Extracts messages and tracks whether any errors occured. If strict mode
+        was enabled, disables it for the tool, else triggers strict recovery.
+        """
+        self.tool_error = False
+        try:
+            tools = super().get_tool_messages(msg, all_tools)
+        except ValidationError as ve:
+            tool_class = ve.model
+            if issubclass(tool_class, ToolMessage):
+                was_strict = (
+                    self.config.use_functions_api
+                    and self.config.use_tools_api
+                    and self._strict_mode_for_tool(tool_class)
+                )
+                # If the result of strict output for a tool using the
+                # OpenAI tools API fails to parse, we infer that the
+                # schema edits necessary for compatibility prevented
+                # adherence to the underlying `ToolMessage` schema and
+                # disable strict output for the tool
+                if was_strict:
+                    name = tool_class.default_value("request")
+                    self.disable_strict_tools_set.add(name)
+                    logging.warning(
+                        f"""
+                        Validation error occured with strict tool format.
+                        Disabling strict mode for the {name} tool.
+                        """
+                    )
+                else:
+                    # We will trigger the strict recovery mechanism to force
+                    # the LLM to correct its output, allowing us to parse
+                    self.tool_error = True
+
+            raise ve
+
+        return tools
+
+    def _get_any_tool_message(self, optional: bool = True) -> type[ToolMessage]:
+        """
+        Returns a `ToolMessage` which wraps all enabled tools, excluding those
+        where strict recovery is disabled. Used in strict recovery.
+        """
+        any_tool_type = Union[  # type: ignore
+            *(
+                self.llm_tools_map[t]
+                for t in self.llm_tools_usable
+                if t not in self.disable_strict_tools_set
+            )
+        ]
+        maybe_optional_type = Optional[any_tool_type] if optional else any_tool_type
+
+        class AnyTool(ToolMessage):
+            purpose: str = "To call a tool/function."
+            request: str = "tool_or_function"
+            tool: maybe_optional_type  # type: ignore
+
+            def response(self, agent: ChatAgent) -> None | str | ChatDocument:
+                # One-time use
+                agent.set_output_format(None)
+
+                if self.tool is None:
+                    return None
+
+                # As the ToolMessage schema accepts invalid
+                # `tool.request` values, reparse with the
+                # corresponding tool
+                request = self.tool.request
+                if request not in agent.llm_tools_map:
+                    return None
+                tool = agent.llm_tools_map[request].parse_raw(self.tool.to_json())
+
+                return agent.handle_tool_message(tool)
+
+            async def response_async(
+                self, agent: ChatAgent
+            ) -> None | str | ChatDocument:
+                # One-time use
+                agent.set_output_format(None)
+
+                if self.tool is None:
+                    return None
+
+                # As the ToolMessage schema accepts invalid
+                # `tool.request` values, reparse with the
+                # corresponding tool
+                request = self.tool.request
+                if request not in agent.llm_tools_map:
+                    return None
+                tool = agent.llm_tools_map[request].parse_raw(self.tool.to_json())
+
+                return await agent.handle_tool_message_async(tool)
+
+        return AnyTool
+
+    def _strict_recovery_instructions(
+        self,
+        tool_type: Optional[type[ToolMessage]] = None,
+        optional: bool = True,
+    ) -> str:
+        """Returns instructions for strict recovery."""
+        optional_instructions = (
+            (
+                "\n"
+                + """
+        If you did NOT intend to do so, `tool` should be null.
+        """
+            )
+            if optional
+            else ""
+        )
+        response_prefix = "If you intended to make such a call, r" if optional else "R"
+        instruction_prefix = "If you do so, b" if optional else "B"
+
+        schema_instructions = (
+            f"""
+        The schema for `tool_or_function` is as follows:
+        {tool_type.llm_function_schema(defaults=True, request=True).parameters}
+        """
+            if tool_type
+            else ""
+        )
+
+        return textwrap.dedent(
+            f"""
+        Your previous attempt to make a tool/function call appears to have failed.
+        {response_prefix}espond with your desired tool/function. Do so with the
+        `tool_or_function` tool/function where `tool` is set to your intended call.
+        {schema_instructions}
+
+        {instruction_prefix}e sure that your corrected call matches your intention
+        in your previous request. For any field with a default value which
+        you did not intend to override in your previous attempt, be sure
+        to set that field to its default value. {optional_instructions}
+        """
+        )
+
+    def truncate_message(
+        self,
+        idx: int,
+        tokens: int = 5,
+        warning: str = "...[Contents truncated!]",
+    ) -> LLMMessage:
+        """Truncate message at idx in msg history to `tokens` tokens"""
+        llm_msg = self.message_history[idx]
+        orig_content = llm_msg.content
+        new_content = (
+            self.parser.truncate_tokens(orig_content, tokens)
+            if self.parser is not None
+            else orig_content[: tokens * 4]  # approx truncation
+        )
+        llm_msg.content = new_content + "\n" + warning
+        return llm_msg
+
+    def _reduce_raw_tool_results(self, message: ChatDocument) -> None:
+        """
+        If message is the result of a ToolMessage that had
+        a `_max_retained_tokens` set to a non-None value, then we replace contents
+        with a placeholder message.
+        """
+        parent_message: ChatDocument | None = message.parent
+        tools = [] if parent_message is None else parent_message.tool_messages
+        truncate_tools = [t for t in tools if t._max_retained_tokens is not None]
+        limiting_tool = truncate_tools[0] if len(truncate_tools) > 0 else None
+        if limiting_tool is not None and limiting_tool._max_retained_tokens is not None:
+            tool_name = limiting_tool.default_value("request")
+            max_tokens: int = limiting_tool._max_retained_tokens
+            truncation_warning = f"""
+                The result of the {tool_name} tool were too large, 
+                and has been truncated to {max_tokens} tokens.
+                To obtain the full result, the tool needs to be re-used.
+            """
+            self.truncate_message(
+                message.metadata.msg_idx, max_tokens, truncation_warning
+            )
+
+    def llm_response(
+        self, message: Optional[str | ChatDocument] = None
+    ) -> Optional[ChatDocument]:
+        """
+        Respond to a single user message, appended to the message history,
+        in "chat" mode
+        Args:
+            message (str|ChatDocument): message or ChatDocument object to respond to.
+                If None, use the self.task_messages
+        Returns:
+            LLM response as a ChatDocument object
+        """
+        if self.llm is None:
+            return None
+
+        # If enabled and a tool error occurred, we recover by generating the tool in
+        # strict json mode
+        if (
+            self.tool_error
+            and self.output_format is None
+            and self._json_schema_available()
+            and self.config.strict_recovery
+        ):
+            AnyTool = self._get_any_tool_message()
+            self.set_output_format(
+                AnyTool,
+                force_tools=True,
+                use=True,
+                handle=True,
+                instructions=True,
+            )
+            recovery_message = self._strict_recovery_instructions(AnyTool)
+
+            if message is None:
+                message = recovery_message
+            elif isinstance(message, str):
+                message = message + recovery_message
+            else:
+                message.content = message.content + recovery_message
+
+            return self.llm_response(message)
+
+        hist, output_len = self._prep_llm_messages(message)
+        if len(hist) == 0:
+            return None
+        tool_choice = (
+            "auto"
+            if isinstance(message, str)
+            else (message.oai_tool_choice if message is not None else "auto")
+        )
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):
+            try:
+                response = self.llm_response_messages(hist, output_len, tool_choice)
+            except openai.BadRequestError as e:
+                if self.any_strict:
+                    self.disable_strict = True
+                    self.set_output_format(None)
+                    logging.warning(
+                        f"""
+                        OpenAI BadRequestError raised with strict mode enabled.
+                        Message: {e.message}
+                        Disabling strict mode and retrying.
+                        """
+                    )
+                    return self.llm_response(message)
+                else:
+                    raise e
+        self.message_history.extend(ChatDocument.to_LLMMessage(response))
+        response.metadata.msg_idx = len(self.message_history) - 1
+        response.metadata.agent_id = self.id
+        if isinstance(message, ChatDocument):
+            self._reduce_raw_tool_results(message)
+        # Preserve trail of tool_ids for OpenAI Assistant fn-calls
+        response.metadata.tool_ids = (
+            []
+            if isinstance(message, str)
+            else message.metadata.tool_ids if message is not None else []
+        )
+
+        return response
+
+    async def llm_response_async(
+        self, message: Optional[str | ChatDocument] = None
+    ) -> Optional[ChatDocument]:
+        """
+        Async version of `llm_response`. See there for details.
+        """
+        if self.llm is None:
+            return None
+
+        # If enabled and a tool error occurred, we recover by generating the tool in
+        # strict json mode
+        if (
+            self.tool_error
+            and self.output_format is None
+            and self._json_schema_available()
+            and self.config.strict_recovery
+        ):
+            AnyTool = self._get_any_tool_message()
+            self.set_output_format(
+                AnyTool,
+                force_tools=True,
+                use=True,
+                handle=True,
+                instructions=True,
+            )
+            recovery_message = self._strict_recovery_instructions(AnyTool)
+
+            if message is None:
+                message = recovery_message
+            elif isinstance(message, str):
+                message = message + recovery_message
+            else:
+                message.content = message.content + recovery_message
+
+            return self.llm_response(message)
+
+        hist, output_len = self._prep_llm_messages(message)
+        if len(hist) == 0:
+            return None
+        tool_choice = (
+            "auto"
+            if isinstance(message, str)
+            else (message.oai_tool_choice if message is not None else "auto")
+        )
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):
+            try:
+                response = await self.llm_response_messages_async(
+                    hist, output_len, tool_choice
+                )
+            except openai.BadRequestError as e:
+                if self.any_strict:
+                    self.disable_strict = True
+                    self.set_output_format(None)
+                    logging.warning(
+                        f"""
+                        OpenAI BadRequestError raised with strict mode enabled.
+                        Message: {e.message}
+                        Disabling strict mode and retrying.
+                        """
+                    )
+                    return await self.llm_response_async(message)
+                else:
+                    raise e
+        self.message_history.extend(ChatDocument.to_LLMMessage(response))
+        response.metadata.msg_idx = len(self.message_history) - 1
+        response.metadata.agent_id = self.id
+        if isinstance(message, ChatDocument):
+            self._reduce_raw_tool_results(message)
+        # Preserve trail of tool_ids for OpenAI Assistant fn-calls
+        response.metadata.tool_ids = (
+            []
+            if isinstance(message, str)
+            else message.metadata.tool_ids if message is not None else []
+        )
+
+        return response
+
+    def init_message_history(self) -> None:
+        """
+        Initialize the message history with the system message and user message
+        """
+        self.message_history = [self._create_system_and_tools_message()]
+        if self.user_message:
+            self.message_history.append(
+                LLMMessage(role=Role.USER, content=self.user_message)
+            )
+
+    def _prep_llm_messages(
+        self,
+        message: Optional[str | ChatDocument] = None,
+        truncate: bool = True,
+    ) -> Tuple[List[LLMMessage], int]:
+        """
+        Prepare messages to be sent to self.llm_response_messages,
+            which is the main method that calls the LLM API to get a response.
+
+        Returns:
+            Tuple[List[LLMMessage], int]: (messages, output_len)
+                messages = Full list of messages to send
+                output_len = max expected number of tokens in response
+        """
+
+        if (
+            not self.llm_can_respond(message)
+            or self.config.llm is None
+            or self.llm is None
+        ):
+            return [], 0
+
+        if message is None and len(self.message_history) > 0:
+            # this means agent has been used to get LLM response already,
+            # and so the last message is an "assistant" response.
+            # We delete this last assistant response and re-generate it.
+            self.clear_history(-1)
+            logger.warning(
+                "Re-generating the last assistant response since message is None"
+            )
+
+        if len(self.message_history) == 0:
+            # initial messages have not yet been loaded, so load them
+            self.init_message_history()
+
+            # for debugging, show the initial message history
+            if settings.debug:
+                print(
+                    f"""
+                [grey37]LLM Initial Msg History:
+                {escape(self.message_history_str())}
+                [/grey37]
+                """
+                )
+        else:
+            assert self.message_history[0].role == Role.SYSTEM
+            # update the system message with the latest tool instructions
+            self.message_history[0] = self._create_system_and_tools_message()
+
+        if message is not None:
+            if (
+                isinstance(message, str)
+                or message.id() != self.message_history[-1].chat_document_id
+            ):
+                # either the message is a str, or it is a fresh ChatDocument
+                # different from the last message in the history
+                llm_msgs = ChatDocument.to_LLMMessage(message, self.oai_tool_calls)
+                # LLM only responds to the content, so only those msgs with
+                # non-empty content should be kept
+                llm_msgs = [m for m in llm_msgs if m.content.strip() != ""]
+                if len(llm_msgs) == 0:
+                    return [], 0
+                # process tools if any
+                done_tools = [m.tool_call_id for m in llm_msgs if m.role == Role.TOOL]
+                self.oai_tool_calls = [
+                    t for t in self.oai_tool_calls if t.id not in done_tools
+                ]
+                self.message_history.extend(llm_msgs)
+
+        hist = self.message_history
+        output_len = self.config.llm.max_output_tokens
+        if (
+            truncate
+            and self.chat_num_tokens(hist)
+            > self.llm.chat_context_length() - self.config.llm.max_output_tokens
+        ):
+            # chat + output > max context length,
+            # so first try to shorten requested output len to fit.
+            output_len = self.llm.chat_context_length() - self.chat_num_tokens(hist)
+            if output_len < self.config.llm.min_output_tokens:
+                # unacceptably small output len, so drop early parts of conv history
+                # if output_len is still too long, then drop early parts of conv history
+                # TODO we should really be doing summarization or other types of
+                #   prompt-size reduction
+                while (
+                    self.chat_num_tokens(hist)
+                    > self.llm.chat_context_length() - self.config.llm.min_output_tokens
+                ):
+                    # try dropping early parts of conv history
+                    # TODO we should really be doing summarization or other types of
+                    #   prompt-size reduction
+                    if len(hist) <= 2:
+                        # We want to preserve the first message (typically system msg)
+                        # and last message (user msg).
+                        raise ValueError(
+                            """
+                        The message history is longer than the max chat context 
+                        length allowed, and we have run out of messages to drop.
+                        HINT: In your `OpenAIGPTConfig` object, try increasing
+                        `chat_context_length` or decreasing `max_output_tokens`.
+                        """
+                        )
+                    # drop the second message, i.e. first msg after the sys msg
+                    # (typically user msg).
+                    ChatDocument.delete_id(hist[1].chat_document_id)
+                    hist = hist[:1] + hist[2:]
+
+                if len(hist) < len(self.message_history):
+                    msg_tokens = self.chat_num_tokens()
+                    logger.warning(
+                        f"""
+                    Chat Model context length is {self.llm.chat_context_length()} 
+                    tokens, but the current message history is {msg_tokens} tokens long.
+                    Dropped the {len(self.message_history) - len(hist)} messages
+                    from early in the conversation history so that history token 
+                    length is {self.chat_num_tokens(hist)}.
+                    This may still not be low enough to allow minimum output length of 
+                    {self.config.llm.min_output_tokens} tokens.
+                    """
+                    )
+
+        if output_len < 0:
+            raise ValueError(
+                f"""
+                Tried to shorten prompt history for chat mode 
+                but even after dropping all messages except system msg and last (
+                user) msg, the history token len {self.chat_num_tokens(hist)} is longer 
+                than the model's max context length {self.llm.chat_context_length()}.
+                Please try shortening the system msg or user prompts.
+                """
+            )
+        if output_len < self.config.llm.min_output_tokens:
+            logger.warning(
+                f"""
+                Tried to shorten prompt history for chat mode 
+                but the feasible output length {output_len} is still
+                less than the minimum output length {self.config.llm.min_output_tokens}.
+                Your chat history is too long for this model, 
+                and the response may be truncated.
+                """
+            )
+        if isinstance(message, ChatDocument):
+            # record the position of the corresponding LLMMessage in
+            # the message_history
+            message.metadata.msg_idx = len(hist) - 1
+            message.metadata.agent_id = self.id
+
+        return hist, output_len
+
+    def _function_args(
+        self,
+    ) -> Tuple[
+        Optional[List[LLMFunctionSpec]],
+        str | Dict[str, str],
+        Optional[List[OpenAIToolSpec]],
+        Optional[Dict[str, Dict[str, str] | str]],
+        Optional[OpenAIJsonSchemaSpec],
+    ]:
+        """
+        Get function/tool spec/output format arguments for
+        OpenAI-compatible LLM API call
+        """
+        functions: Optional[List[LLMFunctionSpec]] = None
+        fun_call: str | Dict[str, str] = "none"
+        tools: Optional[List[OpenAIToolSpec]] = None
+        force_tool: Optional[Dict[str, Dict[str, str] | str]] = None
+        self.any_strict = False
+        if self.config.use_functions_api and len(self.llm_functions_usable) > 0:
+            if not self.config.use_tools_api:
+                functions = [
+                    self.llm_functions_map[f] for f in self.llm_functions_usable
+                ]
+                fun_call = (
+                    "auto"
+                    if self.llm_function_force is None
+                    else self.llm_function_force
+                )
+            else:
+
+                def to_maybe_strict_spec(function: str) -> OpenAIToolSpec:
+                    spec = self.llm_functions_map[function]
+                    strict = self._strict_mode_for_tool(function)
+                    if strict:
+                        self.any_strict = True
+                        strict_spec = copy.deepcopy(spec)
+                        format_schema_for_strict(strict_spec.parameters)
+                    else:
+                        strict_spec = spec
+
+                    return OpenAIToolSpec(
+                        type="function",
+                        strict=strict,
+                        function=strict_spec,
+                    )
+
+                tools = [to_maybe_strict_spec(f) for f in self.llm_functions_usable]
+                force_tool = (
+                    None
+                    if self.llm_function_force is None
+                    else {
+                        "type": "function",
+                        "function": {"name": self.llm_function_force["name"]},
+                    }
+                )
+        output_format = None
+        if self.output_format is not None and self._json_schema_available():
+            self.any_strict = True
+            if issubclass(self.output_format, ToolMessage) and not issubclass(
+                self.output_format, XMLToolMessage
+            ):
+                spec = self.output_format.llm_function_schema(
+                    request=True,
+                    defaults=self.config.output_format_include_defaults,
+                )
+                format_schema_for_strict(spec.parameters)
+
+                output_format = OpenAIJsonSchemaSpec(
+                    # We always require that outputs strictly match the schema
+                    strict=True,
+                    function=spec,
+                )
+            elif issubclass(self.output_format, BaseModel):
+                param_spec = self.output_format.schema()
+                format_schema_for_strict(param_spec)
+
+                output_format = OpenAIJsonSchemaSpec(
+                    # We always require that outputs strictly match the schema
+                    strict=True,
+                    function=LLMFunctionSpec(
+                        name="json_output",
+                        description="Strict Json output format.",
+                        parameters=param_spec,
+                    ),
+                )
+
+        return functions, fun_call, tools, force_tool, output_format
+
+    def llm_response_messages(
+        self,
+        messages: List[LLMMessage],
+        output_len: Optional[int] = None,
+        tool_choice: ToolChoiceTypes | Dict[str, str | Dict[str, str]] = "auto",
+    ) -> ChatDocument:
+        """
+        Respond to a series of messages, e.g. with OpenAI ChatCompletion
+        Args:
+            messages: seq of messages (with role, content fields) sent to LLM
+            output_len: max number of tokens expected in response.
+                    If None, use the LLM's default max_output_tokens.
+        Returns:
+            Document (i.e. with fields "content", "metadata")
+        """
+        assert self.config.llm is not None and self.llm is not None
+        output_len = output_len or self.config.llm.max_output_tokens
+        streamer = noop_fn
+        if self.llm.get_stream():
+            streamer = self.callbacks.start_llm_stream()
+        self.llm.config.streamer = streamer
+        with ExitStack() as stack:  # for conditionally using rich spinner
+            if not self.llm.get_stream() and not settings.quiet:
+                # show rich spinner only if not streaming!
+                # (Why? b/c the intent of showing a spinner is to "show progress",
+                # and we don't need to do that when streaming, since
+                # streaming output already shows progress.)
+                cm = status(
+                    "LLM responding to messages...",
+                    log_if_quiet=False,
+                )
+                stack.enter_context(cm)
+            if self.llm.get_stream() and not settings.quiet:
+                console.print(f"[green]{self.indent}", end="")
+            functions, fun_call, tools, force_tool, output_format = (
+                self._function_args()
+            )
+            assert self.llm is not None
+            response = self.llm.chat(
+                messages,
+                output_len,
+                tools=tools,
+                tool_choice=force_tool or tool_choice,
+                functions=functions,
+                function_call=fun_call,
+                response_format=output_format,
+            )
+        if self.llm.get_stream():
+            self.callbacks.finish_llm_stream(
+                content=str(response),
+                is_tool=self.has_tool_message_attempt(
+                    ChatDocument.from_LLMResponse(response, displayed=True),
+                ),
+            )
+        self.llm.config.streamer = noop_fn
+        if response.cached:
+            self.callbacks.cancel_llm_stream()
+        self._render_llm_response(response)
+        self.update_token_usage(
+            response,  # .usage attrib is updated!
+            messages,
+            self.llm.get_stream(),
+            chat=True,
+            print_response_stats=self.config.show_stats and not settings.quiet,
+        )
+        chat_doc = ChatDocument.from_LLMResponse(response, displayed=True)
+        self.oai_tool_calls = response.oai_tool_calls or []
+        self.oai_tool_id2call.update(
+            {t.id: t for t in self.oai_tool_calls if t.id is not None}
+        )
+
+        # If using strict output format, parse the output JSON
+        self._load_output_format(chat_doc)
+
+        return chat_doc
+
+    async def llm_response_messages_async(
+        self,
+        messages: List[LLMMessage],
+        output_len: Optional[int] = None,
+        tool_choice: ToolChoiceTypes | Dict[str, str | Dict[str, str]] = "auto",
+    ) -> ChatDocument:
+        """
+        Async version of `llm_response_messages`. See there for details.
+        """
+        assert self.config.llm is not None and self.llm is not None
+        output_len = output_len or self.config.llm.max_output_tokens
+        functions, fun_call, tools, force_tool, output_format = self._function_args()
+        assert self.llm is not None
+
+        streamer_async = async_noop_fn
+        if self.llm.get_stream():
+            streamer_async = await self.callbacks.start_llm_stream_async()
+        self.llm.config.streamer_async = streamer_async
+
+        response = await self.llm.achat(
+            messages,
+            output_len,
+            tools=tools,
+            tool_choice=force_tool or tool_choice,
+            functions=functions,
+            function_call=fun_call,
+            response_format=output_format,
+        )
+        if self.llm.get_stream():
+            self.callbacks.finish_llm_stream(
+                content=str(response),
+                is_tool=self.has_tool_message_attempt(
+                    ChatDocument.from_LLMResponse(response, displayed=True),
+                ),
+            )
+        self.llm.config.streamer_async = async_noop_fn
+        if response.cached:
+            self.callbacks.cancel_llm_stream()
+        self._render_llm_response(response)
+        self.update_token_usage(
+            response,  # .usage attrib is updated!
+            messages,
+            self.llm.get_stream(),
+            chat=True,
+            print_response_stats=self.config.show_stats and not settings.quiet,
+        )
+        chat_doc = ChatDocument.from_LLMResponse(response, displayed=True)
+        self.oai_tool_calls = response.oai_tool_calls or []
+        self.oai_tool_id2call.update(
+            {t.id: t for t in self.oai_tool_calls if t.id is not None}
+        )
+
+        # If using strict output format, parse the output JSON
+        self._load_output_format(chat_doc)
+
+        return chat_doc
+
+    def _render_llm_response(
+        self, response: ChatDocument | LLMResponse, citation_only: bool = False
+    ) -> None:
+        is_cached = (
+            response.cached
+            if isinstance(response, LLMResponse)
+            else response.metadata.cached
+        )
+        if self.llm is None:
+            return
+        if not citation_only and (not self.llm.get_stream() or is_cached):
+            # We would have already displayed the msg "live" ONLY if
+            # streaming was enabled, AND we did not find a cached response.
+            # If we are here, it means the response has not yet been displayed.
+            cached = f"[red]{self.indent}(cached)[/red]" if is_cached else ""
+            chat_doc = (
+                response
+                if isinstance(response, ChatDocument)
+                else ChatDocument.from_LLMResponse(response, displayed=True)
+            )
+            # TODO: prepend TOOL: or OAI-TOOL: if it's a tool-call
+            if not settings.quiet:
+                print(cached + "[green]" + escape(str(response)))
+            self.callbacks.show_llm_response(
+                content=str(response),
+                is_tool=self.has_tool_message_attempt(chat_doc),
+                cached=is_cached,
+            )
+        if isinstance(response, LLMResponse):
+            # we are in the context immediately after an LLM responded,
+            # we won't have citations yet, so we're done
+            return
+        if response.metadata.has_citation:
+            if not settings.quiet:
+                print(
+                    "[grey37]SOURCES:\n"
+                    + escape(response.metadata.source)
+                    + "[/grey37]"
+                )
+            self.callbacks.show_llm_response(
+                content=str(response.metadata.source),
+                is_tool=False,
+                cached=False,
+                language="text",
+            )
+
+    def _llm_response_temp_context(self, message: str, prompt: str) -> ChatDocument:
+        """
+        Get LLM response to `prompt` (which presumably includes the `message`
+        somewhere, along with possible large "context" passages),
+        but only include `message` as the USER message, and not the
+        full `prompt`, in the message history.
+        Args:
+            message: the original, relatively short, user request or query
+            prompt: the full prompt potentially containing `message` plus context
+
+        Returns:
+            Document object containing the response.
+        """
+        # we explicitly call THIS class's respond method,
+        # not a derived class's (or else there would be infinite recursion!)
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):  # type: ignore
+            answer_doc = cast(ChatDocument, ChatAgent.llm_response(self, prompt))
+        self.update_last_message(message, role=Role.USER)
+        return answer_doc
+
+    async def _llm_response_temp_context_async(
+        self, message: str, prompt: str
+    ) -> ChatDocument:
+        """
+        Async version of `_llm_response_temp_context`. See there for details.
+        """
+        # we explicitly call THIS class's respond method,
+        # not a derived class's (or else there would be infinite recursion!)
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):  # type: ignore
+            answer_doc = cast(
+                ChatDocument,
+                await ChatAgent.llm_response_async(self, prompt),
+            )
+        self.update_last_message(message, role=Role.USER)
+        return answer_doc
+
+    def llm_response_forget(self, message: str) -> ChatDocument:
+        """
+        LLM Response to single message, and restore message_history.
+        In effect a "one-off" message & response that leaves agent
+        message history state intact.
+
+        Args:
+            message (str): user message
+
+        Returns:
+            A Document object with the response.
+
+        """
+        # explicitly call THIS class's respond method,
+        # not a derived class's (or else there would be infinite recursion!)
+        n_msgs = len(self.message_history)
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):  # type: ignore
+            response = cast(ChatDocument, ChatAgent.llm_response(self, message))
+        # If there is a response, then we will have two additional
+        # messages in the message history, i.e. the user message and the
+        # assistant response. We want to (carefully) remove these two messages.
+        if len(self.message_history) > n_msgs:
+            msg = self.message_history.pop()
+            self._drop_msg_update_tool_calls(msg)
+
+        if len(self.message_history) > n_msgs:
+            msg = self.message_history.pop()
+            self._drop_msg_update_tool_calls(msg)
+
+        # If using strict output format, parse the output JSON
+        self._load_output_format(response)
+
+        return response
+
+    async def llm_response_forget_async(self, message: str) -> ChatDocument:
+        """
+        Async version of `llm_response_forget`. See there for details.
+        """
+        # explicitly call THIS class's respond method,
+        # not a derived class's (or else there would be infinite recursion!)
+        n_msgs = len(self.message_history)
+        with StreamingIfAllowed(self.llm, self.llm.get_stream()):  # type: ignore
+            response = cast(
+                ChatDocument, await ChatAgent.llm_response_async(self, message)
+            )
+        # If there is a response, then we will have two additional
+        # messages in the message history, i.e. the user message and the
+        # assistant response. We want to (carefully) remove these two messages.
+        if len(self.message_history) > n_msgs:
+            msg = self.message_history.pop()
+            self._drop_msg_update_tool_calls(msg)
+
+        if len(self.message_history) > n_msgs:
+            msg = self.message_history.pop()
+            self._drop_msg_update_tool_calls(msg)
+        return response
+
+    def chat_num_tokens(self, messages: Optional[List[LLMMessage]] = None) -> int:
+        """
+        Total number of tokens in the message history so far.
+
+        Args:
+            messages: if provided, compute the number of tokens in this list of
+                messages, rather than the current message history.
+        Returns:
+            int: number of tokens in message history
+        """
+        if self.parser is None:
+            raise ValueError(
+                "ChatAgent.parser is None. "
+                "You must set ChatAgent.parser "
+                "before calling chat_num_tokens()."
+            )
+        hist = messages if messages is not None else self.message_history
+        return sum([self.parser.num_tokens(m.content) for m in hist])
+
+    def message_history_str(self, i: Optional[int] = None) -> str:
+        """
+        Return a string representation of the message history
+        Args:
+            i: if provided, return only the i-th message when i is postive,
+                or last k messages when i = -k.
+        Returns:
+        """
+        if i is None:
+            return "\n".join([str(m) for m in self.message_history])
+        elif i > 0:
+            return str(self.message_history[i])
+        else:
+            return "\n".join([str(m) for m in self.message_history[i:]])