langroid 0.1.60__tar.gz → 0.1.62__tar.gz

{langroid-0.1.60 → langroid-0.1.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.60
+Version: 0.1.62
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -138,7 +138,7 @@ for ideas on what to contribute.
 <summary> <b>:fire: Updates/Releases</b></summary>
 
 - **Sep 2023:**
-  - **Use with local LLama Models:** see tutorial [here](https://langroid.github.io/langroid/tutorials/llama/)
+  - **Use with local LLama Models:** see tutorial [here](https://langroid.github.io/langroid/blog/2023/09/14/using-langroid-with-local-llms/)
   - **Langroid Blog/Newsletter Launched!**: First post is [here](https://substack.com/notes/post/p-136704592) -- Please subscribe to stay updated. 
   - **0.1.56:** Support Azure OpenAI. 
   - **0.1.55:** Improved [`SQLChatAgent`](https://github.com/langroid/langroid/blob/main/langroid/agent/special/sql/sql_chat_agent.py) that efficiently retrieves relevant schema info when translating natural language to SQL.  

{langroid-0.1.60 → langroid-0.1.62}/README.md

@@ -67,7 +67,7 @@ for ideas on what to contribute.
 <summary> <b>:fire: Updates/Releases</b></summary>
 
 - **Sep 2023:**
-  - **Use with local LLama Models:** see tutorial [here](https://langroid.github.io/langroid/tutorials/llama/)
+  - **Use with local LLama Models:** see tutorial [here](https://langroid.github.io/langroid/blog/2023/09/14/using-langroid-with-local-llms/)
   - **Langroid Blog/Newsletter Launched!**: First post is [here](https://substack.com/notes/post/p-136704592) -- Please subscribe to stay updated. 
   - **0.1.56:** Support Azure OpenAI. 
   - **0.1.55:** Improved [`SQLChatAgent`](https://github.com/langroid/langroid/blob/main/langroid/agent/special/sql/sql_chat_agent.py) that efficiently retrieves relevant schema info when translating natural language to SQL.  

{langroid-0.1.60 → langroid-0.1.62}/langroid/agent/base.py

@@ -212,7 +212,7 @@ class Agent(ABC):
     def json_format_rules(self) -> str:
         """
         Specification of JSON formatting rules, based on the currently enabled
-        message classes.
+        usable `ToolMessage`s
 
         Returns:
             str: formatting rules
@@ -221,7 +221,7 @@ class Agent(ABC):
         if len(enabled_classes) == 0:
             return "You can ask questions in natural language."
 
-        json_conditions = "\n\n".join(
+        json_instructions = "\n\n".join(
             [
                 str(msg_cls.default_value("request"))
                 + ":\n"
@@ -230,7 +230,53 @@ class Agent(ABC):
                 if msg_cls.default_value("request") in self.llm_tools_usable
             ]
         )
-        return json_conditions
+        return textwrap.dedent(
+            f"""
+            === ALL AVAILABLE TOOLS and THEIR JSON FORMAT INSTRUCTIONS ===
+            You have access to the following TOOLS to accomplish your task:
+
+            {json_instructions}
+            
+            {INSTRUCTION}            
+            ----------------------------
+            """.lstrip()
+        )
+
+    def tool_instructions(self) -> str:
+        """
+        Instructions (defined via `instructions` classmethod in a
+        ToolMessage class) for currently enabled and usable Tools.
+
+        Returns:
+            str: concatenation of instructions for all usable tools
+        """
+        enabled_classes: List[Type[ToolMessage]] = list(self.llm_tools_map.values())
+        if len(enabled_classes) == 0:
+            return ""
+        instructions = []
+        for msg_cls in enabled_classes:
+            if (
+                hasattr(msg_cls, "instructions")
+                and inspect.ismethod(msg_cls.instructions)
+                and msg_cls.default_value("request") in self.llm_tools_usable
+            ):
+                instructions.append(
+                    textwrap.dedent(
+                        f"""
+                        {msg_cls.default_value("request")}:
+                        {msg_cls.instructions()}
+                        """.lstrip()
+                    )
+                )
+        if len(instructions) == 0:
+            return ""
+        instructions_str = "\n\n".join(instructions)
+        return textwrap.dedent(
+            f"""
+            === GUIDELINES ON SOME TOOLS/FUNCTIONS USAGE ===
+            {instructions_str}
+            """.lstrip()
+        )
 
     def sample_multi_round_dialog(self) -> str:
         """
@@ -247,26 +293,6 @@ class Agent(ABC):
         ]
         return "\n\n".join(sample_convo)
 
-    def json_tool_format_instructions(self) -> str:
-        """
-        Generate a string containing instructions to the LLM on when to format
-        requests/questions as JSON, based on the currently enabled message classes.
-
-        Returns:
-            str: The instructions string.
-        """
-        format_rules = self.json_format_rules()
-
-        return f"""
-        You have access to the following TOOLS to accomplish your task:
-        TOOLS AVAILABLE:
-        {format_rules}
-        
-        {INSTRUCTION}
-        
-        Now start, and be concise!                 
-        """
-
     def agent_response(
         self,
         msg: Optional[str | ChatDocument] = None,
@@ -339,11 +365,18 @@ class Agent(ABC):
         if not user_msg:
             return None
         else:
+            if user_msg.startswith("SYSTEM"):
+                user_msg = user_msg[6:].strip()
+                source = Entity.SYSTEM
+                sender = Entity.SYSTEM
+            else:
+                source = Entity.USER
+                sender = Entity.USER
             return ChatDocument(
                 content=user_msg,
                 metadata=DocMetaData(
-                    source=Entity.USER,
-                    sender=Entity.USER,
+                    source=source,
+                    sender=sender,
                 ),
             )
 
@@ -434,7 +467,12 @@ class Agent(ABC):
             console.print(f"[green]{self.indent}", end="")
             print("[green]" + response.message)
             displayed = True
-        self.update_token_usage(response, prompt, self.llm.get_stream())
+        self.update_token_usage(
+            response,
+            prompt,
+            self.llm.get_stream(),
+            print_response_stats=settings.debug,
+        )
         return ChatDocument.from_LLMResponse(response, displayed)
 
     def get_tool_messages(self, msg: str | ChatDocument) -> List[ToolMessage]:
@@ -619,8 +657,39 @@ class Agent(ABC):
         else:
             return sum([self.parser.num_tokens(m.content) for m in prompt])
 
+    def _print_response_stats(
+        self, chat_length: int, tot_cost: float, response: LLMResponse
+    ) -> None:
+        """
+        Printing LLM response stats.
+
+        Args:
+            chat_length (int): number of messages in the chat
+            tot_cost (float): total cost of the chat so far
+            response (LLMResponse): LLMResponse object
+        """
+        if response.usage:
+            in_tokens = response.usage.prompt_tokens
+            out_tokens = response.usage.completion_tokens
+            llm_response_cost = format(response.usage.cost, ".4f")
+            cumul_cost = format(tot_cost, ".4f")
+            assert isinstance(self.llm, LanguageModel)
+            context_length = self.llm.chat_context_length()
+            max_out = self.config.llm.max_output_tokens
+            print(
+                f"{self.indent}[bold]Stats:[/bold] [magenta] N_MSG={chat_length}, "
+                f"TOKENS: in={in_tokens}, out={out_tokens}, "
+                f"max={max_out}, ctx={context_length}, "
+                f"COST: now=${llm_response_cost}, cumul=${cumul_cost}[/magenta]"
+                ""
+            )
+
     def update_token_usage(
-        self, response: LLMResponse, prompt: str | List[LLMMessage], stream: bool
+        self,
+        response: LLMResponse,
+        prompt: str | List[LLMMessage],
+        stream: bool,
+        print_response_stats: bool = True,
     ) -> None:
         """
         Updates `response.usage` obj (token usage and cost fields).the usage memebr
@@ -653,20 +722,15 @@ class Agent(ABC):
                     cost=cost,
                 )
 
-            if settings.debug and response.usage is not None:
-                print(
-                    textwrap.dedent(
-                        f"""
-                        Stream: {stream}
-                        prompt_tokens: {response.usage.prompt_tokens}
-                        completion_tokens: {response.usage.completion_tokens}
-                        """.lstrip()
-                    )
-                )
             # update total counters
             if response.usage is not None:
                 self.total_llm_token_cost += response.usage.cost
                 self.total_llm_token_usage += response.usage.total_tokens
+                chat_length = 1 if isinstance(prompt, str) else len(prompt)
+                if print_response_stats:
+                    self._print_response_stats(
+                        chat_length, self.total_llm_token_cost, response
+                    )
 
     def compute_token_cost(self, prompt: int, completion: int) -> float:
         price = cast(LanguageModel, self.llm).chat_cost()

{langroid-0.1.60 → langroid-0.1.62}/langroid/agent/chat_agent.py

@@ -1,5 +1,5 @@
-import inspect
 import logging
+import textwrap
 from contextlib import ExitStack
 from typing import Dict, List, Optional, Set, Type, cast, no_type_check
 
@@ -28,8 +28,10 @@ class ChatAgentConfig(AgentConfig):
     Configuration for ChatAgent
     Attributes:
         system_message: system message to include in message sequence
-             (typically defines role and task of agent)
-        user_message: user message to include in message sequence
+             (typically defines role and task of agent).
+             Used only if `task` is not specified in the constructor.
+        user_message: user message to include in message sequence.
+             Used only if `task` is not specified in the constructor.
         use_tools: whether to use our own ToolMessages mechanism
         use_functions_api: whether to use functions native to the LLM API
                 (e.g. OpenAI's `function_call` mechanism)
@@ -70,23 +72,53 @@ class ChatAgent(Agent):
         self.config: ChatAgentConfig = config
         self.message_history: List[LLMMessage] = []
         self.tool_instructions_added: bool = False
-        # system-level instructions for using tools/functions
+        # An agent's "task" is defined by a system msg and an optional user msg;
+        # These are "priming" messages that kick off the agent's conversation.
+        self.system_message: str = self.config.system_message
+        self.user_message: str | None = self.config.user_message
+
+        if task is not None:
+            # if task contains a system msg, we override the config system msg
+            if len(task) > 0 and task[0].role == Role.SYSTEM:
+                self.system_message = task[0].content
+            # if task contains a user msg, we override the config user msg
+            if len(task) > 1 and task[1].role == Role.USER:
+                self.user_message = task[1].content
+
+        # system-level instructions for using tools/functions:
+        # We maintain these as tools/functions are enabled/disabled,
+        # and whenever an LLM response is sought, these are used to
+        # recreate the system message (via `_create_system_and_tools_message`)
+        # each time, so it reflects the current set of enabled tools/functions.
+        # (a) these are general instructions on using certain tools/functions,
+        #   if they are specified in a ToolMessage class as a classmethod `instructions`
         self.system_tool_instructions: str = ""
+        # (b) these are only for the builtin in Langroid TOOLS mechanism:
+        self.system_json_tool_instructions: str = ""
+
         self.llm_functions_map: Dict[str, LLMFunctionSpec] = {}
         self.llm_functions_handled: Set[str] = set()
         self.llm_functions_usable: Set[str] = set()
         self.llm_function_force: Optional[Dict[str, str]] = None
 
-        priming_messages = task
-        if priming_messages is None:
-            priming_messages = [
-                LLMMessage(role=Role.SYSTEM, content=config.system_message),
-            ]
-            if config.user_message:
-                priming_messages.append(
-                    LLMMessage(role=Role.USER, content=config.user_message)
-                )
-        self.task_messages = priming_messages
+    def set_system_message(self, msg: str) -> None:
+        self.system_message = msg
+
+    def set_user_message(self, msg: str) -> None:
+        self.user_message = msg
+
+    @property
+    def task_messages(self) -> List[LLMMessage]:
+        """
+        The task messages are the initial messages that define the task
+        of the agent. There will be at least a system message plus possibly a user msg.
+        Returns:
+            List[LLMMessage]: the task messages
+        """
+        msgs = [self._create_system_and_tools_message()]
+        if self.user_message:
+            msgs.append(LLMMessage(role=Role.USER, content=self.user_message))
+        return msgs
 
     def clear_history(self, start: int = -2) -> None:
         """
@@ -122,31 +154,11 @@ class ChatAgent(Agent):
         Args:
             message (str): system message
         """
-        if len(self.message_history) > 0:
-            if self.message_history[0].role == Role.SYSTEM:
-                self.message_history[0].content = (
-                    self.message_history[0].content + "\n\n" + message
-                )
-        else:
-            if self.task_messages[0].role == Role.SYSTEM:
-                self.task_messages[0].content = (
-                    self.task_messages[0].content + "\n\n" + message
-                )
-
-    def add_user_message(self, message: str) -> None:
-        """
-        Add a user message to the message history.
-        Args:
-            message (str): user message
-        """
-        if len(self.message_history) > 0:
-            self.message_history.append(LLMMessage(role=Role.USER, content=message))
-        else:
-            self.task_messages.append(LLMMessage(role=Role.USER, content=message))
+        self.system_message += "\n\n" + message
 
     def update_last_message(self, message: str, role: str = Role.USER) -> None:
         """
-        Update the last message with role `role` in the message history.
+        Update the last message that has role `role` in the message history.
         Useful when we want to replace a long user prompt, that may contain context
         documents plus a question, with just the question.
         Args:
@@ -161,6 +173,33 @@ class ChatAgent(Agent):
                 self.message_history[i].content = message
                 break
 
+    def _create_system_and_tools_message(self) -> LLMMessage:
+        """
+        (Re-)Create the system message for the LLM of the agent,
+        taking into account any tool instructions that have been added
+        after the agent was initialized.
+
+        The system message will consist of:
+        (a) the system message from the `task` arg in constructor, if any,
+            otherwise the default system message from the config
+        (b) the system tool instructions, if any
+        (c) the system json tool instructions, if any
+
+        Returns:
+            LLMMessage object
+        """
+        content = textwrap.dedent(
+            f"""
+            {self.system_message}
+            
+            {self.system_tool_instructions}
+            
+            {self.system_json_tool_instructions}
+            
+            """.lstrip()
+        )
+        return LLMMessage(role=Role.SYSTEM, content=content)
+
     def enable_message(
         self,
         message_class: Optional[Type[ToolMessage]],
@@ -202,15 +241,7 @@ class ChatAgent(Agent):
                 self.llm_function_force = dict(name=request)
             else:
                 self.llm_function_force = None
-            if hasattr(message_class, "instructions") and inspect.ismethod(
-                message_class.instructions
-            ):
-                # save tool instructions so that when LLM is ready
-                # to respond (i.e. either when a wrapping Task is started,
-                # or when the LLM is directly queried),
-                # we can append to system msg.
-                self.system_tool_instructions += "\n\n" + message_class.instructions()
-        n_usable_tools = len(self.llm_tools_usable)
+
         for t in tools:
             if handle:
                 self.llm_tools_handled.add(t)
@@ -226,12 +257,10 @@ class ChatAgent(Agent):
                 self.llm_tools_usable.discard(t)
                 self.llm_functions_usable.discard(t)
 
-        # TODO we should do this only on demand when we actually are
-        # ready to send the instructions.
-        # But for now leave as is.
-        if len(self.llm_tools_usable) != n_usable_tools and self.config.use_tools:
-            # Update JSON format instructions if the set of usable tools has changed
-            self.update_json_tool_instructions()
+        # Set tool instructions and JSON format instructions
+        if self.config.use_tools:
+            self.system_json_tool_instructions = self.json_format_rules()
+        self.system_tool_instructions = self.tool_instructions()
 
     def disable_message_handling(
         self,
@@ -275,37 +304,6 @@ class ChatAgent(Agent):
                 self.llm_tools_usable.discard(r)
                 self.llm_functions_usable.discard(r)
 
-    def update_json_tool_instructions(self) -> None:
-        """
-        Add special instructions on situations when the LLM should send JSON-formatted
-        messages, and save the index position of these instructions in the
-        message history. Note this specifically only relates to the
-        Langroid JSON tools mechanism, not the LLM-native function-calling.
-        """
-        # Add the instructions as a user message...
-        # TODO need to adapt this based on model type, as some models may
-        # pay more attention to system message.
-        json_instructions = super().json_tool_format_instructions()
-        if not self.tool_instructions_added:
-            self.task_messages.append(
-                LLMMessage(role=Role.USER, content=json_instructions)
-            )
-            self.tool_instructions_added = True
-        else:
-            # json_instructions will contain instructions on ALL tools,
-            # so it is ok to overwrite the last message in the task_messages,
-            # since we know it is a tool-instruction msg.
-            self.task_messages[-1].content = json_instructions
-
-        # Note that task_messages is the initial set of messages created to set up
-        # the task, and they may not yet have been sent to the LLM at this point.
-
-        # But if the task_messages have already been sent to the LLM, then we need to
-        # update the self.message_history as well, since this history will be sent to
-        # the LLM on each round, after appending the latest assistant, user msgs.
-        if len(self.message_history) > 0:
-            self.message_history[-1].content = json_instructions
-
     @no_type_check
     def llm_response(
         self, message: Optional[str | ChatDocument] = None
@@ -327,13 +325,13 @@ class ChatAgent(Agent):
         ), "message can be None only if message_history is empty, i.e. at start."
 
         if len(self.message_history) == 0:
-            # task_messages have not yet been loaded, so load them
-            self.message_history = self.task_messages.copy()
-            # first message is system msg, augment if needed
-            if self.system_tool_instructions != "":
-                self.message_history[0].content += (
-                    "\n\n" + self.system_tool_instructions
+            # initial messages have not yet been loaded, so load them
+            self.message_history = [self._create_system_and_tools_message()]
+            if self.user_message:
+                self.message_history.append(
+                    LLMMessage(role=Role.USER, content=self.user_message)
                 )
+
             # for debugging, show the initial message history
             if settings.debug:
                 print(
@@ -342,6 +340,10 @@ class ChatAgent(Agent):
                 {self.message_history_str()}
                 """
                 )
+        else:
+            assert self.message_history[0].role == Role.SYSTEM
+            # update the system message with the latest tool instructions
+            self.message_history[0] = self._create_system_and_tools_message()
 
         if message is not None:
             llm_msg = ChatDocument.to_LLMMessage(message)
@@ -456,7 +458,9 @@ class ChatAgent(Agent):
                 response_str = response.message
             print(cached + "[green]" + response_str)
         stream = self.llm.get_stream()  # type: ignore
-        self.update_token_usage(response, messages, stream)
+        self.update_token_usage(
+            response, messages, stream, print_response_stats=settings.debug
+        )
         return ChatDocument.from_LLMResponse(response, displayed)
 
     def _llm_response_temp_context(self, message: str, prompt: str) -> ChatDocument:

{langroid-0.1.60 → langroid-0.1.62}/langroid/agent/chat_document.py

@@ -172,6 +172,8 @@ class ChatDocument(Document):
             content = message.content
             fun_call = message.function_call
             sender_name = message.metadata.sender_name
+            if message.metadata.sender == Entity.SYSTEM:
+                sender_role = Role.SYSTEM
             if (
                 message.metadata.parent is not None
                 and message.metadata.parent.function_call is not None

{langroid-0.1.60 → langroid-0.1.62}/langroid/agent/task.py

@@ -12,7 +12,6 @@ from langroid.agent.chat_document import (
     ChatDocMetaData,
     ChatDocument,
 )
-from langroid.language_models.base import LLMMessage, Role
 from langroid.mytypes import Entity
 from langroid.utils.configuration import settings
 from langroid.utils.constants import DONE, NO_ANSWER, USER_QUIT
@@ -82,8 +81,8 @@ class Task:
                 and subsequent response by non-controller. If false, runs for the
                 specified number of turns in `run`, or until `done()` is true.
                 One run of step() is considered a "turn".
-            system_message (str): if not empty, overrides agent's `task_messages[0]`
-            user_message (str): if not empty, overrides agent's `task_messages[1]`
+            system_message (str): if not empty, overrides agent's system_message
+            user_message (str): if not empty, overrides agent's user_message
             restart (bool): if true, resets the agent's message history
             default_human_response (str): default response from user; useful for
                 testing, to avoid interactive input from user.
@@ -99,18 +98,14 @@ class Task:
         """
         if isinstance(agent, ChatAgent) and len(agent.message_history) == 0 or restart:
             agent = cast(ChatAgent, agent)
-            agent.message_history = []
-            # possibly change the task messages
+            agent.clear_history(0)
+            # possibly change the system and user messages
             if system_message:
                 # we always have at least 1 task_message
-                agent.task_messages[0].content = system_message
+                agent.set_system_message(system_message)
             if user_message:
-                agent.task_messages.append(
-                    LLMMessage(
-                        role=Role.USER,
-                        content=user_message,
-                    )
-                )
+                agent.set_user_message(user_message)
+
         self.logger: None | RichFileLogger = None
         self.tsv_logger: None | logging.Logger = None
         self.color_log: bool = True
@@ -231,6 +226,8 @@ class Task:
                 # the CURRENT task's USER entity
                 self.pending_message.metadata.sender = Entity.USER
 
+        self._show_pending_message_if_debug()
+
         if self.caller is not None and self.caller.logger is not None:
             self.logger = self.caller.logger
         else:
@@ -257,8 +254,8 @@ class Task:
 
         Args:
             msg (str|ChatDocument): initial message to process; if None,
-                the LLM will respond to the initial `self.task_messages`
-                which set up the overall task.
+                the LLM will respond to its initial `self.task_messages`
+                which set up and kick off the overall task.
                 The agent tries to achieve this goal by looping
                 over `self.step()` until the task is considered
                 done; this can involve a series of messages produced by Agent,
@@ -325,6 +322,8 @@ class Task:
         n_messages = 0
         if isinstance(self.agent, ChatAgent):
             if self.erase_substeps:
+                # TODO I don't like directly accessing agent message_history. Revisit.
+                # (Pchalasani)
                 del self.agent.message_history[message_history_idx + 2 : n_messages - 1]
             n_messages = len(self.agent.message_history)
         if self.erase_substeps:
@@ -441,11 +440,16 @@ class Task:
             self.pending_sender = responder
             self.log_message(self.pending_sender, self.pending_message, mark=True)
 
+        self._show_pending_message_if_debug()
+        return self.pending_message
+
+    def _show_pending_message_if_debug(self) -> None:
+        if self.pending_message is None:
+            return
         if settings.debug:
             sender_str = str(self.pending_sender)
             msg_str = str(self.pending_message)
             print(f"[red][{sender_str}]{msg_str}")
-        return self.pending_message
 
     def response(self, e: Responder, turns: int = -1) -> Optional[ChatDocument]:
         """
@@ -589,6 +593,7 @@ class Task:
                 Entity.LLM: "green",
                 Entity.USER: "blue",
                 Entity.AGENT: "red",
+                Entity.SYSTEM: "magenta",
             }[msg.metadata.sender]
             f = msg.log_fields()
             tool_type = f.tool_type.rjust(6)

{langroid-0.1.60 → langroid-0.1.62}/langroid/agent/tool_message.py

@@ -56,7 +56,7 @@ INSTRUCTION = """
     In this case you realize there is no available TOOL for this, so you just ask in 
     natural language: "What is the population of France?"
     
-    Whenever possible, AND ONLY IF APPLICABLE, use these TOOL, with the JSON syntax 
+    Whenever possible, AND ONLY IF APPLICABLE, use these TOOLS, with the JSON syntax 
     specified above. When a TOOL is applicable, simply use this syntax, do not write 
     anything else. Only if no TOOL is exactly applicable, ask in natural language. 
     """

{langroid-0.1.60 → langroid-0.1.62}/langroid/language_models/prompt_formatter/llama2_formatter.py

@@ -8,8 +8,8 @@ from langroid.language_models.prompt_formatter.base import PromptFormatter
 logger = logging.getLogger(__name__)
 
 
-BOS: str = ""
-EOS: str = ""
+BOS: str = "<s>"
+EOS: str = "</s>"
 B_INST: str = "[INST]"
 E_INST: str = "[/INST]"
 B_SYS: str = "<<SYS>>\n"

{langroid-0.1.60 → langroid-0.1.62}/langroid/mytypes.py

@@ -18,6 +18,7 @@ class Entity(str, Enum):
     AGENT = "Agent"
     LLM = "LLM"
     USER = "User"
+    SYSTEM = "System"
 
 
 class DocMetaData(BaseModel):

{langroid-0.1.60 → langroid-0.1.62}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.60"
+version = "0.1.62"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"