langroid 0.16.7__py3-none-any.whl → 0.17.0__py3-none-any.whl

langroid/agent/tools/code_file_tool_parse.py

@@ -1,150 +0,0 @@
-""""
-Non-JSON Tool for LLM to specify contents of a code file.
-
-Why Non-JSON?  Because there are numerous issues with even the best LLMs trying
-to return code within JSON strings (e.g. unescaped newlines, quotes, etc.),
-and the problem is even worse with weak LLMs. Json repair methods exist, but
-can't deal with all possible cases.
-
-E.g. see this study from Aider: https://aider.chat/2024/08/14/code-in-json.html
-
-Note: We express the formatting rules with a template since it has several benefits:
-    - all of the formatting rules are in one place, 
-    - we get a parser for free, and don't have to write parsing code,
-    - we get a formatting example generator for free, and don't have to write
-        example generation code.
-    - consistency between the parser and the example generator is guaranteed.        
-"""
-
-from typing import Any, Callable, Dict, List, Tuple, Type
-
-from parse import Parser, compile
-
-from langroid.agent.tool_message import ToolMessage
-from langroid.utils.constants import TOOL, TOOL_END
-
-CODE_FENCE_START = "`" * 3
-CODE_FENCE_END = "`" * 3
-
-
-class CodeFileTool(ToolMessage):
-    """
-    Used by LLM to specify contents of a code file.
-    """
-
-    request: str = "code_file_tool"
-    purpose: str = """
-        To specify the contents of a code file.
-        """
-    file_path: str
-    contents: str
-    language: str
-
-    @classmethod
-    def get_template(cls) -> str:
-        request = cls.default_value("request")
-
-        return f"""
-{{ws}}{TOOL}:{{ws}}{request}
-{{ws}}{{file_path}}
-{CODE_FENCE_START}{{ws}}{{language}}
-{{contents}}
-{CODE_FENCE_END}
-{{ws}}{TOOL_END}{{ws}}
-"""
-
-    @classmethod
-    def parse(cls, string) -> Dict[str, Any]:
-        def parse_ws(string):
-            return (
-                string  # This allows matching any amount of whitespace, including none
-            )
-
-        template = cls.get_template()
-        parser = Parser(template, dict(ws=parse_ws))
-
-        print(f"Parsing string:\n{string}")  # Debug print
-        result = parser.parse(string)
-        print(f"Parse result: {result}")  # Debug print
-
-        if result:
-            return {
-                "request": cls.default_value("request"),
-                "file_path": result["file_path"].strip(),
-                "language": result["language"].strip(),
-                "contents": result["contents"].strip(),
-            }
-        return {}
-
-    @classmethod
-    def format(cls, instance):
-        return cls.get_template().format(
-            request=instance.request,
-            file_path=instance.file_path,
-            language=instance.language,
-            contents=instance.contents,
-            s="",  # no extra spaces
-        )
-
-    @classmethod
-    def create(cls, get_directory: Callable[[], str]) -> Type["CodeFileTool"]:
-        """
-        Create a subclass of CodeFileTool with a static method get_directory,
-        which returns the current directory path, so that all file paths are
-        interpreted as relative to the current directory.
-        """
-
-        class SubCodeFileTool(cls):
-            get_directory: Callable[[], str] = staticmethod(get_directory)
-
-        return SubCodeFileTool
-
-    @classmethod
-    def examples(cls) -> List[ToolMessage | Tuple[str, ToolMessage]]:
-        return [
-            cls(
-                file_path="src/lib.rs",
-                language="rust",
-                contents="""
-                    // function to add two numbers
-                    pub fn add(a: i32, b: i32) -> i32 {
-                        a + b
-                    }
-                    """,
-            )
-        ]
-
-    @classmethod
-    def find_candidates(cls, input_str: str) -> List[str]:
-        """
-        Find all possible (top-level) candidates for
-        CodeFileTool in the input string.
-        """
-        # Use parse.findall to find all instances of the CodeFileTool pattern
-        parser = compile(cls.get_template())
-        matches = list(parser.findall(input_str))
-        candidates = [match.fixed for match in matches]
-        return candidates
-
-    @classmethod
-    def from_string(cls, input_string: str) -> "CodeFileTool":
-        """Parse a string into a CodeFileTool object, using the TEMPLATE."""
-        parsed_data = cls.parse(input_string)
-        if parsed_data:
-            return cls(**parsed_data)
-        raise ValueError("Invalid input string format")
-
-    @classmethod
-    def to_string(cls, instance) -> str:
-        """Convert a CodeFileTool object to a string, using the TEMPLATE."""
-        return cls.format(instance)
-
-    def __str__(self):
-        return self.to_string()
-
-    def __repr__(self):
-        return f"""CodeFileTool(
-            file_path='{self.file_path}', 
-            language='{self.language}',
-            contents='{self.contents}')
-            """

langroid/agent/tools/code_file_tool_pyparsing.py

@@ -1,194 +0,0 @@
-""""
-Non-JSON Tool for LLM to specify contents of a code file.
-
-Why Non-JSON?  Because there are numerous issues with even the best LLMs trying
-to return code within JSON strings (e.g. unescaped newlines, quotes, etc.),
-and the problem is even worse with weak LLMs. Json repair methods exist, but
-can't deal with all possible cases.
-
-E.g. see this study from Aider: https://aider.chat/2024/08/14/code-in-json.html
-
-Note: We express the formatting rules with a template since it has several benefits:
-    - all of the formatting rules are in one place, 
-    - we get a parser for free, and don't have to write parsing code,
-    - we get a formatting example generator for free, and don't have to write
-        example generation code.
-    - consistency between the parser and the example generator is guaranteed.        
-"""
-
-from typing import Any, Callable, Dict, List, Tuple, Type
-
-from pyparsing import (
-    LineEnd,
-    Literal,
-    Optional,
-    ParserElement,
-    SkipTo,
-    White,
-    Word,
-    alphanums,
-    lineEnd,
-    printables,
-)
-
-from langroid.agent.tool_message import ToolMessage
-from langroid.utils.constants import TOOL, TOOL_END
-
-CODE_FENCE_START = "`" * 3
-CODE_FENCE_END = "`" * 3
-
-
-class CodeFileTool(ToolMessage):
-    """
-    Used by LLM to specify contents of a code file.
-    """
-
-    request: str = "code_file_tool"
-    purpose: str = """
-        To specify the contents of a code file.
-        """
-    file_path: str
-    contents: str
-    language: str
-
-    @classmethod
-    def create_parser(cls):
-        TOOL_START = Literal(TOOL + ":")
-        CODE_FENCE = Literal("```")
-
-        file_path = SkipTo(lineEnd)("file_path")
-        language = Word(alphanums)("language")
-        contents = SkipTo(CODE_FENCE)("contents")
-
-        parser = (
-            TOOL_START
-            + Optional(Word(printables), default=cls.default_value("request"))(
-                "request"
-            )
-            + lineEnd
-            + file_path
-            + lineEnd
-            + CODE_FENCE
-            + Optional(White())  # Allow space after ```
-            + language
-            + lineEnd
-            + contents
-            + CODE_FENCE
-            + lineEnd  # Add this line to expect a newline after the closing fence
-            + Literal(TOOL_END)
-        )
-        return parser
-
-    @classmethod
-    def parse(cls, string) -> Dict[str, Any]:
-        parser = cls.create_parser()
-        try:
-            result = parser.parseString(string, parseAll=True)
-            return {
-                "request": result["request"],
-                "file_path": result["file_path"].strip(),
-                "language": result["language"],
-                "contents": result["contents"].strip(),
-            }
-        except Exception as e:
-            print(f"Parsing failed: {e}")
-            return {}
-
-    @classmethod
-    def format(cls, instance) -> str:
-        parser = cls.create_parser()
-
-        def format_element(element):
-            if isinstance(element, Literal):
-                return element.match
-            elif element.resultsName:
-                if element.resultsName == "request":
-                    return instance.request
-                elif element.resultsName == "file_path":
-                    return instance.file_path
-                elif element.resultsName == "language":
-                    return instance.language
-                elif element.resultsName == "contents":
-                    return f"{instance.contents}\n"  # Add newline after contents
-            elif isinstance(element, LineEnd):
-                return "\n"
-            return ""
-
-        def traverse_parser(parser_element):
-            if isinstance(parser_element, ParserElement):
-                if isinstance(parser_element, SkipTo):
-                    return format_element(parser_element)
-                elif hasattr(parser_element, "exprs"):
-                    return "".join(
-                        traverse_parser(expr) for expr in parser_element.exprs
-                    )
-                else:
-                    return format_element(parser_element)
-            return str(parser_element)
-
-        formatted_string = traverse_parser(parser)
-
-        return formatted_string.strip()
-
-    @classmethod
-    def create(cls, get_directory: Callable[[], str]) -> Type["CodeFileTool"]:
-        """
-        Create a subclass of CodeFileTool with a static method get_directory,
-        which returns the current directory path, so that all file paths are
-        interpreted as relative to the current directory.
-        """
-
-        class SubCodeFileTool(cls):
-            get_directory: Callable[[], str] = staticmethod(get_directory)
-
-        return SubCodeFileTool
-
-    @classmethod
-    def examples(cls) -> List[ToolMessage | Tuple[str, ToolMessage]]:
-        return [
-            cls(
-                file_path="src/lib.rs",
-                language="rust",
-                contents="""
-                    // function to add two numbers
-                    pub fn add(a: i32, b: i32) -> i32 {
-                        a + b
-                    }
-                    """,
-            )
-        ]
-
-    @classmethod
-    def find_candidates(cls, input_str: str) -> List[str]:
-        """
-        Find all possible (top-level) candidates for
-        CodeFileTool in the input string.
-        """
-        # Use parse.findall to find all instances of the CodeFileTool pattern
-        parser = compile(cls.get_template())
-        matches = list(parser.findall(input_str))
-        candidates = [match.fixed for match in matches]
-        return candidates
-
-    @classmethod
-    def from_string(cls, input_string: str) -> "CodeFileTool":
-        """Parse a string into a CodeFileTool object, using the TEMPLATE."""
-        parsed_data = cls.parse(input_string)
-        if parsed_data:
-            return cls(**parsed_data)
-        raise ValueError("Invalid input string format")
-
-    @classmethod
-    def to_string(cls, instance) -> str:
-        """Convert a CodeFileTool object to a string, using the TEMPLATE."""
-        return cls.format(instance)
-
-    def __str__(self):
-        return self.to_string()
-
-    def __repr__(self):
-        return f"""CodeFileTool(
-            file_path='{self.file_path}', 
-            language='{self.language}',
-            contents='{self.contents}')
-            """

langroid/agent/tools/code_file_tool_pyparsing2.py

@@ -1,199 +0,0 @@
-""""
-Non-JSON Tool for LLM to specify contents of a code file.
-
-Why Non-JSON?  Because there are numerous issues with even the best LLMs trying
-to return code within JSON strings (e.g. unescaped newlines, quotes, etc.),
-and the problem is even worse with weak LLMs. Json repair methods exist, but
-can't deal with all possible cases.
-
-E.g. see this study from Aider: https://aider.chat/2024/08/14/code-in-json.html
-
-Note: We express the formatting rules with a template since it has several benefits:
-    - all of the formatting rules are in one place, 
-    - we get a parser for free, and don't have to write parsing code,
-    - we get a formatting example generator for free, and don't have to write
-        example generation code.
-    - consistency between the parser and the example generator is guaranteed.        
-"""
-
-from typing import Callable, List, Tuple, Type
-
-from pyparsing import (
-    Literal,
-    Optional,
-    SkipTo,
-    White,
-    Word,
-    alphanums,
-    lineEnd,
-    printables,
-)
-
-from langroid.agent.tool_message import ToolMessage
-from langroid.agent.tools.generic_tool import GenericTool
-from langroid.utils.constants import TOOL, TOOL_END
-
-CODE_FENCE_START = "`" * 3
-CODE_FENCE_END = "`" * 3
-
-
-class CodeFileTool(GenericTool):
-    """
-    Used by LLM to specify contents of a code file.
-    """
-
-    request: str = "code_file_tool"
-    purpose: str = """
-    To specify the <contents> of a code file at <file_path>,
-    containing code in a specific <language>.
-    """
-    file_path: str
-    contents: str
-    language: str
-
-    @classmethod
-    def define_grammar(cls):
-        TOOL_START = Literal(TOOL + ":")
-        CODE_FENCE = Literal("```")
-
-        file_path = SkipTo(lineEnd)("file_path")
-        language = Word(alphanums)("language")
-        contents = SkipTo(lineEnd + CODE_FENCE)("contents")
-
-        grammar = (
-            TOOL_START
-            + Optional(White())
-            + Optional(Word(printables), default=cls.default_value("request"))(
-                "request"
-            )
-            + lineEnd
-            + Optional(White())
-            + file_path
-            + lineEnd
-            + CODE_FENCE
-            + Optional(White())
-            + language
-            + lineEnd
-            + contents
-            + lineEnd
-            + CODE_FENCE
-            + lineEnd
-            + Optional(White())
-            + Literal(TOOL_END)
-            + Optional(White())
-        )
-        return grammar
-
-    @classmethod
-    def create(cls, get_directory: Callable[[], str]) -> Type["CodeFileTool"]:
-        """
-        Create a subclass of CodeFileTool with a static method get_directory,
-        which returns the current directory path, so that all file paths are
-        interpreted as relative to the current directory.
-        """
-
-        class SubCodeFileTool(cls):
-            get_directory: Callable[[], str] = staticmethod(get_directory)
-
-        return SubCodeFileTool
-
-    @classmethod
-    def examples(cls) -> List[ToolMessage | Tuple[str, ToolMessage]]:
-        return [
-            cls(
-                file_path="src/lib.rs",
-                language="rust",
-                contents="""
-                    // function to add two numbers
-                    pub fn add(a: i32, b: i32) -> i32 {
-                        a + b
-                    }
-                    """,
-            )
-        ]
-
-    def __repr__(self):
-        return f"""CodeFileTool(
-            file_path='{self.file_path}', 
-            language='{self.language}',
-            contents='{self.contents}')
-            """
-
-
-if __name__ == "__main__":
-    # Informal test to print instructions for CodeFileTool
-    print("Testing CodeFileTool instructions:")
-    print("-" * 50)
-
-    instructions = CodeFileTool.instructions()
-    print(instructions)
-
-    print("-" * 50)
-    print("End of instructions test")
-
-    # You can add more informal tests here if needed
-    # For example, testing the parse method:
-    print("\nTesting CodeFileTool parse method:")
-    print("-" * 50)
-
-    test_input = """TOOL: code_file_tool
-    src/main.py
-    ```python
-    def hello_world():
-        print("Hello, World!")
-    
-    if __name__ == "__main__":
-        hello_world()
-    ```
-    TOOL_END"""
-
-    parsed_result = CodeFileTool.parse(test_input)
-    print("Parsed result:")
-    for key, value in parsed_result.items():
-        print(f"{key}: {value}")
-
-    print("-" * 50)
-    print("End of parse test")
-
-    # Test format method
-    print("\nTesting CodeFileTool format method:")
-    print("-" * 50)
-    test_instance = CodeFileTool(
-        request="code_file_tool",
-        file_path="tests/test_file.py",
-        language="python",
-        contents="""
-def test_function():
-    assert 1 + 1 == 2
-
-if __name__ == "__main__":
-    test_function()
-""",
-    )
-    formatted_output = CodeFileTool.format(test_instance)
-    print("Formatted output:")
-    print(formatted_output)
-    print("-" * 50)
-    print("End of format test")
-
-    # Additional test: Round-trip (parse -> format -> parse)
-    print("\nTesting CodeFileTool round-trip (parse -> format -> parse):")
-    print("-" * 50)
-    initial_parse = CodeFileTool.parse(test_input)
-    initial_instance = CodeFileTool(**initial_parse)
-    formatted_output = CodeFileTool.format(initial_instance)
-    final_parse = CodeFileTool.parse(formatted_output)
-
-    print("Initial parse:")
-    print(initial_parse)
-    print("\nFormatted output:")
-    print(formatted_output)
-    print("\nFinal parse:")
-    print(final_parse)
-
-    if initial_parse == final_parse:
-        print("\nRound-trip test passed: Initial and final parses match.")
-    else:
-        print("\nRound-trip test failed: Initial and final parses do not match.")
-    print("-" * 50)
-    print("End of round-trip test")

langroid/agent/tools/extract_tool.py

@@ -1,96 +0,0 @@
-"""
-A tool to extract portions from previous (numbered) messages in
-the chat history.
-The idea is that when an LLM wants to (or is asked to) simply extract
-portions of a message verbatim, it should use this tool/function to
-SPECIFY what should be extracted, rather than actually extracting it.
-This will usually be much cheaper and faster than actually writing out the extracted
-text. The handler of this tool/function will then extract the text and send it back.
-"""
-
-from typing import Protocol, no_type_check
-
-from jinja2 import DictLoader, Environment
-
-from langroid.agent.tool_message import ToolMessage
-from langroid.language_models.base import LLMMessage
-
-
-def extract_between(value: str, start_word: str, end_word: str) -> str:
-    """
-    Extract the substring between two words in a string.
-    NOTE: If there are multiple occurrences of the start_word, the first one is used,
-    and if there are multiple occurrences of the end_word, the first one after the
-    start_word is used.
-
-    We do not handle the case of multiple occurrences of the start_word, followed by
-    multiple occurrences of the end_word
-
-    Args:
-        value (str): the string to extract from
-        start_word (str): the word that starts the substring
-        end_word (str): the word that ends the substring
-
-    Returns:
-        str: the substring between the two words
-    """
-    try:
-        start_index = value.index(start_word) + len(start_word)
-        end_index = value.index(end_word, start_index)
-        return value[start_index:end_index].strip()
-    except ValueError:
-        return ""
-
-
-class HasMessageHistory(Protocol):
-    """
-    Defines the fields expected in a class that enables this tool.
-    """
-
-    message_history: list[LLMMessage]
-
-
-class ExtractTool(ToolMessage):
-    request: str = "extract"
-    purpose: str = """
-            To generate a message in the form of a <jinja_template>, 
-            using the Jinja templating language, where the 
-            the i'th message is referred to as msg[i], and integer indices are used 
-            to specify which part of the message to extract, e.g. msg[2][13:45].
-            """
-    jinja_template: str
-
-    @classmethod
-    def instructions(cls) -> str:
-        return """
-        In a conversation with the user, your responses may sometimes use verbatim 
-        extracts of previous messages in the conversation.  You are an expert at 
-        Jinja templating syntax, and you will rely on this syntax whenever you find 
-        yourself wanting to repeat verbatim text from earlier parts of the 
-        conversation.  
-        In your Jinja templates you can use references like {{msg[3][3:100]}} to 
-        indicate that the user should substitute the content of message number 3 
-        (assume first msg is number 1), starting at position 3 and ending at 100.   
-        
-        For example you may respond with something like:
-        
-        The story started like this: {{msg[5][45:89]}}. Then John came home and 
-        {{msg[2][4:19}}. 
-        
-        VERY IMPORTANT --
-        (a) your FIRST priority is to generate  messages that would sound natural 
-        when the jinja templates are rendered. 
-        
-        (b) your NEXT priority is to ALWAYS RELY on the above JINJA scheme when 
-        your intended message would contain verbatim text from previous messages. 
-        
-        (c) Do not simply use large verbatim parts of previous messages, when doing 
-        so may not result in natural responses.         
-        """
-
-    @no_type_check
-    def handle(self: HasMessageHistory) -> str:
-        msg = self.message_history
-        env = Environment(loader=DictLoader({"base": self.jinja_template}))
-        template = env.get_template("base")
-        return template.render(msg=msg)