langroid 0.16.7__py3-none-any.whl → 0.17.1__py3-none-any.whl

langroid/agent/tools/generic_tool.py

@@ -1,165 +0,0 @@
-from abc import abstractmethod
-from typing import Any, Dict, List
-
-from pyparsing import (
-    LineEnd,
-    Literal,
-    ParserElement,
-)
-
-from langroid.agent.tool_message import ToolMessage
-
-
-class GenericTool(ToolMessage):
-    """
-    Abstract class for a tool whose format is defined by a grammar,
-    and not necessarily JSON-based.
-    Especially useful for tools where we need an LLM to return code.
-    Most LLMs, especially weaker ones, have significant issues
-    (related to unescaped newlines, quotes, etc) when returning code within JSON.
-    """
-
-    @classmethod
-    @abstractmethod
-    def define_grammar(cls):
-        """Define the grammar for the specific tool."""
-        pass
-
-    @classmethod
-    def create_parser(cls):
-        """Create a parser based on the defined grammar."""
-        grammar = cls.define_grammar()
-        # Use the grammar to create and return a parser
-        return grammar
-
-    @classmethod
-    def parse(cls, string) -> Dict[str, Any]:
-        parser = cls.create_parser()
-        try:
-            result = parser.parseString(string, parseAll=True)
-            return {
-                name: result[name]
-                for name in result.keys()
-                if name and not name.startswith("_")
-            }
-        except Exception as e:
-            print(f"Parsing failed: {e}")
-            return {}
-
-    @classmethod
-    def instructions(cls) -> str:
-        preamble = "Preamble:\n"
-        for field, field_info in cls.__dict__["__fields__"].items():
-            preamble += (
-                f"<{field}> denotes the value of the `{field}` field "
-                f"(type: {field_info.type_})\n"
-            )
-
-        parser = cls.create_parser()
-
-        def format_element(element):
-            if isinstance(element, Literal):
-                return element.match
-            elif hasattr(element, "resultsName") and element.resultsName:
-                return f"<{element.resultsName}>"
-            elif isinstance(element, LineEnd):
-                return "\n"
-            return ""
-
-        def traverse_parser(parser_element):
-            if isinstance(parser_element, ParserElement):
-                if hasattr(parser_element, "exprs"):
-                    return "".join(
-                        traverse_parser(expr) for expr in parser_element.exprs
-                    )
-                else:
-                    return format_element(parser_element)
-            return str(parser_element)
-
-        template = traverse_parser(parser)
-
-        return f"{preamble}\nFormatted Example:\n{template.strip()}"
-
-    @classmethod
-    def parse(cls, string) -> Dict[str, Any]:
-        parser = cls.create_parser()
-        try:
-            result = parser.parseString(string, parseAll=True)
-            return {
-                name: result[name]
-                for name in result.keys()
-                if name and not name.startswith("_")
-            }
-        except Exception as e:
-            print(f"Parsing failed: {e}")
-            return {}
-
-    @classmethod
-    def format(cls, instance) -> str:
-        parser = cls.create_parser()
-
-        def format_element(element):
-            if isinstance(element, Literal):
-                return element.match
-            elif hasattr(element, "resultsName") and element.resultsName:
-                return getattr(instance, element.resultsName, "")
-            elif isinstance(element, LineEnd):
-                return "\n"
-            return ""
-
-        def traverse_parser(parser_element):
-            if isinstance(parser_element, ParserElement):
-                if hasattr(parser_element, "exprs"):
-                    return "".join(
-                        traverse_parser(expr) for expr in parser_element.exprs
-                    )
-                else:
-                    return format_element(parser_element)
-            return str(parser_element)
-
-        formatted_string = traverse_parser(parser)
-
-        return formatted_string.strip()
-
-    @classmethod
-    def from_string(cls, input_string: str) -> "CodeFileTool":
-        """Parse a string into a CodeFileTool object, using the TEMPLATE."""
-        parsed_data = cls.parse(input_string)
-        if parsed_data:
-            return cls(**parsed_data)
-        raise ValueError("Invalid input string format")
-
-    @classmethod
-    def to_string(cls, instance) -> str:
-        """Convert a CodeFileTool object to a string, using the TEMPLATE."""
-        return cls.format(instance)
-
-    @classmethod
-    def find_candidates(cls, input_str: str) -> List[str]:
-        """
-        Find all possible (top-level) candidates for
-        CodeFileTool in the input string.
-        """
-        parser = cls.create_parser()
-        candidates = []
-
-        for tokens, start, end in parser.scanString(input_str):
-            candidates.append(input_str[start:end])
-
-        return candidates
-
-    def __str__(self):
-        return self.to_string()
-
-    # def __repr__(self) -> str:
-    #     class_name = self.__class__.__name__
-    #     attributes = []
-    #     for key, value in self.__dict__.items():
-    #         if not key.startswith('_'):  # Skip private attributes
-    #             if isinstance(value, str):
-    #                 # Escape quotes and newlines in string values
-    #                 value_repr = f"'{value.replace('\\', '\\\\').replace(\"'\", \"\\'\").replace('\\n', '\\n')}'"
-    #             else:
-    #                 value_repr = repr(value)
-    #             attributes.append(f"{key}={value_repr}")
-    #     return f"{class_name}({', '.join(attributes)})"

langroid/agent/tools/generic_tool_tatsu.py

@@ -1,275 +0,0 @@
-from abc import abstractmethod
-from typing import List
-
-import tatsu
-
-from langroid.agent.tool_message import ToolMessage
-
-
-class GenericTool(ToolMessage):
-    """
-    Abstract class for a tool whose format is defined by a grammar,
-    and not necessarily JSON-based.
-    Especially useful for tools where we need an LLM to return code.
-    Most LLMs, especially weaker ones, have significant issues
-    (related to unescaped newlines, quotes, etc) when returning code within JSON.
-    """
-
-    @classmethod
-    @abstractmethod
-    def tool_grammar(cls) -> str:
-        """Define the grammar for the `tool` rule"""
-        pass
-
-    @classmethod
-    def grammar(cls) -> str:
-        """
-        Full grammar, including templates for rendering.
-        """
-        base_grammar = """
-            @@grammar :: CombinedGrammar
-            @@whitespace :: /[ \\t]+/
-            @@nameguard :: False
-
-            start
-                =
-                "<spec>" ws?
-                request:word ws?
-                tool
-                ws? "</spec>"
-            {:
-                "<spec> " {{request}} {{tool}} " </spec>"
-            :}
-                ;
-
-            ws = /[\\s]+/ ;
-
-            word = /[^\\s<>/]+/ ;
-        """
-        full_grammar = base_grammar + "\n" + cls.tool_grammar()
-        return full_grammar
-
-    @classmethod
-    def parse(cls, s: str):
-        """
-        Parses a string `s` using the grammar and returns an instance of the subclass.
-        """
-        # Build the parser using the provided grammar with model generation
-        parser = tatsu.compile(cls.grammar(), asmodel=True)
-
-        # Parse the input string to get a model object
-        model = parser.parse(s)
-
-        # Convert the model to a dict, filtering only the expected fields
-        data = {k: getattr(model, k) for k in cls.__fields__ if hasattr(model, k)}
-
-        # Create an instance of the subclass with the parsed data
-        model_instance = cls(**data)
-        return model_instance
-
-    def format(self) -> str:
-        """
-        Generates a string representation of the instance based on the grammar.
-        """
-        # Build the parser using the provided grammar with model generation
-        parser = tatsu.compile(self.grammar(), asmodel=True)
-
-        # Create a model instance
-        model_class = parser.model()
-        model = model_class()
-
-        # Set attributes from the instance, excluding fields not in the grammar
-        for field in self.__fields__:
-            if field == "purpose":
-                continue  # Exclude 'purpose' from rendering
-            setattr(model, field, getattr(self, field))
-
-        # Render the model back to text using the grammar's templates
-        generated_string = model.render()
-        return generated_string
-
-    @classmethod
-    def instructions(cls) -> str:
-        """
-        Generates instructions for formatting an instance, including placeholders
-        and an example output with placeholders.
-        """
-
-        def generate_placeholders(field, prefix=""):
-            placeholders = {}
-            if hasattr(field.type_, "__fields__"):
-                # Nested model
-                for sub_field_name, sub_field in field.type_.__fields__.items():
-                    placeholders.update(
-                        generate_placeholders(
-                            sub_field, prefix=f"{prefix}{field.name}."
-                        )
-                    )
-            elif isinstance(field.type_, type) and issubclass(field.type_, list):
-                # List field
-                placeholders[field.name] = (
-                    f"[<{field.name}_item1>,<{field.name}_item2>,...]"
-                )
-            else:
-                placeholders[field.name] = f"<{prefix}{field.name}>"
-            return placeholders
-
-        # Generate placeholders for all fields
-        placeholders = {}
-        for field_name, field in cls.__fields__.items():
-            placeholders.update(generate_placeholders(field))
-
-        # Build the preamble
-        preamble_lines = ["Placeholders for formatting:"]
-        for field_name, placeholder in placeholders.items():
-            field_type = cls.__fields__[field_name].type_.__name__
-            preamble_lines.append(
-                f"- `{placeholder}`: placeholder for `{field_name}` field (type: `{field_type}`)"
-            )
-        preamble = "\n".join(preamble_lines)
-
-        # Create a placeholder instance
-        placeholder_values = {}
-        for field_name in cls.__fields__:
-            placeholder_values[field_name] = placeholders[field_name]
-        placeholder_instance = cls(**placeholder_values)
-
-        # Generate an example output with placeholders
-        parser = tatsu.compile(cls.grammar())
-        ast = placeholder_instance.to_ast()
-        # Use the placeholders in the AST
-        for key, value in ast.items():
-            ast[key] = placeholders.get(key, value)
-        example_output = parser.render(ast)
-
-        # Combine preamble and example output
-        instructions = f"{preamble}\n\nExample format:\n\n{example_output}"
-        return instructions
-
-    @classmethod
-    def from_ast(cls, ast):
-        """
-        Converts an AST into a model instance.
-        """
-        # Since TatSu produces dicts, we can convert the AST dict to the model
-        return cls(**ast)
-
-    def to_ast(self):
-        """
-        Converts the model instance into an AST (dict).
-        """
-        # Since TatSu expects dicts for rendering, we can use the model's dict
-        return self.dict()
-
-    @classmethod
-    def from_string(cls, input_string: str) -> "CodeFileTool":
-        """Parse a string into a CodeFileTool object, using the TEMPLATE."""
-        parsed_data = cls.parse(input_string)
-        if parsed_data:
-            return cls(**parsed_data)
-        raise ValueError("Invalid input string format")
-
-    def to_string(self) -> str:
-        """Convert a CodeFileTool object to a string, using the TEMPLATE."""
-        return self.format()
-
-    @classmethod
-    def find_candidates(cls, s: str) -> List[str]:
-        """
-        Finds all substrings in `s` that start with start_marker and end with end_marker.
-        """
-        start = "<spec>"  # TODO get from TOOL_BEGIN, TOOL_END
-        end = "</spec>"
-        candidates = []
-        start_len = len(start)
-        end_len = len(end)
-        index = 0
-        while index < len(s):
-            start_index = s.find(start, index)
-            if start_index == -1:
-                break
-            end_index = s.find(end, start_index + start_len)
-            if end_index == -1:
-                break
-            candidate = s[start_index : end_index + end_len]
-            # Attempt to parse the candidate to ensure it's valid
-            try:
-                cls.parse(candidate)
-                candidates.append(candidate)
-            except tatsu.exceptions.ParseException:
-                # Ignore invalid candidates
-                pass
-            index = end_index + end_len
-        return candidates
-
-    def __str__(self):
-        return self.to_string()
-
-    # def __repr__(self) -> str:
-    #     class_name = self.__class__.__name__
-    #     attributes = []
-    #     for key, value in self.__dict__.items():
-    #         if not key.startswith('_'):  # Skip private attributes
-    #             if isinstance(value, str):
-    #                 # Escape quotes and newlines in string values
-    #                 value_repr = f"'{value.replace('\\', '\\\\').replace(\"'\", \"\\'\").replace('\\n', '\\n')}'"
-    #             else:
-    #                 value_repr = repr(value)
-    #             attributes.append(f"{key}={value_repr}")
-    #     return f"{class_name}({', '.join(attributes)})"
-
-
-if __name__ == "__main__":
-    # Example subclass
-
-    class MyTool(GenericTool):
-        request: str = "my_tool"
-        purpose: str = "do something"
-        value: int
-
-        @classmethod
-        def tool_grammar(cls) -> str:
-            return """
-                tool = "value:" value:number
-                {:
-                    "value:" {{value}}
-                :}
-                    ;
-    
-                number = /\\d+/
-                ;
-            """
-
-    my_tool = MyTool(value=42)
-
-    # Generate the string from the instance using the grammar
-    generated_string = my_tool.format()
-    print("Formatted string:", generated_string)
-
-    # Parse the string back into an instance using the grammar
-    parsed_instance = MyTool.parse(generated_string)
-    print("Parsed instance:", parsed_instance)
-    print("Parsed value:", parsed_instance.value)
-
-    # Extended example
-    class ExtendedModel(GrammarBasedModel):
-        request: str
-        user_id: int
-        action: str
-        details: str
-
-        @classmethod
-        def rest_grammar(cls) -> str:
-            return """
-                rest = user_id:number ws action:word ws details:text ;
-
-                number = /\d+/ ;
-                text = /.+/ ;
-            """
-
-    input_string_ext = "<spec> user_update 42 delete Account deletion</spec>"
-    extended_instance = ExtendedModel.parse(input_string_ext)
-    print("Parsed extended instance:", extended_instance)
-
-    generated_string_ext = extended_instance.generate()
-    print("Generated extended string:", generated_string_ext)

langroid/agent/tools/grammar_based_model.py

@@ -1,132 +0,0 @@
-from abc import ABC, abstractmethod
-
-from lark import Lark, Token, Tree
-from lark.reconstruct import Reconstructor
-
-from langroid.pydantic_v1 import BaseModel
-
-
-class GrammarBasedModel(BaseModel, ABC):
-    _parse_tree: Tree = None  # Store the parse tree
-    _parser: Lark = None  # Store the parser instance
-
-    @classmethod
-    @abstractmethod
-    def get_grammar(cls) -> str:
-        """
-        Subclasses must implement this method to return their grammar as a string.
-        """
-        pass
-
-    @classmethod
-    def get_token_field_mapping(cls):
-        """
-        Returns a mapping from token types to model field names. Subclasses can override this
-        if their token types and field names differ.
-        """
-        return {}
-
-    @classmethod
-    def parse(cls, text: str):
-        """
-        Parse the input text using the grammar to create an instance of the model.
-        """
-        parser = Lark(cls.get_grammar(), parser="lalr", propagate_positions=True)
-        tree = parser.parse(text)
-        model_instance = cls.from_tree(tree)
-        model_instance._parse_tree = tree  # Store the parse tree in the instance
-        model_instance._parser = parser  # Store the parser in the instance
-        return model_instance
-
-    @classmethod
-    def from_tree(cls, tree: Tree):
-        """
-        Convert a parse tree into a model instance.
-        """
-        data = cls.tree_to_dict(tree)
-        return cls(**data)
-
-    @classmethod
-    def tree_to_dict(cls, tree: Tree):
-        """
-        Recursively convert a parse tree into a dictionary of field values.
-        """
-        data = {}
-        token_field_mapping = cls.get_token_field_mapping()
-        for child in tree.children:
-            if isinstance(child, Tree):
-                data.update(cls.tree_to_dict(child))
-            elif isinstance(child, Token):
-                token_type = child.type
-                field_name = token_field_mapping.get(token_type, token_type.lower())
-                data[field_name] = child.value
-        return data
-
-    def generate(self) -> str:
-        """
-        Generate a string representation of the model instance using the grammar.
-        """
-        if self._parse_tree is None or self._parser is None:
-            raise ValueError("Cannot generate text without parsing first.")
-        # Update the parse tree with current model data
-        self.update_tree_with_model_data(self._parse_tree)
-        reconstructor = Reconstructor(self._parser)
-        text = reconstructor.reconstruct(self._parse_tree)
-        return text
-
-    def update_tree_with_model_data(self, tree: Tree):
-        """
-        Update the parse tree with the current model data.
-        """
-        token_field_mapping = self.get_token_field_mapping()
-        reverse_mapping = {v: k for k, v in token_field_mapping.items()}
-        for idx, child in enumerate(tree.children):
-            if isinstance(child, Tree):
-                self.update_tree_with_model_data(child)
-            elif isinstance(child, Token):
-                field_name = token_field_mapping.get(child.type, child.type.lower())
-                if hasattr(self, field_name):
-                    new_value = getattr(self, field_name)
-                    tree.children[idx] = Token(child.type, str(new_value))
-
-
-# Example subclass
-class MyModel(GrammarBasedModel):
-    name: str
-    age: int
-
-    @classmethod
-    def get_grammar(cls):
-        return """
-            start: "name:" NAME "age:" NUMBER
-            %import common.CNAME -> NAME
-            %import common.INT -> NUMBER
-            %import common.WS
-            %ignore WS
-            """
-
-    @classmethod
-    def get_token_field_mapping(cls):
-        return {
-            "NAME": "name",
-            "NUMBER": "age",
-        }
-
-
-# Usage example
-if __name__ == "__main__":
-    text = "name: Alice age: 30"
-    model = MyModel.parse(text)
-    print("Parsed Model:", model)
-
-    # Generate string from the model
-    generated_text = model.generate()
-    print("Generated Text:", generated_text)
-
-    # Modify the model
-    model.age = 31
-    model.name = "Bob"
-
-    # Generate updated string
-    updated_text = model.generate()
-    print("Updated Generated Text:", updated_text)

langroid/agent/tools/grammar_based_model.py-e

@@ -1,128 +0,0 @@
-from pydantic import BaseModel
-from abc import ABC, abstractmethod
-from lark import Lark, Tree, Token
-from lark.reconstruct import Reconstructor
-
-class GrammarBasedModel(BaseModel, ABC):
-    _parse_tree: Tree = None  # Store the parse tree
-    _parser: Lark = None      # Store the parser instance
-
-    @classmethod
-    @abstractmethod
-    def get_grammar(cls) -> str:
-        """
-        Subclasses must implement this method to return their grammar as a string.
-        """
-        pass
-
-    @classmethod
-    def get_token_field_mapping(cls):
-        """
-        Returns a mapping from token types to model field names. Subclasses can override this
-        if their token types and field names differ.
-        """
-        return {}
-
-    @classmethod
-    def parse(cls, text: str):
-        """
-        Parse the input text using the grammar to create an instance of the model.
-        """
-        parser = Lark(cls.get_grammar(), parser='lalr', propagate_positions=True)
-        tree = parser.parse(text)
-        model_instance = cls.from_tree(tree)
-        model_instance._parse_tree = tree  # Store the parse tree in the instance
-        model_instance._parser = parser    # Store the parser in the instance
-        return model_instance
-
-    @classmethod
-    def from_tree(cls, tree: Tree):
-        """
-        Convert a parse tree into a model instance.
-        """
-        data = cls.tree_to_dict(tree)
-        return cls(**data)
-
-    @classmethod
-    def tree_to_dict(cls, tree: Tree):
-        """
-        Recursively convert a parse tree into a dictionary of field values.
-        """
-        data = {}
-        token_field_mapping = cls.get_token_field_mapping()
-        for child in tree.children:
-            if isinstance(child, Tree):
-                data.update(cls.tree_to_dict(child))
-            elif isinstance(child, Token):
-                token_type = child.type
-                field_name = token_field_mapping.get(token_type, token_type.lower())
-                data[field_name] = child.value
-        return data
-
-    def generate(self) -> str:
-        """
-        Generate a string representation of the model instance using the grammar.
-        """
-        if self._parse_tree is None or self._parser is None:
-            raise ValueError("Cannot generate text without parsing first.")
-        # Update the parse tree with current model data
-        self.update_tree_with_model_data(self._parse_tree)
-        reconstructor = Reconstructor(self._parser)
-        text = reconstructor.reconstruct(self._parse_tree)
-        return text
-
-    def update_tree_with_model_data(self, tree: Tree):
-        """
-        Update the parse tree with the current model data.
-        """
-        token_field_mapping = self.get_token_field_mapping()
-        reverse_mapping = {v: k for k, v in token_field_mapping.items()}
-        for idx, child in enumerate(tree.children):
-            if isinstance(child, Tree):
-                self.update_tree_with_model_data(child)
-            elif isinstance(child, Token):
-                field_name = token_field_mapping.get(child.type, child.type.lower())
-                if hasattr(self, field_name):
-                    new_value = getattr(self, field_name)
-                    tree.children[idx] = Token(child.type, str(new_value))
-
-# Example subclass
-class MyModel(GrammarBasedModel):
-    name: str
-    age: int
-
-    @classmethod
-    def get_grammar(cls):
-        return """
-            start: "name:" NAME "age:" NUMBER
-            %import common.CNAME -> NAME
-            %import common.INT -> NUMBER
-            %import common.WS
-            %ignore WS
-            """
-
-    @classmethod
-    def get_token_field_mapping(cls):
-        return {
-            'NAME': 'name',
-            'NUMBER': 'age',
-        }
-
-# Usage example
-if __name__ == "__main__":
-    text = "name: Alice age: 30"
-    model = MyModel.parse(text)
-    print("Parsed Model:", model)
-
-    # Generate string from the model
-    generated_text = model.generate()
-    print("Generated Text:", generated_text)
-
-    # Modify the model
-    model.age = 31
-    model.name = "Bob"
-
-    # Generate updated string
-    updated_text = model.generate()
-    print("Updated Generated Text:", updated_text)
-