langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/generic_tool_tatsu.py

@@ -0,0 +1,275 @@
+from abc import abstractmethod
+from typing import List
+
+import tatsu
+
+from langroid.agent.tool_message import ToolMessage
+
+
+class GenericTool(ToolMessage):
+    """
+    Abstract class for a tool whose format is defined by a grammar,
+    and not necessarily JSON-based.
+    Especially useful for tools where we need an LLM to return code.
+    Most LLMs, especially weaker ones, have significant issues
+    (related to unescaped newlines, quotes, etc) when returning code within JSON.
+    """
+
+    @classmethod
+    @abstractmethod
+    def tool_grammar(cls) -> str:
+        """Define the grammar for the `tool` rule"""
+        pass
+
+    @classmethod
+    def grammar(cls) -> str:
+        """
+        Full grammar, including templates for rendering.
+        """
+        base_grammar = """
+            @@grammar :: CombinedGrammar
+            @@whitespace :: /[ \\t]+/
+            @@nameguard :: False
+
+            start
+                =
+                "<spec>" ws?
+                request:word ws?
+                tool
+                ws? "</spec>"
+            {:
+                "<spec> " {{request}} {{tool}} " </spec>"
+            :}
+                ;
+
+            ws = /[\\s]+/ ;
+
+            word = /[^\\s<>/]+/ ;
+        """
+        full_grammar = base_grammar + "\n" + cls.tool_grammar()
+        return full_grammar
+
+    @classmethod
+    def parse(cls, s: str):
+        """
+        Parses a string `s` using the grammar and returns an instance of the subclass.
+        """
+        # Build the parser using the provided grammar with model generation
+        parser = tatsu.compile(cls.grammar(), asmodel=True)
+
+        # Parse the input string to get a model object
+        model = parser.parse(s)
+
+        # Convert the model to a dict, filtering only the expected fields
+        data = {k: getattr(model, k) for k in cls.__fields__ if hasattr(model, k)}
+
+        # Create an instance of the subclass with the parsed data
+        model_instance = cls(**data)
+        return model_instance
+
+    def format(self) -> str:
+        """
+        Generates a string representation of the instance based on the grammar.
+        """
+        # Build the parser using the provided grammar with model generation
+        parser = tatsu.compile(self.grammar(), asmodel=True)
+
+        # Create a model instance
+        model_class = parser.model()
+        model = model_class()
+
+        # Set attributes from the instance, excluding fields not in the grammar
+        for field in self.__fields__:
+            if field == "purpose":
+                continue  # Exclude 'purpose' from rendering
+            setattr(model, field, getattr(self, field))
+
+        # Render the model back to text using the grammar's templates
+        generated_string = model.render()
+        return generated_string
+
+    @classmethod
+    def instructions(cls) -> str:
+        """
+        Generates instructions for formatting an instance, including placeholders
+        and an example output with placeholders.
+        """
+
+        def generate_placeholders(field, prefix=""):
+            placeholders = {}
+            if hasattr(field.type_, "__fields__"):
+                # Nested model
+                for sub_field_name, sub_field in field.type_.__fields__.items():
+                    placeholders.update(
+                        generate_placeholders(
+                            sub_field, prefix=f"{prefix}{field.name}."
+                        )
+                    )
+            elif isinstance(field.type_, type) and issubclass(field.type_, list):
+                # List field
+                placeholders[field.name] = (
+                    f"[<{field.name}_item1>,<{field.name}_item2>,...]"
+                )
+            else:
+                placeholders[field.name] = f"<{prefix}{field.name}>"
+            return placeholders
+
+        # Generate placeholders for all fields
+        placeholders = {}
+        for field_name, field in cls.__fields__.items():
+            placeholders.update(generate_placeholders(field))
+
+        # Build the preamble
+        preamble_lines = ["Placeholders for formatting:"]
+        for field_name, placeholder in placeholders.items():
+            field_type = cls.__fields__[field_name].type_.__name__
+            preamble_lines.append(
+                f"- `{placeholder}`: placeholder for `{field_name}` field (type: `{field_type}`)"
+            )
+        preamble = "\n".join(preamble_lines)
+
+        # Create a placeholder instance
+        placeholder_values = {}
+        for field_name in cls.__fields__:
+            placeholder_values[field_name] = placeholders[field_name]
+        placeholder_instance = cls(**placeholder_values)
+
+        # Generate an example output with placeholders
+        parser = tatsu.compile(cls.grammar())
+        ast = placeholder_instance.to_ast()
+        # Use the placeholders in the AST
+        for key, value in ast.items():
+            ast[key] = placeholders.get(key, value)
+        example_output = parser.render(ast)
+
+        # Combine preamble and example output
+        instructions = f"{preamble}\n\nExample format:\n\n{example_output}"
+        return instructions
+
+    @classmethod
+    def from_ast(cls, ast):
+        """
+        Converts an AST into a model instance.
+        """
+        # Since TatSu produces dicts, we can convert the AST dict to the model
+        return cls(**ast)
+
+    def to_ast(self):
+        """
+        Converts the model instance into an AST (dict).
+        """
+        # Since TatSu expects dicts for rendering, we can use the model's dict
+        return self.dict()
+
+    @classmethod
+    def from_string(cls, input_string: str) -> "CodeFileTool":
+        """Parse a string into a CodeFileTool object, using the TEMPLATE."""
+        parsed_data = cls.parse(input_string)
+        if parsed_data:
+            return cls(**parsed_data)
+        raise ValueError("Invalid input string format")
+
+    def to_string(self) -> str:
+        """Convert a CodeFileTool object to a string, using the TEMPLATE."""
+        return self.format()
+
+    @classmethod
+    def find_candidates(cls, s: str) -> List[str]:
+        """
+        Finds all substrings in `s` that start with start_marker and end with end_marker.
+        """
+        start = "<spec>"  # TODO get from TOOL_BEGIN, TOOL_END
+        end = "</spec>"
+        candidates = []
+        start_len = len(start)
+        end_len = len(end)
+        index = 0
+        while index < len(s):
+            start_index = s.find(start, index)
+            if start_index == -1:
+                break
+            end_index = s.find(end, start_index + start_len)
+            if end_index == -1:
+                break
+            candidate = s[start_index : end_index + end_len]
+            # Attempt to parse the candidate to ensure it's valid
+            try:
+                cls.parse(candidate)
+                candidates.append(candidate)
+            except tatsu.exceptions.ParseException:
+                # Ignore invalid candidates
+                pass
+            index = end_index + end_len
+        return candidates
+
+    def __str__(self):
+        return self.to_string()
+
+    # def __repr__(self) -> str:
+    #     class_name = self.__class__.__name__
+    #     attributes = []
+    #     for key, value in self.__dict__.items():
+    #         if not key.startswith('_'):  # Skip private attributes
+    #             if isinstance(value, str):
+    #                 # Escape quotes and newlines in string values
+    #                 value_repr = f"'{value.replace('\\', '\\\\').replace(\"'\", \"\\'\").replace('\\n', '\\n')}'"
+    #             else:
+    #                 value_repr = repr(value)
+    #             attributes.append(f"{key}={value_repr}")
+    #     return f"{class_name}({', '.join(attributes)})"
+
+
+if __name__ == "__main__":
+    # Example subclass
+
+    class MyTool(GenericTool):
+        request: str = "my_tool"
+        purpose: str = "do something"
+        value: int
+
+        @classmethod
+        def tool_grammar(cls) -> str:
+            return """
+                tool = "value:" value:number
+                {:
+                    "value:" {{value}}
+                :}
+                    ;
+    
+                number = /\\d+/
+                ;
+            """
+
+    my_tool = MyTool(value=42)
+
+    # Generate the string from the instance using the grammar
+    generated_string = my_tool.format()
+    print("Formatted string:", generated_string)
+
+    # Parse the string back into an instance using the grammar
+    parsed_instance = MyTool.parse(generated_string)
+    print("Parsed instance:", parsed_instance)
+    print("Parsed value:", parsed_instance.value)
+
+    # Extended example
+    class ExtendedModel(GrammarBasedModel):
+        request: str
+        user_id: int
+        action: str
+        details: str
+
+        @classmethod
+        def rest_grammar(cls) -> str:
+            return """
+                rest = user_id:number ws action:word ws details:text ;
+
+                number = /\d+/ ;
+                text = /.+/ ;
+            """
+
+    input_string_ext = "<spec> user_update 42 delete Account deletion</spec>"
+    extended_instance = ExtendedModel.parse(input_string_ext)
+    print("Parsed extended instance:", extended_instance)
+
+    generated_string_ext = extended_instance.generate()
+    print("Generated extended string:", generated_string_ext)

langroid/agent/tools/grammar_based_model.py

@@ -0,0 +1,132 @@
+from abc import ABC, abstractmethod
+
+from lark import Lark, Token, Tree
+from lark.reconstruct import Reconstructor
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    _parse_tree: Tree = None  # Store the parse tree
+    _parser: Lark = None  # Store the parser instance
+
+    @classmethod
+    @abstractmethod
+    def get_grammar(cls) -> str:
+        """
+        Subclasses must implement this method to return their grammar as a string.
+        """
+        pass
+
+    @classmethod
+    def get_token_field_mapping(cls):
+        """
+        Returns a mapping from token types to model field names. Subclasses can override this
+        if their token types and field names differ.
+        """
+        return {}
+
+    @classmethod
+    def parse(cls, text: str):
+        """
+        Parse the input text using the grammar to create an instance of the model.
+        """
+        parser = Lark(cls.get_grammar(), parser="lalr", propagate_positions=True)
+        tree = parser.parse(text)
+        model_instance = cls.from_tree(tree)
+        model_instance._parse_tree = tree  # Store the parse tree in the instance
+        model_instance._parser = parser  # Store the parser in the instance
+        return model_instance
+
+    @classmethod
+    def from_tree(cls, tree: Tree):
+        """
+        Convert a parse tree into a model instance.
+        """
+        data = cls.tree_to_dict(tree)
+        return cls(**data)
+
+    @classmethod
+    def tree_to_dict(cls, tree: Tree):
+        """
+        Recursively convert a parse tree into a dictionary of field values.
+        """
+        data = {}
+        token_field_mapping = cls.get_token_field_mapping()
+        for child in tree.children:
+            if isinstance(child, Tree):
+                data.update(cls.tree_to_dict(child))
+            elif isinstance(child, Token):
+                token_type = child.type
+                field_name = token_field_mapping.get(token_type, token_type.lower())
+                data[field_name] = child.value
+        return data
+
+    def generate(self) -> str:
+        """
+        Generate a string representation of the model instance using the grammar.
+        """
+        if self._parse_tree is None or self._parser is None:
+            raise ValueError("Cannot generate text without parsing first.")
+        # Update the parse tree with current model data
+        self.update_tree_with_model_data(self._parse_tree)
+        reconstructor = Reconstructor(self._parser)
+        text = reconstructor.reconstruct(self._parse_tree)
+        return text
+
+    def update_tree_with_model_data(self, tree: Tree):
+        """
+        Update the parse tree with the current model data.
+        """
+        token_field_mapping = self.get_token_field_mapping()
+        reverse_mapping = {v: k for k, v in token_field_mapping.items()}
+        for idx, child in enumerate(tree.children):
+            if isinstance(child, Tree):
+                self.update_tree_with_model_data(child)
+            elif isinstance(child, Token):
+                field_name = token_field_mapping.get(child.type, child.type.lower())
+                if hasattr(self, field_name):
+                    new_value = getattr(self, field_name)
+                    tree.children[idx] = Token(child.type, str(new_value))
+
+
+# Example subclass
+class MyModel(GrammarBasedModel):
+    name: str
+    age: int
+
+    @classmethod
+    def get_grammar(cls):
+        return """
+            start: "name:" NAME "age:" NUMBER
+            %import common.CNAME -> NAME
+            %import common.INT -> NUMBER
+            %import common.WS
+            %ignore WS
+            """
+
+    @classmethod
+    def get_token_field_mapping(cls):
+        return {
+            "NAME": "name",
+            "NUMBER": "age",
+        }
+
+
+# Usage example
+if __name__ == "__main__":
+    text = "name: Alice age: 30"
+    model = MyModel.parse(text)
+    print("Parsed Model:", model)
+
+    # Generate string from the model
+    generated_text = model.generate()
+    print("Generated Text:", generated_text)
+
+    # Modify the model
+    model.age = 31
+    model.name = "Bob"
+
+    # Generate updated string
+    updated_text = model.generate()
+    print("Updated Generated Text:", updated_text)

langroid/agent/tools/grammar_based_model.py-e

@@ -0,0 +1,128 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+from lark import Lark, Tree, Token
+from lark.reconstruct import Reconstructor
+
+class GrammarBasedModel(BaseModel, ABC):
+    _parse_tree: Tree = None  # Store the parse tree
+    _parser: Lark = None      # Store the parser instance
+
+    @classmethod
+    @abstractmethod
+    def get_grammar(cls) -> str:
+        """
+        Subclasses must implement this method to return their grammar as a string.
+        """
+        pass
+
+    @classmethod
+    def get_token_field_mapping(cls):
+        """
+        Returns a mapping from token types to model field names. Subclasses can override this
+        if their token types and field names differ.
+        """
+        return {}
+
+    @classmethod
+    def parse(cls, text: str):
+        """
+        Parse the input text using the grammar to create an instance of the model.
+        """
+        parser = Lark(cls.get_grammar(), parser='lalr', propagate_positions=True)
+        tree = parser.parse(text)
+        model_instance = cls.from_tree(tree)
+        model_instance._parse_tree = tree  # Store the parse tree in the instance
+        model_instance._parser = parser    # Store the parser in the instance
+        return model_instance
+
+    @classmethod
+    def from_tree(cls, tree: Tree):
+        """
+        Convert a parse tree into a model instance.
+        """
+        data = cls.tree_to_dict(tree)
+        return cls(**data)
+
+    @classmethod
+    def tree_to_dict(cls, tree: Tree):
+        """
+        Recursively convert a parse tree into a dictionary of field values.
+        """
+        data = {}
+        token_field_mapping = cls.get_token_field_mapping()
+        for child in tree.children:
+            if isinstance(child, Tree):
+                data.update(cls.tree_to_dict(child))
+            elif isinstance(child, Token):
+                token_type = child.type
+                field_name = token_field_mapping.get(token_type, token_type.lower())
+                data[field_name] = child.value
+        return data
+
+    def generate(self) -> str:
+        """
+        Generate a string representation of the model instance using the grammar.
+        """
+        if self._parse_tree is None or self._parser is None:
+            raise ValueError("Cannot generate text without parsing first.")
+        # Update the parse tree with current model data
+        self.update_tree_with_model_data(self._parse_tree)
+        reconstructor = Reconstructor(self._parser)
+        text = reconstructor.reconstruct(self._parse_tree)
+        return text
+
+    def update_tree_with_model_data(self, tree: Tree):
+        """
+        Update the parse tree with the current model data.
+        """
+        token_field_mapping = self.get_token_field_mapping()
+        reverse_mapping = {v: k for k, v in token_field_mapping.items()}
+        for idx, child in enumerate(tree.children):
+            if isinstance(child, Tree):
+                self.update_tree_with_model_data(child)
+            elif isinstance(child, Token):
+                field_name = token_field_mapping.get(child.type, child.type.lower())
+                if hasattr(self, field_name):
+                    new_value = getattr(self, field_name)
+                    tree.children[idx] = Token(child.type, str(new_value))
+
+# Example subclass
+class MyModel(GrammarBasedModel):
+    name: str
+    age: int
+
+    @classmethod
+    def get_grammar(cls):
+        return """
+            start: "name:" NAME "age:" NUMBER
+            %import common.CNAME -> NAME
+            %import common.INT -> NUMBER
+            %import common.WS
+            %ignore WS
+            """
+
+    @classmethod
+    def get_token_field_mapping(cls):
+        return {
+            'NAME': 'name',
+            'NUMBER': 'age',
+        }
+
+# Usage example
+if __name__ == "__main__":
+    text = "name: Alice age: 30"
+    model = MyModel.parse(text)
+    print("Parsed Model:", model)
+
+    # Generate string from the model
+    generated_text = model.generate()
+    print("Generated Text:", generated_text)
+
+    # Modify the model
+    model.age = 31
+    model.name = "Bob"
+
+    # Generate updated string
+    updated_text = model.generate()
+    print("Updated Generated Text:", updated_text)
+    

langroid/agent/tools/grammar_based_model_lark.py

@@ -0,0 +1,156 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from lark import Lark, Transformer, Visitor
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_rule(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> "GrammarBasedModel":
+        parser = Lark(cls.grammar(), start=cls.start_rule())
+        tree = parser.parse(text)
+
+        class TreeToDict(Transformer):
+            def __init__(self, field_mappings):
+                self.field_mappings = field_mappings
+
+            def __default__(self, data, children, meta):
+                for field, rule in self.field_mappings.items():
+                    if data == rule:
+                        return {field: children[0]}
+                return children
+
+            def start(self, items):
+                result = {}
+                for item in items:
+                    if isinstance(item, dict):
+                        result.update(item)
+                return result
+
+        transformer = TreeToDict(cls.field_mappings())
+        data = transformer.transform(tree)
+        return cls(**data)
+
+    def generate(self) -> str:
+        parser = Lark(self.grammar(), start=self.start_rule())
+
+        class ModelToString(Visitor):
+            def __init__(self, model):
+                self.model = model
+                self.result = []
+
+            def __default__(self, tree):
+                if tree.data in self.model.field_mappings().values():
+                    field = next(
+                        k
+                        for k, v in self.model.field_mappings().items()
+                        if v == tree.data
+                    )
+                    value = getattr(self.model, field)
+                    self.result.append(f"{' '.join(tree.children)} {value}")
+                else:
+                    for child in tree.children:
+                        if isinstance(child, str):
+                            self.result.append(child)
+                        else:
+                            self.visit(child)
+
+        visitor = ModelToString(self)
+        tree = parser.parse(" ".join(self.grammar().split()))
+        visitor.visit(tree)
+        return " ".join(visitor.result)
+
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls):
+        return """
+        start: "<spec>" name age city "</spec>"
+        name: "name:" WORD
+        age: "age" "is" NUMBER
+        city: "lives" "in" WORD
+        %import common.WORD
+        %import common.NUMBER
+        %import common.WS
+        %ignore WS
+        """
+
+    @classmethod
+    def start_rule(cls):
+        return "start"
+
+    @classmethod
+    def field_mappings(cls):
+        return {"name": "name", "age": "age", "city": "city"}
+
+
+if __name__ == "__main__":
+    # Test parsing
+    test_string = """
+    <spec>
+    name: John
+    age is 30
+    lives in Tokyo
+    </spec>
+    """
+    parsed_person = PersonSpec.parse(test_string)
+    print("Parsed person:", parsed_person)
+
+    # Test generating
+    new_person = PersonSpec(name="Alice", age=25, city="NewYork")
+    generated_string = new_person.generate()
+    print("\nGenerated string:")
+    print(generated_string)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_string)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert new_person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")
+
+    # Test with modified grammar
+    class ModifiedPersonSpec(PersonSpec):
+        @classmethod
+        def grammar(cls):
+            return """
+            start: "<person>" name age city "</person>"
+            name: "Name:" WORD
+            age: "Age:" NUMBER "years"
+            city: "City:" WORD
+            %import common.WORD
+            %import common.NUMBER
+            %import common.WS
+            %ignore WS
+            """
+
+    modified_person = ModifiedPersonSpec(name="Bob", age=40, city="London")
+    modified_string = modified_person.generate()
+    print("\nModified grammar generated string:")
+    print(modified_string)
+
+    parsed_modified = ModifiedPersonSpec.parse(modified_string)
+    print("Parsed modified person:", parsed_modified)
+    assert modified_person == parsed_modified, "Modified grammar round-trip failed"
+    print("Modified grammar round-trip test passed!")