langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/grammar_based_model_lark.py-e

@@ -0,0 +1,153 @@
+from pydantic import BaseModel
+from lark import Lark, Transformer, Visitor
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_rule(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> 'GrammarBasedModel':
+        parser = Lark(cls.grammar(), start=cls.start_rule())
+        tree = parser.parse(text)
+        
+        class TreeToDict(Transformer):
+            def __init__(self, field_mappings):
+                self.field_mappings = field_mappings
+            
+            def __default__(self, data, children, meta):
+                for field, rule in self.field_mappings.items():
+                    if data == rule:
+                        return {field: children[0]}
+                return children
+            
+            def start(self, items):
+                result = {}
+                for item in items:
+                    if isinstance(item, dict):
+                        result.update(item)
+                return result
+
+        transformer = TreeToDict(cls.field_mappings())
+        data = transformer.transform(tree)
+        return cls(**data)
+
+    def generate(self) -> str:
+        parser = Lark(self.grammar(), start=self.start_rule())
+        
+        class ModelToString(Visitor):
+            def __init__(self, model):
+                self.model = model
+                self.result = []
+
+            def __default__(self, tree):
+                if tree.data in self.model.field_mappings().values():
+                    field = next(k for k, v in self.model.field_mappings().items() if v == tree.data)
+                    value = getattr(self.model, field)
+                    self.result.append(f"{' '.join(tree.children)} {value}")
+                else:
+                    for child in tree.children:
+                        if isinstance(child, str):
+                            self.result.append(child)
+                        else:
+                            self.visit(child)
+
+        visitor = ModelToString(self)
+        tree = parser.parse(" ".join(self.grammar().split()))
+        visitor.visit(tree)
+        return " ".join(visitor.result)
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls):
+        return """
+        start: "<spec>" name age city "</spec>"
+        name: "name:" WORD
+        age: "age" "is" NUMBER
+        city: "lives" "in" WORD
+        %import common.WORD
+        %import common.NUMBER
+        %import common.WS
+        %ignore WS
+        """
+
+    @classmethod
+    def start_rule(cls):
+        return "start"
+
+    @classmethod
+    def field_mappings(cls):
+        return {
+            "name": "name",
+            "age": "age",
+            "city": "city"
+        }
+
+if __name__ == "__main__":
+    # Test parsing
+    test_string = """
+    <spec>
+    name: John
+    age is 30
+    lives in Tokyo
+    </spec>
+    """
+    parsed_person = PersonSpec.parse(test_string)
+    print("Parsed person:", parsed_person)
+
+    # Test generating
+    new_person = PersonSpec(name="Alice", age=25, city="NewYork")
+    generated_string = new_person.generate()
+    print("\nGenerated string:")
+    print(generated_string)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_string)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert new_person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")
+
+    # Test with modified grammar
+    class ModifiedPersonSpec(PersonSpec):
+        @classmethod
+        def grammar(cls):
+            return """
+            start: "<person>" name age city "</person>"
+            name: "Name:" WORD
+            age: "Age:" NUMBER "years"
+            city: "City:" WORD
+            %import common.WORD
+            %import common.NUMBER
+            %import common.WS
+            %ignore WS
+            """
+
+    modified_person = ModifiedPersonSpec(name="Bob", age=40, city="London")
+    modified_string = modified_person.generate()
+    print("\nModified grammar generated string:")
+    print(modified_string)
+
+    parsed_modified = ModifiedPersonSpec.parse(modified_string)
+    print("Parsed modified person:", parsed_modified)
+    assert modified_person == parsed_modified, "Modified grammar round-trip failed"
+    print("Modified grammar round-trip test passed!")
+    
+    

langroid/agent/tools/grammar_based_model_parse.py

@@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+
+from parse import compile
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls, string: str):
+        parser = compile(cls.grammar())
+        result = parser.parse(string)
+        if result is None:
+            raise ValueError("Invalid string format")
+        return cls(**result.named)
+
+    def generate(self) -> str:
+        return self.grammar().format(**self.dict())
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class Person(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls) -> str:
+        return """
+{:s}<spec>{:s}
+{:s}name={name:S}{:s}
+{:s}age={age:d}{:s}
+{:s}city={city:S}{:s}
+{:s}</spec>{:s}
+"""
+
+
+class SimpleFormat(GrammarBasedModel):
+    key: str
+    value: str
+
+    @classmethod
+    def grammar(cls) -> str:
+        return "{:s}{key:S}{:s}:{:s}{value:S}{:s}"
+
+
+if __name__ == "__main__":
+    # Test Person class
+    input_string = """
+    <spec>
+    name=John Doe
+    age=30
+    city=New York
+    </spec>
+    """
+
+    person = Person.parse(input_string)
+    print("Parsed person:", person)
+
+    generated_string = person.generate()
+    print("Generated string:")
+    print(generated_string)
+
+    # Test SimpleFormat class
+    simple_input = "  Hello  :  World  "
+    simple = SimpleFormat.parse(simple_input)
+    print("Parsed simple format:", simple)
+
+    simple_generated = simple.generate()
+    print("Generated simple format:", simple_generated)
+
+    # Test with different whitespace
+    input_string2 = "<spec>\nname=Jane Smith\n\n  age=25\n\t\tcity=London\n</spec>"
+    person2 = Person.parse(input_string2)
+    print("Parsed person with different whitespace:", person2)
+
+    generated_string2 = person2.generate()
+    print("Generated string for person2:")
+    print(generated_string2)

langroid/agent/tools/grammar_based_model_parse.py-e

@@ -0,0 +1,80 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+from parse import compile
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls, string: str):
+        parser = compile(cls.grammar())
+        result = parser.parse(string)
+        if result is None:
+            raise ValueError("Invalid string format")
+        return cls(**result.named)
+
+    def generate(self) -> str:
+        return self.grammar().format(**self.dict())
+
+    class Config:
+        arbitrary_types_allowed = True
+
+class Person(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls) -> str:
+        return """
+{:s}<spec>{:s}
+{:s}name={name:S}{:s}
+{:s}age={age:d}{:s}
+{:s}city={city:S}{:s}
+{:s}</spec>{:s}
+"""
+
+class SimpleFormat(GrammarBasedModel):
+    key: str
+    value: str
+
+    @classmethod
+    def grammar(cls) -> str:
+        return "{:s}{key:S}{:s}:{:s}{value:S}{:s}"
+
+if __name__ == "__main__":
+    # Test Person class
+    input_string = """
+    <spec>
+    name=John Doe
+    age=30
+    city=New York
+    </spec>
+    """
+    
+    person = Person.parse(input_string)
+    print("Parsed person:", person)
+
+    generated_string = person.generate()
+    print("Generated string:")
+    print(generated_string)
+
+    # Test SimpleFormat class
+    simple_input = "  Hello  :  World  "
+    simple = SimpleFormat.parse(simple_input)
+    print("Parsed simple format:", simple)
+
+    simple_generated = simple.generate()
+    print("Generated simple format:", simple_generated)
+
+    # Test with different whitespace
+    input_string2 = "<spec>\nname=Jane Smith\n\n  age=25\n\t\tcity=London\n</spec>"
+    person2 = Person.parse(input_string2)
+    print("Parsed person with different whitespace:", person2)
+
+    generated_string2 = person2.generate()
+    print("Generated string for person2:")
+    print(generated_string2)

langroid/agent/tools/grammar_based_model_parsimonious.py

@@ -0,0 +1,129 @@
+from abc import ABC, abstractmethod
+from typing import ClassVar, Dict
+
+from parsimonious import Grammar, NodeVisitor
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    grammar: ClassVar[str]
+    start_token: ClassVar[str]
+    end_token: ClassVar[str]
+    field_mappings: ClassVar[Dict[str, str]]
+
+    @classmethod
+    @abstractmethod
+    def get_grammar(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> "GrammarBasedModel":
+        grammar = Grammar(cls.get_grammar())
+        tree = grammar.parse(text[len(cls.start_token) : -len(cls.end_token)].strip())
+
+        class ModelVisitor(NodeVisitor):
+            def __init__(self, field_mappings):
+                self.field_mappings = field_mappings
+                self.data = {}
+
+            def generic_visit(self, node, visited_children):
+                return visited_children or node.text
+
+            def visit_start(self, node, visited_children):
+                return self.data
+
+            def __getattr__(self, name):
+                if name.startswith("visit_"):
+                    field = name[6:]
+                    if field in self.field_mappings.values():
+
+                        def visit_method(node, visited_children):
+                            model_field = next(
+                                k for k, v in self.field_mappings.items() if v == field
+                            )
+                            self.data[model_field] = node.text.strip()
+                            return node.text
+
+                        return visit_method
+                return super().__getattribute__(name)
+
+        visitor = ModelVisitor(cls.field_mappings)
+        model_dict = visitor.visit(tree)
+        return cls(**model_dict)
+
+    def generate(self) -> str:
+        grammar = Grammar(self.get_grammar())
+
+        class ModelGenerator(NodeVisitor):
+            def __init__(self, model):
+                self.model = model
+
+            def generic_visit(self, node, visited_children):
+                return "".join(filter(None, visited_children))
+
+            def __getattr__(self, name):
+                if name.startswith("visit_"):
+                    field = name[6:]
+                    if field in self.model.field_mappings.values():
+                        model_field = next(
+                            k
+                            for k, v in self.model.field_mappings.items()
+                            if v == field
+                        )
+                        return lambda node, children: str(
+                            getattr(self.model, model_field)
+                        )
+                return lambda node, children: node.text
+
+        generator = ModelGenerator(self)
+        generated_content = generator.visit(grammar["start"])
+        return f"{self.start_token}\n{generated_content}\n{self.end_token}"
+
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    grammar = r"""
+        start = name_line age_line city_line
+        name_line = "name:" ws name newline
+        age_line = "age is" ws age newline
+        city_line = "lives in" ws city newline?
+        name = ~r"[^\n]+"
+        age = ~r"\d+"
+        city = ~r"[^\n]+"
+        ws = ~r"\s+"
+        newline = ~r"\n"
+    """
+    start_token = "<spec>"
+    end_token = "</spec>"
+    field_mappings = {"name": "name", "age": "age", "city": "city"}
+
+    @classmethod
+    def get_grammar(cls):
+        return cls.grammar
+
+
+if __name__ == "__main__":
+    # Test parsing
+    input_str = """<spec>
+name: John Doe
+age is 30
+lives in New York
+</spec>"""
+    person = PersonSpec.parse(input_str)
+    print("Parsed person:", person)
+
+    # Test generation
+    generated_str = person.generate()
+    print("\nGenerated string:")
+    print(generated_str)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_str)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")

langroid/agent/tools/grammar_based_model_parsimonious.py-e

@@ -0,0 +1,120 @@
+from typing import Dict, ClassVar
+from pydantic import BaseModel
+from parsimonious import Grammar, NodeVisitor
+from abc import ABC, abstractmethod
+
+class GrammarBasedModel(BaseModel, ABC):
+    grammar: ClassVar[str]
+    start_token: ClassVar[str]
+    end_token: ClassVar[str]
+    field_mappings: ClassVar[Dict[str, str]]
+
+    @classmethod
+    @abstractmethod
+    def get_grammar(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> 'GrammarBasedModel':
+        grammar = Grammar(cls.get_grammar())
+        tree = grammar.parse(text[len(cls.start_token):-len(cls.end_token)].strip())
+
+        class ModelVisitor(NodeVisitor):
+            def __init__(self, field_mappings):
+                self.field_mappings = field_mappings
+                self.data = {}
+
+            def generic_visit(self, node, visited_children):
+                return visited_children or node.text
+
+            def visit_start(self, node, visited_children):
+                return self.data
+
+            def __getattr__(self, name):
+                if name.startswith('visit_'):
+                    field = name[6:]
+                    if field in self.field_mappings.values():
+                        def visit_method(node, visited_children):
+                            model_field = next(k for k, v in self.field_mappings.items() if v == field)
+                            self.data[model_field] = node.text.strip()
+                            return node.text
+                        return visit_method
+                return super().__getattribute__(name)
+
+        visitor = ModelVisitor(cls.field_mappings)
+        model_dict = visitor.visit(tree)
+        return cls(**model_dict)
+
+    def generate(self) -> str:
+        grammar = Grammar(self.get_grammar())
+
+        class ModelGenerator(NodeVisitor):
+            def __init__(self, model):
+                self.model = model
+
+            def generic_visit(self, node, visited_children):
+                return ''.join(filter(None, visited_children))
+
+            def __getattr__(self, name):
+                if name.startswith('visit_'):
+                    field = name[6:]
+                    if field in self.model.field_mappings.values():
+                        model_field = next(k for k, v in self.model.field_mappings.items() if v == field)
+                        return lambda node, children: str(getattr(self.model, model_field))
+                return lambda node, children: node.text
+
+        generator = ModelGenerator(self)
+        generated_content = generator.visit(grammar['start'])
+        return f"{self.start_token}\n{generated_content}\n{self.end_token}"
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    grammar = r"""
+        start = name_line age_line city_line
+        name_line = "name:" ws name newline
+        age_line = "age is" ws age newline
+        city_line = "lives in" ws city newline?
+        name = ~r"[^\n]+"
+        age = ~r"\d+"
+        city = ~r"[^\n]+"
+        ws = ~r"\s+"
+        newline = ~r"\n"
+    """
+    start_token = "<spec>"
+    end_token = "</spec>"
+    field_mappings = {
+        "name": "name",
+        "age": "age",
+        "city": "city"
+    }
+
+    @classmethod
+    def get_grammar(cls):
+        return cls.grammar
+
+if __name__ == "__main__":
+    # Test parsing
+    input_str = """<spec>
+name: John Doe
+age is 30
+lives in New York
+</spec>"""
+    person = PersonSpec.parse(input_str)
+    print("Parsed person:", person)
+
+    # Test generation
+    generated_str = person.generate()
+    print("\nGenerated string:")
+    print(generated_str)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_str)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")
+    
+    

langroid/agent/tools/grammar_based_model_pyparsing.py

@@ -0,0 +1,105 @@
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from pyparsing import *
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> ParserElement:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> "GrammarBasedModel":
+        full_grammar = (
+            Suppress(cls.start_token()) + cls.grammar() + Suppress(cls.end_token())
+        )
+        parsed = full_grammar.parseString(text, parseAll=True)
+        return cls(
+            **{
+                field: parsed[token][-1]
+                for field, token in cls.field_mappings().items()
+            }
+        )
+
+    def generate(self) -> str:
+        result = [self.start_token()]
+        for field, token in self.field_mappings().items():
+            value = getattr(self, field)
+            if token == "name":
+                result.append(f"name: {value}")
+            elif token == "age":
+                result.append(f"age is {value}")
+            elif token == "city":
+                result.append(f"lives in {value}")
+        result.append(self.end_token())
+        return "\n".join(result)
+
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls):
+        name = Group(Literal("name:") + Word(alphas))("name")
+        age = Group(Literal("age is") + Word(nums))("age")
+        city = Group(Literal("lives in") + Word(alphas))("city")
+        return name + age + city
+
+    @classmethod
+    def start_token(cls):
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls):
+        return "</spec>"
+
+    @classmethod
+    def field_mappings(cls):
+        return {"name": "name", "age": "age", "city": "city"}
+
+
+if __name__ == "__main__":
+    # Test parsing
+    test_string = """
+    <spec>
+    name: John
+    age is 30
+    lives in Tokyo
+    </spec>
+    """
+    parsed_person = PersonSpec.parse(test_string)
+    print("Parsed person:", parsed_person)
+
+    # Test generating
+    new_person = PersonSpec(name="Alice", age=25, city="NewYork")
+    generated_string = new_person.generate()
+    print("\nGenerated string:")
+    print(generated_string)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_string)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert new_person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")