langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/formatted_model_parsimon.py

@@ -0,0 +1,194 @@
+from abc import ABC, abstractmethod
+
+from parsimonious import Grammar, NodeVisitor, exceptions
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_visitor(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def format(cls, instance: "FormattingModel") -> str:
+        spec = cls.format_spec()
+        formatted = spec.format(**instance.dict())
+        return f"{cls.start_token()}\n{formatted}\n{cls.end_token()}"
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> "FormattingModel":
+        lines = formatted_string.strip().split("\n")
+        if lines[0] != cls.start_token() or lines[-1] != cls.end_token():
+            raise ValueError("Invalid start or end token")
+        content = "\n".join(lines[1:-1])
+
+        grammar = Grammar(cls.parse_spec())
+        try:
+            tree = grammar.parse(content)
+        except exceptions.ParseError as e:
+            raise ValueError(f"Failed to parse content: {e}\nContent:\n{content}")
+
+        visitor = cls.parse_visitor()()
+        try:
+            visitor_result = visitor.visit(tree)
+        except Exception as e:
+            raise ValueError(f"Error during tree visitation: {e}")
+
+        if not all(field in visitor_result for field in cls.__fields__):
+            missing_fields = [
+                field for field in cls.__fields__ if field not in visitor_result
+            ]
+            raise ValueError(
+                f"Missing fields after parsing: {', '.join(missing_fields)}"
+            )
+
+        return cls(**visitor_result)
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls):
+        return "code_file_model\n{file_path}\n```{language}\n{code}\n```"
+
+    @classmethod
+    def parse_spec(cls):
+        return """
+        model = "code_file_model" newline file_path newline code_block
+        file_path = line
+        code_block = "```" language newline code "```"
+        language = ~"[^\\n]+"
+        code = (!(~"```") (newline / any_char))*
+        line = ~"[^\\n]+"
+        newline = ~"\\s*\\n\\s*"
+        any_char = ~"."
+        """
+
+    @classmethod
+    def parse_visitor(cls):
+        class Visitor(NodeVisitor):
+            def __init__(self):
+                self.data = {}
+
+            def visit_model(self, node, visited_children):
+                return self.data
+
+            def visit_file_path(self, node, visited_children):
+                self.data["file_path"] = node.text.strip()
+
+            def visit_language(self, node, visited_children):
+                self.data["language"] = node.text.strip()
+
+            def visit_code(self, node, visited_children):
+                self.data["code"] = node.text.strip()
+
+            def generic_visit(self, node, visited_children):
+                return visited_children or node
+
+        return Visitor
+
+    @classmethod
+    def start_token(cls):
+        return "<format>"
+
+    @classmethod
+    def end_token(cls):
+        return "</format>"
+
+
+# Test cases
+if __name__ == "__main__":
+    # Test formatting
+    code_file = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    formatted = CodeFileModel.format(code_file)
+    expected_format = """<format>
+code_file_model
+src/main.py
+```Python
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    assert (
+        formatted == expected_format
+    ), f"Formatting failed. Expected:\n{expected_format}\nGot:\n{formatted}"
+    print("Formatting test passed.")
+
+    # Test parsing
+    parsed = CodeFileModel.parse(formatted)
+    assert (
+        parsed == code_file
+    ), f"Parsing failed. Expected:\n{code_file}\nGot:\n{parsed}"
+    print("Parsing test passed.")
+
+    # Test round-trip
+    round_trip = CodeFileModel.parse(CodeFileModel.format(code_file))
+    assert (
+        round_trip == code_file
+    ), f"Round-trip failed. Expected:\n{code_file}\nGot:\n{round_trip}"
+    print("Round-trip test passed.")
+
+    # Test with different values
+    code_file2 = CodeFileModel(
+        language="JavaScript",
+        file_path="src/app.js",
+        code="function greet() {\n  console.log('Hello, World!');\n}",
+    )
+    formatted2 = CodeFileModel.format(code_file2)
+    parsed2 = CodeFileModel.parse(formatted2)
+    assert (
+        parsed2 == code_file2
+    ), f"Parsing failed for different values. Expected:\n{code_file2}\nGot:\n{parsed2}"
+    print("Different values test passed.")
+
+    # Test tolerant parsing
+    tolerant_input = """<format>
+code_file_model
+   src/main.py   
+
+```  Python  
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    parsed_tolerant = CodeFileModel.parse(tolerant_input)
+    expected_tolerant = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    assert (
+        parsed_tolerant == expected_tolerant
+    ), f"Tolerant parsing failed. Expected:\n{expected_tolerant}\nGot:\n{parsed_tolerant}"
+    print("Tolerant parsing test passed.")
+
+    print("All tests passed successfully!")

langroid/agent/tools/formatted_model_parsimon.py-e

@@ -0,0 +1,186 @@
+from abc import ABC, abstractmethod
+from pydantic import BaseModel
+from parsimonious import Grammar, NodeVisitor, exceptions
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_visitor(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def format(cls, instance: 'FormattingModel') -> str:
+        spec = cls.format_spec()
+        formatted = spec.format(**instance.dict())
+        return f"{cls.start_token()}\n{formatted}\n{cls.end_token()}"
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> 'FormattingModel':
+        lines = formatted_string.strip().split('\n')
+        if lines[0] != cls.start_token() or lines[-1] != cls.end_token():
+            raise ValueError("Invalid start or end token")
+        content = '\n'.join(lines[1:-1])
+        
+        grammar = Grammar(cls.parse_spec())
+        try:
+            tree = grammar.parse(content)
+        except exceptions.ParseError as e:
+            raise ValueError(f"Failed to parse content: {e}\nContent:\n{content}")
+        
+        visitor = cls.parse_visitor()()
+        try:
+            visitor_result = visitor.visit(tree)
+        except Exception as e:
+            raise ValueError(f"Error during tree visitation: {e}")
+        
+        if not all(field in visitor_result for field in cls.__fields__):
+            missing_fields = [field for field in cls.__fields__ if field not in visitor_result]
+            raise ValueError(f"Missing fields after parsing: {', '.join(missing_fields)}")
+        
+        return cls(**visitor_result)
+
+
+from parsimonious import Grammar, NodeVisitor
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls):
+        return "code_file_model\n{file_path}\n```{language}\n{code}\n```"
+
+    @classmethod
+    def parse_spec(cls):
+        return """
+        model = "code_file_model" newline file_path newline code_block
+        file_path = line
+        code_block = "```" language newline code "```"
+        language = ~"[^\\n]+"
+        code = (!(~"```") (newline / any_char))*
+        line = ~"[^\\n]+"
+        newline = ~"\\s*\\n\\s*"
+        any_char = ~"."
+        """
+
+    @classmethod
+    def parse_visitor(cls):
+        class Visitor(NodeVisitor):
+            def __init__(self):
+                self.data = {}
+
+            def visit_model(self, node, visited_children):
+                return self.data
+
+            def visit_file_path(self, node, visited_children):
+                self.data['file_path'] = node.text.strip()
+
+            def visit_language(self, node, visited_children):
+                self.data['language'] = node.text.strip()
+
+            def visit_code(self, node, visited_children):
+                self.data['code'] = node.text.strip()
+
+            def generic_visit(self, node, visited_children):
+                return visited_children or node
+
+        return Visitor
+
+    @classmethod
+    def start_token(cls):
+        return '<format>'
+
+    @classmethod
+    def end_token(cls):
+        return '</format>'
+        
+
+        
+        
+
+
+        
+        
+# Test cases
+if __name__ == "__main__":
+    # Test formatting
+    code_file = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')"
+    )
+    formatted = CodeFileModel.format(code_file)
+    expected_format = """<format>
+code_file_model
+src/main.py
+```Python
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    assert formatted == expected_format, f"Formatting failed. Expected:\n{expected_format}\nGot:\n{formatted}"
+    print("Formatting test passed.")
+
+    # Test parsing
+    parsed = CodeFileModel.parse(formatted)
+    assert parsed == code_file, f"Parsing failed. Expected:\n{code_file}\nGot:\n{parsed}"
+    print("Parsing test passed.")
+
+    # Test round-trip
+    round_trip = CodeFileModel.parse(CodeFileModel.format(code_file))
+    assert round_trip == code_file, f"Round-trip failed. Expected:\n{code_file}\nGot:\n{round_trip}"
+    print("Round-trip test passed.")
+
+    # Test with different values
+    code_file2 = CodeFileModel(
+        language="JavaScript",
+        file_path="src/app.js",
+        code="function greet() {\n  console.log('Hello, World!');\n}"
+    )
+    formatted2 = CodeFileModel.format(code_file2)
+    parsed2 = CodeFileModel.parse(formatted2)
+    assert parsed2 == code_file2, f"Parsing failed for different values. Expected:\n{code_file2}\nGot:\n{parsed2}"
+    print("Different values test passed.")
+
+    # Test tolerant parsing
+    tolerant_input = """<format>
+code_file_model
+   src/main.py   
+
+```  Python  
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    parsed_tolerant = CodeFileModel.parse(tolerant_input)
+    expected_tolerant = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')"
+    )
+    assert parsed_tolerant == expected_tolerant, f"Tolerant parsing failed. Expected:\n{expected_tolerant}\nGot:\n{parsed_tolerant}"
+    print("Tolerant parsing test passed.")
+
+    print("All tests passed successfully!")
+    

langroid/agent/tools/formatted_model_pyparsing.py

@@ -0,0 +1,169 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Type, TypeVar
+
+from pyparsing import (
+    LineEnd,
+    ParseException,
+    ParserElement,
+    Regex,
+    SkipTo,
+    Suppress,
+    Word,
+    alphanums,
+)
+
+from langroid.pydantic_v1 import BaseModel
+
+T = TypeVar("T", bound="FormattingModel")
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()) : -len(cls.end_token())].strip()
+        try:
+            _, parser = cls.format_spec()
+            parsed = parser.parseString(content)
+            data = {
+                field: parsed[token].strip()
+                for field, token in cls.field_mappings().items()
+            }
+            return cls(**data)
+        except ParseException as e:
+            print(f"Parsing error: {e}")
+            raise
+
+    def generate(self) -> str:
+        template, _ = self.format_spec()
+        for field, token in self.field_mappings().items():
+            value = getattr(self, field)
+            template = template.replace(f"{{{token}}}", str(value))
+        return f"{self.start_token()}\n{template}\n{self.end_token()}"
+
+
+class MyFormattedModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        template = "name: {NAME}\n{AGE} is the age\nlives in {CITY}"
+        name = Suppress("name:") + Word(alphanums + " ")("NAME") + LineEnd()
+        age = Word(alphanums)("AGE") + Suppress("is the age") + LineEnd()
+        city = Suppress("lives in") + SkipTo(LineEnd())("CITY")
+        parser = name + age + city
+        return template, parser
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {"name": "NAME", "age": "AGE", "city": "CITY"}
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        template = "code_file_model\nfile_path: {FILE_PATH}\n```{LANGUAGE}\n{CODE}\n```"
+        file_path = (
+            Suppress("code_file_model")
+            + LineEnd()
+            + Suppress("file_path:")
+            + SkipTo(LineEnd())("FILE_PATH")
+            + LineEnd()
+        )
+        code_block = (
+            Suppress("```")
+            + Word(alphanums)("LANGUAGE")
+            + LineEnd()
+            + Regex(r"(?s).*?(?=```)").set_parse_action(lambda s, l, t: t[0].strip())(
+                "CODE"
+            )
+            + Suppress("```")
+        )
+        parser = file_path + code_block
+        return template, parser
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {"file_path": "FILE_PATH", "language": "LANGUAGE", "code": "CODE"}
+
+
+if __name__ == "__main__":
+    # Test MyFormattedModel
+    model = MyFormattedModel(name="John", age=30, city="Tokyo")
+    generated = model.generate()
+    print("Generated MyFormattedModel string:")
+    print(generated)
+    print()
+
+    parsed = MyFormattedModel.parse(generated)
+    print("Parsed MyFormattedModel object:")
+    print(parsed)
+    print()
+
+    print("MyFormattedModel Round-trip test:")
+    assert model == parsed, "MyFormattedModel: Original != Parsed"
+    print("Passed!")
+    print()
+
+    # Test CodeFileModel
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code='def hello():\n    print("Hello, World!")',
+    )
+    code_generated = code_model.generate()
+    print("Generated CodeFileModel string:")
+    print(code_generated)
+    print()
+
+    code_parsed = CodeFileModel.parse(code_generated)
+    print("Parsed CodeFileModel object:")
+    print(code_parsed)
+    print()
+
+    print("CodeFileModel Round-trip test:")
+    assert code_model == code_parsed, "CodeFileModel: Original != Parsed"
+    print("Passed!")
+
+    print("\nAll tests passed successfully!")

langroid/agent/tools/formatted_model_pyparsing.py-e

@@ -0,0 +1,149 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+from typing import Dict, Type, TypeVar
+from pyparsing import (
+    Word, alphanums, Suppress, SkipTo, LineEnd, 
+    QuotedString, delimitedList, Group, OneOrMore, ParseException,
+    ParserElement, Regex
+)
+
+T = TypeVar('T', bound='FormattingModel')
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()):-len(cls.end_token())].strip()
+        try:
+            _, parser = cls.format_spec()
+            parsed = parser.parseString(content)
+            data = {field: parsed[token].strip() for field, token in cls.field_mappings().items()}
+            return cls(**data)
+        except ParseException as e:
+            print(f"Parsing error: {e}")
+            raise
+
+    def generate(self) -> str:
+        template, _ = self.format_spec()
+        for field, token in self.field_mappings().items():
+            value = getattr(self, field)
+            template = template.replace(f"{{{token}}}", str(value))
+        return f"{self.start_token()}\n{template}\n{self.end_token()}"
+
+class MyFormattedModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        template = "name: {NAME}\n{AGE} is the age\nlives in {CITY}"
+        name = Suppress("name:") + Word(alphanums + " ")("NAME") + LineEnd()
+        age = Word(alphanums)("AGE") + Suppress("is the age") + LineEnd()
+        city = Suppress("lives in") + SkipTo(LineEnd())("CITY")
+        parser = name + age + city
+        return template, parser
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {
+            "name": "NAME",
+            "age": "AGE",
+            "city": "CITY"
+        }
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> tuple[str, ParserElement]:
+        template = "code_file_model\nfile_path: {FILE_PATH}\n```{LANGUAGE}\n{CODE}\n```"
+        file_path = Suppress("code_file_model") + LineEnd() + Suppress("file_path:") + SkipTo(LineEnd())("FILE_PATH") + LineEnd()
+        code_block = Suppress("```") + Word(alphanums)("LANGUAGE") + LineEnd() + Regex(r"(?s).*?(?=```)").set_parse_action(lambda s, l, t: t[0].strip())("CODE") + Suppress("```")
+        parser = file_path + code_block
+        return template, parser
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {
+            "file_path": "FILE_PATH",
+            "language": "LANGUAGE",
+            "code": "CODE"
+        }
+
+if __name__ == "__main__":
+    # Test MyFormattedModel
+    model = MyFormattedModel(name="John", age=30, city="Tokyo")
+    generated = model.generate()
+    print("Generated MyFormattedModel string:")
+    print(generated)
+    print()
+
+    parsed = MyFormattedModel.parse(generated)
+    print("Parsed MyFormattedModel object:")
+    print(parsed)
+    print()
+
+    print("MyFormattedModel Round-trip test:")
+    assert model == parsed, "MyFormattedModel: Original != Parsed"
+    print("Passed!")
+    print()
+
+    # Test CodeFileModel
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code="def hello():\n    print(\"Hello, World!\")"
+    )
+    code_generated = code_model.generate()
+    print("Generated CodeFileModel string:")
+    print(code_generated)
+    print()
+
+    code_parsed = CodeFileModel.parse(code_generated)
+    print("Parsed CodeFileModel object:")
+    print(code_parsed)
+    print()
+
+    print("CodeFileModel Round-trip test:")
+    assert code_model == code_parsed, "CodeFileModel: Original != Parsed"
+    print("Passed!")
+
+    print("\nAll tests passed successfully!")