langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/formatted_model_jinja.py

@@ -0,0 +1,133 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Type, TypeVar
+
+from jinja2 import BaseLoader, Environment
+
+from langroid.pydantic_v1 import BaseModel
+
+T = TypeVar("T", bound="FormattingModel")
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()) : -len(cls.end_token())].strip()
+        pattern = cls.format_spec()
+        for field in cls.__fields__:
+            pattern = pattern.replace(f"{{{{{field}}}}}", f"(?P<{field}>.*?)")
+        pattern = pattern.replace("\n", "\\n")
+
+        match = re.match(pattern, content, re.DOTALL)
+        if not match:
+            raise ValueError("Failed to parse the input string")
+
+        parsed_data = {k: v.strip() for k, v in match.groupdict().items()}
+        return cls(**parsed_data)
+
+    def generate(self) -> str:
+        env = Environment(loader=BaseLoader())
+        template = env.from_string(self.format_spec())
+        content = template.render(**self.dict())
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return (
+            "code_file_model\nfile_path: {{file_path}}\n```{{language}}\n{{code}}\n```"
+        )
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+
+if __name__ == "__main__":
+    # Test CodeFileModel
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code='def hello():\n    print("Hello, World!")',
+    )
+
+    print("Original CodeFileModel:")
+    print(code_model)
+    print()
+
+    generated = code_model.generate()
+    print("Generated string:")
+    print(generated)
+    print()
+
+    parsed = CodeFileModel.parse(generated)
+    print("Parsed CodeFileModel:")
+    print(parsed)
+    print()
+
+    print("Round-trip test:")
+    assert (
+        code_model == parsed
+    ), "Round-trip test failed: original and parsed models are not equal"
+    print("Passed!")
+
+    # Test with different values
+    another_model = CodeFileModel(
+        language="javascript",
+        file_path="src/app.js",
+        code="function greet(name) {\n    console.log(`Hello, ${name}!`);\n}",
+    )
+
+    print("\nAnother CodeFileModel:")
+    print(another_model)
+    print()
+
+    another_generated = another_model.generate()
+    print("Another generated string:")
+    print(another_generated)
+    print()
+
+    another_parsed = CodeFileModel.parse(another_generated)
+    print("Another parsed CodeFileModel:")
+    print(another_parsed)
+    print()
+
+    print("Another round-trip test:")
+    assert (
+        another_model == another_parsed
+    ), "Another round-trip test failed: original and parsed models are not equal"
+    print("Passed!")
+
+    # Test error handling
+    print("\nTesting error handling:")
+    try:
+        CodeFileModel.parse("Invalid format string")
+        assert False, "Should have raised a ValueError"
+    except ValueError as e:
+        print(f"Correctly raised ValueError: {e}")
+
+    print("\nAll tests passed successfully!")

langroid/agent/tools/formatted_model_jinja.py-e

@@ -0,0 +1,122 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+from typing import Dict, Type, TypeVar
+from jinja2 import Environment, BaseLoader
+import re
+
+T = TypeVar('T', bound='FormattingModel')
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()):-len(cls.end_token())].strip()
+        pattern = cls.format_spec()
+        for field in cls.__fields__:
+            pattern = pattern.replace(f"{{{{{field}}}}}", f"(?P<{field}>.*?)")
+        pattern = pattern.replace("\n", "\\n")
+        
+        match = re.match(pattern, content, re.DOTALL)
+        if not match:
+            raise ValueError("Failed to parse the input string")
+        
+        parsed_data = {k: v.strip() for k, v in match.groupdict().items()}
+        return cls(**parsed_data)
+
+    def generate(self) -> str:
+        env = Environment(loader=BaseLoader())
+        template = env.from_string(self.format_spec())
+        content = template.render(**self.dict())
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "code_file_model\nfile_path: {{file_path}}\n```{{language}}\n{{code}}\n```"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+if __name__ == "__main__":
+    # Test CodeFileModel
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code="def hello():\n    print(\"Hello, World!\")"
+    )
+    
+    print("Original CodeFileModel:")
+    print(code_model)
+    print()
+
+    generated = code_model.generate()
+    print("Generated string:")
+    print(generated)
+    print()
+
+    parsed = CodeFileModel.parse(generated)
+    print("Parsed CodeFileModel:")
+    print(parsed)
+    print()
+
+    print("Round-trip test:")
+    assert code_model == parsed, "Round-trip test failed: original and parsed models are not equal"
+    print("Passed!")
+
+    # Test with different values
+    another_model = CodeFileModel(
+        language="javascript",
+        file_path="src/app.js",
+        code="function greet(name) {\n    console.log(`Hello, ${name}!`);\n}"
+    )
+    
+    print("\nAnother CodeFileModel:")
+    print(another_model)
+    print()
+
+    another_generated = another_model.generate()
+    print("Another generated string:")
+    print(another_generated)
+    print()
+
+    another_parsed = CodeFileModel.parse(another_generated)
+    print("Another parsed CodeFileModel:")
+    print(another_parsed)
+    print()
+
+    print("Another round-trip test:")
+    assert another_model == another_parsed, "Another round-trip test failed: original and parsed models are not equal"
+    print("Passed!")
+
+    # Test error handling
+    print("\nTesting error handling:")
+    try:
+        CodeFileModel.parse("Invalid format string")
+        assert False, "Should have raised a ValueError"
+    except ValueError as e:
+        print(f"Correctly raised ValueError: {e}")
+
+    print("\nAll tests passed successfully!")

langroid/agent/tools/formatted_model_jinja2.py

@@ -0,0 +1,145 @@
+from abc import ABC, abstractmethod
+from typing import Type, TypeVar
+
+from jinja2 import BaseLoader, Environment
+from parse import parse as str_parse
+from parse import with_pattern
+
+from langroid.pydantic_v1 import BaseModel
+
+T = TypeVar("T", bound="FormattingModel")
+
+
+@with_pattern(r"[\s\S]*?")
+def _match_multiline(text):
+    return text.strip()
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()) : -len(cls.end_token())].strip()
+        result = str_parse(
+            cls.parse_spec(),
+            content,
+            dict(multiline=_match_multiline),
+            case_sensitive=False,
+        )
+        if result is None:
+            raise ValueError(
+                f"Failed to parse the input string using spec: {cls.parse_spec()}\nInput: {content}"
+            )
+        return cls(**result.named)
+
+    def generate(self) -> str:
+        env = Environment(loader=BaseLoader(), trim_blocks=True, lstrip_blocks=True)
+        template = env.from_string(self.format_spec())
+        content = template.render(**self.dict())
+        return f"{self.start_token()}\n{content.strip()}\n{self.end_token()}"
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return (
+            "code_file_model\n"
+            "file_path: {{- file_path -}}\n"
+            "```{{- language -}}\n"
+            "{{ code }}"
+            "```"
+        )
+
+    @classmethod
+    def parse_spec(cls) -> str:
+        return "code_file_model\n" "file_path:{:s}\n" "```{:s}\n" "{:multiline}" "```"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+
+# Test code
+if __name__ == "__main__":
+    # Test with extra whitespace
+    test_string = """
+    <format>
+        code_file_model
+        file_path:     src/main.py    
+        ```   python   
+        def hello():
+            print("Hello, World!")
+        
+        ```
+    </format>
+    """
+
+    parsed = CodeFileModel.parse(test_string)
+    print("Parsed model:")
+    print(parsed)
+
+    generated = parsed.generate()
+    print("\nGenerated string:")
+    print(generated)
+
+    reparsed = CodeFileModel.parse(generated)
+    print("\nReparsed model:")
+    print(reparsed)
+
+    print("\nRound trip test:")
+    assert parsed == reparsed, "Round trip test failed"
+    print("Passed!")
+
+    # Test with different values and whitespace
+    another_test = """
+    <format>
+    code_file_model
+    file_path:src/app.js
+    ``` javascript
+    function greet(name) {
+        console.log(`Hello, ${name}!`);
+    }
+    ```
+    </format>
+    """
+
+    another_parsed = CodeFileModel.parse(another_test)
+    print("\nAnother parsed model:")
+    print(another_parsed)
+
+    another_generated = another_parsed.generate()
+    print("\nAnother generated string:")
+    print(another_generated)
+
+    print("\nAnother round trip test:")
+    assert another_parsed == CodeFileModel.parse(
+        another_generated
+    ), "Another round trip test failed"
+    print("Passed!")

langroid/agent/tools/formatted_model_jinja2.py-e

@@ -0,0 +1,135 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+from typing import Type, TypeVar
+from jinja2 import Environment, BaseLoader
+from parse import parse as str_parse, with_pattern
+
+T = TypeVar('T', bound='FormattingModel')
+
+@with_pattern(r'[\s\S]*?')
+def _match_multiline(text):
+    return text.strip()
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        content = text.strip()[len(cls.start_token()):-len(cls.end_token())].strip()
+        result = str_parse(cls.parse_spec(), content, dict(multiline=_match_multiline), case_sensitive=False)
+        if result is None:
+            raise ValueError(f"Failed to parse the input string using spec: {cls.parse_spec()}\nInput: {content}")
+        return cls(**result.named)
+
+    def generate(self) -> str:
+        env = Environment(loader=BaseLoader(), trim_blocks=True, lstrip_blocks=True)
+        template = env.from_string(self.format_spec())
+        content = template.render(**self.dict())
+        return f"{self.start_token()}\n{content.strip()}\n{self.end_token()}"
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return (
+            "code_file_model\n"
+            "file_path: {{- file_path -}}\n"
+            "```{{- language -}}\n"
+            "{{ code }}"
+            "```"
+        )
+
+    @classmethod
+    def parse_spec(cls) -> str:
+        return (
+            "code_file_model\n"
+            "file_path:{:s}\n"
+            "```{:s}\n"
+            "{:multiline}"
+            "```"
+        )
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+# Test code
+if __name__ == "__main__":
+    # Test with extra whitespace
+    test_string = """
+    <format>
+        code_file_model
+        file_path:     src/main.py    
+        ```   python   
+        def hello():
+            print("Hello, World!")
+        
+        ```
+    </format>
+    """
+
+    parsed = CodeFileModel.parse(test_string)
+    print("Parsed model:")
+    print(parsed)
+
+    generated = parsed.generate()
+    print("\nGenerated string:")
+    print(generated)
+
+    reparsed = CodeFileModel.parse(generated)
+    print("\nReparsed model:")
+    print(reparsed)
+
+    print("\nRound trip test:")
+    assert parsed == reparsed, "Round trip test failed"
+    print("Passed!")
+
+    # Test with different values and whitespace
+    another_test = """
+    <format>
+    code_file_model
+    file_path:src/app.js
+    ``` javascript
+    function greet(name) {
+        console.log(`Hello, ${name}!`);
+    }
+    ```
+    </format>
+    """
+
+    another_parsed = CodeFileModel.parse(another_test)
+    print("\nAnother parsed model:")
+    print(another_parsed)
+
+    another_generated = another_parsed.generate()
+    print("\nAnother generated string:")
+    print(another_generated)
+
+    print("\nAnother round trip test:")
+    assert another_parsed == CodeFileModel.parse(another_generated), "Another round trip test failed"
+    print("Passed!")

langroid/agent/tools/formatted_model_lark2.py

@@ -0,0 +1,168 @@
+from abc import ABC, abstractmethod
+
+from lark import Lark, Transformer, v_args
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def format(cls, instance: "FormattingModel") -> str:
+        spec = cls.format_spec()
+        formatted = spec.format(**instance.dict())
+        return f"{cls.start_token()}\n{formatted}\n{cls.end_token()}"
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> "FormattingModel":
+        lines = formatted_string.strip().split("\n")
+        if lines[0] != cls.start_token() or lines[-1] != cls.end_token():
+            raise ValueError("Invalid start or end token")
+        content = "\n".join(lines[1:-1])
+
+        parser = Lark(cls.parse_spec(), start="start", parser="lalr")
+
+        @v_args(inline=True)
+        class TreeToDict(Transformer):
+            def start(self, _, file_path, code_block):
+                return {
+                    "file_path": file_path,
+                    "language": code_block.children[1],
+                    "code": code_block.children[3],
+                }
+
+            def file_path(self, path):
+                return path.value
+
+            def language(self, lang):
+                return lang.value
+
+            def code(self, code):
+                return code.value.strip()
+
+        tree = parser.parse(content)
+        data = TreeToDict().transform(tree)
+        return cls(**data)
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls):
+        return "code_file_model\n{file_path}\n```{language}\n{code}\n```"
+
+    @classmethod
+    def parse_spec(cls):
+        return """
+            start: "code_file_model" NEWLINE file_path NEWLINE code_block
+            file_path: /[^\\n]+/
+            code_block: "```" language NEWLINE code "```"
+            language: /[^\\n]+/
+            code: /.+?(?=\\n```)/s
+            NEWLINE: "\\n"
+            %import common.WS
+            %ignore WS
+        """
+
+    @classmethod
+    def start_token(cls):
+        return "<format>"
+
+    @classmethod
+    def end_token(cls):
+        return "</format>"
+
+
+# Test cases
+if __name__ == "__main__":
+    # Test formatting
+    code_file = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    formatted = CodeFileModel.format(code_file)
+    expected_format = """<format>
+code_file_model
+src/main.py
+```Python
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    assert (
+        formatted == expected_format
+    ), f"Formatting failed. Expected:\n{expected_format}\nGot:\n{formatted}"
+    print("Formatting test passed.")
+
+    # Test parsing
+    parsed = CodeFileModel.parse(formatted)
+    assert (
+        parsed == code_file
+    ), f"Parsing failed. Expected:\n{code_file}\nGot:\n{parsed}"
+    print("Parsing test passed.")
+
+    # Test round-trip
+    round_trip = CodeFileModel.parse(CodeFileModel.format(code_file))
+    assert (
+        round_trip == code_file
+    ), f"Round-trip failed. Expected:\n{code_file}\nGot:\n{round_trip}"
+    print("Round-trip test passed.")
+
+    # Test with different values
+    code_file2 = CodeFileModel(
+        language="JavaScript",
+        file_path="src/app.js",
+        code="function greet() {\n  console.log('Hello, World!');\n}",
+    )
+    formatted2 = CodeFileModel.format(code_file2)
+    parsed2 = CodeFileModel.parse(formatted2)
+    assert (
+        parsed2 == code_file2
+    ), f"Parsing failed for different values. Expected:\n{code_file2}\nGot:\n{parsed2}"
+    print("Different values test passed.")
+
+    # Test tolerant parsing
+    tolerant_input = """<format>
+code_file_model
+   src/main.py   
+
+```  Python  
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    parsed_tolerant = CodeFileModel.parse(tolerant_input)
+    expected_tolerant = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    assert (
+        parsed_tolerant == expected_tolerant
+    ), f"Tolerant parsing failed. Expected:\n{expected_tolerant}\nGot:\n{parsed_tolerant}"
+    print("Tolerant parsing test passed.")
+
+    print("All tests passed successfully!")