langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/formatted_model_regex2.py

@@ -0,0 +1,250 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Dict, Type, TypeVar
+
+from langroid.pydantic_v1 import BaseModel
+
+T = TypeVar("T", bound="FormattingModel")
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        # Remove start and end tokens
+        content = text.strip()[len(cls.start_token()) : -len(cls.end_token())].strip()
+
+        # Create regex pattern from format_spec
+        pattern = cls.format_spec()
+        for field, token in cls.field_mappings().items():
+            pattern = pattern.replace(token, f"(?P<{field}>.*?)")
+
+        # Extract data using regex
+        match = re.match(pattern, content, re.DOTALL)
+        if not match:
+            raise ValueError("Invalid format")
+
+        # Create instance with extracted data
+        data = {field: match.group(field).strip() for field in cls.field_mappings()}
+        return cls(**data)
+
+    def generate(self) -> str:
+        # Start with the format spec
+        result = self.format_spec()
+
+        # Replace tokens with actual values
+        for field, token in self.field_mappings().items():
+            value = getattr(self, field)
+            # Use re.sub to replace tokens, treating the replacement as a literal string
+            result = re.sub(re.escape(token), lambda m: str(value), result)
+
+        # Remove raw string markers and extra escapes
+        result = result.replace(r"\s*", "").replace(r"\n", "\n")
+
+        # Wrap with start and end tokens
+        return f"{self.start_token()}\n{result}\n{self.end_token()}"
+
+
+class MyFormattedModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name: {NAME}\n{AGE} is the age\nlives in {CITY}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {"name": "{NAME}", "age": "{AGE}", "city": "{CITY}"}
+
+
+if __name__ == "__main__":
+    # Test object to string
+    model = MyFormattedModel(name="John", age=30, city="Tokyo")
+    generated = model.generate()
+    print("Generated string:")
+    print(generated)
+    print()
+
+    # Test string to object
+    parsed = MyFormattedModel.parse(generated)
+    print("Parsed object:")
+    print(parsed)
+    print()
+
+    # Test round-trip
+    print("Round-trip test:")
+    print("Original == Parsed:", model == parsed)
+
+    # Test with different values
+    another_model = MyFormattedModel(name="Alice", age=25, city="New York")
+    another_generated = another_model.generate()
+    print("\nAnother generated string:")
+    print(another_generated)
+    print()
+
+    another_parsed = MyFormattedModel.parse(another_generated)
+    print("Another parsed object:")
+    print(another_parsed)
+    print("Another Original == Another Parsed:", another_model == another_parsed)
+
+    # code file model
+    class CodeFileModel(FormattingModel):
+        language: str
+        file_path: str
+        code: str
+
+        @classmethod
+        def format_spec(cls) -> str:
+            return "code_file_model\nfile_path: {FILE_PATH}\n```{LANGUAGE}\n{CODE}\n```"
+
+        @classmethod
+        def start_token(cls) -> str:
+            return "<format>"
+
+        @classmethod
+        def end_token(cls) -> str:
+            return "</format>"
+
+        @classmethod
+        def field_mappings(cls) -> Dict[str, str]:
+            return {
+                "file_path": "{FILE_PATH}",
+                "language": "{LANGUAGE}",
+                "code": "{CODE}",
+            }
+
+    print("\nTesting CodeFileModel:")
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code='def hello():\n    print("Hello, World!")',
+    )
+    code_generated = code_model.generate()
+    print("Generated CodeFileModel string:")
+    print(code_generated)
+    print()
+
+    code_parsed = CodeFileModel.parse(code_generated)
+    print("Parsed CodeFileModel object:")
+    print(code_parsed)
+    print()
+
+    print("CodeFileModel Round-trip test:")
+    print("Original == Parsed:", code_model == code_parsed)
+
+    # tolerant format
+    #
+    class CodeFileModel(FormattingModel):
+        language: str
+        file_path: str
+        code: str
+
+        @classmethod
+        def format_spec(cls) -> str:
+            return (
+                r"code_file_model\s*\n"
+                r"file_path:\s*{FILE_PATH}\s*\n"
+                r"```\s*{LANGUAGE}\s*\n"
+                r"{CODE}\s*"
+                r"```"
+            )
+
+        @classmethod
+        def start_token(cls) -> str:
+            return "<format>"
+
+        @classmethod
+        def end_token(cls) -> str:
+            return "</format>"
+
+        @classmethod
+        def field_mappings(cls) -> Dict[str, str]:
+            return {
+                "file_path": "{FILE_PATH}",
+                "language": "{LANGUAGE}",
+                "code": "{CODE}",
+            }
+
+    print("\nTesting CodeFileModel with various whitespace variations:")
+
+    test_strings = [
+        # Standard format
+        """<format>
+code_file_model
+file_path: src/main.py
+```python
+def hello():
+    print("Hello, World!")
+```
+</format>""",
+        # Extra whitespace
+        """<format>
+code_file_model  
+file_path:     src/main.py    
+```   python   
+def hello():
+    print("Hello, World!")
+```
+</format>""",
+        # Extra newlines
+        """<format>
+code_file_model
+
+file_path: src/main.py
+
+```python
+
+def hello():
+    print("Hello, World!")
+
+```
+
+</format>""",
+    ]
+
+    for i, test_string in enumerate(test_strings, 1):
+        print(f"\nTest {i}:")
+        print("Input string:")
+        print(test_string)
+
+        parsed = CodeFileModel.parse(test_string)
+        print("\nParsed object:")
+        print(parsed)
+
+        regenerated = parsed.generate()
+        print("\nRegenerated string:")
+        print(regenerated)
+
+        reparsed = CodeFileModel.parse(regenerated)
+        print("\nRound-trip test:")
+        print("Original parsed == Reparsed:", parsed == reparsed)
+        print("-" * 50)

langroid/agent/tools/formatted_model_regex2.py-e

@@ -0,0 +1,253 @@
+from pydantic import BaseModel
+from abc import ABC, abstractmethod
+import re
+from typing import Dict, Type, TypeVar
+
+T = TypeVar('T', bound='FormattingModel')
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls: Type[T], text: str) -> T:
+        # Remove start and end tokens
+        content = text.strip()[len(cls.start_token()):-len(cls.end_token())].strip()
+        
+        # Create regex pattern from format_spec
+        pattern = cls.format_spec()
+        for field, token in cls.field_mappings().items():
+            pattern = pattern.replace(token, f"(?P<{field}>.*?)")
+        
+        # Extract data using regex
+        match = re.match(pattern, content, re.DOTALL)
+        if not match:
+            raise ValueError("Invalid format")
+        
+        # Create instance with extracted data
+        data = {field: match.group(field).strip() for field in cls.field_mappings()}
+        return cls(**data)
+
+    def generate(self) -> str:
+            # Start with the format spec
+            result = self.format_spec()
+            
+            # Replace tokens with actual values
+            for field, token in self.field_mappings().items():
+                value = getattr(self, field)
+                # Use re.sub to replace tokens, treating the replacement as a literal string
+                result = re.sub(re.escape(token), lambda m: str(value), result)
+            
+            # Remove raw string markers and extra escapes
+            result = result.replace(r'\s*', '').replace(r'\n', '\n')
+            
+            # Wrap with start and end tokens
+            return f"{self.start_token()}\n{result}\n{self.end_token()}"
+        
+
+class MyFormattedModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name: {NAME}\n{AGE} is the age\nlives in {CITY}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<format>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</format>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {
+            "name": "{NAME}",
+            "age": "{AGE}",
+            "city": "{CITY}"
+        }
+
+if __name__ == "__main__":
+    # Test object to string
+    model = MyFormattedModel(name="John", age=30, city="Tokyo")
+    generated = model.generate()
+    print("Generated string:")
+    print(generated)
+    print()
+
+    # Test string to object
+    parsed = MyFormattedModel.parse(generated)
+    print("Parsed object:")
+    print(parsed)
+    print()
+
+    # Test round-trip
+    print("Round-trip test:")
+    print("Original == Parsed:", model == parsed)
+
+    # Test with different values
+    another_model = MyFormattedModel(name="Alice", age=25, city="New York")
+    another_generated = another_model.generate()
+    print("\nAnother generated string:")
+    print(another_generated)
+    print()
+
+    another_parsed = MyFormattedModel.parse(another_generated)
+    print("Another parsed object:")
+    print(another_parsed)
+    print("Another Original == Another Parsed:", another_model == another_parsed)
+
+
+    # code file model
+    class CodeFileModel(FormattingModel):
+        language: str
+        file_path: str
+        code: str
+    
+        @classmethod
+        def format_spec(cls) -> str:
+            return "code_file_model\nfile_path: {FILE_PATH}\n```{LANGUAGE}\n{CODE}\n```"
+    
+        @classmethod
+        def start_token(cls) -> str:
+            return "<format>"
+    
+        @classmethod
+        def end_token(cls) -> str:
+            return "</format>"
+    
+        @classmethod
+        def field_mappings(cls) -> Dict[str, str]:
+            return {
+                "file_path": "{FILE_PATH}",
+                "language": "{LANGUAGE}",
+                "code": "{CODE}"
+            }
+            
+    
+    print("\nTesting CodeFileModel:")
+    code_model = CodeFileModel(
+        language="python",
+        file_path="src/main.py",
+        code="def hello():\n    print(\"Hello, World!\")"
+    )
+    code_generated = code_model.generate()
+    print("Generated CodeFileModel string:")
+    print(code_generated)
+    print()
+
+    code_parsed = CodeFileModel.parse(code_generated)
+    print("Parsed CodeFileModel object:")
+    print(code_parsed)
+    print()
+
+    print("CodeFileModel Round-trip test:")
+    print("Original == Parsed:", code_model == code_parsed)
+
+    # tolerant format
+    # 
+    class CodeFileModel(FormattingModel):
+        language: str
+        file_path: str
+        code: str
+    
+        @classmethod
+        def format_spec(cls) -> str:
+            return (
+                r"code_file_model\s*\n"
+                r"file_path:\s*{FILE_PATH}\s*\n"
+                r"```\s*{LANGUAGE}\s*\n"
+                r"{CODE}\s*"
+                r"```"
+            )
+    
+        @classmethod
+        def start_token(cls) -> str:
+            return "<format>"
+    
+        @classmethod
+        def end_token(cls) -> str:
+            return "</format>"
+    
+        @classmethod
+        def field_mappings(cls) -> Dict[str, str]:
+            return {
+                "file_path": "{FILE_PATH}",
+                "language": "{LANGUAGE}",
+                "code": "{CODE}"
+            }
+
+    print("\nTesting CodeFileModel with various whitespace variations:")
+
+    test_strings = [
+        # Standard format
+        """<format>
+code_file_model
+file_path: src/main.py
+```python
+def hello():
+    print("Hello, World!")
+```
+</format>""",
+        # Extra whitespace
+        """<format>
+code_file_model  
+file_path:     src/main.py    
+```   python   
+def hello():
+    print("Hello, World!")
+```
+</format>""",
+        # Extra newlines
+        """<format>
+code_file_model
+
+file_path: src/main.py
+
+```python
+
+def hello():
+    print("Hello, World!")
+
+```
+
+</format>"""
+    ]
+
+    for i, test_string in enumerate(test_strings, 1):
+        print(f"\nTest {i}:")
+        print("Input string:")
+        print(test_string)
+        
+        parsed = CodeFileModel.parse(test_string)
+        print("\nParsed object:")
+        print(parsed)
+        
+        regenerated = parsed.generate()
+        print("\nRegenerated string:")
+        print(regenerated)
+        
+        reparsed = CodeFileModel.parse(regenerated)
+        print("\nRound-trip test:")
+        print("Original parsed == Reparsed:", parsed == reparsed)
+        print("-" * 50)        

langroid/agent/tools/formatted_model_tatsu.py

@@ -0,0 +1,172 @@
+from abc import ABC, abstractmethod
+
+import tatsu
+from tatsu.model import ModelBuilderSemantics
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def parse_spec(cls):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def format(cls, instance: "FormattingModel") -> str:
+        spec = cls.format_spec()
+        formatted = spec.format(**instance.dict())
+        return f"{cls.start_token()}\n{formatted}\n{cls.end_token()}"
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> "FormattingModel":
+        lines = formatted_string.strip().split("\n")
+        if lines[0] != cls.start_token() or lines[-1] != cls.end_token():
+            raise ValueError("Invalid start or end token")
+        content = "\n".join(lines[1:-1])
+
+        parser = tatsu.compile(cls.parse_spec())
+        ast = parser.parse(content)
+        return cls(**ast)
+
+
+class CodeFileModel(FormattingModel):
+    language: str
+    file_path: str
+    code: str
+
+    @classmethod
+    def format_spec(cls):
+        return "code_file_model\n{file_path}\n```{language}\n{code}\n```"
+
+    @classmethod
+    def parse_spec(cls):
+        return """
+            @@grammar::CodeFileModel
+
+            start = header file_path language code $ ;
+
+            header = "code_file_model" ~;
+            file_path = /[^\n]+/ ~;
+            language = "```" /[a-zA-Z]+/ ~;
+            code = /(?s).*?(?=```)/ "```" ~;
+
+            @@whitespace :: /\s*/
+        """
+
+    @classmethod
+    def start_token(cls):
+        return "<format>"
+
+    @classmethod
+    def end_token(cls):
+        return "</format>"
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> "CodeFileModel":
+        lines = formatted_string.strip().split("\n")
+        if lines[0] != cls.start_token() or lines[-1] != cls.end_token():
+            raise ValueError("Invalid start or end token")
+        content = "\n".join(lines[1:-1])
+
+        class CodeFileModelSemantics(ModelBuilderSemantics):
+            def file_path(self, ast):
+                return ast.strip()
+
+            def language(self, ast):
+                return ast[1].strip()
+
+            def code(self, ast):
+                return ast[0].strip()
+
+        parser = tatsu.compile(cls.parse_spec(), semantics=CodeFileModelSemantics())
+        ast = parser.parse(content)
+        return cls(**ast)
+
+
+# Test cases
+if __name__ == "__main__":
+    # Test formatting
+    code_file = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    formatted = CodeFileModel.format(code_file)
+    expected_format = """<format>
+code_file_model
+src/main.py
+```Python
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    assert (
+        formatted == expected_format
+    ), f"Formatting failed. Expected:\n{expected_format}\nGot:\n{formatted}"
+    print("Formatting test passed.")
+
+    # Test parsing
+    parsed = CodeFileModel.parse(formatted)
+    assert (
+        parsed == code_file
+    ), f"Parsing failed. Expected:\n{code_file}\nGot:\n{parsed}"
+    print("Parsing test passed.")
+
+    # Test round-trip
+    round_trip = CodeFileModel.parse(CodeFileModel.format(code_file))
+    assert (
+        round_trip == code_file
+    ), f"Round-trip failed. Expected:\n{code_file}\nGot:\n{round_trip}"
+    print("Round-trip test passed.")
+
+    # Test with different values
+    code_file2 = CodeFileModel(
+        language="JavaScript",
+        file_path="src/app.js",
+        code="function greet() {\n  console.log('Hello, World!');\n}",
+    )
+    formatted2 = CodeFileModel.format(code_file2)
+    parsed2 = CodeFileModel.parse(formatted2)
+    assert (
+        parsed2 == code_file2
+    ), f"Parsing failed for different values. Expected:\n{code_file2}\nGot:\n{parsed2}"
+    print("Different values test passed.")
+
+    # Test tolerant parsing
+    tolerant_input = """<format>
+code_file_model
+   src/main.py   
+
+```  Python  
+def hello():
+    print('Hello, World!')
+```
+</format>"""
+    parsed_tolerant = CodeFileModel.parse(tolerant_input)
+    expected_tolerant = CodeFileModel(
+        language="Python",
+        file_path="src/main.py",
+        code="def hello():\n    print('Hello, World!')",
+    )
+    assert (
+        parsed_tolerant == expected_tolerant
+    ), f"Tolerant parsing failed. Expected:\n{expected_tolerant}\nGot:\n{parsed_tolerant}"
+    print("Tolerant parsing test passed.")
+
+    print("All tests passed successfully!")