langroid 0.16.5__py3-none-any.whl → 0.16.7__py3-none-any.whl

langroid/agent/tools/grammar_based_model_pyparsing.py-e

@@ -0,0 +1,103 @@
+from pydantic import BaseModel
+from pyparsing import *
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+class GrammarBasedModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def grammar(cls) -> ParserElement:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> 'GrammarBasedModel':
+        full_grammar = (
+            Suppress(cls.start_token()) +
+            cls.grammar() +
+            Suppress(cls.end_token())
+        )
+        parsed = full_grammar.parseString(text, parseAll=True)
+        return cls(**{field: parsed[token][-1] for field, token in cls.field_mappings().items()})
+
+    def generate(self) -> str:
+        result = [self.start_token()]
+        for field, token in self.field_mappings().items():
+            value = getattr(self, field)
+            if token == "name":
+                result.append(f"name: {value}")
+            elif token == "age":
+                result.append(f"age is {value}")
+            elif token == "city":
+                result.append(f"lives in {value}")
+        result.append(self.end_token())
+        return "\n".join(result)
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def grammar(cls):
+        name = Group(Literal("name:") + Word(alphas))("name")
+        age = Group(Literal("age is") + Word(nums))("age")
+        city = Group(Literal("lives in") + Word(alphas))("city")
+        return name + age + city
+
+    @classmethod
+    def start_token(cls):
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls):
+        return "</spec>"
+
+    @classmethod
+    def field_mappings(cls):
+        return {
+            "name": "name",
+            "age": "age",
+            "city": "city"
+        }
+
+if __name__ == "__main__":
+    # Test parsing
+    test_string = """
+    <spec>
+    name: John
+    age is 30
+    lives in Tokyo
+    </spec>
+    """
+    parsed_person = PersonSpec.parse(test_string)
+    print("Parsed person:", parsed_person)
+
+    # Test generating
+    new_person = PersonSpec(name="Alice", age=25, city="NewYork")
+    generated_string = new_person.generate()
+    print("\nGenerated string:")
+    print(generated_string)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_string)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert new_person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")
+    
+    

langroid/agent/tools/grammar_based_model_regex.py

@@ -0,0 +1,139 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def _create_regex_pattern(cls) -> str:
+        spec = cls.format_spec()
+        for field, placeholder in cls.field_mappings().items():
+            spec = spec.replace(placeholder, f"(?P<{field}>.*?)")
+        pattern = (
+            f"{re.escape(cls.start_token())}\\s*{spec}\\s*{re.escape(cls.end_token())}"
+        )
+        return pattern
+
+    @classmethod
+    def parse(cls, text: str) -> "FormattingModel":
+        pattern = cls._create_regex_pattern()
+        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+        if match:
+            return cls(**{k: v.strip() for k, v in match.groupdict().items()})
+        raise ValueError(
+            f"Text does not match the expected format. Pattern: {pattern}, Text: {text}"
+        )
+
+    def generate(self) -> str:
+        content = self.format_spec()
+        # Remove \s* patterns
+        content = re.sub(r"\\s\*", " ", content)
+        for field, placeholder in self.field_mappings().items():
+            content = content.replace(placeholder, str(getattr(self, field)))
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+
+class PersonModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name:\\s*{name}\\s*age is\\s*{age}\\s*lives in\\s*{city}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</spec>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {"name": "{name}", "age": "{age}", "city": "{city}"}
+
+
+def test_round_trip(model_class, input_string):
+    # Parse the input string
+    parsed_model = model_class.parse(input_string)
+    print(f"Parsed model: {parsed_model}")
+
+    # Generate a string from the parsed model
+    generated_string = parsed_model.generate()
+    print(f"Generated string:\n{generated_string}")
+
+    # Parse the generated string
+    reparsed_model = model_class.parse(generated_string)
+    print(f"Reparsed model: {reparsed_model}")
+
+    # Assert that the original parsed model and the reparsed model are equal
+    assert (
+        parsed_model == reparsed_model
+    ), "Round trip failed: original and reparsed models are not equal"
+
+    # Assert that all fields are present and have the correct types
+    for field, field_type in model_class.__annotations__.items():
+        assert hasattr(parsed_model, field), f"Field {field} is missing"
+        assert isinstance(
+            getattr(parsed_model, field), field_type
+        ), f"Field {field} has incorrect type"
+
+    print("Round trip test passed successfully!")
+
+
+if __name__ == "__main__":
+    # Test case 1: Standard formatting
+    test_string1 = """
+    <spec>
+    name: John Doe
+    age is 30
+    lives in New York
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string1)
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Test case 2: Varying whitespace
+    test_string2 = "<spec>name:   Alice  \nage is    25   \nlives in    Tokyo</spec>"
+    test_round_trip(PersonModel, test_string2)
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Test case 3: Multiline values
+    test_string3 = """
+    <spec>
+    name: Bob
+    Smith
+    age is 40
+    lives in San
+    Francisco
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string3)
+
+    print("All tests passed successfully!")

langroid/agent/tools/grammar_based_model_regex.py-e

@@ -0,0 +1,130 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+from pydantic import BaseModel
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        pass
+
+    @classmethod
+    def _create_regex_pattern(cls) -> str:
+        spec = cls.format_spec()
+        for field, placeholder in cls.field_mappings().items():
+            spec = spec.replace(placeholder, f"(?P<{field}>.*?)")
+        pattern = f"{re.escape(cls.start_token())}\\s*{spec}\\s*{re.escape(cls.end_token())}"
+        return pattern
+
+    @classmethod
+    def parse(cls, text: str) -> 'FormattingModel':
+        pattern = cls._create_regex_pattern()
+        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+        if match:
+            return cls(**{k: v.strip() for k, v in match.groupdict().items()})
+        raise ValueError(f"Text does not match the expected format. Pattern: {pattern}, Text: {text}")
+
+    def generate(self) -> str:
+        content = self.format_spec()
+        # Remove \s* patterns
+        content = re.sub(r'\\s\*', ' ', content)
+        for field, placeholder in self.field_mappings().items():
+            content = content.replace(placeholder, str(getattr(self, field)))
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+class PersonModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name:\\s*{name}\\s*age is\\s*{age}\\s*lives in\\s*{city}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</spec>"
+
+    @classmethod
+    def field_mappings(cls) -> Dict[str, str]:
+        return {
+            "name": "{name}",
+            "age": "{age}",
+            "city": "{city}"
+        }
+
+def test_round_trip(model_class, input_string):
+    # Parse the input string
+    parsed_model = model_class.parse(input_string)
+    print(f"Parsed model: {parsed_model}")
+
+    # Generate a string from the parsed model
+    generated_string = parsed_model.generate()
+    print(f"Generated string:\n{generated_string}")
+
+    # Parse the generated string
+    reparsed_model = model_class.parse(generated_string)
+    print(f"Reparsed model: {reparsed_model}")
+
+    # Assert that the original parsed model and the reparsed model are equal
+    assert parsed_model == reparsed_model, "Round trip failed: original and reparsed models are not equal"
+
+    # Assert that all fields are present and have the correct types
+    for field, field_type in model_class.__annotations__.items():
+        assert hasattr(parsed_model, field), f"Field {field} is missing"
+        assert isinstance(getattr(parsed_model, field), field_type), f"Field {field} has incorrect type"
+
+    print("Round trip test passed successfully!")
+
+if __name__ == "__main__":
+    # Test case 1: Standard formatting
+    test_string1 = """
+    <spec>
+    name: John Doe
+    age is 30
+    lives in New York
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string1)
+
+    print("\n" + "="*50 + "\n")
+
+    # Test case 2: Varying whitespace
+    test_string2 = "<spec>name:   Alice  \nage is    25   \nlives in    Tokyo</spec>"
+    test_round_trip(PersonModel, test_string2)
+
+    print("\n" + "="*50 + "\n")
+
+    # Test case 3: Multiline values
+    test_string3 = """
+    <spec>
+    name: Bob
+    Smith
+    age is 40
+    lives in San
+    Francisco
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string3)
+
+    print("All tests passed successfully!")

langroid/agent/tools/grammar_based_model_regex2.py

@@ -0,0 +1,124 @@
+import re
+from abc import ABC, abstractmethod
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def _create_regex_pattern(cls) -> str:
+        spec = cls.format_spec()
+        # Replace {field_name} with (?P<field_name>.*?)
+        pattern = re.sub(r"\{(\w+)\}", lambda m: f"(?P<{m.group(1)}>.*?)", spec)
+        # Replace newlines with \s* to allow flexible whitespace
+        pattern = pattern.replace("\n", r"\s*")
+        return f"{re.escape(cls.start_token())}\\s*{pattern}\\s*{re.escape(cls.end_token())}"
+
+    @classmethod
+    def parse(cls, text: str) -> "FormattingModel":
+        pattern = cls._create_regex_pattern()
+        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+        if match:
+            return cls(**{k: v.strip() for k, v in match.groupdict().items()})
+        raise ValueError(
+            f"Text does not match the expected format. Pattern: {pattern}, Text: {text}"
+        )
+
+    def generate(self) -> str:
+        content = self.format_spec().format(**self.dict())
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+
+class PersonModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name: {name}\nage is {age}\nlives in {city}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</spec>"
+
+
+def test_round_trip(model_class, input_string):
+    # Parse the input string
+    parsed_model = model_class.parse(input_string)
+    print(f"Parsed model: {parsed_model}")
+
+    # Generate a string from the parsed model
+    generated_string = parsed_model.generate()
+    print(f"Generated string:\n{generated_string}")
+
+    # Parse the generated string
+    reparsed_model = model_class.parse(generated_string)
+    print(f"Reparsed model: {reparsed_model}")
+
+    # Assert that the original parsed model and the reparsed model are equal
+    assert (
+        parsed_model == reparsed_model
+    ), "Round trip failed: original and reparsed models are not equal"
+
+    # Assert that all fields are present and have the correct types
+    for field, field_type in model_class.__annotations__.items():
+        assert hasattr(parsed_model, field), f"Field {field} is missing"
+        assert isinstance(
+            getattr(parsed_model, field), field_type
+        ), f"Field {field} has incorrect type"
+
+    print("Round trip test passed successfully!")
+
+
+if __name__ == "__main__":
+    # Test case 1: Standard formatting
+    test_string1 = """
+    <spec>
+    name: John Doe
+    age is 30
+    lives in New York
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string1)
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Test case 2: Varying whitespace
+    test_string2 = "<spec>name:   Alice  \nage is    25   \nlives in    Tokyo</spec>"
+    test_round_trip(PersonModel, test_string2)
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Test case 3: Multiline values
+    test_string3 = """
+    <spec>
+    name: Bob
+    Smith
+    age is 40
+    lives in San
+    Francisco
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string3)
+
+    print("All tests passed successfully!")

langroid/agent/tools/grammar_based_model_regex2.py-e

@@ -0,0 +1,116 @@
+import re
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+from pydantic import BaseModel
+
+class FormattingModel(BaseModel, ABC):
+    @classmethod
+    @abstractmethod
+    def format_spec(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def start_token(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def end_token(cls) -> str:
+        pass
+
+    @classmethod
+    def _create_regex_pattern(cls) -> str:
+        spec = cls.format_spec()
+        # Replace {field_name} with (?P<field_name>.*?)
+        pattern = re.sub(r'\{(\w+)\}', lambda m: f'(?P<{m.group(1)}>.*?)', spec)
+        # Replace newlines with \s* to allow flexible whitespace
+        pattern = pattern.replace('\n', r'\s*')
+        return f"{re.escape(cls.start_token())}\\s*{pattern}\\s*{re.escape(cls.end_token())}"
+
+    @classmethod
+    def parse(cls, text: str) -> 'FormattingModel':
+        pattern = cls._create_regex_pattern()
+        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+        if match:
+            return cls(**{k: v.strip() for k, v in match.groupdict().items()})
+        raise ValueError(f"Text does not match the expected format. Pattern: {pattern}, Text: {text}")
+
+    def generate(self) -> str:
+        content = self.format_spec().format(**self.dict())
+        return f"{self.start_token()}\n{content}\n{self.end_token()}"
+
+class PersonModel(FormattingModel):
+    name: str
+    age: int
+    city: str
+
+    @classmethod
+    def format_spec(cls) -> str:
+        return "name: {name}\nage is {age}\nlives in {city}"
+
+    @classmethod
+    def start_token(cls) -> str:
+        return "<spec>"
+
+    @classmethod
+    def end_token(cls) -> str:
+        return "</spec>"
+
+def test_round_trip(model_class, input_string):
+    # Parse the input string
+    parsed_model = model_class.parse(input_string)
+    print(f"Parsed model: {parsed_model}")
+
+    # Generate a string from the parsed model
+    generated_string = parsed_model.generate()
+    print(f"Generated string:\n{generated_string}")
+
+    # Parse the generated string
+    reparsed_model = model_class.parse(generated_string)
+    print(f"Reparsed model: {reparsed_model}")
+
+    # Assert that the original parsed model and the reparsed model are equal
+    assert parsed_model == reparsed_model, "Round trip failed: original and reparsed models are not equal"
+
+    # Assert that all fields are present and have the correct types
+    for field, field_type in model_class.__annotations__.items():
+        assert hasattr(parsed_model, field), f"Field {field} is missing"
+        assert isinstance(getattr(parsed_model, field), field_type), f"Field {field} has incorrect type"
+
+    print("Round trip test passed successfully!")
+
+if __name__ == "__main__":
+    # Test case 1: Standard formatting
+    test_string1 = """
+    <spec>
+    name: John Doe
+    age is 30
+    lives in New York
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string1)
+
+    print("\n" + "="*50 + "\n")
+
+    # Test case 2: Varying whitespace
+    test_string2 = "<spec>name:   Alice  \nage is    25   \nlives in    Tokyo</spec>"
+    test_round_trip(PersonModel, test_string2)
+
+    print("\n" + "="*50 + "\n")
+
+    # Test case 3: Multiline values
+    test_string3 = """
+    <spec>
+    name: Bob
+    Smith
+    age is 40
+    lives in San
+    Francisco
+    </spec>
+    """
+    test_round_trip(PersonModel, test_string3)
+
+    print("All tests passed successfully!")
+    
+    

langroid/agent/tools/grammar_based_model_tatsu.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+from typing import ClassVar, Dict
+
+from tatsu import compile as compile_grammar
+from tatsu.model import ModelBuilder
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class GrammarBasedModel(BaseModel, ABC):
+    grammar: ClassVar[str]
+    start_token: ClassVar[str]
+    end_token: ClassVar[str]
+    field_mappings: ClassVar[Dict[str, str]]
+
+    @classmethod
+    @abstractmethod
+    def get_grammar(cls) -> str:
+        pass
+
+    @classmethod
+    def parse(cls, text: str) -> "GrammarBasedModel":
+        parser = compile_grammar(cls.get_grammar())
+        ast = parser.parse(text, start="start")
+        model_dict = {
+            field: getattr(ast, rule) for field, rule in cls.field_mappings.items()
+        }
+        return cls(**model_dict)
+
+    def generate(self) -> str:
+        grammar = compile_grammar(self.get_grammar())
+        model_builder = ModelBuilder()
+        for field, rule in self.field_mappings.items():
+            setattr(model_builder, rule, getattr(self, field))
+        ast = model_builder.start()
+        return f"{self.start_token}\n{grammar.parse(str(ast), start='start')}\n{self.end_token}"
+
+
+class PersonSpec(GrammarBasedModel):
+    name: str
+    age: int
+    city: str
+
+    grammar = """
+        start = name_line age_line city_line;
+        name_line = 'name:' /\s*/ name:/.+/ EOL;
+        age_line = 'age is' /\s*/ age:/\d+/ EOL;
+        city_line = 'lives in' /\s*/ city:/.+/ EOL;
+        EOL = /\r?\n/;
+    """
+    start_token = "<spec>"
+    end_token = "</spec>"
+    field_mappings = {"name": "name", "age": "age", "city": "city"}
+
+    @classmethod
+    def get_grammar(cls):
+        return cls.grammar
+
+
+if __name__ == "__main__":
+    # Test parsing
+    input_str = """<spec>
+name: John Doe
+age is 30
+lives in New York
+</spec>"""
+    person = PersonSpec.parse(input_str)
+    print("Parsed person:", person)
+
+    # Test generation
+    generated_str = person.generate()
+    print("\nGenerated string:")
+    print(generated_str)
+
+    # Test round-trip
+    round_trip_person = PersonSpec.parse(generated_str)
+    print("\nRound-trip parsed person:", round_trip_person)
+
+    assert person == round_trip_person, "Round-trip parsing failed"
+    print("\nRound-trip test passed!")