machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/ast/tests/test_foreach_desugar.py

@@ -0,0 +1,245 @@
+"""Test for-each statement desugaring to while loops."""
+
+from machine_dialect.ast import Identifier
+from machine_dialect.ast.call_expression import CallExpression
+from machine_dialect.ast.literals import OrderedListLiteral, StringLiteral, WholeNumberLiteral
+from machine_dialect.ast.statements import BlockStatement, ForEachStatement, SetStatement
+from machine_dialect.lexer import Token, TokenType
+
+
+class TestForEachDesugaring:
+    """Test that for-each statements correctly desugar to while loops."""
+
+    def test_basic_foreach_desugaring(self) -> None:
+        """Test basic for-each loop desugaring."""
+        # Create tokens
+        for_token = Token(TokenType.KW_FOR, "for", 1, 1)
+        item_token = Token(TokenType.MISC_IDENT, "item", 1, 10)
+        collection_token = Token(TokenType.MISC_IDENT, "items", 1, 20)
+
+        # Create the for-each statement:
+        # For each `item` in `items`:
+        #     body
+        item_id = Identifier(item_token, "item")
+        collection_id = Identifier(collection_token, "items")
+
+        # Create a simple body
+        body = BlockStatement(for_token)
+        body.statements = [
+            SetStatement(
+                Token(TokenType.KW_SET, "Set", 2, 1),
+                Identifier(Token(TokenType.MISC_IDENT, "result", 2, 5), "result"),
+                item_id,
+            )
+        ]
+
+        foreach_stmt = ForEachStatement(for_token, item=item_id, collection=collection_id, body=body)
+
+        # Desugar the for-each statement
+        desugared = foreach_stmt.desugar()
+
+        # Should return a BlockStatement containing initialization and while loop
+        assert isinstance(desugared, BlockStatement)
+        assert len(desugared.statements) == 3  # init_index, init_length, while_stmt
+
+        # Check initialization statements
+        init_index = desugared.statements[0]
+        assert isinstance(init_index, SetStatement)
+        assert isinstance(init_index.name, Identifier)
+        assert init_index.name.value.startswith("$foreach_idx_")  # Synthetic variable
+        assert isinstance(init_index.value, WholeNumberLiteral)
+        assert init_index.value.value == 0
+
+        init_length = desugared.statements[1]
+        assert isinstance(init_length, SetStatement)
+        assert isinstance(init_length.name, Identifier)
+        assert init_length.name.value.startswith("$foreach_len_")  # Synthetic variable
+        assert init_length.value is not None
+        assert isinstance(init_length.value, CallExpression)
+        assert init_length.value.function_name is not None
+        assert isinstance(init_length.value.function_name, Identifier)
+        assert init_length.value.function_name.value == "len"
+
+        # The desugared ForEachStatement returns a BlockStatement containing:
+        # [0] Set $foreach_idx_N to 0
+        # [1] Set $foreach_len_N to len(collection)
+        # [2] WhileStatement with the loop logic
+        while_stmt = desugared.statements[2]
+
+        # Verify the while statement structure
+        from machine_dialect.ast.expressions import CollectionAccessExpression, InfixExpression
+        from machine_dialect.ast.statements import WhileStatement
+
+        assert isinstance(while_stmt, WhileStatement)
+
+        # Check condition: index < length
+        assert isinstance(while_stmt.condition, InfixExpression)
+        assert while_stmt.condition.operator == "<"
+        assert isinstance(while_stmt.condition.left, Identifier)
+        assert while_stmt.condition.left.value.startswith("$foreach_idx_")
+        assert isinstance(while_stmt.condition.right, Identifier)
+        assert while_stmt.condition.right.value.startswith("$foreach_len_")
+
+        # Check while body
+        assert isinstance(while_stmt.body, BlockStatement)
+        assert len(while_stmt.body.statements) >= 3  # set item, original body, increment
+
+        # First statement should set item = collection[index]
+        first_stmt = while_stmt.body.statements[0]
+        assert isinstance(first_stmt, SetStatement)
+        assert first_stmt.name is not None
+        assert first_stmt.name.value == "item"  # The original loop variable
+        assert isinstance(first_stmt.value, CollectionAccessExpression)
+
+        # Last statement should increment index
+        last_stmt = while_stmt.body.statements[-1]
+        assert isinstance(last_stmt, SetStatement)
+        assert last_stmt.name is not None
+        assert last_stmt.name.value.startswith("$foreach_idx_")
+        assert isinstance(last_stmt.value, InfixExpression)
+        assert last_stmt.value.operator == "+"
+
+    def test_foreach_with_literal_collection(self) -> None:
+        """Test for-each with a literal list as collection."""
+        # Create tokens
+        for_token = Token(TokenType.KW_FOR, "for", 1, 1)
+        item_token = Token(TokenType.MISC_IDENT, "fruit", 1, 10)
+
+        # Create a literal list
+        list_token = Token(TokenType.MISC_IDENT, "[", 1, 20)
+        str1_token = Token(TokenType.LIT_TEXT, "apple", 1, 22)
+        str2_token = Token(TokenType.LIT_TEXT, "banana", 1, 30)
+
+        collection = OrderedListLiteral(
+            list_token, [StringLiteral(str1_token, "apple"), StringLiteral(str2_token, "banana")]
+        )
+
+        # Create for-each with literal collection
+        foreach_stmt = ForEachStatement(
+            for_token,
+            item=Identifier(item_token, "fruit"),
+            collection=collection,
+            body=BlockStatement(for_token),  # Empty body
+        )
+
+        # Desugar
+        desugared = foreach_stmt.desugar()
+
+        # Should still produce valid desugared form
+        assert isinstance(desugared, BlockStatement)
+        assert len(desugared.statements) == 3
+
+    def test_foreach_empty_body(self) -> None:
+        """Test for-each with empty body."""
+        for_token = Token(TokenType.KW_FOR, "for", 1, 1)
+
+        foreach_stmt = ForEachStatement(
+            for_token,
+            item=Identifier(Token(TokenType.MISC_IDENT, "x", 1, 10), "x"),
+            collection=Identifier(Token(TokenType.MISC_IDENT, "xs", 1, 15), "xs"),
+            body=None,
+        )
+
+        desugared = foreach_stmt.desugar()
+
+        # Should still produce valid structure
+        assert isinstance(desugared, BlockStatement)
+        assert len(desugared.statements) == 3
+
+    def test_foreach_malformed_missing_parts(self) -> None:
+        """Test for-each with missing item or collection."""
+        for_token = Token(TokenType.KW_FOR, "for", 1, 1)
+
+        # Missing collection
+        foreach_stmt = ForEachStatement(
+            for_token,
+            item=Identifier(Token(TokenType.MISC_IDENT, "x", 1, 10), "x"),
+            collection=None,
+            body=BlockStatement(for_token),
+        )
+
+        desugared = foreach_stmt.desugar()
+
+        # Should return an empty while statement for malformed input
+        from machine_dialect.ast.statements import WhileStatement
+
+        assert isinstance(desugared, WhileStatement)
+        assert desugared.condition is None
+        assert desugared.body is None
+
+    def test_gensym_uniqueness(self) -> None:
+        """Test that gensym generates unique variable names."""
+        # Reset counter for predictable testing
+        original_counter = ForEachStatement._gensym_counter
+        ForEachStatement._gensym_counter = 0
+
+        try:
+            # Generate multiple synthetic variables
+            var1 = ForEachStatement._gensym("test")
+            var2 = ForEachStatement._gensym("test")
+            var3 = ForEachStatement._gensym("other")
+
+            # All should be unique
+            assert var1.value == "$test_1"
+            assert var2.value == "$test_2"
+            assert var3.value == "$other_3"
+
+            # All should have $ prefix (invalid for user variables)
+            assert all(v.value.startswith("$") for v in [var1, var2, var3])
+        finally:
+            # Restore original counter
+            ForEachStatement._gensym_counter = original_counter
+
+    def test_nested_foreach_unique_variables(self) -> None:
+        """Test that nested for-each loops get unique synthetic variables."""
+        # Create outer for-each
+        outer_foreach = ForEachStatement(
+            Token(TokenType.KW_FOR, "for", 1, 1),
+            item=Identifier(Token(TokenType.MISC_IDENT, "x", 1, 10), "x"),
+            collection=Identifier(Token(TokenType.MISC_IDENT, "xs", 1, 15), "xs"),
+            body=BlockStatement(Token(TokenType.KW_FOR, "for", 1, 1)),
+        )
+
+        # Create inner for-each
+        inner_foreach = ForEachStatement(
+            Token(TokenType.KW_FOR, "for", 2, 1),
+            item=Identifier(Token(TokenType.MISC_IDENT, "y", 2, 10), "y"),
+            collection=Identifier(Token(TokenType.MISC_IDENT, "ys", 2, 15), "ys"),
+            body=BlockStatement(Token(TokenType.KW_FOR, "for", 2, 1)),
+        )
+
+        # Desugar both
+        outer_desugared = outer_foreach.desugar()
+        inner_desugared = inner_foreach.desugar()
+
+        # Extract synthetic variable names from both
+        assert isinstance(outer_desugared, BlockStatement)
+        assert isinstance(inner_desugared, BlockStatement)
+
+        # Cast to SetStatement and check name existence
+        outer_set0 = outer_desugared.statements[0]
+        outer_set1 = outer_desugared.statements[1]
+        inner_set0 = inner_desugared.statements[0]
+        inner_set1 = inner_desugared.statements[1]
+
+        assert isinstance(outer_set0, SetStatement)
+        assert isinstance(outer_set1, SetStatement)
+        assert isinstance(inner_set0, SetStatement)
+        assert isinstance(inner_set1, SetStatement)
+
+        assert outer_set0.name is not None
+        assert outer_set1.name is not None
+        assert inner_set0.name is not None
+        assert inner_set1.name is not None
+
+        outer_index_var = outer_set0.name.value
+        outer_length_var = outer_set1.name.value
+        inner_index_var = inner_set0.name.value
+        inner_length_var = inner_set1.name.value
+
+        # All should be unique
+        all_vars = {outer_index_var, outer_length_var, inner_index_var, inner_length_var}
+        assert len(all_vars) == 4, "All synthetic variables should be unique"
+
+        # All should start with $
+        assert all(v.startswith("$") for v in all_vars)

machine_dialect/cfg/__init__.py

@@ -0,0 +1,6 @@
+"""Context Free Grammar module for Machine Dialect™ with GPT-5 integration."""
+
+from .openai_generation import generate_with_openai, validate_model_support
+from .parser import CFGParser
+
+__all__ = ["CFGParser", "generate_with_openai", "validate_model_support"]

machine_dialect/cfg/config.py

@@ -0,0 +1,156 @@
+"""Configuration module for Machine Dialect™ AI API settings."""
+
+import configparser
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class AIAPIConfig:
+    """Configuration for AI API settings.
+
+    Attributes:
+        model: The AI model to use (e.g., 'gpt-5', 'gpt-5-mini').
+        key: The API key for authentication.
+    """
+
+    model: str | None = None
+    key: str | None = None
+
+    def is_valid(self) -> bool:
+        """Check if the configuration has all required fields.
+
+        Returns:
+            True if both model and key are set, False otherwise.
+        """
+        return self.model is not None and self.key is not None
+
+    def with_defaults(self) -> "AIAPIConfig":
+        """Return a config with default values filled in.
+
+        Returns:
+            Config with defaults applied where values are missing.
+        """
+        if self.model is None:
+            self.model = "gpt-5"  # Default to GPT-5 for CFG support
+        return self
+
+
+class ConfigLoader:
+    """Loader for Machine Dialect™ configuration."""
+
+    CONFIG_FILE_NAME = ".mdconfig"
+    ENV_MODEL_KEY = "MD_AI_API_MODEL"
+    ENV_API_KEY = "MD_AI_API_KEY"
+
+    def __init__(self) -> None:
+        """Initialize the configuration loader."""
+        self._config: AIAPIConfig | None = None
+
+    def load(self) -> AIAPIConfig:
+        """Load AI API configuration from file or environment.
+
+        Priority order:
+        1. .mdconfig file in user's home directory
+        2. Environment variables (MD_AI_API_MODEL and MD_AI_API_KEY)
+        3. Legacy environment variable (OPENAI_API_KEY) for backward compatibility
+
+        Returns:
+            AIAPIConfig object with loaded settings.
+        """
+        if self._config is not None:
+            return self._config
+
+        config = AIAPIConfig()
+
+        # Try to load from .mdconfig file
+        config_file_path = Path.home() / self.CONFIG_FILE_NAME
+        if config_file_path.exists():
+            config = self._load_from_file(config_file_path)
+
+        # Override or fill in with environment variables
+        env_model = os.getenv(self.ENV_MODEL_KEY)
+        if env_model:
+            config.model = env_model
+
+        env_key = os.getenv(self.ENV_API_KEY)
+        if env_key:
+            config.key = env_key
+        elif not config.key:
+            # Fallback to legacy OPENAI_API_KEY for backward compatibility
+            config.key = os.getenv("OPENAI_API_KEY")
+
+        self._config = config
+        return config
+
+    def _load_from_file(self, config_file_path: Path) -> AIAPIConfig:
+        """Load configuration from a .mdconfig file.
+
+        Args:
+            config_file_path: Path to the configuration file.
+
+        Returns:
+            AIAPIConfig object with settings from the file.
+        """
+        config_parser = configparser.ConfigParser()
+        config_parser.read(config_file_path)
+
+        ai_config = AIAPIConfig()
+
+        try:
+            ai_section = config_parser["ai-api"]
+            ai_config.model = ai_section.get("model")
+            ai_config.key = ai_section.get("key")
+        except (configparser.NoSectionError, KeyError):
+            # Section doesn't exist, return empty config
+            pass
+
+        return ai_config
+
+    def get_error_message(self) -> str:
+        """Get a helpful error message for missing configuration.
+
+        Returns:
+            Error message with instructions for setting up configuration.
+        """
+        home_dir = Path.home()
+        config_path = home_dir / self.CONFIG_FILE_NAME
+
+        return f"""AI API configuration not found.
+
+Please configure the AI API in one of the following ways:
+
+1. Create a {self.CONFIG_FILE_NAME} file in your home directory ({config_path}):
+
+   [ai-api]
+   model = gpt-5
+   key = your_api_key_here
+
+   Note: Only GPT-5 models (gpt-5, gpt-5-mini, gpt-5-nano) support
+   context-free grammar constraints required for Machine Dialect™ generation.
+
+2. Set environment variables:
+
+   export {self.ENV_MODEL_KEY}=gpt-5
+   export {self.ENV_API_KEY}=your_api_key_here
+
+3. For backward compatibility, you can also use:
+
+   export OPENAI_API_KEY=your_api_key_here
+   (Note: Model will default to gpt-5 if not specified)
+
+To get an API key: https://platform.openai.com/api-keys"""
+
+
+def get_ai_config() -> AIAPIConfig:
+    """Get the AI API configuration.
+
+    This is a convenience function that creates a ConfigLoader
+    and loads the configuration.
+
+    Returns:
+        AIAPIConfig object with current settings.
+    """
+    loader = ConfigLoader()
+    return loader.load()

machine_dialect/cfg/examples.py

@@ -0,0 +1,221 @@
+"""Examples and usage of the CFG module for Machine Dialect™.
+
+This module provides demonstration examples of how to use the CFG (Context-Free Grammar)
+parser for Machine Dialect™ code. It includes examples of parsing variable assignments,
+conditional statements, logical operations, and code validation.
+
+The examples show:
+- Basic parsing of Machine Dialect™ syntax
+- Handling conditional statements with if/else blocks
+- Working with logical operations and boolean values
+- Code validation to check syntax correctness
+- Pretty-printing of Abstract Syntax Trees (AST)
+
+Example:
+    Run all examples from the command line::
+
+        $ python -m machine_dialect.cfg.examples
+"""
+
+from machine_dialect.cfg import CFGParser
+
+
+def example_parse_code() -> None:
+    """Demonstrate parsing Machine Dialect™ code with the CFG parser.
+
+    This function shows three examples of parsing Machine Dialect™ code:
+    1. Simple variable assignment and arithmetic operations
+    2. Conditional statements with if/else blocks
+    3. Logical operations with boolean values
+
+    Each example prints the original code, attempts to parse it, and displays
+    the resulting Abstract Syntax Tree (AST) if successful.
+
+    Raises:
+        ValueError: If any of the code examples fail to parse.
+
+    Example:
+        >>> example_parse_code()
+        Example 1: Simple arithmetic
+        Code: ...
+        Parse successful!
+        AST: ...
+    """
+    parser = CFGParser()
+
+    # Example 1: Simple variable assignment and output
+    code1 = """
+    Set `x` to _10_.
+    Set `y` to _20_.
+    Set `sum` to `x` + `y`.
+    Say `sum`.
+    """
+
+    print("Example 1: Simple arithmetic")
+    print("Code:", code1)
+    try:
+        tree = parser.parse(code1)
+        print("Parse successful!")
+        print("AST:")
+        print(parser.pretty_print(tree))
+    except ValueError as e:
+        print(f"Parse failed: {e}")
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Example 2: Conditional statement
+    code2 = """
+    Set `age` to _18_.
+    If `age` is greater than _17_ then:
+    > Say _"You are an adult."_.
+    Else:
+    > Say _"You are a minor."_.
+    """
+
+    print("Example 2: Conditional")
+    print("Code:", code2)
+    try:
+        tree = parser.parse(code2)
+        print("Parse successful!")
+        print("AST:")
+        print(parser.pretty_print(tree))
+    except ValueError as e:
+        print(f"Parse failed: {e}")
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Example 3: Logical operations
+    code3 = """
+    Set `is_raining` to _yes_.
+    Set `have_umbrella` to _no_.
+    Set `get_wet` to `is_raining` and not `have_umbrella`.
+    If `get_wet` then:
+    > Say _"You will get wet!"_.
+    """
+
+    print("Example 3: Logical operations")
+    print("Code:", code3)
+    try:
+        tree = parser.parse(code3)
+        print("Parse successful!")
+        print("AST:")
+        print(parser.pretty_print(tree))
+    except ValueError as e:
+        print(f"Parse failed: {e}")
+
+
+def example_generate_prompt() -> None:
+    """Demonstrate creating prompts for GPT-5 CFG generation.
+
+    This function serves as a placeholder for future CFG generation
+    functionality using GPT-5 or similar language models. Once implemented,
+    it will show how to create prompts that guide AI models to generate
+    valid Machine Dialect™ code following the CFG rules.
+
+    Note:
+        This functionality is not yet implemented and will be added
+        in a future version.
+
+    Todo:
+        * Implement prompt generation for CFG-based code generation
+        * Add examples of different prompt types
+        * Include validation of generated code
+    """
+    # Placeholder for CFG generation examples
+    print("CFG generation functionality coming soon.")
+
+
+def example_validate_code() -> None:
+    """Demonstrate validation of Machine Dialect™ code syntax.
+
+    This function shows examples of both valid and invalid Machine Dialect™
+    code to illustrate the validation capabilities of the CFG parser.
+    It demonstrates common syntax errors like missing backticks around
+    variables and missing periods at the end of statements.
+
+    The function validates:
+    - Valid code with proper syntax (backticks, periods)
+    - Invalid code with missing syntax elements
+
+    Example:
+        >>> example_validate_code()
+        Validating valid code:
+        Set `name` to "Alice".
+        Say name.
+        ✓ Code is valid!
+    """
+    parser = CFGParser()
+
+    # Valid code
+    valid_code = """
+    Set `name` to _"Alice"_.
+    Say `name`.
+    """
+
+    print("Validating valid code:")
+    print(valid_code)
+    if parser.validate(valid_code):
+        print("✓ Code is valid!")
+    else:
+        print("✗ Code is invalid!")
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Invalid code
+    invalid_code = """
+    Set x to 10
+    Say x
+    """
+
+    print("Validating invalid code (missing backticks and periods):")
+    print(invalid_code)
+    if parser.validate(invalid_code):
+        print("✓ Code is valid!")
+    else:
+        print("✗ Code is invalid!")
+
+
+def main() -> None:
+    """Run all CFG parser examples in sequence.
+
+    This function executes all the example functions to demonstrate
+    the full capabilities of the CFG parser for Machine Dialect™.
+    It runs parsing examples, generation prompt examples, and
+    validation examples, separating each section with visual dividers
+    for clarity.
+
+    The execution order is:
+    1. Parsing examples - demonstrating code parsing
+    2. Generation prompt examples - placeholder for future features
+    3. Validation examples - showing syntax validation
+
+    Example:
+        >>> main()
+        ============================================================
+        CFG Parser Examples
+        ============================================================
+        ...
+    """
+    print("=" * 60)
+    print("CFG Parser Examples")
+    print("=" * 60)
+    print()
+
+    print("1. PARSING EXAMPLES")
+    print("-" * 40)
+    example_parse_code()
+
+    print("\n2. GENERATION PROMPT EXAMPLES")
+    print("-" * 40)
+    example_generate_prompt()
+
+    print("\n3. VALIDATION EXAMPLES")
+    print("-" * 40)
+    example_validate_code()
+
+    print("\n" + "=" * 60)
+    print("Examples complete!")
+
+
+if __name__ == "__main__":
+    main()