machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/parser/tests/test_utility_statements.py

@@ -0,0 +1,423 @@
+"""Tests for Utility statements (functions) in Machine Dialect™."""
+
+from machine_dialect.ast import BlockStatement, SetStatement, UtilityStatement
+from machine_dialect.parser import Parser
+
+
+class TestUtilityStatements:
+    """Test parsing of Utility statements (functions)."""
+
+    def test_simple_utility_without_parameters(self) -> None:
+        """Test parsing a simple utility without parameters."""
+        source = """### **Utility**: `calculate pi`
+
+<details>
+<summary>Calculates the value of pi.</summary>
+
+> Define `result` as Number.
+> Set `result` to _3.14159_.
+> Give back `result`.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0, f"Parser errors: {parser.errors}"
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "calculate pi"
+        assert len(utility_stmt.inputs) == 0
+        assert len(utility_stmt.outputs) == 0
+        assert isinstance(utility_stmt.body, BlockStatement)
+        assert len(utility_stmt.body.statements) == 3  # Define + Set + Give back
+
+        from machine_dialect.ast import DefineStatement
+
+        # Check first statement: Define `result` as Number.
+        define_stmt = utility_stmt.body.statements[0]
+        assert isinstance(define_stmt, DefineStatement)
+        assert define_stmt.name.value == "result"
+
+        # Check second statement: Set `result` to _3.14159_.
+        set_stmt = utility_stmt.body.statements[1]
+        assert isinstance(set_stmt, SetStatement)
+        assert set_stmt.name and set_stmt.name.value == "result"
+
+    def test_utility_with_inputs_and_outputs(self) -> None:
+        """Test parsing a utility with input and output parameters."""
+        source = """### **Utility**: `add two numbers`
+
+<details>
+<summary>Adds two numbers and returns the result</summary>
+
+> Define `result` as Whole Number.
+> Set `result` to `addend 1` + `addend 2`.
+
+</details>
+
+#### Inputs:
+- `addend 1` **as** Whole Number (required)
+- `addend 2` **as** Whole Number (required)
+
+#### Outputs:
+- `result` **as** Yes/No (default: _Empty_)"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0, f"Parser errors: {parser.errors}"
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "add two numbers"
+        assert len(utility_stmt.inputs) == 2
+        assert len(utility_stmt.outputs) == 1
+
+        # Check inputs
+        assert utility_stmt.inputs[0].name.value == "addend 1"
+        assert utility_stmt.inputs[0].type_name == "Whole Number"
+        assert utility_stmt.inputs[0].is_required is True
+
+        assert utility_stmt.inputs[1].name.value == "addend 2"
+        assert utility_stmt.inputs[1].type_name == "Whole Number"
+        assert utility_stmt.inputs[1].is_required is True
+
+        # Check outputs (now using Output class)
+        from machine_dialect.ast import EmptyLiteral, Output
+
+        assert len(utility_stmt.outputs) == 1
+        assert isinstance(utility_stmt.outputs[0], Output)
+        assert utility_stmt.outputs[0].name.value == "result"
+        assert utility_stmt.outputs[0].type_name == "Yes/No"
+        # Check that it has a default value of Empty
+        assert utility_stmt.outputs[0].default_value is not None
+        assert isinstance(utility_stmt.outputs[0].default_value, EmptyLiteral)
+
+    def test_utility_with_heading_level(self) -> None:
+        """Test that utility heading level (###) is parsed correctly."""
+        source = """### **Utility**: `double value`
+
+<details>
+<summary>Doubles the input value.</summary>
+
+> Define `result` as Number.
+> Set `result` to `value` * _2_.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "double value"
+
+    def test_utility_with_multi_word_name(self) -> None:
+        """Test utility with multi-word name in backticks."""
+        source = """### **Utility**: `calculate compound interest`
+
+<details>
+<summary>Calculates compound interest.</summary>
+
+> Define `amount` as Number.
+> Set `amount` to _1000_.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "calculate compound interest"
+
+    def test_utility_with_empty_body(self) -> None:
+        """Test utility with no statements in body."""
+        source = """### **Utility**: `identity function`
+
+<details>
+<summary>Returns nothing.</summary>
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "identity function"
+        assert len(utility_stmt.body.statements) == 0
+
+    def test_multiple_utilities(self) -> None:
+        """Test parsing multiple utilities in one program."""
+        source = """### **Utility**: `first utility`
+
+<details>
+<summary>First utility.</summary>
+
+> Define `x` as Number.
+> Set `x` to _1_.
+
+</details>
+
+### **Utility**: `second utility`
+
+<details>
+<summary>Second utility.</summary>
+
+> Define `second_y` as Number.
+> Set `second_y` to _2_.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 2
+
+        # Check first utility
+        first_utility = program.statements[0]
+        assert isinstance(first_utility, UtilityStatement)
+        assert first_utility.name.value == "first utility"
+        assert len(first_utility.body.statements) == 2  # Define + Set
+
+        # Check second utility
+        second_utility = program.statements[1]
+        assert isinstance(second_utility, UtilityStatement)
+        assert second_utility.name.value == "second utility"
+        assert len(second_utility.body.statements) == 2  # Define + Set
+
+    def test_utility_with_complex_body(self) -> None:
+        """Test utility with complex body including conditionals."""
+        source = """### **Utility**: `absolute value`
+
+<details>
+<summary>Returns the absolute value of a number.</summary>
+
+> Define `result` as Number.
+> If `number` < _0_ then:
+> > Set `result` to -`number`.
+> Else:
+> > Set `result` to `number`.
+>
+> Give back `result`.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 1
+
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "absolute value"
+        assert utility_stmt.description == "Returns the absolute value of a number."
+        assert len(utility_stmt.body.statements) == 3  # Define + If statement + Give back statement
+
+    def test_mixed_statements_with_utility(self) -> None:
+        """Test that utilities can coexist with actions and interactions."""
+        source = """### **Action**: `private method`
+
+<details>
+<summary>A private action.</summary>
+
+> Define `x` as Number.
+> Set `x` to _1_.
+
+</details>
+
+### **Utility**: `helper function`
+
+<details>
+<summary>A utility function.</summary>
+
+> Give back _42_.
+
+</details>
+
+### **Interaction**: `public method`
+
+<details>
+<summary>A public interaction.</summary>
+
+> Define `y` as Number.
+> Set `y` to _2_.
+
+</details>"""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 3
+
+        # Check action
+        from machine_dialect.ast import ActionStatement, InteractionStatement
+
+        action_stmt = program.statements[0]
+        assert isinstance(action_stmt, ActionStatement)
+        assert action_stmt.name.value == "private method"
+
+        # Check utility
+        utility_stmt = program.statements[1]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "helper function"
+
+        # Check interaction
+        interaction_stmt = program.statements[2]
+        assert isinstance(interaction_stmt, InteractionStatement)
+        assert interaction_stmt.name.value == "public method"
+
+    def test_utility_with_recursive_call(self) -> None:
+        """Test utility with recursive call (Fibonacci example)."""
+        source = """### **Utility**: `Fibonacci`
+
+<details>
+<summary>Calculate the nth Fibonacci number recursively</summary>
+
+> If `n` is less than or equal to _1_:
+> >
+> > Give back `n`.
+> >
+> Else:
+> >
+> > Define `n_minus_1` as Whole Number.
+> > Set `n_minus_1` to `n` - _1_.
+> > Define `n_minus_2` as Whole Number.
+> > Set `n_minus_2` to `n` - _2_.
+> > Define `fib_1` as Whole Number.
+> > Set `fib_1` using `Fibonacci` with `n_minus_1`.
+> > Define `fib_2` as Whole Number.
+> > Set `fib_2` using `Fibonacci` with `n_minus_2`.
+> > Define `result` as Whole Number.
+> > Set `result` to `fib_1` + `fib_2`.
+> > Give back `result`.
+
+</details>
+
+#### Inputs:
+
+- `n` **as** Whole Number (required)
+
+#### Outputs:
+
+- `result` **as** Whole Number
+
+Define `m` as Whole Number.
+Set `m` to _10_.
+
+Define `final result` as Whole Number.
+Set `final result` using `Fibonacci` with `m`.
+
+Say `final result`."""
+
+        parser = Parser()
+        program = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 0, f"Parser errors: {parser.errors}"
+        assert (
+            len(program.statements) == 6
+        )  # Utility + Define m + Set m + Define final result + Set final result using + Say
+
+        # Check the utility statement
+        utility_stmt = program.statements[0]
+        assert isinstance(utility_stmt, UtilityStatement)
+        assert utility_stmt.name.value == "Fibonacci"
+        assert utility_stmt.description == "Calculate the nth Fibonacci number recursively"
+
+        # Check inputs
+        assert len(utility_stmt.inputs) == 1
+        assert utility_stmt.inputs[0].name.value == "n"
+        assert utility_stmt.inputs[0].type_name == "Whole Number"
+        assert utility_stmt.inputs[0].is_required is True
+
+        # Check outputs
+        assert len(utility_stmt.outputs) == 1
+        assert utility_stmt.outputs[0].name.value == "result"
+        assert utility_stmt.outputs[0].type_name == "Whole Number"
+
+        # Check the body contains an if statement
+        from machine_dialect.ast import DefineStatement, IfStatement, ReturnStatement, SayStatement
+
+        assert len(utility_stmt.body.statements) == 1
+        if_stmt = utility_stmt.body.statements[0]
+        assert isinstance(if_stmt, IfStatement)
+
+        # Check the if branch (n <= 1)
+        assert isinstance(if_stmt.consequence, BlockStatement)
+        assert len(if_stmt.consequence.statements) == 1
+        assert isinstance(if_stmt.consequence.statements[0], ReturnStatement)
+
+        # Check the else branch (recursive case)
+        assert if_stmt.alternative is not None
+        assert isinstance(if_stmt.alternative, BlockStatement)
+        else_statements = if_stmt.alternative.statements
+
+        # Should have: 2 Define, 2 Set for n_minus_1 and n_minus_2
+        # Then 2 Define, 2 Set using for fib_1 and fib_2
+        # Then 1 Define, 1 Set for result, and 1 Give back
+        assert len(else_statements) == 11
+
+        # Check recursive calls (Set using statements with CallExpression)
+        from machine_dialect.ast import CallExpression
+
+        recursive_calls = [
+            stmt
+            for stmt in else_statements
+            if isinstance(stmt, SetStatement) and isinstance(stmt.value, CallExpression)
+        ]
+        assert len(recursive_calls) == 2
+
+        # Both should call "Fibonacci"
+        from machine_dialect.ast import Identifier
+
+        for call in recursive_calls:
+            assert isinstance(call.value, CallExpression)
+            assert call.value.function_name is not None
+            assert isinstance(call.value.function_name, Identifier)
+            assert call.value.function_name.value == "Fibonacci"
+
+        # Check the main program statements after the utility
+        assert isinstance(program.statements[1], DefineStatement)
+        assert program.statements[1].name.value == "m"
+
+        assert isinstance(program.statements[2], SetStatement)
+        set_m = program.statements[2]
+        assert set_m.name is not None
+        assert set_m.name.value == "m"
+
+        assert isinstance(program.statements[3], DefineStatement)
+        assert program.statements[3].name.value == "final result"
+
+        # Check the utility call in main program
+        assert isinstance(program.statements[4], SetStatement)
+        set_final = program.statements[4]
+        assert set_final.name is not None
+        assert set_final.name.value == "final result"
+        assert isinstance(set_final.value, CallExpression)
+        assert set_final.value.function_name is not None
+        assert isinstance(set_final.value.function_name, Identifier)
+        assert set_final.value.function_name.value == "Fibonacci"
+
+        # Check the Say statement
+        assert isinstance(program.statements[5], SayStatement)
+        say_stmt = program.statements[5]
+        # The Say statement should reference the final result identifier
+        assert isinstance(say_stmt.expression, Identifier)
+        assert say_stmt.expression.value == "final result"

machine_dialect/parser/token_buffer.py

@@ -0,0 +1,159 @@
+"""Token buffer for streaming tokens from lexer to parser.
+
+This module provides a TokenBuffer class that maintains a small buffer of tokens
+and streams them from the lexer to the parser on demand, instead of generating
+all tokens upfront.
+"""
+
+from machine_dialect.lexer.tokens import Token, TokenType
+
+# Buffer size constant - number of tokens to keep in the buffer
+BUFFER_SIZE = 4
+
+
+class TokenBuffer:
+    """Buffer for streaming tokens from lexer to parser.
+
+    Maintains a small buffer of tokens and fetches new tokens from the lexer
+    as needed. This allows for memory-efficient parsing of large files.
+
+    Attributes:
+        _lexer: The lexer instance to get tokens from.
+        _buffer: Internal buffer of tokens.
+        _eof_reached: Whether EOF has been reached.
+    """
+
+    def __init__(self, lexer) -> None:  # type: ignore
+        """Initialize the token buffer with a lexer.
+
+        Args:
+            lexer: The lexer instance to stream tokens from.
+        """
+        from machine_dialect.lexer import Lexer
+
+        self._lexer: Lexer = lexer
+        self._buffer: list[Token] = []
+        self._at_line_start: list[bool] = []  # Track if each token is at line start
+        self._eof_reached = False
+        self._in_block = False  # Track block context
+        self._in_list_context = False  # Track list definition context
+
+        # Pre-fill the buffer
+        self._fill_buffer()
+
+    def _fill_buffer(self) -> None:
+        """Fill the buffer with tokens from the lexer up to BUFFER_SIZE."""
+        while len(self._buffer) < BUFFER_SIZE and not self._eof_reached:
+            token = self._get_next_token()
+            if token is not None:
+                self._buffer.append(token)
+                # Determine if token is at line start based on its column position
+                # Column 1 means it's at the start of a line
+                at_line_start = token.position == 1
+                self._at_line_start.append(at_line_start)
+                if token.type == TokenType.MISC_EOF:
+                    self._eof_reached = True
+            else:
+                # No more tokens available
+                self._eof_reached = True
+                # Add EOF token if buffer is empty
+                if not self._buffer:
+                    self._buffer.append(
+                        Token(TokenType.MISC_EOF, "", line=self._lexer.line, position=self._lexer.column)
+                    )
+                    self._at_line_start.append(False)
+
+    def _get_next_token(self) -> Token | None:
+        """Get the next token from the lexer.
+
+        Returns:
+            The next token, or None if no more tokens are available.
+        """
+        # Pass the block and list contexts to the lexer
+        return self._lexer.next_token(in_block=self._in_block, in_list_context=self._in_list_context)
+
+    def current(self) -> Token | None:
+        """Get the current token without consuming it.
+
+        Returns:
+            The current token, or None if no tokens are available.
+        """
+        if self._buffer:
+            return self._buffer[0]
+        return None
+
+    def peek(self, offset: int = 1) -> Token | None:
+        """Peek at a token at the given offset without consuming tokens.
+
+        Args:
+            offset: How many tokens ahead to look (1 = next token).
+
+        Returns:
+            The token at the given offset, or None if not available.
+        """
+        # Ensure we have enough tokens in the buffer
+        while len(self._buffer) <= offset and not self._eof_reached:
+            token = self._get_next_token()
+            if token is not None:
+                self._buffer.append(token)
+                if token.type == TokenType.MISC_EOF:
+                    self._eof_reached = True
+            else:
+                self._eof_reached = True
+                break
+
+        if offset < len(self._buffer):
+            return self._buffer[offset]
+
+        # Return EOF token if we're past the buffer
+        return Token(TokenType.MISC_EOF, "", line=self._lexer.line, position=self._lexer.column)
+
+    def advance(self) -> None:
+        """Consume the current token and advance to the next one."""
+        if self._buffer:
+            self._buffer.pop(0)
+            if self._at_line_start:
+                self._at_line_start.pop(0)
+            # Refill the buffer to maintain BUFFER_SIZE tokens
+            self._fill_buffer()
+
+    def has_tokens(self) -> bool:
+        """Check if there are more tokens available.
+
+        Returns:
+            True if there are tokens available, False otherwise.
+        """
+        return bool(self._buffer)
+
+    def set_block_context(self, in_block: bool) -> None:
+        """Set the block parsing context.
+
+        Args:
+            in_block: Whether we're currently parsing inside a block.
+        """
+        self._in_block = in_block
+
+    def set_list_context(self, in_list: bool) -> None:
+        """Set the list definition parsing context.
+
+        Args:
+            in_list: Whether we're currently parsing a list definition.
+        """
+        if self._in_list_context != in_list:
+            self._in_list_context = in_list
+
+            # Update token types in the buffer based on new context
+            for i, token in enumerate(self._buffer):
+                if token.literal == "-" and i < len(self._at_line_start):
+                    at_line_start = self._at_line_start[i]
+
+                    if in_list and at_line_start:
+                        # In list context and at line start: should be PUNCT_DASH
+                        if token.type == TokenType.OP_MINUS:
+                            self._buffer[i] = Token(TokenType.PUNCT_DASH, token.literal, token.line, token.position)
+                    elif not in_list or not at_line_start:
+                        # Not in list context or not at line start: should be OP_MINUS
+                        if token.type == TokenType.PUNCT_DASH:
+                            self._buffer[i] = Token(TokenType.OP_MINUS, token.literal, token.line, token.position)
+        else:
+            self._in_list_context = in_list

machine_dialect/repl/__init__.py

@@ -0,0 +1,3 @@
+from machine_dialect.repl.repl import main
+
+__all__ = ["main"]