machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/parser/tests/test_no_none_in_ast.py

@@ -0,0 +1,94 @@
+"""Tests to ensure parser never returns None in AST nodes.
+
+When errors occur, the parser should always return ErrorExpression or
+ErrorStatement nodes to preserve the AST structure, never None.
+"""
+
+import pytest
+
+from machine_dialect.ast import ErrorStatement, ExpressionStatement
+from machine_dialect.parser import Parser
+
+
+class TestNoNoneInAST:
+    """Test that parser always creates AST nodes, never None."""
+
+    @pytest.mark.parametrize(
+        "source",
+        [
+            "(42",  # Missing closing parenthesis
+            "((42)",  # Missing one closing parenthesis
+            "()",  # Empty parentheses
+            "( + 42)",  # Operator at start inside parens
+            "Set x 42.",  # Missing 'to' keyword
+            "Set @ to 42.",  # Illegal character as identifier
+            "42 + @ + 5.",  # Illegal character in expression
+        ],
+    )
+    def test_errors_produce_error_nodes_not_none(self, source: str) -> None:
+        """Test that parsing errors result in Error nodes, not None."""
+        parser = Parser()
+        program = parser.parse(source)
+
+        # Should have at least one error
+        assert parser.has_errors()
+
+        # Should have at least one statement
+        assert len(program.statements) > 0
+
+        # Check all statements are not None
+        for stmt in program.statements:
+            assert stmt is not None
+
+            # If it's an expression statement, check the expression
+            if isinstance(stmt, ExpressionStatement):
+                assert stmt.expression is not None
+                # Many of these should be ErrorExpressions
+                if parser.has_errors() and not isinstance(stmt, ErrorStatement):
+                    # If there were errors and it's not an ErrorStatement,
+                    # the expression might be an ErrorExpression
+                    pass  # This is fine, we just care it's not None
+
+    def test_invalid_float_produces_error_expression(self) -> None:
+        """Test that invalid float literals produce ErrorExpression."""
+        # This would need to bypass the lexer somehow to test the parser directly
+        # Since the lexer validates floats, this is hard to test in integration
+        # We trust that our changes to _parse_float_literal work
+        pass
+
+    def test_invalid_integer_produces_error_expression(self) -> None:
+        """Test that invalid integer literals produce ErrorExpression."""
+        # This would need to bypass the lexer somehow to test the parser directly
+        # Since the lexer validates integers, this is hard to test in integration
+        # We trust that our changes to _parse_integer_literal work
+        pass
+
+    def test_nested_errors_still_create_nodes(self) -> None:
+        """Test that nested parsing errors still create AST nodes."""
+        source = "Set x to (42 + (5 * @))."
+        parser = Parser()
+        program = parser.parse(source)
+
+        # Should have errors
+        assert parser.has_errors()
+
+        # Should have a statement
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert stmt is not None
+
+    def test_multiple_grouped_expression_errors(self) -> None:
+        """Test multiple grouped expression errors."""
+        source = "(42 + (5 * 3"  # Missing two closing parens
+        parser = Parser()
+        program = parser.parse(source)
+
+        # Should have error(s)
+        assert parser.has_errors()
+
+        # Should have a statement with an expression
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert stmt is not None
+        if isinstance(stmt, ExpressionStatement):
+            assert stmt.expression is not None

machine_dialect/parser/tests/test_panic_mode_recovery.py

@@ -0,0 +1,171 @@
+"""Tests for parser panic mode recovery.
+
+This module tests the parser's panic mode recovery mechanism which allows
+the parser to recover from errors and continue parsing to find more errors.
+"""
+
+from machine_dialect.errors.exceptions import MDSyntaxError
+from machine_dialect.parser import Parser
+
+
+class TestPanicModeRecovery:
+    """Test panic mode recovery in the parser."""
+
+    def test_recovery_at_period_boundary(self) -> None:
+        """Test that panic mode stops at period boundaries."""
+        source = "Define `x` as Whole Number. Set @ to 42. Set `x` to 5."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have one error for the illegal @ character
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+        assert "@" in str(parser.errors[0])
+
+        # Should have parsed the statements successfully
+        assert len(program.statements) == 3  # Define + 2 Sets
+        # First statement should be Define
+        assert program.statements[0] is not None
+        # Second statement should be incomplete (error due to @)
+        assert program.statements[1] is not None
+        # Third statement should be complete
+        assert program.statements[2] is not None
+
+    def test_recovery_at_eof_boundary(self) -> None:
+        """Test that panic mode stops at EOF."""
+        source = "Set @ to 42"  # No period at end
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have one error for the illegal @ character
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+
+        # Should have one statement (incomplete due to error)
+        assert len(program.statements) == 1
+
+    def test_multiple_errors_with_recovery(self) -> None:
+        """Test that multiple errors are collected with recovery between them."""
+        source = "Define `x` as Whole Number. Set @ to 42. Set # to 10. Set `x` to 5."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have two errors for the illegal characters
+        assert len(parser.errors) == 2
+        assert "@" in str(parser.errors[0])
+        assert "#" in str(parser.errors[1])
+        # @ is now MDSyntaxError (illegal token), # remains MDNameError (unexpected identifier)
+
+        # Should have four statements (Define + 3 Sets)
+        assert len(program.statements) == 4
+
+    def test_recovery_in_expression_context(self) -> None:
+        """Test panic recovery when error occurs in expression parsing."""
+        source = "Define `x` as Whole Number. Define `y` as Whole Number. Set `x` to @ + 5. Set `y` to 10."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have one error for the illegal @ in expression
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+
+        # Should have four statements (2 defines + 2 sets)
+        assert len(program.statements) == 4
+
+    def test_recovery_with_missing_keyword(self) -> None:
+        """Test panic recovery when 'to' keyword is missing."""
+        source = "Define `x` as Whole Number. Define `y` as Whole Number. Set `x` 42. Set `y` to 10."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have one error for missing 'to'
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+
+        # Should have four statements (2 defines + 2 sets)
+        assert len(program.statements) == 4
+
+    def test_recovery_with_consecutive_errors(self) -> None:
+        """Test panic recovery with errors in consecutive statements."""
+        source = "Set @ to 42. Set # to 10."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have two errors
+        assert len(parser.errors) == 2
+
+        # Should have two statements (both incomplete)
+        assert len(program.statements) == 2
+
+    def test_panic_counter_limit(self) -> None:
+        """Test that panic counter prevents infinite loops."""
+        # Create a source with many errors (more than 20)
+        statements = [f"Set @{i} to {i}." for i in range(25)]
+        source = " ".join(statements)
+
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Parser should stop after 20 panic recoveries
+        # We might have fewer errors if parser stops early
+        assert len(parser.errors) <= 20
+        assert len(program.statements) <= 20
+
+    def test_recovery_preserves_valid_parts(self) -> None:
+        """Test that valid parts of statements are preserved during recovery."""
+        source = "give back @. give back 42."
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have one error for illegal @
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+
+        # Should have two return statements
+        assert len(program.statements) == 2
+        # First should be incomplete, second should be complete
+        from machine_dialect.ast import ReturnStatement
+
+        if isinstance(program.statements[1], ReturnStatement):
+            assert program.statements[1].return_value is not None
+
+    def test_no_recovery_for_valid_code(self) -> None:
+        """Test that valid code doesn't trigger panic recovery."""
+        source = """Define `x` as Whole Number.
+Define `y` as Whole Number.
+Set `x` to _42_.
+Set `y` to _10_.
+give back `x`."""
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have no errors
+        assert len(parser.errors) == 0
+
+        # Should have five complete statements (2 defines + 2 sets + 1 return)
+        assert len(program.statements) == 5
+
+    def test_recovery_with_mixed_statement_types(self) -> None:
+        """Test panic recovery across different statement types."""
+        source = """Define `x` as Whole Number.
+Set @ to 42.
+give back #.
+Set `x` to _5_."""
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have two errors (@ is illegal, # is illegal)
+        assert len(parser.errors) == 2
+
+        # Should have four statements (1 define + 3 attempts)
+        assert len(program.statements) == 4

machine_dialect/parser/tests/test_parse_errors.py

@@ -0,0 +1,114 @@
+"""Tests for parser error handling.
+
+This module tests the parser's ability to properly report errors
+when encountering invalid syntax or missing parse functions.
+"""
+
+import pytest
+
+from machine_dialect.errors.exceptions import MDSyntaxError
+from machine_dialect.parser import Parser
+
+
+class TestParseErrors:
+    """Test parsing error handling."""
+
+    @pytest.mark.parametrize(
+        "source,expected_literal,expected_message",
+        [
+            ("* 42", "*", "unexpected token '*' at start of expression"),  # Multiplication operator at start
+            ("+ 5", "+", "unexpected token '+' at start of expression"),  # Plus operator at start
+            ("/ 10", "/", "unexpected token '/' at start of expression"),  # Division operator at start
+            (") x", ")", "No suitable parse function was found to handle ')'"),  # Right parenthesis at start
+            ("} x", "}", "No suitable parse function was found to handle '}'"),  # Right brace at start
+            (", x", ",", "No suitable parse function was found to handle ','"),  # Comma at start
+            ("; x", ";", "No suitable parse function was found to handle ';'"),  # Semicolon at start
+        ],
+    )
+    def test_no_prefix_parse_function_error(self, source: str, expected_literal: str, expected_message: str) -> None:
+        """Test error reporting when no prefix parse function exists.
+
+        Args:
+            source: The source code that should trigger an error.
+            expected_literal: The literal that should appear in the error message.
+            expected_message: The expected error message.
+        """
+        parser = Parser()
+
+        program = parser.parse(source, check_semantics=False)
+
+        # Should have exactly one error
+        assert len(parser.errors) == 1
+
+        # Check the error type and message
+        error = parser.errors[0]
+        assert isinstance(error, MDSyntaxError)
+        assert expected_message in str(error)
+
+        # The program should still be created
+        assert program is not None
+        # Parser continues after errors, so we'll have statements with ErrorExpression
+        # Check that the first statement has an error expression
+        if program.statements:
+            from machine_dialect.ast import ErrorExpression, ExpressionStatement
+
+            stmt = program.statements[0]
+            assert isinstance(stmt, ExpressionStatement)
+            assert isinstance(stmt.expression, ErrorExpression)
+
+    def test_multiple_parse_errors(self) -> None:
+        """Test that multiple parse errors are collected."""
+        source = "* 42. + 5. / 10."
+        parser = Parser()
+
+        parser.parse(source, check_semantics=False)
+
+        # Should have three errors (one for each invalid prefix)
+        assert len(parser.errors) == 3
+
+        # Check the error messages
+        expected_messages = [
+            "unexpected token '*' at start of expression",
+            "unexpected token '+' at start of expression",
+            "unexpected token '/' at start of expression",
+        ]
+
+        for error, expected_msg in zip(parser.errors, expected_messages, strict=True):
+            assert isinstance(error, MDSyntaxError)
+            assert expected_msg in str(error), f"Expected '{expected_msg}' in '{error!s}'"
+
+    def test_error_location_tracking(self) -> None:
+        """Test that errors track correct line and column positions."""
+        source = "   * 42"  # 3 spaces before *
+        parser = Parser()
+
+        _ = parser.parse(source, check_semantics=False)
+
+        assert len(parser.errors) == 1
+        error = parser.errors[0]
+
+        # The * should be at column 4 (1-indexed)
+        assert "column 4" in str(error)
+        assert "line 1" in str(error)
+
+    def test_valid_expression_no_errors(self) -> None:
+        """Test that valid expressions don't produce parse errors."""
+        valid_sources = [
+            "42",
+            "-42",
+            "not Yes",
+            "x",
+            "`my variable`",
+            "_123_",
+            "Yes",
+            "No",
+        ]
+
+        for source in valid_sources:
+            parser = Parser()
+
+            program = parser.parse(source, check_semantics=False)
+
+            # Should have no errors
+            assert len(parser.errors) == 0, f"Unexpected error for source: {source}"
+            assert len(program.statements) == 1

machine_dialect/parser/tests/test_possessive_syntax.py

@@ -0,0 +1,182 @@
+"""Tests for possessive syntax parsing."""
+
+from machine_dialect.ast.expressions import CollectionAccessExpression, ErrorExpression, Identifier
+from machine_dialect.ast.statements import IfStatement, SetStatement
+from machine_dialect.lexer.lexer import Lexer
+from machine_dialect.lexer.tokens import TokenType
+from machine_dialect.parser.parser import Parser
+
+
+class TestPossessiveSyntax:
+    """Test parsing of possessive property access syntax."""
+
+    def test_lexer_recognizes_possessive(self) -> None:
+        """Test that lexer properly tokenizes `person`'s."""
+        lexer = Lexer('`person`\'s "name"')
+
+        # First token should be the possessive token
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_APOSTROPHE_S
+        assert token.literal == "person"
+
+        # Next should be the property name as a string literal
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_TEXT
+        assert token.literal == '"name"'
+
+    def test_parse_possessive_property_access(self) -> None:
+        """Test parsing `person`'s _"name"_."""
+        source = 'Set `result` to `person`\'s _"name"_.'
+        parser = Parser()
+        program = parser.parse(source)
+
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert stmt.__class__.__name__ == "SetStatement"
+        assert isinstance(stmt, SetStatement)  # Type assertion for MyPy
+
+        # Check the value is a CollectionAccessExpression
+        expr = stmt.value
+        assert isinstance(expr, CollectionAccessExpression)
+        assert expr.access_type == "property"
+        assert isinstance(expr.collection, Identifier)
+        assert expr.collection.value == "person"
+        assert expr.accessor == "name"
+
+    def test_parse_possessive_in_set_statement(self) -> None:
+        """Test parsing Set `user` to `person`'s _"name"_."""
+        source = 'Set `user` to `person`\'s _"name"_.'
+        parser = Parser()
+        program = parser.parse(source)
+
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert stmt.__class__.__name__ == "SetStatement"
+        assert isinstance(stmt, SetStatement)  # Type assertion for MyPy
+
+        # Check the value is a CollectionAccessExpression
+        assert isinstance(stmt.value, CollectionAccessExpression)
+        assert stmt.value.access_type == "property"
+        coll = stmt.value.collection
+        assert isinstance(coll, Identifier)
+        assert coll.value == "person"
+        assert stmt.value.accessor == "name"
+
+    def test_parse_possessive_mutation(self) -> None:
+        """Test parsing Set `person`'s _"age"_ to _31_."""
+        source = 'Set `person`\'s _"age"_ to _31_.'
+        parser = Parser()
+        program = parser.parse(source)
+
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+
+        # This should parse as a collection mutation statement
+        # The parser should recognize this pattern and create appropriate AST
+        assert stmt is not None
+        # The implementation might vary - could be SetStatement with special handling
+        # or CollectionMutationStatement
+
+    def test_parse_multiple_possessive_access(self) -> None:
+        """Test parsing chained possessive access."""
+        source = 'Set `result` to `user`\'s _"email"_.'
+        parser = Parser()
+        program = parser.parse(source)
+
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert isinstance(stmt, SetStatement)  # Type assertion for MyPy
+        expr = stmt.value
+
+        assert isinstance(expr, CollectionAccessExpression)
+        assert expr.access_type == "property"
+        coll = expr.collection
+        assert isinstance(coll, Identifier)
+        assert coll.value == "user"
+        assert expr.accessor == "email"
+
+    def test_possessive_with_different_properties(self) -> None:
+        """Test possessive with various property names."""
+        test_cases = [
+            ('`config`\'s _"host"_', "config", "host"),
+            ('`user`\'s _"premium"_', "user", "premium"),
+            ('`settings`\'s _"timeout"_', "settings", "timeout"),
+            ('`person`\'s _"phone"_', "person", "phone"),
+        ]
+
+        parser = Parser()
+
+        for possessive_expr, dict_name, prop_name in test_cases:
+            source = f"Set `result` to {possessive_expr}."
+            program = parser.parse(source)
+
+            assert len(program.statements) == 1
+            stmt = program.statements[0]
+            assert isinstance(stmt, SetStatement)  # Type assertion for MyPy
+            expr = stmt.value
+
+            assert isinstance(expr, CollectionAccessExpression)
+            assert expr.access_type == "property"
+            coll = expr.collection
+            assert isinstance(coll, Identifier)
+            assert coll.value == dict_name
+            assert expr.accessor == prop_name
+
+    def test_lexer_apostrophe_without_s(self) -> None:
+        """Test that regular apostrophes don't trigger possessive."""
+        lexer = Lexer("`don't`")
+
+        # Should be parsed as a regular identifier
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "don't"
+
+    def test_lexer_apostrophe_s_without_backticks(self) -> None:
+        """Test that 's without backticks doesn't trigger possessive."""
+        lexer = Lexer('person\'s "name"')
+
+        # Should parse as regular identifier with contraction
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "person's"
+
+        token = lexer.next_token()
+        # Now expecting a string literal
+        assert token.type == TokenType.LIT_TEXT
+        assert token.literal == '"name"'
+
+    def test_possessive_missing_property_name(self) -> None:
+        """Test error when property name is missing after possessive."""
+        source = "Set `result` to `person`'s."
+        parser = Parser()
+        program = parser.parse(source)
+
+        # Should have an error in parsing
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert isinstance(stmt, SetStatement)  # Type assertion for MyPy
+        expr = stmt.value
+
+        # The value should be an error expression
+        assert isinstance(expr, ErrorExpression)
+        assert "Expected string literal" in expr.message
+
+    def test_possessive_in_conditional(self) -> None:
+        """Test possessive in if statement condition."""
+        source = 'If `user`\'s _"premium"_ then:\n> Say _"Premium user"_.'
+        parser = Parser()
+        program = parser.parse(source)
+
+        assert len(program.statements) == 1
+        stmt = program.statements[0]
+        assert stmt.__class__.__name__ == "IfStatement"
+        assert isinstance(stmt, IfStatement)  # Type assertion for MyPy
+
+        # Check the condition contains possessive access
+        condition = stmt.condition
+        assert isinstance(condition, CollectionAccessExpression)
+        assert condition.access_type == "property"
+        coll = condition.collection
+        assert isinstance(coll, Identifier)
+        assert coll.value == "user"
+        assert condition.accessor == "premium"