machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/lexer/tests/test_comments.py

@@ -0,0 +1,200 @@
+"""Tests for comment token recognition."""
+
+from machine_dialect.lexer.lexer import Lexer
+from machine_dialect.lexer.tokens import TokenType
+
+
+class TestComments:
+    """Test comment token recognition."""
+
+    def test_simple_comment(self) -> None:
+        """Test simple comment within summary tags."""
+        source = "<summary>This is a comment</summary>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[0].literal == "<summary>"
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "This is a comment"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+        assert tokens[2].literal == "</summary>"
+        assert tokens[3].type == TokenType.MISC_EOF
+
+    def test_multiline_comment(self) -> None:
+        """Test multiline comment within summary tags."""
+        source = """<summary>
+This is a comment
+that spans multiple lines
+</summary>"""
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "\nThis is a comment\nthat spans multiple lines\n"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+        assert tokens[3].type == TokenType.MISC_EOF
+
+    def test_empty_comment(self) -> None:
+        """Test empty comment within summary tags."""
+        source = "<summary></summary>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == ""
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+        assert tokens[3].type == TokenType.MISC_EOF
+
+    def test_comment_with_code_before_and_after(self) -> None:
+        """Test comment with code before and after."""
+        source = "set x to 10. <summary>This is a comment</summary> set y to 20."
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        # Check first part: set x to 10.
+        assert tokens[0].type == TokenType.KW_SET
+        assert tokens[1].type == TokenType.MISC_IDENT
+        assert tokens[1].literal == "x"
+        assert tokens[2].type == TokenType.KW_TO
+        assert tokens[3].type == TokenType.LIT_WHOLE_NUMBER
+        assert tokens[3].literal == "10"
+        assert tokens[4].type == TokenType.PUNCT_PERIOD
+
+        # Check comment part
+        assert tokens[5].type == TokenType.TAG_SUMMARY_START
+        assert tokens[6].type == TokenType.MISC_COMMENT
+        assert tokens[6].literal == "This is a comment"
+        assert tokens[7].type == TokenType.TAG_SUMMARY_END
+
+        # Check second part: set y to 20.
+        assert tokens[8].type == TokenType.KW_SET
+        assert tokens[9].type == TokenType.MISC_IDENT
+        assert tokens[9].literal == "y"
+        assert tokens[10].type == TokenType.KW_TO
+        assert tokens[11].type == TokenType.LIT_WHOLE_NUMBER
+        assert tokens[11].literal == "20"
+        assert tokens[12].type == TokenType.PUNCT_PERIOD
+        assert tokens[13].type == TokenType.MISC_EOF
+
+    def test_comment_case_insensitive_tags(self) -> None:
+        """Test that summary tags are case-insensitive."""
+        source = "<SUMMARY>This is a comment</SUMMARY>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[0].literal == "<summary>"  # Canonical form
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "This is a comment"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+        assert tokens[2].literal == "</summary>"
+
+    def test_mixed_case_tags(self) -> None:
+        """Test mixed case summary tags."""
+        source = "<SuMmArY>Mixed case comment</sUmMaRy>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "Mixed case comment"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+
+    def test_comment_with_special_characters(self) -> None:
+        """Test comment containing special characters."""
+        source = "<summary>Comment with special chars: !@#$%^&*()_+-=[]{}|;:'\",.<>?/`~</summary>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "Comment with special chars: !@#$%^&*()_+-=[]{}|;:'\",.<>?/`~"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END
+
+    def test_unclosed_summary_tag(self) -> None:
+        """Test that unclosed summary tag creates a comment up to EOF."""
+        source = "<summary>This is a comment without closing tag"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        # Should tokenize as summary tag followed by comment content up to EOF
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "This is a comment without closing tag"
+        assert tokens[2].type == TokenType.MISC_EOF
+
+    def test_nested_tags_in_comment(self) -> None:
+        """Test comment containing nested tags."""
+        source = "<summary>Comment with <tag> and </tag> inside</summary>"
+        lexer = Lexer(source)
+
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            tokens.append(token)
+            if token.type == TokenType.MISC_EOF:
+                break
+
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.TAG_SUMMARY_START
+        assert tokens[1].type == TokenType.MISC_COMMENT
+        assert tokens[1].literal == "Comment with <tag> and </tag> inside"
+        assert tokens[2].type == TokenType.TAG_SUMMARY_END

machine_dialect/lexer/tests/test_double_asterisk_keywords.py

@@ -0,0 +1,127 @@
+from machine_dialect.lexer import Lexer, TokenType
+from machine_dialect.lexer.tests.helpers import collect_all_tokens
+
+
+class TestDoubleAsteriskKeywords:
+    def test_wrapped_keyword_define(self) -> None:
+        """Test double-asterisk-wrapped keyword 'define'."""
+        source = "**define**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.KW_DEFINE
+        assert tokens[0].literal == "define"
+
+    def test_wrapped_keyword_rule(self) -> None:
+        """Test double-asterisk-wrapped keyword 'rule'."""
+        source = "**rule**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.KW_RULE
+        assert tokens[0].literal == "rule"
+
+    def test_wrapped_keyword_set(self) -> None:
+        """Test double-asterisk-wrapped keyword 'Set'."""
+        source = "**Set**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.KW_SET
+        assert tokens[0].literal == "Set"
+
+    def test_wrapped_multi_word_keyword(self) -> None:
+        """Test double-asterisk-wrapped multi-word keyword."""
+        source = "**give back**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.KW_RETURN
+        assert tokens[0].literal == "give back"
+
+    def test_unwrapped_keyword(self) -> None:
+        """Test unwrapped keyword (backward compatibility)."""
+        source = "define"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.KW_DEFINE
+        assert tokens[0].literal == "define"
+
+    def test_incomplete_wrapped_keyword(self) -> None:
+        """Test incomplete wrapped keyword (missing closing asterisks)."""
+        source = "**define"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 2
+        assert tokens[0].type == TokenType.OP_TWO_STARS
+        assert tokens[0].literal == "**"
+        assert tokens[1].type == TokenType.KW_DEFINE
+        assert tokens[1].literal == "define"
+
+    def test_non_keyword_wrapped(self) -> None:
+        """Test non-keyword wrapped in double asterisks."""
+        source = "**notakeyword**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.OP_TWO_STARS
+        assert tokens[0].literal == "**"
+        assert tokens[1].type == TokenType.MISC_IDENT
+        assert tokens[1].literal == "notakeyword"
+        assert tokens[2].type == TokenType.OP_TWO_STARS
+        assert tokens[2].literal == "**"
+
+    def test_mixed_usage_in_expression(self) -> None:
+        """Test both wrapped and unwrapped keywords in same expression."""
+        source = "**define** a rule that takes"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 5
+        assert tokens[0].type == TokenType.KW_DEFINE
+        assert tokens[0].literal == "define"
+        assert tokens[1].type == TokenType.MISC_STOPWORD  # "a"
+        assert tokens[2].type == TokenType.KW_RULE
+        assert tokens[2].literal == "rule"
+        assert tokens[3].type == TokenType.MISC_STOPWORD  # "that"
+        assert tokens[4].type == TokenType.KW_TAKE
+        assert tokens[4].literal == "takes"
+
+    def test_operator_usage(self) -> None:
+        """Test that ** operator still works correctly."""
+        source = "2 ** 3"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.LIT_WHOLE_NUMBER
+        assert tokens[0].literal == "2"
+        assert tokens[1].type == TokenType.OP_TWO_STARS
+        assert tokens[1].literal == "**"
+        assert tokens[2].type == TokenType.LIT_WHOLE_NUMBER
+        assert tokens[2].literal == "3"
+
+    def test_stopword_wrapped(self) -> None:
+        """Test stopword wrapped in double asterisks (should not be recognized as keyword)."""
+        source = "**the**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.OP_TWO_STARS
+        assert tokens[0].literal == "**"
+        assert tokens[1].type == TokenType.MISC_STOPWORD
+        assert tokens[1].literal == "the"
+        assert tokens[2].type == TokenType.OP_TWO_STARS
+        assert tokens[2].literal == "**"
+
+    def test_boolean_literal_wrapped(self) -> None:
+        """Test boolean literals wrapped in double asterisks (should not be recognized as keyword)."""
+        source = "**Yes**"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.OP_TWO_STARS
+        assert tokens[0].literal == "**"
+        assert tokens[1].type == TokenType.LIT_YES
+        assert tokens[1].literal == "Yes"
+        assert tokens[2].type == TokenType.OP_TWO_STARS
+        assert tokens[2].literal == "**"

machine_dialect/lexer/tests/test_lexer_position.py

@@ -0,0 +1,113 @@
+from machine_dialect.lexer.lexer import Lexer
+from machine_dialect.lexer.tests.helpers import collect_all_tokens
+from machine_dialect.lexer.tokens import Token, TokenType
+
+
+class TestLexerPosition:
+    def test_single_line_positions(self) -> None:
+        """Test that tokens on a single line have correct positions."""
+        source = "Set x = 42"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        expected = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_IDENT, "x", line=1, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=1, position=7),
+            Token(TokenType.LIT_WHOLE_NUMBER, "42", line=1, position=9),
+        ]
+
+        assert tokens == expected
+
+    def test_multiline_positions(self) -> None:
+        """Test that tokens across multiple lines have correct line numbers."""
+        source = """if Yes then
+    give back 42
+else
+    gives back 0"""
+
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        expected = [
+            Token(TokenType.KW_IF, "if", line=1, position=1),
+            Token(TokenType.LIT_YES, "Yes", line=1, position=4),
+            Token(TokenType.KW_THEN, "then", line=1, position=8),
+            Token(TokenType.KW_RETURN, "give back", line=2, position=5),
+            Token(TokenType.LIT_WHOLE_NUMBER, "42", line=2, position=15),
+            Token(TokenType.KW_ELSE, "else", line=3, position=1),
+            Token(TokenType.KW_RETURN, "gives back", line=4, position=5),
+            Token(TokenType.LIT_WHOLE_NUMBER, "0", line=4, position=16),
+        ]
+
+        assert tokens == expected
+
+    def test_string_literal_position(self) -> None:
+        """Test that string literals maintain correct position."""
+        source = 'Set msg = "hello world"'
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        expected = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_IDENT, "msg", line=1, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=1, position=9),
+            Token(TokenType.LIT_TEXT, '"hello world"', line=1, position=11),
+        ]
+
+        assert tokens == expected
+
+    def test_empty_lines_position(self) -> None:
+        """Test position tracking with empty lines."""
+        source = """Set x = 1
+
+Set y = 2"""
+
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        expected = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_IDENT, "x", line=1, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=1, position=7),
+            Token(TokenType.LIT_WHOLE_NUMBER, "1", line=1, position=9),
+            Token(TokenType.KW_SET, "Set", line=3, position=1),
+            Token(TokenType.MISC_IDENT, "y", line=3, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=3, position=7),
+            Token(TokenType.LIT_WHOLE_NUMBER, "2", line=3, position=9),
+        ]
+
+        assert tokens == expected
+
+    def test_tab_position(self) -> None:
+        """Test position tracking with tabs."""
+        source = "Set\tx\t=\t42"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        # Tabs count as single characters for position
+        expected = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_IDENT, "x", line=1, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=1, position=7),
+            Token(TokenType.LIT_WHOLE_NUMBER, "42", line=1, position=9),
+        ]
+
+        assert tokens == expected
+
+    def test_illegal_character_position(self) -> None:
+        """Test that illegal characters have correct position."""
+        source = "Set x = @"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        # Lexer no longer reports errors (parser will handle them)
+
+        expected = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_IDENT, "x", line=1, position=5),
+            Token(TokenType.OP_ASSIGN, "=", line=1, position=7),
+            Token(TokenType.MISC_ILLEGAL, "@", line=1, position=9),
+        ]
+
+        assert tokens == expected