machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/lexer/tests/test_list_tokens.py

@@ -0,0 +1,282 @@
+"""Test list-related token recognition in the lexer."""
+
+from machine_dialect.lexer import Lexer, TokenType
+
+
+class TestListMarkers:
+    """Test recognition of list markers (dash vs minus)."""
+
+    def test_dash_at_line_start(self) -> None:
+        """Test that dash at line start is recognized as PUNCT_DASH in list context."""
+        lexer = Lexer('- _"apple"_')
+
+        # Without list context, it's OP_MINUS
+        token = lexer.next_token(in_list_context=False)
+        assert token.type == TokenType.OP_MINUS
+        assert token.literal == "-"
+
+        # Reset lexer
+        lexer = Lexer('- _"apple"_')
+
+        # With list context, it's PUNCT_DASH
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+        assert token.literal == "-"
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.LIT_TEXT
+        assert token.literal == '"apple"'
+
+    def test_dash_after_whitespace(self) -> None:
+        """Test that dash after whitespace at line start is PUNCT_DASH in list context."""
+        lexer = Lexer('  - _"apple"_')
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+        assert token.literal == "-"
+
+    def test_dash_after_block_marker(self) -> None:
+        """Test that dash after block marker (>) is PUNCT_DASH in list context."""
+        lexer = Lexer('> - _"apple"_')
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.OP_GT
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+        assert token.literal == "-"
+
+    def test_dash_in_expression(self) -> None:
+        """Test that dash in expression context is OP_MINUS."""
+        lexer = Lexer("_5_ - _3_")
+
+        # First number
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+
+        # Minus operator
+        token = lexer.next_token()
+        assert token.type == TokenType.OP_MINUS
+        assert token.literal == "-"
+
+        # Second number
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+
+    def test_multiple_dashes_at_line_start(self) -> None:
+        """Test that --- at line start is PUNCT_FRONTMATTER."""
+        lexer = Lexer("---")
+
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_FRONTMATTER
+        assert token.literal == "---"
+
+    def test_dash_on_new_line(self) -> None:
+        """Test dash recognition across multiple lines in list context."""
+        source = """Set `x` to _5_.
+- _"apple"_
+- _"banana"_"""
+
+        lexer = Lexer(source)
+
+        # First line: Set `x` to _5_ (not in list context)
+        token = lexer.next_token()
+        assert token.type == TokenType.KW_SET
+
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+
+        token = lexer.next_token()
+        assert token.type == TokenType.KW_TO
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_PERIOD
+
+        # Second line: - _"apple"_ (in list context)
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+        assert token.literal == "-"
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.LIT_TEXT
+
+        # Third line: - _"banana"_ (in list context)
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+        assert token.literal == "-"
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.LIT_TEXT
+
+
+class TestListKeywords:
+    """Test new keywords for list operations."""
+
+    def test_list_operation_keywords(self) -> None:
+        """Test recognition of list operation keywords."""
+        keywords = [
+            ("add", TokenType.KW_ADD),
+            ("remove", TokenType.KW_REMOVE),
+            ("insert", TokenType.KW_INSERT),
+            ("has", TokenType.KW_HAS),
+        ]
+
+        for literal, expected_type in keywords:
+            lexer = Lexer(literal)
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == literal
+
+    def test_list_type_keywords(self) -> None:
+        """Test recognition of list type keywords."""
+        keywords = [
+            ("Ordered List", TokenType.KW_ORDERED_LIST),
+            ("Unordered List", TokenType.KW_UNORDERED_LIST),
+            ("Named List", TokenType.KW_NAMED_LIST),
+        ]
+
+        for literal, expected_type in keywords:
+            lexer = Lexer(literal)
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == literal
+
+    def test_list_access_keywords(self) -> None:
+        """Test recognition of list access keywords."""
+        keywords = [
+            ("first", TokenType.KW_FIRST),
+            ("second", TokenType.KW_SECOND),
+            ("third", TokenType.KW_THIRD),
+            ("last", TokenType.KW_LAST),
+            ("item", TokenType.KW_ITEM),
+            ("of", TokenType.KW_OF),
+        ]
+
+        for literal, expected_type in keywords:
+            lexer = Lexer(literal)
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == literal
+
+    def test_iteration_keywords(self) -> None:
+        """Test recognition of iteration keywords."""
+        keywords = [
+            ("for", TokenType.KW_FOR),
+            ("each", TokenType.KW_EACH),
+            ("in", TokenType.KW_IN),
+        ]
+
+        for literal, expected_type in keywords:
+            lexer = Lexer(literal)
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == literal
+
+    def test_named_list_keywords(self) -> None:
+        """Test recognition of named list keywords."""
+        keywords = [
+            ("name", TokenType.KW_NAME),
+            ("names", TokenType.KW_NAME),  # Plural maps to same token type
+            ("content", TokenType.KW_CONTENT),
+            ("contents", TokenType.KW_CONTENT),  # Plural maps to same token type
+        ]
+
+        for literal, expected_type in keywords:
+            lexer = Lexer(literal)
+            token = lexer.next_token()
+            assert token.type == expected_type
+            # Literals are preserved as-is in the token
+            assert token.literal == literal
+
+
+class TestComplexListScenarios:
+    """Test complex scenarios involving list tokens."""
+
+    def test_list_with_colon(self) -> None:
+        """Test dash followed by identifier and colon (named list syntax)."""
+        lexer = Lexer("- name: `value`")
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.KW_NAME
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_COLON
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "value"
+
+    def test_numbered_list_marker(self) -> None:
+        """Test numbered list markers (1., 2., etc)."""
+        lexer = Lexer("1. first\n2. second")
+
+        # 1.
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+        assert token.literal == "1"
+
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_PERIOD
+
+        # first
+        token = lexer.next_token()
+        assert token.type == TokenType.KW_FIRST
+
+        # 2.
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+        assert token.literal == "2"
+
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_PERIOD
+
+        # second
+        token = lexer.next_token()
+        assert token.type == TokenType.KW_SECOND
+
+    def test_expression_with_negative_number(self) -> None:
+        """Test that negative numbers still work correctly."""
+        lexer = Lexer("_-42_")
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+        assert token.literal == "-42"
+
+    def test_subtraction_vs_list_marker(self) -> None:
+        """Test differentiating subtraction from list markers."""
+        # Subtraction (not in list context)
+        lexer = Lexer("`x` - `y`")
+
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+
+        token = lexer.next_token()
+        assert token.type == TokenType.OP_MINUS
+
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+
+        # List marker on new line (in list context)
+        lexer = Lexer('\n- _"apple"_')
+
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+
+    def test_list_with_negative_number(self) -> None:
+        """Test list items that include negative numbers."""
+        lexer = Lexer("- _-42_")
+
+        # First dash is list marker in list context
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.PUNCT_DASH
+
+        # The literal with negative number
+        token = lexer.next_token(in_list_context=True)
+        assert token.type == TokenType.LIT_WHOLE_NUMBER
+        assert token.literal == "-42"

machine_dialect/lexer/tests/test_stopwords.py

@@ -0,0 +1,80 @@
+import pytest
+
+from machine_dialect.lexer import Lexer
+from machine_dialect.lexer.tests.helpers import stream_and_assert_tokens
+from machine_dialect.lexer.tokens import Token, TokenType
+
+
+class TestStopwords:
+    @pytest.mark.parametrize(
+        "input_text,expected_tokens",
+        [
+            # Common stopwords
+            ("the", [Token(TokenType.MISC_STOPWORD, "the", line=1, position=1)]),
+            ("a", [Token(TokenType.MISC_STOPWORD, "a", line=1, position=1)]),
+            ("an", [Token(TokenType.MISC_STOPWORD, "an", line=1, position=1)]),
+            ("on", [Token(TokenType.MISC_STOPWORD, "on", line=1, position=1)]),
+            ("at", [Token(TokenType.MISC_STOPWORD, "at", line=1, position=1)]),
+            ("by", [Token(TokenType.MISC_STOPWORD, "by", line=1, position=1)]),
+            ("about", [Token(TokenType.MISC_STOPWORD, "about", line=1, position=1)]),
+            ("against", [Token(TokenType.MISC_STOPWORD, "against", line=1, position=1)]),
+            ("between", [Token(TokenType.MISC_STOPWORD, "between", line=1, position=1)]),
+            ("into", [Token(TokenType.MISC_STOPWORD, "into", line=1, position=1)]),
+            ("through", [Token(TokenType.MISC_STOPWORD, "through", line=1, position=1)]),
+            ("during", [Token(TokenType.MISC_STOPWORD, "during", line=1, position=1)]),
+            ("before", [Token(TokenType.MISC_STOPWORD, "before", line=1, position=1)]),
+            ("after", [Token(TokenType.MISC_STOPWORD, "after", line=1, position=1)]),
+            ("above", [Token(TokenType.MISC_STOPWORD, "above", line=1, position=1)]),
+            ("below", [Token(TokenType.MISC_STOPWORD, "below", line=1, position=1)]),
+            ("up", [Token(TokenType.MISC_STOPWORD, "up", line=1, position=1)]),
+            ("down", [Token(TokenType.MISC_STOPWORD, "down", line=1, position=1)]),
+            ("out", [Token(TokenType.MISC_STOPWORD, "out", line=1, position=1)]),
+            ("off", [Token(TokenType.MISC_STOPWORD, "off", line=1, position=1)]),
+            ("over", [Token(TokenType.MISC_STOPWORD, "over", line=1, position=1)]),
+            ("under", [Token(TokenType.MISC_STOPWORD, "under", line=1, position=1)]),
+            ("again", [Token(TokenType.MISC_STOPWORD, "again", line=1, position=1)]),
+            ("further", [Token(TokenType.MISC_STOPWORD, "further", line=1, position=1)]),
+            ("once", [Token(TokenType.MISC_STOPWORD, "once", line=1, position=1)]),
+            # Case-insensitive stopword detection
+            ("The", [Token(TokenType.MISC_STOPWORD, "The", line=1, position=1)]),
+            ("THE", [Token(TokenType.MISC_STOPWORD, "THE", line=1, position=1)]),
+            # Non-stopwords should be identifiers
+            ("variable", [Token(TokenType.MISC_IDENT, "variable", line=1, position=1)]),
+            ("myfunction", [Token(TokenType.MISC_IDENT, "myfunction", line=1, position=1)]),
+            ("data", [Token(TokenType.MISC_IDENT, "data", line=1, position=1)]),
+        ],
+    )
+    def test_stopword_detection(self, input_text: str, expected_tokens: list[Token]) -> None:
+        lexer = Lexer(input_text)
+        stream_and_assert_tokens(lexer, expected_tokens)
+
+    def test_stopwords_mixed_with_code(self) -> None:
+        input_text = "Set the `value` to 5"
+        lexer = Lexer(input_text)
+
+        # Expected tokens: "Set" (keyword), "the" (stopword), "value" (ident), "to" (keyword), "5" (int)
+        expected_tokens = [
+            Token(TokenType.KW_SET, "Set", line=1, position=1),
+            Token(TokenType.MISC_STOPWORD, "the", line=1, position=5),
+            Token(TokenType.MISC_IDENT, "value", line=1, position=10),
+            Token(TokenType.KW_TO, "to", line=1, position=17),
+            Token(TokenType.LIT_WHOLE_NUMBER, "5", line=1, position=20),
+        ]
+
+        stream_and_assert_tokens(lexer, expected_tokens)
+
+    def test_parser_ignores_stopwords(self) -> None:
+        from machine_dialect.parser import Parser
+
+        # Test that parser skips stopwords correctly
+        input_text = "Define `x` as Whole Number. Set the `x` to _5_"
+        parser = Parser()
+        program = parser.parse(input_text)
+
+        # The parser should skip "the" stopword and parse correctly
+        assert len(program.statements) == 2  # Define + Set
+        # First statement is Define, second is Set
+        assert program.statements[1].token.type == TokenType.KW_SET
+
+        # Check no parsing errors
+        assert len(parser.errors) == 0

machine_dialect/lexer/tests/test_strict_equality.py

@@ -0,0 +1,129 @@
+"""Tests for strict equality operators in the lexer.
+
+This module tests that the lexer correctly recognizes strict equality
+and strict inequality operators in their various natural language forms.
+"""
+
+import pytest
+
+from machine_dialect.lexer import Lexer
+from machine_dialect.lexer.tokens import TokenType
+
+
+class TestStrictEqualityOperators:
+    """Test the lexer's handling of strict equality operators."""
+
+    @pytest.mark.parametrize(
+        "source,expected_token_type,expected_literal",
+        [
+            # Strict equality operators
+            ("is strictly equal to", TokenType.OP_STRICT_EQ, "is strictly equal to"),
+            ("is exactly equal to", TokenType.OP_STRICT_EQ, "is exactly equal to"),
+            ("is identical to", TokenType.OP_STRICT_EQ, "is identical to"),
+            # Strict inequality operators
+            ("is not strictly equal to", TokenType.OP_STRICT_NOT_EQ, "is not strictly equal to"),
+            ("is not exactly equal to", TokenType.OP_STRICT_NOT_EQ, "is not exactly equal to"),
+            ("is not identical to", TokenType.OP_STRICT_NOT_EQ, "is not identical to"),
+            # Value equality (for comparison)
+            ("is equal to", TokenType.OP_EQ, "is equal to"),
+            ("equals", TokenType.OP_EQ, "equals"),
+            ("is the same as", TokenType.OP_EQ, "is the same as"),
+            # Value inequality (for comparison)
+            ("is not equal to", TokenType.OP_NOT_EQ, "is not equal to"),
+            ("does not equal", TokenType.OP_NOT_EQ, "does not equal"),
+            ("is different from", TokenType.OP_NOT_EQ, "is different from"),
+        ],
+    )
+    def test_strict_equality_operators(
+        self, source: str, expected_token_type: TokenType, expected_literal: str
+    ) -> None:
+        """Test that strict equality operators are correctly tokenized.
+
+        Args:
+            source: The source string containing the operator.
+            expected_token_type: The expected token type.
+            expected_literal: The expected literal value.
+        """
+        lexer = Lexer(source)
+        token = lexer.next_token()
+
+        assert token.type == expected_token_type
+        assert token.literal == expected_literal
+
+    def test_strict_equality_in_expression(self) -> None:
+        """Test strict equality operators in complete expressions."""
+        source = "if x is strictly equal to 5 then give back Yes"
+        lexer = Lexer(source)
+
+        expected_tokens = [
+            (TokenType.KW_IF, "if"),
+            (TokenType.MISC_IDENT, "x"),
+            (TokenType.OP_STRICT_EQ, "is strictly equal to"),
+            (TokenType.LIT_WHOLE_NUMBER, "5"),
+            (TokenType.KW_THEN, "then"),
+            (TokenType.KW_RETURN, "give back"),
+            (TokenType.LIT_YES, "Yes"),
+            (TokenType.MISC_EOF, ""),
+        ]
+
+        for expected_type, expected_literal in expected_tokens:
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == expected_literal
+
+    def test_strict_inequality_in_expression(self) -> None:
+        """Test strict inequality operators in complete expressions."""
+        source = "if `value` is not identical to `null` then `process`"
+        lexer = Lexer(source)
+
+        expected_tokens = [
+            (TokenType.KW_IF, "if"),
+            (TokenType.MISC_IDENT, "value"),
+            (TokenType.OP_STRICT_NOT_EQ, "is not identical to"),
+            (TokenType.MISC_IDENT, "null"),
+            (TokenType.KW_THEN, "then"),
+            (TokenType.MISC_IDENT, "process"),
+            (TokenType.MISC_EOF, ""),
+        ]
+
+        for expected_type, expected_literal in expected_tokens:
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == expected_literal
+
+    def test_mixed_equality_operators(self) -> None:
+        """Test that different equality operators are distinguished correctly."""
+        source = "`a` equals `b` and `c` is strictly equal to `d`"
+        lexer = Lexer(source)
+
+        expected_tokens = [
+            (TokenType.MISC_IDENT, "a"),
+            (TokenType.OP_EQ, "equals"),
+            (TokenType.MISC_IDENT, "b"),
+            (TokenType.KW_AND, "and"),
+            (TokenType.MISC_IDENT, "c"),
+            (TokenType.OP_STRICT_EQ, "is strictly equal to"),
+            (TokenType.MISC_IDENT, "d"),
+            (TokenType.MISC_EOF, ""),
+        ]
+
+        for expected_type, expected_literal in expected_tokens:
+            token = lexer.next_token()
+            assert token.type == expected_type
+            assert token.literal == expected_literal
+
+    def test_case_insensitive_strict_operators(self) -> None:
+        """Test that strict operators are case-insensitive."""
+        test_cases = [
+            ("Is Strictly Equal To", TokenType.OP_STRICT_EQ),
+            ("IS EXACTLY EQUAL TO", TokenType.OP_STRICT_EQ),
+            ("Is Identical To", TokenType.OP_STRICT_EQ),
+            ("IS NOT STRICTLY EQUAL TO", TokenType.OP_STRICT_NOT_EQ),
+            ("Is Not Exactly Equal To", TokenType.OP_STRICT_NOT_EQ),
+            ("is NOT identical TO", TokenType.OP_STRICT_NOT_EQ),
+        ]
+
+        for source, expected_type in test_cases:
+            lexer = Lexer(source)
+            token = lexer.next_token()
+            assert token.type == expected_type

machine_dialect/lexer/tests/test_token.py

@@ -0,0 +1,41 @@
+from machine_dialect.lexer.tokens import Token, TokenType
+
+
+class TestToken:
+    def test_token_with_line_and_position(self) -> None:
+        """Test that Token includes line and position information."""
+        token = Token(type=TokenType.MISC_IDENT, literal="test", line=1, position=1)
+
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "test"
+        assert token.line == 1
+        assert token.position == 1
+
+    def test_token_string_representation_with_location(self) -> None:
+        """Test string representation includes line and position."""
+        token = Token(type=TokenType.KW_IF, literal="if", line=5, position=10)
+
+        expected = "Type: TokenType.KW_IF, Literal: if, Line: 5, Position: 10"
+        assert str(token) == expected
+
+    def test_token_equality_with_location(self) -> None:
+        """Test that tokens are equal if all attributes match."""
+        token1 = Token(type=TokenType.LIT_WHOLE_NUMBER, literal="42", line=1, position=1)
+        token2 = Token(type=TokenType.LIT_WHOLE_NUMBER, literal="42", line=1, position=1)
+        token3 = Token(
+            type=TokenType.LIT_WHOLE_NUMBER,
+            literal="42",
+            line=2,  # Different line
+            position=1,
+        )
+
+        assert token1 == token2
+        assert token1 != token3
+
+    def test_token_creation_with_defaults(self) -> None:
+        """Test Token creation with default line and position values."""
+        # This test assumes we might want default values for backward compatibility
+        token = Token(type=TokenType.OP_PLUS, literal="+", line=1, position=1)
+
+        assert token.line == 1
+        assert token.position == 1