machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/lexer/tests/helpers.py

@@ -0,0 +1,86 @@
+from typing import TYPE_CHECKING
+
+from machine_dialect.lexer.tokens import Token, TokenType
+
+if TYPE_CHECKING:
+    from machine_dialect.lexer import Lexer
+
+
+def assert_expected_token(actual: Token, expected: Token) -> None:
+    """Assert that an actual token matches the expected token.
+
+    Args:
+        actual: The token received from the lexer.
+        expected: The expected token.
+    """
+    assert actual.type == expected.type, f"Token type mismatch: got {actual.type}, expected {expected.type}"
+    assert actual.literal == expected.literal, (
+        f"Token literal mismatch: got '{actual.literal}', expected '{expected.literal}'"
+    )
+    assert actual.line == expected.line, f"Token line mismatch: got {actual.line}, expected {expected.line}"
+    assert actual.position == expected.position, (
+        f"Token position mismatch: got {actual.position}, expected {expected.position}"
+    )
+
+
+def assert_eof(token: Token) -> None:
+    """Assert that a token is an EOF token.
+
+    Args:
+        token: The token to check.
+    """
+    assert token.type == TokenType.MISC_EOF, f"Expected EOF token, got {token.type}"
+
+
+def stream_and_assert_tokens(lexer: "Lexer", expected_tokens: list[Token]) -> None:
+    """Stream tokens from lexer and assert they match expected tokens.
+
+    This helper function:
+    1. Streams tokens one by one from the lexer
+    2. Asserts each token matches the expected token
+    3. Verifies the count matches
+    4. Asserts EOF is reached after all expected tokens
+
+    Args:
+        lexer: The lexer instance to stream tokens from.
+        expected_tokens: List of expected tokens (not including EOF).
+    """
+    actual_count = 0
+
+    for i, expected in enumerate(expected_tokens):
+        actual = lexer.next_token()
+        assert actual.type != TokenType.MISC_EOF, f"Got EOF at token {i}, expected {len(expected_tokens)} tokens"
+        assert_expected_token(actual, expected)
+        actual_count += 1
+
+    # Verify we get EOF next
+    eof_token = lexer.next_token()
+    assert_eof(eof_token)
+
+    # Verify count
+    assert actual_count == len(expected_tokens), f"Expected {len(expected_tokens)} tokens, got {actual_count}"
+
+
+def token(token_type: TokenType, literal: str, line: int = 1, position: int = 0) -> Token:
+    """Helper function to create tokens with default line and position values for tests."""
+    return Token(token_type, literal, line, position)
+
+
+def collect_all_tokens(lexer: "Lexer") -> list[Token]:
+    """Collect all tokens from lexer until EOF (excluding EOF token).
+
+    This is useful for tests that need to examine all tokens but don't
+    want to repeatedly write the streaming loop.
+
+    Args:
+        lexer: The lexer instance to stream tokens from.
+
+    Returns:
+        List of all tokens excluding the EOF token.
+    """
+    tokens = []
+    token = lexer.next_token()
+    while token.type != TokenType.MISC_EOF:
+        tokens.append(token)
+        token = lexer.next_token()
+    return tokens

machine_dialect/lexer/tests/test_apostrophe_identifiers.py

@@ -0,0 +1,122 @@
+"""Tests for apostrophe support in identifiers."""
+
+from machine_dialect.lexer.lexer import Lexer
+from machine_dialect.lexer.tokens import TokenType, is_valid_identifier
+
+
+class TestApostropheIdentifiers:
+    """Test apostrophe support in identifiers."""
+
+    def test_valid_apostrophe_identifiers(self) -> None:
+        """Test that valid apostrophe patterns are accepted."""
+        valid_identifiers = [
+            "don't",
+            "can't",
+            "won't",
+            "I'm",
+            "it's",
+            "person's",
+            "person's name",
+            "I don't like",
+            "can't wait",
+            "won't stop",
+            "it's working",
+            "user's data",
+            "server's response",
+        ]
+
+        for identifier in valid_identifiers:
+            assert is_valid_identifier(identifier), f"'{identifier}' should be valid"
+
+    def test_invalid_apostrophe_identifiers(self) -> None:
+        """Test that invalid apostrophe patterns are rejected."""
+        invalid_identifiers = [
+            "'hello",  # Starts with apostrophe
+            "hello'",  # Ends with apostrophe
+            "'",  # Just apostrophe
+            "hello 'world",  # Word starts with apostrophe
+            "hello world'",  # Word ends with apostrophe
+            "'hello world",  # Starts with apostrophe
+            "hello world'",  # Ends with apostrophe
+        ]
+
+        for identifier in invalid_identifiers:
+            assert not is_valid_identifier(identifier), f"'{identifier}' should be invalid"
+
+    def test_backtick_identifiers_with_apostrophes(self) -> None:
+        """Test that backtick-wrapped identifiers with apostrophes work correctly."""
+        test_cases = [
+            ("Set `don't` to _true_.", "don't", TokenType.MISC_IDENT),
+            ("Set `I'm happy` to _yes_.", "I'm happy", TokenType.MISC_IDENT),
+            ('Set `person\'s name` to _"John"_.', "person's name", TokenType.MISC_IDENT),
+            ("Set `I don't like` to _no_.", "I don't like", TokenType.MISC_IDENT),
+            ("Set `can't wait` to _true_.", "can't wait", TokenType.MISC_IDENT),
+        ]
+
+        for source, expected_ident, expected_type in test_cases:
+            lexer = Lexer(source)
+
+            # Skip "Set"
+            token = lexer.next_token()
+            assert token.type == TokenType.KW_SET
+
+            # Get the identifier
+            token = lexer.next_token()
+            assert token.type == expected_type, f"Expected {expected_type}, got {token.type}"
+            assert token.literal == expected_ident, f"Expected '{expected_ident}', got '{token.literal}'"
+
+    def test_invalid_apostrophe_patterns_in_backticks(self) -> None:
+        """Test that invalid apostrophe patterns still fail in backticks."""
+        invalid_sources = [
+            "`'hello`",  # Starts with apostrophe
+            "`hello'`",  # Ends with apostrophe
+            "`hello 'world`",  # Word starts with apostrophe
+            "`hello world'`",  # Word ends with apostrophe
+        ]
+
+        for source in invalid_sources:
+            lexer = Lexer(source)
+            token = lexer.next_token()
+            # Should return MISC_ILLEGAL since the identifier is invalid
+            assert token.type == TokenType.MISC_ILLEGAL, f"Source '{source}' should produce MISC_ILLEGAL"
+
+    def test_apostrophe_s_possessive(self) -> None:
+        """Test that possessive 's pattern works correctly."""
+        # Test with our new syntax: possessive followed by string literal
+        source = '`person`\'s "name"'
+        lexer = Lexer(source)
+
+        # Should get the possessive token
+        token = lexer.next_token()
+        assert token.type == TokenType.PUNCT_APOSTROPHE_S
+        assert token.literal == "person"
+
+        # Next should be a string literal
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_TEXT
+        assert token.literal == '"name"'
+
+        # Also test with a non-keyword identifier in backticks
+        source2 = "`person`'s `property`"
+        lexer2 = Lexer(source2)
+
+        token = lexer2.next_token()
+        assert token.type == TokenType.PUNCT_APOSTROPHE_S
+        assert token.literal == "person"
+
+        token = lexer2.next_token()
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "property"
+
+    def test_mixed_valid_characters(self) -> None:
+        """Test identifiers with mixed valid characters including apostrophes."""
+        valid_identifiers = [
+            "user_1's_data",
+            "don't-stop",
+            "can't_wait_2",
+            "person's-item",
+            "it's_working-now",
+        ]
+
+        for identifier in valid_identifiers:
+            assert is_valid_identifier(identifier), f"'{identifier}' should be valid"

machine_dialect/lexer/tests/test_backtick_identifiers.py

@@ -0,0 +1,140 @@
+from machine_dialect.lexer import Lexer, TokenType
+from machine_dialect.lexer.tests.helpers import collect_all_tokens
+
+
+class TestBacktickIdentifiers:
+    def test_backtick_wrapped_identifier(self) -> None:
+        """Test backtick-wrapped identifier."""
+        source = "`identifier`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "identifier"
+
+    def test_backtick_wrapped_keyword(self) -> None:
+        """Test that backtick-wrapped keywords become identifiers."""
+        source = "`define`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "define"
+
+    def test_backtick_wrapped_number(self) -> None:
+        """Test that backtick-wrapped numbers are not valid identifiers."""
+        source = "`42`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        # `42` is not a valid identifier, so backtick is illegal, then 42, then backtick
+        assert len(tokens) == 3
+        assert tokens[0].type == TokenType.MISC_ILLEGAL
+        assert tokens[0].literal == "`"
+        assert tokens[1].type == TokenType.LIT_WHOLE_NUMBER
+        assert tokens[1].literal == "42"
+        assert tokens[2].type == TokenType.MISC_ILLEGAL
+        assert tokens[2].literal == "`"
+
+    def test_empty_backticks(self) -> None:
+        """Test that empty backticks are treated as illegal."""
+        source = "``"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        # Empty content is not a valid identifier, so both backticks are illegal
+        assert len(tokens) == 2
+        assert tokens[0].type == TokenType.MISC_ILLEGAL
+        assert tokens[0].literal == "`"
+        assert tokens[1].type == TokenType.MISC_ILLEGAL
+        assert tokens[1].literal == "`"
+
+    def test_unwrapped_identifier(self) -> None:
+        """Test unwrapped identifier (backward compatibility)."""
+        source = "identifier"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "identifier"
+
+    def test_mixed_usage_in_expression(self) -> None:
+        """Test both wrapped and unwrapped identifiers in same expression."""
+        source = "Set `x` to y"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+        assert len(tokens) == 4
+        assert tokens[0].type == TokenType.KW_SET
+        assert tokens[0].literal == "Set"
+        assert tokens[1].type == TokenType.MISC_IDENT
+        assert tokens[1].literal == "x"
+        assert tokens[2].type == TokenType.KW_TO
+        assert tokens[2].literal == "to"
+        assert tokens[3].type == TokenType.MISC_IDENT
+        assert tokens[3].literal == "y"
+
+    def test_unclosed_backtick(self) -> None:
+        """Test unclosed backtick without closing backtick is treated as illegal."""
+        source = "`unclosed"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        # Without closing backtick, the opening backtick is illegal
+        assert len(tokens) == 2
+        assert tokens[0].type == TokenType.MISC_ILLEGAL
+        assert tokens[0].literal == "`"
+        assert tokens[1].type == TokenType.MISC_IDENT
+        assert tokens[1].literal == "unclosed"
+
+    def test_backtick_with_spaces(self) -> None:
+        """Test backtick with spaces inside."""
+        source = "`with spaces`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "with spaces"
+
+    def test_triple_backticks_still_work(self) -> None:
+        """Test that triple backticks still work as string literals."""
+        source = "```code block```"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.LIT_TRIPLE_BACKTICK
+        assert tokens[0].literal == "code block"
+
+    def test_backtick_with_hyphens(self) -> None:
+        """Test backtick with hyphens inside."""
+        source = "`my-identifier`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "my-identifier"
+
+    def test_backtick_with_spaces_and_hyphens(self) -> None:
+        """Test backtick with both spaces and hyphens."""
+        source = "`my-complex identifier`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "my-complex identifier"
+
+    def test_backtick_wrapped_type_keyword(self) -> None:
+        """Test that type keywords like 'number' become identifiers in backticks."""
+        source = "`number`"
+        lexer = Lexer(source)
+        tokens = collect_all_tokens(lexer)
+
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.MISC_IDENT
+        assert tokens[0].literal == "number"

machine_dialect/lexer/tests/test_boolean_literals.py

@@ -0,0 +1,108 @@
+from machine_dialect.lexer import Lexer, Token, TokenMetaType, TokenType
+from machine_dialect.lexer.tests.helpers import assert_eof
+
+
+def is_literal_token(token: Token) -> bool:
+    return token.type.meta_type == TokenMetaType.LIT
+
+
+class TestBooleanLiterals:
+    def test_wrapped_yes(self) -> None:
+        """Test underscore-wrapped Yes literal."""
+        source = "_Yes_"
+        lexer = Lexer(source)
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_YES
+        assert token.literal == "Yes"  # Canonical form without underscores
+        assert is_literal_token(token)
+
+        assert_eof(lexer.next_token())
+
+    def test_wrapped_no(self) -> None:
+        """Test underscore-wrapped No literal."""
+        source = "_No_"
+        lexer = Lexer(source)
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_NO
+        assert token.literal == "No"  # Canonical form without underscores
+        assert is_literal_token(token)
+
+        assert_eof(lexer.next_token())
+
+    def test_unwrapped_yes(self) -> None:
+        """Test unwrapped Yes literal."""
+        source = "Yes"
+        lexer = Lexer(source)
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_YES
+        assert token.literal == "Yes"
+        assert is_literal_token(token)
+
+        assert_eof(lexer.next_token())
+
+    def test_unwrapped_no(self) -> None:
+        """Test unwrapped No literal."""
+        source = "No"
+        lexer = Lexer(source)
+
+        token = lexer.next_token()
+        assert token.type == TokenType.LIT_NO
+        assert token.literal == "No"
+        assert is_literal_token(token)
+
+        assert_eof(lexer.next_token())
+
+    def test_boolean_in_expression(self) -> None:
+        """Test boolean literals in expressions."""
+        source = "if x > 0 then give back _Yes_ else give back No"
+        lexer = Lexer(source)
+
+        # Collect all tokens
+        tokens = []
+        while True:
+            token = lexer.next_token()
+            if token.type == TokenType.MISC_EOF:
+                break
+            tokens.append(token)
+
+        # Find the boolean tokens
+        boolean_tokens = [t for t in tokens if t.type in (TokenType.LIT_YES, TokenType.LIT_NO)]
+        assert len(boolean_tokens) == 2
+
+        # Both booleans are stored in canonical form
+        assert boolean_tokens[0].type == TokenType.LIT_YES
+        assert boolean_tokens[0].literal == "Yes"
+
+        assert boolean_tokens[1].type == TokenType.LIT_NO
+        assert boolean_tokens[1].literal == "No"
+
+    def test_lowercase_yes_no(self) -> None:
+        """Test that lowercase yes/no are recognized as boolean literals."""
+        source = "yes no"
+        lexer = Lexer(source)
+
+        # Lowercase yes/no are recognized as boolean literals
+        token1 = lexer.next_token()
+        assert token1.type == TokenType.LIT_YES
+        assert token1.literal == "Yes"  # Canonical form
+
+        token2 = lexer.next_token()
+        assert token2.type == TokenType.LIT_NO
+        assert token2.literal == "No"  # Canonical form
+
+        assert_eof(lexer.next_token())
+
+    def test_incomplete_wrapped_boolean(self) -> None:
+        """Test incomplete wrapped boolean falls back to identifier."""
+        source = "_Yes"  # Missing closing underscore
+        lexer = Lexer(source)
+
+        token = lexer.next_token()
+        assert token.type == TokenType.MISC_IDENT
+        assert token.literal == "_Yes"
+        assert not is_literal_token(token)
+
+        assert_eof(lexer.next_token())

machine_dialect/lexer/tests/test_case_insensitive_keywords.py

@@ -0,0 +1,188 @@
+from machine_dialect.lexer.lexer import Lexer
+from machine_dialect.lexer.tests.helpers import collect_all_tokens
+from machine_dialect.lexer.tokens import TokenType, keywords_mapping
+
+
+class TestCaseInsensitiveKeywords:
+    """Test case-insensitive keyword matching while preserving canonical form."""
+
+    def test_all_keywords_case_variations(self) -> None:
+        """Test that all keywords in keywords_mapping are case-insensitive."""
+        for canonical_form, token_type in keywords_mapping.items():
+            # Test different case variations
+            test_cases = [
+                canonical_form,  # Original form
+                canonical_form.lower(),  # Lowercase
+                canonical_form.upper(),  # Uppercase
+                canonical_form.title(),  # Title case
+            ]
+
+            # Add a mixed case variant
+            if len(canonical_form) > 2:
+                # Create a mixed case like "fLoAt" or "tRuE"
+                mixed = ""
+                for i, char in enumerate(canonical_form):
+                    if char == " ":
+                        mixed += " "
+                    elif i % 2 == 0:
+                        mixed += char.lower()
+                    else:
+                        mixed += char.upper()
+                test_cases.append(mixed)
+
+            for variant in test_cases:
+                # Skip if variant is the same as one we already tested
+                if test_cases.count(variant) > 1:
+                    continue
+
+                lexer = Lexer(variant)
+                tokens = collect_all_tokens(lexer)
+                assert len(tokens) == 1, f"Expected 1 token for '{variant}', got {len(tokens)}"
+                assert tokens[0].type == token_type, f"Expected {token_type} for '{variant}', got {tokens[0].type}"
+                # Special case for boolean literals which canonicalize to Yes/No
+                if token_type in (TokenType.LIT_YES, TokenType.LIT_NO):
+                    expected = "Yes" if token_type == TokenType.LIT_YES else "No"
+                    assert tokens[0].literal == expected, (
+                        f"Expected literal '{expected}' for '{variant}', got '{tokens[0].literal}'"
+                    )
+                else:
+                    assert tokens[0].literal == canonical_form, (
+                        f"Expected literal '{canonical_form}' for '{variant}', got '{tokens[0].literal}'"
+                    )
+
+    def test_double_asterisk_keywords_all_cases(self) -> None:
+        """Test that all keywords work with double-asterisk wrapping in different cases."""
+        # Test a subset of keywords with double asterisks
+        test_keywords = ["define", "Float", "Integer", "Boolean", "rule", "Set", "Tell"]
+
+        for keyword in test_keywords:
+            if keyword not in keywords_mapping:
+                continue
+
+            token_type = keywords_mapping[keyword]
+            test_cases = [
+                f"**{keyword}**",
+                f"**{keyword.lower()}**",
+                f"**{keyword.upper()}**",
+            ]
+
+            for source in test_cases:
+                lexer = Lexer(source)
+                tokens = collect_all_tokens(lexer)
+                assert len(tokens) == 1
+                assert tokens[0].type == token_type
+                # Special handling for boolean literals
+                if token_type in (TokenType.LIT_YES, TokenType.LIT_NO):
+                    expected = "Yes" if token_type == TokenType.LIT_YES else "No"
+                    assert tokens[0].literal == expected
+                else:
+                    assert tokens[0].literal == keyword
+
+    def test_backtick_keywords_all_cases(self) -> None:
+        """Test that keywords in backticks become identifiers (case-insensitive)."""
+        # Test a subset of keywords with backticks
+        test_keywords = ["Float", "Integer", "True", "False", "define", "rule"]
+
+        for keyword in test_keywords:
+            if keyword not in keywords_mapping:
+                continue
+
+            test_cases = [
+                f"`{keyword}`",
+                f"`{keyword.lower()}`",
+                f"`{keyword.upper()}`",
+            ]
+
+            for source in test_cases:
+                lexer = Lexer(source)
+                tokens = collect_all_tokens(lexer)
+                assert len(tokens) == 1
+                # Backticks force content to be identifiers
+                assert tokens[0].type == TokenType.MISC_IDENT
+                # The literal should be the actual text within backticks
+                assert tokens[0].literal.lower() == keyword.lower()
+
+    def test_underscore_wrapped_booleans_all_cases(self) -> None:
+        """Test underscore-wrapped boolean literals in different cases."""
+        # Test both True/False and Yes/No inputs
+        test_inputs = [
+            ("True", TokenType.LIT_YES, "Yes"),
+            ("False", TokenType.LIT_NO, "No"),
+            ("Yes", TokenType.LIT_YES, "Yes"),
+            ("No", TokenType.LIT_NO, "No"),
+        ]
+
+        for input_form, token_type, expected_literal in test_inputs:
+            test_cases = [
+                f"_{input_form}_",
+                f"_{input_form.lower()}_",
+                f"_{input_form.upper()}_",
+            ]
+
+            for source in test_cases:
+                lexer = Lexer(source)
+                tokens = collect_all_tokens(lexer)
+                assert len(tokens) == 1
+                assert tokens[0].type == token_type
+                assert tokens[0].literal == expected_literal
+
+    def test_identifiers_preserve_case(self) -> None:
+        """Test that non-keyword identifiers preserve their case."""
+        # These should NOT match any keywords
+        test_cases = [
+            ("myVariable", "myVariable"),
+            ("MyVariable", "MyVariable"),
+            ("MYVARIABLE", "MYVARIABLE"),
+            ("userName", "userName"),
+            ("floatValue", "floatValue"),  # Contains "float" but not a keyword
+            ("integerCount", "integerCount"),  # Contains "integer" but not a keyword
+        ]
+
+        for source, expected_literal in test_cases:
+            lexer = Lexer(source)
+            tokens = collect_all_tokens(lexer)
+            assert len(tokens) == 1
+            assert tokens[0].type == TokenType.MISC_IDENT
+            assert tokens[0].literal == expected_literal
+
+    def test_complex_expression_mixed_case(self) -> None:
+        """Test complex expressions with mixed case keywords."""
+        test_cases = [
+            ("SET x AS integer", ["Set", "x", "as", "integer"]),
+            ("set X as INTEGER", ["Set", "X", "as", "INTEGER"]),
+            ("define RULE myFunc", ["define", "rule", "myFunc"]),
+            ("DEFINE rule MyFunc", ["define", "rule", "MyFunc"]),
+            ("if YES then GIVE BACK no", ["if", "Yes", "then", "give back", "No"]),
+        ]
+
+        for source, expected_literals in test_cases:
+            lexer = Lexer(source)
+            tokens = collect_all_tokens(lexer)
+            assert len(tokens) == len(expected_literals)
+
+            for token, expected_literal in zip(tokens, expected_literals, strict=False):
+                assert token.literal == expected_literal
+
+    def test_multi_word_keywords_preserve_spacing(self) -> None:
+        """Test that multi-word keywords preserve internal spacing but are case-insensitive."""
+        multi_word_keywords = [
+            ("give back", TokenType.KW_RETURN),
+            ("gives back", TokenType.KW_RETURN),
+        ]
+
+        for canonical_form, token_type in multi_word_keywords:
+            # Test with different cases but same spacing
+            test_cases = [
+                canonical_form,
+                canonical_form.upper(),
+                canonical_form.title(),
+                # Mixed case for each word
+                " ".join(word.upper() if i % 2 == 0 else word.lower() for i, word in enumerate(canonical_form.split())),
+            ]
+
+            for variant in test_cases:
+                lexer = Lexer(variant)
+                tokens = collect_all_tokens(lexer)
+                assert len(tokens) == 1
+                assert tokens[0].type == token_type
+                assert tokens[0].literal == canonical_form