machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/errors/tests/test_expected_token_errors.py

@@ -0,0 +1,188 @@
+"""Tests for parser expected token errors.
+
+This module tests the parser's ability to detect and report syntax errors
+when expected tokens are not found during parsing.
+"""
+
+from machine_dialect.errors.exceptions import MDNameError, MDSyntaxError
+from machine_dialect.parser import Parser
+
+
+class TestExpectedTokenErrors:
+    """Test cases for expected token error handling."""
+
+    def test_missing_identifier_after_set(self) -> None:
+        """Test error when Set statement is missing the identifier."""
+        source = "Set 42 to X"  # 42 is not a valid identifier
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have name error when non-identifier used as identifier
+        assert parser.has_errors() is True
+        # With panic recovery, we get 1 error and skip to EOF (no period)
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDNameError)
+
+        # Error should mention expected identifier and the illegal character
+        error_msg = str(parser.errors[0])
+        assert "identifier" in error_msg.lower()
+        assert "42" in error_msg  # The illegal character should be in the message
+
+    def test_missing_to_keyword(self) -> None:
+        """Test error when Set statement is missing the 'to' keyword."""
+        source = "Set `X` 42"  # Missing 'to' keyword
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have two errors: undefined variable + syntax error
+        assert parser.has_errors() is True
+        assert len(parser.errors) == 2
+        # First is NameError for undefined variable
+        assert isinstance(parser.errors[0], MDNameError)
+        # Second is the syntax error for missing 'to'
+        assert isinstance(parser.errors[1], MDSyntaxError)
+
+        # Syntax error should mention expected 'to' keyword
+        error_msg = str(parser.errors[1])
+        assert "TokenType.KW_TO" in error_msg or "to" in error_msg.lower()
+
+    def test_multiple_expected_token_errors(self) -> None:
+        """Test multiple expected token errors in one parse."""
+        # Add periods so panic recovery stops at statement boundaries
+        source = """Set 42 to X.
+Set price 3.14.
+Set to "hello".
+"""
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have multiple errors (including undefined variable errors)
+        assert parser.has_errors() is True
+        # With periods, panic recovery allows finding syntax + name errors
+        assert len(parser.errors) == 4
+
+        # Check for expected error types - mix of syntax and name errors
+        # Line 1: "Set 42" - MDNameError (42 is not valid identifier)
+        # Line 2: "Set price" - MDNameError (undefined variable) + MDSyntaxError (missing 'to')
+        # Line 3: "Set to" - MDNameError ('to' is not valid identifier)
+        name_errors = [e for e in parser.errors if isinstance(e, MDNameError)]
+        syntax_errors = [e for e in parser.errors if isinstance(e, MDSyntaxError)]
+        assert len(name_errors) == 3  # 3 name errors
+        assert len(syntax_errors) == 1  # 1 syntax error
+
+    def test_empty_identifier(self) -> None:
+        """Test error with empty backtick identifier."""
+        source = "Set `` to 42"  # Empty backticks produce illegal tokens
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have errors (from lexer producing illegal tokens)
+        # Empty backticks produce two illegal backtick characters
+        assert parser.has_errors() is True
+
+    def test_unclosed_backtick(self) -> None:
+        """Test error with unclosed backtick identifier."""
+        source = "Set `X to 42"  # Missing closing backtick
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have an error (either from lexer or parser)
+        assert parser.has_errors() is True
+
+    def test_error_location_info(self) -> None:
+        """Test that expected token errors have correct location information."""
+        source = "Set 42 to X"  # Error at position of 42
+        parser = Parser()
+
+        parser.parse(source)
+
+        # With panic recovery, we get 1 name error (42 is not valid identifier)
+        assert len(parser.errors) == 1
+        # The error should be MDNameError since 42 is not a valid identifier
+        error = parser.errors[0]
+        assert isinstance(error, MDNameError)
+
+        # Check that error has location information
+        assert hasattr(error, "_line")
+        assert hasattr(error, "_column")
+        assert error._line == 1
+        assert error._column == 5  # Points to '42'
+
+    def test_error_message_content(self) -> None:
+        """Test that error messages contain helpful information."""
+        source = "Set `X` something"  # 'to' keyword missing
+        parser = Parser()
+
+        parser.parse(source)
+
+        assert len(parser.errors) == 2  # Name error + syntax error
+        # Get the syntax error
+        error = parser.errors[1] if isinstance(parser.errors[1], MDSyntaxError) else parser.errors[0]
+        error_msg = str(error)
+
+        # Error message should contain what was expected and what was found
+        assert "expected" in error_msg.lower() or "Expected" in error_msg
+        # The parser now expects 'to' after the merged identifier 'X something'
+        # and finds EOF, so check for that
+        assert "TokenType.KW_TO" in error_msg or "to" in error_msg.lower()
+
+    def test_parser_continues_after_expected_token_error(self) -> None:
+        """Test that parser continues parsing after encountering expected token errors."""
+        source = """Set 42 to X.
+Set `price` to 3.14.
+Set `Z` 99.
+"""
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        # Should have errors for first and third statements
+        assert parser.has_errors() is True
+        # We expect 4 errors:
+        # Line 1: expected identifier (42 is invalid)
+        # Line 2: undefined variable 'price'
+        # Line 3: undefined variable 'Z' + missing 'to'
+        assert len(parser.errors) == 4
+
+        # The parser should attempt to parse all statements, even if some fail
+        # Due to error recovery, we may get fewer successfully parsed statements
+        # But we should get at least the valid one (second statement)
+        assert len(program.statements) >= 1
+
+        # Check that we have the valid statement
+        from machine_dialect.ast import SetStatement
+
+        valid_statements = [
+            s for s in program.statements if isinstance(s, SetStatement) and s.name and s.name.value == "price"
+        ]
+        assert len(valid_statements) == 1
+
+    def test_consecutive_errors(self) -> None:
+        """Test handling of consecutive expected token errors."""
+        source = "Set Set Set"  # Multiple Set keywords without proper syntax
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have multiple errors
+        assert parser.has_errors() is True
+        assert len(parser.errors) == 1
+
+    def test_eof_during_parsing(self) -> None:
+        """Test error when EOF is encountered while expecting a token."""
+        source = "Define `X` as Empty. Set `X`"  # Missing 'to' and value
+        parser = Parser()
+
+        parser.parse(source)
+
+        # Should have an error for missing 'to'
+        assert parser.has_errors() is True
+        assert len(parser.errors) == 1
+        # Find syntax error (may not be first if there are name errors)
+        syntax_errors = [e for e in parser.errors if isinstance(e, MDSyntaxError)]
+        assert len(syntax_errors) == 1

machine_dialect/errors/tests/test_name_errors.py

@@ -0,0 +1,118 @@
+"""Tests for parser error handling.
+
+This module tests the parser's ability to collect and report errors
+from the lexer, including lexical errors like illegal characters.
+"""
+
+from machine_dialect.errors.exceptions import MDSyntaxError
+from machine_dialect.parser import Parser
+
+
+class TestParserErrors:
+    """Test cases for parser error handling."""
+
+    def test_parser_collects_lexer_errors(self) -> None:
+        """Test that parser reports errors for illegal tokens during parsing."""
+        # Source with illegal character
+        source = "Define `X` as Empty. Set `X` to @."
+        # Lexer instantiation moved to Parser.parse()
+        parser = Parser()
+
+        # Errors are reported during parsing, not before
+        parser.parse(source)
+
+        # Parser should have reported the error for @ as a syntax error (illegal token)
+        assert len(parser.errors) == 1
+        assert isinstance(parser.errors[0], MDSyntaxError)
+        assert "@" in str(parser.errors[0])
+
+    def test_parser_has_errors_method(self) -> None:
+        """Test the has_errors() method."""
+        # Valid source - no errors
+        source = "Define `X` as Whole Number. Set `X` to 42."
+        parser = Parser()
+        parser.parse(source)
+
+        assert parser.has_errors() is False
+        assert len(parser.errors) == 0
+
+        # Invalid source - with illegal character
+        source_with_error = "Define `Y` as Empty. Set `Y` to §."  # § is not a valid token
+        parser_with_error = Parser()
+        parser_with_error.parse(source_with_error)
+
+        assert parser_with_error.has_errors() is True
+        assert len(parser_with_error.errors) == 1
+
+    def test_parser_collects_multiple_errors(self) -> None:
+        """Test that parser reports multiple errors through panic recovery."""
+        # Source with multiple illegal characters - periods are mandatory
+        source = (
+            "Define `A` as Empty. Define `B` as Empty. Define `C` as Empty. Set `A` to @. Set `B` to $. Set `C` to %."
+        )
+        parser = Parser()
+        parser.parse(source)
+
+        # Should have 3 errors for illegal characters (all syntax errors)
+        assert len(parser.errors) == 3
+        assert all(isinstance(error, MDSyntaxError) for error in parser.errors)
+
+        # Check that all illegal characters are in the errors
+        error_messages = [str(error) for error in parser.errors]
+        assert any("@" in msg for msg in error_messages)
+        assert any("$" in msg for msg in error_messages)
+        assert any("%" in msg for msg in error_messages)
+
+    def test_parser_continues_after_lexer_errors(self) -> None:
+        """Test that parser continues parsing despite lexer errors."""
+        # Source with an error but valid structure
+        source = "Define `X` as Empty. Define `result` as Whole Number. Set `X` to @. Set `result` to _123_."
+        # Lexer instantiation moved to Parser.parse()
+        parser = Parser()
+
+        # Parse the program
+        program = parser.parse(source)
+
+        # Should have one error for illegal character
+        assert parser.has_errors() is True
+        assert len(parser.errors) == 1
+
+        # But should still parse the valid statements
+        assert len(program.statements) == 4  # 2 defines + 2 sets
+        # Type assertions to help mypy
+        from machine_dialect.ast import DefineStatement, SetStatement
+
+        assert isinstance(program.statements[0], DefineStatement)
+        assert isinstance(program.statements[1], DefineStatement)
+        assert isinstance(program.statements[2], SetStatement)
+        assert isinstance(program.statements[3], SetStatement)
+        assert program.statements[2].name is not None
+        assert program.statements[2].name.value == "X"
+        assert program.statements[3].name is not None
+        assert program.statements[3].name.value == "result"
+
+    def test_empty_source_no_errors(self) -> None:
+        """Test that empty source produces no errors."""
+        source = ""
+        parser = Parser()
+
+        program = parser.parse(source)
+
+        assert parser.has_errors() is False
+        assert len(parser.errors) == 0
+        assert len(program.statements) == 0
+
+    def test_parser_error_details(self) -> None:
+        """Test that parser errors contain correct location information."""
+        source = "Define `X` as Empty. Set `X` to &."
+        parser = Parser()
+        parser.parse(source)
+
+        assert len(parser.errors) == 1
+        error = parser.errors[0]
+
+        # Check error has location information
+        assert hasattr(error, "_line")
+        assert hasattr(error, "_column")
+        assert error._line == 1  # First line
+        assert error._column > 0  # Should have a column position

machine_dialect/helpers/stopwords.py

@@ -0,0 +1,225 @@
+"""English stopwords module for Machine Dialect™.
+
+This module provides a curated set of English stopwords commonly used in
+natural language processing. These words are typically filtered out during
+text analysis as they carry little semantic meaning on their own.
+
+The stopwords include common articles, prepositions, pronouns, conjunctions,
+and other functional words that appear frequently in English text.
+
+Attributes:
+    ENGLISH_STOPWORDS (set[str]): A set containing common English stopwords
+        in lowercase. Includes contractions and their expanded forms.
+"""
+
+ENGLISH_STOPWORDS = {
+    "a",
+    "about",
+    "above",
+    "after",
+    "again",
+    "against",
+    "ain",
+    "all",
+    "am",
+    "an",
+    "and",
+    "any",
+    "are",
+    "aren",
+    "aren't",
+    "as",
+    "at",
+    "be",
+    "because",
+    "been",
+    "before",
+    "being",
+    "below",
+    "between",
+    "both",
+    "but",
+    "by",
+    "can",
+    "could",
+    "couldn",
+    "couldn't",
+    "d",
+    "did",
+    "didn",
+    "didn't",
+    "do",
+    "does",
+    "doesn",
+    "doesn't",
+    "doing",
+    "don",
+    "don't",
+    "down",
+    "during",
+    "each",
+    "few",
+    "for",
+    "from",
+    "further",
+    "had",
+    "hadn",
+    "hadn't",
+    "has",
+    "hasn",
+    "hasn't",
+    "have",
+    "haven",
+    "haven't",
+    "having",
+    "he",
+    "he'd",
+    "he'll",
+    "he's",
+    "here's",
+    "her",
+    "here",
+    "hers",
+    "herself",
+    "him",
+    "himself",
+    "his",
+    "how",
+    "how's",
+    "i",
+    "i'd",
+    "i'll",
+    "i'm",
+    "i've",
+    "if",
+    "in",
+    "into",
+    "is",
+    "isn",
+    "isn't",
+    "it",
+    "it'd",
+    "it'll",
+    "it's",
+    "its",
+    "itself",
+    "just",
+    "ll",
+    "m",
+    "ma",
+    "me",
+    "mightn",
+    "mightn't",
+    "more",
+    "most",
+    "mustn",
+    "mustn't",
+    "my",
+    "myself",
+    "needn",
+    "needn't",
+    "no",
+    "nor",
+    "not",
+    "now",
+    "o",
+    "of",
+    "off",
+    "on",
+    "once",
+    "only",
+    "or",
+    "other",
+    "ought",
+    "our",
+    "ours",
+    "ourselves",
+    "out",
+    "over",
+    "own",
+    "re",
+    "s",
+    "same",
+    "shan",
+    "shan't",
+    "she",
+    "she'd",
+    "she'll",
+    "she's",
+    "should",
+    "should've",
+    "shouldn",
+    "shouldn't",
+    "so",
+    "some",
+    "such",
+    "t",
+    "than",
+    "that",
+    "that'll",
+    "that's",
+    "the",
+    "their",
+    "theirs",
+    "them",
+    "themselves",
+    "then",
+    "there",
+    "there's",
+    "they'd",
+    "they'll",
+    "they're",
+    "they've",
+    "these",
+    "they",
+    "this",
+    "those",
+    "through",
+    "to",
+    "too",
+    "under",
+    "until",
+    "up",
+    "ve",
+    "very",
+    "was",
+    "wasn",
+    "wasn't",
+    "we",
+    "we'd",
+    "we'll",
+    "we're",
+    "we've",
+    "were",
+    "weren",
+    "weren't",
+    "what",
+    "what's",
+    "when",
+    "when's",
+    "where's",
+    "where",
+    "which",
+    "while",
+    "who",
+    "who's",
+    "whom",
+    "why",
+    "will",
+    "why's",
+    "with",
+    "won",
+    "won't",
+    "would",
+    "wouldn",
+    "wouldn't",
+    "you",
+    "you'd",
+    "you'll",
+    "you're",
+    "you've",
+    "your",
+    "yours",
+    "yourself",
+    "yourselves",
+}

machine_dialect/helpers/validators.py

@@ -0,0 +1,30 @@
+from rfc3986 import uri_reference
+from rfc3986.validators import Validator
+
+
+def is_valid_url(url: str, *, require_scheme: bool = True) -> bool:
+    try:
+        uri = uri_reference(url).normalize()
+
+        # Use the new Validator API
+        validator = Validator()
+
+        # Special handling for certain schemes that don't require host
+        if uri.scheme in ("mailto", "data", "file"):
+            # These schemes have their own validation rules
+            if require_scheme:
+                validator = validator.require_presence_of("scheme")
+        else:
+            # For typical URLs (http, https, ftp, etc.), require host
+            if require_scheme:
+                validator = validator.require_presence_of("scheme", "host")
+            else:
+                validator = validator.require_presence_of("host")
+
+        # Check all components
+        validator = validator.check_validity_of("scheme", "userinfo", "host", "port", "path", "query", "fragment")
+
+        validator.validate(uri)
+        return True
+    except Exception:
+        return False

machine_dialect/lexer/__init__.py

@@ -0,0 +1,9 @@
+from .lexer import Lexer
+from .tokens import Token, TokenMetaType, TokenType
+
+__all__ = [
+    "Lexer",
+    "Token",
+    "TokenMetaType",
+    "TokenType",
+]

machine_dialect/lexer/constants.py

@@ -0,0 +1,23 @@
+from machine_dialect.lexer.tokens import TokenType
+
+# Single-character tokens
+CHAR_TO_TOKEN_MAP = {
+    "+": TokenType.OP_PLUS,
+    "-": TokenType.OP_MINUS,
+    "*": TokenType.OP_STAR,
+    "/": TokenType.OP_DIVISION,
+    "^": TokenType.OP_CARET,
+    "=": TokenType.OP_ASSIGN,
+    "<": TokenType.OP_LT,
+    ">": TokenType.OP_GT,
+    "(": TokenType.DELIM_LPAREN,
+    ")": TokenType.DELIM_RPAREN,
+    "{": TokenType.DELIM_LBRACE,
+    "}": TokenType.DELIM_RBRACE,
+    ";": TokenType.PUNCT_SEMICOLON,
+    ",": TokenType.PUNCT_COMMA,
+    ".": TokenType.PUNCT_PERIOD,
+    ":": TokenType.PUNCT_COLON,
+    "#": TokenType.PUNCT_HASH,
+    "\\": TokenType.PUNCT_BACKSLASH,
+}