python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/delphiast_lexer.py

@@ -0,0 +1,399 @@
+"""Linear Delphi/Object Pascal lexer ported from DelphiAST/SimpleParser.
+
+The implementation intentionally uses direct character dispatch rather than a
+backtracking regular-expression lexer.  Source: DelphiAST, MPL-2.0,
+https://github.com/RomanYankovsky/DelphiAST
+"""
+
+from __future__ import annotations
+
+import re
+
+from .delphiast_tokens import Token, TokenKind
+from .lark_tokens import KEYWORDS
+from .lexical_scanner import scan_brace_comment
+
+_KEYWORDS = frozenset(value.casefold() for value in KEYWORDS)
+_COMPOUND_SYMBOLS = frozenset(
+    {
+        ":=",
+        "<=",
+        ">=",
+        "<>",
+        "..",
+        "**",
+        "+=",
+        "-=",
+        "*=",
+        "/=",
+        "><",
+        "=>",
+    }
+)
+_HEX_DIGITS = frozenset("0123456789abcdefABCDEF")
+_WHITESPACE_RE = re.compile(r"\s+")
+_IDENTIFIER_PART_RE = re.compile(r"\w*")
+_INITIALIZER_SPECIAL_RE = re.compile(r"['/{}\[\]()]")
+_LARGE_INITIALIZER_CANDIDATE_RE = re.compile(r"(?<!:)=\s*\(")
+_SIMPLE_TOKEN_RE = re.compile(
+    r"\s+"
+    r"|(?:[^\W\d]|_)\w*"
+    r"|\d+(?:\.(?!\.)\d*)?(?:[eE][+-]?\d+)?"
+    r"|:=|<=|>=|<>|\.\.|\*\*|\+=|-=|\*=|/=|><|=>"
+    r"|[;,:.=+\-*/()\[\]^@<>]"
+    r"|.",
+    re.DOTALL,
+)
+_SIMPLE_SOURCE_MARKERS = ("'", "{", "//", "(*", "#", "$", "%", "&")
+
+
+class DelphiAstLexer:
+    def __init__(self, source: str, file_name: str = "") -> None:
+        self.source = source
+        self.file_name = file_name
+        self._length = len(source)
+        self._index = 0
+        self._line = 1
+        self._column = 1
+        self._previous_significant = ""
+
+    def tokenize(self, *, include_trivia: bool = True) -> list[Token]:
+        if self._is_simple_source():
+            return self._tokenize_simple()
+        tokens: list[Token] = []
+        while self._index < self._length:
+            character = self.source[self._index]
+            if character.isspace():
+                whitespace = _WHITESPACE_RE.match(self.source, self._index)
+                assert whitespace is not None
+                self._advance_to(whitespace.end())
+                continue
+            if not include_trivia:
+                trivia_end = self._trivia_end()
+                if trivia_end is not None:
+                    self._advance_to(trivia_end)
+                    continue
+            token = self._next_token()
+            if include_trivia or token.kind not in {
+                TokenKind.COMMENT,
+                TokenKind.DIRECTIVE,
+            }:
+                tokens.append(token)
+            if token.kind not in {TokenKind.COMMENT, TokenKind.DIRECTIVE}:
+                self._previous_significant = token.normalized
+            if token.end <= token.start:  # Defensive forward-progress guard.
+                self._advance_to(self._index + 1)
+        tokens.append(
+            Token(
+                TokenKind.EOF,
+                "",
+                self._line,
+                self._column,
+                self._index,
+                self._index,
+                self.file_name,
+            )
+        )
+        return tokens
+
+    def _is_simple_source(self) -> bool:
+        return not any(marker in self.source for marker in _SIMPLE_SOURCE_MARKERS) and (
+            len(self.source) < 64 * 1024
+            or _LARGE_INITIALIZER_CANDIDATE_RE.search(self.source) is None
+        )
+
+    def _tokenize_simple(self) -> list[Token]:
+        tokens: list[Token] = []
+        line = 1
+        column = 1
+        for match in _SIMPLE_TOKEN_RE.finditer(self.source):
+            value = match.group()
+            if value.isspace():
+                newline_count = value.count("\n")
+                if newline_count:
+                    line += newline_count
+                    column = len(value.rsplit("\n", 1)[-1]) + 1
+                else:
+                    column += len(value)
+                continue
+            first = value[0]
+            if _identifier_start(first):
+                kind = (
+                    TokenKind.KEYWORD
+                    if value.casefold() in _KEYWORDS
+                    else TokenKind.IDENTIFIER
+                )
+            elif first.isdigit():
+                kind = TokenKind.NUMBER
+            elif value in _COMPOUND_SYMBOLS or first in ";,:.=+-*/()[]^@<>":
+                kind = TokenKind.SYMBOL
+            else:
+                kind = TokenKind.UNKNOWN
+            tokens.append(
+                Token(
+                    kind,
+                    value,
+                    line,
+                    column,
+                    match.start(),
+                    match.end(),
+                    self.file_name,
+                )
+            )
+            column += len(value)
+        self._index = self._length
+        self._line = line
+        self._column = column
+        tokens.append(
+            Token(
+                TokenKind.EOF,
+                "",
+                line,
+                column,
+                self._length,
+                self._length,
+                self.file_name,
+            )
+        )
+        return tokens
+
+    def _trivia_end(self) -> int | None:
+        start = self._index
+        character = self.source[start]
+        following = self.source[start + 1] if start + 1 < self._length else ""
+        if character == "/" and following == "/":
+            end = self.source.find("\n", start + 2)
+            return self._length if end < 0 else end
+        if character == "{":
+            end, _ = scan_brace_comment(self.source, start)
+            return end
+        if character == "(" and following == "*":
+            close = self.source.find("*)", start + 2)
+            return self._length if close < 0 else close + 2
+        return None
+
+    def _next_token(self) -> Token:
+        start = self._index
+        line = self._line
+        column = self._column
+        character = self.source[start]
+        following = self.source[start + 1] if start + 1 < self._length else ""
+
+        if character == "/" and following == "/":
+            end = self.source.find("\n", start + 2)
+            end = self._length if end < 0 else end
+            return self._emit(TokenKind.COMMENT, start, end, line, column)
+        if character == "{":
+            end, _ = scan_brace_comment(self.source, start)
+            kind = TokenKind.DIRECTIVE if following == "$" else TokenKind.COMMENT
+            return self._emit(kind, start, end, line, column)
+        if character == "(" and following == "*":
+            end = self.source.find("*)", start + 2)
+            end = self._length if end < 0 else end + 2
+            directive = start + 2 < self._length and self.source[start + 2] == "$"
+            return self._emit(
+                TokenKind.DIRECTIVE if directive else TokenKind.COMMENT,
+                start,
+                end,
+                line,
+                column,
+            )
+        if character == "'":
+            end = self._quoted_end(start)
+            return self._emit(TokenKind.STRING, start, end, line, column)
+        if character == "(" and self._previous_significant == "=":
+            initializer_end = self._large_initializer_end(start)
+            if initializer_end is not None:
+                return self._emit(
+                    TokenKind.UNKNOWN,
+                    start,
+                    initializer_end,
+                    line,
+                    column,
+                    value="<large-initializer>",
+                )
+        if character == "#":
+            end = start + 1
+            if end < self._length and self.source[end] == "$":
+                end += 1
+                while end < self._length and self.source[end] in _HEX_DIGITS:
+                    end += 1
+            else:
+                while end < self._length and self.source[end].isdigit():
+                    end += 1
+            return self._emit(TokenKind.CHAR_CODE, start, max(end, start + 1), line, column)
+        if character == "$":
+            end = start + 1
+            while end < self._length and self.source[end] in _HEX_DIGITS:
+                end += 1
+            return self._emit(TokenKind.NUMBER, start, end, line, column)
+        if character == "%" and following in {"0", "1"}:
+            end = start + 1
+            while end < self._length and self.source[end] in {"0", "1"}:
+                end += 1
+            return self._emit(TokenKind.NUMBER, start, end, line, column)
+        if character == "&" and following.isdigit():
+            end = start + 1
+            while end < self._length and self.source[end] in "01234567":
+                end += 1
+            return self._emit(TokenKind.NUMBER, start, end, line, column)
+        if character == "&" and _identifier_start(following):
+            end = start + 2
+            identifier_tail = _IDENTIFIER_PART_RE.match(self.source, end)
+            assert identifier_tail is not None
+            end = identifier_tail.end()
+            token = self._emit(TokenKind.IDENTIFIER, start, end, line, column)
+            return Token(
+                token.kind,
+                token.value[1:],
+                token.line,
+                token.column,
+                token.start,
+                token.end,
+                token.file_name,
+            )
+        if _identifier_start(character):
+            end = start + 1
+            identifier_tail = _IDENTIFIER_PART_RE.match(self.source, end)
+            assert identifier_tail is not None
+            end = identifier_tail.end()
+            value = self.source[start:end]
+            kind = TokenKind.KEYWORD if value.casefold() in _KEYWORDS else TokenKind.IDENTIFIER
+            return self._emit(kind, start, end, line, column)
+        if character.isdigit():
+            end = self._number_end(start)
+            return self._emit(TokenKind.NUMBER, start, end, line, column)
+
+        compound = self.source[start : start + 2]
+        if compound in _COMPOUND_SYMBOLS:
+            return self._emit(TokenKind.SYMBOL, start, start + 2, line, column)
+        if character in ";,:.=+-*/()[]^@<>":
+            return self._emit(TokenKind.SYMBOL, start, start + 1, line, column)
+        return self._emit(TokenKind.UNKNOWN, start, start + 1, line, column)
+
+    def _number_end(self, start: int) -> int:
+        end = start
+        while end < self._length and self.source[end].isdigit():
+            end += 1
+        if (
+            end < self._length
+            and self.source[end] == "."
+            and self.source[end : end + 2] != ".."
+        ):
+            end += 1
+            while end < self._length and self.source[end].isdigit():
+                end += 1
+        if end < self._length and self.source[end] in {"e", "E"}:
+            exponent = end + 1
+            if exponent < self._length and self.source[exponent] in {"+", "-"}:
+                exponent += 1
+            digit_start = exponent
+            while exponent < self._length and self.source[exponent].isdigit():
+                exponent += 1
+            if exponent > digit_start:
+                end = exponent
+        return end
+
+    def _quoted_end(self, start: int) -> int:
+        for delimiter in ("'''''", "'''"):
+            opening_end = start + len(delimiter)
+            if not self.source.startswith(delimiter, start):
+                continue
+            if self.source[opening_end : opening_end + 1] not in {"\r", "\n"}:
+                continue
+            close = self.source.find(delimiter, opening_end)
+            return self._length if close < 0 else close + len(delimiter)
+        end = start + 1
+        while end < self._length:
+            if self.source[end] in {"\r", "\n"}:
+                return end
+            if self.source[end] != "'":
+                end += 1
+                continue
+            if end + 1 < self._length and self.source[end + 1] == "'":
+                end += 2
+                continue
+            return end + 1
+        return self._length
+
+    def _large_initializer_end(self, start: int) -> int | None:
+        first_special = _INITIALIZER_SPECIAL_RE.search(self.source, start + 1)
+        if first_special is not None and first_special.group() == ")":
+            end = first_special.end()
+            return end if end - start >= 64 * 1024 else None
+        stack = [")"]
+        index = start + 1
+        while index < self._length:
+            special = _INITIALIZER_SPECIAL_RE.search(self.source, index)
+            if special is None:
+                return None
+            index = special.start()
+            character = special.group()
+            following = self.source[index + 1] if index + 1 < self._length else ""
+            if character == "'":
+                index = self._quoted_end(index)
+                continue
+            if character == "/" and following == "/":
+                newline = self.source.find("\n", index + 2)
+                index = self._length if newline < 0 else newline + 1
+                continue
+            if character == "{":
+                index, _ = scan_brace_comment(self.source, index)
+                continue
+            if character == "(" and following == "*":
+                close = self.source.find("*)", index + 2)
+                index = self._length if close < 0 else close + 2
+                continue
+            if character == "(":
+                stack.append(")")
+            elif character == "[":
+                stack.append("]")
+            elif stack and character == stack[-1]:
+                stack.pop()
+                if not stack:
+                    end = index + 1
+                    return end if end - start >= 64 * 1024 else None
+            index = special.end()
+        return None
+
+    def _emit(
+        self,
+        kind: TokenKind,
+        start: int,
+        end: int,
+        line: int,
+        column: int,
+        *,
+        value: str | None = None,
+    ) -> Token:
+        token_value = self.source[start:end] if value is None else value
+        self._advance_to(end)
+        return Token(kind, token_value, line, column, start, end, self.file_name)
+
+    def _advance_to(self, end: int) -> None:
+        if end <= self._index:
+            return
+        newline_count = self.source.count("\n", self._index, end)
+        if newline_count:
+            self._line += newline_count
+            last_newline = self.source.rfind("\n", self._index, end)
+            self._column = end - last_newline
+        else:
+            self._column += end - self._index
+        self._index = end
+
+
+def large_initializer_end(source: str, start: int) -> int | None:
+    """Return the closing offset for an initializer large enough to compact."""
+    return DelphiAstLexer(source)._large_initializer_end(start)
+
+
+def _identifier_start(character: str) -> bool:
+    return bool(character) and (character == "_" or character.isalpha())
+
+
+def _identifier_part(character: str) -> bool:
+    return character == "_" or character.isalnum()
+
+
+__all__ = ["DelphiAstLexer", "large_initializer_end"]