python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/parser.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any
+
+from .comment_builder import build_comment_nodes
+from .consts import AttributeName
+from .grammar import build_grammar
+from .lark_builder import build_syntax_tree
+from .lexical_scanner import scan_brace_comment
+from .nodes import SyntaxNode
+from .parser_backend import (
+    ParserBackend,
+    ParserMode,
+    normalize_backend,
+    normalize_mode,
+    parse_delphiast_syntax_tree,
+)
+from .preprocessor import (
+    IncludeLoader,
+    PreprocessedSource,
+    Preprocessor,
+    PreprocessorOptions,
+)
+from .semantic import SymbolIndex
+from .semantic_builder import SemanticBuilder, SemanticModel
+
+StringTransform = Callable[[str], str]
+
+
+@dataclass
+class ParseResult:
+    root: SyntaxNode
+    comments: list
+    preprocessed: PreprocessedSource
+    semantic: SemanticModel | None = None
+    problems: list = field(default_factory=list)
+
+
+class DelphiParseError(Exception):
+    """Stable public parse failure independent of the selected parser backend."""
+
+    def __init__(
+        self,
+        file_name: str,
+        message: str,
+        *,
+        line: int | None = None,
+        column: int | None = None,
+    ) -> None:
+        self.file_name = file_name
+        self.line = line
+        self.column = column
+        location = file_name
+        if line is not None:
+            location += f":{line}"
+            if column is not None:
+                location += f":{column}"
+        super().__init__(f"{location}: {message}")
+
+    @classmethod
+    def from_lark(cls, file_name: str, error: BaseException) -> DelphiParseError:
+        line = getattr(error, "line", None)
+        column = getattr(error, "column", None)
+        message = str(error).splitlines()[0].strip()
+        if line is not None and column is not None:
+            location_suffix = f" at line {line}, column {column}."
+            message = message.removesuffix(location_suffix)
+        return cls(
+            file_name,
+            message or type(error).__name__,
+            line=line if isinstance(line, int) else None,
+            column=column if isinstance(column, int) else None,
+        )
+
+
+class DelphiParser:
+    def __init__(
+        self,
+        *,
+        include_paths: Iterable[str] = (),
+        defines: Iterable[str] = (),
+        include_loader: IncludeLoader | None = None,
+        preprocessor_options: PreprocessorOptions | None = None,
+        interface_only: bool = False,
+        on_handle_string: StringTransform | None = None,
+        backend: ParserBackend | str = ParserBackend.DELPHIAST,
+        mode: ParserMode | str = ParserMode.STRICT,
+        legacy_fallback_max_chars: int = 32 * 1024,
+    ) -> None:
+        self._include_paths = list(include_paths)
+        self._defines = list(defines)
+        self._include_loader = include_loader
+        self._preprocessor_options = preprocessor_options
+        self._interface_only = interface_only
+        self._on_handle_string = on_handle_string
+        self._backend = normalize_backend(backend)
+        self._mode = normalize_mode(mode)
+        self._legacy_fallback_max_chars = max(0, legacy_fallback_max_chars)
+
+    @property
+    def backend(self) -> ParserBackend:
+        return self._backend
+
+    @property
+    def mode(self) -> ParserMode:
+        return self._mode
+
+    def parse(
+        self,
+        text: str,
+        file_name: str,
+        *,
+        build_semantic: bool = False,
+        index: SymbolIndex | None = None,
+        include_loader: IncludeLoader | None = None,
+        interface_only: bool | None = None,
+        on_handle_string: StringTransform | None = None,
+        build_source_map: bool = True,
+        build_expressions: bool = True,
+        build_node_positions: bool = True,
+        build_comments: bool = True,
+    ) -> ParseResult:
+        effective_include_loader = include_loader if include_loader is not None else self._include_loader
+        effective_interface_only = self._interface_only if interface_only is None else interface_only
+        effective_string_hook = on_handle_string if on_handle_string is not None else self._on_handle_string
+
+        preprocessor = Preprocessor(
+            defines=self._defines,
+            include_paths=self._include_paths,
+            include_loader=effective_include_loader,
+            options=self._preprocessor_options,
+            track_source_map=build_source_map,
+            collect_comments=build_comments,
+        )
+        preprocessed = preprocessor.process(text, file_name)
+        parse_text = self._to_interface_only(preprocessed.text) if effective_interface_only else preprocessed.text
+        problems: list[Any]
+        if self._backend is ParserBackend.LARK:
+            tree = self._parse_lark(parse_text, file_name)
+            root = self._build_lark_syntax_tree(
+                tree,
+                file_name,
+                string_transform=effective_string_hook,
+            )
+            problems = []
+        else:
+            try:
+                output = parse_delphiast_syntax_tree(
+                    parse_text,
+                    file_name,
+                    build_expressions=build_expressions,
+                    build_node_positions=build_node_positions,
+                )
+            except RecursionError:
+                raise DelphiParseError(
+                    file_name,
+                    "parser nesting limit exceeded",
+                ) from None
+            root = output.root
+            problems = output.problems
+            if (
+                self._mode is ParserMode.STRICT
+                and len(parse_text) <= self._legacy_fallback_max_chars
+            ):
+                tree = self._parse_lark(parse_text, file_name)
+                root = self._build_lark_syntax_tree(
+                    tree,
+                    file_name,
+                    string_transform=effective_string_hook,
+                )
+                problems = []
+            if effective_string_hook is not None:
+                name = root.get_attribute(AttributeName.anName)
+                if name:
+                    root.set_attribute(AttributeName.anName, effective_string_hook(name))
+        comments = build_comment_nodes(preprocessed.comments) if build_comments else []
+        problems = [*problems, *preprocessed.problems]
+        semantic = (
+            SemanticBuilder(source_maps={file_name: preprocessed}).build(
+                root,
+                index=index,
+            )
+            if build_semantic
+            else None
+        )
+        return ParseResult(
+            root=root,
+            comments=comments,
+            preprocessed=preprocessed,
+            semantic=semantic,
+            problems=problems,
+        )
+
+    def _parse_lark(self, text: str, file_name: str) -> Any:
+        parser = _get_lark_parser()
+        try:
+            return parser.parse(text)
+        except RecursionError:
+            raise DelphiParseError(
+                file_name,
+                "parser nesting limit exceeded",
+            ) from None
+        except Exception as error:
+            if type(error).__module__.partition(".")[0] != "lark":
+                raise
+            raise DelphiParseError.from_lark(file_name, error) from None
+
+    def _build_lark_syntax_tree(
+        self,
+        tree: Any,
+        file_name: str,
+        *,
+        string_transform: StringTransform | None,
+    ) -> SyntaxNode:
+        try:
+            return build_syntax_tree(
+                tree,
+                file_name,
+                string_transform=string_transform,
+            )
+        except Exception as error:
+            nested_error = getattr(error, "orig_exc", error)
+            if isinstance(nested_error, RecursionError):
+                raise DelphiParseError(
+                    file_name,
+                    "parser nesting limit exceeded",
+                ) from None
+            raise
+
+    def _to_interface_only(self, text: str) -> str:
+        if not text:
+            return text
+        lower = text.casefold().lstrip()
+        if not lower.startswith('unit'):
+            return text
+
+        i = 0
+        n = len(text)
+        while i < n:
+            ch = text[i]
+            nxt = text[i + 1] if i + 1 < n else ''
+
+            if ch == "'" and not (ch == '{' or (ch == '(' and nxt == '*')):
+                i += 1
+                while i < n:
+                    if text[i] == "'":
+                        if i + 1 < n and text[i + 1] == "'":
+                            i += 2
+                            continue
+                        i += 1
+                        break
+                    i += 1
+                continue
+
+            if ch == '/' and nxt == '/':
+                i += 2
+                while i < n and text[i] not in {'\n', '\r'}:
+                    i += 1
+                continue
+
+            if ch == '{':
+                i, _ = scan_brace_comment(text, i)
+                continue
+
+            if ch == '(' and nxt == '*':
+                i += 2
+                while i + 1 < n and not (text[i] == '*' and text[i + 1] == ')'):
+                    i += 1
+                if i + 1 < n:
+                    i += 2
+                continue
+
+            if ch.isalpha() or ch == '_':
+                start = i
+                i += 1
+                while i < n and (text[i].isalnum() or text[i] == '_'):
+                    i += 1
+                word = text[start:i]
+                if word.casefold() == 'implementation':
+                    return text[:start] + '\nend.\n'
+                continue
+
+            i += 1
+        return text
+
+
+@lru_cache(maxsize=1)
+def _get_lark_parser():
+    try:
+        from lark import Lark
+    except ImportError as exc:  # pragma: no cover - runtime dependency
+        raise RuntimeError('lark is required to parse delphi source') from exc
+    grammar = build_grammar()
+    return Lark(
+        grammar,
+        parser='lalr',
+        lexer='contextual',
+        propagate_positions=True,
+        maybe_placeholders=False,
+    )
+
+
+def parse(
+    text: str,
+    file_name: str,
+    *,
+    include_paths: Iterable[str] = (),
+    defines: Iterable[str] = (),
+    include_loader: IncludeLoader | None = None,
+    preprocessor_options: PreprocessorOptions | None = None,
+    build_semantic: bool = False,
+    build_source_map: bool = True,
+    build_expressions: bool = True,
+    build_node_positions: bool = True,
+    build_comments: bool = True,
+    index: SymbolIndex | None = None,
+    interface_only: bool = False,
+    on_handle_string: StringTransform | None = None,
+) -> ParseResult:
+    parser = DelphiParser(
+        include_paths=include_paths,
+        defines=defines,
+        include_loader=include_loader,
+        preprocessor_options=preprocessor_options,
+        interface_only=interface_only,
+        on_handle_string=on_handle_string,
+    )
+    return parser.parse(
+        text,
+        file_name,
+        build_semantic=build_semantic,
+        build_source_map=build_source_map,
+        build_expressions=build_expressions,
+        build_node_positions=build_node_positions,
+        build_comments=build_comments,
+        index=index,
+        interface_only=interface_only,
+        on_handle_string=on_handle_string,
+    )

delphi_lsp/parser_backend.py

@@ -0,0 +1,139 @@
+"""Parser backend selection for the DelphiAST Python migration.
+
+The DelphiAST-compatible implementation is the default.  The legacy Lark
+backend remains available explicitly while compatibility coverage is expanded.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+import sys
+from enum import Enum
+
+from .delphiast_parser import DelphiAstParseOutput, DelphiAstParser
+from .nodes import SyntaxNode
+
+
+class ParserBackend(str, Enum):
+    DELPHIAST = "delphiast"
+    HYBRID = "hybrid"
+    LARK = "lark"
+
+
+class ParserImplementation(str, Enum):
+    PYTHON = "python"
+    RUST = "rust"
+
+
+class ParserMode(str, Enum):
+    TOLERANT = "tolerant"
+    STRICT = "strict"
+
+
+def normalize_backend(value: ParserBackend | str) -> ParserBackend:
+    try:
+        return value if isinstance(value, ParserBackend) else ParserBackend(value.casefold())
+    except (AttributeError, ValueError) as error:
+        raise ValueError(f"Unsupported parser backend: {value!r}.") from error
+
+
+def normalize_mode(value: ParserMode | str) -> ParserMode:
+    try:
+        return value if isinstance(value, ParserMode) else ParserMode(value.casefold())
+    except (AttributeError, ValueError) as error:
+        raise ValueError(f"Unsupported parser mode: {value!r}.") from error
+
+
+def native_backend_available() -> bool:
+    """Return ``True`` when the native Rust parser extension is installed.
+
+    The import is performed inside the function so that the main package
+    never fails on start when the native extension is absent.
+    """
+    if sys.platform != "win32" or platform.machine().casefold() not in {
+        "amd64",
+        "x86_64",
+    }:
+        return False
+    try:
+        __import__("delphi_lsp.delphi_lsp_native")
+    except ImportError:
+        return False
+    return True
+
+
+def resolve_parser_implementation() -> ParserImplementation:
+    """Decide which implementation to use based on ``DELPHI_LSP_PARSER``.
+
+    The environment variable is read on every call so that tests can
+    ``monkeypatch`` it between calls.
+    """
+    raw = os.environ.get("DELPHI_LSP_PARSER", "")
+    value = raw.casefold() if raw else ""
+
+    if value == "auto":
+        if native_backend_available():
+            return ParserImplementation.RUST
+        return ParserImplementation.PYTHON
+
+    if value == "python":
+        return ParserImplementation.PYTHON
+
+    if value == "rust":
+        if native_backend_available():
+            return ParserImplementation.RUST
+        raise RuntimeError(
+            "Rust parser backend requested but the native extension is not installed."
+        )
+
+    # Unknown / misspelled value → silently fall back to Python.
+    return ParserImplementation.PYTHON
+
+
+def parse_delphiast_syntax_tree(
+    text: str,
+    file_name: str,
+    *,
+    build_expressions: bool = True,
+    build_node_positions: bool = True,
+) -> DelphiAstParseOutput:
+    impl = resolve_parser_implementation()
+    if impl is ParserImplementation.RUST:
+        return _parse_native(text, file_name)
+    return DelphiAstParser(
+        text,
+        file_name,
+        build_expressions=build_expressions,
+        build_node_positions=build_node_positions,
+    ).parse()
+
+
+def _parse_native(text: str, file_name: str) -> DelphiAstParseOutput:
+    """Use the native Rust parser backend and reconstruct SyntaxNode objects."""
+    from .delphi_lsp_native import parse_to_serialized  # type: ignore[import-not-found]
+
+    serialized = parse_to_serialized(text, file_name)
+    from .native_bridge import reconstruct_syntax_tree
+
+    root = reconstruct_syntax_tree(serialized)
+    return DelphiAstParseOutput(root=root, problems=[])
+
+
+def build_delphiast_syntax_tree(text: str, file_name: str) -> SyntaxNode:
+    """Compatibility helper for callers that only need the root node."""
+
+    return parse_delphiast_syntax_tree(text, file_name).root
+
+
+__all__ = [
+    "ParserBackend",
+    "ParserImplementation",
+    "ParserMode",
+    "build_delphiast_syntax_tree",
+    "native_backend_available",
+    "normalize_backend",
+    "normalize_mode",
+    "parse_delphiast_syntax_tree",
+    "resolve_parser_implementation",
+]