python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/native_bridge.py

@@ -0,0 +1,300 @@
+"""Reconstruct SyntaxNode trees from the Rust parser's serialized output.
+
+The Rust parser (``parse_to_serialized``) produces a deterministic text
+representation identical to Python's ``serialize_syntax_tree``.  This module
+provides the inverse: parsing that text back into real
+``delphi_lsp.nodes.SyntaxNode`` objects so the rest of the LSP can operate on
+native AST nodes instead of strings.
+"""
+
+from __future__ import annotations
+
+from typing import cast
+
+from .consts import ATTRIBUTE_NAME_STRINGS, AttributeName, SyntaxNodeType
+from .nodes import CommentNode, SyntaxNode, ValuedSyntaxNode
+
+
+def _unescape_value(value: str) -> str:
+    """Unescape a serialized node value (reverse of ``_escape_value``)."""
+    result: list[str] = []
+    i = 0
+    while i < len(value):
+        if value[i] == "\\" and i + 1 < len(value):
+            nxt = value[i + 1]
+            if nxt == "\\":
+                result.append("\\")
+                i += 2
+            elif nxt == "n":
+                result.append("\n")
+                i += 2
+            elif nxt == "r":
+                result.append("\r")
+                i += 2
+            elif nxt == "t":
+                result.append("\t")
+                i += 2
+            else:
+                result.append(value[i])
+                i += 1
+        else:
+            result.append(value[i])
+            i += 1
+    return "".join(result)
+
+
+def _syntax_node_type_from_name(name: str) -> SyntaxNodeType:
+    """Map a name string like ``'ntUnit'`` back to the ``SyntaxNodeType`` enum."""
+    try:
+        return SyntaxNodeType[name]
+    except KeyError:
+        if name.startswith("ntUnknown(") and name.endswith(")"):
+            try:
+                val = int(name[len("ntUnknown("):-1])
+                return SyntaxNodeType(val)
+            except (ValueError, OverflowError):
+                return SyntaxNodeType.ntUnknown
+        return SyntaxNodeType.ntUnknown
+
+
+def _parse_kv_pairs(token: str) -> list[tuple[str, str]]:
+    """Parse ``key=value`` pairs from a token string.
+
+    Values may contain spaces but are delimited by the next whitespace
+    followed by a ``key=`` pattern.
+    """
+    pairs: list[tuple[str, str]] = []
+    i = 0
+    length = len(token)
+    while i < length:
+        while i < length and token[i].isspace():
+            i += 1
+        if i >= length:
+            break
+
+        eq_pos = token.find("=", i)
+        if eq_pos == -1:
+            break
+        key = token[i:eq_pos]
+        i = eq_pos + 1
+
+        while i < length and token[i].isspace():
+            i += 1
+
+        if i >= length:
+            pairs.append((key, ""))
+            break
+
+        value_start = i
+        while i < length:
+            if token[i].isspace():
+                j = i
+                while j < length and token[j].isspace():
+                    j += 1
+                if j < length and token[j].isalpha():
+                    k = j
+                    while k < length and (token[k].isalnum() or token[k] == "_"):
+                        k += 1
+                    if k < length and token[k] == "=":
+                        break
+                i += 1
+            else:
+                i += 1
+
+        value = token[value_start:i]
+        pairs.append((key, value))
+
+    return pairs
+
+
+def _parse_node_line(line: str) -> tuple[str, list[tuple[str, str]]]:
+    """Parse a node line into (type_name, key_value_pairs).
+
+    The first whitespace-delimited token that does not contain ``=``
+    is the type name.  Everything after that is parsed as key=value pairs.
+    """
+    stripped = line.lstrip(" ")
+    # Find the first whitespace-delimited token
+    first_space = stripped.find(" ")
+    if first_space == -1:
+        # Only the type name, no key-value pairs
+        return stripped, []
+
+    first_token = stripped[:first_space]
+    if "=" not in first_token:
+        # First token is the type name
+        rest = stripped[first_space:]
+        kv_pairs = _parse_kv_pairs(rest)
+        return first_token, kv_pairs
+
+    # First token contains '=' — fall back to treating everything as kv pairs
+    kv_pairs = _parse_kv_pairs(stripped)
+    return kv_pairs[0][0] if kv_pairs else "", kv_pairs[1:] if len(kv_pairs) > 1 else []
+
+
+def _attribute_name_to_index(name: str) -> int:
+    """Map an attribute name string to its AttributeName int value."""
+    for i, s in enumerate(ATTRIBUTE_NAME_STRINGS):
+        if s == name:
+            return i
+    return len(ATTRIBUTE_NAME_STRINGS)
+
+
+def _looks_like_subclass_field(line: str) -> bool:
+    """Check if a line looks like a subclass field (value= or text=)."""
+    return line.startswith(("value=", "text="))
+
+
+def _apply_subclass_field(node: SyntaxNode, line: str) -> SyntaxNode:
+    """Apply a subclass field (value= or text=) to a node.
+
+    If the node is a plain SyntaxNode and needs a subclass attribute,
+    convert it to the appropriate subclass. Returns the (possibly new) node.
+    """
+    if line.startswith("value="):
+        # Convert to ValuedSyntaxNode if needed
+        if isinstance(node, SyntaxNode) and not isinstance(node, ValuedSyntaxNode):
+            vnode: ValuedSyntaxNode = ValuedSyntaxNode(node.typ)
+            vnode._col = node._col
+            vnode._line = node._line
+            vnode._file_name = node._file_name
+            vnode._attributes = list(node._attributes)
+            vnode._child_nodes = list(node._child_nodes)
+            vnode._source_positions = node._source_positions
+            vnode._token_end_line = node._token_end_line
+            vnode._token_end_col = node._token_end_col
+        else:
+            vnode = cast(ValuedSyntaxNode, node)
+        vnode.value = _unescape_value(line[len("value="):])
+        return vnode
+    elif line.startswith("text="):
+        # Convert to CommentNode if needed
+        if isinstance(node, SyntaxNode) and not isinstance(node, CommentNode):
+            cnode: CommentNode = CommentNode(node.typ)
+            cnode._col = node._col
+            cnode._line = node._line
+            cnode._file_name = node._file_name
+            cnode._attributes = list(node._attributes)
+            cnode._child_nodes = list(node._child_nodes)
+            cnode._source_positions = node._source_positions
+            cnode._token_end_line = node._token_end_line
+            cnode._token_end_col = node._token_end_col
+        else:
+            cnode = cast(CommentNode, node)
+        cnode.text = _unescape_value(line[len("text="):])
+        return cnode
+    return node
+
+
+def _parse_single_node(line: str, parent_file_name: str) -> tuple[SyntaxNode, int]:
+    """Parse a single node line and return (node, next_line_index)."""
+    type_name, kv_pairs = _parse_node_line(line.lstrip(" "))
+    if not type_name:
+        raise ValueError(f"Cannot parse node line: {line!r}")
+
+    typ = _syntax_node_type_from_name(type_name)
+
+    node = SyntaxNode(typ)
+
+    for key, value in kv_pairs:
+        if key == "line":
+            node.line = int(value)
+        elif key == "col":
+            node.col = int(value)
+        elif key == "end_line":
+            node.token_end_line = int(value)
+        elif key == "end_col":
+            node.token_end_col = int(value)
+        elif key == "file_name":
+            node.file_name = _unescape_value(value)
+        elif key == "source_positions":
+            positions: list[tuple[int, int]] = []
+            for pos in value.split(","):
+                if ":" in pos:
+                    l, c = pos.split(":", 1)
+                    positions.append((int(l), int(c)))
+            node.source_positions = tuple(positions)
+        else:
+            attr_idx = _attribute_name_to_index(key)
+            node.set_attribute(AttributeName(attr_idx), _unescape_value(value))
+
+    return node, 0
+
+
+def _parse_node_block(
+    lines: list[str],
+    start_idx: int,
+    expected_indent: int,
+    parent_file_name: str,
+) -> tuple[SyntaxNode, int]:
+    """Parse a block of nodes at a given indentation level."""
+    if start_idx >= len(lines):
+        raise ValueError(f"Expected node at line index {start_idx}, but no lines remain")
+
+    first_line = lines[start_idx]
+    if not first_line.strip():
+        raise ValueError(f"Empty line at index {start_idx}")
+
+    node, _ = _parse_single_node(first_line, parent_file_name)
+
+    child_indent = expected_indent + 2
+    idx = start_idx + 1
+
+    while idx < len(lines):
+        line = lines[idx]
+        if not line.strip():
+            idx += 1
+            continue
+
+        stripped = line.lstrip(" ")
+        current_indent = len(line) - len(stripped)
+
+        if current_indent < child_indent:
+            break
+
+        if current_indent == child_indent:
+            if _looks_like_subclass_field(stripped):
+                node = _apply_subclass_field(node, stripped)
+                idx += 1
+            else:
+                # Parse the current line as a child and recurse into its subtree
+                child_node, next_idx = _parse_node_block(lines, idx, child_indent, parent_file_name)
+                node.add_child(child_node)
+                idx = next_idx
+        else:
+            break
+
+    return node, idx
+
+
+def reconstruct_syntax_tree(serialized: str) -> SyntaxNode:
+    """Reconstruct a ``SyntaxNode`` tree from a serialized string.
+
+    The input format matches exactly the output of
+    ``serialize_syntax_tree`` / the Rust ``parse_to_serialized`` function:
+
+    * Header line: ``# file_name=<value>``
+    * One line per node: ``  TYPE line=N col=M end_line=N end_col=M [attr=val …]``
+    * Subclass fields: ``  value=<val>`` or ``  text=<text>``
+    * Children are indented one level deeper.
+
+    Returns
+    -------
+    SyntaxNode
+        The root node of the reconstructed tree.
+    """
+    lines = serialized.split("\n") if serialized else []
+
+    root_file_name = ""
+    line_idx = 0
+    if lines and lines[0].startswith("# file_name="):
+        root_file_name = _unescape_value(lines[0][len("# file_name="):])
+        line_idx = 1
+
+    root, _ = _parse_node_block(lines, line_idx, 0, root_file_name)
+    # Set file_name on the root node so re-serialization includes the header
+    root.file_name = root_file_name
+    return root
+
+
+__all__ = ["reconstruct_syntax_tree"]

delphi_lsp/navigation_cache.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import stat
+import tempfile
+from collections.abc import Mapping
+from pathlib import Path
+
+NAVIGATION_SHARD_SCHEMA = 4
+_MAX_SHARD_BYTES = 256 * 1024**2
+_KEY_PREFIX = b"python-delphi-lsp-navigation-shard-v4\0"
+
+
+def navigation_cache_key(text: str, defines: tuple[str, ...]) -> str:
+    digest = hashlib.sha256()
+    digest.update(_KEY_PREFIX)
+    digest.update(
+        json.dumps(
+            sorted(define.casefold() for define in defines),
+            ensure_ascii=False,
+            separators=(",", ":"),
+        ).encode("utf-8")
+    )
+    digest.update(b"\0")
+    digest.update(text.encode("utf-8"))
+    return digest.hexdigest()
+
+
+class NavigationShardStore:
+    """Content-addressed JSON storage for detached navigation symbols."""
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+
+    def load(self, cache_key: str) -> Mapping[str, object] | None:
+        path = self._path(cache_key)
+        if self.root.is_symlink() or path.parent.is_symlink():
+            return None
+        try:
+            descriptor = os.open(
+                path,
+                os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
+            )
+            try:
+                info = os.fstat(descriptor)
+                if (
+                    not stat.S_ISREG(info.st_mode)
+                    or info.st_size > _MAX_SHARD_BYTES
+                    or (
+                        os.name != "nt"
+                        and (
+                            info.st_uid != os.getuid()
+                            or info.st_mode & 0o077
+                        )
+                    )
+                ):
+                    return None
+                with os.fdopen(descriptor, "rb", closefd=False) as stream:
+                    raw = stream.read(_MAX_SHARD_BYTES + 1)
+                if len(raw) > _MAX_SHARD_BYTES:
+                    return None
+            finally:
+                os.close(descriptor)
+            payload = json.loads(raw.decode("utf-8"))
+        except (OSError, UnicodeError, ValueError):
+            return None
+        if (
+            not isinstance(payload, dict)
+            or payload.get("schema") != NAVIGATION_SHARD_SCHEMA
+            or payload.get("cache_key") != cache_key
+            or not isinstance(payload.get("symbols"), list)
+        ):
+            return None
+        return payload
+
+    def store(self, cache_key: str, payload: Mapping[str, object]) -> None:
+        path = self._path(cache_key)
+        self._ensure_directory(path.parent)
+        record = {
+            **dict(payload),
+            "schema": NAVIGATION_SHARD_SCHEMA,
+            "cache_key": cache_key,
+        }
+        data = json.dumps(
+            record,
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+            allow_nan=False,
+        ).encode("utf-8")
+        if len(data) > _MAX_SHARD_BYTES:
+            return
+        descriptor, temporary = tempfile.mkstemp(prefix=".navigation-", dir=path.parent)
+        try:
+            if os.name != "nt":
+                os.fchmod(descriptor, 0o600)
+            with os.fdopen(descriptor, "wb") as stream:
+                stream.write(data)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(temporary, path)
+            if os.name != "nt":
+                os.chmod(path, 0o600)
+        finally:
+            try:
+                os.unlink(temporary)
+            except FileNotFoundError:
+                pass
+
+    def prune(self, live_keys: set[str], max_bytes: int) -> None:
+        if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes < 0:
+            raise ValueError("navigation cache byte budget must be non-negative")
+        if not self.root.exists() or self.root.is_symlink():
+            return
+        retained: list[tuple[int, str, Path, int]] = []
+        bucket_paths: list[Path] = []
+        try:
+            buckets = tuple(self.root.iterdir())
+        except OSError:
+            return
+        for bucket in buckets:
+            if bucket.is_symlink() or not bucket.is_dir():
+                continue
+            bucket_paths.append(bucket)
+            try:
+                shards = tuple(bucket.iterdir())
+            except OSError:
+                continue
+            for shard in shards:
+                if (
+                    shard.is_symlink()
+                    or not shard.is_file()
+                    or shard.suffix != ".json"
+                ):
+                    continue
+                cache_key = shard.stem
+                try:
+                    info = shard.stat(follow_symlinks=False)
+                except OSError:
+                    continue
+                if cache_key not in live_keys:
+                    try:
+                        shard.unlink()
+                    except OSError:
+                        pass
+                    continue
+                retained.append(
+                    (info.st_mtime_ns, str(shard), shard, info.st_size)
+                )
+
+        retained_bytes = sum(item[3] for item in retained)
+        for _mtime, _name, shard, size in sorted(retained):
+            if retained_bytes <= max_bytes:
+                break
+            try:
+                shard.unlink()
+            except OSError:
+                continue
+            retained_bytes -= size
+        for bucket in bucket_paths:
+            try:
+                bucket.rmdir()
+            except OSError:
+                pass
+
+    def _path(self, cache_key: str) -> Path:
+        if (
+            len(cache_key) != 64
+            or any(character not in "0123456789abcdef" for character in cache_key)
+        ):
+            raise ValueError("navigation cache key is malformed")
+        return self.root / cache_key[:2] / f"{cache_key}.json"
+
+    def _ensure_directory(self, directory: Path) -> None:
+        for path in (self.root, directory):
+            if path.exists() and path.is_symlink():
+                raise OSError("navigation cache path is unsafe")
+            path.mkdir(mode=0o700, parents=True, exist_ok=True)
+            if os.name != "nt":
+                os.chmod(path, 0o700)
+
+
+__all__ = [
+    "NAVIGATION_SHARD_SCHEMA",
+    "NavigationShardStore",
+    "navigation_cache_key",
+]