python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/response_cache.py

@@ -0,0 +1,465 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import stat
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from .project_discovery import DelphiProjectDiscovery, discover_delphi_project
+
+RESPONSE_CACHE_SCHEMA_VERSION = 1
+RESPONSE_CACHE_PRODUCER_VERSION = "batch-response-v1"
+DEFAULT_RESPONSE_CACHE_BYTES = 2 * 1024 * 1024 * 1024
+_MAX_HEADER_BYTES = 16 * 1024
+_MAX_RECORD_BYTES = 1024 * 1024 * 1024
+_MAX_REVISION_MANIFEST_BYTES = 64 * 1024 * 1024
+_REVISION_MANIFEST_NAME = "content-revision-v1.json"
+_REVISION_MANIFEST_PRODUCER = "workspace-content-v1"
+_REVISION_MANIFEST_SCHEMA = 1
+_REVISION_PREFIX = b"python-delphi-lsp-workspace-content-v1\0"
+_KEY_PREFIX = b"python-delphi-lsp-response-cache-v1\0"
+
+
+@dataclass(frozen=True)
+class ResponseCacheKey:
+    workspace_revision: str
+    command: str
+    layer: str
+    query: str
+    output_format: str
+    deep_projects: bool
+
+    def to_mapping(self) -> dict[str, object]:
+        return {
+            "command": self.command,
+            "deep_projects": self.deep_projects,
+            "layer": self.layer,
+            "output_format": self.output_format,
+            "producer_version": RESPONSE_CACHE_PRODUCER_VERSION,
+            "query": self.query,
+            "schema_version": RESPONSE_CACHE_SCHEMA_VERSION,
+            "workspace_revision": self.workspace_revision,
+        }
+
+    @property
+    def cache_key(self) -> str:
+        digest = hashlib.sha256()
+        digest.update(_KEY_PREFIX)
+        digest.update(_canonical_json(self.to_mapping()))
+        return digest.hexdigest()
+
+
+@dataclass(frozen=True)
+class WorkspaceContentSnapshot:
+    revision: str
+    discovery: DelphiProjectDiscovery
+
+
+class ResponseCacheStore:
+    def __init__(
+        self,
+        root: str | Path,
+        *,
+        max_bytes: int = DEFAULT_RESPONSE_CACHE_BYTES,
+    ) -> None:
+        if (
+            isinstance(max_bytes, bool)
+            or not isinstance(max_bytes, int)
+            or max_bytes < 0
+        ):
+            raise ValueError("response cache byte budget must be non-negative")
+        self.root = Path(root)
+        self.max_bytes = max_bytes
+
+    def load(self, key: ResponseCacheKey) -> bytes | None:
+        path = self.path_for_key(key)
+        if self._has_unsafe_directory(path.parent):
+            return None
+        try:
+            descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+            try:
+                info = os.fstat(descriptor)
+                if (
+                    not stat.S_ISREG(info.st_mode)
+                    or info.st_size > _MAX_RECORD_BYTES
+                    or (
+                        os.name != "nt"
+                        and (info.st_uid != os.getuid() or info.st_mode & 0o077)
+                    )
+                ):
+                    return None
+                with os.fdopen(descriptor, "rb", closefd=False) as stream:
+                    data = stream.read(_MAX_RECORD_BYTES + 1)
+            finally:
+                os.close(descriptor)
+        except OSError:
+            return None
+        if len(data) > _MAX_RECORD_BYTES:
+            return None
+        header_bytes, separator, payload = data.partition(b"\n")
+        if not separator or len(header_bytes) > _MAX_HEADER_BYTES:
+            return None
+        try:
+            header = json.loads(header_bytes.decode("utf-8"))
+        except (UnicodeError, ValueError, RecursionError):
+            return None
+        if not isinstance(header, dict):
+            return None
+        if (
+            header.get("schema_version") != RESPONSE_CACHE_SCHEMA_VERSION
+            or header.get("producer_version") != RESPONSE_CACHE_PRODUCER_VERSION
+            or header.get("cache_key") != key.cache_key
+            or header.get("key") != key.to_mapping()
+            or header.get("payload_size") != len(payload)
+            or header.get("payload_sha256") != hashlib.sha256(payload).hexdigest()
+        ):
+            return None
+        return payload
+
+    def store(self, key: ResponseCacheKey, payload: bytes) -> None:
+        if not isinstance(payload, bytes):
+            raise TypeError("response cache payload must be bytes")
+        header = {
+            "cache_key": key.cache_key,
+            "key": key.to_mapping(),
+            "payload_sha256": hashlib.sha256(payload).hexdigest(),
+            "payload_size": len(payload),
+            "producer_version": RESPONSE_CACHE_PRODUCER_VERSION,
+            "schema_version": RESPONSE_CACHE_SCHEMA_VERSION,
+        }
+        header_bytes = _canonical_json(header)
+        if len(header_bytes) > _MAX_HEADER_BYTES:
+            raise ValueError("response cache header is too large")
+        record_size = len(header_bytes) + 1 + len(payload)
+        if record_size > _MAX_RECORD_BYTES:
+            raise ValueError("response cache record is too large")
+        if record_size > self.max_bytes:
+            return
+        path = self.path_for_key(key)
+        self._ensure_directory(path.parent)
+        descriptor, temporary = tempfile.mkstemp(prefix=".response-", dir=path.parent)
+        try:
+            if os.name != "nt":
+                os.fchmod(descriptor, 0o600)
+            with os.fdopen(descriptor, "wb") as stream:
+                stream.write(header_bytes)
+                stream.write(b"\n")
+                stream.write(payload)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(temporary, path)
+            if os.name != "nt":
+                os.chmod(path, 0o600)
+            self._prune(path)
+        finally:
+            try:
+                os.unlink(temporary)
+            except FileNotFoundError:
+                pass
+
+    def path_for_key(self, key: ResponseCacheKey) -> Path:
+        cache_key = key.cache_key
+        return self.root / cache_key[:2] / f"{cache_key}.bin"
+
+    def _prune(self, current: Path) -> None:
+        if not self.root.exists() or self._has_unsafe_directory(self.root):
+            return
+        records: list[tuple[bool, int, str, Path, int]] = []
+        try:
+            buckets = tuple(self.root.iterdir())
+        except OSError:
+            return
+        for bucket in buckets:
+            if bucket.is_symlink() or not bucket.is_dir():
+                continue
+            try:
+                entries = tuple(bucket.iterdir())
+            except OSError:
+                continue
+            for entry in entries:
+                if entry.is_symlink() or entry.suffix != ".bin":
+                    continue
+                try:
+                    info = entry.stat(follow_symlinks=False)
+                except OSError:
+                    continue
+                if not stat.S_ISREG(info.st_mode):
+                    continue
+                records.append(
+                    (
+                        entry == current,
+                        info.st_mtime_ns,
+                        str(entry),
+                        entry,
+                        info.st_size,
+                    )
+                )
+        retained_bytes = sum(item[4] for item in records)
+        for _is_current, _mtime, _name, entry, size in sorted(records):
+            if retained_bytes <= self.max_bytes:
+                break
+            try:
+                entry.unlink()
+            except OSError:
+                continue
+            retained_bytes -= size
+
+    def _has_unsafe_directory(self, directory: Path) -> bool:
+        return any(
+            path.is_symlink() for path in (self.root.parent, self.root, directory)
+        )
+
+    def _ensure_directory(self, directory: Path) -> None:
+        if self._has_unsafe_directory(directory):
+            raise OSError("response cache path is unsafe")
+        self.root.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+        for path in (self.root, directory):
+            if path.exists() and path.is_symlink():
+                raise OSError("response cache path is unsafe")
+            path.mkdir(mode=0o700, parents=True, exist_ok=True)
+            if os.name != "nt":
+                os.chmod(path, 0o700)
+
+
+def workspace_content_revision(
+    root: str | Path,
+    *,
+    project_file: str | Path | None = None,
+) -> str:
+    return workspace_content_snapshot(root, project_file=project_file).revision
+
+
+def workspace_content_snapshot(
+    root: str | Path,
+    *,
+    project_file: str | Path | None = None,
+) -> WorkspaceContentSnapshot:
+    root_path = Path(root).expanduser().resolve()
+    resolved_project: Path | None = None
+    if project_file is not None:
+        resolved_project = Path(project_file).expanduser()
+        if not resolved_project.is_absolute():
+            resolved_project = root_path / resolved_project
+        resolved_project = resolved_project.resolve()
+    discovery = discover_delphi_project(
+        root_path,
+        project_file=resolved_project,
+        scan_workspace_sources=True,
+    )
+    paths = {
+        *(Path(value).expanduser().resolve() for value in discovery.project_files),
+        *(Path(value).expanduser().resolve() for value in discovery.config_files),
+        *(Path(value).expanduser().resolve() for value in discovery.source_files),
+    }
+    if resolved_project is not None:
+        paths.add(resolved_project)
+
+    digest = hashlib.sha256()
+    digest.update(_REVISION_PREFIX)
+    selection = {
+        "defines": sorted(discovery.defines),
+        "include_paths": sorted(
+            _display_path(Path(value), root_path) for value in discovery.include_paths
+        ),
+        "project_file": ""
+        if resolved_project is None
+        else _display_path(resolved_project, root_path),
+        "root": str(root_path),
+        "search_paths": sorted(
+            _display_path(Path(value), root_path) for value in discovery.search_paths
+        ),
+    }
+    digest.update(_canonical_json(selection))
+    cached_records = _load_revision_manifest(root_path)
+    current_records: dict[str, dict[str, object]] = {}
+    for path in sorted(
+        paths, key=lambda item: (_display_path(item, root_path).casefold(), str(item))
+    ):
+        display = _display_path(path, root_path)
+        digest.update(display.encode("utf-8", errors="surrogatepass"))
+        digest.update(b"\0")
+        metadata = _file_metadata(path)
+        content_digest = _cached_content_digest(
+            cached_records.get(display),
+            metadata,
+        )
+        if content_digest is None:
+            content_digest = _hash_file(path)
+            verified_metadata = _file_metadata(path)
+            if content_digest is not None and verified_metadata == metadata:
+                metadata = verified_metadata
+            else:
+                metadata = None
+        if content_digest is None:
+            digest.update(b"missing")
+        else:
+            digest.update(bytes.fromhex(content_digest))
+            if metadata is not None:
+                current_records[display] = {
+                    **metadata,
+                    "sha256": content_digest,
+                }
+        digest.update(b"\0")
+    if current_records != cached_records:
+        try:
+            _store_revision_manifest(root_path, current_records)
+        except (OSError, ValueError):
+            pass
+    return WorkspaceContentSnapshot(
+        revision=f"workspace_content_v1_{digest.hexdigest()}",
+        discovery=discovery,
+    )
+
+
+def _file_metadata(path: Path) -> dict[str, int] | None:
+    try:
+        info = path.stat()
+    except OSError:
+        return None
+    if not stat.S_ISREG(info.st_mode):
+        return None
+    return {
+        "ctime_ns": info.st_ctime_ns,
+        "device": info.st_dev,
+        "inode": info.st_ino,
+        "mtime_ns": info.st_mtime_ns,
+        "size": info.st_size,
+    }
+
+
+def _cached_content_digest(
+    record: object,
+    metadata: dict[str, int] | None,
+) -> str | None:
+    if not isinstance(record, dict) or metadata is None:
+        return None
+    if any(record.get(key) != value for key, value in metadata.items()):
+        return None
+    content_digest = record.get("sha256")
+    if (
+        not isinstance(content_digest, str)
+        or len(content_digest) != 64
+        or any(character not in "0123456789abcdef" for character in content_digest)
+    ):
+        return None
+    return content_digest
+
+
+def _hash_file(path: Path) -> str | None:
+    try:
+        with path.open("rb") as stream:
+            digest = hashlib.sha256()
+            while chunk := stream.read(1024 * 1024):
+                digest.update(chunk)
+    except OSError:
+        return None
+    return digest.hexdigest()
+
+
+def _load_revision_manifest(root: Path) -> dict[str, object]:
+    path = root / ".delphi-lsp" / _REVISION_MANIFEST_NAME
+    if path.parent.is_symlink():
+        return {}
+    try:
+        descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        try:
+            info = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_size > _MAX_REVISION_MANIFEST_BYTES
+                or (
+                    os.name != "nt"
+                    and (info.st_uid != os.getuid() or info.st_mode & 0o077)
+                )
+            ):
+                return {}
+            with os.fdopen(descriptor, "rb", closefd=False) as stream:
+                data = stream.read(_MAX_REVISION_MANIFEST_BYTES + 1)
+        finally:
+            os.close(descriptor)
+    except OSError:
+        return {}
+    if len(data) > _MAX_REVISION_MANIFEST_BYTES:
+        return {}
+    try:
+        manifest = json.loads(data.decode("utf-8"))
+    except (UnicodeError, ValueError, RecursionError):
+        return {}
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("schema_version") != _REVISION_MANIFEST_SCHEMA
+        or manifest.get("producer_version") != _REVISION_MANIFEST_PRODUCER
+        or not isinstance(manifest.get("records"), dict)
+    ):
+        return {}
+    return manifest["records"]
+
+
+def _store_revision_manifest(
+    root: Path,
+    records: dict[str, dict[str, object]],
+) -> None:
+    directory = root / ".delphi-lsp"
+    path = directory / _REVISION_MANIFEST_NAME
+    if directory.is_symlink():
+        raise OSError("workspace revision manifest path is unsafe")
+    directory.mkdir(mode=0o700, parents=True, exist_ok=True)
+    if os.name != "nt":
+        os.chmod(directory, 0o700)
+    encoded = _canonical_json(
+        {
+            "producer_version": _REVISION_MANIFEST_PRODUCER,
+            "records": records,
+            "schema_version": _REVISION_MANIFEST_SCHEMA,
+        }
+    )
+    if len(encoded) > _MAX_REVISION_MANIFEST_BYTES:
+        raise ValueError("workspace revision manifest is too large")
+    descriptor, temporary = tempfile.mkstemp(prefix=".revision-", dir=directory)
+    try:
+        if os.name != "nt":
+            os.fchmod(descriptor, 0o600)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(encoded)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+        if os.name != "nt":
+            os.chmod(path, 0o600)
+    finally:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+
+
+def _display_path(path: Path, root: Path) -> str:
+    resolved = path.expanduser().resolve()
+    try:
+        return resolved.relative_to(root).as_posix()
+    except ValueError:
+        return str(resolved)
+
+
+def _canonical_json(payload: object) -> bytes:
+    return json.dumps(
+        payload,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    ).encode("utf-8")
+
+
+__all__ = [
+    "DEFAULT_RESPONSE_CACHE_BYTES",
+    "RESPONSE_CACHE_PRODUCER_VERSION",
+    "RESPONSE_CACHE_SCHEMA_VERSION",
+    "ResponseCacheKey",
+    "ResponseCacheStore",
+    "WorkspaceContentSnapshot",
+    "workspace_content_revision",
+    "workspace_content_snapshot",
+]