python-delphi-lsp 2.1.0__py3-none-any.whl → 2.2.0__py3-none-any.whl

delphi_lsp/_version.py

@@ -1 +1 @@
-__version__ = "2.1.0"
+__version__ = "2.2.0"

delphi_lsp/agent_cache.py

@@ -20,6 +20,8 @@ import threading
 import time
 from types import ModuleType
 
+from watchfiles import watch
+
 from ._version import __version__
 from .agent_context import AgentContext
 from .agent_protocol import AgentProtocolError
@@ -32,10 +34,15 @@ DEFAULT_IDLE_TIMEOUT = 1800
 DEFAULT_STARTUP_TIMEOUT = 120.0
 _MAX_MESSAGE_BYTES = 1024 * 1024
 _CONNECTION_TIMEOUT = 2.0
+_CLIENT_RESPONSE_TIMEOUT = 120.0
 _MEMORY_SIZE = re.compile(r"^(?P<count>[1-9][0-9]*)(?P<suffix>[KMG]?)$", re.IGNORECASE)
 _STARTUP_DIAGNOSTIC_BYTES = 16 * 1024
 _STARTUP_TOKEN_RE = re.compile(r"(?i)(token\b[^\n\r]*?:?\s*['\"]?[A-Za-z0-9_-]+['\"]?|\b[a-zA-Z0-9_-]{32,})")
 _START_LOCK_INCOMPLETE_GRACE_SECONDS = 1.0
+_CACHE_REVISION_CHECK_INTERVAL_SECONDS = 3600.0
+_WATCHED_SUFFIXES = frozenset(
+    {".pas", ".pp", ".inc", ".dpr", ".dpk", ".dproj", ".cfg"}
+)
 
 
 def estimate_deep_size(value: object) -> int:
@@ -475,7 +482,7 @@ def _start_lock(root: str | Path, timeout: float):
 def _client_exchange(metadata: CacheMetadata, request: dict[str, object]) -> CacheClientResponse:
     try:
         with socket.create_connection(("127.0.0.1", metadata.port), timeout=2) as connection:
-            connection.settimeout(3)
+            connection.settimeout(_CLIENT_RESPONSE_TIMEOUT)
             request_without_token = {key: value for key, value in request.items() if key != "token"}
             connection.sendall(json.dumps({"token": metadata.token, **request_without_token}, separators=(",", ":")).encode("utf-8") + b"\n")
             response = _read_line(connection)
@@ -541,6 +548,7 @@ class _CacheService:
             metadata.project_file or None,
             workers=metadata.workers,
             worker_memory_budget_bytes=metadata.max_memory_bytes,
+            revision_check_interval_seconds=_CACHE_REVISION_CHECK_INTERVAL_SECONDS,
         )
         self.budget = CacheBudget(metadata.max_memory_bytes)
         self.stats = CacheStats()
@@ -556,15 +564,14 @@ class _CacheService:
     def prewarm(self) -> None:
         started = time.monotonic()
         try:
-            self.context.handle({"action": "find", "query": "", "max_items": 1, "max_chars": 256})
-            self.last_revision = self.context.workspace.workspace_revision
+            self.last_revision = self.context.prewarm_navigation()
             self.cache_state = "warm"
         except AgentProtocolError as error:
             if error.code != "project_required":
                 raise
             self.cache_state = "ready"
         self.last_budget = self.budget.enforce(
-            measure=lambda: estimate_deep_size(self.context.cache_roots()),
+            measure=lambda: self.context.estimated_cache_bytes,
             evict_auxiliary=self.context.evict_auxiliary_caches,
             evict_navigation=self.context.evict_navigation_caches,
         )
@@ -605,7 +612,7 @@ class _CacheService:
             if before != after:
                 self.stats.invalidations += 1
             self.last_budget = self.budget.enforce(
-                measure=lambda: estimate_deep_size(self.context.cache_roots()),
+                measure=lambda: self.context.estimated_cache_bytes,
                 evict_auxiliary=self.context.evict_auxiliary_caches,
                 evict_navigation=self.context.evict_navigation_caches,
             )
@@ -642,10 +649,31 @@ class _CacheService:
             "parallel_seconds": self.context.parallel_stats.elapsed_seconds,
             "parallel_fallbacks": self.stats.parallel_fallbacks,
             "idle_timeout": self.metadata.idle_timeout, "idle_remaining": max(0.0, self.metadata.idle_timeout - idle),
-            "workspace_revision": self.context.workspace.workspace_revision,
+            "workspace_revision": self.last_revision,
         }
 
 
+def _watch_filter(_change: object, path: str) -> bool:
+    return Path(path).suffix.casefold() in _WATCHED_SUFFIXES
+
+
+def _watch_workspace(service: _CacheService) -> None:
+    try:
+        for changes in watch(
+            service.metadata.root,
+            watch_filter=_watch_filter,
+            stop_event=service.shutdown,
+            debounce=50,
+            step=20,
+            recursive=True,
+            raise_interrupt=False,
+        ):
+            if changes:
+                service.context.invalidate_revision_cache()
+    except (OSError, RuntimeError):
+        service.context.invalidate_revision_cache()
+
+
 def _serve_connection(connection: socket.socket, service: _CacheService) -> None:
     try:
         line = _read_line(connection)
@@ -691,9 +719,17 @@ def run_cache_daemon(
         idle_timeout,
         time.time(),
     )
+    watcher: threading.Thread | None = None
     try:
         service = _CacheService(metadata)
         service.prewarm()
+        watcher = threading.Thread(
+            target=_watch_workspace,
+            args=(service,),
+            name="delphi-cache-watcher",
+            daemon=True,
+        )
+        watcher.start()
         _write_metadata(metadata)
         while not service.shutdown.is_set() and time.monotonic() - service.last_activity < idle_timeout:
             try:
@@ -704,6 +740,10 @@ def run_cache_daemon(
                 connection.settimeout(_CONNECTION_TIMEOUT)
                 _serve_connection(connection, service)
     finally:
+        if "service" in locals():
+            service.shutdown.set()
+        if watcher is not None:
+            watcher.join(timeout=2.0)
         listener.close()
         _remove_metadata_if_owned(metadata)
 

delphi_lsp/agent_context.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 from bisect import bisect_left, bisect_right
+from collections import OrderedDict
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from heapq import heappop, heappush
 import hashlib
 import json
 from pathlib import Path, PureWindowsPath
+import sys
+import time
 import unicodedata
 
 from .agent_protocol import (
@@ -22,12 +25,17 @@ from .agent_metrics import build_workspace_metrics, project_metric_item, unit_me
 from .agent_relations import ProjectRelationIndex, RelationTarget
 from .agent_workspace import AgentUnit, AgentWorkspace, unit_display_path, unit_source_path, unit_target_id
 from .consts import AttributeName, SyntaxNodeType
-from .lsp_server import multiline_string_block_end
+from .lsp_server import build_outline_semantic_model, multiline_string_block_end
 from .nodes import CompoundSyntaxNode, SyntaxNode
 from .parser import DelphiParser
-from .semantic import Scope, ScopeKind, Symbol, SymbolKind
+from .semantic import NamedTypeRef, Scope, ScopeKind, Symbol, SymbolKind
 from .metrics import ProjectMetrics
-from .parallel_outline import OutlineResult, OutlineTask, ParallelBuildStats, run_outline_tasks
+from .parallel_outline import (
+    ParallelBuildStats,
+    ParallelOutlineError,
+    run_outline_tasks,
+)
+from .source_reader import read_source_text
 
 
 _ROUTINE_KINDS = frozenset(
@@ -90,9 +98,10 @@ _CALLING_CONVENTIONS = frozenset(
     {"cdecl", "pascal", "register", "safecall", "stdcall", "winapi"}
 )
 _SOURCE_CHUNK_CHARS = 6000
+_RANKED_QUERY_CACHE_SIZE = 16
 
 
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
 class _Token:
     value: str
     start: int
@@ -137,6 +146,18 @@ class _SourceDocument:
         self.parser_spans: dict[str, tuple[int, int] | None] = {}
         self._full_parse_attempted = False
         self._full_parse_result: object | None = None
+        self.retained_bytes = (
+            sys.getsizeof(self)
+            + sys.getsizeof(self.text)
+            + sys.getsizeof(self.line_starts)
+            + len(self.line_starts) * 32
+            + sys.getsizeof(self.tokens)
+            + len(self.tokens) * 160
+            + sys.getsizeof(self.token_starts)
+            + len(self.token_starts) * 28
+            + sys.getsizeof(self.directive_starts)
+            + len(self.directive_starts) * 28
+        )
 
     def offset(self, line: int, column: int = 1) -> int:
         if not self.line_starts:
@@ -185,7 +206,78 @@ class _SourceDocument:
         return self._full_parse_result
 
 
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
+class _SourceSpec:
+    source_path: Path
+    display_path: str
+    defines: tuple[str, ...]
+    include_paths: tuple[str, ...]
+
+
+class _SourceStore:
+    """Load expensive tokenized source documents only when source evidence is requested."""
+
+    def __init__(
+        self,
+        specs: Mapping[Path, _SourceSpec],
+        *,
+        max_loaded_bytes: int,
+    ) -> None:
+        self._specs = dict(specs)
+        self._loaded: OrderedDict[Path, _SourceDocument] = OrderedDict()
+        self._loaded_bytes = 0
+        self._max_loaded_bytes = max(0, max_loaded_bytes)
+
+    def __getitem__(self, source_path: Path) -> _SourceDocument:
+        cached = self._loaded.pop(source_path, None)
+        if cached is not None:
+            self._loaded[source_path] = cached
+            return cached
+
+        spec = self._specs[source_path]
+        document = _SourceDocument(
+            spec.source_path,
+            spec.display_path,
+            read_source_text(spec.source_path),
+            defines=spec.defines,
+            include_paths=spec.include_paths,
+        )
+        if document.retained_bytes > self._max_loaded_bytes:
+            return document
+        while (
+            self._loaded
+            and self._loaded_bytes + document.retained_bytes > self._max_loaded_bytes
+        ):
+            _, evicted = self._loaded.popitem(last=False)
+            self._loaded_bytes -= evicted.retained_bytes
+        self._loaded[source_path] = document
+        self._loaded_bytes += document.retained_bytes
+        return document
+
+    @property
+    def loaded_count(self) -> int:
+        return len(self._loaded)
+
+    @property
+    def retained_bytes(self) -> int:
+        return self._loaded_bytes
+
+    @property
+    def metadata_bytes(self) -> int:
+        return 512 + sum(
+            256
+            + sys.getsizeof(path)
+            + sys.getsizeof(spec)
+            + sys.getsizeof(spec.display_path)
+            for path, spec in self._specs.items()
+        )
+
+    def clear_loaded(self) -> None:
+        self._loaded.clear()
+        self._loaded_bytes = 0
+
+
+@dataclass(frozen=True, slots=True)
 class _RawSymbol:
     symbol: Symbol
     source_path: Path
@@ -198,7 +290,115 @@ class _RawSymbol:
     signature: str
 
 
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
+class _NavigationTask:
+    ordinal: int
+    source_path: str
+    display_path: str
+    unit_name: str
+    unit_path: str
+    unit_id: str
+    unit_has_error: bool
+    defines: tuple[str, ...]
+    include_paths: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _NavigationResult:
+    ordinal: int
+    source_path: str
+    text: str
+    model: None
+    lines_processed: int
+    symbols_discovered: int
+    read_error: str
+    raw_symbols: tuple[_RawSymbol, ...]
+
+
+def _parse_navigation_task(task: _NavigationTask) -> _NavigationResult:
+    source_path = Path(task.source_path)
+    try:
+        text = read_source_text(source_path)
+    except (OSError, UnicodeError) as error:
+        return _NavigationResult(
+            task.ordinal,
+            task.source_path,
+            "",
+            None,
+            0,
+            0,
+            str(error),
+            (),
+        )
+    try:
+        model = build_outline_semantic_model(
+            text,
+            task.source_path,
+            defines=task.defines,
+        )
+        document = _SourceDocument(
+            source_path,
+            task.display_path,
+            text,
+            defines=task.defines,
+            include_paths=task.include_paths,
+        )
+        unit = AgentUnit(
+            unit_id=task.unit_id,
+            name=task.unit_name,
+            path=task.unit_path,
+            has_error=task.unit_has_error,
+        )
+        symbols = _collect_raw_symbols(model.unit_scope, unit, source_path, document)
+        raw_symbols = _detach_raw_symbols(
+            _exclude_routine_locals(symbols, document),
+            task.unit_name,
+        )
+    except Exception as error:
+        raise ParallelOutlineError(
+            f"failed to build navigation shard for {task.source_path}: {error}"
+        ) from error
+    return _NavigationResult(
+        task.ordinal,
+        task.source_path,
+        "",
+        None,
+        text.count("\n") + (0 if not text or text.endswith(("\n", "\r")) else 1),
+        len(raw_symbols),
+        "",
+        raw_symbols,
+    )
+
+
+def _detach_raw_symbols(
+    raw_symbols: list[_RawSymbol],
+    unit_name: str,
+) -> tuple[_RawSymbol, ...]:
+    detached_scope = Scope(kind=ScopeKind.UNIT, name=unit_name)
+    detached: list[_RawSymbol] = []
+    for raw in raw_symbols:
+        symbol = raw.symbol
+        flat_symbol = Symbol(
+            name=symbol.name,
+            kind=symbol.kind,
+            decl_range=symbol.decl_range,
+            name_range=symbol.name_range,
+            scope=detached_scope,
+            visibility=symbol.visibility,
+            type_ref=NamedTypeRef(symbol.type_ref.display_name()),
+            modifiers=set(symbol.modifiers),
+            attributes=dict(symbol.attributes),
+            doc=symbol.doc,
+            base_types=tuple(
+                NamedTypeRef(base_type.display_name())
+                for base_type in symbol.base_types
+            ),
+        )
+        detached.append(replace(raw, symbol=flat_symbol))
+    return tuple(detached)
+
+
+@dataclass(frozen=True, slots=True)
 class _SymbolEntry:
     symbol: Symbol
     source_path: Path
@@ -228,14 +428,15 @@ class _SymbolEntry:
         }
 
 
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
 class _Registry:
     project_id: str
     revision: str
     entries: tuple[_SymbolEntry, ...]
     by_target: dict[str, _SymbolEntry]
-    sources: dict[Path, _SourceDocument]
-    ranked_queries: dict[str, tuple[_SymbolEntry, ...]]
+    sources: _SourceStore
+    ranked_queries: OrderedDict[str, tuple[_SymbolEntry, ...]]
+    static_retained_bytes: int
 
 
 class AgentContext:
@@ -245,10 +446,16 @@ class AgentContext:
         *,
         workers: int = 0,
         worker_memory_budget_bytes: int | None = None,
+        revision_check_interval_seconds: float = 0.0,
     ) -> None:
         self._workspace = workspace
         self._workers = workers
         self._worker_memory_budget_bytes = worker_memory_budget_bytes
+        self._revision_check_interval_seconds = max(
+            0.0,
+            revision_check_interval_seconds,
+        )
+        self._last_revision_check_at = 0.0
         self._parallel_stats = ParallelBuildStats(0, 0, 0, 0.0, 0)
         project_id = workspace.active_project_id
         self._focus = Focus(project_id=project_id) if project_id else Focus()
@@ -266,11 +473,13 @@ class AgentContext:
         *,
         workers: int = 0,
         worker_memory_budget_bytes: int | None = None,
+        revision_check_interval_seconds: float = 0.0,
     ) -> AgentContext:
         return cls(
             AgentWorkspace.open(root, project_file=project_file),
             workers=workers,
             worker_memory_budget_bytes=worker_memory_budget_bytes,
+            revision_check_interval_seconds=revision_check_interval_seconds,
         )
 
     @property
@@ -293,16 +502,46 @@ class AgentContext:
             self._metrics,
         )
 
+    @property
+    def estimated_cache_bytes(self) -> int:
+        retained = self._workspace.estimated_cache_bytes
+        if self._registry is not None:
+            retained += (
+                self._registry.static_retained_bytes
+                + self._registry.sources.retained_bytes
+                + sum(
+                    sys.getsizeof(query)
+                    + sys.getsizeof(ranked)
+                    + len(ranked) * 8
+                    for query, ranked in self._registry.ranked_queries.items()
+                )
+            )
+        if self._relation_index is not None:
+            retained += max(4096, len(self._registry.entries) * 512) if self._registry else 4096
+        if self._metrics is not None:
+            retained += 4096 + len(self._metrics.units) * 1024
+        return retained
+
     def evict_auxiliary_caches(self) -> None:
         self._relation_index = None
         self._metrics = None
         self._metrics_revision = ""
+        if self._registry is not None:
+            self._registry.sources.clear_loaded()
 
     def evict_navigation_caches(self) -> None:
         self.evict_auxiliary_caches()
         self._registry = None
         self._workspace.evict_recomputable_caches()
 
+    def prewarm_navigation(self) -> str:
+        revision = self._refresh_workspace("")
+        self._require_registry(revision)
+        return revision
+
+    def invalidate_revision_cache(self) -> None:
+        self._last_revision_check_at = 0.0
+
     def handle(self, request: AgentRequest | Mapping[str, object]) -> AgentResponse:
         parsed = _validated_request(request)
         revision = self._refresh_workspace(parsed.project_id)
@@ -328,11 +567,12 @@ class AgentContext:
             return self._handle_focus(parsed, revision)
         if parsed.action == "find":
             registry = self._require_registry(revision)
-            ranked = registry.ranked_queries.get(parsed.query)
+            ranked = registry.ranked_queries.pop(parsed.query, None)
             if ranked is None:
                 ranked = tuple(_ranked_entries(registry.entries, parsed.query))
-                registry.ranked_queries.clear()
-                registry.ranked_queries[parsed.query] = ranked
+            registry.ranked_queries[parsed.query] = ranked
+            while len(registry.ranked_queries) > _RANKED_QUERY_CACHE_SIZE:
+                registry.ranked_queries.popitem(last=False)
             items = [entry.card() for entry in ranked]
             return self._response(parsed, revision, items)
         if parsed.action == "inspect":
@@ -345,10 +585,25 @@ class AgentContext:
     def _refresh_workspace(self, requested_project_id: str) -> str:
         previous_project_id = self._workspace.active_project_id
         selected_project_id = requested_project_id or previous_project_id
-        if selected_project_id:
+        now = time.monotonic()
+        selection_changed = bool(
+            requested_project_id
+            and requested_project_id != previous_project_id
+        )
+        revision_is_fresh = (
+            not selection_changed
+            and self._revision_check_interval_seconds > 0.0
+            and now - self._last_revision_check_at
+            < self._revision_check_interval_seconds
+        )
+        if revision_is_fresh:
+            revision = self._last_revision
+        elif selected_project_id:
             revision = self._workspace._select_project_with_revision(selected_project_id)
         else:
             revision = self._workspace.workspace_revision
+        if not revision_is_fresh:
+            self._last_revision_check_at = now
         current_project_id = self._workspace.active_project_id
 
         if current_project_id != previous_project_id:
@@ -731,36 +986,39 @@ def _build_registry(
     worker_memory_budget_bytes: int | None = None,
 ) -> tuple[_Registry, ParallelBuildStats]:
     raw_symbols: list[_RawSymbol] = []
-    sources: dict[Path, _SourceDocument] = {}
     units = tuple(workspace.units)
+    source_specs = {
+        unit_source_path(workspace.root, unit): _SourceSpec(
+            unit_source_path(workspace.root, unit),
+            unit_display_path(workspace.root, unit),
+            workspace.defines,
+            workspace.include_paths,
+        )
+        for unit in units
+    }
 
-    def consume_result(result: OutlineResult) -> None:
-        unit = units[result.ordinal]
-        source_path = unit_source_path(workspace.root, unit)
-        display_path = unit_display_path(workspace.root, unit)
-        if result.read_error or result.model is None:
+    def consume_result(result: _NavigationResult) -> None:
+        if result.read_error:
+            unit = units[result.ordinal]
+            display_path = unit_display_path(workspace.root, unit)
             raise AgentProtocolError(
                 "source_unavailable",
                 f"Could not read selected source {display_path}.",
             )
-        document = _SourceDocument(
-            source_path,
-            display_path,
-            result.text,
-            defines=workspace.defines,
-            include_paths=workspace.include_paths,
-        )
-        sources[source_path] = document
-        unit_symbols = _collect_raw_symbols(result.model.unit_scope, unit, source_path, document)
-        raw_symbols.extend(_exclude_routine_locals(unit_symbols, document))
+        raw_symbols.extend(result.raw_symbols)
 
     outline_batch = run_outline_tasks(
         (
-            OutlineTask(
+            _NavigationTask(
                 ordinal,
                 str(unit_source_path(workspace.root, unit)),
+                unit_display_path(workspace.root, unit),
+                unit.name,
+                unit.path,
+                unit.unit_id,
+                unit.has_error,
                 workspace.defines,
-                True,
+                workspace.include_paths,
             )
             for ordinal, unit in enumerate(units)
         ),
@@ -768,6 +1026,7 @@ def _build_registry(
         memory_budget_bytes=worker_memory_budget_bytes,
         on_complete=consume_result,
         retain_results=False,
+        task_runner=_parse_navigation_task,
     )
 
     ordered = sorted(raw_symbols, key=_raw_sort_key)
@@ -859,6 +1118,9 @@ def _build_registry(
         )
 
     entries_tuple = tuple(sorted(with_parents, key=_entry_sort_key))
+    source_cache_bytes = _source_cache_budget(worker_memory_budget_bytes)
+    sources = _SourceStore(source_specs, max_loaded_bytes=source_cache_bytes)
+    static_retained_bytes = _estimate_registry_bytes(entries_tuple, sources)
     return (
         _Registry(
             project_id=project_id,
@@ -866,12 +1128,40 @@ def _build_registry(
             entries=entries_tuple,
             by_target={entry.target_id: entry for entry in entries_tuple},
             sources=sources,
-            ranked_queries={},
+            ranked_queries=OrderedDict(),
+            static_retained_bytes=static_retained_bytes,
         ),
         outline_batch.stats,
     )
 
 
+def _source_cache_budget(total_budget_bytes: int | None) -> int:
+    if total_budget_bytes is None:
+        return 64 * 1024**2
+    return max(0, min(128 * 1024**2, total_budget_bytes // 4))
+
+
+def _estimate_registry_bytes(
+    entries: tuple[_SymbolEntry, ...],
+    sources: _SourceStore,
+) -> int:
+    retained = 4096 + sources.metadata_bytes
+    for entry in entries:
+        retained += (
+            1536
+            + sys.getsizeof(entry.path)
+            + sys.getsizeof(entry.unit_id)
+            + sys.getsizeof(entry.unit_name)
+            + sys.getsizeof(entry.qualified_name)
+            + sys.getsizeof(entry.owner)
+            + sys.getsizeof(entry.signature)
+            + sys.getsizeof(entry.target_id)
+            + sys.getsizeof(entry.parent_target_id)
+        )
+    retained += len(entries) * 96
+    return retained
+
+
 def _stable_path_component(value: str) -> str:
     normalized = unicodedata.normalize("NFC", value).replace("\\", "_").replace("/", "_")
     return normalized or "unknown"