python-delphi-lsp 3.2.1__tar.gz → 3.5.0__tar.gz

{python_delphi_lsp-3.2.1 → python_delphi_lsp-3.5.0}/MANIFEST.in

@@ -12,12 +12,14 @@ include scripts/benchmark_fpc_corpus.py
 include scripts/prepare_delphi_corpus.py
 include scripts/benchmark_cpg.py
 include scripts/benchmark_parallel_cache.py
+include scripts/benchmark_response_cache.py
 include scripts/bootstrap_vllm_opencode_test.py
 include scripts/bootstrap_vllm_codebase_skill_test.py
 include scripts/bootstrap_vllm_opencode_test.ps1
 include scripts/bootstrap_vllm_opencode_test.sh
 include scripts/generate_release_evidence.py
 include scripts/generate_progress_pdf.py
+include scripts/outline_gap_detector.py
 include scripts/prepare_ornith_cache.py
 include scripts/run_opencode_lsp_probe.py
 include scripts/run_openrouter_github_e2e.py

{python_delphi_lsp-3.2.1/python_delphi_lsp.egg-info → python_delphi_lsp-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-delphi-lsp
-Version: 3.2.1
+Version: 3.5.0
 Summary: Python Delphi/Object Pascal parser, semantic indexer, and language server.
 Author: Dark Light
 License-Expression: MPL-2.0
@@ -45,7 +45,7 @@ Dynamic: license-file
 
 `python-delphi-lsp` parses Delphi/Object Pascal, builds semantic and project
 indexes, serves LSP, and provides bounded codebase navigation for agents.
-Version 3.2.1 is authored by Dark Light and supports Windows, macOS, and Linux.
+Version 3.5.0 is authored by Dark Light and supports Windows, macOS, and Linux.
 
 ## Install and quick start
 
@@ -226,7 +226,8 @@ delphi-lsp-agent worker --root PATH [--project-file FILE] [--workers auto|N]
 ```
 
 `--deep-projects` opts into per-project dependency parsing for the `projects`
-layer. It is off by default because it dominates run time on real repositories:
+and `problems` layers. It is off by default because it dominates run time on
+real repositories:
 on a 60-project corpus it cost 36.9 s against 1.0 s for the same index, and it
 adds nothing to the unit model set. Without it the `projects` and `problems`
 layers report `deep_indexed: false`, and `index` records `deep_projects` at the
@@ -416,6 +417,21 @@ task, CPU, and eight-worker caps without the cache-budget term. An explicit valu
 from 1 through 32 overrides the automatic CPU and memory caps but is still
 limited by the number of tasks.
 
+Standalone `view` and `index` commands cache their exact completed response
+bytes under `.delphi-lsp/response-cache`. A strong content revision covers the
+selected source, include, project, and configuration files, so any relevant
+content change produces a miss. Records are content-addressed, owner-only,
+atomically replaced, size-bounded, and never use pickle. Batch commands bypass
+the older per-unit artifact shards; an unchanged request can therefore return
+the validated response without rebuilding semantic models. To reproduce the
+large-corpus cold/warm gate, run:
+
+```bash
+python scripts/benchmark_response_cache.py \
+  --fpc-root /path/to/fpc-corpus \
+  --delphi-root /path/to/delphi-corpus
+```
+
 Worker processes build compact, detached navigation shards and exit before
 retained-cache accounting. Full semantic object graphs and source text are not
 retained in the parent. Tokenized source documents are loaded only for source

{python_delphi_lsp-3.2.1 → python_delphi_lsp-3.5.0}/README.md

@@ -2,7 +2,7 @@
 
 `python-delphi-lsp` parses Delphi/Object Pascal, builds semantic and project
 indexes, serves LSP, and provides bounded codebase navigation for agents.
-Version 3.2.1 is authored by Dark Light and supports Windows, macOS, and Linux.
+Version 3.5.0 is authored by Dark Light and supports Windows, macOS, and Linux.
 
 ## Install and quick start
 
@@ -183,7 +183,8 @@ delphi-lsp-agent worker --root PATH [--project-file FILE] [--workers auto|N]
 ```
 
 `--deep-projects` opts into per-project dependency parsing for the `projects`
-layer. It is off by default because it dominates run time on real repositories:
+and `problems` layers. It is off by default because it dominates run time on
+real repositories:
 on a 60-project corpus it cost 36.9 s against 1.0 s for the same index, and it
 adds nothing to the unit model set. Without it the `projects` and `problems`
 layers report `deep_indexed: false`, and `index` records `deep_projects` at the
@@ -373,6 +374,21 @@ task, CPU, and eight-worker caps without the cache-budget term. An explicit valu
 from 1 through 32 overrides the automatic CPU and memory caps but is still
 limited by the number of tasks.
 
+Standalone `view` and `index` commands cache their exact completed response
+bytes under `.delphi-lsp/response-cache`. A strong content revision covers the
+selected source, include, project, and configuration files, so any relevant
+content change produces a miss. Records are content-addressed, owner-only,
+atomically replaced, size-bounded, and never use pickle. Batch commands bypass
+the older per-unit artifact shards; an unchanged request can therefore return
+the validated response without rebuilding semantic models. To reproduce the
+large-corpus cold/warm gate, run:
+
+```bash
+python scripts/benchmark_response_cache.py \
+  --fpc-root /path/to/fpc-corpus \
+  --delphi-root /path/to/delphi-corpus
+```
+
 Worker processes build compact, detached navigation shards and exit before
 retained-cache accounting. Full semantic object graphs and source text are not
 retained in the parent. Tokenized source documents are loaded only for source

python_delphi_lsp-3.5.0/delphi_lsp/_version.py

@@ -0,0 +1 @@
+__version__ = "3.5.0"

{python_delphi_lsp-3.2.1 → python_delphi_lsp-3.5.0}/delphi_lsp/agent_cache.py

@@ -8,6 +8,8 @@ import math
 import os
 import re
 import secrets
+import select
+import signal
 import socket
 import stat
 import subprocess
@@ -20,7 +22,7 @@ from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, fields, is_dataclass, replace
 from functools import lru_cache
 from pathlib import Path
-from types import ModuleType
+from types import FrameType, ModuleType
 
 from watchfiles import watch
 
@@ -50,6 +52,9 @@ _START_LOCK_INCOMPLETE_GRACE_SECONDS = 1.0
 _CACHE_REVISION_CHECK_INTERVAL_SECONDS = 3600.0
 _WATCHER_READY_TIMEOUT_SECONDS = 1.0
 _WATCHER_READY_POLL_MILLISECONDS = 50
+_STOP_TIMEOUT_SECONDS = 15.0
+_STOP_SIGNAL_DELAY_SECONDS = 3.0
+_STOP_FORCE_TIMEOUT_SECONDS = 2.0
 _WATCHED_SUFFIXES = frozenset(
     {".pas", ".pp", ".inc", ".dpr", ".dpk", ".dproj", ".cfg"}
 )
@@ -356,6 +361,54 @@ class CacheClientError(RuntimeError):
         super().__init__(message)
 
 
+class _RequestGuard:
+    def __init__(
+        self,
+        service: _CacheService,
+        connection: socket.socket,
+        *,
+        enforce_rss_budget: bool = True,
+    ) -> None:
+        self._service = service
+        self._connection = connection
+        self._enforce_rss_budget = enforce_rss_budget
+
+    def check(self) -> None:
+        if self._service.shutdown.is_set():
+            raise CacheClientError(
+                "request_cancelled",
+                "Cache request was cancelled.",
+            )
+        try:
+            readable, _, _ = select.select(
+                (self._connection,),
+                (),
+                (),
+                0,
+            )
+            if readable and not self._connection.recv(1, socket.MSG_PEEK):
+                raise CacheClientError(
+                    "client_disconnected",
+                    "Cache client disconnected.",
+                )
+        except CacheClientError:
+            raise
+        except (OSError, ValueError) as error:
+            raise CacheClientError(
+                "client_disconnected",
+                "Cache client disconnected.",
+            ) from error
+        if (
+            self._enforce_rss_budget
+            and self._service._measure_retained_bytes()
+            > self._service.metadata.max_memory_bytes
+        ):
+            raise CacheClientError(
+                "memory_budget_exceeded",
+                "Cache request exceeded the configured retained-memory budget.",
+            )
+
+
 def cache_metadata_path(root: str | Path) -> Path:
     return Path(root).resolve() / ".delphi-lsp" / "agent-cache" / "daemon.json"
 
@@ -752,6 +805,8 @@ class _CacheService:
         self.budget = CacheBudget(metadata.max_memory_bytes)
         self.stats = CacheStats()
         self.lock = threading.Lock()
+        self._request_state_lock = threading.Lock()
+        self._active_requests = 0
         self.started = time.monotonic()
         self.last_activity = self.started
         self.cache_state = "warming"
@@ -803,11 +858,15 @@ class _CacheService:
         context = self._initialize_context()
         try:
             self.last_revision = context.prewarm_navigation()
-            self.cache_state = "warm"
         except AgentProtocolError as error:
             if error.code != "project_required":
                 raise
-            self.cache_state = "ready"
+        prewarm_rss = current_process_rss_bytes()
+        if prewarm_rss > 0:
+            # Parsing can leave a high allocator RSS watermark even though the
+            # retained navigation data is already covered by its own estimate.
+            # Track only subsequent, otherwise unaccounted-for RSS growth.
+            self._rss_baseline_bytes = prewarm_rss
         self._enforce_budget()
         self.prewarm_seconds = time.monotonic() - started
         self.stats.parallel_fallbacks = context.parallel_stats.fallbacks
@@ -836,7 +895,12 @@ class _CacheService:
         )
         self.prewarm_thread.start()
 
-    def request(self, request: dict[str, object]) -> CacheClientResponse:
+    def request(
+        self,
+        request: dict[str, object],
+        *,
+        cancel_check: Callable[[], None] | None = None,
+    ) -> CacheClientResponse:
         action = request.get("action")
         if action == "status":
             warning = "" if request.get("_startup_probe") is True else self._consume_warning()
@@ -863,21 +927,69 @@ class _CacheService:
         self.watcher_ready.wait(timeout=_WATCHER_READY_TIMEOUT_SECONDS)
         if self.watcher_failed.is_set():
             self.context.invalidate_revision_cache()
-        with self.lock:
+        with self._request_state_lock:
+            self._active_requests += 1
+        lock_acquired = False
+        try:
+            while not self.lock.acquire(timeout=0.05):
+                if self.shutdown.is_set():
+                    raise CacheClientError(
+                        "request_cancelled",
+                        "Cache request was cancelled.",
+                    )
+                if cancel_check is not None:
+                    cancel_check()
+            lock_acquired = True
+            if self.shutdown.is_set():
+                raise CacheClientError(
+                    "request_cancelled",
+                    "Cache request was cancelled.",
+                )
+            if cancel_check is not None:
+                cancel_check()
             self.last_activity = time.monotonic()
             before = self.last_revision
             was_warm = self.context.navigation_cache_is_warm
+            request_aborted = False
             try:
                 try:
-                    response = self.context.handle(request).to_mapping()
+                    handled = (
+                        self.context.handle(request)
+                        if cancel_check is None
+                        else self.context.handle(
+                            request,
+                            cancel_check=cancel_check,
+                        )
+                    )
+                    if self.shutdown.is_set():
+                        raise CacheClientError(
+                            "request_cancelled",
+                            "Cache request was cancelled.",
+                        )
+                    if cancel_check is not None:
+                        cancel_check()
+                    response = handled.to_mapping()
+                except CacheClientError:
+                    request_aborted = True
+                    raise
                 except AgentProtocolError as error:
-                    raise CacheClientError(error.code, error.message) from None
+                    raise CacheClientError(
+                        error.code,
+                        error.message,
+                    ) from None
                 except (OSError, UnicodeError):
-                    raise CacheClientError("source_unavailable", "Source is unavailable.") from None
+                    raise CacheClientError(
+                        "source_unavailable",
+                        "Source is unavailable.",
+                    ) from None
                 except Exception:
-                    raise CacheClientError("internal_error", "Internal cache error.") from None
+                    raise CacheClientError(
+                        "internal_error",
+                        "Internal cache error.",
+                    ) from None
             finally:
-                self._enforce_budget()
+                if not request_aborted and not self.shutdown.is_set():
+                    self._enforce_budget()
             after = str(response["workspace_revision"])
             self.last_revision = after
             self.stats.requests += 1
@@ -885,10 +997,20 @@ class _CacheService:
                 self.stats.warm_hits += 1
             else:
                 self.stats.rebuilds += 1
-                self.stats.parallel_fallbacks += self.context.parallel_stats.fallbacks
+                self.stats.parallel_fallbacks += (
+                    self.context.parallel_stats.fallbacks
+                )
             if before != after:
                 self.stats.invalidations += 1
-            return CacheClientResponse(response, self._consume_warning())
+            return CacheClientResponse(
+                response,
+                self._consume_warning(),
+            )
+        finally:
+            if lock_acquired:
+                self.lock.release()
+            with self._request_state_lock:
+                self._active_requests -= 1
 
     def _enforce_budget(self) -> None:
         self.last_budget = self.budget.enforce(
@@ -932,6 +1054,8 @@ class _CacheService:
             if context is not None
             else None
         )
+        with self._request_state_lock:
+            busy = self._active_requests > 0
         return {
             "pid": self.metadata.pid, "root": self.metadata.root, "project_file": self.metadata.project_file,
             "version": self.metadata.version, "uptime": now - self.started, "idle_seconds": idle,
@@ -958,6 +1082,7 @@ class _CacheService:
             "workspace_revision": self.last_revision,
             "startup_error": self.startup_error,
             "watcher_active": not self.watcher_failed.is_set(),
+            "busy": busy,
         }
 
 
@@ -1053,7 +1178,16 @@ def _serve_connection(connection: socket.socket, service: _CacheService) -> None
             response: dict[str, object] = {"error": {"code": "authentication_failed", "message": "authentication failed"}}
         else:
             try:
-                result = service.request(request)
+                action = request.get("action")
+                guard = None if action in {"status", "stop"} else _RequestGuard(
+                    service,
+                    connection,
+                    enforce_rss_budget=action in {"trace", "cpg"},
+                )
+                result = service.request(
+                    request,
+                    cancel_check=guard.check if guard is not None else None,
+                )
                 response = {"payload": result.payload, "warning": result.warning}
             except CacheClientError as error:
                 response = {"error": {"code": error.code, "message": error.message}}
@@ -1105,6 +1239,11 @@ def run_cache_daemon(
         max_disk_cache_bytes,
     )
     watcher: threading.Thread | None = None
+    previous_sigterm: (
+        Callable[[int, FrameType | None], object]
+        | int
+        | None
+    ) = None
     connection_slots = threading.BoundedSemaphore(_CONNECTION_WORKERS)
     connection_pool = ThreadPoolExecutor(
         max_workers=_CONNECTION_WORKERS,
@@ -1112,6 +1251,16 @@ def run_cache_daemon(
     )
     try:
         service = _CacheService(metadata, defer_context=True)
+        if threading.current_thread() is threading.main_thread():
+            previous_sigterm = signal.getsignal(signal.SIGTERM)
+
+            def request_shutdown(
+                _signum: int,
+                _frame: FrameType | None,
+            ) -> None:
+                service.shutdown.set()
+
+            signal.signal(signal.SIGTERM, request_shutdown)
         _write_metadata(metadata)
         watcher = threading.Thread(
             target=_watch_workspace,
@@ -1150,6 +1299,8 @@ def run_cache_daemon(
         _remove_metadata_if_owned(metadata)
         if watcher is not None:
             watcher.join(timeout=2.0)
+        if previous_sigterm is not None:
+            signal.signal(signal.SIGTERM, previous_sigterm)
 
 
 def _start_cache_unlocked(
@@ -1339,16 +1490,47 @@ def stop_cache(root: str | Path) -> None:
         _client_exchange(metadata, {"action": "stop"})
     except CacheClientError:
         pass
-    deadline = time.monotonic() + 3
+    started = time.monotonic()
+    deadline = started + _STOP_TIMEOUT_SECONDS
+    signal_sent = False
     stopped = False
-    while time.monotonic() < deadline:
+    while True:
+        now = time.monotonic()
+        if now >= deadline:
+            break
         if _reap_child_if_exited(metadata.pid):
             stopped = True
             break
         if not _pid_alive(metadata.pid):
             stopped = True
             break
+        if (
+            not signal_sent
+            and os.name != "nt"
+            and now - started >= _STOP_SIGNAL_DELAY_SECONDS
+        ):
+            with contextlib.suppress(OSError):
+                os.kill(metadata.pid, signal.SIGTERM)
+            signal_sent = True
         time.sleep(0.05)
+    if not stopped and _pid_alive(metadata.pid):
+        current = _read_metadata(root)
+        if current == metadata:
+            force_signal = (
+                signal.SIGKILL
+                if os.name != "nt" and hasattr(signal, "SIGKILL")
+                else signal.SIGTERM
+            )
+            with contextlib.suppress(OSError):
+                os.kill(metadata.pid, force_signal)
+            force_deadline = time.monotonic() + _STOP_FORCE_TIMEOUT_SECONDS
+            while time.monotonic() < force_deadline:
+                if _reap_child_if_exited(metadata.pid) or not _pid_alive(
+                    metadata.pid
+                ):
+                    stopped = True
+                    break
+                time.sleep(0.05)
     if not stopped and _pid_alive(metadata.pid):
         raise CacheClientError("stop_failed", "Cache daemon did not stop.")
     _remove_metadata_if_owned(metadata)

{python_delphi_lsp-3.2.1 → python_delphi_lsp-3.5.0}/delphi_lsp/agent_cli.py

@@ -25,7 +25,14 @@ from .agent_cache import (
     watch_workspace_changes,
 )
 from .agent_context import AgentContext
-from .agent_layers import build_codebase_index, layer_payload, render_layer
+from .agent_layers import (
+    build_codebase_index,
+    build_metrics_payload,
+    build_references_payload,
+    layer_payload,
+    render_layer,
+    render_payload,
+)
 from .agent_protocol import (
     SCHEMA_VERSION,
     SUPPORTED_ACTIONS,
@@ -39,6 +46,12 @@ from .agent_templates import install_opencode_support, install_skill
 from .agent_wiki import WikiExportError, WikiProgressEvent, export_okf_wiki
 from .parallel_outline import ParallelOutlineError, parse_worker_setting
 from .project_config import ProjectConfigError
+from .response_cache import (
+    ResponseCacheKey,
+    ResponseCacheStore,
+    WorkspaceContentSnapshot,
+    workspace_content_snapshot,
+)
 
 _MAX_WORKER_RECORD_BYTES = 1024 * 1024
 _INVALID_JSON_MESSAGE = "Invalid JSON request."
@@ -110,6 +123,11 @@ def build_parser() -> argparse.ArgumentParser:
     wiki_export.add_argument("--project-file", type=Path)
     wiki_export.add_argument("--out", type=Path, default=Path(".delphi-lsp") / "wiki")
     wiki_export.add_argument("--workers", type=parse_worker_setting, default=0)
+    wiki_export.add_argument(
+        "--max-memory",
+        type=parse_memory_size,
+        help="Hard process-tree memory limit (default: min(4G, 90%% of system RAM)).",
+    )
     wiki_export.add_argument("--force", action="store_true")
     wiki_export.add_argument(
         "--quiet",
@@ -317,36 +335,130 @@ def _discard_broken_stdout() -> None:
 def _view(args: argparse.Namespace) -> None:
     args.root = _workspace_root(args.root)
     args.project_file = _project_file(args.root, args.project_file)
-    index = build_codebase_index(
-        args.root,
-        project_file=args.project_file,
-        index_projects=args.deep_projects or args.layer == "problems",
-        workers=args.workers,
+    store, cache_key, snapshot = _batch_response_cache(
+        args,
+        command="view",
+        layer=args.layer,
+        query=args.query,
+        output_format=args.format,
     )
-    sys.stdout.write(render_layer(index, args.layer, query=args.query, output_format=args.format))
+    cached = store.load(cache_key)
+    if cached is not None:
+        sys.stdout.write(cached.decode("utf-8"))
+        return
+    if args.layer == "references":
+        payload = build_references_payload(
+            args.root,
+            query=args.query,
+            project_file=args.project_file,
+            workers=args.workers,
+        )
+        rendered = render_payload(payload, output_format=args.format)
+    elif args.layer == "metrics":
+        payload = build_metrics_payload(
+            args.root,
+            query=args.query,
+            project_file=args.project_file,
+            workers=args.workers,
+            discovery=snapshot.discovery,
+        )
+        if args.format == "json":
+            rendered = _json_text(payload)
+        else:
+            rendered = render_payload(payload, output_format=args.format)
+    else:
+        index = build_codebase_index(
+            args.root,
+            project_file=args.project_file,
+            index_projects=args.deep_projects,
+            workers=args.workers,
+            use_artifact_cache=False,
+            discovery=snapshot.discovery,
+        )
+        rendered = render_layer(
+            index,
+            args.layer,
+            query=args.query,
+            output_format=args.format,
+        )
+    encoded = rendered.encode("utf-8")
+    _store_batch_response(store, cache_key, encoded)
+    sys.stdout.write(rendered)
 
 
 def _index(args: argparse.Namespace) -> None:
     args.root = _workspace_root(args.root)
     args.project_file = _project_file(args.root, args.project_file)
     output = _resolve_output_path(args.root, args.out)
-    index = build_codebase_index(
-        args.root,
-        project_file=args.project_file,
-        index_projects=args.deep_projects,
-        workers=args.workers,
+    store, cache_key, snapshot = _batch_response_cache(
+        args,
+        command="index",
+        layer="",
+        query="",
+        output_format="json",
     )
-    payload = {
-        "deep_projects": bool(args.deep_projects),
-        "overview": layer_payload(index, "overview"),
-        "projects": layer_payload(index, "projects"),
-        "problems": layer_payload(index, "problems"),
-    }
+    encoded = store.load(cache_key)
+    if encoded is None:
+        index = build_codebase_index(
+            args.root,
+            project_file=args.project_file,
+            index_projects=args.deep_projects,
+            workers=args.workers,
+            use_artifact_cache=False,
+            discovery=snapshot.discovery,
+        )
+        payload = {
+            "deep_projects": bool(args.deep_projects),
+            "overview": layer_payload(index, "overview"),
+            "projects": layer_payload(index, "projects"),
+            "problems": layer_payload(index, "problems"),
+        }
+        encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode(
+            "utf-8"
+        )
+        _store_batch_response(store, cache_key, encoded)
     output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    output.write_bytes(encoded)
     print(output)
 
 
+def _batch_response_cache(
+    args: argparse.Namespace,
+    *,
+    command: str,
+    layer: str,
+    query: str,
+    output_format: str,
+) -> tuple[ResponseCacheStore, ResponseCacheKey, WorkspaceContentSnapshot]:
+    snapshot = workspace_content_snapshot(
+        args.root,
+        project_file=args.project_file,
+    )
+    return (
+        ResponseCacheStore(args.root / ".delphi-lsp" / "response-cache"),
+        ResponseCacheKey(
+            workspace_revision=snapshot.revision,
+            command=command,
+            layer=layer,
+            query=query,
+            output_format=output_format,
+            deep_projects=bool(args.deep_projects),
+        ),
+        snapshot,
+    )
+
+
+def _store_batch_response(
+    store: ResponseCacheStore,
+    key: ResponseCacheKey,
+    payload: bytes,
+) -> None:
+    try:
+        store.store(key, payload)
+    except (OSError, ValueError):
+        pass
+
+
 def _wiki_export(args: argparse.Namespace) -> None:
     args.root = _workspace_root(args.root)
     args.project_file = _project_file(args.root, args.project_file)
@@ -358,6 +470,7 @@ def _wiki_export(args: argparse.Namespace) -> None:
             output,
             project_file=args.project_file,
             workers=args.workers,
+            max_memory_bytes=args.max_memory,
             force=args.force,
             on_progress=renderer,
         )
@@ -501,7 +614,11 @@ def _open_worker_context(
 
 
 def _write_json(payload: object) -> None:
-    sys.stdout.write(json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")
+    sys.stdout.write(_json_text(payload))
+
+
+def _json_text(payload: object) -> str:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n"
 
 
 def _write_warning(warning: str) -> None: