python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/agent_layers.py

@@ -0,0 +1,1410 @@
+from __future__ import annotations
+
+import json
+from collections import OrderedDict
+from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, cast
+
+from .agent_context import AgentContext
+from .agent_metrics import build_path_metrics
+from .agent_protocol import AgentRequest
+from .lsp_server import outline_unit_name
+from .parallel_outline import (
+    OutlineResult,
+    OutlineTask,
+    ParallelBuildStats,
+    run_outline_tasks,
+)
+from .parse_artifacts import OUTLINE_PRODUCER_VERSION
+from .progress import ProgressCallback, ProgressEvent
+from .project_config import ProjectPathConfig
+from .project_discovery import (
+    DelphiProjectDiscovery,
+    discover_delphi_project,
+    populate_workspace_sources,
+)
+from .project_indexer import ProjectIndexer, ProjectIndexResult
+from .semantic import Scope, SourceRange, Symbol, SymbolIndex, SymbolKind
+from .semantic_builder import SemanticModel
+from .source_reader import read_source_text
+
+_MAX_IMPLEMENTATION_ITEMS = 50
+_MAX_IMPLEMENTATION_SOURCE_FILES = 4
+_MAX_REFERENCE_ITEMS = 50
+_MAX_REFERENCE_PAGES = 8
+# A references view performs an internal find followed by one or more trace
+# requests.  Keep that bounded request on the workspace revision opened for it;
+# the 60-second window is deliberately well above the 20-second query budget.
+_REFERENCE_INITIAL_REVISION_REUSE_SECONDS = 60.0
+_REFERENCE_LIMITATIONS = [
+    "Only resolved semantic references are reported; text matches are excluded."
+]
+
+
+@dataclass
+class CodebaseIndex:
+    root: str
+    discovery: DelphiProjectDiscovery
+    models: Mapping[str, SemanticModel]
+    symbol_index: SymbolIndex
+    project_results: dict[str, ProjectIndexResult]
+    parallel_stats: ParallelBuildStats = ParallelBuildStats(0, 0, 0, 0.0, 0)
+
+
+@dataclass(frozen=True, slots=True)
+class _ProjectIndexTask:
+    ordinal: int
+    source_path: str
+    search_paths: tuple[str, ...]
+    include_paths: tuple[str, ...]
+    defines: tuple[str, ...]
+    retain_project_syntax: bool
+    source_roots: tuple[str, ...]
+    project_config: ProjectPathConfig | None
+
+
+@dataclass(frozen=True, slots=True)
+class _ProjectIndexTaskResult:
+    ordinal: int
+    source_path: str
+    result: ProjectIndexResult
+
+
+def _index_project_task(task: _ProjectIndexTask) -> _ProjectIndexTaskResult:
+    indexer = ProjectIndexer(
+        search_paths=task.search_paths,
+        include_paths=task.include_paths,
+        defines=task.defines,
+        source_roots=task.source_roots,
+        project_config=task.project_config,
+        retain_syntax=task.retain_project_syntax,
+    )
+    raw_result = indexer.index(task.source_path)
+    result = raw_result
+    if not task.retain_project_syntax:
+        result = replace(
+            raw_result,
+            parsed_units=[
+                replace(unit, syntax_tree=None, preprocessed=None)
+                for unit in raw_result.parsed_units
+            ],
+        )
+    return _ProjectIndexTaskResult(task.ordinal, task.source_path, result)
+
+
+def build_codebase_index(
+    root: str | Path,
+    *,
+    project_file: str | Path | None = None,
+    index_projects: bool = False,
+    main_projects_only: bool = False,
+    scan_workspace_sources: bool | None = None,
+    retain_project_syntax: bool = True,
+    retain_project_results: bool = True,
+    retain_outline_models: bool = True,
+    project_source_roots: Iterable[str | Path] = (),
+    on_progress: ProgressCallback | None = None,
+    on_project_result: Callable[[str, ProjectIndexResult], None] | None = None,
+    on_outline_result: Callable[[OutlineResult], None] | None = None,
+    workers: int = 0,
+    project_workers: int | None = None,
+    outline_memory_budget_bytes: int | None = None,
+    outline_heartbeat: Callable[[], object] | None = None,
+    outline_force_heartbeat: Callable[[], object] | None = None,
+    outline_active_worker_limit: Callable[[], int] | None = None,
+    use_artifact_cache: bool = True,
+    discovery: DelphiProjectDiscovery | None = None,
+    outline_source_files: Iterable[str] | None = None,
+) -> CodebaseIndex:
+    progress = _MonotonicProgress(on_progress)
+    root_path = Path(root).expanduser().resolve()
+    if scan_workspace_sources is None:
+        scan_workspace_sources = not (main_projects_only and index_projects)
+    if discovery is None:
+        discovery = discover_delphi_project(
+            root_path,
+            project_file=project_file,
+            main_projects_only=main_projects_only,
+            scan_workspace_sources=scan_workspace_sources,
+            on_progress=progress,
+        )
+    project_results: dict[str, ProjectIndexResult] = {}
+    if index_projects and main_projects_only:
+        streamed_sources = (
+            {source.casefold(): source for source in discovery.source_files}
+            if scan_workspace_sources
+            else {}
+        )
+
+        def accept_project_result(
+            path: str,
+            result: ProjectIndexResult,
+        ) -> None:
+            _add_project_result_sources(streamed_sources, result)
+            if on_project_result is not None:
+                on_project_result(path, result)
+
+        project_results = _build_project_results(
+            discovery,
+            retain_project_syntax=retain_project_syntax,
+            retain_results=retain_project_results,
+            source_roots=tuple(project_source_roots),
+            progress=progress,
+            on_progress=on_progress,
+            on_result=accept_project_result,
+            workers=workers if project_workers is None else project_workers,
+        )
+        if retain_project_results:
+            _set_project_source_inventory(
+                discovery,
+                project_results,
+                merge_existing=scan_workspace_sources,
+            )
+        else:
+            _set_source_inventory(discovery, streamed_sources)
+        if not discovery.project_files:
+            populate_workspace_sources(discovery, on_progress=progress)
+
+    models: dict[str, SemanticModel] = {}
+    lines_processed = 0
+    symbols_discovered = 0
+    completed_outlines = 0
+    sources_to_outline = (
+        tuple(discovery.source_files)
+        if outline_source_files is None
+        else tuple(outline_source_files)
+    )
+    outline_tasks = tuple(
+        OutlineTask(
+            ordinal,
+            source,
+            tuple(discovery.defines),
+            False,
+            workspace_root=str(root_path),
+            include_paths=tuple(discovery.include_paths),
+            project_config=discovery.project_config,
+            producer_version=OUTLINE_PRODUCER_VERSION,
+        )
+        for ordinal, source in enumerate(sources_to_outline)
+        if Path(source).suffix.casefold() in {".pas", ".dpr", ".dpk", ".inc"}
+    )
+    outline_task_count = len(outline_tasks)
+
+    def on_outline_complete(result: OutlineResult) -> None:
+        nonlocal completed_outlines, lines_processed, symbols_discovered
+        completed_outlines += 1
+        if not result.read_error:
+            lines_processed += result.lines_processed
+            symbols_discovered += result.symbols_discovered
+        _emit_progress(
+            progress,
+            "outline",
+            result.source_path,
+            len(discovery.source_files),
+            completed_outlines,
+            outline_task_count,
+            "source unreadable" if result.read_error else "source outlined",
+            lines_processed=lines_processed,
+            symbols_discovered=symbols_discovered,
+        )
+        if on_outline_result is not None:
+            on_outline_result(result)
+
+    outline_batch = run_outline_tasks(
+        outline_tasks,
+        configured_workers=workers,
+        use_artifact_cache=use_artifact_cache,
+        memory_budget_bytes=outline_memory_budget_bytes,
+        include_files=discovery.source_files,
+        on_complete=on_outline_complete,
+        retain_results=retain_outline_models,
+        heartbeat=outline_heartbeat,
+        force_heartbeat=outline_force_heartbeat,
+        active_worker_limit=outline_active_worker_limit,
+    )
+    for result in outline_batch.results:
+        if result.read_error or result.model is None:
+            continue
+        models[result.source_path] = result.model
+
+    symbol_index = SymbolIndex()
+    for model in models.values():
+        symbol_index.register_unit(model.unit_scope.name, model.unit_scope)
+    for model in models.values():
+        model.index = symbol_index
+
+    _emit_progress(
+        progress,
+        "relations",
+        str(root_path),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        "semantic relations indexed",
+        lines_processed=lines_processed,
+        symbols_discovered=sum(len(items) for items in symbol_index.name_index.values()),
+    )
+
+    if index_projects and not main_projects_only:
+        project_results = _build_project_results(
+            discovery,
+            retain_project_syntax=retain_project_syntax,
+            retain_results=retain_project_results,
+            source_roots=tuple(project_source_roots),
+            progress=progress,
+            on_progress=on_progress,
+            on_result=on_project_result,
+            workers=workers if project_workers is None else project_workers,
+        )
+
+    index = CodebaseIndex(
+        root=str(root_path),
+        discovery=discovery,
+        models=models,
+        symbol_index=symbol_index,
+        project_results=project_results,
+        parallel_stats=outline_batch.stats,
+    )
+    _emit_progress(
+        progress,
+        "complete",
+        str(root_path),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        "codebase index complete",
+        lines_processed=lines_processed,
+        symbols_discovered=sum(len(items) for items in symbol_index.name_index.values()),
+    )
+    return index
+
+
+def build_units_payload(
+    root: str | Path,
+    *,
+    query: str,
+    project_file: str | Path | None = None,
+    index_projects: bool = False,
+    workers: int = 0,
+    use_artifact_cache: bool = True,
+    discovery: DelphiProjectDiscovery | None = None,
+) -> dict[str, Any]:
+    root_path = Path(root).expanduser().resolve()
+    if discovery is None:
+        discovery = discover_delphi_project(
+            root_path,
+            project_file=project_file,
+            scan_workspace_sources=True,
+        )
+    needle = query.casefold().strip()
+    candidates = tuple(
+        source
+        for source in discovery.source_files
+        if _source_matches_units_query(source, needle)
+    )
+    index = build_codebase_index(
+        root_path,
+        project_file=project_file,
+        index_projects=index_projects,
+        workers=workers,
+        use_artifact_cache=use_artifact_cache,
+        discovery=discovery,
+        outline_source_files=candidates,
+    )
+    return _units_payload(index, query=query)
+
+
+def _source_matches_units_query(source: str, needle: str) -> bool:
+    if not needle or needle in source.casefold():
+        return True
+    try:
+        text = read_source_text(Path(source))
+    except OSError:
+        return False
+    return needle in outline_unit_name(text, source).casefold()
+
+
+def _build_project_results(
+    discovery: DelphiProjectDiscovery,
+    *,
+    retain_project_syntax: bool,
+    retain_results: bool,
+    source_roots: tuple[str | Path, ...],
+    progress: ProgressCallback,
+    on_progress: ProgressCallback | None,
+    on_result: Callable[[str, ProjectIndexResult], None] | None,
+    workers: int,
+) -> dict[str, ProjectIndexResult]:
+    project_results: dict[str, ProjectIndexResult] = {}
+    project_total = len(discovery.project_files)
+    if project_total > 1 and workers != 1:
+        completed = 0
+
+        def on_complete(result: _ProjectIndexTaskResult) -> None:
+            nonlocal completed
+            completed += 1
+            _emit_progress(
+                on_progress,
+                "projects",
+                result.source_path,
+                project_total,
+                completed,
+                project_total,
+                "main project indexed",
+            )
+            if on_result is not None:
+                on_result(result.source_path, result.result)
+
+        tasks = tuple(
+            _ProjectIndexTask(
+                ordinal=ordinal,
+                source_path=project,
+                search_paths=tuple(discovery.search_paths),
+                include_paths=tuple(discovery.include_paths),
+                defines=tuple(discovery.defines),
+                retain_project_syntax=retain_project_syntax,
+                source_roots=tuple(str(root) for root in source_roots),
+                project_config=discovery.project_config,
+            )
+            for ordinal, project in enumerate(discovery.project_files)
+        )
+        batch = run_outline_tasks(
+            cast(Iterable[OutlineTask], tasks),
+            configured_workers=workers,
+            use_artifact_cache=False,
+            on_complete=cast(Callable[[OutlineResult], None], on_complete),
+            retain_results=retain_results,
+            task_runner=cast(
+                Callable[[OutlineTask], OutlineResult],
+                _index_project_task,
+            ),
+        )
+        results = cast(tuple[_ProjectIndexTaskResult, ...], batch.results)
+        return {
+            result.source_path: result.result
+            for result in results
+        } if retain_results else {}
+
+    for ordinal, project in enumerate(discovery.project_files, start=1):
+        indexer = ProjectIndexer(
+            search_paths=discovery.search_paths,
+            include_paths=discovery.include_paths,
+            defines=discovery.defines,
+            on_progress=progress,
+            source_roots=source_roots,
+            project_config=discovery.project_config,
+            retain_syntax=retain_project_syntax,
+        )
+        raw_result = indexer.index(project)
+        result = raw_result
+        if not retain_project_syntax:
+            result = replace(
+                raw_result,
+                parsed_units=[
+                    replace(unit, syntax_tree=None, preprocessed=None)
+                    for unit in raw_result.parsed_units
+                ],
+            )
+        # Do not invoke streaming consumers while the parser/indexer and its
+        # raw syntax/preprocessor graph are still reachable.  The compact
+        # result is self-contained, so releasing those owners first keeps a
+        # sequential project export genuinely bounded.
+        del raw_result
+        del indexer
+        if on_result is not None:
+            on_result(project, result)
+        if retain_results:
+            project_results[project] = result
+        _emit_progress(
+            on_progress,
+            "projects",
+            project,
+            project_total,
+            ordinal,
+            project_total,
+            "main project indexed",
+        )
+    return project_results
+
+
+def _add_project_result_sources(
+    sources: dict[str, str],
+    result: ProjectIndexResult,
+) -> None:
+    for unit in result.parsed_units:
+        resolved = str(Path(unit.path).expanduser().resolve())
+        sources.setdefault(resolved.casefold(), resolved)
+    for include in result.include_files:
+        resolved = str(Path(include.path).expanduser().resolve())
+        sources.setdefault(resolved.casefold(), resolved)
+
+
+def _set_source_inventory(
+    discovery: DelphiProjectDiscovery,
+    sources: Mapping[str, str],
+) -> None:
+    discovery.source_files = sorted(sources.values(), key=str.casefold)
+    discovery.unit_paths = {}
+    for source in discovery.source_files:
+        key = Path(source).stem.casefold()
+        discovery.unit_paths.setdefault(key, []).append(source)
+
+
+def _set_project_source_inventory(
+    discovery: DelphiProjectDiscovery,
+    project_results: dict[str, ProjectIndexResult],
+    *,
+    merge_existing: bool = False,
+) -> None:
+    sources = (
+        {source.casefold(): source for source in discovery.source_files}
+        if merge_existing
+        else {}
+    )
+    for result in project_results.values():
+        for unit in result.parsed_units:
+            resolved = str(Path(unit.path).expanduser().resolve())
+            sources.setdefault(resolved.casefold(), resolved)
+        for include in result.include_files:
+            resolved = str(Path(include.path).expanduser().resolve())
+            sources.setdefault(resolved.casefold(), resolved)
+    discovery.source_files = sorted(sources.values(), key=str.casefold)
+    discovery.unit_paths = {}
+    for source in discovery.source_files:
+        key = Path(source).stem.casefold()
+        discovery.unit_paths.setdefault(key, []).append(source)
+
+
+def _emit_progress(
+    callback: ProgressCallback | None,
+    phase: str,
+    path: str,
+    files_discovered: int,
+    files_completed: int,
+    files_total: int | None,
+    detail: str,
+    *,
+    lines_processed: int = 0,
+    symbols_discovered: int = 0,
+) -> None:
+    if callback is not None:
+        callback(
+            ProgressEvent(
+                phase,
+                "delphi",
+                path,
+                files_discovered,
+                files_completed,
+                files_total,
+                lines_processed,
+                symbols_discovered,
+                0,
+                detail,
+            )
+        )
+
+
+class _MonotonicProgress:
+    def __init__(self, callback: ProgressCallback | None) -> None:
+        self._callback = callback
+        self._files_discovered = 0
+        self._files_completed = 0
+        self._files_total: int | None = None
+        self._lines_processed = 0
+        self._symbols_discovered = 0
+        self._cached_files = 0
+
+    def __call__(self, event: ProgressEvent) -> None:
+        self._files_discovered = max(self._files_discovered, event.files_discovered)
+        self._files_completed = max(self._files_completed, event.files_completed)
+        if event.files_total is not None:
+            self._files_total = max(self._files_total or 0, event.files_total)
+        self._lines_processed = max(self._lines_processed, event.lines_processed)
+        self._symbols_discovered = max(self._symbols_discovered, event.symbols_discovered)
+        self._cached_files = max(self._cached_files, event.cached_files)
+        if self._callback is not None:
+            self._callback(
+                replace(
+                    event,
+                    files_discovered=self._files_discovered,
+                    files_completed=self._files_completed,
+                    files_total=self._files_total,
+                    lines_processed=self._lines_processed,
+                    symbols_discovered=self._symbols_discovered,
+                    cached_files=self._cached_files,
+                )
+            )
+
+
+def render_layer(
+    index: CodebaseIndex,
+    layer: str,
+    *,
+    query: str = "",
+    output_format: str = "markdown",
+) -> str:
+    payload = layer_payload(index, layer, query=query)
+    return render_payload(payload, output_format=output_format)
+
+
+def render_payload(
+    payload: dict[str, Any],
+    *,
+    output_format: str = "markdown",
+) -> str:
+    if output_format == "json":
+        return json.dumps(payload, indent=2, sort_keys=True) + "\n"
+    return _render_markdown(payload)
+
+
+def layer_payload(index: CodebaseIndex, layer: str, *, query: str = "") -> dict[str, Any]:
+    normalized_layer = layer.casefold()
+    if normalized_layer == "overview":
+        return _overview_payload(index)
+    if normalized_layer == "projects":
+        return _projects_payload(index)
+    if normalized_layer == "units":
+        return _units_payload(index, query=query)
+    if normalized_layer == "unit":
+        return _unit_payload(index, query=query)
+    if normalized_layer == "symbols":
+        return _symbols_payload(index, query=query)
+    if normalized_layer == "symbol":
+        return _symbol_payload(index, query=query)
+    if normalized_layer == "implementation":
+        return _implementation_payload(index, query=query)
+    if normalized_layer == "references":
+        return _references_payload(index, query=query)
+    if normalized_layer == "problems":
+        return _problems_payload(index)
+    if normalized_layer == "metrics":
+        return _metrics_payload(index, query=query)
+    raise ValueError(f"Unknown layer: {layer}")
+
+
+def _overview_payload(index: CodebaseIndex) -> dict[str, Any]:
+    return {
+        "layer": "overview",
+        "root": index.root,
+        "project_count": len(index.discovery.project_files),
+        "source_count": len(index.discovery.source_files),
+        "unit_count": len(index.models),
+        "search_paths": index.discovery.search_paths,
+        "include_paths": index.discovery.include_paths,
+        "defines": index.discovery.defines,
+        "problems": [_problem_item(problem) for problem in index.discovery.problems],
+        "projects": index.discovery.project_files,
+    }
+
+
+def _projects_payload(index: CodebaseIndex) -> dict[str, Any]:
+    items: list[dict[str, Any]] = []
+    if index.project_results:
+        for project, result in index.project_results.items():
+            items.append(
+                {
+                    "path": project,
+                    "parsed_units": [
+                        {"name": unit.name, "path": unit.path, "has_error": unit.has_error}
+                        for unit in result.parsed_units
+                    ],
+                    "include_files": [{"name": item.name, "path": item.path} for item in result.include_files],
+                    "not_found_units": result.not_found_units,
+                    "problems": [
+                        {"kind": problem.problem_type.value, "file": problem.file_name, "message": problem.description}
+                        for problem in result.problems
+                    ],
+                }
+            )
+    else:
+        for project in index.discovery.project_files:
+            items.append(
+                {
+                    "path": project,
+                    "parsed_units": [],
+                    "include_files": [],
+                    "not_found_units": [],
+                    "problems": [],
+                    "deep_indexed": False,
+                }
+            )
+    return {"layer": "projects", "root": index.root, "items": items}
+
+
+def _units_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    needle = query.casefold().strip()
+    items = []
+    for file_name, model in sorted(index.models.items(), key=lambda item: item[0].casefold()):
+        if needle and needle not in model.unit_scope.name.casefold() and needle not in file_name.casefold():
+            continue
+        items.append(
+            {
+                "name": model.unit_scope.name,
+                "path": file_name,
+                "symbol_count": sum(1 for _ in _iter_symbols(model.unit_scope, include_unit=False)),
+            }
+        )
+    return {"layer": "units", "root": index.root, "items": items}
+
+
+def _unit_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    needle = query.casefold().strip()
+    items = []
+    for file_name, model in sorted(index.models.items(), key=lambda item: item[0].casefold()):
+        if needle and needle not in model.unit_scope.name.casefold() and needle not in file_name.casefold():
+            continue
+        items.append(
+            {
+                "name": model.unit_scope.name,
+                "path": file_name,
+                "symbols": [_symbol_item(symbol) for symbol in _iter_symbols(model.unit_scope, include_unit=False)],
+            }
+        )
+    return {"layer": "unit", "root": index.root, "items": items}
+
+
+def _symbols_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    needle = query.casefold().strip()
+    symbols = []
+    for symbol in _all_symbols(index):
+        if symbol.kind.value == "unit":
+            continue
+        if needle and needle not in symbol.name.casefold():
+            continue
+        symbols.append(_symbol_item(symbol))
+    symbols.sort(key=lambda item: (item["name"].casefold(), item["path"].casefold(), item["line"]))
+    return {"layer": "symbols", "root": index.root, "query": query, "items": symbols}
+
+
+def _symbol_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    needle = query.casefold().strip()
+    matches = []
+    for symbol in _all_symbols(index):
+        if needle and needle not in symbol.name.casefold():
+            continue
+        item = _symbol_item(symbol)
+        if symbol.member_scope is not None:
+            item["children"] = [_symbol_item(child) for child in _iter_symbols(symbol.member_scope, include_unit=False)]
+        matches.append(item)
+    return {"layer": "symbol", "root": index.root, "query": query, "items": matches[:50]}
+
+
+def _implementation_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    needle = query.casefold().strip()
+    if not needle:
+        return {
+            "layer": "implementation",
+            "root": index.root,
+            "query": query,
+            "items": [],
+            "message": "Pass a class, routine, or member name in query to read focused source.",
+        }
+
+    all_symbols = tuple(_all_symbols(index))
+    matches = [symbol for symbol in all_symbols if symbol.name.casefold() == needle]
+    if not matches:
+        matches = [
+            symbol
+            for symbol in all_symbols
+            if symbol.kind.value != "unit" and needle in symbol.name.casefold()
+        ]
+    unique_matches: dict[tuple[str, str, int, int, str], Symbol] = {}
+    for symbol in matches:
+        unique_matches.setdefault(
+            (
+                symbol.decl_range.file_name,
+                symbol.name.casefold(),
+                symbol.decl_range.start_line,
+                symbol.decl_range.start_col,
+                symbol.kind.value,
+            ),
+            symbol,
+        )
+    matches = sorted(
+        unique_matches.values(),
+        key=lambda symbol: (
+            symbol.decl_range.file_name.casefold(),
+            symbol.decl_range.start_line,
+            symbol.decl_range.start_col,
+            symbol.name.casefold(),
+            symbol.kind.value,
+        ),
+    )[:_MAX_IMPLEMENTATION_ITEMS]
+
+    items: list[dict[str, Any]] = []
+    source_cache = _BoundedSourceCache(_MAX_IMPLEMENTATION_SOURCE_FILES)
+    routine_lookup = _build_routine_lookup(all_symbols)
+    seen_keys: set[tuple[str, str, int, str]] = set()
+    for symbol in matches:
+        item = _implementation_item(symbol, source_cache, routine_lookup)
+        if item is None:
+            continue
+        key = (item["name"].casefold(), item["path"], item["line"], item["kind"])
+        if key in seen_keys:
+            continue
+        seen_keys.add(key)
+        items.append(item)
+    items.sort(key=lambda item: (item["path"].casefold(), item["line"], item["name"].casefold()))
+    return {"layer": "implementation", "root": index.root, "query": query, "items": items}
+
+
+class _BoundedSourceCache(OrderedDict[str, list[str]]):
+    def __init__(self, max_entries: int) -> None:
+        super().__init__()
+        self.max_entries = max_entries
+
+    def get(self, key: str, default: list[str] | None = None) -> list[str] | None:
+        value = super().get(key, default)
+        if key in self:
+            self.move_to_end(key)
+        return value
+
+    def __setitem__(self, key: str, value: list[str]) -> None:
+        super().__setitem__(key, value)
+        self.move_to_end(key)
+        while len(self) > self.max_entries:
+            self.popitem(last=False)
+
+
+@dataclass(frozen=True)
+class _RoutineLookup:
+    by_name: dict[tuple[str, str], tuple[Symbol, ...]]
+    by_owner: dict[tuple[str, str], tuple[Symbol, ...]]
+
+
+def _build_routine_lookup(symbols: Iterable[Symbol]) -> _RoutineLookup:
+    by_name_lists: dict[tuple[str, str], list[Symbol]] = {}
+    by_owner_lists: dict[tuple[str, str], list[Symbol]] = {}
+    for symbol in symbols:
+        if symbol.kind not in _ROUTINE_KINDS:
+            continue
+        normalized_name = symbol.name.casefold()
+        path = symbol.decl_range.file_name
+        by_name_lists.setdefault((path, normalized_name), []).append(symbol)
+        owner, separator, _member = normalized_name.rpartition(".")
+        if separator:
+            by_owner_lists.setdefault((path, owner), []).append(symbol)
+    return _RoutineLookup(
+        by_name={key: tuple(value) for key, value in by_name_lists.items()},
+        by_owner={key: tuple(value) for key, value in by_owner_lists.items()},
+    )
+
+
+def _implementation_item(
+    symbol: Symbol,
+    source_cache: MutableMapping[str, list[str]],
+    routine_lookup: _RoutineLookup,
+) -> dict[str, Any] | None:
+    fragments: list[dict[str, Any]] = []
+    if symbol.kind in {SymbolKind.CLASS, SymbolKind.RECORD, SymbolKind.INTERFACE}:
+        declaration = _source_fragment(symbol.decl_range, "declaration", source_cache)
+        if declaration is not None:
+            fragments.append(declaration)
+        fragments.extend(
+            _implementation_fragments_for_type(
+                symbol, source_cache, routine_lookup
+            )
+        )
+    elif symbol.kind in _ROUTINE_KINDS:
+        fragment = _source_fragment(symbol.decl_range, "implementation", source_cache)
+        if fragment is not None:
+            fragments.append(fragment)
+    else:
+        fragment = _source_fragment(symbol.decl_range, "declaration", source_cache)
+        if fragment is not None:
+            fragments.append(fragment)
+        fragments.extend(
+            _implementation_fragments_for_member(
+                symbol, source_cache, routine_lookup
+            )
+        )
+    if not fragments:
+        return None
+    item = _symbol_item(symbol)
+    item["fragments"] = fragments
+    return item
+
+
+def _implementation_fragments_for_type(
+    symbol: Symbol,
+    source_cache: MutableMapping[str, list[str]],
+    routine_lookup: _RoutineLookup,
+) -> list[dict[str, Any]]:
+    fragments: list[dict[str, Any]] = []
+    seen_ranges: set[SourceRange] = set()
+    candidates = routine_lookup.by_owner.get(
+        (symbol.decl_range.file_name, symbol.name.casefold()), ()
+    )
+    for candidate in candidates:
+        fragment = _source_fragment(candidate.decl_range, "implementation", source_cache)
+        if fragment is None or candidate.decl_range in seen_ranges:
+            continue
+        seen_ranges.add(candidate.decl_range)
+        fragment["symbol"] = candidate.name
+        fragments.append(fragment)
+    fragments.sort(key=lambda item: (item["range"]["start_line"], item["range"]["start_col"]))
+    return fragments
+
+
+def _implementation_fragments_for_member(
+    symbol: Symbol,
+    source_cache: MutableMapping[str, list[str]],
+    routine_lookup: _RoutineLookup,
+) -> list[dict[str, Any]]:
+    owner = symbol.scope.owner
+    if owner is None or owner.kind not in {SymbolKind.CLASS, SymbolKind.RECORD, SymbolKind.INTERFACE}:
+        return []
+    qualified_name = f"{owner.name.casefold()}.{symbol.name.casefold()}"
+    fragments: list[dict[str, Any]] = []
+    candidates = routine_lookup.by_name.get(
+        (symbol.decl_range.file_name, qualified_name), ()
+    )
+    for candidate in candidates:
+        fragment = _source_fragment(candidate.decl_range, "implementation", source_cache)
+        if fragment is not None:
+            fragment["symbol"] = candidate.name
+            fragments.append(fragment)
+    return fragments
+
+
+def _source_fragment(
+    source_range: SourceRange,
+    fragment_kind: str,
+    source_cache: MutableMapping[str, list[str]],
+) -> dict[str, Any] | None:
+    lines = source_cache.get(source_range.file_name)
+    if lines is None:
+        try:
+            text = read_source_text(Path(source_range.file_name))
+        except (OSError, UnicodeError):
+            return None
+        lines = text.splitlines(keepends=True)
+        source_cache[source_range.file_name] = lines
+    if source_range.start_line < 1 or source_range.start_line > len(lines):
+        return None
+    end_line = min(max(source_range.end_line, source_range.start_line), len(lines))
+    snippet = "".join(lines[source_range.start_line - 1 : end_line]).rstrip("\r\n")
+    return {
+        "fragment_kind": fragment_kind,
+        "range": _range_item(source_range),
+        "line_count": end_line - source_range.start_line + 1,
+        "text": snippet,
+    }
+
+
+def _references_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    return build_references_payload(index.root, query=query)
+
+
+def build_references_payload(
+    root: str | Path,
+    *,
+    query: str,
+    project_file: str | Path | None = None,
+    workers: int = 0,
+) -> dict[str, Any]:
+    root_path = Path(root).expanduser().resolve()
+    payload: dict[str, Any] = {
+        "layer": "references",
+        "root": str(root_path),
+        "query": query,
+        "target": None,
+        "items": [],
+        "completeness": "sound_partial",
+        "scan_complete": False,
+        "items_truncated": False,
+        "limitations": list(_REFERENCE_LIMITATIONS),
+    }
+    symbol_query = query.strip()
+    selector_path = ""
+    if "::" in symbol_query:
+        path_part, symbol_part = symbol_query.rsplit("::", 1)
+        if path_part.strip() and symbol_part.strip():
+            selector_path = path_part.strip().replace("\\", "/").casefold()
+            symbol_query = symbol_part.strip()
+    needle = symbol_query.casefold()
+    if not needle:
+        payload["message"] = (
+            "Pass one exact symbol name or qualified name in query to find "
+            "resolved semantic references."
+        )
+        return payload
+
+    context = AgentContext.open(
+        root_path,
+        project_file=project_file,
+        workers=workers,
+        revision_check_interval_seconds=_REFERENCE_INITIAL_REVISION_REUSE_SECONDS,
+        reference_query=symbol_query,
+        reference_path=selector_path,
+    )
+    found = context.handle(
+        AgentRequest(
+            action="find",
+            query=symbol_query,
+            max_items=50,
+            max_chars=40000,
+        )
+    )
+    candidates = _mapping_items(found.result)
+    exact_targets: dict[str, dict[str, object]] = {}
+    for candidate in candidates:
+        name = candidate.get("name")
+        qualified_name = candidate.get("qualified_name")
+        candidate_path = candidate.get("path")
+        target_id = candidate.get("target_id")
+        if not (
+            isinstance(name, str)
+            and isinstance(qualified_name, str)
+            and (name.casefold() == needle or qualified_name.casefold() == needle)
+        ):
+            continue
+        if selector_path and not (
+            isinstance(candidate_path, str)
+            and candidate_path.replace("\\", "/").casefold() == selector_path
+        ):
+            continue
+        if isinstance(target_id, str):
+            exact_targets.setdefault(target_id, dict(candidate))
+
+    logical_targets = _logical_reference_targets(exact_targets.values())
+    if not logical_targets:
+        payload["scan_complete"] = not found.page.truncated
+        payload["message"] = f"No exact symbol target found for query: {query}"
+        return payload
+    if len(logical_targets) > 1:
+        payload["candidates"] = logical_targets
+        payload["items_truncated"] = found.page.truncated
+        payload["message"] = (
+            "The query resolves to multiple symbols; pass one candidate as "
+            "'<path>::<qualified_name>'."
+        )
+        return payload
+
+    target = logical_targets[0]
+    payload["target"] = target
+    target_id = target["target_id"]
+    assert isinstance(target_id, str)
+    trace_request = AgentRequest(
+        action="trace",
+        target_id=target_id,
+        relation="references",
+        max_items=_MAX_REFERENCE_ITEMS,
+        max_chars=40000,
+    )
+    trace_items: list[dict[str, object]] = []
+    remaining_cursor = ""
+    for _page_number in range(_MAX_REFERENCE_PAGES):
+        traced = context.handle(trace_request)
+        trace_items.extend(_mapping_items(traced.result))
+        remaining_cursor = traced.page.next_cursor
+        if not remaining_cursor:
+            break
+        trace_request = replace(trace_request, cursor=remaining_cursor)
+
+    metadata = next(
+        (
+            item
+            for item in trace_items
+            if item.get("item_type") == "relation_metadata"
+        ),
+        None,
+    )
+    reference_items = [
+        _reference_view_item(target, item)
+        for item in trace_items
+        if item.get("item_type") == "relation"
+    ]
+    payload["items"] = _dedupe_reference_view_items(reference_items)
+    if metadata is None:
+        payload["items_truncated"] = True
+        payload["message"] = "Reference metadata was truncated by the bounded trace."
+        return payload
+
+    payload["completeness"] = metadata.get("completeness", "sound_partial")
+    payload["scan_complete"] = metadata.get("scan_complete") is True
+    payload["items_truncated"] = (
+        bool(remaining_cursor) or metadata.get("scan_complete") is not True
+    )
+    for key in (
+        "unresolved_references",
+        "ambiguous_references",
+        "deep_problem_count",
+        "problems",
+        "problems_truncated",
+    ):
+        if key in metadata:
+            payload[key] = metadata[key]
+    limitations = metadata.get("limitations")
+    if isinstance(limitations, list):
+        payload["limitations"] = limitations
+    return payload
+
+
+def _mapping_items(value: object) -> list[dict[str, object]]:
+    if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
+        return []
+    return [dict(item) for item in value if isinstance(item, Mapping)]
+
+
+def _logical_reference_targets(
+    targets: Iterable[dict[str, object]],
+) -> list[dict[str, object]]:
+    logical: dict[tuple[object, object, object], dict[str, object]] = {}
+    for target in targets:
+        key = (
+            target.get("path"),
+            target.get("qualified_name"),
+            target.get("kind"),
+        )
+        logical.setdefault(key, target)
+    return sorted(
+        logical.values(),
+        key=lambda target: (
+            str(target.get("qualified_name", "")).casefold(),
+            str(target.get("path", "")).casefold(),
+            _integer(target.get("line")),
+        ),
+    )
+
+
+def _reference_view_item(
+    target: Mapping[str, object],
+    relation: Mapping[str, object],
+) -> dict[str, object]:
+    evidence = relation.get("evidence")
+    referencing_symbol = relation.get("referencing_symbol")
+    evidence_item = dict(evidence) if isinstance(evidence, Mapping) else {}
+    symbol_item = (
+        dict(referencing_symbol)
+        if isinstance(referencing_symbol, Mapping)
+        else {}
+    )
+    return {
+        "name": target.get("name", ""),
+        "target_id": target.get("target_id", ""),
+        "path": evidence_item.get("path", ""),
+        "line": evidence_item.get("line", 0),
+        "column": evidence_item.get("column", 0),
+        "kind": evidence_item.get("kind", ""),
+        "referencing_symbol": symbol_item,
+    }
+
+
+def _dedupe_reference_view_items(
+    items: Iterable[dict[str, object]],
+) -> list[dict[str, object]]:
+    groups: dict[tuple[object, ...], list[dict[str, object]]] = {}
+    for item in items:
+        key = (
+            item["target_id"],
+            item["path"],
+            item["line"],
+            item["column"],
+            item["kind"],
+        )
+        groups.setdefault(key, []).append(item)
+    unique: list[dict[str, object]] = []
+    seen_exact: set[str] = set()
+    for group in groups.values():
+        rich = [
+            item
+            for item in group
+            if isinstance(item.get("referencing_symbol"), dict)
+            and bool(item["referencing_symbol"])
+        ]
+        for item in rich or group[:1]:
+            exact_key = json.dumps(
+                item,
+                ensure_ascii=False,
+                sort_keys=True,
+                separators=(",", ":"),
+            )
+            if exact_key in seen_exact:
+                continue
+            seen_exact.add(exact_key)
+            unique.append(item)
+    return sorted(
+        unique,
+        key=lambda item: (
+            str(item["path"]).casefold(),
+            _integer(item["line"]),
+            _integer(item["column"]),
+            str(item["kind"]),
+            str(item["target_id"]),
+            json.dumps(
+                item,
+                ensure_ascii=False,
+                sort_keys=True,
+                separators=(",", ":"),
+            ),
+        ),
+    )
+
+
+def _integer(value: object) -> int:
+    return value if isinstance(value, int) else 0
+
+
+def _problems_payload(index: CodebaseIndex) -> dict[str, Any]:
+    items = [_problem_item(problem) for problem in index.discovery.problems]
+    for project, result in index.project_results.items():
+        for problem in result.problems:
+            items.append(
+                {
+                    "kind": problem.problem_type.value,
+                    "origin": project,
+                    "message": problem.description,
+                    "file": problem.file_name,
+                }
+            )
+    return {
+        "layer": "problems",
+        "root": index.root,
+        "items": items,
+        "deep_indexed": bool(index.project_results),
+    }
+
+
+def _metrics_payload(index: CodebaseIndex, *, query: str) -> dict[str, Any]:
+    return _metrics_payload_from_discovery(
+        index.root,
+        index.discovery,
+        query=query,
+    )
+
+
+def build_metrics_payload(
+    root: str | Path,
+    *,
+    query: str = "",
+    project_file: str | Path | None = None,
+    workers: int = 0,
+    discovery: DelphiProjectDiscovery | None = None,
+) -> dict[str, Any]:
+    root_path = Path(root).expanduser().resolve()
+    if discovery is None:
+        discovery = discover_delphi_project(
+            root_path,
+            project_file=project_file,
+            scan_workspace_sources=True,
+        )
+    return _metrics_payload_from_discovery(
+        str(root_path),
+        discovery,
+        query=query,
+        workers=workers,
+    )
+
+
+def _metrics_payload_from_discovery(
+    root: str,
+    discovery: DelphiProjectDiscovery,
+    *,
+    query: str,
+    workers: int = 0,
+) -> dict[str, Any]:
+    project_name = "Workspace"
+    if len(discovery.project_files) == 1:
+        project_name = Path(discovery.project_files[0]).stem
+    metrics = build_path_metrics(
+        root,
+        discovery.source_files,
+        defines=discovery.defines,
+        include_paths=discovery.include_paths,
+        project_config=discovery.project_config,
+        project_name=project_name,
+        workers=workers,
+    )
+    needle = query.casefold().strip()
+    units = [
+        unit.to_mapping(detail=True)
+        for unit in metrics.file_metrics
+        if not needle or needle in unit.name.casefold() or needle in unit.path.casefold()
+    ]
+    return {
+        "layer": "metrics",
+        "root": root,
+        "project": metrics.to_mapping(),
+        "items": units,
+    }
+
+
+def _all_symbols(index: CodebaseIndex) -> Iterable[Symbol]:
+    for model in index.models.values():
+        yield from _iter_symbols(model.unit_scope)
+
+
+def _iter_symbols(scope: Scope, *, include_unit: bool = True, seen: set[int] | None = None) -> Iterable[Symbol]:
+    if seen is None:
+        seen = set()
+    scope_id = id(scope)
+    if scope_id in seen:
+        return
+    seen.add(scope_id)
+    for symbols in scope.symbols.values():
+        for symbol in symbols:
+            if include_unit or symbol.kind.value != "unit":
+                yield symbol
+            if symbol.member_scope is not None:
+                yield from _iter_symbols(symbol.member_scope, include_unit=include_unit, seen=seen)
+
+
+def _symbol_item(symbol: Symbol) -> dict[str, Any]:
+    return {
+        "name": symbol.name,
+        "kind": symbol.kind.value,
+        "path": symbol.decl_range.file_name,
+        "line": symbol.decl_range.start_line,
+        "column": symbol.decl_range.start_col,
+        "range": _range_item(symbol.decl_range),
+        "visibility": symbol.visibility.value,
+        "type": symbol.type_ref.display_name(),
+        "owner": symbol.scope.owner.name if symbol.scope.owner is not None else symbol.scope.name,
+    }
+
+
+_ROUTINE_KINDS = {
+    SymbolKind.PROCEDURE,
+    SymbolKind.FUNCTION,
+    SymbolKind.CONSTRUCTOR,
+    SymbolKind.DESTRUCTOR,
+}
+
+
+def _range_item(source_range: SourceRange) -> dict[str, int | str]:
+    return {
+        "path": source_range.file_name,
+        "start_line": source_range.start_line,
+        "start_col": source_range.start_col,
+        "end_line": source_range.end_line,
+        "end_col": source_range.end_col,
+    }
+
+
+def _problem_item(problem: Any) -> dict[str, str]:
+    return {
+        "kind": problem.kind,
+        "origin": problem.origin,
+        "message": problem.message,
+    }
+
+
+def _render_markdown(payload: dict[str, Any]) -> str:
+    lines = [f"# Delphi Codebase Layer: {payload['layer']}", ""]
+    if payload["layer"] == "overview":
+        lines.extend(
+            [
+                f"- Root: `{payload['root']}`",
+                f"- Projects: {payload['project_count']}",
+                f"- Sources: {payload['source_count']}",
+                f"- Units: {payload['unit_count']}",
+                f"- Defines: {', '.join(payload['defines']) or '(none)'}",
+                "",
+                "## Search Paths",
+            ]
+        )
+        lines.extend(f"- `{path}`" for path in payload["search_paths"])
+        lines.append("")
+        lines.append("## Include Paths")
+        lines.extend(f"- `{path}`" for path in payload["include_paths"])
+    elif payload["layer"] in {"units", "symbols"}:
+        for item in payload["items"]:
+            lines.append(f"- `{item['name']}` {item.get('kind', 'unit')} at `{item['path']}:{item.get('line', 1)}`")
+    elif payload["layer"] == "references":
+        if payload.get("message"):
+            lines.append(str(payload["message"]))
+        for item in payload["items"]:
+            referencing_symbol = item.get("referencing_symbol", {})
+            owner = (
+                referencing_symbol.get("qualified_name", "")
+                if isinstance(referencing_symbol, dict)
+                else ""
+            )
+            lines.append(
+                f"- `{item['name']}` {item['kind']} at "
+                f"`{item['path']}:{item['line']}:{item['column']}`"
+                + (f" from `{owner}`" if owner else "")
+            )
+        lines.extend(
+            [
+                "",
+                f"- Completeness: `{payload['completeness']}`",
+                f"- Scan complete: `{str(payload['scan_complete']).lower()}`",
+                f"- Items truncated: `{str(payload['items_truncated']).lower()}`",
+            ]
+        )
+    elif payload["layer"] in {"unit", "symbol"}:
+        for item in payload["items"]:
+            lines.append(f"## {item['name']}")
+            lines.append(f"- Path: `{item['path']}`")
+            for symbol in item.get("symbols", item.get("children", [])):
+                lines.append(
+                    f"- `{symbol['name']}` {symbol['kind']} at `{symbol['path']}:{symbol['line']}`"
+                )
+    elif payload["layer"] == "implementation":
+        if payload.get("message"):
+            lines.append(payload["message"])
+        for item in payload["items"]:
+            lines.append(f"## {item['name']}")
+            lines.append(f"- Kind: `{item['kind']}`")
+            lines.append(f"- Path: `{item['path']}`")
+            for fragment in item["fragments"]:
+                source_range = fragment["range"]
+                lines.append("")
+                lines.append(
+                    f"### {fragment['fragment_kind']} `{source_range['path']}:{source_range['start_line']}`"
+                )
+                if fragment.get("symbol"):
+                    lines.append(f"- Symbol: `{fragment['symbol']}`")
+                lines.append(f"- Lines: {fragment['line_count']}")
+                lines.append("")
+                lines.append("```pascal")
+                lines.append(fragment["text"])
+                lines.append("```")
+    elif payload["layer"] == "projects":
+        for item in payload["items"]:
+            lines.append(f"## `{item['path']}`")
+            for unit in item["parsed_units"]:
+                lines.append(f"- Unit `{unit['name']}` at `{unit['path']}`")
+    elif payload["layer"] == "problems":
+        for item in payload["items"]:
+            lines.append(f"- {item['kind']}: {item['message']} (`{item.get('origin', '')}`)")
+    elif payload["layer"] == "metrics":
+        project = payload["project"]
+        lines.extend(
+            [
+                f"- Project LOC: {project['total_loc']}",
+                f"- Project LOC with includes: {project['total_loc_with_includes']}",
+                f"- Units: {project['unit_count']}",
+                f"- Maintainability index: {project['maintainability_index']:.2f}",
+            ]
+        )
+        for item in payload["items"]:
+            lines.extend(
+                [
+                    "",
+                    f"## {item['name']}",
+                    f"- Path: `{item['path']}`",
+                    f"- LOC: {item['lines']['total_lines']}",
+                    f"- Cyclomatic maximum: {item['cyclomatic']['maximum']}",
+                    f"- Maintainability index: {item['maintainability_index']:.2f}",
+                    f"- Instability: {item['instability']:.3f}",
+                    f"- Abstractness: {item['abstractness']:.3f}",
+                    f"- Distance: {item['distance']:.3f}",
+                ]
+            )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+__all__ = [
+    "CodebaseIndex",
+    "build_codebase_index",
+    "build_metrics_payload",
+    "build_references_payload",
+    "layer_payload",
+    "render_layer",
+    "render_payload",
+]