python-delphi-lsp 2.0.2__py3-none-any.whl → 2.0.4__py3-none-any.whl

delphi_lsp/__init__.py

@@ -48,6 +48,7 @@ from .semantic_builder import SemanticBuilder, SemanticModel, SemanticProblem
 from .workspace import WorkspaceSemanticResult, build_workspace_semantics
 from .lsp_server import LspWorkspaceState, create_server
 from .project_discovery import DelphiProjectDiscovery, DiscoveryProblem, discover_delphi_project
+from .progress import ProgressCallback, ProgressEvent
 from .project_indexer import (
     GetUnitSyntaxHook,
     IncludeFileInfo,
@@ -136,6 +137,8 @@ __all__ = [
     'DelphiProjectDiscovery',
     'DiscoveryProblem',
     'discover_delphi_project',
+    'ProgressCallback',
+    'ProgressEvent',
     'ProjectIndexer',
     'ProjectIndexResult',
     'ProjectProblemType',

delphi_lsp/_version.py

@@ -1 +1 @@
-__version__ = "2.0.2"
+__version__ = "2.0.4"

delphi_lsp/agent_context.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from bisect import bisect_left, bisect_right
 from collections.abc import Mapping
 from dataclasses import dataclass
+from heapq import heappop, heappush
 import hashlib
 import json
 from pathlib import Path, PureWindowsPath
@@ -14,13 +15,12 @@ from .agent_protocol import (
     AgentResponse,
     ContextBudget,
     Focus,
-    Page,
     make_target_id,
     paginate_items,
 )
 from .agent_metrics import build_workspace_metrics, project_metric_item, unit_metric_item
 from .agent_relations import ProjectRelationIndex, RelationTarget
-from .agent_workspace import AgentWorkspace, unit_display_path, unit_source_path, unit_target_id
+from .agent_workspace import AgentUnit, AgentWorkspace, unit_display_path, unit_source_path, unit_target_id
 from .consts import AttributeName, SyntaxNodeType
 from .lsp_server import build_outline_semantic_model, multiline_string_block_end
 from .nodes import CompoundSyntaxNode, SyntaxNode
@@ -121,6 +121,10 @@ class _SourceDocument:
         self.tokens = tuple(_lex_delphi(text))
         self.token_starts = tuple(token.start for token in self.tokens)
         self.directive_starts = tuple(token.start for token in self.tokens if token.directive)
+        self.declaration_section_indexes = [0]
+        self.declaration_section_checkpoints: dict[int, tuple[str, int, int, int]] = {
+            0: ("", 0, 0, 0)
+        }
         words = [token for token in self.tokens if token.word and not token.escaped]
         self.unit_kind = next(
             (token.value for token in words if token.value in {"unit", "program", "library", "package"}),
@@ -129,6 +133,7 @@ class _SourceDocument:
         implementation = next((token for token in words if token.value == "implementation"), None)
         self.implementation_line = self.line_col(implementation.start)[0] if implementation else 0
         self.routine_spans: dict[int, tuple[int, int] | None] = {}
+        self.routine_token_spans: dict[int, tuple[int, int, int] | None] = {}
         self.parser_spans: dict[str, tuple[int, int] | None] = {}
         self._full_parse_attempted = False
         self._full_parse_result: object | None = None
@@ -230,6 +235,7 @@ class _Registry:
     entries: tuple[_SymbolEntry, ...]
     by_target: dict[str, _SymbolEntry]
     sources: dict[Path, _SourceDocument]
+    ranked_queries: dict[str, tuple[_SymbolEntry, ...]]
 
 
 class AgentContext:
@@ -280,7 +286,12 @@ class AgentContext:
             return self._handle_focus(parsed, revision)
         if parsed.action == "find":
             registry = self._require_registry(revision)
-            items = [entry.card() for entry in _ranked_entries(registry.entries, parsed.query)]
+            ranked = registry.ranked_queries.get(parsed.query)
+            if ranked is None:
+                ranked = tuple(_ranked_entries(registry.entries, parsed.query))
+                registry.ranked_queries.clear()
+                registry.ranked_queries[parsed.query] = ranked
+            items = [entry.card() for entry in ranked]
             return self._response(parsed, revision, items)
         if parsed.action == "inspect":
             registry = self._require_registry(revision)
@@ -293,9 +304,10 @@ class AgentContext:
         previous_project_id = self._workspace.active_project_id
         selected_project_id = requested_project_id or previous_project_id
         if selected_project_id:
-            self._workspace.select_project(selected_project_id)
+            revision = self._workspace._select_project_with_revision(selected_project_id)
+        else:
+            revision = self._workspace.workspace_revision
         current_project_id = self._workspace.active_project_id
-        revision = self._workspace.workspace_revision
 
         if current_project_id != previous_project_id:
             self._registry = None
@@ -683,7 +695,14 @@ def _build_registry(workspace: AgentWorkspace, project_id: str, revision: str) -
             include_paths=workspace.include_paths,
         )
         sources[source_path] = document
-        model = build_outline_semantic_model(text, str(source_path))
+        if workspace.defines:
+            model = build_outline_semantic_model(
+                text,
+                str(source_path),
+                defines=workspace.defines,
+            )
+        else:
+            model = build_outline_semantic_model(text, str(source_path))
         unit_symbols = _collect_raw_symbols(model.unit_scope, unit, source_path, document)
         raw_symbols.extend(_exclude_routine_locals(unit_symbols, document))
 
@@ -782,6 +801,7 @@ def _build_registry(workspace: AgentWorkspace, project_id: str, revision: str) -
         entries=entries_tuple,
         by_target={entry.target_id: entry for entry in entries_tuple},
         sources=sources,
+        ranked_queries={},
     )
 
 
@@ -925,32 +945,47 @@ def _correct_outline_symbol_kind(document: _SourceDocument, symbol: Symbol) -> N
         symbol.kind = SymbolKind.TYPE
 
 
+def _advance_declaration_section(
+    state: tuple[str, int, int, int],
+    token: _Token,
+) -> tuple[str, int, int, int]:
+    section, parentheses, brackets, angles = state
+    if token.directive:
+        return state
+    if token.value == "(":
+        parentheses += 1
+    elif token.value == ")":
+        parentheses = max(0, parentheses - 1)
+    elif token.value == "[":
+        brackets += 1
+    elif token.value == "]":
+        brackets = max(0, brackets - 1)
+    elif token.value == "<":
+        angles += 1
+    elif token.value == ">":
+        angles = max(0, angles - 1)
+    elif not parentheses and not brackets and not angles and token.word:
+        if token.value in {"const", "resourcestring", "threadvar", "type", "var"}:
+            section = token.value
+        elif token.value in {"implementation", "initialization", "finalization"}:
+            section = ""
+    return section, parentheses, brackets, angles
+
+
 def _declaration_section(document: _SourceDocument, offset: int) -> str:
-    section = ""
-    parentheses = 0
-    brackets = 0
-    angles = 0
-    for token in document.tokens[:document.first_token_index(offset)]:
-        if token.directive:
-            continue
-        if token.value == "(":
-            parentheses += 1
-        elif token.value == ")":
-            parentheses = max(0, parentheses - 1)
-        elif token.value == "[":
-            brackets += 1
-        elif token.value == "]":
-            brackets = max(0, brackets - 1)
-        elif token.value == "<":
-            angles += 1
-        elif token.value == ">":
-            angles = max(0, angles - 1)
-        elif not any((parentheses, brackets, angles)) and token.word:
-            if token.value in {"const", "resourcestring", "threadvar", "type", "var"}:
-                section = token.value
-            elif token.value in {"implementation", "initialization", "finalization"}:
-                section = ""
-    return section
+    target_index = document.first_token_index(offset)
+    cached = document.declaration_section_checkpoints.get(target_index)
+    if cached is not None:
+        return cached[0]
+    checkpoint_position = bisect_right(document.declaration_section_indexes, target_index) - 1
+    checkpoint_index = document.declaration_section_indexes[checkpoint_position]
+    state = document.declaration_section_checkpoints[checkpoint_index]
+    for token in document.tokens[checkpoint_index:target_index]:
+        state = _advance_declaration_section(state, token)
+    insert_at = bisect_left(document.declaration_section_indexes, target_index)
+    document.declaration_section_indexes.insert(insert_at, target_index)
+    document.declaration_section_checkpoints[target_index] = state
+    return state[0]
 
 
 def _declared_symbol_name(document: _SourceDocument, symbol: Symbol) -> str:
@@ -1208,20 +1243,47 @@ def _exclude_routine_locals(
         span = _raw_routine_span(raw, document)
         if span is not None:
             containers.append((span[0], span[1], raw))
+    if not containers:
+        return symbols
 
-    filtered: list[_RawSymbol] = []
-    for raw in symbols:
-        offset = document.offset(
-            raw.symbol.decl_range.start_line,
-            raw.symbol.decl_range.start_col,
+    containers.sort(key=lambda item: (item[0], item[1]))
+    positioned = sorted(
+        (
+            document.offset(
+                raw.symbol.decl_range.start_line,
+                raw.symbol.decl_range.start_col,
+            ),
+            order,
+            raw,
         )
-        if any(
-            start < offset < end and raw is not container
-            for start, end, container in containers
+        for order, raw in enumerate(symbols)
+    )
+    active_ends: list[tuple[int, int, int]] = []
+    active_ids: set[int] = set()
+    excluded_orders: set[int] = set()
+    container_index = 0
+    for offset, order, raw in positioned:
+        while (
+            container_index < len(containers)
+            and containers[container_index][0] < offset
         ):
-            continue
-        filtered.append(raw)
-    return filtered
+            _, end, container = containers[container_index]
+            container_id = id(container)
+            heappush(active_ends, (end, container_index, container_id))
+            active_ids.add(container_id)
+            container_index += 1
+        while active_ends and active_ends[0][0] <= offset:
+            _, _, container_id = heappop(active_ends)
+            active_ids.discard(container_id)
+        raw_id = id(raw)
+        if active_ids and (raw_id not in active_ids or len(active_ids) > 1):
+            excluded_orders.add(order)
+
+    return [
+        raw
+        for order, raw in enumerate(symbols)
+        if order not in excluded_orders
+    ]
 
 
 def _raw_routine_span(
@@ -1523,6 +1585,7 @@ def _routine_span(
         document.tokens,
         document.token_starts,
         token_index,
+        cache=document.routine_token_spans,
     )
     span = (start, found[1]) if found is not None else None
     document.routine_spans[start] = span
@@ -1534,6 +1597,7 @@ def _find_routine_token_span(
     token_starts: tuple[int, ...],
     start_index: int,
     *,
+    cache: dict[int, tuple[int, int, int] | None] | None = None,
     depth: int = 0,
 ) -> tuple[int, int, int] | None:
     if depth > 64:
@@ -1541,56 +1605,100 @@ def _find_routine_token_span(
     routine_index = _routine_keyword_index(tokens, start_index)
     if routine_index is None:
         return None
+    spans = cache if cache is not None else {}
+    missing = object()
+    cached = spans.get(routine_index, missing)
+    if cached is not missing:
+        if cached is None:
+            return None
+        return tokens[start_index].start, cached[1], cached[2]
+
     heading_end = _heading_semicolon_index(tokens, routine_index)
     if heading_end is None:
+        spans[routine_index] = None
         return None
 
-    index = heading_end + 1
-    while index < len(tokens):
+    frames: list[list[int]] = [[routine_index, heading_end + 1]]
+
+    def reject_active_frames() -> None:
+        for active_routine_index, _ in frames:
+            spans[active_routine_index] = None
+        frames.clear()
+
+    while frames:
+        frame = frames[-1]
+        frame_routine_index, index = frame
+        if index >= len(tokens):
+            reject_active_frames()
+            continue
+
         token = tokens[index]
         if token.directive:
-            return None
+            reject_active_frames()
+            continue
         if token.word and not token.escaped:
             if token.value in _NO_BODY_DIRECTIVES:
-                return None
+                spans[frame_routine_index] = None
+                frames.pop()
+                continue
             if token.value in {"implementation", "initialization", "finalization"}:
-                return None
+                reject_active_frames()
+                continue
             if (
                 token.value in _STRUCTURED_TYPE_WORDS
                 and _is_structured_type_opener(tokens, index)
             ):
                 if index + 1 < len(tokens) and tokens[index + 1].value == ";":
-                    index += 2
+                    frame[1] = index + 2
                     continue
                 structured_end = _match_end_terminated_block(tokens, index)
                 if structured_end is None:
-                    return None
-                index = bisect_left(token_starts, structured_end)
+                    reject_active_frames()
+                    continue
+                frame[1] = bisect_left(token_starts, structured_end)
                 continue
             if token.value == "end":
-                return None
+                reject_active_frames()
+                continue
             if token.value in {"begin", "asm"}:
                 end = _match_end_terminated_block(tokens, index)
                 if end is None:
-                    return None
+                    reject_active_frames()
+                    continue
                 end_index = bisect_left(token_starts, end)
-                return tokens[start_index].start, end, end_index
-            if token.value in _ROUTINE_WORDS and _is_nested_routine_declaration(tokens, index):
-                nested = _find_routine_token_span(
-                    tokens,
-                    token_starts,
-                    index,
-                    depth=depth + 1,
+                spans[frame_routine_index] = (
+                    tokens[frame_routine_index].start,
+                    end,
+                    end_index,
                 )
+                frames.pop()
+                continue
+            if token.value in _ROUTINE_WORDS and _is_nested_routine_declaration(tokens, index):
+                nested_routine_index = _routine_keyword_index(tokens, index)
+                if nested_routine_index is None:
+                    frame[1] = index + 1
+                    continue
+                nested = spans.get(nested_routine_index, missing)
+                if nested is missing:
+                    nested_heading_end = _heading_semicolon_index(tokens, nested_routine_index)
+                    if nested_heading_end is None:
+                        spans[nested_routine_index] = None
+                        continue
+                    frames.append([nested_routine_index, nested_heading_end + 1])
+                    continue
                 if nested is not None:
-                    index = max(index + 1, nested[2])
+                    frame[1] = max(index + 1, nested[2])
                     continue
                 skipped = _routine_declaration_end_index(tokens, index)
                 if skipped is not None:
-                    index = skipped + 1
+                    frame[1] = skipped + 1
                     continue
-        index += 1
-    return None
+        frame[1] = index + 1
+
+    result = spans.get(routine_index)
+    if result is None:
+        return None
+    return tokens[start_index].start, result[1], result[2]
 
 
 def _routine_keyword_index(

delphi_lsp/agent_layers.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any, Iterable
 import json
 
-from .lsp_server import build_outline_semantic_model, outline_source
+from .lsp_server import build_outline_semantic_model
 from .metrics import analyze_project
 from .project_discovery import DelphiProjectDiscovery, discover_delphi_project
 from .project_indexer import ProjectIndexResult, ProjectIndexer
+from .progress import ProgressCallback, ProgressEvent
 from .semantic import Scope, SourceRange, Symbol, SymbolIndex, SymbolKind
 from .semantic_builder import SemanticModel
 from .source_reader import read_source_text
@@ -28,10 +29,14 @@ def build_codebase_index(
     *,
     project_file: str | Path | None = None,
     index_projects: bool = False,
+    on_progress: ProgressCallback | None = None,
 ) -> CodebaseIndex:
-    discovery = discover_delphi_project(root, project_file=project_file)
+    progress = _MonotonicProgress(on_progress)
+    discovery = discover_delphi_project(root, project_file=project_file, on_progress=progress)
     models: dict[str, SemanticModel] = {}
-    for source in discovery.source_files:
+    lines_processed = 0
+    symbols_discovered = 0
+    for completed, source in enumerate(discovery.source_files, start=1):
         path = Path(source)
         if path.suffix.casefold() not in {".pas", ".dpr", ".dpk", ".inc"}:
             continue
@@ -39,7 +44,25 @@ def build_codebase_index(
             text = read_source_text(path)
         except (OSError, UnicodeError):
             continue
-        models[source] = build_outline_semantic_model(outline_source(text), source)
+        model = build_outline_semantic_model(
+            text,
+            source,
+            defines=discovery.defines,
+        )
+        models[source] = model
+        lines_processed += text.count("\n") + (0 if not text or text.endswith("\n") else 1)
+        symbols_discovered += sum(len(items) for items in model.index.name_index.values())
+        _emit_progress(
+            progress,
+            "outline",
+            source,
+            len(discovery.source_files),
+            completed,
+            len(discovery.source_files),
+            "source outlined",
+            lines_processed=lines_processed,
+            symbols_discovered=symbols_discovered,
+        )
 
     symbol_index = SymbolIndex()
     for model in models.values():
@@ -47,6 +70,18 @@ def build_codebase_index(
     for model in models.values():
         model.index = symbol_index
 
+    _emit_progress(
+        progress,
+        "relations",
+        str(Path(root).expanduser().resolve()),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        "semantic relations indexed",
+        lines_processed=lines_processed,
+        symbols_discovered=sum(len(items) for items in symbol_index.name_index.values()),
+    )
+
     project_results: dict[str, ProjectIndexResult] = {}
     if index_projects:
         for project in discovery.project_files:
@@ -54,16 +89,90 @@ def build_codebase_index(
                 search_paths=discovery.search_paths,
                 include_paths=discovery.include_paths,
                 defines=discovery.defines,
+                on_progress=progress,
             )
             project_results[project] = indexer.index(project)
 
-    return CodebaseIndex(
+    index = CodebaseIndex(
         root=str(Path(root).expanduser().resolve()),
         discovery=discovery,
         models=models,
         symbol_index=symbol_index,
         project_results=project_results,
     )
+    _emit_progress(
+        progress,
+        "complete",
+        str(Path(root).expanduser().resolve()),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        len(discovery.source_files),
+        "codebase index complete",
+        lines_processed=lines_processed,
+        symbols_discovered=sum(len(items) for items in symbol_index.name_index.values()),
+    )
+    return index
+
+
+def _emit_progress(
+    callback: ProgressCallback | None,
+    phase: str,
+    path: str,
+    files_discovered: int,
+    files_completed: int,
+    files_total: int | None,
+    detail: str,
+    *,
+    lines_processed: int = 0,
+    symbols_discovered: int = 0,
+) -> None:
+    if callback is not None:
+        callback(
+            ProgressEvent(
+                phase,
+                "delphi",
+                path,
+                files_discovered,
+                files_completed,
+                files_total,
+                lines_processed,
+                symbols_discovered,
+                0,
+                detail,
+            )
+        )
+
+
+class _MonotonicProgress:
+    def __init__(self, callback: ProgressCallback | None) -> None:
+        self._callback = callback
+        self._files_discovered = 0
+        self._files_completed = 0
+        self._files_total: int | None = None
+        self._lines_processed = 0
+        self._symbols_discovered = 0
+        self._cached_files = 0
+
+    def __call__(self, event: ProgressEvent) -> None:
+        self._files_discovered = max(self._files_discovered, event.files_discovered)
+        self._files_completed = max(self._files_completed, event.files_completed)
+        if event.files_total is not None:
+            self._files_total = max(self._files_total or 0, event.files_total)
+        self._lines_processed = max(self._lines_processed, event.lines_processed)
+        self._symbols_discovered = max(self._symbols_discovered, event.symbols_discovered)
+        self._cached_files = max(self._cached_files, event.cached_files)
+        if self._callback is not None:
+            self._callback(
+                replace(
+                    event,
+                    files_discovered=self._files_discovered,
+                    files_completed=self._files_completed,
+                    files_total=self._files_total,
+                    lines_processed=self._lines_processed,
+                    symbols_discovered=self._symbols_discovered,
+                    cached_files=self._cached_files,
+                )
+            )
 
 
 def render_layer(

delphi_lsp/agent_templates.py

@@ -239,27 +239,30 @@ Prefer `summary` and `declaration`, narrow `max_items` and `max_chars`, and requ
 def _agent_markdown() -> str:
     return """---
 description: Inspect Delphi and Object Pascal codebases through python-delphi-lsp.
-mode: subagent
+mode: all
 temperature: 0
-tools:
-  delphi_codebase: true
-  skill: true
-  lsp: false
-  bash: false
-  read: false
-  glob: false
-  grep: false
-  edit: false
-  write: false
-  task: false
-  webfetch: false
-  todowrite: false
 permission:
-  "*": deny
   delphi_codebase: allow
   skill:
     "*": deny
     python-delphi-lsp: allow
+  lsp: deny
+  bash: deny
+  read: deny
+  glob: deny
+  grep: deny
+  list: deny
+  edit: deny
+  write: deny
+  patch: deny
+  task: deny
+  webfetch: deny
+  websearch: deny
+  question: deny
+  todowrite: deny
+  todoread: deny
+  codebase_map: deny
+  code_guidelines: deny
 ---
 
 Load `python-delphi-lsp` first, then use only `delphi_codebase` for Delphi and

delphi_lsp/agent_workspace.py

@@ -264,6 +264,9 @@ class AgentWorkspace:
         return f"workspace_v2_{fingerprint}"
 
     def select_project(self, project_id: str) -> None:
+        self._select_project_with_revision(project_id)
+
+    def _select_project_with_revision(self, project_id: str) -> str:
         if project_id not in self._project_paths:
             raise AgentProtocolError("project_not_found", f"Project not found: {project_id}.")
         project_path = self._project_paths[project_id]
@@ -280,7 +283,7 @@ class AgentWorkspace:
             fingerprint = _selection_fingerprint(discovery, cached.result, root=self._root)
             if fingerprint == cached.fingerprint:
                 self._activate_project(project_id, discovery, cached.result)
-                return
+                return f"workspace_v2_{fingerprint}"
 
         if project_path is None:
             result = _catalog_workspace_sources(discovery)
@@ -292,11 +295,13 @@ class AgentWorkspace:
                 source_transform=_outline_agent_source,
             )
             result = indexer.index(str(project_path))
+        fingerprint = _selection_fingerprint(discovery, result, root=self._root)
         self._project_cache[project_id] = _ProjectCache(
             result=result,
-            fingerprint=_selection_fingerprint(discovery, result, root=self._root),
+            fingerprint=fingerprint,
         )
         self._activate_project(project_id, discovery, result)
+        return f"workspace_v2_{fingerprint}"
 
     def _activate_project(
         self,