python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/parallel_outline.py

@@ -0,0 +1,662 @@
+from __future__ import annotations
+
+import contextlib
+import multiprocessing
+import os
+from collections.abc import Callable, Iterable
+from concurrent.futures import (
+    FIRST_COMPLETED,
+    Future,
+    ProcessPoolExecutor,
+    as_completed,
+    wait,
+)
+from concurrent.futures.process import BrokenProcessPool
+from dataclasses import dataclass
+from pathlib import Path
+from time import monotonic, perf_counter
+
+from .lsp_server import build_outline_semantic_model
+from .parse_artifacts import (
+    OUTLINE_PRODUCER_VERSION,
+    ArtifactKey,
+    ArtifactStore,
+    CachedOutlineArtifact,
+    artifact_cache_disabled,
+    artifact_key,
+    load_outline_artifact,
+    store_outline_artifact,
+)
+from .project_config import ProjectPathConfig, workspace_include_loader
+from .semantic_builder import SemanticModel
+from .source_reader import read_source_text
+
+_MEBIBYTE = 1024 * 1024
+_WORKER_MEMORY_BYTES = 64 * _MEBIBYTE
+_AUTO_WORKER_LIMIT = 8
+
+
+class ParallelOutlineError(RuntimeError):
+    """Raised when a parallel outline operation cannot complete."""
+
+
+@dataclass(frozen=True)
+class OutlineTask:
+    ordinal: int
+    source_path: str
+    defines: tuple[str, ...]
+    return_text: bool
+    workspace_root: str = ""
+    include_paths: tuple[str, ...] = ()
+    project_config: ProjectPathConfig | None = None
+    producer_version: str = OUTLINE_PRODUCER_VERSION
+
+
+@dataclass(frozen=True)
+class _OutlineWorkItem:
+    ordinal: int
+    source_path: str
+    return_text: bool
+
+
+@dataclass(frozen=True)
+class _OutlineWorkerContext:
+    defines: tuple[str, ...]
+    include_paths: tuple[str, ...]
+    project_config: ProjectPathConfig | None
+    include_files: tuple[str, ...] = ()
+
+
+_outline_worker_context: _OutlineWorkerContext | None = None
+_outline_worker_include_loader: Callable[[str, str], tuple[str, str] | None] | None = (
+    None
+)
+
+
+@dataclass(frozen=True)
+class OutlineResult:
+    ordinal: int
+    source_path: str
+    text: str
+    model: SemanticModel | None
+    lines_processed: int
+    symbols_discovered: int
+    read_error: str = ""
+
+
+@dataclass(frozen=True)
+class ParallelBuildStats:
+    configured_workers: int
+    effective_workers: int
+    files_completed: int
+    elapsed_seconds: float
+    fallbacks: int
+
+
+@dataclass(frozen=True)
+class OutlineBatch:
+    results: tuple[OutlineResult, ...]
+    stats: ParallelBuildStats
+
+
+@dataclass(frozen=True)
+class _PreparedArtifact:
+    store: ArtifactStore
+    key: ArtifactKey
+
+
+def parse_worker_setting(value: str) -> int:
+    if value == "auto":
+        return 0
+    try:
+        workers = int(value)
+    except ValueError as error:
+        raise ValueError(
+            "workers must be 'auto' or an integer from 1 through 32"
+        ) from error
+    if not 1 <= workers <= 32:
+        raise ValueError("workers must be 'auto' or an integer from 1 through 32")
+    return workers
+
+
+def resolve_worker_count(
+    configured_workers: int,
+    *,
+    task_count: int,
+    cpu_count: int | None = None,
+    memory_budget_bytes: int | None = None,
+) -> int:
+    if not 0 <= configured_workers <= 32:
+        raise ValueError("workers must be 'auto' or an integer from 1 through 32")
+    if task_count <= 0:
+        return 0
+    if configured_workers:
+        return min(configured_workers, task_count)
+
+    detected_cpus = os.cpu_count() if cpu_count is None else cpu_count
+    candidates = [task_count, _AUTO_WORKER_LIMIT, max(1, (detected_cpus or 1) - 1)]
+    if memory_budget_bytes is not None:
+        candidates.append(max(1, memory_budget_bytes // _WORKER_MEMORY_BYTES))
+    return min(candidates)
+
+
+def _parse_outline_source(
+    ordinal: int,
+    source_path: str,
+    return_text: bool,
+    *,
+    defines: tuple[str, ...],
+    include_paths: tuple[str, ...],
+    include_loader: Callable[[str, str], tuple[str, str] | None] | None,
+) -> OutlineResult:
+    try:
+        text = read_source_text(Path(source_path))
+    except (OSError, UnicodeError) as error:
+        return OutlineResult(
+            ordinal=ordinal,
+            source_path=source_path,
+            text="",
+            model=None,
+            lines_processed=0,
+            symbols_discovered=0,
+            read_error=str(error),
+        )
+
+    try:
+        model = build_outline_semantic_model(
+            text,
+            source_path,
+            defines=defines,
+            include_paths=include_paths,
+            include_loader=include_loader,
+        )
+    except Exception as error:
+        raise ParallelOutlineError(f"failed to parse {source_path}: {error}") from error
+    return OutlineResult(
+        ordinal=ordinal,
+        source_path=source_path,
+        text=text if return_text else "",
+        model=model,
+        lines_processed=text.count("\n")
+        + (0 if not text or text.endswith(("\n", "\r")) else 1),
+        symbols_discovered=sum(len(items) for items in model.index.name_index.values()),
+    )
+
+
+def _parse_outline_task(task: OutlineTask) -> OutlineResult:
+    return _parse_outline_source(
+        task.ordinal,
+        task.source_path,
+        task.return_text,
+        defines=task.defines,
+        include_paths=task.include_paths,
+        include_loader=workspace_include_loader(
+            task.project_config,
+            task.include_paths,
+        ),
+    )
+
+
+def _initialize_outline_worker(context: _OutlineWorkerContext) -> None:
+    global _outline_worker_context, _outline_worker_include_loader
+    _outline_worker_context = context
+    _outline_worker_include_loader = _inventory_include_loader(context)
+
+
+def _inventory_include_loader(
+    context: _OutlineWorkerContext,
+) -> Callable[[str, str], tuple[str, str] | None] | None:
+    if not context.include_files:
+        return workspace_include_loader(
+            context.project_config,
+            context.include_paths,
+        )
+    known_files = {
+        os.path.normcase(os.path.normpath(path)): path for path in context.include_files
+    }
+    search_paths = tuple(os.path.normpath(path) for path in context.include_paths)
+
+    def load(parent_file: str, include_name: str) -> tuple[str, str] | None:
+        normalized_name = include_name.replace("\\", os.sep).replace("/", os.sep)
+        parent = os.path.dirname(os.path.normpath(parent_file))
+        for base in (parent, *search_paths):
+            candidate = os.path.normcase(
+                os.path.normpath(os.path.join(base, normalized_name))
+            )
+            selected = known_files.get(candidate)
+            if selected is None:
+                continue
+            selected_path = Path(selected)
+            if (
+                context.project_config is not None
+                and context.project_config.excludes_workspace_path(selected_path)
+            ):
+                continue
+            try:
+                return read_source_text(selected_path), str(selected_path)
+            except (OSError, UnicodeError):
+                continue
+        return None
+
+    return load
+
+
+def _parse_outline_work_item(item: _OutlineWorkItem) -> OutlineResult:
+    context = _outline_worker_context
+    if context is None:
+        raise ParallelOutlineError("outline worker context was not initialized")
+    return _parse_outline_source(
+        item.ordinal,
+        item.source_path,
+        item.return_text,
+        defines=context.defines,
+        include_paths=context.include_paths,
+        include_loader=_outline_worker_include_loader,
+    )
+
+
+def _shared_worker_context(
+    tasks: tuple[OutlineTask, ...],
+    *,
+    include_files: tuple[str, ...],
+) -> _OutlineWorkerContext | None:
+    if not tasks:
+        return None
+    first = tasks[0]
+    context = _OutlineWorkerContext(
+        first.defines,
+        first.include_paths,
+        first.project_config,
+        include_files,
+    )
+    if all(
+        task.defines == context.defines
+        and task.include_paths == context.include_paths
+        and task.project_config == context.project_config
+        for task in tasks[1:]
+    ):
+        return context
+    return None
+
+
+def run_outline_tasks(
+    tasks: Iterable[OutlineTask],
+    *,
+    configured_workers: int,
+    use_artifact_cache: bool = True,
+    memory_budget_bytes: int | None = None,
+    cpu_count: int | None = None,
+    on_complete: Callable[[OutlineResult], None] | None = None,
+    retain_results: bool = True,
+    task_runner: Callable[[OutlineTask], OutlineResult] | None = None,
+    include_files: Iterable[str | Path] = (),
+    heartbeat: Callable[[], object] | None = None,
+    force_heartbeat: Callable[[], object] | None = None,
+    active_worker_limit: Callable[[], int] | None = None,
+) -> OutlineBatch:
+    task_list = tuple(tasks)
+    selected_include_files = tuple(str(path) for path in include_files)
+    operation = _parse_outline_task if task_runner is None else task_runner
+    started = perf_counter()
+    effective_workers = resolve_worker_count(
+        configured_workers,
+        task_count=len(task_list),
+        cpu_count=cpu_count,
+        memory_budget_bytes=memory_budget_bytes,
+    )
+    if task_runner is None and use_artifact_cache and not artifact_cache_disabled():
+        cached_results, pending_tasks, prepared = _prepare_artifacts(task_list)
+    else:
+        cached_results, pending_tasks, prepared = [], task_list, {}
+    accepted: list[OutlineResult] = []
+    cached_completed = len(cached_results)
+    for result in cached_results:
+        if on_complete is not None:
+            on_complete(result)
+        if retain_results:
+            accepted.append(result)
+
+    if not pending_tasks:
+        return _batch(
+            configured_workers,
+            effective_workers,
+            accepted,
+            started,
+            files_completed=cached_completed,
+            fallbacks=0,
+        )
+
+    def complete(result: OutlineResult) -> None:
+        artifact = prepared.get((result.ordinal, result.source_path))
+        if artifact is not None and not result.read_error and result.model is not None:
+            with contextlib.suppress(Exception):
+                store_outline_artifact(
+                    artifact.store,
+                    artifact.key,
+                    result.source_path,
+                    result.model,
+                    lines_processed=result.lines_processed,
+                    symbols_discovered=result.symbols_discovered,
+                )
+        if on_complete is not None:
+            on_complete(result)
+
+    dispatch_workers = min(effective_workers, len(pending_tasks))
+    if dispatch_workers <= 1:
+        results, serial_completed = _run_serial(
+            pending_tasks,
+            complete,
+            retain_results,
+            operation,
+        )
+        accepted.extend(results)
+        _prune_artifacts(prepared)
+        return _batch(
+            configured_workers,
+            effective_workers,
+            accepted,
+            started,
+            files_completed=cached_completed + serial_completed,
+            fallbacks=0,
+        )
+
+    worker_context = (
+        _shared_worker_context(
+            pending_tasks,
+            include_files=selected_include_files,
+        )
+        if task_runner is None
+        else None
+    )
+    futures: dict[Future[OutlineResult], OutlineTask] = {}
+    executor: ProcessPoolExecutor | None = None
+    parallel_completed = 0
+    try:
+        context = multiprocessing.get_context("spawn")
+        if worker_context is None:
+            active_executor = ProcessPoolExecutor(
+                max_workers=dispatch_workers,
+                mp_context=context,
+            )
+        else:
+            active_executor = ProcessPoolExecutor(
+                max_workers=dispatch_workers,
+                mp_context=context,
+                initializer=_initialize_outline_worker,
+                initargs=(worker_context,),
+            )
+        executor = active_executor
+
+        def submit(task: OutlineTask) -> Future[OutlineResult]:
+            if worker_context is None:
+                return active_executor.submit(operation, task)
+            return active_executor.submit(
+                _parse_outline_work_item,
+                _OutlineWorkItem(
+                    task.ordinal,
+                    task.source_path,
+                    task.return_text,
+                ),
+            )
+
+        task_iterator = iter(_largest_sources_first(pending_tasks))
+        tasks_exhausted = False
+
+        def desired_window() -> int:
+            if active_worker_limit is None:
+                return dispatch_workers
+            return max(1, min(dispatch_workers, active_worker_limit()))
+
+        def fill_submission_window() -> None:
+            nonlocal tasks_exhausted
+            while not tasks_exhausted and len(futures) < desired_window():
+                try:
+                    task = next(task_iterator)
+                except StopIteration:
+                    tasks_exhausted = True
+                    return
+                futures[submit(task)] = task
+
+        fill_submission_window()
+        next_heartbeat = monotonic()
+        if heartbeat is not None:
+            (force_heartbeat or heartbeat)()
+            next_heartbeat = monotonic() + 0.5
+        while futures:
+            if heartbeat is None:
+                future = next(iter(as_completed(tuple(futures))))
+            else:
+                while True:
+                    timeout = max(0.0, next_heartbeat - monotonic())
+                    completed, _pending = wait(
+                        tuple(futures),
+                        timeout=timeout,
+                        return_when=FIRST_COMPLETED,
+                    )
+                    now = monotonic()
+                    if now >= next_heartbeat:
+                        heartbeat()
+                        next_heartbeat = now + 0.5
+                    if completed:
+                        future = next(iter(completed))
+                        break
+            task = futures.pop(future)
+            if heartbeat is not None and parallel_completed == 0:
+                (force_heartbeat or heartbeat)()
+            try:
+                result = future.result()
+            except ParallelOutlineError:
+                raise
+            except Exception as error:
+                wrapped = ParallelOutlineError(
+                    f"parallel outline execution failed for {task.source_path}: {error}"
+                )
+                if configured_workers == 0 and parallel_completed == 0:
+                    _cancel_futures(futures)
+                    _shutdown_failed_executor(executor)
+                    results, serial_completed = _run_serial(
+                        pending_tasks,
+                        complete,
+                        retain_results,
+                        operation,
+                    )
+                    accepted.extend(results)
+                    _prune_artifacts(prepared)
+                    return _batch(
+                        configured_workers,
+                        1,
+                        accepted,
+                        started,
+                        files_completed=cached_completed + serial_completed,
+                        fallbacks=1,
+                    )
+                raise wrapped from error
+            parallel_completed += 1
+            complete(result)
+            if retain_results:
+                accepted.append(result)
+            fill_submission_window()
+    except ParallelOutlineError:
+        _cancel_futures(futures)
+        _shutdown_failed_executor(executor)
+        raise
+    except (OSError, BrokenProcessPool) as error:
+        _cancel_futures(futures)
+        _shutdown_failed_executor(executor)
+        if configured_workers == 0 and parallel_completed == 0:
+            results, serial_completed = _run_serial(
+                pending_tasks,
+                complete,
+                retain_results,
+                operation,
+            )
+            accepted.extend(results)
+            _prune_artifacts(prepared)
+            return _batch(
+                configured_workers,
+                1,
+                accepted,
+                started,
+                files_completed=cached_completed + serial_completed,
+                fallbacks=1,
+            )
+        raise ParallelOutlineError(
+            f"parallel outline execution failed: {error}"
+        ) from error
+    except BaseException:
+        _cancel_futures(futures)
+        _shutdown_failed_executor(executor)
+        raise
+
+    assert executor is not None
+    executor.shutdown(wait=True, cancel_futures=False)
+    _prune_artifacts(prepared)
+
+    return _batch(
+        configured_workers,
+        effective_workers,
+        accepted,
+        started,
+        files_completed=cached_completed + parallel_completed,
+        fallbacks=0,
+    )
+
+
+def _prepare_artifacts(
+    tasks: tuple[OutlineTask, ...],
+) -> tuple[
+    list[OutlineResult],
+    tuple[OutlineTask, ...],
+    dict[tuple[int, str], _PreparedArtifact],
+]:
+    cached_results: list[OutlineResult] = []
+    pending_tasks: list[OutlineTask] = []
+    prepared: dict[tuple[int, str], _PreparedArtifact] = {}
+    stores: dict[str, ArtifactStore] = {}
+    for task in tasks:
+        if not task.workspace_root:
+            pending_tasks.append(task)
+            continue
+        text = ""
+        cached: CachedOutlineArtifact | None = None
+        cache_check_succeeded = False
+        with contextlib.suppress(Exception):
+            text = read_source_text(Path(task.source_path))
+            key = artifact_key(
+                task.workspace_root,
+                task.source_path,
+                text,
+                task.defines,
+                include_paths=task.include_paths,
+                project_config=task.project_config,
+                producer_version=task.producer_version,
+            )
+            store = stores.setdefault(
+                task.workspace_root,
+                ArtifactStore(Path(task.workspace_root) / ".delphi-lsp" / "artifacts"),
+            )
+            prepared[(task.ordinal, task.source_path)] = _PreparedArtifact(store, key)
+            cached = load_outline_artifact(store, key, task.source_path)
+            cache_check_succeeded = True
+        if not cache_check_succeeded:
+            pending_tasks.append(task)
+            continue
+        if cached is None:
+            pending_tasks.append(task)
+            continue
+        cached_results.append(
+            OutlineResult(
+                ordinal=task.ordinal,
+                source_path=task.source_path,
+                text=text if task.return_text else "",
+                model=cached.model,
+                lines_processed=cached.lines_processed,
+                symbols_discovered=cached.symbols_discovered,
+            )
+        )
+    return cached_results, tuple(pending_tasks), prepared
+
+
+def _prune_artifacts(
+    prepared: dict[tuple[int, str], _PreparedArtifact],
+) -> None:
+    live_keys_by_root: dict[Path, set[str]] = {}
+    stores: dict[Path, ArtifactStore] = {}
+    for artifact in prepared.values():
+        live_keys_by_root.setdefault(artifact.store.root, set()).add(
+            artifact.key.cache_key
+        )
+        stores[artifact.store.root] = artifact.store
+    for root, live_keys in live_keys_by_root.items():
+        with contextlib.suppress(Exception):
+            stores[root].prune(live_keys)
+
+
+def _largest_sources_first(tasks: tuple[OutlineTask, ...]) -> tuple[OutlineTask, ...]:
+    sizes: dict[str, int] = {}
+    for task in tasks:
+        try:
+            sizes[task.source_path] = Path(task.source_path).stat().st_size
+        except OSError:
+            sizes[task.source_path] = -1
+    return tuple(
+        sorted(
+            tasks,
+            key=lambda task: (
+                -sizes[task.source_path],
+                task.ordinal,
+            ),
+        )
+    )
+
+
+def _run_serial(
+    tasks: tuple[OutlineTask, ...],
+    on_complete: Callable[[OutlineResult], None] | None,
+    retain_results: bool,
+    operation: Callable[[OutlineTask], OutlineResult],
+) -> tuple[list[OutlineResult], int]:
+    results = []
+    for task in tasks:
+        result = operation(task)
+        if on_complete is not None:
+            on_complete(result)
+        if retain_results:
+            results.append(result)
+    return results, len(tasks)
+
+
+def _cancel_futures(futures: dict[Future[OutlineResult], OutlineTask]) -> None:
+    for future in futures:
+        future.cancel()
+
+
+def _shutdown_failed_executor(executor: ProcessPoolExecutor | None) -> None:
+    if executor is None:
+        return
+    try:
+        executor.shutdown(wait=False, cancel_futures=True)
+    except BaseException:
+        pass
+
+
+def _batch(
+    configured_workers: int,
+    effective_workers: int,
+    results: list[OutlineResult],
+    started: float,
+    *,
+    files_completed: int,
+    fallbacks: int,
+) -> OutlineBatch:
+    ordered_results = tuple(sorted(results, key=lambda result: result.ordinal))
+    return OutlineBatch(
+        results=ordered_results,
+        stats=ParallelBuildStats(
+            configured_workers=configured_workers,
+            effective_workers=effective_workers,
+            files_completed=files_completed,
+            elapsed_seconds=perf_counter() - started,
+            fallbacks=fallbacks,
+        ),
+    )