python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/agent_metrics.py

@@ -0,0 +1,526 @@
+from __future__ import annotations
+
+import pickle
+import zlib
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from .agent_protocol import make_target_id
+from .agent_workspace import (
+    AgentWorkspace,
+    unit_display_path,
+    unit_source_path,
+    unit_target_id,
+)
+from .metrics import (
+    MetricProblem,
+    ProjectMetrics,
+    UnitMetrics,
+    aggregate_project_metrics,
+    analyze_include,
+    analyze_unit,
+)
+from .parallel_outline import resolve_worker_count, run_outline_tasks
+from .parser_backend import ParserMode
+from .preprocessor import IncludeLoader
+from .project_config import ProjectPathConfig, workspace_include_loader
+from .source_reader import read_source_text
+
+_LARGE_PROJECT_UNIT_THRESHOLD = 256
+_LARGE_PROJECT_BYTE_THRESHOLD = 16 * 1024 * 1024
+_METRIC_TASK_BATCH_SIZE = 4
+_LARGE_METRIC_TASK_THRESHOLD = 256
+_LARGE_METRIC_BATCHES_PER_WORKER = 12
+MetricProgressCallback = Callable[[int, int, str], None]
+
+
+@dataclass(frozen=True, slots=True)
+class _MetricTask:
+    ordinal: int
+    source_path: str
+    display_path: str
+    unit_id: str
+    defines: tuple[str, ...]
+    include_paths: tuple[str, ...]
+    parser_mode: ParserMode
+    project_config: ProjectPathConfig | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _MetricResult:
+    ordinal: int
+    source_path: str
+    metrics: UnitMetrics | None
+    problem: MetricProblem | None
+
+
+@dataclass(frozen=True, slots=True)
+class _MetricTaskBatch:
+    ordinal: int
+    source_path: str
+    tasks: tuple[_MetricTask, ...]
+    compress_vocabulary: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class _MetricResultBatch:
+    ordinal: int
+    source_path: str
+    results: tuple[_MetricResult, ...]
+    vocabulary_blob: bytes = b""
+
+
+def _analyze_metric_task(task: _MetricTask) -> _MetricResult:
+    return _analyze_metric_task_with_loader(
+        task,
+        workspace_include_loader(
+            task.project_config,
+            task.include_paths,
+        ),
+    )
+
+
+def _analyze_metric_task_with_loader(
+    task: _MetricTask,
+    include_loader: IncludeLoader | None,
+) -> _MetricResult:
+    try:
+        source = read_source_text(Path(task.source_path))
+    except (OSError, UnicodeError):
+        return _MetricResult(
+            task.ordinal,
+            task.source_path,
+            None,
+            MetricProblem(
+                kind="source_unavailable",
+                path=task.display_path,
+                message="Could not read source.",
+            ),
+        )
+    metrics = analyze_unit(
+        source,
+        task.display_path,
+        defines=task.defines,
+        include_paths=task.include_paths,
+        include_loader=include_loader,
+        parser_mode=task.parser_mode,
+    )
+    return _MetricResult(
+        task.ordinal,
+        task.source_path,
+        replace(metrics, unit_id=task.unit_id),
+        None,
+    )
+
+
+def _analyze_metric_batch(batch: _MetricTaskBatch) -> _MetricResultBatch:
+    results: list[_MetricResult] = []
+    operators: set[str] = set()
+    operands: set[str] = set()
+    vocabulary_result_index: int | None = None
+    include_loaders: dict[
+        tuple[ProjectPathConfig | None, tuple[str, ...]],
+        IncludeLoader | None,
+    ] = {}
+    for task in batch.tasks:
+        context = (task.project_config, task.include_paths)
+        if context not in include_loaders:
+            include_loaders[context] = workspace_include_loader(*context)
+        result = _analyze_metric_task_with_loader(task, include_loaders[context])
+        if result.metrics is not None:
+            operators.update(result.metrics._operator_vocabulary)
+            operands.update(result.metrics._operand_vocabulary)
+            if vocabulary_result_index is None:
+                vocabulary_result_index = len(results)
+            result = replace(
+                result,
+                metrics=replace(
+                    result.metrics,
+                    _operator_vocabulary=frozenset(),
+                    _operand_vocabulary=frozenset(),
+                ),
+            )
+        results.append(result)
+    vocabulary_blob = b""
+    if vocabulary_result_index is not None and batch.compress_vocabulary:
+        vocabulary_blob = zlib.compress(
+            pickle.dumps(
+                (frozenset(operators), frozenset(operands)),
+                protocol=pickle.HIGHEST_PROTOCOL,
+            ),
+            level=1,
+        )
+    elif vocabulary_result_index is not None:
+        vocabulary_result = results[vocabulary_result_index]
+        assert vocabulary_result.metrics is not None
+        results[vocabulary_result_index] = replace(
+            vocabulary_result,
+            metrics=replace(
+                vocabulary_result.metrics,
+                _operator_vocabulary=frozenset(operators),
+                _operand_vocabulary=frozenset(operands),
+            ),
+        )
+    return _MetricResultBatch(
+        ordinal=batch.ordinal,
+        source_path=batch.source_path,
+        results=tuple(results),
+        vocabulary_blob=vocabulary_blob,
+    )
+
+
+def _metric_task_batches(
+    tasks: Iterable[_MetricTask],
+    *,
+    batch_count: int | None = None,
+) -> tuple[_MetricTaskBatch, ...]:
+    task_list = tuple(tasks)
+    if not task_list:
+        return ()
+    selected_batch_count = batch_count or (
+        (len(task_list) + _METRIC_TASK_BATCH_SIZE - 1) // _METRIC_TASK_BATCH_SIZE
+    )
+    selected_batch_count = min(max(1, selected_batch_count), len(task_list))
+    buckets: list[list[_MetricTask]] = [[] for _ in range(selected_batch_count)]
+    sizes: dict[str, int] = {}
+    for task in task_list:
+        try:
+            sizes[task.source_path] = Path(task.source_path).stat().st_size
+        except OSError:
+            sizes[task.source_path] = -1
+    ordered = sorted(
+        task_list,
+        key=lambda task: (-sizes[task.source_path], task.ordinal),
+    )
+    bucket_sizes = [0] * selected_batch_count
+    for index, task in enumerate(ordered):
+        if batch_count is None:
+            bucket_index = index % selected_batch_count
+        else:
+            bucket_index = min(
+                range(selected_batch_count),
+                key=lambda candidate: (bucket_sizes[candidate], candidate),
+            )
+        buckets[bucket_index].append(task)
+        bucket_sizes[bucket_index] += max(0, sizes[task.source_path])
+    return tuple(
+        _MetricTaskBatch(
+            ordinal=index,
+            source_path=bucket[0].source_path,
+            tasks=tuple(bucket),
+            compress_vocabulary=batch_count is not None,
+        )
+        for index, bucket in enumerate(buckets)
+        if bucket
+    )
+
+
+def _run_metric_tasks(
+    tasks: Iterable[_MetricTask],
+    *,
+    configured_workers: int,
+    memory_budget_bytes: int | None = None,
+    on_complete: Callable[[_MetricResult], None],
+    cancel_check: Callable[[], None] | None = None,
+) -> None:
+    if cancel_check is not None:
+        cancel_check()
+    task_list = tuple(tasks)
+    batch_count = None
+    if (
+        len(task_list) >= _LARGE_METRIC_TASK_THRESHOLD
+        and configured_workers != 1
+    ):
+        batch_count = (
+            resolve_worker_count(
+                configured_workers,
+                task_count=len(task_list),
+                memory_budget_bytes=memory_budget_bytes,
+            )
+            * _LARGE_METRIC_BATCHES_PER_WORKER
+        )
+
+    def consume_batch(batch: _MetricResultBatch) -> None:
+        if cancel_check is not None:
+            cancel_check()
+        results = list(batch.results)
+        if batch.vocabulary_blob:
+            operators, operands = pickle.loads(zlib.decompress(batch.vocabulary_blob))
+            for index, result in enumerate(results):
+                if result.metrics is None:
+                    continue
+                results[index] = replace(
+                    result,
+                    metrics=replace(
+                        result.metrics,
+                        _operator_vocabulary=operators,
+                        _operand_vocabulary=operands,
+                    ),
+                )
+                break
+        for result in results:
+            if cancel_check is not None:
+                cancel_check()
+            on_complete(result)
+
+    run_outline_tasks(
+        _metric_task_batches(task_list, batch_count=batch_count),
+        configured_workers=configured_workers,
+        memory_budget_bytes=memory_budget_bytes,
+        on_complete=consume_batch,
+        retain_results=False,
+        task_runner=_analyze_metric_batch,
+    )
+
+
+def build_workspace_metrics(
+    workspace: AgentWorkspace,
+    *,
+    workers: int = 0,
+    worker_memory_budget_bytes: int | None = None,
+    cancel_check: Callable[[], None] | None = None,
+) -> ProjectMetrics:
+    units = tuple(workspace.units)
+    source_bytes = 0
+    for unit in units:
+        if cancel_check is not None:
+            cancel_check()
+        try:
+            source_bytes += unit_source_path(workspace.root, unit).stat().st_size
+        except OSError:
+            continue
+    parser_mode = (
+        ParserMode.TOLERANT
+        if len(units) >= _LARGE_PROJECT_UNIT_THRESHOLD
+        or source_bytes >= _LARGE_PROJECT_BYTE_THRESHOLD
+        else ParserMode.STRICT
+    )
+    analyzed_units: list[UnitMetrics] = []
+    include_metrics: list[UnitMetrics] = []
+    problems: list[MetricProblem] = []
+
+    def consume_result(result: _MetricResult) -> None:
+        if result.metrics is not None:
+            analyzed_units.append(result.metrics)
+        if result.problem is not None:
+            problems.append(result.problem)
+
+    _run_metric_tasks(
+        (
+            _MetricTask(
+                ordinal,
+                str(unit_source_path(workspace.root, unit)),
+                unit_display_path(workspace.root, unit),
+                unit_target_id(workspace.root, unit),
+                workspace.defines,
+                workspace.include_paths,
+                parser_mode,
+                workspace.project_config,
+            )
+            for ordinal, unit in enumerate(units)
+        ),
+        configured_workers=workers,
+        memory_budget_bytes=worker_memory_budget_bytes,
+        on_complete=consume_result,
+        cancel_check=cancel_check,
+    )
+
+    include_loc = 0
+    seen_includes: set[str] = set()
+    for include_file in workspace.include_files:
+        if cancel_check is not None:
+            cancel_check()
+        recorded_path = include_file["path"]
+        display_path = workspace.include_display_path(recorded_path)
+        normalized_path = str(display_path).replace("\\", "/").casefold()
+        if normalized_path in seen_includes:
+            continue
+        seen_includes.add(normalized_path)
+        source_path = workspace.include_source_path(recorded_path)
+        try:
+            include_source = read_source_text(source_path)
+            item = replace(
+                analyze_include(include_source, display_path),
+                unit_id=make_target_id(
+                    "unit",
+                    display_path,
+                    Path(include_file["name"]).stem,
+                ),
+            )
+            include_metrics.append(item)
+            include_loc += item.lines.total_lines
+        except (OSError, UnicodeError):
+            problems.append(
+                MetricProblem(
+                    kind="include_unavailable",
+                    path=display_path,
+                    message="Could not read include file.",
+                )
+            )
+
+    project = workspace.active_project
+    metrics = aggregate_project_metrics(
+        analyzed_units,
+        include_loc=include_loc,
+        include_metrics=include_metrics,
+        project_id=workspace.active_project_id,
+        project_name=project.name if project is not None else "",
+    )
+    if problems:
+        metrics = replace(
+            metrics,
+            problems=(
+                *metrics.problems,
+                *sorted(problems, key=lambda item: (item.path.casefold(), item.kind)),
+            ),
+        )
+    return metrics
+
+
+def build_path_metrics(
+    root: str | Path,
+    source_files: Iterable[str | Path],
+    *,
+    defines: Iterable[str] = (),
+    include_paths: Iterable[str | Path] = (),
+    project_config: ProjectPathConfig | None = None,
+    project_name: str = "",
+    workers: int = 0,
+    on_progress: MetricProgressCallback | None = None,
+) -> ProjectMetrics:
+    """Analyze file paths without retaining the complete source corpus in RAM."""
+
+    root_path = Path(root).expanduser().resolve()
+    unique_paths: dict[str, Path] = {}
+    for value in source_files:
+        path = Path(value).expanduser()
+        if not path.is_absolute():
+            path = root_path / path
+        path = path.resolve()
+        if (
+            project_config is not None
+            and project_config.excludes_workspace_path(path)
+        ):
+            continue
+        unique_paths.setdefault(str(path).replace("\\", "/").casefold(), path)
+    unit_paths = [
+        path for path in unique_paths.values() if path.suffix.casefold() in {".pas", ".dpr", ".dpk"}
+    ]
+    include_files = [
+        path for path in unique_paths.values() if path.suffix.casefold() == ".inc"
+    ]
+    source_bytes = 0
+    for path in unit_paths:
+        try:
+            source_bytes += path.stat().st_size
+        except OSError:
+            continue
+    parser_mode = (
+        ParserMode.TOLERANT
+        if len(unit_paths) >= _LARGE_PROJECT_UNIT_THRESHOLD
+        or source_bytes >= _LARGE_PROJECT_BYTE_THRESHOLD
+        else ParserMode.STRICT
+    )
+    define_values = tuple(defines)
+    include_path_values = tuple(str(Path(value)) for value in include_paths)
+    analyzed_units: list[UnitMetrics] = []
+    include_metrics: list[UnitMetrics] = []
+    problems: list[MetricProblem] = []
+    total_files = len(unit_paths) + len(include_files)
+    completed_files = 0
+
+    def consume_result(result: _MetricResult) -> None:
+        nonlocal completed_files
+        if result.metrics is not None:
+            analyzed_units.append(result.metrics)
+        if result.problem is not None:
+            problems.append(result.problem)
+        completed_files += 1
+        if on_progress is not None:
+            on_progress(completed_files, total_files, result.source_path)
+
+    _run_metric_tasks(
+        (
+            _MetricTask(
+                ordinal,
+                str(path),
+                _display_path(path, root_path),
+                "",
+                define_values,
+                include_path_values,
+                parser_mode,
+                project_config,
+            )
+            for ordinal, path in enumerate(unit_paths)
+        ),
+        configured_workers=workers,
+        on_complete=consume_result,
+    )
+
+    include_loc = 0
+    for path in include_files:
+        try:
+            source = read_source_text(path)
+            item = analyze_include(source, _display_path(path, root_path))
+            include_metrics.append(item)
+            include_loc += item.lines.total_lines
+        except (OSError, UnicodeError):
+            problems.append(
+                MetricProblem(
+                    kind="include_unavailable",
+                    path=_display_path(path, root_path),
+                    message="Could not read include file.",
+                )
+            )
+        finally:
+            completed_files += 1
+            if on_progress is not None:
+                on_progress(completed_files, total_files, str(path))
+    if total_files == 0 and on_progress is not None:
+        on_progress(1, 1, "")
+    metrics = aggregate_project_metrics(
+        analyzed_units,
+        include_loc=include_loc,
+        include_metrics=include_metrics,
+        project_name=project_name,
+    )
+    if problems:
+        metrics = replace(
+            metrics,
+            problems=(
+                *metrics.problems,
+                *sorted(problems, key=lambda item: (item.path.casefold(), item.kind)),
+            ),
+        )
+    return metrics
+
+
+def project_metric_item(metrics: ProjectMetrics) -> dict[str, object]:
+    return {"item_type": "project_metrics", **metrics.to_mapping()}
+
+
+def unit_metric_item(metrics: UnitMetrics, *, detail: bool = False) -> dict[str, object]:
+    return {"item_type": "unit_metrics", **metrics.to_mapping(detail=detail)}
+
+
+def _workspace_path(root: Path, value: str) -> Path:
+    path = Path(value).expanduser()
+    if not path.is_absolute():
+        path = root / path
+    return path.resolve()
+
+
+def _display_path(path: Path, root: Path) -> str:
+    try:
+        return path.relative_to(root).as_posix()
+    except ValueError:
+        return path.as_posix()
+
+
+__all__ = [
+    "build_path_metrics",
+    "build_workspace_metrics",
+    "project_metric_item",
+    "unit_metric_item",
+]