python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl

delphi_lsp/source_reader.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def read_source_text(path: Path) -> str:
+    data = path.read_bytes()
+    if data.startswith(b'\xff\xfe') or data.startswith(b'\xfe\xff'):
+        text = data.decode('utf-16')
+    elif data.startswith(b'\xef\xbb\xbf'):
+        text = data.decode('utf-8-sig')
+    else:
+        try:
+            text = data.decode('utf-8')
+        except UnicodeDecodeError:
+            text = data.decode('latin-1')
+    return text.replace('\r\n', '\n').replace('\r', '\n')

delphi_lsp/wiki_external_sort.py

@@ -0,0 +1,392 @@
+"""Streaming JSONL sorter for wiki export artifacts.
+
+The implementation keeps memory usage bounded by writing bounded sorted runs to
+temporary files and performing a streaming heap merge. When a system ``sort``
+command is available, it is preferred to keep the Python heap cold.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from collections.abc import Callable, Iterator
+from heapq import heappop, heappush
+from itertools import count
+from pathlib import Path
+from subprocess import SubprocessError
+from typing import Any, TextIO
+
+SortKey = Callable[[dict[str, Any]], tuple[Any, ...] | list[Any] | Any]
+_SYSTEM_SORT_MEMORY = "16M"
+
+
+class WikiExternalSortError(RuntimeError):
+    """Raised when a JSONL record or key cannot be sorted deterministically."""
+
+
+def sort_jsonl_records(
+    input_path: str | Path,
+    output_path: str | Path,
+    *,
+    key: SortKey | None = None,
+    fields: tuple[str, ...] | list[str] | None = None,
+    max_records_per_run: int = 50_000,
+    max_bytes_per_run: int = 8 * 1024 * 1024,
+    temp_dir: str | Path | None = None,
+    heartbeat: Callable[[], object] | None = None,
+    force_heartbeat: Callable[[], object] | None = None,
+) -> None:
+    """Sort JSONL ``input_path`` into ``output_path`` deterministically.
+
+    Either ``key`` or ``fields`` must be provided. ``fields`` must be an ordered
+    tuple/list of object keys used as a fixed sort tuple.
+    """
+
+    if not isinstance(max_records_per_run, int) or max_records_per_run <= 0:
+        raise ValueError("max_records_per_run must be a positive integer")
+    if not isinstance(max_bytes_per_run, int) or max_bytes_per_run <= 0:
+        raise ValueError("max_bytes_per_run must be a positive integer")
+
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+    if not input_path.is_file():
+        raise FileNotFoundError(f"input JSONL path does not exist: {input_path}")
+    if output_path.exists() and output_path.is_dir():
+        raise ValueError("output_path must be a file path")
+
+    if key is None:
+        if not fields:
+            raise ValueError("key or fields must be provided")
+        field_names = tuple(fields)
+        if not all(isinstance(field, str) for field in field_names):
+            raise TypeError("fields must contain only strings")
+
+        def key(record: dict[str, Any]) -> tuple[Any, ...]:
+            return tuple(record.get(field) for field in field_names)
+
+    try:
+        temp_root = Path(tempfile.mkdtemp(dir=temp_dir))
+        try:
+            if _system_sort_available():
+                # Prefer external sort. The caller receives deterministic output
+                # either from this path or fallback merge below.
+                _system_sort_records(
+                    input_path,
+                    output_path,
+                    key=key,
+                    temp_root=temp_root,
+                    heartbeat=heartbeat,
+                    force_heartbeat=force_heartbeat,
+                )
+                return
+        except (OSError, SubprocessError, TypeError, UnicodeError, ValueError, WikiExternalSortError):
+            # A missing or incompatible platform sorter is exactly what the
+            # bounded Python merge path is for.
+            pass
+
+        runs = _build_sorted_runs(
+            input_path,
+            key=key,
+            max_records_per_run=max_records_per_run,
+            max_bytes_per_run=max_bytes_per_run,
+            temp_root=temp_root,
+            heartbeat=heartbeat,
+        )
+        _merge_sorted_runs(
+            runs,
+            output_path=output_path,
+            key=key,
+            heartbeat=heartbeat,
+        )
+    finally:
+        temp_root_path = locals().get("temp_root")
+        if temp_root_path is not None:
+            shutil.rmtree(temp_root_path, ignore_errors=True)
+
+
+def _system_sort_available() -> bool:
+    return shutil.which("sort") is not None
+
+
+def _system_sort_records(
+    input_path: Path,
+    output_path: Path,
+    *,
+    key: SortKey,
+    temp_root: Path,
+    heartbeat: Callable[[], object] | None,
+    force_heartbeat: Callable[[], object] | None,
+) -> None:
+    keyed_path = temp_root / "system-sort-keyed.jsonl"
+    sorted_path = temp_root / "system-sort-sorted.jsonl"
+
+    key_stream = output_with_keys(input_path, key)
+    with keyed_path.open("w", encoding="utf-8") as handle:
+        for key_text, index, raw in key_stream:
+            handle.write(f"{key_text}\x1f{index:020d}\x1f{raw}\n")
+            if heartbeat is not None and (index + 1) % 50_000 == 0:
+                heartbeat()
+
+    sort_binary = shutil.which("sort") or "sort"
+    command = [
+        sort_binary,
+        "-S",
+        _SYSTEM_SORT_MEMORY,
+        "--parallel",
+        "1",
+        "-T",
+        str(temp_root),
+        "-s",
+        "-t",
+        "\x1f",
+        "-k1,1",
+        "-k2,2",
+        str(keyed_path),
+    ]
+    env = os.environ.copy()
+    env["LC_ALL"] = "C"
+
+    stderr = ""
+    with sorted_path.open("w", encoding="utf-8") as sorted_handle:
+        if heartbeat is None:
+            result = subprocess.run(
+                command,
+                check=False,
+                stdout=sorted_handle,
+                stderr=subprocess.PIPE,
+                text=True,
+                env=env,
+            )
+            returncode = result.returncode
+            stderr = result.stderr
+        else:
+            with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as stderr_handle:
+                process = subprocess.Popen(
+                    command,
+                    stdout=sorted_handle,
+                    stderr=stderr_handle,
+                    text=True,
+                    env=env,
+                )
+                try:
+                    (force_heartbeat or heartbeat)()
+                    while True:
+                        try:
+                            returncode = process.wait(timeout=0.5)
+                            break
+                        except subprocess.TimeoutExpired:
+                            heartbeat()
+                except BaseException:
+                    process.terminate()
+                    process.wait()
+                    raise
+                stderr_handle.seek(0)
+                stderr = stderr_handle.read()
+    if returncode != 0:
+        raise WikiExternalSortError(
+            "system sort failed" + (f": {stderr.strip()}" if stderr else "")
+        )
+
+    temp_output_path = _temp_output_path(output_path)
+    with (
+        temp_output_path.open("w", encoding="utf-8") as output_handle,
+        sorted_path.open("r", encoding="utf-8") as sorted_handle,
+    ):
+        for line in sorted_handle:
+            text = line.rstrip("\n")
+            if not text:
+                continue
+            key_part, _sequence, record = text.split("\x1f", 2)
+            if not key_part:
+                continue
+            output_handle.write(record)
+            output_handle.write("\n")
+    os.replace(temp_output_path, output_path)
+
+
+def _build_sorted_runs(
+    input_path: Path,
+    *,
+    key: SortKey,
+    max_records_per_run: int,
+    max_bytes_per_run: int,
+    temp_root: Path,
+    heartbeat: Callable[[], object] | None,
+) -> list[Path]:
+    runs: list[Path] = []
+    buffer: list[tuple[str, int, str]] = []
+    buffer_bytes = 0
+    next_sequence = count(0)
+
+    def flush() -> None:
+        nonlocal buffer, buffer_bytes
+        if not buffer:
+            return
+        buffer.sort(key=lambda item: (item[0], item[1]))
+        run_path = temp_root / f".wiki-sort-run-{len(runs):04d}.jsonl"
+        with run_path.open("w", encoding="utf-8") as handle:
+            for _record_key, _sequence, raw in buffer:
+                handle.write(raw)
+                handle.write("\n")
+        runs.append(run_path)
+        buffer = []
+        buffer_bytes = 0
+        if heartbeat is not None:
+            heartbeat()
+
+    for raw, payload in _iter_records(input_path):
+        record_key = _serialize_key(_normalize_key(key(payload)))
+        sequence = next(next_sequence)
+        line_bytes = len(raw.encode("utf-8")) + len(record_key.encode("utf-8"))
+        if buffer and (
+            len(buffer) >= max_records_per_run
+            or buffer_bytes + line_bytes > max_bytes_per_run
+        ):
+            flush()
+        buffer.append((record_key, sequence, raw))
+        buffer_bytes += line_bytes
+    flush()
+    return runs
+
+
+def _merge_sorted_runs(
+    runs: list[Path],
+    *,
+    output_path: Path,
+    key: SortKey,
+    heartbeat: Callable[[], object] | None,
+) -> None:
+    class _RunState:
+        __slots__ = ("handle", "index", "line", "path")
+
+        def __init__(self, path: Path, index: int, handle: TextIO) -> None:
+            self.path = path
+            self.index = index
+            self.handle = handle
+            self.line: str | None = None
+
+    def close_run(state: _RunState) -> None:
+        try:
+            state.handle.close()
+        except OSError:
+            pass
+        try:
+            state.path.unlink()
+        except OSError:
+            pass
+
+    entries: list[tuple[str, int, int, _RunState]] = []
+    states: list[_RunState] = []
+
+    def advance(state: _RunState, ordinal: int) -> None:
+        if state.line is None:
+            state.line = state.handle.readline().rstrip("\n")
+        if not state.line:
+            return
+        payload = json.loads(state.line)
+        if not isinstance(payload, dict):
+            raise WikiExternalSortError("sorted run contains a non-object record")
+        record_key = _serialize_key(_normalize_key(key(payload)))
+        heappush(entries, (record_key, state.index, ordinal, state))
+
+    output_temp = _temp_output_path(output_path)
+    order = count(0)
+    try:
+        for index, run_path in enumerate(runs):
+            handle = run_path.open("r", encoding="utf-8")
+            state = _RunState(run_path, index, handle)
+            states.append(state)
+            try:
+                advance(state, next(order))
+            except Exception:
+                close_run(state)
+                raise
+
+        with output_temp.open("w", encoding="utf-8") as output_handle:
+            records_written = 0
+            while entries:
+                _record_key, _run_index, _ordinal, state = heappop(entries)
+                if not state.line:
+                    raise WikiExternalSortError("corrupted merge queue state")
+                output_handle.write(state.line)
+                output_handle.write("\n")
+                records_written += 1
+                if heartbeat is not None and records_written % 50_000 == 0:
+                    heartbeat()
+                state.line = None
+                try:
+                    advance(state, next(order))
+                except Exception:
+                    close_run(state)
+                    raise
+
+        os.replace(output_temp, output_path)
+    finally:
+        for state in states:
+            close_run(state)
+
+
+def _iter_records(input_path: Path) -> Iterator[tuple[str, dict[str, Any]]]:
+    with input_path.open("r", encoding="utf-8") as handle:
+        for line_number, text in enumerate(handle, start=1):
+            raw = text.rstrip("\n")
+            if raw == "":
+                continue
+            try:
+                payload = json.loads(raw)
+            except json.JSONDecodeError as exc:
+                raise WikiExternalSortError(
+                    f"invalid JSON in {input_path} at line {line_number}"
+                ) from exc
+            if not isinstance(payload, dict):
+                raise WikiExternalSortError(
+                    f"record at line {line_number} in {input_path} is not a JSON object"
+                )
+            yield raw, payload
+
+
+def output_with_keys(
+    input_path: Path,
+    key: SortKey,
+) -> Iterator[tuple[str, int, str]]:
+    for sequence, (raw, payload) in enumerate(_iter_records(input_path)):
+        yield _serialize_key(_normalize_key(key(payload))), sequence, raw
+
+
+def _normalize_key(raw_key: Any) -> tuple[Any, ...]:
+    if isinstance(raw_key, tuple):
+        return raw_key
+    if isinstance(raw_key, list):
+        return tuple(raw_key)
+    return (raw_key,)
+
+
+def _serialize_key(value: tuple[Any, ...]) -> str:
+    return "\x1e".join(_serialize_key_part(item) for item in value)
+
+
+def _serialize_key_part(value: Any) -> str:
+    if value is None:
+        return "n"
+    if isinstance(value, bool):
+        return "b1" if value else "b0"
+    if isinstance(value, int):
+        limit = 10**100
+        if abs(value) >= limit:
+            raise WikiExternalSortError("integer sort key exceeds 100 digits")
+        if value < 0:
+            return f"i0{limit + value:0101d}"
+        return f"i1{value:0101d}"
+    if isinstance(value, str):
+        return "s" + value.encode("utf-8").hex()
+    raise WikiExternalSortError(f"unsupported sort-key component: {value!r}")
+
+
+def _temp_output_path(output_path: Path) -> Path:
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temp_name = tempfile.mkstemp(prefix=".wiki-sort-", suffix=".tmp", dir=str(output_path.parent))
+    os.close(fd)
+    return Path(temp_name)