python-constricter 0.4.1__cp314-cp314-win_amd64.whl

constricter/cli/lsp/protocol.py

@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: MIT
+"""The language server protocol's wire format, as `--infer-with`'s connections speak it (see `hints`).
+
+Each message is framed by headers and a blank line, then `Content-Length` bytes of JSON. A hint's
+type is its label after its `:`; basedpyright is asked for variable types only.
+"""
+
+import json
+import shutil
+import subprocess  # asks a checker whether it runs
+import sys
+from pathlib import Path
+from typing import IO, Final, NamedTuple, TypeAlias, cast
+
+Json: TypeAlias = dict[str, "Json"] | list["Json"] | str | int | float | bool | None
+Object: TypeAlias = dict[str, Json]
+_HEADER_END: Final = b"\r\n"
+_LENGTH: Final = "content-length"
+_TYPE_HINT: Final = 1  # `InlayHintKind.Type`
+# Only variable types: argument names and return types are hints too, but nothing here.
+_BASEDPYRIGHT_SECTION: Final = "basedpyright.analysis"
+_BASEDPYRIGHT: Final[Object] = {
+    "inlayHints": {
+        "variableTypes": True,
+        "callArgumentNames": False,
+        "functionReturnTypes": False,
+        "genericTypes": False,
+    },
+}
+
+
+class Server(NamedTuple):
+    """A checker's language server: how it's started, and how many are worth running at once.
+
+    Its executable, the arguments that start it over stdio, and the most of it to run (up to
+    `MAX_SERVERS`). And the memory its servers take, as measured: `memory` bytes each before the
+    files, and `per_byte` more for each byte of them: each server's again where each holds the
+    `whole` program, once among them all where each holds what it's asked about.
+    """
+
+    executable: str
+    args: tuple[str, ...]
+    most: int
+    memory: int
+    per_byte: int
+    whole: bool = False
+
+    def fitting(self, budget: int, size: int) -> int:
+        """Count the servers that fit in `budget` bytes, for files of `size` bytes together.
+
+        Returns:
+          It: none where one doesn't fit.
+
+        """
+        if self.whole:
+            return budget // (self.memory + self.per_byte * size)
+        return max(0, budget - self.per_byte * size) // self.memory
+
+
+MAX_SERVERS: Final = 4  # a checker's, at most: each holds its own copy of the program it checks
+SERVERS: Final = {
+    # One thread each: more servers check more at once (sqlalchemy's hints: 10.8s with one, 7.2s with four).
+    # Each holds the whole program: 1.2 GB for sqlalchemy's 8 MB, 3.4 for pandas's 21.
+    "basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS, 600 << 20, 130, whole=True),
+    # Parallel itself, and still quicker shared out: a hinted check of pandas takes 36s with one, 25s
+    # with two, 22s with three and 21s with four (1.2 GB with one, 1.7 with two, 2.5 with four).
+    "ty": Server("ty", ("server",), MAX_SERVERS, 420 << 20, 38),
+    # Parallel too: pandas's takes 132s with one, 58s with two and 32s with four (2.8 GB, 3.2, 4.2);
+    # the standard library's 39 MB, 4.3 GB with one and 5.6 with four.
+    "pyrefly": Server("pyrefly", ("lsp",), MAX_SERVERS, 480 << 20, 120),
+}
+_PROBE: Final = "--version"  # quick for each; basedpyright-langserver's exits 1 all the same
+_PROBE_TIMEOUT: Final = 10.0  # seconds it may take
+_CANT_RUN: Final = frozenset({126, 127})  # the shell's "can't execute" and "not found": a broken shim's
+
+
+def executable(checker: str) -> str | None:
+    """Find `checker`'s language server: on `PATH`, or beside this Python (an unactivated venv).
+
+    The first that runs (a version manager's shim can be found, and not run), else the first found.
+
+    Returns:
+      Its path, or `None` if it isn't installed.
+
+    """
+    name: str = SERVERS[checker].executable
+    found: list[str] = list(
+        dict.fromkeys(
+            path
+            for path in (shutil.which(name), shutil.which(name, path=str(Path(sys.executable).parent)))
+            if path is not None
+        ),
+    )
+    return next((path for path in found if _runs(path)), found[0] if found else None)
+
+
+def runs(checker: str) -> bool:
+    """Check that `checker` is installed, and runs.
+
+    Returns:
+      Whether it does.
+
+    """
+    found: str | None = executable(checker)
+    return found is not None and _runs(found)
+
+
+def _runs(path: str) -> bool:
+    """Check that an executable runs: its `--version` starts, whatever it answers.
+
+    Returns:
+      Whether it does.
+
+    """
+    try:
+        done: subprocess.CompletedProcess[bytes] = subprocess.run(
+            [path, _PROBE],
+            capture_output=True,
+            timeout=_PROBE_TIMEOUT,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+    return done.returncode not in _CANT_RUN
+
+
+class HintError(Exception):
+    """The type checker couldn't be started, or failed to answer."""
+
+
+class HungError(HintError):
+    """The type checker's server said nothing for too long."""
+
+
+def write(stream: IO[bytes], data: bytes) -> None:
+    """Write `data` to `stream`, and flush it: the server reads it now."""
+    _ = stream.write(data)
+    stream.flush()
+
+
+def settings(section: Json) -> Json:
+    """Answer a `workspace/configuration` item: only basedpyright's inlay hints are set.
+
+    Returns:
+      The section's settings, or `None` for the server's defaults.
+
+    """
+    return _BASEDPYRIGHT if section == _BASEDPYRIGHT_SECTION else None
+
+
+def label(hint: Object) -> str | None:
+    """Read a variable-type hint's type: its label (a string, or parts joined) after its `:`.
+
+    Returns:
+      The type's text, or `None` for any other hint (a parameter's name, a return type).
+
+    """
+    raw: Json = hint.get("label")
+    text: str = (
+        raw
+        if isinstance(raw, str)
+        else "".join(str(cast("Object", part).get("value", "")) for part in cast("list[Json]", raw or []))
+    )
+    if hint.get("kind", _TYPE_HINT) != _TYPE_HINT or not text.startswith(":"):
+        return None
+    return text[1:].strip() or None
+
+
+def message(stream: IO[bytes]) -> Object | None:
+    """Read one framed message: headers, a blank line, then `Content-Length` bytes of JSON.
+
+    Returns:
+      It, or `None` at the end of the stream (or a frame without a length).
+
+    """
+    headers: dict[str, str] = {}
+    line: bytes
+    for line in iter(stream.readline, b""):
+        if line == _HEADER_END:
+            break
+        name: str
+        _separator: str
+        value: str
+        name, _separator, value = line.decode("ascii").partition(":")
+        headers[name.strip().lower()] = value.strip()
+    if _LENGTH not in headers:
+        return None
+    body: bytes = stream.read(int(headers[_LENGTH]))
+    try:
+        return cast("Object", json.loads(body))
+    except ValueError:  # cut off: the server died in the middle of it
+        return None

constricter/cli/lsp/remembered.py

@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: MIT
+"""The files a type checker's server hung on, remembered between runs.
+
+A server that hangs on a file costs the run its timeout twice, and a restart each time: once with
+the file among others, once with it alone. The file's text, with the checker and its build (its
+installed distribution's version, else its executable's size and when it changed), names an empty
+file in the cache; while there's one, the file is left without hints from the start. So is the file
+at that path for `_RECENT` seconds, whatever it holds: a second pass over files the first one fixed
+sees the text it fixed. A new version of the checker, or a changed file after that, is asked again.
+"""
+
+import contextlib
+import hashlib
+import time
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Final
+
+from constricter.fix.index.stubs.installed import cache_directory
+
+_RECENT: Final = 3600.0  # seconds a hang is remembered by the file's path, as well as by its text
+# Where an executable's environment keeps its distributions, from its `bin` (or `Scripts`) directory.
+_DISTRIBUTIONS: Final = ("lib/python*/site-packages", "Lib/site-packages")
+
+
+def _marker(checker: str, command: Sequence[str], named: str) -> Path:
+    """Name what remembers that `checker`, started by `command`, hung on a file `named` so.
+
+    Returns:
+      Its path in the cache.
+
+    """
+    made: str = ""
+    with contextlib.suppress(OSError):
+        made = build(checker, Path(command[0]))
+    key: str = "\0".join((checker, command[0], made, named))
+    return cache_directory().with_name("hung") / hashlib.sha256(key.encode("utf-8", "replace")).hexdigest()
+
+
+def build(checker: str, executable: Path) -> str:
+    """Say which build of `checker` `executable` starts: its distribution's, where its environment has one.
+
+    Not the executable's own where there is: an install writes the script again for the same version.
+
+    Returns:
+      The distribution's folder's name, which holds its version; else the executable's size and when
+      it changed.
+
+    """
+    pattern: str
+    for pattern in _DISTRIBUTIONS:
+        found: list[Path]
+        if found := sorted(executable.parent.parent.glob(f"{pattern}/{checker}-*.dist-info")):
+            return found[0].name
+    return f"{executable.stat().st_size}:{executable.stat().st_mtime_ns}"
+
+
+def hung(checker: str, command: Sequence[str], text: str, path: Path | None = None) -> bool:
+    """Check whether `checker` hung on a file holding `text` in an earlier run, or at `path` of late.
+
+    Returns:
+      Whether it did.
+
+    """
+    if _marker(checker, command, text).exists():
+        return True
+    if path is None:
+        return False
+    with contextlib.suppress(OSError):
+        return time.time() - _marker(checker, command, str(path.absolute())).stat().st_mtime < _RECENT
+    return False
+
+
+def remember(checker: str, command: Sequence[str], text: str, path: Path | None = None) -> None:
+    """Remember that `checker` hung on a file holding `text`, at `path`; not where the cache can't be."""
+    each: str
+    for each in (text,) if path is None else (text, str(path.absolute())):
+        marker: Path = _marker(checker, command, each)
+        with contextlib.suppress(OSError):
+            marker.parent.mkdir(parents=True, exist_ok=True)
+            marker.touch()

constricter/cli/lsp/stored.py

@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: MIT
+"""A type checker's hints for a round of files, kept between runs: asked for once while nothing changes.
+
+A file's hints rest on the whole program the checker reads, so a round's are kept whole, by what
+they rest on (`key`): the checker and its build (see `remembered.build`), the text of every file
+asked about, the workspace's settings files (`pyproject.toml`, `pyrightconfig.json`, ...), the
+installed distributions of the environments a checker may read (the interpreter's, `VIRTUAL_ENV`'s
+and the workspace's own `.venv`), and the code here that asks and reads. Not a file of the
+workspace's that isn't asked about: walking a tree costs as much as a round can save, and a
+project's files are asked about together. The first round of a project at the
+same commit asks again only when one of those changed; a later round's texts are `--fix`'s, so it's
+kept as well, for the next run with the same fixes. A round read from the store keeps the files the
+checker hung on (see `Session.abandoned`) as it found them.
+
+Each entry is a file in the cache, read again within `_KEPT` seconds of its last use; older ones are
+removed as a new one is written. A store that can't be read or written is passed over.
+"""
+
+import contextlib
+import hashlib
+import json
+import os
+import time
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Final, TypeAlias, TypedDict, cast
+
+from constricter.cli.lsp import connection, protocol, remembered
+from constricter.cli.lsp.connection import FileHints, Found
+from constricter.fix.core.known import Offered
+from constricter.fix.index.stubs.installed import cache_directory, search_path
+
+_KEPT: Final = 14 * 24 * 3600.0  # seconds an entry is kept unused
+# A workspace's files that set how a checker reads it.
+_SETTINGS: Final = frozenset(
+    {"pyproject.toml", "setup.cfg", "pyrightconfig.json", "ty.toml", "pyrefly.toml", "mypy.ini"},
+)
+_LOCAL_ENVIRONMENT: Final = ".venv"
+_VARIABLES: Final = ("VIRTUAL_ENV", "CONDA_PREFIX", "PYTHONPATH")
+_SUFFIX: Final = ".json"
+_WHERE: Final = ":"  # between a hint's line and column, in an entry
+# One file's hints as an entry holds them: by `line:column`, the text and the edits' text and imports.
+_Entry: TypeAlias = dict[str, list[str | list[str] | None]]
+
+
+class _Round(TypedDict):
+    """A round as the store holds it: each file's hints, and the files the checker hung on."""
+
+    found: dict[str, _Entry]
+    hung: list[str]
+
+
+def key(checker: str, command: Sequence[str], root: Path, files: Mapping[Path, str]) -> str:
+    """Name a round of `checker`'s hints for `files` (each path's current text), in workspace `root`.
+
+    Returns:
+      A digest of everything they rest on (see the module docstring).
+
+    """
+    build: str = ""
+    with contextlib.suppress(OSError):
+        build = remembered.build(checker, Path(command[0]))
+    texts: list[str] = [checker, *command, build, *(os.environ.get(name, "") for name in _VARIABLES)]
+    parts: list[bytes] = [Path(cast("str", code.__file__)).read_bytes() for code in (connection, protocol)]
+    asked: dict[str, str] = {_relative(path, root): text for path, text in files.items()}
+    name: str
+    for name in sorted(asked):
+        texts += [name, asked[name]]
+    for name in sorted(_SETTINGS):
+        with contextlib.suppress(OSError):
+            parts += [name.encode("utf-8"), (root / name).read_bytes()]
+    texts += _distributions(root)
+    parts += [each.encode("utf-8", "replace") for each in texts]
+    return hashlib.sha256(b"\0".join(f"{len(part)}:".encode() + part for part in parts)).hexdigest()
+
+
+def load(name: str, root: Path, paths: Iterable[Path]) -> tuple[Found, frozenset[Path]] | None:
+    """Read the round stored as `name` (see `key`) for `paths`, in workspace `root`; and mark it used.
+
+    Returns:
+      Each path's hints and those the checker hung on; `None` if there's no such round, or it can't
+      be read.
+
+    """
+    entry: Path = _entry(name)
+    named: dict[str, Path] = {_relative(path, root): path for path in paths}
+    read: tuple[Found, frozenset[Path]]
+    try:
+        read = _read(entry, named)
+    except (OSError, ValueError, KeyError, TypeError, AttributeError):
+        return None
+    with contextlib.suppress(OSError):
+        entry.touch()
+    return read
+
+
+def store(name: str, root: Path, found: Found, hung: frozenset[Path]) -> None:
+    """Keep a round's hints as `name` (see `key`), and remove the entries no run has read of late."""
+    entry: Path = _entry(name)
+    kept: _Round = {
+        "found": {_relative(path, root): _entry_of(hints) for path, hints in found.items()},
+        "hung": sorted(_relative(path, root) for path in hung),
+    }
+    with contextlib.suppress(OSError):
+        entry.parent.mkdir(parents=True, exist_ok=True)
+        written: Path = entry.with_name(f"{entry.name}.{os.getpid()}")
+        _ = written.write_text(json.dumps(kept), encoding="utf-8")
+        _ = written.replace(entry)  # whole, or not at all: another run may read it meanwhile
+        now: float = time.time()
+        old: Path
+        for old in entry.parent.glob(f"*{_SUFFIX}"):
+            with contextlib.suppress(OSError):
+                if now - old.stat().st_mtime > _KEPT:
+                    old.unlink()
+
+
+def _read(entry: Path, named: Mapping[str, Path]) -> tuple[Found, frozenset[Path]]:
+    """Read a stored round's hints for the paths `named` by the store's names for them.
+
+    Returns:
+      Each path's hints, and those the checker hung on.
+
+    """
+    kept: _Round = cast("_Round", json.loads(entry.read_text(encoding="utf-8")))
+    found: Found = {path: _hints(kept["found"].get(each, {})) for each, path in named.items()}
+    return found, frozenset(named[each] for each in kept["hung"])
+
+
+def _entry(name: str) -> Path:
+    """Find where the round `name` is kept.
+
+    Returns:
+      Its path in the cache.
+
+    """
+    return cache_directory().with_name("hints") / f"{name}{_SUFFIX}"
+
+
+def _relative(path: Path, root: Path) -> str:
+    """Write `path` as the store keeps it: under `root`, relative to it.
+
+    Returns:
+      It, as text.
+
+    """
+    absolute: Path = path if path.is_absolute() else Path.cwd() / path
+    with contextlib.suppress(ValueError):
+        return absolute.relative_to(root).as_posix()
+    return absolute.as_posix()
+
+
+def _distributions(root: Path) -> list[str]:
+    """List the installed distributions a checker may read: each environment's, by folder and name.
+
+    Returns:
+      Each `*.dist-info` folder's path, which holds its version, in a fixed order.
+
+    """
+    folders: list[Path] = list(search_path())
+    local: Path = root / _LOCAL_ENVIRONMENT
+    folders += [*sorted(local.glob("lib/python3*/site-packages")), local / "Lib" / "site-packages"]
+    return [str(found) for folder in folders for found in sorted(folder.glob("*.dist-info"))]
+
+
+def _entry_of(hints: FileHints) -> _Entry:
+    """Write one file's hints as an entry holds them.
+
+    Returns:
+      Them, by `line:column`.
+
+    """
+    return {
+        f"{line}{_WHERE}{column}": [
+            text,
+            None if offered is None else offered.text,
+            None if offered is None else list(offered.imports),
+        ]
+        for (line, column), (text, offered) in hints.items()
+    }
+
+
+def _hints(entry: _Entry) -> FileHints:
+    """Read one file's hints from its entry.
+
+    Returns:
+      Them, by line and column.
+
+    """
+    found: FileHints = {}
+    where: str
+    parts: list[str | list[str] | None]
+    for where, parts in entry.items():
+        line: str
+        column: str
+        line, _, column = where.partition(_WHERE)
+        text: str = cast("str", parts[0])
+        shown: str | None = cast("str | None", parts[1])
+        imports: list[str] = cast("list[str] | None", parts[2]) or []
+        found[int(line), int(column)] = (text, None if shown is None else Offered(shown, tuple(imports)))
+    return found