python-constricter 0.4.1__cp314-cp314-win_amd64.whl

constricter/fix/index/stubs/installed.py

@@ -0,0 +1,425 @@
+# SPDX-License-Identifier: MIT
+"""Installed packages' declared types, for `--fix` to type calls into them as it does checked files'.
+
+`with_installed` adds to the cross-file index the modules the checked files import from installed
+packages that declare their types: a stub package (`numpy-stubs`), a `py.typed` package's stubs
+(`.pyi`) or source, or a lone stub module. Each is found on the search path (`search_path`: the
+running interpreter's, and the active virtual environment's) as the import system would, and read
+as `modules.read` reads a checked file, but never checked or fixed; the modules they import from
+their own package (re-exports) are read too, up to `LIMIT` modules in all. Each module read is
+cached (`cached`) under the user's cache directory, by its file's path, size and modification time
+and constricter's version and reading code: a package upgraded, or a changed constricter, reads it
+again. Each version and reading code has its own part of the cache, swept as it's written to, a day
+apart at most (`swept`): what's unread for a month goes, and what's over its cap.
+"""
+
+import contextlib
+import hashlib
+import os
+import pickle  # the cache: written and read by constricter alone, in the user's own directory
+import sys
+import sysconfig
+import tempfile
+import time
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import replace
+from functools import lru_cache
+from pathlib import Path
+from typing import Final, cast
+
+from constricter import __version__
+from constricter.fix.index.modules import STUB, SUFFIX, Index, Memo, Module, read
+
+LIMIT: Final = 2000  # the most installed modules one run reads
+_HOPS: Final = 5  # re-exports followed to a type variable
+_STUBS: Final = "-stubs"
+_TYPED: Final = "py.typed"
+_PACKAGE: Final = "__init__"
+_UNDERSCORE: Final = "_"
+_VIRTUAL_ENV: Final = "VIRTUAL_ENV"
+_STDLIB: Final = sys.stdlib_module_names
+_CACHE_HOME: Final = "XDG_CACHE_HOME"
+_LOCAL_APP_DATA: Final = "LOCALAPPDATA"  # Windows'
+_CACHE: Final = "constricter"
+_ENTRIES: Final = "*.pickle"
+_SWEPT: Final = ".swept"  # in a build's part of the cache: touched when that build last swept it
+_DAY: Final = 86400.0  # the seconds between a build's sweeps
+_KEPT: Final = 30 * _DAY  # the seconds an entry is kept unread, and another build's part unswept
+_CAP: Final = 256 << 20  # the bytes a build's entries may take: past it, the longest unread go
+
+
+def search_path() -> tuple[Path, ...]:
+    """List the directories installed packages are found in: the interpreter's, then the active environment's.
+
+    The environment in `VIRTUAL_ENV`, where the checked code runs, even when constricter runs in
+    another (pre-commit's, pipx's).
+
+    Returns:
+      Them, the standard library's own left out.
+
+    """
+    stdlib: set[Path] = {Path(sysconfig.get_paths()[key]).resolve() for key in ("stdlib", "platstdlib")}
+    active: str | None = os.environ.get(_VIRTUAL_ENV)
+    extra: list[str] = (
+        [] if active is None else [str(p) for p in sorted(Path(active).glob("lib/python3*/site-packages"))]
+    )
+    extra += [] if active is None else [str(Path(active) / "Lib" / "site-packages")]  # Windows
+    found: dict[Path, None] = {}
+    entry: str
+    for entry in (*sys.path, *extra):
+        path: Path = Path(entry or ".").resolve()
+        if path.is_dir() and path not in stdlib:
+            found[path] = None
+    return tuple(found)
+
+
+def with_installed(catalog: Index, search: Sequence[Path]) -> Index:
+    """Add the installed modules the checked files import to `catalog` (see the module docstring).
+
+    Returns:
+      The index, with them.
+
+    """
+    wanted: list[str] = _imported(catalog.modules.values())  # grows as it's walked
+    found: dict[str, Module] = {}
+    seen: set[str] = set(catalog.modules)
+    name: str
+    for name in wanted:
+        if len(found) >= LIMIT:
+            break
+        path: Path | None
+        module: Module | None
+        if (
+            name in seen
+            or (path := locate(name, tuple(search))) is None
+            or (module := cached(path, name)) is None
+        ):
+            seen.add(name)
+            continue
+        seen.add(name)
+        found[name] = module
+        top: str = name.partition(".")[0].lstrip(_UNDERSCORE)
+        # Walked as it grows: the modules this one re-exports from, its package's private twin's
+        # too (`pytest`'s, from `_pytest`).
+        wanted.extend(  # ruff: ignore[loop-iterator-mutation]
+            other for other in _imported([module]) if other.partition(".")[0].lstrip(_UNDERSCORE) == top
+        )
+    if not found:
+        return catalog
+    every: dict[str, Module] = {**catalog.modules, **found}
+    every.update((name, _generics(every, module)) for name, module in found.items())
+    # The checked files' repeated names kept; what was worked out of the modules there were, not.
+    return catalog._replace(modules=every, names=sorted(every), memo=Memo())
+
+
+def _generics(modules: Mapping[str, Module], module: Module) -> Module:
+    """Drop the classes a subscripted base made look generic whose bases pass no type variable.
+
+    `class float64(floating[_64Bit])` binds every parameter it inherits: it's written bare.
+
+    Returns:
+      The module, its `generics` corrected.
+
+    """
+    bases: Mapping[str, frozenset[str]] = {} if module.declared is None else module.declared.bases
+    bound: frozenset[str] = frozenset(
+        name
+        for name, names in bases.items()
+        if name in module.generics and not any(_is_type_var(modules, module, found) for found in names)
+    )
+    return replace(module, generics=module.generics - bound) if bound else module
+
+
+def _is_type_var(modules: Mapping[str, Module], module: Module, name: str, hops: int = _HOPS) -> bool:
+    """Check whether `name` is a type variable in `module`: its own, or one it imports from an indexed module.
+
+    Returns:
+      Whether it is (unknown: no).
+
+    """
+    if name in module.type_vars or (module.declared is not None and name in module.declared.variables):
+        return True
+    origin: tuple[str, str | None] | None = module.names.get(name) or module.guarded.get(name)
+    onward: Module | None = None if origin is None or origin[1] is None else modules.get(origin[0])
+    return (
+        hops > 0
+        and onward is not None
+        and onward is not module
+        and origin is not None
+        and _is_type_var(modules, onward, origin[1] or "", hops - 1)
+    )
+
+
+def cache_directory() -> Path:
+    """Find where installed modules' reads are cached: `$XDG_CACHE_HOME`, `%LOCALAPPDATA%`, or `~/.cache`.
+
+    Returns:
+      Its `constricter` directory (not made yet).
+
+    """
+    base: str | None = os.environ.get(_CACHE_HOME) or os.environ.get(_LOCAL_APP_DATA)
+    return (Path(base) if base else Path.home() / ".cache") / _CACHE / "installed"
+
+
+def cached(path: Path, name: str) -> Module | None:
+    """Read an installed module (see `modules.read`), from the cache if it's there and still current.
+
+    A cache that can't be read or written (a read-only home, a corrupt entry) is passed over: the
+    module is read from its file.
+
+    Returns:
+      Its module, or `None` if it can't be read or parsed.
+
+    """
+    try:
+        stat: os.stat_result = path.stat()
+    except OSError:
+        return None
+    key: str = f"{name}\0{path.resolve()}\0{stat.st_size}\0{stat.st_mtime_ns}"
+    entry: Path = _own() / f"{hashlib.sha256(key.encode()).hexdigest()}.pickle"
+    loaded: Module | None
+    if (loaded := _load(entry)) is not None:
+        return loaded
+    module: Module | None
+    if (module := read(path, name)) is not None:
+        _store(entry, module)
+    return module
+
+
+@lru_cache(maxsize=1)
+def _code() -> str:
+    """Fingerprint the code that reads a module and shapes what's cached (`constricter.fix`'s).
+
+    A development build keeps its version as its modules' fields change: an entry pickled by another
+    unpickles with them shifted.
+
+    Returns:
+      Its sources' digest.
+
+    """
+    sources: list[Path] = sorted(Path(__file__).parents[1].rglob(f"*{SUFFIX}"))
+    return hashlib.sha256(b"\0".join(source.read_bytes() for source in sources)).hexdigest()
+
+
+def _own() -> Path:
+    """Find this build's part of the cache: another version's or build's entries unpickle wrong.
+
+    Returns:
+      Its directory, named by the version and `_code` (not made yet).
+
+    """
+    return cache_directory() / hashlib.sha256(f"{__version__}\0{_code()}".encode()).hexdigest()[:16]
+
+
+def _load(entry: Path) -> Module | None:
+    """Read a cache entry, and mark one read after half its keep as read now.
+
+    Returns:
+      Its module, or `None` if there's none, or it can't be read.
+
+    """
+    with contextlib.suppress(OSError, pickle.UnpicklingError, EOFError, AttributeError, TypeError):
+        module: Module | None = _module(cast("object", pickle.loads(entry.read_bytes())))
+        if _age(entry, time.time()) > _KEPT / 2:
+            os.utime(entry)
+        return module
+    return None
+
+
+def _age(path: Path, now: float) -> float:
+    """Find how long ago `path` was written or marked read.
+
+    Returns:
+      The seconds; none for one that can't be read, which is then kept.
+
+    """
+    try:
+        return now - path.stat().st_mtime
+    except OSError:
+        return 0.0
+
+
+def _dropped(part: Path) -> None:
+    """Remove a build's part of the cache: its entries, what a stopped write left, and its directory."""
+    with contextlib.suppress(OSError):
+        each: Path
+        for each in (*part.glob(_ENTRIES), *part.glob("*.tmp"), part / _SWEPT):
+            each.unlink(missing_ok=True)
+        part.rmdir()
+
+
+@lru_cache(maxsize=4)  # once a process for a cache, as it first writes there
+def swept(own: Path) -> None:
+    """Sweep the cache, a day or more after this build last did.
+
+    The entries no build's part holds (the cache's, before each had one) go, and another build's
+    part unswept for `_KEPT`: one still used sweeps, and marks its own. Of this build's entries,
+    those unread for `_KEPT`, then the longest unread until the rest fit `_CAP`.
+    """
+    now: float = time.time()
+    if (own / _SWEPT).exists() and _age(own / _SWEPT, now) < _DAY:
+        return
+    with contextlib.suppress(OSError):
+        (own / _SWEPT).touch()
+        other: Path
+        for other in own.parent.iterdir():
+            if other.match(_ENTRIES):
+                other.unlink(missing_ok=True)
+            elif other != own and len(other.name) == len(own.name) and _age(other / _SWEPT, now) > _KEPT:
+                _dropped(other)
+        held: list[tuple[float, int, Path]] = sorted(
+            (stat.st_mtime, stat.st_size, entry) for entry in own.glob(_ENTRIES) for stat in (entry.stat(),)
+        )
+        size: int = sum(taken for _, taken, _ in held)
+        written: float
+        taken: int
+        entry: Path
+        for written, taken, entry in held:
+            if now - written <= _KEPT and size <= _CAP:
+                break
+            entry.unlink(missing_ok=True)
+            size -= taken
+
+
+def _module(found: object) -> Module | None:
+    return found if isinstance(found, Module) else None
+
+
+def _store(entry: Path, module: Module) -> None:
+    """Write a module's read to its cache entry, whole or not at all (another run may be reading it)."""
+    with contextlib.suppress(OSError):
+        entry.parent.mkdir(parents=True, exist_ok=True)
+        handle: int
+        temporary: str
+        handle, temporary = tempfile.mkstemp(dir=entry.parent, suffix=".tmp")
+        os.close(handle)
+        _ = Path(temporary).write_bytes(pickle.dumps(module))
+        _ = Path(temporary).replace(entry)
+        swept(entry.parent)
+
+
+def _imported(found: Iterable[Module]) -> list[str]:
+    """List the modules `found` import from outside the standard library, and those they may name.
+
+    `from pkg import util` may name the module `pkg.util` as well as something `pkg` defines; a call
+    through an imported module (`pkg.util.f()` after `import pkg.util`) names each module on its way.
+
+    Returns:
+      Their names, in order, each once.
+
+    """
+    names: dict[str, None] = {}
+    module: Module
+    for module in found:
+        origin: tuple[str, str | None]
+        for origin in (*module.names.values(), *module.guarded.values()):
+            if origin[0].partition(".")[0] not in _STDLIB and origin[0] != module.name:
+                names[origin[0]] = None
+                if origin[1] is not None:
+                    names[f"{origin[0]}.{origin[1]}"] = None
+        callee: str
+        for callee in module.called:
+            head: str
+            parts: list[str]
+            head, *parts = callee.split(".")
+            through: tuple[str, str | None] = module.names[head]
+            if through[1] is None and through[0].partition(".")[0] not in _STDLIB:
+                names.update(
+                    dict.fromkeys(".".join([through[0], *parts[:end]]) for end in range(1, len(parts))),
+                )
+    return list(names)
+
+
+@lru_cache(maxsize=4096)
+def locate(name: str, search: tuple[Path, ...]) -> Path | None:
+    """Find the file an installed module `name` declares its types in, as the import system would.
+
+    In the first directory of `search` that has its package: its stub package (`pkg-stubs`) first,
+    then the package itself if it's typed (`py.typed`), a stub (`.pyi`) before its source; or a lone
+    stub module (`mod.pyi`).
+
+    Returns:
+      The file, or `None` if it isn't installed, or doesn't declare its types.
+
+    """
+    top: str
+    rest: list[str]
+    top, *rest = name.split(".")
+    directory: Path
+    for directory in search:
+        root: Path | None
+        stub: bool
+        here: bool
+        root, stub, here = _package(directory, top)
+        if root is not None:
+            return _module_file(root.joinpath(*rest))
+        if not rest and stub:
+            return directory / f"{top}{STUB}"
+        if here:
+            return None  # installed here, untyped: an earlier directory shadows any later one
+    return None
+
+
+def unseen(name: str) -> Module | None:
+    """Read an installed module that declares no types, for its classes' bases alone.
+
+    Never part of the index: nothing it declares types a call, as no type checker reads it. What
+    its classes inherit from still says where a class under one ends (see `constricter.fix.index.beyond`).
+
+    Returns:
+      Its module, read from its source as `cached` reads one; `None` for the standard library's, or
+      one that isn't installed.
+
+    """
+    if not name or name.partition(".")[0] in _STDLIB:
+        return None
+    return _unseen(name, _searched(tuple(sys.path), os.environ.get(_VIRTUAL_ENV)))
+
+
+@lru_cache(maxsize=4)
+def _searched(_path: tuple[str, ...], _active: str | None) -> tuple[Path, ...]:  # `search_path`, for both
+    return search_path()
+
+
+@lru_cache(maxsize=1024)  # asked for each class under one of its classes
+def _unseen(name: str, search: tuple[Path, ...]) -> Module | None:
+    paths: Iterable[Path | None] = (
+        _module_file(directory.joinpath(*name.split("."))) for directory in search
+    )
+    path: Path | None = next((path for path in paths if path is not None), None)
+    return None if path is None else cached(path, name)
+
+
+@lru_cache(maxsize=4096)  # asked for each of a package's modules: the same answer for them all
+def _package(directory: Path, top: str) -> tuple[Path | None, bool, bool]:
+    """Find what `directory` has of the top-level package or module `top`.
+
+    Returns:
+      Where its types are declared (its stub package, else the package itself if it's typed), or
+      `None`; whether it has a lone stub module for it; and whether it's installed there at all.
+
+    """
+    roots: list[Path] = [directory / f"{top}{_STUBS}"]
+    if (directory / top / _TYPED).is_file():
+        roots.append(directory / top)
+    return (
+        next((root for root in roots if root.is_dir()), None),
+        (directory / f"{top}{STUB}").is_file(),
+        (directory / top).is_dir() or (directory / f"{top}{SUFFIX}").is_file(),
+    )
+
+
+def _module_file(base: Path) -> Path | None:
+    """Find a module's file under a package's root: a package's `__init__`, or a module, stub first.
+
+    Returns:
+      It, or `None`.
+
+    """
+    candidates: tuple[Path, ...] = (
+        base / f"{_PACKAGE}{STUB}",
+        base / f"{_PACKAGE}{SUFFIX}",
+        base.with_name(f"{base.name}{STUB}"),
+        base.with_name(f"{base.name}{SUFFIX}"),
+    )
+    return next((path for path in candidates if path.is_file()), None)

constricter/fix/index/stubs/names.py

@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: MIT
+"""What a name in a stub refers to, across the installed modules the index holds (`Resolver`).
+
+An alias, a class or a type variable, where it's declared, a class's bases in order, and what a
+call's name refers to through its module's imports; `stub_signatures.Reader` reads signatures with
+it.
+"""
+
+import ast
+import builtins
+from collections.abc import Iterator, Mapping
+from typing import Final, TypeAlias
+
+from constricter.fix.core.known import Origin
+from constricter.fix.index import project
+from constricter.fix.index.modules import Module
+from constricter.fix.index.stubs import classnames
+from constricter.fix.index.stubs.atoms import (
+    BUILTINS_MODULE,
+    CLASS,
+    VAR,
+    Atom,
+    Scope,
+    atom_path,
+    parse_text,
+)
+from constricter.fix.index.stubs.declared import Alias, Class, Declarations, Variable
+
+ARG: Final = "arg"
+_BUILTINS: Final = frozenset(dir(builtins))
+ALIAS_DEPTH: Final = 20  # aliases followed before giving up
+_UNFOLLOWED: Final = "?"  # a lineage's base that can't be followed
+
+
+Resolved: TypeAlias = Atom | tuple[Alias, Scope, Origin]
+
+
+def call_origin(modules: Mapping[str, Module], target: Module, callee: str) -> Origin | None:
+    """Find what a call's name refers to: `f`, `np.empty`, `pkg.util.f` through the module's imports.
+
+    A package's attribute, re-exported from anywhere (`numpy.empty`), not just what it defines.
+
+    Returns:
+      Its origin, or `None`.
+
+    """
+    head: str
+    parts: list[str]
+    head, *parts = callee.split(".")
+    origin: Origin | None = classnames.as_module(modules, target.names.get(head))
+    part: str
+    for part in parts:
+        if origin is None or origin[1] is not None:
+            return None
+        submodule: str = f"{origin[0]}.{part}"
+        origin = (submodule, None) if submodule in modules else (origin[0], part)
+    return origin
+
+
+class Resolver:
+    """Resolves a stub's names across `modules` (see the module docstring)."""
+
+    def __init__(self, modules: Mapping[str, Module]) -> None:
+        """Resolve in `modules`."""
+        self.modules: Mapping[str, Module] = modules
+
+    def resolve(self, expr: ast.Name | ast.Attribute, scope: Scope) -> Resolved | None:
+        """Resolve a name an annotation uses: a type parameter, a class, an alias, a type variable.
+
+        Returns:
+          It as an atom (`arg`: a parameter an alias's argument binds), or an alias with the scope
+          its value is read in; or `None` if it's none of them.
+
+        """
+        origin: Origin | None
+        if isinstance(expr, ast.Name) and expr.id in scope.params:
+            param: str | tuple[ast.expr, Scope] | None = scope.params[expr.id]
+            if isinstance(param, tuple):
+                return Atom(ARG, name=ast.unparse(param[0]), scope=param[1])
+            return Atom(VAR, name=expr.id, bound=param, scope=scope)
+        if (origin := self._origin(expr, scope.module)) is None or origin[1] is None:
+            return None
+        origin = project.canonical_origin(self.modules, origin)
+        defining: Module | None = self.modules.get(origin[0])
+        if defining is None or not defining.reads(origin[1]):
+            return Atom(CLASS, origin=origin)
+        return self._declared(defining, origin)
+
+    def resolved_base(self, base: str, module: Module) -> Resolved | None:
+        """Resolve a base `module`'s class names, as written there (`Base`, `pkg.Base[T]`).
+
+        Returns:
+          What it refers to, or `None` for a base that isn't a name.
+
+        """
+        parsed: ast.expr = parse_text(base)
+        parsed = parsed.value if isinstance(parsed, ast.Subscript) else parsed
+        return (
+            self.resolve(parsed, Scope(module, {}, module.name))
+            if isinstance(parsed, ast.Name | ast.Attribute)
+            else None
+        )
+
+    def lineage(self, origin: Origin, hops: int) -> Iterator[str]:
+        """Walk an installed class and its ancestors, depth first (see `Methods.lineage`).
+
+        Yields:
+          Where each is defined (`numpy.float64`), then `?` if a base can't be followed.
+
+        """
+        yield f"{origin[0]}.{origin[1]}"
+        module: Module = self.modules[origin[0]]
+        klass: Class | None
+        # No class: a checked file's protocol, or one its module defines twice (attrs's `AttrsInstance_`).
+        if module.declared is None or (klass := module.declared.classes.get(origin[1] or "")) is None:
+            yield _UNFOLLOWED
+            return
+        base: str
+        for base in klass.bases:
+            resolved: Resolved | None = self.resolved_base(base, module)
+            if not isinstance(resolved, Atom) or resolved.kind != CLASS:
+                yield _UNFOLLOWED
+            elif resolved.module is None:
+                yield atom_path(resolved)
+            elif hops < ALIAS_DEPTH:
+                yield from self.lineage(resolved.origin, hops + 1)
+            else:  # a cycle of bases
+                yield _UNFOLLOWED
+
+    def is_class(self, target: Module, name: str) -> bool:
+        """Check whether a dotted name the module writes (`np.float64`) is an installed class or its alias.
+
+        Returns:
+          Whether it is.
+
+        """
+        origin: Origin | None = call_origin(self.modules, target, name)
+        resolved: Resolved | None = (
+            None
+            if origin is None or origin[1] is None
+            else self._declared_of(project.canonical_origin(self.modules, origin))
+        )
+        if resolved is not None and not isinstance(resolved, Atom):  # an alias: of a class?
+            value: ast.expr = parse_text(resolved[0].value)
+            base: ast.expr = value.value if isinstance(value, ast.Subscript) else value
+            resolved = self.resolve(base, resolved[1]) if isinstance(base, ast.Name | ast.Attribute) else None
+        return isinstance(resolved, Atom) and resolved.kind == CLASS and resolved.module is not None
+
+    def _declared_of(self, origin: Origin) -> Resolved | None:
+        defining: Module | None = self.modules.get(origin[0])
+        return None if defining is None or not defining.reads(origin[1]) else self._declared(defining, origin)
+
+    @staticmethod
+    def _declared(module: Module, origin: Origin) -> Resolved | None:
+        """Read what an installed module defines under `origin`'s name: an alias, a type variable, a class.
+
+        Returns:
+          It, or `None` for anything else (a function, a variable).
+
+        """
+        name: str = origin[1] or ""
+        declared: Declarations | None = module.typing
+        own: Scope = Scope(module, {}, module.name)
+        if declared is not None and name in declared.aliases:
+            return declared.aliases[name], own, origin
+        variable: Variable | None
+        if (variable := None if declared is None else declared.variables.get(name)) is not None:
+            return Atom(VAR, name=name, bound=variable.bound, scope=own, constrained=variable.constrained)
+        if (
+            name in module.classes
+            or name in module.generics
+            or (declared is not None and name in declared.protocols)
+        ):
+            return Atom(CLASS, origin=origin, module=module)
+        return None
+
+    def _origin(self, expr: ast.Name | ast.Attribute, module: Module) -> Origin | None:
+        """Find what a (dotted) name refers to in `module`: an import, its own, or a builtin.
+
+        Returns:
+          Its origin (a module's: `(name, None)`), or `None`.
+
+        """
+        if isinstance(expr, ast.Name):
+            declared: Declarations | None = module.typing
+            return (
+                ((module.name, expr.id) if declared is not None and expr.id in declared.aliases else None)
+                or module.names.get(expr.id)
+                or module.guarded.get(expr.id)
+                or ((BUILTINS_MODULE, expr.id) if expr.id in _BUILTINS else None)
+            )
+        if not isinstance(expr.value, ast.Name | ast.Attribute):
+            return None
+        base: Origin | None
+        if (base := self._origin(expr.value, module)) is None:
+            return None
+        if base[1] is None:  # a module's attribute, or its submodule
+            submodule: str = f"{base[0]}.{expr.attr}"
+            return (submodule, None) if submodule in self.modules else (base[0], expr.attr)
+        return None