python-constricter 0.4.1__cp311-cp311-win32.whl

constricter/rules/narrowing.py

@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: MIT
+"""Value flow over a module's scopes, and its findings (LVA008, LVA009, LVA010) as offences with fixes."""
+
+import ast
+from collections.abc import Sequence
+from functools import lru_cache
+from typing import Final
+
+from constricter.offences import NARROW, Edit, Fix, FixPolicy, Offence
+from constricter.rules.annotations import node_name
+from constricter.rules.flow import Finding, Kind
+from constricter.rules.scope import Scope
+from constricter.rules.walked import of_type
+
+_TYPE_CHECKING: Final = "TYPE_CHECKING"
+
+
+def flow_offences(found: list[Finding], policy: FixPolicy) -> list[Offence]:
+    """Report value-flow findings as offences, a finding's kind as its code.
+
+    Each declaration LVA008 or LVA010 would narrow gets one fix, a guess (`--unsafe-fixes`: a
+    declared type can be wider on purpose): the LVA008 finding's narrowed type if there's one,
+    since it's already within the members the values use, else the union without its unused
+    members, on the first LVA010 finding. Two rewrites of one annotation can't both apply. `policy`
+    decides whether it's offered (`narrow`), and whether it's trusted.
+
+    Returns:
+      The offences.
+
+    """
+    offences: list[Offence] = []
+    fixed: set[tuple[int, int]] = set()
+    finding: Finding
+    narrowed: set[tuple[int, int]] = {(f.line, f.col) for f in found if f.kind is Kind.NARROWABLE}
+    for finding in sorted(found, key=lambda f: f.kind is not Kind.NARROWABLE):
+        where: tuple[int, int] = (finding.line, finding.col)
+        edit: Fix | None = None
+        if (
+            finding.span
+            and finding.rewrite
+            and where not in fixed
+            and (finding.kind is Kind.NARROWABLE or where not in narrowed)
+        ):
+            fixed.add(where)
+            reason: str = (
+                "the values it's bound to"
+                if finding.kind is Kind.NARROWABLE
+                else "the members its values use"
+            )
+            kinds: frozenset[str] = frozenset({NARROW})
+            if policy.allows(kinds):
+                edit = Fix(
+                    finding.rewrite,
+                    reason,
+                    unsafe=not policy.trusts(kinds),
+                    edit=Edit.REPLACE,
+                    span=finding.span,
+                    kinds=kinds,
+                )
+        offences.append(Offence(*where, finding.name, finding.kind.value, edit, detail=finding.detail))
+    return offences
+
+
+def module_flow(tree: ast.Module, scopes: Sequence[Scope]) -> list[Finding]:
+    """Compare each name's values with its declared type, in each of `scopes`.
+
+    Returns:
+      Every finding, in source order.
+
+    """
+    escaped: frozenset[str]
+    checking_only: frozenset[str]
+    escaped, checking_only = module_names(tree)
+    return sorted(found for scope in scopes for found in scope.value_flow(escaped, checking_only))
+
+
+@lru_cache(maxsize=16)  # `module_flow` runs once per round of the checker's `_returned`, on the same tree
+def module_names(tree: ast.Module) -> tuple[frozenset[str], frozenset[str]]:
+    """Find the names value flow leaves alone in a module.
+
+    Returns:
+      Those a `global` or `nonlocal` writes from elsewhere, and those annotated only for type
+      checkers (`if TYPE_CHECKING: x: str`), whose runtime values may differ on purpose.
+
+    """
+    escaped: set[str] = set()
+    checking_only: set[str] = set()
+    node: ast.AST
+    for node in of_type(tree, ast.Global, ast.Nonlocal, ast.If):
+        if isinstance(node, ast.Global | ast.Nonlocal):
+            escaped.update(node.names)
+        elif isinstance(node, ast.If) and node_name(node.test) == _TYPE_CHECKING:  # the only other kind
+            checking_only.update(
+                stmt.target.id
+                for stmt in node.body
+                if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name)
+            )
+    return frozenset(escaped), frozenset(checking_only)

constricter/rules/parsed.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: MIT
+"""Parse a module once: the cross-file index and the check share its tree, in the same process.
+
+The CLI reads every file to index what it offers other files (`fix.index.project`), then checks each; both
+parse it the same way (`parse`), and the index's tree is kept (`keep`) for the check to take
+(`take`), which keeps it again once it's done: a file may be checked a second time. One `--fix`
+changed is freed (`take`, by the text it had). What's kept is capped (`budget`): a tree takes about
+26 bytes of memory for each byte of source, so past the cap a file is parsed again, as it always
+was. The cap is a quarter of the machine's memory (`sized`), and never under 1 GB of trees.
+"""
+
+import ast
+import io
+import os
+import tokenize
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from constricter.rules.tables import Tables
+
+_FLOOR: Final = 40 << 20  # bytes of source whose trees any run may keep (about 1 GB of them)
+_TREE: Final = 26  # bytes of memory a tree takes for each byte of its source
+_SHARE: Final = 4  # the kept trees' part of the machine's memory: a quarter
+# A control group's memory limit (a container's), where there is one: `max` for none.
+_GROUP_LIMIT: Final = Path("/sys/fs/cgroup/memory.max")
+
+
+def memory() -> int | None:
+    """Read how much memory this process has to use: the machine's, or its control group's limit if lower.
+
+    Returns:
+      It, in bytes; `None` where the system doesn't say (Windows has no `sysconf`).
+
+    """
+    # Looked up, not named: mypyc's mypy pass on Windows rejects `os.sysconf`.
+    sysconf: Callable[[str], int] | None
+    if (sysconf := getattr(os, "sysconf", None)) is None:
+        return None
+    try:
+        total: int = sysconf("SC_PHYS_PAGES") * sysconf("SC_PAGE_SIZE")
+    except (ValueError, OSError):
+        return None
+    try:
+        return min(total, int(_GROUP_LIMIT.read_text(encoding="ascii")))
+    except (OSError, ValueError):  # no control group, or no limit in it
+        return total
+
+
+def sized(available: int | None) -> int:
+    """Size the kept trees' budget for a machine with `available` bytes of memory (see `memory`).
+
+    Returns:
+      The bytes of source whose trees a run keeps: a quarter of the memory's worth, `_FLOOR` at
+      least, and where the memory isn't known.
+
+    """
+    return _FLOOR if available is None else max(_FLOOR, available // _SHARE // _TREE)
+
+
+BUDGET: Final = sized(memory())  # bytes of source whose trees a run keeps, over all its processes
+# A kept tree, and the module's own tables the index read from it (the check reads the same).
+Kept: TypeAlias = tuple[ast.Module, "Tables"]
+
+
+@dataclass
+class _Kept:
+    """The trees kept in this process, by their source's text, and what's left of its share."""
+
+    trees: dict[str, Kept] = field(default_factory=dict)
+    left: int = BUDGET
+
+
+_KEPT: Final = _Kept()
+
+
+def budget(processes: int) -> None:
+    """Share the budget among `processes`: each keeps its part (a worker's initializer, or the CLI's own)."""
+    _KEPT.trees.clear()
+    _KEPT.left = BUDGET // max(processes, 1)
+
+
+def parse(source: str | bytes, filename: str = "<unknown>") -> ast.Module:
+    """Parse `source` with its `# type:` comments; without them if one is misplaced.
+
+    Python's warnings about the source (an invalid escape sequence in a string) are left unsaid: they're
+    about the code checked, not findings, and it's the checked project's to see them when it runs.
+
+    Returns:
+      The module. Raises `SyntaxError`.
+
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", SyntaxWarning)
+        warnings.simplefilter("ignore", DeprecationWarning)  # what Python 3.11 warns about them with
+        try:
+            return ast.parse(source, filename, type_comments=True)
+        except SyntaxError:  # a misplaced `# type:` comment, or a real error raised again here
+            return ast.parse(source, filename)
+
+
+def text(data: bytes) -> str:
+    """Decode a module's bytes as Python finds their encoding (a PEP 263 declaration, a BOM, else UTF-8).
+
+    Returns:
+      Its text. Raises `SyntaxError` for an unknown or contradictory encoding, and
+      `UnicodeDecodeError` for bytes it doesn't hold.
+
+    """
+    return data.decode(tokenize.detect_encoding(io.BytesIO(data).readline)[0])
+
+
+def keep(source: str, kept: Kept) -> None:
+    """Keep `source`'s tree and tables for the check to take, if this process's share of the budget allows."""
+    if len(source) <= _KEPT.left and source not in _KEPT.trees:
+        _KEPT.trees[source] = kept
+        _KEPT.left -= len(source)
+
+
+def take(source: str) -> "Kept | None":
+    """Take the tree and tables kept for `source`, freeing them (two files alike in every byte share them).
+
+    Only for exactly the text they were read from: a file `--fix` has changed since is parsed again.
+
+    Returns:
+      Them, or `None` if none were kept.
+
+    """
+    kept: Kept | None
+    if (kept := _KEPT.trees.pop(source, None)) is not None:
+        _KEPT.left += len(source)
+    return kept

constricter/rules/quoted.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: MIT
+"""Quoted annotations: a string annotation as the expression it quotes, and a quoted part as its text."""
+
+import ast
+from functools import lru_cache
+from typing import Final
+
+from constricter.offences import BUILTIN_GENERICS, UNION_SYNTAX
+
+_LITERAL: Final = "Literal"  # its strings are values, not quoted types
+_ANNOTATED: Final = "Annotated"  # its first argument alone is a type
+_QUOTES: Final = frozenset("'\"")
+_QUALIFIERS: Final = frozenset({"ClassVar", "Final"})  # around an attribute's type
+
+
+def _head(node: ast.expr) -> str:
+    """Read the last name a subscript's head is written with (`Literal`, for `typing.Literal`).
+
+    Returns:
+      It.
+
+    """
+    return ast.unparse(node).rpartition(".")[2]
+
+
+@lru_cache(maxsize=256)
+def parsed(annotation: ast.expr) -> ast.expr:
+    """Unwrap a string annotation.
+
+    `is_vague` and `depth` both call this on the same annotation; cached so it's parsed once.
+
+    Returns:
+      Its parsed expression, or the annotation itself if it isn't a string.
+
+    """
+    if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str):
+        try:
+            return ast.parse(annotation.value, mode="eval").body
+        except SyntaxError:
+            return annotation
+    return annotation
+
+
+def written(annotation: ast.expr) -> str:
+    """Write an annotation as text `--fix` infers from: a quoted one, or a quoted part, as what it quotes.
+
+    `"list[Node]"` and `list["Node"]` are both `list[Node]`. Not a `Literal`'s strings, nor an
+    `Annotated`'s metadata, which are values; nor a string that isn't an expression.
+
+    Returns:
+      The text.
+
+    """
+    text: str = ast.unparse(annotation)
+    return ast.unparse(_unquoted(annotation)) if _QUOTES.intersection(text) else text
+
+
+def unqualified(annotation: ast.expr) -> str:
+    """Write an attribute's annotation (see `written`), less the `ClassVar` or `Final` around its type.
+
+    Returns:
+      The text; nothing for a bare `Final`, which leaves the type to the value.
+
+    """
+    name: str
+    inner: ast.expr
+    match annotation:
+        case ast.Name(id=name) | ast.Attribute(attr=name) if name in _QUALIFIERS:
+            return ""
+        case ast.Subscript(value=ast.Name(id=name) | ast.Attribute(attr=name), slice=inner) if (
+            name in _QUALIFIERS
+        ):
+            return unqualified(inner)
+        case _:
+            return written(annotation)
+
+
+def _unquoted(node: ast.expr) -> ast.expr:
+    """Rebuild an annotation with each quoted type in it read as an expression (see `written`).
+
+    Returns:
+      The annotation: a new tree wherever it differs (the module's own is shared).
+
+    """
+    head: ast.expr
+    inner: ast.expr
+    first: ast.expr
+    rest: list[ast.expr]
+    parts: list[ast.expr]
+    left: ast.expr
+    right: ast.expr
+    found: ast.expr = node
+    match node:
+        case ast.Constant(value=str()):
+            found = parsed(node)
+            found = found if found is node else _unquoted(found)
+        case ast.Subscript(value=head) if _head(head) == _LITERAL:
+            pass
+        case ast.Subscript(value=head, slice=ast.Tuple(elts=[first, *rest])) if _head(head) == _ANNOTATED:
+            found = ast.Subscript(head, ast.Tuple([_unquoted(first), *rest]))
+        case ast.Subscript(value=head, slice=inner):
+            found = ast.Subscript(head, _unquoted(inner))
+        case ast.Tuple(elts=parts):
+            found = ast.Tuple([_unquoted(part) for part in parts])
+        case ast.List(elts=parts):
+            found = ast.List([_unquoted(part) for part in parts])
+        case ast.BinOp(left=left, op=ast.BitOr(), right=right):
+            found = ast.BinOp(_unquoted(left), ast.BitOr(), _unquoted(right))
+        case _:
+            pass
+    return found
+
+
+def quote(annotation: str) -> str:
+    """Quote an annotation: in double quotes, unless it has one or a backslash (a `Literal`'s string).
+
+    Returns:
+      It, as a string literal.
+
+    """
+    plain: bool = not {'"', "\\"} & set(annotation)
+    return f'"{annotation}"' if plain else ast.unparse(ast.Constant(annotation))
+
+
+@lru_cache(maxsize=4096)  # a module's fixes are a few types, each asked again and again
+def newer(annotation: str, oldest: tuple[int, int] | None) -> bool:
+    """Check whether `annotation` is written as Pythons from `oldest` on don't all read a type.
+
+    A union by `|` before 3.10, any subscript before 3.9 (a builtin or standard-library class's).
+
+    Returns:
+      Whether it is; never where `oldest` isn't known, or the annotation is quoted already.
+
+    """
+    if oldest is None or oldest >= UNION_SYNTAX or annotation.startswith(('"', "'")):
+        return False
+    return any(
+        (isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr))
+        or (oldest < BUILTIN_GENERICS and isinstance(node, ast.Subscript))
+        for node in ast.walk(ast.parse(annotation, mode="eval"))
+    )

constricter/rules/rebinding.py

@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: MIT
+"""A first binding's fix, refitted to every value the name is bound to later, as a checker holds it to.
+
+`x = 1` then `x = None` declares `x: int | None` before the first binding (annotated there, mypy
+wouldn't narrow it to the `int` it's bound to), and `total = 0` then `total += 0.5` declares a
+`float`; another type that doesn't fit (`x = 1` then `x = "a"`, a class and its base) leaves it
+untyped: a union is better left to the author. A `Self` takes only `Self`. A later binding whose
+type isn't known may be anything, and one whose type is a guess may be wrong: either makes the fix
+a guess, and leaves a type checker's hint (`--infer-with`) no fix at all.
+"""
+
+from typing import Final, NamedTuple
+
+from constricter.fix.core.known import Inference
+from constricter.fix.values.hinted import KINDS as HINTED
+from constricter.offences import UNIONS, Edit, Fix, Offence
+from constricter.rules.flow import Binding, Hierarchy, members
+
+_NONE: Final = "None"
+_ONLY_NONE: Final = frozenset({_NONE})
+_OPTIONAL: Final = "optional"  # the fix kind of a `None` added to a type
+REBOUND: Final = "rebound"  # the fix kind (and guessing mechanism) of a later binding's type
+# The types a later binding may widen a first one's to: the numeric tower's (`int`, then `float`).
+_NUMBERS: Final = frozenset({"bool", "int", "float", "complex"})
+_UNION: Final = 3  # the most types a widening's union names
+
+
+class Refit(NamedTuple):
+    """A fix refitted: its inference, what it rests on if a guess, whether it is one, and its edit."""
+
+    found: Inference
+    origins: frozenset[str]
+    unsafe: bool
+    edit: Edit
+    span: tuple[int, int]
+
+
+class _Later(NamedTuple):
+    """What a name's later bindings say: the types that don't fit its fix, their guesses, any unknown."""
+
+    misfits: frozenset[str]
+    origins: frozenset[str]  # what the later bindings' guessed types rest on
+    unknown: bool
+
+
+def refit(
+    o: Offence,
+    fix: Fix,
+    rest: list[Binding],
+    hierarchy: Hierarchy,
+    module: tuple[str | None, bool],
+) -> Refit | Fix | None:
+    """Refit `o`'s `fix` to the name's `rest` of bindings (see the module docstring).
+
+    `module`: how the module spells `Self`, if it imports it; and whether values of two or three
+    types that fit no one of them are declared their union, a widening (`fix-widen`'s `unions`).
+
+    Returns:
+      `fix` itself, if it takes them all as it is; else its refit, to offer; or `None`, for no fix.
+
+    """
+    # `--fix`'s annotations are always readable; one that weren't would be its own one member.
+    declared: frozenset[str] = members(fix.annotation) or frozenset({fix.annotation})
+    later: _Later = _later(rest, declared, hierarchy)
+    # A hint is one value's type alone: it says nothing of a later one's.
+    if (later.misfits or later.unknown) and (module[0] in declared or HINTED & fix.kinds):
+        return None
+    if not (later.misfits or later.unknown or later.origins):
+        return fix
+    rebound: frozenset[str] = frozenset({REBOUND}) if later.unknown else frozenset()
+    kinds: frozenset[str] = fix.kinds | later.origins | rebound
+    origins: frozenset[str] = later.origins | rebound | (fix.kinds if fix.unsafe else frozenset[str]())
+    unsafe: bool = fix.unsafe or later.unknown or bool(later.origins)
+    if not later.misfits:
+        return Refit(Inference(fix.annotation, fix.reason, kinds), origins, unsafe, fix.edit, fix.span)
+    atoms: frozenset[str] = declared | later.misfits
+    if module[0] in atoms:  # a `Self` takes only `Self`, and is taken by nothing else
+        return None
+    joined: tuple[str, frozenset[str]] | None
+    # A union of known types: one that's a guess makes it a guess, resting on what that one does, as
+    # a second pass would declare it once the first had declared the guess.
+    if (joined := _joined(atoms - {_NONE}, hierarchy, unions=module[1] and not later.unknown)) is None:
+        return None
+    return Refit(
+        Inference(
+            joined[0] + (" | None" if _NONE in atoms else ""),
+            f"{fix.reason}, and every value it's bound to later",
+            kinds | {_OPTIONAL if later.misfits == _ONLY_NONE else REBOUND} | joined[1],
+        ),
+        origins | joined[1],
+        unsafe or bool(joined[1]),
+        Edit.DECLARE,
+        fix.span if fix.edit is Edit.DECLARE else (o.line, o.col),
+    )
+
+
+def _joined(
+    atoms: frozenset[str],
+    hierarchy: Hierarchy,
+    *,
+    unions: bool,
+) -> tuple[str, frozenset[str]] | None:
+    """Write the one type a name bound to values of types `atoms` is declared.
+
+    The one they all fit (see `_widest`); else, with `unions`, the union of two or three of them: a
+    widening, which a checker may hold a later use to.
+
+    Returns:
+      The type, and the widening's kind if it's one; `None` where there's neither.
+
+    """
+    widest: str | None
+    if (widest := _widest(atoms, hierarchy)) is not None:
+        return widest, frozenset()
+    return (" | ".join(sorted(atoms)), frozenset({UNIONS})) if unions and len(atoms) <= _UNION else None
+
+
+def _later(rest: list[Binding], declared: frozenset[str], hierarchy: Hierarchy) -> _Later:
+    """Read what the later bindings of a name declared as the union `declared` say.
+
+    Returns:
+      Their types that don't fit it, what their guesses rest on, and whether any is unknown.
+
+    """
+    misfits: set[str] = set()
+    origins: frozenset[str] = frozenset()
+    unknown: bool = False
+    binding: Binding
+    for binding in rest:
+        text: str | None = binding.value or (None if binding.guess is None else binding.guess[0])
+        found: frozenset[str] | None
+        if text is None or (found := members(text)) is None:
+            unknown = True
+            continue
+        if binding.value is None and binding.guess is not None:
+            origins |= binding.guess[1]
+        misfits.update(atom for atom in found if not _fits(atom, declared, hierarchy))
+    return _Later(frozenset(misfits), origins, unknown)
+
+
+def _fits(atom: str, declared: frozenset[str], hierarchy: Hierarchy) -> bool:
+    """Check whether a value of type `atom` fits a name declared as the union `declared`.
+
+    As value flow's `Hierarchy.fits` has it, but a generic only fits the same text: an annotation that
+    says `list[int]` doesn't take a `list[str]`.
+
+    Returns:
+      Whether it does.
+
+    """
+    return any(atom == member or member in hierarchy.wider(atom) for member in declared)
+
+
+def _widest(atoms: frozenset[str], hierarchy: Hierarchy) -> str | None:
+    """Find the one type all of `atoms` fit: itself, if it's one; else the widest, if they're numbers.
+
+    Not a class's base (`Base` for `Base` and `Sub`): a checker may see only one of them bound (the
+    other, say, under `if not TYPE_CHECKING:`), and narrow to it.
+
+    Returns:
+      It, or `None`.
+
+    """
+    if len(atoms) == 1:
+        return next(iter(atoms))
+    widest: list[str] = [
+        atom for atom in atoms if all(_fits(other, frozenset({atom}), hierarchy) for other in atoms)
+    ]
+    return widest[0] if widest and atoms <= _NUMBERS else None