python-constricter 0.4.1__cp312-cp312-win32.whl

constricter/cli/command.py

@@ -0,0 +1,547 @@
+# SPDX-License-Identifier: MIT
+"""The `constricter` command (see README)."""
+
+import codecs
+import contextlib
+import difflib
+import importlib
+import json
+import sys
+from collections.abc import Callable, Sequence
+from dataclasses import replace
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING, Final, NamedTuple, TextIO, TypeAlias, cast
+
+from constricter import notebook
+from constricter.cli import baseline, collecting, schedule
+from constricter.cli.lsp.protocol import HintError
+from constricter.cli.options import Mode, Options, Output
+from constricter.cli.paths import STDIN, python_files
+from constricter.cli.reading import encoding_of, read_text
+from constricter.cli.report import Format, Result, fix_reasons, render, statistics
+from constricter.cli.rounds import checked_all, settled
+from constricter.cli.runs import Added, BaselineRun, CheckRun, CoverageRun, FileRun
+from constricter.cli.workers import kept
+from constricter.fix.core import fixes
+from constricter.fix.core.known import Outside, Returns
+from constricter.noqa import lines, unsuppressed
+from constricter.offences import (
+    DEFAULT_CHECKS,
+    Checks,
+    Edit,
+    Offence,
+)
+from constricter.rules import parsed
+from constricter.rules.checker import Checked, Coverage, annotation_coverage, checked_source
+
+if TYPE_CHECKING:  # slow to import, and only needed for `--infer-with`
+    from constricter.cli.lsp import hints
+
+EXIT_CLEAN: Final = 0
+EXIT_FOUND: Final = 1
+EXIT_ERROR: Final = 2
+_ALL: Final = 100  # percent
+
+
+def _source(raw: str, name: Path) -> tuple[str, list[notebook.Line]]:
+    """Extract the Python in `raw`: a notebook's code cells, joined.
+
+    Returns:
+      The source, and each line's cell if it's a notebook.
+
+    """
+    return notebook.parse(raw, str(name)) if name.suffix == notebook.SUFFIX else (raw, [])
+
+
+def checked_text(
+    raw: str,
+    name: Path,
+    checks: Checks = DEFAULT_CHECKS,
+    *,
+    outside: Outside | None = None,
+) -> Checked:
+    """Check `raw`, the text of `name`: the offences no `# noqa` suppresses.
+
+    A notebook's code cells are checked as one module, and each offence placed in its cell.
+    Raises `ValueError` for a `.ipynb` that isn't a notebook.
+
+    Returns:
+      The unsuppressed offences, a notebook's placed in their cells, and what its functions return
+      (see `Checked`).
+
+    """
+    source: str
+    where: list[notebook.Line]
+    source, where = _source(raw, name)
+    checked: Checked = checked_source(source, str(name), checks, outside=outside)
+    offences: list[Offence] = unsuppressed(checked.offences, lines(source))
+    return checked._replace(offences=[_placed(o, where) for o in offences] if where else offences)
+
+
+def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
+    """Place an offence from a notebook's joined module in its cell, its declaration's line too.
+
+    Returns:
+      The offence, its `line` (and a `Edit.DECLARE` fix's statement line) counted in its `cell`; a
+      fix that would add an import isn't offered.
+
+    """
+    line: notebook.Line = where[offence.line - 1]
+    placed: Offence = replace(offence, line=line.line, cell=line.cell)
+    # A notebook's cells have no import block.
+    if offence.edit is not None and (offence.edit.imports or offence.edit.guarded):
+        return replace(placed, edit=None)
+    if offence.edit is not None and offence.edit.edit is Edit.DECLARE:
+        statement: int = where[offence.edit.span[0] - 1].line
+        placed = replace(placed, edit=offence.edit._replace(span=(statement, offence.edit.span[1])))
+    return placed
+
+
+# One changed part of a fixed file (the file, or a notebook's cell): its label, old and new lines.
+_Change: TypeAlias = tuple[str, list[str], list[str]]
+# One JSON value a coverage report holds, and one file's row of them.
+_Scalar: TypeAlias = str | int | float
+_Row: TypeAlias = dict[str, _Scalar]
+
+
+class Fixed(NamedTuple):
+    """`raw`, fixed: its new text, and each changed part (the file, or a notebook's cell)."""
+
+    text: str
+    changes: list[_Change]  # each part's label and old and new lines
+
+
+def _fixed(raw: str, name: Path, offences: Sequence[Offence]) -> Fixed:
+    """Add each fixable offence's annotation to `raw`, the text of `name`.
+
+    Returns:
+      The new text, and each changed part.
+
+    """
+    if name.suffix == notebook.SUFFIX:
+        text: str
+        cells: list[notebook.Cell]
+        text, cells = notebook.fix(raw, offences)
+        return Fixed(text, [(f"{name}:cell {c.number}", c.old, c.new) for c in cells])
+    old: list[str] = lines(raw)
+    new: list[str] = fixes.apply(old, offences)
+    return Fixed("".join(new), [(str(name), old, new)] if new != old else [])
+
+
+def _diff(raw: str, name: Path, offences: Sequence[Offence]) -> str:
+    return "".join(
+        "".join(difflib.unified_diff(old, new, label, label))
+        for label, old, new in _fixed(raw, name, offences).changes
+    )
+
+
+def fix_file(path: Path, offences: Sequence[Offence]) -> int:
+    """Add each fixable offence's annotation to `path` (a notebook's, in its cells), in its encoding.
+
+    Raises `UnicodeEncodeError`, leaving the file as it was, when an annotation can't be written in
+    the file's encoding (a PEP 263 declaration's).
+
+    Returns:
+      How many offences were fixed.
+
+    """
+    count: int
+    if not (count := sum(1 for o in offences if o.edit is not None)):
+        return 0
+    data: bytes = path.read_bytes()
+    encoding: str = encoding_of(path, data)
+    _ = path.write_bytes(_fixed(data.decode(encoding), path, offences).text.encode(encoding))
+    return count
+
+
+def _shown_lines(raw: str, name: Path) -> dict[tuple[int | None, int], str]:
+    """Map each line of `raw` (a notebook's, in its cells) to its text, as offences are placed.
+
+    Returns:
+      Each line's text, by its cell (`None` outside a notebook) and line number.
+
+    """
+    source: str
+    where: list[notebook.Line]
+    source, where = _source(raw, name)
+    return {
+        (where[index].cell if where else None, where[index].line if where else index + 1): text
+        for index, text in enumerate(source.splitlines())
+    }
+
+
+def _checked(path: Path, name: Path, checks: Checks, outside: Outside) -> tuple[str, Checked]:
+    """Read `path` and check it as `name`; raises what reading or parsing it does.
+
+    Returns:
+      Its text, and what checking it found.
+
+    """
+    raw: str = read_text(path)
+    return raw, checked_text(raw, name, checks, outside=outside)
+
+
+def _read_checked(
+    path: Path,
+    name: Path,
+    checks: Checks,
+    outside: Outside,
+) -> tuple[str, Checked, str]:
+    """Read and check `path`, turning a read or parse error into a message instead of raising.
+
+    Returns:
+      Its text and what checking it found (nothing on error), and the error message (empty on success).
+
+    """
+    raw: str
+    checked: Checked
+    try:
+        raw, checked = _checked(path, name, checks, outside)
+    except (OSError, ValueError, SyntaxError) as error:  # UnicodeDecodeError is a ValueError
+        return "", Checked([], Returns()), f"{name}: error: {error}"
+    return raw, checked, ""
+
+
+def _results(raw: str, name: Path, offences: Sequence[Offence], options: Options) -> list[Result]:
+    """Turn the offences in `raw`, the text of `name`, into the results to report.
+
+    Returns:
+      Each one the options don't filter out, with its source line and its fix as a text edit.
+
+    """
+    shown: dict[tuple[int | None, int], str] = _shown_lines(raw, name)
+    # A notebook's cells have no file lines for an edit to point at.
+    text: list[str] = [] if name.suffix == notebook.SUFFIX else lines(raw)
+    return [
+        r._replace(
+            source=shown.get((r.offence.cell, r.offence.line), ""),
+            replacements=fixes.replacements(text, r.offence) if text else (),
+        )
+        for r in options.filter.results(name, offences)
+    ]
+
+
+def _check_path(path: Path, outside: Outside, options: Options) -> CheckRun:
+    """Check (and fix, or diff) one file, given what's known of it from outside it.
+
+    Returns:
+      What it found; a file that can't be read or parsed is an error.
+
+    """
+    name: Path = options.input.name(path)
+    raw: str
+    checked: Checked
+    error: str
+    raw, checked, error = _read_checked(path, name, options.checks, outside)
+    if error:
+        return CheckRun(error=error)
+    return replace(
+        _handled(path, (raw, name), checked.offences, options),
+        returned=checked.returned,
+        calls=checked.calls,
+    )
+
+
+def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], options: Options) -> CheckRun:
+    """Report, fix or diff the offences found in `path` (`text`: its text, and its name).
+
+    Returns:
+      What was found, and fixed.
+
+    """
+    raw: str
+    name: Path
+    raw, name = text
+    baselined: int
+    offences, baselined = options.filter.unbaselined(name, offences)
+    results: list[Result] = _results(raw, name, offences, options)
+    fixing: list[Offence] = [
+        r.offence for r in results if r.offence.edit is not None and options.guesses.applies(r.offence)
+    ]
+    if options.mode is Mode.DIFF:
+        return CheckRun(text=_diff(raw, name, fixing))
+    if options.mode is not Mode.FIX:
+        return CheckRun(results, baselined)
+    left: list[Result] = [r for r in results if r.offence not in fixing]
+    if path == STDIN:  # the fixed source goes to stdout
+        return CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
+    fixed: int
+    try:
+        fixed = fix_file(path, fixing)
+    except UnicodeEncodeError as failure:  # the file is left as it was, its offences unfixed
+        # Its canonical name: PyPy reports `latin1` where CPython says `latin-1`.
+        encoding: str = codecs.lookup(failure.encoding).name
+        message: str = f"an annotation can't be written in its encoding, {encoding}; left as it was"
+        return CheckRun(results, baselined, error=f"{name}: error: {message}")
+    if fixed:
+        _ = parsed.take(raw)  # the tree of the text it had: no check will ask for it again
+    return CheckRun(
+        left,
+        baselined,
+        fixed,
+        made=[r._replace(replacements=(), fixed=True) for r in results if r.offence in fixing],
+        rounds=(_added(raw, name, fixing),),
+    )
+
+
+def _added(raw: str, name: Path, fixing: Sequence[Offence]) -> Added:
+    """Find the lines fixing `fixing` adds to `raw`, the text of `name` (see `fixes.inserted`).
+
+    Returns:
+      Each one's cell (`None` outside a notebook) and the line it goes before; a notebook's cells
+      get declarations alone.
+
+    """
+    if name.suffix != notebook.SUFFIX:
+        return tuple((None, line) for line in fixes.inserted(lines(raw), fixing))
+    declared: list[tuple[int, int]] = [
+        (o.cell or 0, o.edit.span[0]) for o in fixing if o.edit is not None and o.edit.edit is Edit.DECLARE
+    ]
+    return tuple(sorted(declared))
+
+
+def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
+    """Check one file, unfiltered, for --write-baseline.
+
+    Returns:
+      Every offence found; a file that can't be read or parsed is an error.
+
+    """
+    name: Path = options.input.name(path)
+    checked: Checked
+    error: str
+    _, checked, error = _read_checked(path, name, options.checks, outside)
+    return BaselineRun(error=error) if error else BaselineRun(checked.offences, returned=checked.returned)
+
+
+def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
+    """Count one file's typed first bindings.
+
+    Returns:
+      The counts; a file that can't be read or parsed is an error.
+
+    """
+    name: Path = options.input.name(path)
+    try:
+        return CoverageRun(annotation_coverage(_source(read_text(path), name)[0], options.checks))
+    except (OSError, ValueError, SyntaxError) as error:
+        return CoverageRun(error=f"{name}: error: {error}")
+
+
+def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
+    """Check every file (see `_check_rounds`), its rounds by one set of worker processes.
+
+    Returns:
+      The names, and what each file found.
+
+    """
+    schedule.forget()
+    with kept():
+        return _check_rounds(options)
+
+
+def _check_rounds(options: Options) -> tuple[list[Path], list[FileRun]]:
+    """Check every file (`--jobs` at a time), in order.
+
+    With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
+    hints first, and with `--fix`, each file a round changed is asked again and checked again, as its
+    new annotations change what the checker infers (see `settled`). A baseline records offences,
+    and coverage counts annotations: hints change neither, only what `--fix` offers.
+
+    Returns:
+      The names, and what each file found.
+
+    """
+    paths: list[Path] = list(
+        python_files(options.input.paths, options.input.exclude, tracked=options.input.gitignore),
+    )
+    check: Callable[[Path, Outside], FileRun]
+    if options.mode is Mode.COVERAGE:
+        check = partial(_cover_path, options=options)
+    elif options.mode is Mode.WRITE_BASELINE:
+        check = partial(_baseline_path, options=options)
+    else:
+        check = partial(_check_path, options=options)
+    names: list[Path] = [options.input.name(path) for path in paths]
+    if options.mode in {Mode.COVERAGE, Mode.WRITE_BASELINE}:
+        return names, checked_all(paths, check, options)[0]
+    if not options.infer_with:
+        return names, settled(paths, check, options, None)
+    session: hints.Session
+    # Imported only for `--infer-with`: it's slow to import, for every run's start.
+    with cast("type[hints.Session]", importlib.import_module("constricter.cli.lsp.hints").Session)(
+        options.infer_with,
+        Path.cwd(),
+        options.jobs,
+        options.infer_memory,
+        kept=not options.infer_fresh,
+    ) as session:
+        runs: list[FileRun] = settled(paths, check, options, session)
+        checker: str
+        path: Path
+        for checker, path in session.abandoned:
+            name: Path = options.input.name(path)
+            _ = sys.stderr.write(f"constricter: warning: {checker} hung on {name} twice: no hints for it\n")
+    return names, runs
+
+
+def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
+    """Print the results (`runs` is check/fix/diff mode's: every other mode has its own printing).
+
+    Returns:
+      The exit status.
+
+    """
+    checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
+    results: list[Result] = [result for run in checked for result in run.results]
+    output: Output = options.output
+    text: bool = output.fmt in {Format.TEXT, Format.FULL}
+    # With `--show-fixes`, the fixes `--fix` made too: after the rest, as text or JSON.
+    shown: list[Result] = [*results, *(made for run in checked if output.show_fixes for made in run.made)]
+    line: str
+    for line in (
+        statistics(results)
+        if text and output.statistics
+        else render(output.fmt, shown if output.fmt is Format.JSON else results)
+    ):
+        _ = sys.stdout.write(f"{line}\n")
+    for line in fix_reasons(shown) if text and output.show_fixes else ():
+        _ = sys.stdout.write(f"{line}\n")
+    errors: int = sum(r.offence.is_error(r.level) for r in results)
+    if text and not output.quiet:
+        parts: list[str] = [
+            f"Found {errors} error(s) and {len(results) - errors} warning(s) in {files} file(s)",
+        ]
+        if options.mode is Mode.FIX:
+            parts.append(f"fixed {sum(run.fixed for run in checked)}")
+            guesses: int
+            if guesses := sum(r.offence.unsafe for r in results):
+                likely: int = sum(r.offence.likely for r in results)
+                parts.append(
+                    f"{guesses} more with --unsafe-fixes" + f" ({likely} with --likely)" * bool(likely),
+                )
+        if options.filter.baseline_file is not None:
+            parts.append(f"{sum(run.baselined for run in checked)} baselined")
+        _ = sys.stdout.write("; ".join(parts) + ".\n")
+    return EXIT_FOUND if errors else EXIT_CLEAN
+
+
+def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun]) -> int:
+    """Print each file's and the total annotation coverage (`runs` is --coverage mode's).
+
+    Returns:
+      The exit status.
+
+    """
+    covered: Sequence[CoverageRun] = cast("Sequence[CoverageRun]", runs)
+    counted: list[tuple[Path, Coverage]] = [
+        (path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
+    ]
+    total: Coverage = Coverage(
+        sum(c.typed for _, c in counted),
+        sum(c.total for _, c in counted),
+        sum(c.widened for _, c in counted),
+    )
+    if options.output.fmt is Format.JSON:
+        report: dict[str, _Scalar | list[_Row]] = {
+            **_counts(total),
+            "files": [{"path": str(path), **_counts(c)} for path, c in counted],
+        }
+        _ = sys.stdout.write(json.dumps(report, indent=2) + "\n")
+    else:
+        path: Path
+        c: Coverage
+        for path, c in counted:
+            _ = sys.stdout.write(f"{path}: {_typed(c)}\n")
+        _ = sys.stdout.write(f"Total: {_typed(total)} in {len(counted)} file(s).\n")
+    threshold: float | None = options.output.fail_under
+    return EXIT_FOUND if threshold is not None and total.percent < threshold else EXIT_CLEAN
+
+
+def _counts(found: Coverage) -> _Row:
+    """Name a coverage's counts, for `--format=json`.
+
+    Returns:
+      Them, by name.
+
+    """
+    return {
+        "typed": found.typed,
+        "total": found.total,
+        "widened": found.widened,
+        "percent": round(found.percent, 1),
+    }
+
+
+def _typed(found: Coverage) -> str:
+    """Write a coverage as text: `3/4 typed (75.0%)`, then what a widening annotated, if anything.
+
+    Returns:
+      It.
+
+    """
+    wider: str = f", {found.widened} widened" if found.widened else ""
+    return f"{found.typed}/{found.total} typed ({found.percent:.1f}%){wider}"
+
+
+def _run(options: Options) -> int:
+    """Check, fix, diff, count or baseline, printing what that finds.
+
+    Returns:
+      The exit status.
+
+    """
+    names: list[Path]
+    runs: list[FileRun]
+    with collecting.by_hand():
+        try:
+            names, runs = _check_all(options)
+        except HintError as error:
+            _ = sys.stderr.write(f"constricter: error: {error}\n")
+            return EXIT_ERROR
+    _ = sys.stderr.write("".join(f"{run.error}\n" for run in runs if run.error))
+    failed: bool = any(run.error for run in runs)
+    status: int
+    if options.mode is Mode.WRITE_BASELINE and options.filter.baseline_file is not None:
+        file: Path = options.filter.baseline_file
+        baselined: Sequence[BaselineRun] = cast("Sequence[BaselineRun]", runs)
+        found: dict[str, list[Offence]] = {
+            baseline.key(n, file): run.found for n, run in zip(names, baselined, strict=True)
+        }
+        _ = sys.stdout.write(f"Wrote {baseline.write(file, found)} offence(s) to {file}.\n")
+        status = EXIT_CLEAN
+    elif options.mode is Mode.COVERAGE:
+        status = _coverage(options, names, runs)
+    elif options.mode is Mode.DIFF:
+        checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
+        diffs: str = "".join(run.text for run in checked)
+        _ = sys.stdout.write(diffs)
+        status = EXIT_FOUND if diffs else EXIT_CLEAN
+    elif options.mode is Mode.FIX and options.input.paths == [STDIN] and runs and not failed:
+        fixed: CheckRun = cast("CheckRun", runs[0])
+        _ = sys.stdout.write(fixed.text)  # standard input, fixed, is the whole output
+        status = EXIT_FOUND if any(r.offence.is_error(r.level) for r in fixed.results) else EXIT_CLEAN
+    else:
+        status = _report(options, runs, len(names))
+    return EXIT_ERROR if failed else status
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the command.
+
+    Returns:
+      Its exit status.
+
+    """
+    options: Options = Options.parse(argv)
+    file: Path | None = options.output.output_file
+    stream: TextIO
+    with (
+        file.open("w", encoding="utf-8", newline="\n")
+        if file
+        else contextlib.nullcontext(sys.stdout) as stream,
+        contextlib.redirect_stdout(stream),
+    ):
+        status: int = _run(options)
+    return EXIT_CLEAN if options.output.exit_zero and status == EXIT_FOUND else status