python-constricter 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- constricter/__init__.py +1 -1
- constricter/__main__.py +19 -2
- constricter/cli/command.py +103 -55
- constricter/cli/config.py +28 -1
- constricter/cli/hints.py +18 -12
- constricter/cli/options.py +29 -1
- constricter/cli/protocol.py +25 -5
- constricter/cli/remembered.py +52 -0
- constricter/cli/report.py +7 -5
- constricter/cli/runs.py +49 -2
- constricter/cli/schedule.py +221 -68
- constricter/cli/traced.py +191 -79
- constricter/cli/workers.py +85 -23
- constricter/fix/core/fixes.py +51 -7
- constricter/fix/core/imports.py +100 -2
- constricter/fix/core/inherited.py +117 -17
- constricter/fix/core/known.py +177 -49
- constricter/fix/core/signatures.py +22 -4
- constricter/fix/index/atoms.py +18 -3
- constricter/fix/index/awaits.py +90 -0
- constricter/fix/index/beyond.py +87 -29
- constricter/fix/index/callers.py +2 -1
- constricter/fix/index/chained.py +69 -0
- constricter/fix/index/classnames.py +17 -0
- constricter/fix/index/declared.py +68 -4
- constricter/fix/index/decorated.py +3 -1
- constricter/fix/index/installed.py +35 -3
- constricter/fix/index/linked.py +142 -0
- constricter/fix/index/loose.py +98 -0
- constricter/fix/index/managed.py +63 -0
- constricter/fix/index/modules.py +331 -47
- constricter/fix/index/offers.py +4 -1
- constricter/fix/index/order.py +10 -3
- constricter/fix/index/own_methods.py +67 -0
- constricter/fix/index/own_overloads.py +14 -1
- constricter/fix/index/own_types.py +161 -0
- constricter/fix/index/plain.py +3 -1
- constricter/fix/index/project.py +137 -153
- constricter/fix/index/sides.py +102 -7
- constricter/fix/index/strings.py +46 -0
- constricter/fix/index/stubbed.py +51 -12
- constricter/fix/index/taken.py +45 -0
- constricter/fix/index/tuples.py +54 -28
- constricter/fix/libraries/library.py +95 -7
- constricter/fix/libraries/opened.py +57 -16
- constricter/fix/libraries/overloads.py +32 -6
- constricter/fix/libraries/stdlib.py +64 -13
- constricter/fix/libraries/walks.py +49 -0
- constricter/fix/tables/attributes.json +2 -0
- constricter/fix/tables/classes.json +1 -0
- constricter/fix/tables/generic_attributes.json +2 -0
- constricter/fix/tables/method_overloads.json +14 -6
- constricter/fix/tables/method_signatures.json +50 -20
- constricter/fix/tables/methods.json +1 -0
- constricter/fix/tables/overloads.json +38 -37
- constricter/fix/tables/scalars.json +2 -0
- constricter/fix/tables/source.json +1 -1
- constricter/fix/tables/type_parameters.json +1 -0
- constricter/fix/values/aliased.py +9 -1
- constricter/fix/values/bodies.py +76 -0
- constricter/fix/values/callables.py +321 -0
- constricter/fix/values/called.py +10 -12
- constricter/fix/values/classvars.py +124 -21
- constricter/fix/values/decided.py +22 -2
- constricter/fix/values/doubts.py +104 -27
- constricter/fix/values/entered.py +192 -34
- constricter/fix/values/fills.py +160 -13
- constricter/fix/values/guesses.py +51 -19
- constricter/fix/values/inference.py +48 -54
- constricter/fix/values/members.py +273 -34
- constricter/fix/values/narrowed.py +60 -4
- constricter/fix/values/operated.py +7 -6
- constricter/fix/values/ordered.py +74 -0
- constricter/fix/values/returned.py +319 -224
- constricter/fix/values/returns.py +44 -1
- constricter/fix/values/shapes.py +132 -32
- constricter/fix/values/targets.py +42 -15
- constricter/fix/values/unchecked.py +162 -0
- constricter/offences.py +45 -6
- constricter/recording.py +571 -0
- constricter/rules/annotations.py +30 -10
- constricter/rules/binding.py +229 -37
- constricter/rules/calls.py +58 -8
- constricter/rules/checker.py +177 -70
- constricter/rules/keys.py +79 -0
- constricter/rules/late.py +349 -10
- constricter/rules/quoted.py +32 -0
- constricter/rules/rebinding.py +35 -10
- constricter/rules/recorded.py +12 -8
- constricter/rules/scope.py +241 -74
- constricter/rules/syntax.py +20 -0
- constricter/rules/tables.py +18 -4
- constricter/rules/widened.py +322 -0
- constricter/trace.py +247 -254
- {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/METADATA +42 -40
- python_constricter-0.3.5.dist-info/RECORD +140 -0
- {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/entry_points.txt +1 -1
- python_constricter-0.3.3.dist-info/RECORD +0 -122
- {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/WHEEL +0 -0
- {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py
CHANGED
constricter/__main__.py
CHANGED
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
|
-
"""`python -m constricter
|
|
2
|
+
"""`python -m constricter`, and the `constricter` script."""
|
|
3
3
|
|
|
4
|
+
import contextlib
|
|
5
|
+
import os
|
|
4
6
|
import sys
|
|
5
7
|
|
|
6
8
|
from constricter.cli.command import main
|
|
7
9
|
|
|
10
|
+
|
|
11
|
+
def run() -> None:
|
|
12
|
+
"""Run the command, and leave with its status at once.
|
|
13
|
+
|
|
14
|
+
Not by the interpreter's own exit, which frees every parsed file and the index an object at a
|
|
15
|
+
time: a fifth of a check of a thousand files. What the command wrote is flushed first; nothing
|
|
16
|
+
else waits on the exit (its workers have ended, its output file is closed).
|
|
17
|
+
"""
|
|
18
|
+
status: int = main()
|
|
19
|
+
with contextlib.suppress(OSError, ValueError): # a reader that left, a stream closed already
|
|
20
|
+
_ = sys.stdout.flush()
|
|
21
|
+
_ = sys.stderr.flush()
|
|
22
|
+
os._exit(status)
|
|
23
|
+
|
|
24
|
+
|
|
8
25
|
if __name__ == "__main__": # not when `--jobs` workers import it
|
|
9
|
-
|
|
26
|
+
run()
|
constricter/cli/command.py
CHANGED
|
@@ -21,11 +21,11 @@ from constricter.cli.options import Mode, Options, Output
|
|
|
21
21
|
from constricter.cli.paths import STDIN, python_files
|
|
22
22
|
from constricter.cli.protocol import HintError
|
|
23
23
|
from constricter.cli.report import Format, Result, fix_reasons, render, statistics
|
|
24
|
-
from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
|
|
25
|
-
from constricter.cli.workers import Workers
|
|
24
|
+
from constricter.cli.runs import Added, BaselineRun, CheckRun, CoverageRun, FileRun, merged
|
|
25
|
+
from constricter.cli.workers import Workers, kept, started
|
|
26
26
|
from constricter.fix.core import fixes
|
|
27
27
|
from constricter.fix.core.known import Callee, Hints, Outside, Returns
|
|
28
|
-
from constricter.fix.index import callers, decorated, installed, plain, project
|
|
28
|
+
from constricter.fix.index import callers, decorated, installed, loose, plain, project
|
|
29
29
|
from constricter.noqa import lines, unsuppressed
|
|
30
30
|
from constricter.offences import (
|
|
31
31
|
DEFAULT_CHECKS,
|
|
@@ -127,6 +127,7 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
|
|
|
127
127
|
|
|
128
128
|
|
|
129
129
|
# One changed part of a fixed file (the file, or a notebook's cell): its label, old and new lines.
|
|
130
|
+
_Hinted: TypeAlias = dict[Path, tuple[Hints, ...]] # each file's hints, a checker's each
|
|
130
131
|
_Change: TypeAlias = tuple[str, list[str], list[str]]
|
|
131
132
|
# One JSON value a coverage report holds, and one file's row of them.
|
|
132
133
|
_Scalar: TypeAlias = str | int | float
|
|
@@ -308,7 +309,29 @@ def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], option
|
|
|
308
309
|
return CheckRun(results, baselined, error=f"{name}: error: {message}")
|
|
309
310
|
if fixed:
|
|
310
311
|
_ = parsed.take(raw) # the tree of the text it had: no check will ask for it again
|
|
311
|
-
return CheckRun(
|
|
312
|
+
return CheckRun(
|
|
313
|
+
left,
|
|
314
|
+
baselined,
|
|
315
|
+
fixed,
|
|
316
|
+
made=[r._replace(replacements=(), fixed=True) for r in results if r.offence in fixing],
|
|
317
|
+
rounds=(_added(raw, name, fixing),),
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _added(raw: str, name: Path, fixing: Sequence[Offence]) -> Added:
|
|
322
|
+
"""Find the lines fixing `fixing` adds to `raw`, the text of `name` (see `fixes.inserted`).
|
|
323
|
+
|
|
324
|
+
Returns:
|
|
325
|
+
Each one's cell (`None` outside a notebook) and the line it goes before; a notebook's cells
|
|
326
|
+
get declarations alone.
|
|
327
|
+
|
|
328
|
+
"""
|
|
329
|
+
if name.suffix != notebook.SUFFIX:
|
|
330
|
+
return tuple((None, line) for line in fixes.inserted(lines(raw), fixing))
|
|
331
|
+
declared: list[tuple[int, int]] = [
|
|
332
|
+
(o.cell or 0, o.edit.span[0]) for o in fixing if o.edit is not None and o.edit.edit is Edit.DECLARE
|
|
333
|
+
]
|
|
334
|
+
return tuple(sorted(declared))
|
|
312
335
|
|
|
313
336
|
|
|
314
337
|
def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
|
|
@@ -340,6 +363,17 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
|
|
|
340
363
|
|
|
341
364
|
|
|
342
365
|
def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
366
|
+
"""Check every file (see `_check_rounds`), its rounds by one set of worker processes.
|
|
367
|
+
|
|
368
|
+
Returns:
|
|
369
|
+
The names, and what each file found.
|
|
370
|
+
|
|
371
|
+
"""
|
|
372
|
+
with kept():
|
|
373
|
+
return _check_rounds(options)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _check_rounds(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
343
377
|
"""Check every file (`--jobs` at a time), in order.
|
|
344
378
|
|
|
345
379
|
With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
|
|
@@ -401,7 +435,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
|
401
435
|
index: int
|
|
402
436
|
run: FileRun
|
|
403
437
|
for index, run in zip(again, redone, strict=True):
|
|
404
|
-
runs[index] =
|
|
438
|
+
runs[index] = merged(runs[index], run)
|
|
405
439
|
again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
|
|
406
440
|
checker: str
|
|
407
441
|
path: Path
|
|
@@ -447,7 +481,7 @@ def _called_again(
|
|
|
447
481
|
calling = [
|
|
448
482
|
at
|
|
449
483
|
for at in calling
|
|
450
|
-
if
|
|
484
|
+
if loose.returned(modules, paths[at], {}) != loose.returned(before, paths[at], {})
|
|
451
485
|
]
|
|
452
486
|
return runs, _checked_more(paths, runs, calling, (check, options, session), modules)
|
|
453
487
|
|
|
@@ -473,7 +507,7 @@ def _checked_more(
|
|
|
473
507
|
at: int
|
|
474
508
|
run: FileRun
|
|
475
509
|
for at, run in zip(which, redone, strict=True):
|
|
476
|
-
runs[at] =
|
|
510
|
+
runs[at] = merged(runs[at], run) if how[1].mode is Mode.FIX else run
|
|
477
511
|
return modules
|
|
478
512
|
|
|
479
513
|
|
|
@@ -509,44 +543,29 @@ def _checked_all(
|
|
|
509
543
|
What each file found, and the index of every file's module.
|
|
510
544
|
|
|
511
545
|
"""
|
|
512
|
-
|
|
546
|
+
# Before a worker starts: one forked while the servers' threads run could wait on a lock one holds.
|
|
547
|
+
hinted: _Hinted = {} if session is None else session.hints(_texts(paths))
|
|
513
548
|
if options.input.trace is not None:
|
|
514
549
|
hinted = traced.merged(hinted, traced.load(options.input.trace), paths)
|
|
515
550
|
coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
if not coverage:
|
|
527
|
-
modules = plain.settled(
|
|
528
|
-
decorated.passed(installed.with_installed(modules, installed.search_path())),
|
|
529
|
-
options.checks.plain_bases,
|
|
530
|
-
)
|
|
531
|
-
return schedule.checked(
|
|
532
|
-
paths,
|
|
533
|
-
check,
|
|
534
|
-
pool,
|
|
535
|
-
(modules, hinted),
|
|
536
|
-
_merged if options.mode is Mode.FIX else None,
|
|
551
|
+
pool: Workers | None = started(paths, options.jobs) if options.jobs != 1 and len(paths) > 1 else None
|
|
552
|
+
if modules is None and pool:
|
|
553
|
+
modules = project.Index({}, []) if coverage else pool.index()
|
|
554
|
+
elif modules is None:
|
|
555
|
+
modules = project.Index({}, []) if coverage else project.index(paths)
|
|
556
|
+
collecting.indexed()
|
|
557
|
+
if not coverage:
|
|
558
|
+
modules = plain.settled(
|
|
559
|
+
decorated.passed(installed.with_installed(modules, installed.search_path())),
|
|
560
|
+
options.checks.plain_bases,
|
|
537
561
|
)
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
"""
|
|
547
|
-
first: CheckRun = cast("CheckRun", before)
|
|
548
|
-
later: CheckRun = cast("CheckRun", after)
|
|
549
|
-
return replace(later, fixed=first.fixed + later.fixed)
|
|
562
|
+
return schedule.checked(
|
|
563
|
+
paths,
|
|
564
|
+
check,
|
|
565
|
+
pool,
|
|
566
|
+
(modules, hinted),
|
|
567
|
+
merged if options.mode is Mode.FIX else None,
|
|
568
|
+
)
|
|
550
569
|
|
|
551
570
|
|
|
552
571
|
def _texts(paths: Sequence[Path]) -> dict[Path, str]:
|
|
@@ -579,10 +598,16 @@ def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
|
|
|
579
598
|
results: list[Result] = [result for run in checked for result in run.results]
|
|
580
599
|
output: Output = options.output
|
|
581
600
|
text: bool = output.fmt in {Format.TEXT, Format.FULL}
|
|
601
|
+
# With `--show-fixes`, the fixes `--fix` made too: after the rest, as text or JSON.
|
|
602
|
+
shown: list[Result] = [*results, *(made for run in checked if output.show_fixes for made in run.made)]
|
|
582
603
|
line: str
|
|
583
|
-
for line in
|
|
604
|
+
for line in (
|
|
605
|
+
statistics(results)
|
|
606
|
+
if text and output.statistics
|
|
607
|
+
else render(output.fmt, shown if output.fmt is Format.JSON else results)
|
|
608
|
+
):
|
|
584
609
|
_ = sys.stdout.write(f"{line}\n")
|
|
585
|
-
for line in fix_reasons(
|
|
610
|
+
for line in fix_reasons(shown) if text and output.show_fixes else ():
|
|
586
611
|
_ = sys.stdout.write(f"{line}\n")
|
|
587
612
|
errors: int = sum(r.offence.is_error(r.level) for r in results)
|
|
588
613
|
if text and not output.quiet:
|
|
@@ -611,30 +636,53 @@ def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun])
|
|
|
611
636
|
counted: list[tuple[Path, Coverage]] = [
|
|
612
637
|
(path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
|
|
613
638
|
]
|
|
614
|
-
total: Coverage = Coverage(
|
|
639
|
+
total: Coverage = Coverage(
|
|
640
|
+
sum(c.typed for _, c in counted),
|
|
641
|
+
sum(c.total for _, c in counted),
|
|
642
|
+
sum(c.widened for _, c in counted),
|
|
643
|
+
)
|
|
615
644
|
if options.output.fmt is Format.JSON:
|
|
616
645
|
report: dict[str, _Scalar | list[_Row]] = {
|
|
617
|
-
|
|
618
|
-
"
|
|
619
|
-
"percent": round(total.percent, 1),
|
|
620
|
-
"files": [
|
|
621
|
-
{"path": str(path), "typed": c.typed, "total": c.total, "percent": round(c.percent, 1)}
|
|
622
|
-
for path, c in counted
|
|
623
|
-
],
|
|
646
|
+
**_counts(total),
|
|
647
|
+
"files": [{"path": str(path), **_counts(c)} for path, c in counted],
|
|
624
648
|
}
|
|
625
649
|
_ = sys.stdout.write(json.dumps(report, indent=2) + "\n")
|
|
626
650
|
else:
|
|
627
651
|
path: Path
|
|
628
652
|
c: Coverage
|
|
629
653
|
for path, c in counted:
|
|
630
|
-
_ = sys.stdout.write(f"{path}: {
|
|
631
|
-
_ = sys.stdout.write(
|
|
632
|
-
f"Total: {total.typed}/{total.total} typed ({total.percent:.1f}%) in {len(counted)} file(s).\n",
|
|
633
|
-
)
|
|
654
|
+
_ = sys.stdout.write(f"{path}: {_typed(c)}\n")
|
|
655
|
+
_ = sys.stdout.write(f"Total: {_typed(total)} in {len(counted)} file(s).\n")
|
|
634
656
|
threshold: float | None = options.output.fail_under
|
|
635
657
|
return EXIT_FOUND if threshold is not None and total.percent < threshold else EXIT_CLEAN
|
|
636
658
|
|
|
637
659
|
|
|
660
|
+
def _counts(found: Coverage) -> _Row:
|
|
661
|
+
"""Name a coverage's counts, for `--format=json`.
|
|
662
|
+
|
|
663
|
+
Returns:
|
|
664
|
+
Them, by name.
|
|
665
|
+
|
|
666
|
+
"""
|
|
667
|
+
return {
|
|
668
|
+
"typed": found.typed,
|
|
669
|
+
"total": found.total,
|
|
670
|
+
"widened": found.widened,
|
|
671
|
+
"percent": round(found.percent, 1),
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _typed(found: Coverage) -> str:
|
|
676
|
+
"""Write a coverage as text: `3/4 typed (75.0%)`, then what a widening annotated, if anything.
|
|
677
|
+
|
|
678
|
+
Returns:
|
|
679
|
+
It.
|
|
680
|
+
|
|
681
|
+
"""
|
|
682
|
+
wider: str = f", {found.widened} widened" if found.widened else ""
|
|
683
|
+
return f"{found.typed}/{found.total} typed ({found.percent:.1f}%){wider}"
|
|
684
|
+
|
|
685
|
+
|
|
638
686
|
def _run(options: Options) -> int:
|
|
639
687
|
"""Check, fix, diff, count or baseline, printing what that finds.
|
|
640
688
|
|
constricter/cli/config.py
CHANGED
|
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias, cast
|
|
|
10
10
|
|
|
11
11
|
from constricter.cli.protocol import SERVERS
|
|
12
12
|
from constricter.jsonc import is_int
|
|
13
|
-
from constricter.offences import FIX_KINDS, LEVELS, MESSAGES
|
|
13
|
+
from constricter.offences import FIX_KINDS, LEVELS, MESSAGES, WIDEN_KINDS
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from datetime import date, datetime, time
|
|
@@ -32,6 +32,7 @@ def unknown_codes(codes: Sequence[str]) -> list[str]:
|
|
|
32
32
|
|
|
33
33
|
DEFAULT_BASELINE: Final = "constricter-baseline.json"
|
|
34
34
|
_MIN_PYTHON: Final = "min_python"
|
|
35
|
+
_ALL_KINDS: Final = "all" # `fix-widen`'s: every wider type
|
|
35
36
|
_VERSION: Final = re.compile(r"\d+\.\d+") # `min-python`'s: `3.11`
|
|
36
37
|
# A lower bound among `requires-python`'s specifiers: `>=3.11`, `~=3.11`, `==3.11.*`, `>3.10`.
|
|
37
38
|
_LOWER_BOUND: Final = re.compile(r"(?:>=|~=|==|>)\s*(\d+)(?:\.(\d+))?")
|
|
@@ -160,6 +161,31 @@ def unknown_fix_kinds(kinds: Sequence[str]) -> list[str]:
|
|
|
160
161
|
return [kind for kind in kinds if kind not in FIX_KINDS]
|
|
161
162
|
|
|
162
163
|
|
|
164
|
+
def unknown_widen_kinds(kinds: Sequence[str]) -> list[str]:
|
|
165
|
+
"""Check `kinds` against `WIDEN_KINDS`' ids, and `all`.
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
Those that aren't one.
|
|
169
|
+
|
|
170
|
+
"""
|
|
171
|
+
return [kind for kind in kinds if kind not in WIDEN_KINDS | {_ALL_KINDS}]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def widen_kinds(kinds: Sequence[str]) -> list[str]:
|
|
175
|
+
"""Read `fix-widen`'s kinds, `all` as every one.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
Them.
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
return sorted(WIDEN_KINDS) if _ALL_KINDS in kinds else list(kinds)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _widen_kinds(value: _Toml) -> list[str] | None:
|
|
185
|
+
kinds: list[str] | None = [value] if isinstance(value, str) else _strings(value)
|
|
186
|
+
return None if kinds is None or unknown_widen_kinds(kinds) else widen_kinds(kinds)
|
|
187
|
+
|
|
188
|
+
|
|
163
189
|
def _fix_kinds(value: _Toml) -> list[str] | None:
|
|
164
190
|
kinds: list[str] | None = _strings(value)
|
|
165
191
|
return None if kinds is None or unknown_fix_kinds(kinds) else kinds
|
|
@@ -246,6 +272,7 @@ _READERS: dict[str, Callable[[_Toml], Default | None]] = {
|
|
|
246
272
|
"fix-select": _fix_kinds,
|
|
247
273
|
"fix-ignore": _fix_kinds,
|
|
248
274
|
"unsafe-fix-select": _fix_kinds,
|
|
275
|
+
"fix-widen": _widen_kinds,
|
|
249
276
|
"fix-plain-bases": _strings,
|
|
250
277
|
"type-comments": _flag,
|
|
251
278
|
"all-scopes": _flag,
|
constricter/cli/hints.py
CHANGED
|
@@ -33,7 +33,7 @@ from functools import partial
|
|
|
33
33
|
from pathlib import Path
|
|
34
34
|
from typing import IO, Final, NamedTuple, Self, TypeAlias, cast
|
|
35
35
|
|
|
36
|
-
from constricter.cli import edits, guard, protocol
|
|
36
|
+
from constricter.cli import edits, guard, protocol, remembered
|
|
37
37
|
from constricter.cli.protocol import SERVERS, HintError, HungError, Server
|
|
38
38
|
from constricter.fix.core.known import Hints, Offered
|
|
39
39
|
|
|
@@ -47,8 +47,6 @@ _Task: TypeAlias = Callable[[], _Found] # one server's work: the hints of each
|
|
|
47
47
|
|
|
48
48
|
_BATCH: Final = 8 # files a free server takes at a time: few, so none is left with the slow ones
|
|
49
49
|
MEMORY: Final = 8 << 30 # the most a checker's servers use together, by default (`--infer-memory`)
|
|
50
|
-
_SERVER_MEMORY: Final = 600 << 20 # a server's memory, before the files (basedpyright's, measured)
|
|
51
|
-
_MEMORY_PER_BYTE: Final = 130 # and for each byte of them: 1.2 GB for SQLAlchemy's 8 MB, 3.4 for pandas' 21
|
|
52
50
|
_MEMINFO: Final = Path("/proc/meminfo")
|
|
53
51
|
_FILES_PER_SERVER: Final = 32 # fewer files than this each don't pay for another server's start
|
|
54
52
|
# Seconds without a word from a server before it's taken for hung. Not an answer's wait: pandas'
|
|
@@ -228,11 +226,10 @@ class Checker:
|
|
|
228
226
|
"""One checker's servers, and which of them has each file.
|
|
229
227
|
|
|
230
228
|
It has as many as its files need (one per `_FILES_PER_SERVER`), up to the command's `--jobs`, the
|
|
231
|
-
checker's own most (`Server.most
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
One server it always has, whatever the memory.
|
|
229
|
+
checker's own most (`Server.most`), and as many as fit in its memory: `--infer-memory` (never
|
|
230
|
+
more than is available), or by default `MEMORY` (never more than half what's available). Each
|
|
231
|
+
server loads the whole program it checks, or its share of it, by its checker's own measure (see
|
|
232
|
+
`Server.fitting`). One server it always has, whatever the memory.
|
|
236
233
|
|
|
237
234
|
The files are handed out a few at a time (`_BATCH`), the biggest first, to whichever server is
|
|
238
235
|
free: one file's analysis can take far longer than another's, and none waits on a busy one. A
|
|
@@ -244,7 +241,8 @@ class Checker:
|
|
|
244
241
|
self.name: str = name
|
|
245
242
|
self.root: Path = root
|
|
246
243
|
self.command: list[str] = command(name)
|
|
247
|
-
self.
|
|
244
|
+
self.server: Server = SERVERS[name]
|
|
245
|
+
self.most: int = max(1, min(servers, self.server.most))
|
|
248
246
|
self.memory: int = budget(memory, available_memory())
|
|
249
247
|
self.servers: list[Connection] = []
|
|
250
248
|
self.assigned: dict[Path, int] = {} # each file's server, by its place in `servers`
|
|
@@ -258,13 +256,14 @@ class Checker:
|
|
|
258
256
|
|
|
259
257
|
"""
|
|
260
258
|
size: int = sum(len(text) for text in files.values())
|
|
261
|
-
fit: int = max(1, self.memory
|
|
259
|
+
fit: int = max(1, self.server.fitting(self.memory, size))
|
|
262
260
|
return max(len(self.servers), min(self.most, -(-len(files) // _FILES_PER_SERVER), fit))
|
|
263
261
|
|
|
264
262
|
def tasks(self, files: Mapping[Path, str]) -> list[_Task]:
|
|
265
263
|
"""Make each server's work on `files`: its own files first, then new ones as it's free for them.
|
|
266
264
|
|
|
267
|
-
A file abandoned (see `_alone`) isn't
|
|
265
|
+
A file abandoned (see `_alone`), in this run or an earlier one (see `remembered`), isn't
|
|
266
|
+
asked about again.
|
|
268
267
|
|
|
269
268
|
Returns:
|
|
270
269
|
A task per server, giving the hints of every file it was sent.
|
|
@@ -274,8 +273,14 @@ class Checker:
|
|
|
274
273
|
path: Path
|
|
275
274
|
for path in files.keys() & self.assigned.keys() - self.abandoned:
|
|
276
275
|
own.setdefault(self.assigned[path], {})[path] = files[path]
|
|
276
|
+
# One it hung on in an earlier run, unchanged since: left without hints from the start.
|
|
277
|
+
self.abandoned.update(
|
|
278
|
+
path
|
|
279
|
+
for path in files.keys() - self.assigned.keys() - self.abandoned
|
|
280
|
+
if remembered.hung(self.name, self.command, files[path])
|
|
281
|
+
)
|
|
277
282
|
fresh: list[Path] = sorted(
|
|
278
|
-
files.keys() - self.assigned.keys(),
|
|
283
|
+
files.keys() - self.assigned.keys() - self.abandoned,
|
|
279
284
|
key=lambda new: (-len(files[new]), new),
|
|
280
285
|
)
|
|
281
286
|
batches: queue.SimpleQueue[dict[Path, str]] = queue.SimpleQueue()
|
|
@@ -352,6 +357,7 @@ class Checker:
|
|
|
352
357
|
found.update(server.answers(server.ask({path: text})))
|
|
353
358
|
except HungError as hung:
|
|
354
359
|
self.abandoned.add(path)
|
|
360
|
+
remembered.remember(self.name, self.command, text)
|
|
355
361
|
if len(self.abandoned) > _MOST_ABANDONED:
|
|
356
362
|
error: str = f"{self.name} hung on {len(self.abandoned)} file(s), each asked about alone"
|
|
357
363
|
raise HintError(error) from hung
|
constricter/cli/options.py
CHANGED
|
@@ -18,6 +18,8 @@ from constricter.cli.config import (
|
|
|
18
18
|
project_root,
|
|
19
19
|
unknown_codes,
|
|
20
20
|
unknown_fix_kinds,
|
|
21
|
+
unknown_widen_kinds,
|
|
22
|
+
widen_kinds,
|
|
21
23
|
)
|
|
22
24
|
from constricter.cli.explain import explain
|
|
23
25
|
from constricter.cli.paths import STDIN, excluded
|
|
@@ -133,6 +135,24 @@ def _fix_kinds(text: str) -> list[str]:
|
|
|
133
135
|
return kinds
|
|
134
136
|
|
|
135
137
|
|
|
138
|
+
def _widen_kinds(text: str) -> list[str]:
|
|
139
|
+
"""Read a comma-separated list of the wider types `--fix` may write (`untyped-parameters`), or `all`.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Them.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
argparse.ArgumentTypeError: One isn't a widening's id.
|
|
146
|
+
|
|
147
|
+
"""
|
|
148
|
+
kinds: list[str] = [kind.strip() for kind in text.split(",") if kind.strip()]
|
|
149
|
+
unknown: list[str]
|
|
150
|
+
if unknown := unknown_widen_kinds(kinds):
|
|
151
|
+
message: str = f"no wider type is called {', '.join(unknown)} (see docs/FIXES.md)"
|
|
152
|
+
raise argparse.ArgumentTypeError(message)
|
|
153
|
+
return widen_kinds(kinds)
|
|
154
|
+
|
|
155
|
+
|
|
136
156
|
def _version(text: str) -> tuple[int, int]:
|
|
137
157
|
"""Read `--min-python`: a Python version (`3.11`).
|
|
138
158
|
|
|
@@ -333,6 +353,13 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
333
353
|
metavar="KINDS",
|
|
334
354
|
help="treat guesses from these mechanisms (constructor, narrow) as certain",
|
|
335
355
|
)
|
|
356
|
+
_ = parser.add_argument(
|
|
357
|
+
"--fix-widen",
|
|
358
|
+
type=_widen_kinds,
|
|
359
|
+
default=[],
|
|
360
|
+
metavar="KINDS",
|
|
361
|
+
help="also write these wider types, each marked (untyped-parameters, unions, ...; all: every one)",
|
|
362
|
+
)
|
|
336
363
|
_ = parser.add_argument(
|
|
337
364
|
"--fix-plain-bases",
|
|
338
365
|
type=_listed,
|
|
@@ -367,7 +394,7 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
367
394
|
_ = parser.add_argument(
|
|
368
395
|
"--show-fixes",
|
|
369
396
|
action="store_true",
|
|
370
|
-
help="after the report, list each fix and how the value decided it
|
|
397
|
+
help="after the report, list each fix and how the value decided it; with --fix, those made too",
|
|
371
398
|
)
|
|
372
399
|
_ = parser.add_argument(
|
|
373
400
|
"--statistics",
|
|
@@ -594,6 +621,7 @@ class Options:
|
|
|
594
621
|
frozenset(cast("list[str]", args.fix_select)),
|
|
595
622
|
frozenset(cast("list[str]", args.fix_ignore)),
|
|
596
623
|
frozenset(cast("list[str]", args.unsafe_fix_select)),
|
|
624
|
+
frozenset(cast("list[str]", args.fix_widen)),
|
|
597
625
|
),
|
|
598
626
|
min_python=cast("tuple[int, int] | None", args.min_python),
|
|
599
627
|
plain_bases=(*PLAIN_BASES, *cast("list[str]", args.fix_plain_bases)),
|
constricter/cli/protocol.py
CHANGED
|
@@ -33,21 +33,41 @@ class Server(NamedTuple):
|
|
|
33
33
|
"""A checker's language server: how it's started, and how many are worth running at once.
|
|
34
34
|
|
|
35
35
|
Its executable, the arguments that start it over stdio, and the most of it to run (up to
|
|
36
|
-
`MAX_SERVERS
|
|
36
|
+
`MAX_SERVERS`). And the memory its servers take, as measured: `memory` bytes each before the
|
|
37
|
+
files, and `per_byte` more for each byte of them: each server's again where each holds the
|
|
38
|
+
`whole` program, once among them all where each holds what it's asked about.
|
|
37
39
|
"""
|
|
38
40
|
|
|
39
41
|
executable: str
|
|
40
42
|
args: tuple[str, ...]
|
|
41
43
|
most: int
|
|
44
|
+
memory: int
|
|
45
|
+
per_byte: int
|
|
46
|
+
whole: bool = False
|
|
47
|
+
|
|
48
|
+
def fitting(self, budget: int, size: int) -> int:
|
|
49
|
+
"""Count the servers that fit in `budget` bytes, for files of `size` bytes together.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
It: none where one doesn't fit.
|
|
53
|
+
|
|
54
|
+
"""
|
|
55
|
+
if self.whole:
|
|
56
|
+
return budget // (self.memory + self.per_byte * size)
|
|
57
|
+
return max(0, budget - self.per_byte * size) // self.memory
|
|
42
58
|
|
|
43
59
|
|
|
44
60
|
MAX_SERVERS: Final = 4 # a checker's, at most: each holds its own copy of the program it checks
|
|
45
61
|
SERVERS: Final = {
|
|
46
62
|
# One thread each: more servers check more at once (sqlalchemy's hints: 10.8s with one, 7.2s with four).
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
63
|
+
# Each holds the whole program: 1.2 GB for sqlalchemy's 8 MB, 3.4 for pandas's 21.
|
|
64
|
+
"basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS, 600 << 20, 130, whole=True),
|
|
65
|
+
# Parallel itself, and still quicker shared out: a hinted check of pandas takes 36s with one, 25s
|
|
66
|
+
# with two, 22s with three and 21s with four (1.2 GB with one, 1.7 with two, 2.5 with four).
|
|
67
|
+
"ty": Server("ty", ("server",), MAX_SERVERS, 420 << 20, 38),
|
|
68
|
+
# Parallel too: pandas's takes 132s with one, 58s with two and 32s with four (2.8 GB, 3.2, 4.2);
|
|
69
|
+
# the standard library's 39 MB, 4.3 GB with one and 5.6 with four.
|
|
70
|
+
"pyrefly": Server("pyrefly", ("lsp",), MAX_SERVERS, 480 << 20, 120),
|
|
51
71
|
}
|
|
52
72
|
_PROBE: Final = "--version" # quick for each; basedpyright-langserver's exits 1 all the same
|
|
53
73
|
_PROBE_TIMEOUT: Final = 10.0 # seconds it may take
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
"""The files a type checker's server hung on, remembered between runs.
|
|
3
|
+
|
|
4
|
+
A server that hangs on a file costs the run its timeout twice, and a restart each time: once with
|
|
5
|
+
the file among others, once with it alone. The file's text, with the checker and its executable as
|
|
6
|
+
it was (its size and when it changed), names an empty file in the cache; while there's one, the
|
|
7
|
+
file is left without hints from the start. A new version of the checker, or a changed file, is
|
|
8
|
+
asked again.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import hashlib
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from constricter.fix.index.installed import cache_directory
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _marker(checker: str, command: Sequence[str], text: str) -> Path:
|
|
20
|
+
"""Name what remembers that `checker`, started by `command`, hung on a file holding `text`.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
Its path in the cache.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
version: str = ""
|
|
27
|
+
with contextlib.suppress(OSError):
|
|
28
|
+
version = _version(Path(command[0]))
|
|
29
|
+
key: str = "\0".join((checker, command[0], version, text))
|
|
30
|
+
return cache_directory().with_name("hung") / hashlib.sha256(key.encode("utf-8", "replace")).hexdigest()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _version(executable: Path) -> str: # which build of it: its size and when it changed
|
|
34
|
+
return f"{executable.stat().st_size}:{executable.stat().st_mtime_ns}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def hung(checker: str, command: Sequence[str], text: str) -> bool:
|
|
38
|
+
"""Check whether `checker` hung on a file holding `text`, in an earlier run.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
Whether it did.
|
|
42
|
+
|
|
43
|
+
"""
|
|
44
|
+
return _marker(checker, command, text).exists()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def remember(checker: str, command: Sequence[str], text: str) -> None:
|
|
48
|
+
"""Remember that `checker` hung on a file holding `text`; nothing, where the cache can't be written."""
|
|
49
|
+
marker: Path = _marker(checker, command, text)
|
|
50
|
+
with contextlib.suppress(OSError):
|
|
51
|
+
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
marker.touch()
|
constricter/cli/report.py
CHANGED
|
@@ -40,6 +40,7 @@ class Result(NamedTuple):
|
|
|
40
40
|
level: Level
|
|
41
41
|
source: str = "" # the offending line, for `--format=full`
|
|
42
42
|
replacements: tuple[Replacement, ...] = () # its fix as text edits, for SARIF and rdjson
|
|
43
|
+
fixed: bool = False # `--fix` made its fix: listed with `--show-fixes`, on the line it was on
|
|
43
44
|
|
|
44
45
|
@property
|
|
45
46
|
def suggestions(self) -> tuple[Replacement, ...]:
|
|
@@ -208,6 +209,7 @@ def _json(results: Sequence[Result]) -> Iterator[str]:
|
|
|
208
209
|
"severity": r.severity,
|
|
209
210
|
"message": r.offence.message,
|
|
210
211
|
"cell": r.offence.cell,
|
|
212
|
+
"fixed": r.fixed,
|
|
211
213
|
"fix": None
|
|
212
214
|
if r.offence.fix is None
|
|
213
215
|
else {
|
|
@@ -360,11 +362,11 @@ def render(fmt: Format, results: Sequence[Result]) -> Iterator[str]:
|
|
|
360
362
|
|
|
361
363
|
|
|
362
364
|
def fix_reasons(results: Sequence[Result]) -> Iterator[str]:
|
|
363
|
-
"""List the annotation each fixable result would get, and how its value decided it.
|
|
365
|
+
"""List the annotation each fixable result would get, or got, and how its value decided it.
|
|
364
366
|
|
|
365
367
|
Yields:
|
|
366
|
-
A line each: where, the name, the annotation and its reason;
|
|
367
|
-
`--unsafe-fixes`.
|
|
368
|
+
A line each: where, the name (`fixed`, for a fix `--fix` made), the annotation and its reason;
|
|
369
|
+
a guess left unmade says it needs `--unsafe-fixes`.
|
|
368
370
|
|
|
369
371
|
"""
|
|
370
372
|
r: Result
|
|
@@ -372,11 +374,11 @@ def fix_reasons(results: Sequence[Result]) -> Iterator[str]:
|
|
|
372
374
|
if r.offence.fix is not None:
|
|
373
375
|
cell: str = "" if r.offence.cell is None else f"cell {r.offence.cell}:"
|
|
374
376
|
where: str = f"{r.path}:{cell}{r.offence.line}:{r.offence.col + 1}"
|
|
375
|
-
guess: str = " (a guess: --unsafe-fixes)" if r.offence.unsafe else ""
|
|
377
|
+
guess: str = " (a guess: --unsafe-fixes)" if r.offence.unsafe and not r.fixed else ""
|
|
376
378
|
kinds: str = ", ".join(sorted(r.offence.edit.kinds if r.offence.edit else ()))
|
|
377
379
|
written: str = f"`{r.offence.fix}`" if r.offence.fix else "drop its annotation"
|
|
378
380
|
decided: str = f"{written}, from {r.offence.reason} [{kinds}]"
|
|
379
|
-
yield f"{where}: fix {r.offence.name!r}: {decided}{guess}"
|
|
381
|
+
yield f"{where}: {'fixed' if r.fixed else 'fix'} {r.offence.name!r}: {decided}{guess}"
|
|
380
382
|
|
|
381
383
|
|
|
382
384
|
def statistics(results: Sequence[Result]) -> Iterator[str]:
|