python-constricter 0.3.2__py3-none-any.whl → 0.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- constricter/__init__.py +1 -1
- constricter/__main__.py +19 -2
- constricter/cli/command.py +111 -56
- constricter/cli/config.py +30 -2
- constricter/cli/hints.py +18 -12
- constricter/cli/options.py +57 -3
- constricter/cli/protocol.py +25 -5
- constricter/cli/remembered.py +52 -0
- constricter/cli/report.py +7 -5
- constricter/cli/runs.py +49 -2
- constricter/cli/schedule.py +241 -68
- constricter/cli/traced.py +401 -0
- constricter/cli/workers.py +85 -23
- constricter/fix/core/asked.py +56 -0
- constricter/fix/core/fixes.py +63 -9
- constricter/fix/core/imports.py +196 -10
- constricter/fix/core/inherited.py +128 -15
- constricter/fix/core/known.py +271 -55
- constricter/fix/core/signatures.py +29 -6
- constricter/fix/index/atoms.py +22 -4
- constricter/fix/index/awaits.py +90 -0
- constricter/fix/index/beyond.py +148 -0
- constricter/fix/index/callers.py +3 -2
- constricter/fix/index/chained.py +69 -0
- constricter/fix/index/classnames.py +17 -0
- constricter/fix/index/declared.py +83 -3
- constricter/fix/index/decorated.py +4 -2
- constricter/fix/index/fixtures.py +176 -0
- constricter/fix/index/installed.py +68 -15
- constricter/fix/index/linked.py +142 -0
- constricter/fix/index/loose.py +98 -0
- constricter/fix/index/managed.py +63 -0
- constricter/fix/index/modules.py +368 -46
- constricter/fix/index/offers.py +4 -1
- constricter/fix/index/order.py +13 -3
- constricter/fix/index/own_methods.py +67 -0
- constricter/fix/index/own_overloads.py +70 -0
- constricter/fix/index/own_types.py +161 -0
- constricter/fix/index/plain.py +13 -8
- constricter/fix/index/project.py +222 -180
- constricter/fix/index/read.py +70 -0
- constricter/fix/index/sides.py +102 -7
- constricter/fix/index/strings.py +46 -0
- constricter/fix/index/stubbed.py +86 -80
- constricter/fix/index/taken.py +45 -0
- constricter/fix/index/tuples.py +54 -28
- constricter/fix/libraries/library.py +208 -6
- constricter/fix/libraries/opened.py +88 -15
- constricter/fix/libraries/overloads.py +155 -14
- constricter/fix/libraries/stdlib.py +278 -24
- constricter/fix/libraries/walks.py +49 -0
- constricter/fix/tables/attributes.json +25 -0
- constricter/fix/tables/awaited.json +69 -0
- constricter/fix/tables/awaited_overloads.json +14 -0
- constricter/fix/tables/classes.json +3 -1
- constricter/fix/tables/elements.json +156 -0
- constricter/fix/tables/functions.json +0 -1
- constricter/fix/tables/generic_attributes.json +201 -2
- constricter/fix/tables/method_overloads.json +146 -99
- constricter/fix/tables/method_signatures.json +229 -40
- constricter/fix/tables/methods.json +35 -10
- constricter/fix/tables/overloads.json +88 -63
- constricter/fix/tables/scalars.json +5 -0
- constricter/fix/tables/source.json +4 -1
- constricter/fix/tables/subscriptable.json +1 -0
- constricter/fix/tables/type_parameters.json +3 -0
- constricter/fix/tables/variables.json +21 -0
- constricter/fix/values/aliased.py +9 -1
- constricter/fix/values/bodies.py +76 -0
- constricter/fix/values/callables.py +321 -0
- constricter/fix/values/called.py +10 -12
- constricter/fix/values/classvars.py +216 -40
- constricter/fix/values/decided.py +74 -11
- constricter/fix/values/displays.py +87 -0
- constricter/fix/values/doubts.py +152 -28
- constricter/fix/values/entered.py +209 -30
- constricter/fix/values/fills.py +208 -30
- constricter/fix/values/guesses.py +80 -21
- constricter/fix/values/hinted.py +94 -7
- constricter/fix/values/inference.py +145 -146
- constricter/fix/values/members.py +326 -39
- constricter/fix/values/narrowed.py +60 -4
- constricter/fix/values/operated.py +200 -0
- constricter/fix/values/ordered.py +74 -0
- constricter/fix/values/returned.py +487 -226
- constricter/fix/values/returns.py +45 -1
- constricter/fix/values/shapes.py +267 -52
- constricter/fix/values/targets.py +51 -18
- constricter/fix/values/unchecked.py +162 -0
- constricter/offences.py +48 -6
- constricter/recording.py +571 -0
- constricter/rules/annotations.py +52 -13
- constricter/rules/binding.py +237 -48
- constricter/rules/calls.py +222 -26
- constricter/rules/checker.py +301 -77
- constricter/rules/decorators.py +12 -0
- constricter/rules/keys.py +79 -0
- constricter/rules/late.py +351 -12
- constricter/rules/parsed.py +42 -3
- constricter/rules/quoted.py +32 -0
- constricter/rules/rebinding.py +38 -11
- constricter/rules/recorded.py +53 -10
- constricter/rules/scope.py +291 -88
- constricter/rules/syntax.py +20 -0
- constricter/rules/tables.py +79 -9
- constricter/rules/widened.py +322 -0
- constricter/trace.py +378 -0
- {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/METADATA +42 -38
- python_constricter-0.3.5.dist-info/RECORD +140 -0
- {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/entry_points.txt +1 -1
- python_constricter-0.3.2.dist-info/RECORD +0 -110
- {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/WHEEL +0 -0
- {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py
CHANGED
constricter/__main__.py
CHANGED
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
|
-
"""`python -m constricter
|
|
2
|
+
"""`python -m constricter`, and the `constricter` script."""
|
|
3
3
|
|
|
4
|
+
import contextlib
|
|
5
|
+
import os
|
|
4
6
|
import sys
|
|
5
7
|
|
|
6
8
|
from constricter.cli.command import main
|
|
7
9
|
|
|
10
|
+
|
|
11
|
+
def run() -> None:
|
|
12
|
+
"""Run the command, and leave with its status at once.
|
|
13
|
+
|
|
14
|
+
Not by the interpreter's own exit, which frees every parsed file and the index an object at a
|
|
15
|
+
time: a fifth of a check of a thousand files. What the command wrote is flushed first; nothing
|
|
16
|
+
else waits on the exit (its workers have ended, its output file is closed).
|
|
17
|
+
"""
|
|
18
|
+
status: int = main()
|
|
19
|
+
with contextlib.suppress(OSError, ValueError): # a reader that left, a stream closed already
|
|
20
|
+
_ = sys.stdout.flush()
|
|
21
|
+
_ = sys.stderr.flush()
|
|
22
|
+
os._exit(status)
|
|
23
|
+
|
|
24
|
+
|
|
8
25
|
if __name__ == "__main__": # not when `--jobs` workers import it
|
|
9
|
-
|
|
26
|
+
run()
|
constricter/cli/command.py
CHANGED
|
@@ -16,16 +16,16 @@ from pathlib import Path
|
|
|
16
16
|
from typing import TYPE_CHECKING, Final, NamedTuple, TextIO, TypeAlias, cast
|
|
17
17
|
|
|
18
18
|
from constricter import notebook
|
|
19
|
-
from constricter.cli import baseline, collecting, schedule
|
|
19
|
+
from constricter.cli import baseline, collecting, schedule, traced
|
|
20
20
|
from constricter.cli.options import Mode, Options, Output
|
|
21
21
|
from constricter.cli.paths import STDIN, python_files
|
|
22
22
|
from constricter.cli.protocol import HintError
|
|
23
23
|
from constricter.cli.report import Format, Result, fix_reasons, render, statistics
|
|
24
|
-
from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
|
|
25
|
-
from constricter.cli.workers import Workers
|
|
24
|
+
from constricter.cli.runs import Added, BaselineRun, CheckRun, CoverageRun, FileRun, merged
|
|
25
|
+
from constricter.cli.workers import Workers, kept, started
|
|
26
26
|
from constricter.fix.core import fixes
|
|
27
27
|
from constricter.fix.core.known import Callee, Hints, Outside, Returns
|
|
28
|
-
from constricter.fix.index import callers, decorated, installed, plain, project
|
|
28
|
+
from constricter.fix.index import callers, decorated, installed, loose, plain, project
|
|
29
29
|
from constricter.noqa import lines, unsuppressed
|
|
30
30
|
from constricter.offences import (
|
|
31
31
|
DEFAULT_CHECKS,
|
|
@@ -33,6 +33,7 @@ from constricter.offences import (
|
|
|
33
33
|
Edit,
|
|
34
34
|
Offence,
|
|
35
35
|
)
|
|
36
|
+
from constricter.rules import parsed
|
|
36
37
|
from constricter.rules.checker import Checked, Coverage, annotation_coverage, checked_source
|
|
37
38
|
|
|
38
39
|
if TYPE_CHECKING: # slow to import, and only needed for `--infer-with`
|
|
@@ -126,6 +127,7 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
|
|
|
126
127
|
|
|
127
128
|
|
|
128
129
|
# One changed part of a fixed file (the file, or a notebook's cell): its label, old and new lines.
|
|
130
|
+
_Hinted: TypeAlias = dict[Path, tuple[Hints, ...]] # each file's hints, a checker's each
|
|
129
131
|
_Change: TypeAlias = tuple[str, list[str], list[str]]
|
|
130
132
|
# One JSON value a coverage report holds, and one file's row of them.
|
|
131
133
|
_Scalar: TypeAlias = str | int | float
|
|
@@ -297,13 +299,39 @@ def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], option
|
|
|
297
299
|
left: list[Result] = [r for r in results if r.offence not in fixing]
|
|
298
300
|
if path == STDIN: # the fixed source goes to stdout
|
|
299
301
|
return CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
|
|
302
|
+
fixed: int
|
|
300
303
|
try:
|
|
301
|
-
|
|
304
|
+
fixed = fix_file(path, fixing)
|
|
302
305
|
except UnicodeEncodeError as failure: # the file is left as it was, its offences unfixed
|
|
303
306
|
# Its canonical name: PyPy reports `latin1` where CPython says `latin-1`.
|
|
304
307
|
encoding: str = codecs.lookup(failure.encoding).name
|
|
305
308
|
message: str = f"an annotation can't be written in its encoding, {encoding}; left as it was"
|
|
306
309
|
return CheckRun(results, baselined, error=f"{name}: error: {message}")
|
|
310
|
+
if fixed:
|
|
311
|
+
_ = parsed.take(raw) # the tree of the text it had: no check will ask for it again
|
|
312
|
+
return CheckRun(
|
|
313
|
+
left,
|
|
314
|
+
baselined,
|
|
315
|
+
fixed,
|
|
316
|
+
made=[r._replace(replacements=(), fixed=True) for r in results if r.offence in fixing],
|
|
317
|
+
rounds=(_added(raw, name, fixing),),
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _added(raw: str, name: Path, fixing: Sequence[Offence]) -> Added:
|
|
322
|
+
"""Find the lines fixing `fixing` adds to `raw`, the text of `name` (see `fixes.inserted`).
|
|
323
|
+
|
|
324
|
+
Returns:
|
|
325
|
+
Each one's cell (`None` outside a notebook) and the line it goes before; a notebook's cells
|
|
326
|
+
get declarations alone.
|
|
327
|
+
|
|
328
|
+
"""
|
|
329
|
+
if name.suffix != notebook.SUFFIX:
|
|
330
|
+
return tuple((None, line) for line in fixes.inserted(lines(raw), fixing))
|
|
331
|
+
declared: list[tuple[int, int]] = [
|
|
332
|
+
(o.cell or 0, o.edit.span[0]) for o in fixing if o.edit is not None and o.edit.edit is Edit.DECLARE
|
|
333
|
+
]
|
|
334
|
+
return tuple(sorted(declared))
|
|
307
335
|
|
|
308
336
|
|
|
309
337
|
def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
|
|
@@ -335,6 +363,17 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
|
|
|
335
363
|
|
|
336
364
|
|
|
337
365
|
def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
366
|
+
"""Check every file (see `_check_rounds`), its rounds by one set of worker processes.
|
|
367
|
+
|
|
368
|
+
Returns:
|
|
369
|
+
The names, and what each file found.
|
|
370
|
+
|
|
371
|
+
"""
|
|
372
|
+
with kept():
|
|
373
|
+
return _check_rounds(options)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _check_rounds(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
338
377
|
"""Check every file (`--jobs` at a time), in order.
|
|
339
378
|
|
|
340
379
|
With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
|
|
@@ -396,7 +435,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
|
396
435
|
index: int
|
|
397
436
|
run: FileRun
|
|
398
437
|
for index, run in zip(again, redone, strict=True):
|
|
399
|
-
runs[index] =
|
|
438
|
+
runs[index] = merged(runs[index], run)
|
|
400
439
|
again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
|
|
401
440
|
checker: str
|
|
402
441
|
path: Path
|
|
@@ -442,7 +481,7 @@ def _called_again(
|
|
|
442
481
|
calling = [
|
|
443
482
|
at
|
|
444
483
|
for at in calling
|
|
445
|
-
if
|
|
484
|
+
if loose.returned(modules, paths[at], {}) != loose.returned(before, paths[at], {})
|
|
446
485
|
]
|
|
447
486
|
return runs, _checked_more(paths, runs, calling, (check, options, session), modules)
|
|
448
487
|
|
|
@@ -468,7 +507,7 @@ def _checked_more(
|
|
|
468
507
|
at: int
|
|
469
508
|
run: FileRun
|
|
470
509
|
for at, run in zip(which, redone, strict=True):
|
|
471
|
-
runs[at] =
|
|
510
|
+
runs[at] = merged(runs[at], run) if how[1].mode is Mode.FIX else run
|
|
472
511
|
return modules
|
|
473
512
|
|
|
474
513
|
|
|
@@ -504,42 +543,29 @@ def _checked_all(
|
|
|
504
543
|
What each file found, and the index of every file's module.
|
|
505
544
|
|
|
506
545
|
"""
|
|
507
|
-
|
|
546
|
+
# Before a worker starts: one forked while the servers' threads run could wait on a lock one holds.
|
|
547
|
+
hinted: _Hinted = {} if session is None else session.hints(_texts(paths))
|
|
548
|
+
if options.input.trace is not None:
|
|
549
|
+
hinted = traced.merged(hinted, traced.load(options.input.trace), paths)
|
|
508
550
|
coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
if not coverage:
|
|
520
|
-
modules = plain.settled(
|
|
521
|
-
decorated.passed(installed.with_installed(modules, installed.search_path())),
|
|
522
|
-
options.checks.plain_bases,
|
|
523
|
-
)
|
|
524
|
-
return schedule.checked(
|
|
525
|
-
paths,
|
|
526
|
-
check,
|
|
527
|
-
pool,
|
|
528
|
-
(modules, hinted),
|
|
529
|
-
_merged if options.mode is Mode.FIX else None,
|
|
551
|
+
pool: Workers | None = started(paths, options.jobs) if options.jobs != 1 and len(paths) > 1 else None
|
|
552
|
+
if modules is None and pool:
|
|
553
|
+
modules = project.Index({}, []) if coverage else pool.index()
|
|
554
|
+
elif modules is None:
|
|
555
|
+
modules = project.Index({}, []) if coverage else project.index(paths)
|
|
556
|
+
collecting.indexed()
|
|
557
|
+
if not coverage:
|
|
558
|
+
modules = plain.settled(
|
|
559
|
+
decorated.passed(installed.with_installed(modules, installed.search_path())),
|
|
560
|
+
options.checks.plain_bases,
|
|
530
561
|
)
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
"""
|
|
540
|
-
first: CheckRun = cast("CheckRun", before)
|
|
541
|
-
later: CheckRun = cast("CheckRun", after)
|
|
542
|
-
return replace(later, fixed=first.fixed + later.fixed)
|
|
562
|
+
return schedule.checked(
|
|
563
|
+
paths,
|
|
564
|
+
check,
|
|
565
|
+
pool,
|
|
566
|
+
(modules, hinted),
|
|
567
|
+
merged if options.mode is Mode.FIX else None,
|
|
568
|
+
)
|
|
543
569
|
|
|
544
570
|
|
|
545
571
|
def _texts(paths: Sequence[Path]) -> dict[Path, str]:
|
|
@@ -572,10 +598,16 @@ def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
|
|
|
572
598
|
results: list[Result] = [result for run in checked for result in run.results]
|
|
573
599
|
output: Output = options.output
|
|
574
600
|
text: bool = output.fmt in {Format.TEXT, Format.FULL}
|
|
601
|
+
# With `--show-fixes`, the fixes `--fix` made too: after the rest, as text or JSON.
|
|
602
|
+
shown: list[Result] = [*results, *(made for run in checked if output.show_fixes for made in run.made)]
|
|
575
603
|
line: str
|
|
576
|
-
for line in
|
|
604
|
+
for line in (
|
|
605
|
+
statistics(results)
|
|
606
|
+
if text and output.statistics
|
|
607
|
+
else render(output.fmt, shown if output.fmt is Format.JSON else results)
|
|
608
|
+
):
|
|
577
609
|
_ = sys.stdout.write(f"{line}\n")
|
|
578
|
-
for line in fix_reasons(
|
|
610
|
+
for line in fix_reasons(shown) if text and output.show_fixes else ():
|
|
579
611
|
_ = sys.stdout.write(f"{line}\n")
|
|
580
612
|
errors: int = sum(r.offence.is_error(r.level) for r in results)
|
|
581
613
|
if text and not output.quiet:
|
|
@@ -604,30 +636,53 @@ def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun])
|
|
|
604
636
|
counted: list[tuple[Path, Coverage]] = [
|
|
605
637
|
(path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
|
|
606
638
|
]
|
|
607
|
-
total: Coverage = Coverage(
|
|
639
|
+
total: Coverage = Coverage(
|
|
640
|
+
sum(c.typed for _, c in counted),
|
|
641
|
+
sum(c.total for _, c in counted),
|
|
642
|
+
sum(c.widened for _, c in counted),
|
|
643
|
+
)
|
|
608
644
|
if options.output.fmt is Format.JSON:
|
|
609
645
|
report: dict[str, _Scalar | list[_Row]] = {
|
|
610
|
-
|
|
611
|
-
"
|
|
612
|
-
"percent": round(total.percent, 1),
|
|
613
|
-
"files": [
|
|
614
|
-
{"path": str(path), "typed": c.typed, "total": c.total, "percent": round(c.percent, 1)}
|
|
615
|
-
for path, c in counted
|
|
616
|
-
],
|
|
646
|
+
**_counts(total),
|
|
647
|
+
"files": [{"path": str(path), **_counts(c)} for path, c in counted],
|
|
617
648
|
}
|
|
618
649
|
_ = sys.stdout.write(json.dumps(report, indent=2) + "\n")
|
|
619
650
|
else:
|
|
620
651
|
path: Path
|
|
621
652
|
c: Coverage
|
|
622
653
|
for path, c in counted:
|
|
623
|
-
_ = sys.stdout.write(f"{path}: {
|
|
624
|
-
_ = sys.stdout.write(
|
|
625
|
-
f"Total: {total.typed}/{total.total} typed ({total.percent:.1f}%) in {len(counted)} file(s).\n",
|
|
626
|
-
)
|
|
654
|
+
_ = sys.stdout.write(f"{path}: {_typed(c)}\n")
|
|
655
|
+
_ = sys.stdout.write(f"Total: {_typed(total)} in {len(counted)} file(s).\n")
|
|
627
656
|
threshold: float | None = options.output.fail_under
|
|
628
657
|
return EXIT_FOUND if threshold is not None and total.percent < threshold else EXIT_CLEAN
|
|
629
658
|
|
|
630
659
|
|
|
660
|
+
def _counts(found: Coverage) -> _Row:
|
|
661
|
+
"""Name a coverage's counts, for `--format=json`.
|
|
662
|
+
|
|
663
|
+
Returns:
|
|
664
|
+
Them, by name.
|
|
665
|
+
|
|
666
|
+
"""
|
|
667
|
+
return {
|
|
668
|
+
"typed": found.typed,
|
|
669
|
+
"total": found.total,
|
|
670
|
+
"widened": found.widened,
|
|
671
|
+
"percent": round(found.percent, 1),
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _typed(found: Coverage) -> str:
|
|
676
|
+
"""Write a coverage as text: `3/4 typed (75.0%)`, then what a widening annotated, if anything.
|
|
677
|
+
|
|
678
|
+
Returns:
|
|
679
|
+
It.
|
|
680
|
+
|
|
681
|
+
"""
|
|
682
|
+
wider: str = f", {found.widened} widened" if found.widened else ""
|
|
683
|
+
return f"{found.typed}/{found.total} typed ({found.percent:.1f}%){wider}"
|
|
684
|
+
|
|
685
|
+
|
|
631
686
|
def _run(options: Options) -> int:
|
|
632
687
|
"""Check, fix, diff, count or baseline, printing what that finds.
|
|
633
688
|
|
constricter/cli/config.py
CHANGED
|
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias, cast
|
|
|
10
10
|
|
|
11
11
|
from constricter.cli.protocol import SERVERS
|
|
12
12
|
from constricter.jsonc import is_int
|
|
13
|
-
from constricter.offences import FIX_KINDS, LEVELS, MESSAGES
|
|
13
|
+
from constricter.offences import FIX_KINDS, LEVELS, MESSAGES, WIDEN_KINDS
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from datetime import date, datetime, time
|
|
@@ -32,6 +32,7 @@ def unknown_codes(codes: Sequence[str]) -> list[str]:
|
|
|
32
32
|
|
|
33
33
|
DEFAULT_BASELINE: Final = "constricter-baseline.json"
|
|
34
34
|
_MIN_PYTHON: Final = "min_python"
|
|
35
|
+
_ALL_KINDS: Final = "all" # `fix-widen`'s: every wider type
|
|
35
36
|
_VERSION: Final = re.compile(r"\d+\.\d+") # `min-python`'s: `3.11`
|
|
36
37
|
# A lower bound among `requires-python`'s specifiers: `>=3.11`, `~=3.11`, `==3.11.*`, `>3.10`.
|
|
37
38
|
_LOWER_BOUND: Final = re.compile(r"(?:>=|~=|==|>)\s*(\d+)(?:\.(\d+))?")
|
|
@@ -160,6 +161,31 @@ def unknown_fix_kinds(kinds: Sequence[str]) -> list[str]:
|
|
|
160
161
|
return [kind for kind in kinds if kind not in FIX_KINDS]
|
|
161
162
|
|
|
162
163
|
|
|
164
|
+
def unknown_widen_kinds(kinds: Sequence[str]) -> list[str]:
|
|
165
|
+
"""Check `kinds` against `WIDEN_KINDS`' ids, and `all`.
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
Those that aren't one.
|
|
169
|
+
|
|
170
|
+
"""
|
|
171
|
+
return [kind for kind in kinds if kind not in WIDEN_KINDS | {_ALL_KINDS}]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def widen_kinds(kinds: Sequence[str]) -> list[str]:
|
|
175
|
+
"""Read `fix-widen`'s kinds, `all` as every one.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
Them.
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
return sorted(WIDEN_KINDS) if _ALL_KINDS in kinds else list(kinds)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _widen_kinds(value: _Toml) -> list[str] | None:
|
|
185
|
+
kinds: list[str] | None = [value] if isinstance(value, str) else _strings(value)
|
|
186
|
+
return None if kinds is None or unknown_widen_kinds(kinds) else widen_kinds(kinds)
|
|
187
|
+
|
|
188
|
+
|
|
163
189
|
def _fix_kinds(value: _Toml) -> list[str] | None:
|
|
164
190
|
kinds: list[str] | None = _strings(value)
|
|
165
191
|
return None if kinds is None or unknown_fix_kinds(kinds) else kinds
|
|
@@ -223,7 +249,7 @@ def _lists(value: _Toml, read: Callable[[_Toml], list[str] | None]) -> dict[str,
|
|
|
223
249
|
|
|
224
250
|
|
|
225
251
|
def _baseline(value: _Toml, root: Path) -> str | None:
|
|
226
|
-
"""Read a baseline path, relative to `root` (the pyproject.toml that names it).
|
|
252
|
+
"""Read a baseline's or a trace's path, relative to `root` (the pyproject.toml that names it).
|
|
227
253
|
|
|
228
254
|
Returns:
|
|
229
255
|
The resolved path, or `None` if `value` isn't a non-empty string.
|
|
@@ -246,6 +272,7 @@ _READERS: dict[str, Callable[[_Toml], Default | None]] = {
|
|
|
246
272
|
"fix-select": _fix_kinds,
|
|
247
273
|
"fix-ignore": _fix_kinds,
|
|
248
274
|
"unsafe-fix-select": _fix_kinds,
|
|
275
|
+
"fix-widen": _widen_kinds,
|
|
249
276
|
"fix-plain-bases": _strings,
|
|
250
277
|
"type-comments": _flag,
|
|
251
278
|
"all-scopes": _flag,
|
|
@@ -276,6 +303,7 @@ def config_defaults(start: Path) -> dict[str, Default]:
|
|
|
276
303
|
readers: dict[str, Callable[[_Toml], Default | None]] = {
|
|
277
304
|
**_READERS,
|
|
278
305
|
"baseline": partial(_baseline, root=path.parent),
|
|
306
|
+
"infer-from": partial(_baseline, root=path.parent),
|
|
279
307
|
}
|
|
280
308
|
defaults: dict[str, Default] = {}
|
|
281
309
|
key: str
|
constricter/cli/hints.py
CHANGED
|
@@ -33,7 +33,7 @@ from functools import partial
|
|
|
33
33
|
from pathlib import Path
|
|
34
34
|
from typing import IO, Final, NamedTuple, Self, TypeAlias, cast
|
|
35
35
|
|
|
36
|
-
from constricter.cli import edits, guard, protocol
|
|
36
|
+
from constricter.cli import edits, guard, protocol, remembered
|
|
37
37
|
from constricter.cli.protocol import SERVERS, HintError, HungError, Server
|
|
38
38
|
from constricter.fix.core.known import Hints, Offered
|
|
39
39
|
|
|
@@ -47,8 +47,6 @@ _Task: TypeAlias = Callable[[], _Found] # one server's work: the hints of each
|
|
|
47
47
|
|
|
48
48
|
_BATCH: Final = 8 # files a free server takes at a time: few, so none is left with the slow ones
|
|
49
49
|
MEMORY: Final = 8 << 30 # the most a checker's servers use together, by default (`--infer-memory`)
|
|
50
|
-
_SERVER_MEMORY: Final = 600 << 20 # a server's memory, before the files (basedpyright's, measured)
|
|
51
|
-
_MEMORY_PER_BYTE: Final = 130 # and for each byte of them: 1.2 GB for SQLAlchemy's 8 MB, 3.4 for pandas' 21
|
|
52
50
|
_MEMINFO: Final = Path("/proc/meminfo")
|
|
53
51
|
_FILES_PER_SERVER: Final = 32 # fewer files than this each don't pay for another server's start
|
|
54
52
|
# Seconds without a word from a server before it's taken for hung. Not an answer's wait: pandas'
|
|
@@ -228,11 +226,10 @@ class Checker:
|
|
|
228
226
|
"""One checker's servers, and which of them has each file.
|
|
229
227
|
|
|
230
228
|
It has as many as its files need (one per `_FILES_PER_SERVER`), up to the command's `--jobs`, the
|
|
231
|
-
checker's own most (`Server.most
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
One server it always has, whatever the memory.
|
|
229
|
+
checker's own most (`Server.most`), and as many as fit in its memory: `--infer-memory` (never
|
|
230
|
+
more than is available), or by default `MEMORY` (never more than half what's available). Each
|
|
231
|
+
server loads the whole program it checks, or its share of it, by its checker's own measure (see
|
|
232
|
+
`Server.fitting`). One server it always has, whatever the memory.
|
|
236
233
|
|
|
237
234
|
The files are handed out a few at a time (`_BATCH`), the biggest first, to whichever server is
|
|
238
235
|
free: one file's analysis can take far longer than another's, and none waits on a busy one. A
|
|
@@ -244,7 +241,8 @@ class Checker:
|
|
|
244
241
|
self.name: str = name
|
|
245
242
|
self.root: Path = root
|
|
246
243
|
self.command: list[str] = command(name)
|
|
247
|
-
self.
|
|
244
|
+
self.server: Server = SERVERS[name]
|
|
245
|
+
self.most: int = max(1, min(servers, self.server.most))
|
|
248
246
|
self.memory: int = budget(memory, available_memory())
|
|
249
247
|
self.servers: list[Connection] = []
|
|
250
248
|
self.assigned: dict[Path, int] = {} # each file's server, by its place in `servers`
|
|
@@ -258,13 +256,14 @@ class Checker:
|
|
|
258
256
|
|
|
259
257
|
"""
|
|
260
258
|
size: int = sum(len(text) for text in files.values())
|
|
261
|
-
fit: int = max(1, self.memory
|
|
259
|
+
fit: int = max(1, self.server.fitting(self.memory, size))
|
|
262
260
|
return max(len(self.servers), min(self.most, -(-len(files) // _FILES_PER_SERVER), fit))
|
|
263
261
|
|
|
264
262
|
def tasks(self, files: Mapping[Path, str]) -> list[_Task]:
|
|
265
263
|
"""Make each server's work on `files`: its own files first, then new ones as it's free for them.
|
|
266
264
|
|
|
267
|
-
A file abandoned (see `_alone`) isn't
|
|
265
|
+
A file abandoned (see `_alone`), in this run or an earlier one (see `remembered`), isn't
|
|
266
|
+
asked about again.
|
|
268
267
|
|
|
269
268
|
Returns:
|
|
270
269
|
A task per server, giving the hints of every file it was sent.
|
|
@@ -274,8 +273,14 @@ class Checker:
|
|
|
274
273
|
path: Path
|
|
275
274
|
for path in files.keys() & self.assigned.keys() - self.abandoned:
|
|
276
275
|
own.setdefault(self.assigned[path], {})[path] = files[path]
|
|
276
|
+
# One it hung on in an earlier run, unchanged since: left without hints from the start.
|
|
277
|
+
self.abandoned.update(
|
|
278
|
+
path
|
|
279
|
+
for path in files.keys() - self.assigned.keys() - self.abandoned
|
|
280
|
+
if remembered.hung(self.name, self.command, files[path])
|
|
281
|
+
)
|
|
277
282
|
fresh: list[Path] = sorted(
|
|
278
|
-
files.keys() - self.assigned.keys(),
|
|
283
|
+
files.keys() - self.assigned.keys() - self.abandoned,
|
|
279
284
|
key=lambda new: (-len(files[new]), new),
|
|
280
285
|
)
|
|
281
286
|
batches: queue.SimpleQueue[dict[Path, str]] = queue.SimpleQueue()
|
|
@@ -352,6 +357,7 @@ class Checker:
|
|
|
352
357
|
found.update(server.answers(server.ask({path: text})))
|
|
353
358
|
except HungError as hung:
|
|
354
359
|
self.abandoned.add(path)
|
|
360
|
+
remembered.remember(self.name, self.command, text)
|
|
355
361
|
if len(self.abandoned) > _MOST_ABANDONED:
|
|
356
362
|
error: str = f"{self.name} hung on {len(self.abandoned)} file(s), each asked about alone"
|
|
357
363
|
raise HintError(error) from hung
|
constricter/cli/options.py
CHANGED
|
@@ -11,13 +11,15 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Final, cast
|
|
12
12
|
|
|
13
13
|
from constricter import __version__
|
|
14
|
-
from constricter.cli import baseline
|
|
14
|
+
from constricter.cli import baseline, traced
|
|
15
15
|
from constricter.cli.config import (
|
|
16
16
|
DEFAULT_BASELINE,
|
|
17
17
|
config_defaults,
|
|
18
18
|
project_root,
|
|
19
19
|
unknown_codes,
|
|
20
20
|
unknown_fix_kinds,
|
|
21
|
+
unknown_widen_kinds,
|
|
22
|
+
widen_kinds,
|
|
21
23
|
)
|
|
22
24
|
from constricter.cli.explain import explain
|
|
23
25
|
from constricter.cli.paths import STDIN, excluded
|
|
@@ -133,6 +135,24 @@ def _fix_kinds(text: str) -> list[str]:
|
|
|
133
135
|
return kinds
|
|
134
136
|
|
|
135
137
|
|
|
138
|
+
def _widen_kinds(text: str) -> list[str]:
|
|
139
|
+
"""Read a comma-separated list of the wider types `--fix` may write (`untyped-parameters`), or `all`.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Them.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
argparse.ArgumentTypeError: One isn't a widening's id.
|
|
146
|
+
|
|
147
|
+
"""
|
|
148
|
+
kinds: list[str] = [kind.strip() for kind in text.split(",") if kind.strip()]
|
|
149
|
+
unknown: list[str]
|
|
150
|
+
if unknown := unknown_widen_kinds(kinds):
|
|
151
|
+
message: str = f"no wider type is called {', '.join(unknown)} (see docs/FIXES.md)"
|
|
152
|
+
raise argparse.ArgumentTypeError(message)
|
|
153
|
+
return widen_kinds(kinds)
|
|
154
|
+
|
|
155
|
+
|
|
136
156
|
def _version(text: str) -> tuple[int, int]:
|
|
137
157
|
"""Read `--min-python`: a Python version (`3.11`).
|
|
138
158
|
|
|
@@ -333,6 +353,13 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
333
353
|
metavar="KINDS",
|
|
334
354
|
help="treat guesses from these mechanisms (constructor, narrow) as certain",
|
|
335
355
|
)
|
|
356
|
+
_ = parser.add_argument(
|
|
357
|
+
"--fix-widen",
|
|
358
|
+
type=_widen_kinds,
|
|
359
|
+
default=[],
|
|
360
|
+
metavar="KINDS",
|
|
361
|
+
help="also write these wider types, each marked (untyped-parameters, unions, ...; all: every one)",
|
|
362
|
+
)
|
|
336
363
|
_ = parser.add_argument(
|
|
337
364
|
"--fix-plain-bases",
|
|
338
365
|
type=_listed,
|
|
@@ -353,6 +380,12 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
353
380
|
metavar="GB",
|
|
354
381
|
help="with --infer-with: the most memory each checker's servers use together (default: 8)",
|
|
355
382
|
)
|
|
383
|
+
_ = parser.add_argument(
|
|
384
|
+
"--infer-from",
|
|
385
|
+
type=Path,
|
|
386
|
+
metavar="FILE",
|
|
387
|
+
help="type what --fix can't with the types `python -m constricter.trace` recorded, as guesses",
|
|
388
|
+
)
|
|
356
389
|
_ = parser.add_argument(
|
|
357
390
|
"--diff",
|
|
358
391
|
action="store_true",
|
|
@@ -361,7 +394,7 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
361
394
|
_ = parser.add_argument(
|
|
362
395
|
"--show-fixes",
|
|
363
396
|
action="store_true",
|
|
364
|
-
help="after the report, list each fix and how the value decided it
|
|
397
|
+
help="after the report, list each fix and how the value decided it; with --fix, those made too",
|
|
365
398
|
)
|
|
366
399
|
_ = parser.add_argument(
|
|
367
400
|
"--statistics",
|
|
@@ -440,6 +473,7 @@ class Input:
|
|
|
440
473
|
paths: list[Path]
|
|
441
474
|
exclude: list[str]
|
|
442
475
|
stdin_name: Path = STDIN
|
|
476
|
+
trace: Path | None = None # the trace whose types `--fix` guesses with (`--infer-from`)
|
|
443
477
|
|
|
444
478
|
def name(self, path: Path) -> Path:
|
|
445
479
|
"""Name `path` for reports.
|
|
@@ -566,7 +600,12 @@ class Options:
|
|
|
566
600
|
parser.error("`-` (standard input) must be the only path")
|
|
567
601
|
mode: Mode = _mode(parser, args)
|
|
568
602
|
return cls(
|
|
569
|
-
input=Input(
|
|
603
|
+
input=Input(
|
|
604
|
+
paths,
|
|
605
|
+
cast("list[str]", args.exclude),
|
|
606
|
+
cast("Path", args.stdin_filename),
|
|
607
|
+
_trace(parser, cast("Path | None", args.infer_from)),
|
|
608
|
+
),
|
|
570
609
|
checks=Checks(
|
|
571
610
|
type_comments=cast("bool", args.type_comments),
|
|
572
611
|
all_scopes=cast("bool", args.all_scopes),
|
|
@@ -582,6 +621,7 @@ class Options:
|
|
|
582
621
|
frozenset(cast("list[str]", args.fix_select)),
|
|
583
622
|
frozenset(cast("list[str]", args.fix_ignore)),
|
|
584
623
|
frozenset(cast("list[str]", args.unsafe_fix_select)),
|
|
624
|
+
frozenset(cast("list[str]", args.fix_widen)),
|
|
585
625
|
),
|
|
586
626
|
min_python=cast("tuple[int, int] | None", args.min_python),
|
|
587
627
|
plain_bases=(*PLAIN_BASES, *cast("list[str]", args.fix_plain_bases)),
|
|
@@ -605,6 +645,20 @@ class Options:
|
|
|
605
645
|
)
|
|
606
646
|
|
|
607
647
|
|
|
648
|
+
def _trace(parser: argparse.ArgumentParser, path: Path | None) -> Path | None:
|
|
649
|
+
"""Check that `--infer-from`'s file is a trace, read here once for the run.
|
|
650
|
+
|
|
651
|
+
Returns:
|
|
652
|
+
The path.
|
|
653
|
+
|
|
654
|
+
"""
|
|
655
|
+
try:
|
|
656
|
+
_ = None if path is None else traced.load(path)
|
|
657
|
+
except ValueError as error:
|
|
658
|
+
parser.error(str(error))
|
|
659
|
+
return path
|
|
660
|
+
|
|
661
|
+
|
|
608
662
|
def _maximal(args: argparse.Namespace) -> argparse.Namespace:
|
|
609
663
|
"""Apply `--max-fix` (`--max`, every fix, every installed checker) and `--max`.
|
|
610
664
|
|
constricter/cli/protocol.py
CHANGED
|
@@ -33,21 +33,41 @@ class Server(NamedTuple):
|
|
|
33
33
|
"""A checker's language server: how it's started, and how many are worth running at once.
|
|
34
34
|
|
|
35
35
|
Its executable, the arguments that start it over stdio, and the most of it to run (up to
|
|
36
|
-
`MAX_SERVERS
|
|
36
|
+
`MAX_SERVERS`). And the memory its servers take, as measured: `memory` bytes each before the
|
|
37
|
+
files, and `per_byte` more for each byte of them: each server's again where each holds the
|
|
38
|
+
`whole` program, once among them all where each holds what it's asked about.
|
|
37
39
|
"""
|
|
38
40
|
|
|
39
41
|
executable: str
|
|
40
42
|
args: tuple[str, ...]
|
|
41
43
|
most: int
|
|
44
|
+
memory: int
|
|
45
|
+
per_byte: int
|
|
46
|
+
whole: bool = False
|
|
47
|
+
|
|
48
|
+
def fitting(self, budget: int, size: int) -> int:
|
|
49
|
+
"""Count the servers that fit in `budget` bytes, for files of `size` bytes together.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
It: none where one doesn't fit.
|
|
53
|
+
|
|
54
|
+
"""
|
|
55
|
+
if self.whole:
|
|
56
|
+
return budget // (self.memory + self.per_byte * size)
|
|
57
|
+
return max(0, budget - self.per_byte * size) // self.memory
|
|
42
58
|
|
|
43
59
|
|
|
44
60
|
MAX_SERVERS: Final = 4 # a checker's, at most: each holds its own copy of the program it checks
|
|
45
61
|
SERVERS: Final = {
|
|
46
62
|
# One thread each: more servers check more at once (sqlalchemy's hints: 10.8s with one, 7.2s with four).
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
63
|
+
# Each holds the whole program: 1.2 GB for sqlalchemy's 8 MB, 3.4 for pandas's 21.
|
|
64
|
+
"basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS, 600 << 20, 130, whole=True),
|
|
65
|
+
# Parallel itself, and still quicker shared out: a hinted check of pandas takes 36s with one, 25s
|
|
66
|
+
# with two, 22s with three and 21s with four (1.2 GB with one, 1.7 with two, 2.5 with four).
|
|
67
|
+
"ty": Server("ty", ("server",), MAX_SERVERS, 420 << 20, 38),
|
|
68
|
+
# Parallel too: pandas's takes 132s with one, 58s with two and 32s with four (2.8 GB, 3.2, 4.2);
|
|
69
|
+
# the standard library's 39 MB, 4.3 GB with one and 5.6 with four.
|
|
70
|
+
"pyrefly": Server("pyrefly", ("lsp",), MAX_SERVERS, 480 << 20, 120),
|
|
51
71
|
}
|
|
52
72
|
_PROBE: Final = "--version" # quick for each; basedpyright-langserver's exits 1 all the same
|
|
53
73
|
_PROBE_TIMEOUT: Final = 10.0 # seconds it may take
|