python-constricter 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- constricter/__init__.py +1 -1
- constricter/cli/collecting.py +47 -0
- constricter/cli/command.py +114 -259
- constricter/cli/config.py +8 -15
- constricter/cli/hints.py +9 -103
- constricter/cli/options.py +1 -1
- constricter/cli/paths.py +31 -0
- constricter/cli/protocol.py +110 -0
- constricter/cli/runs.py +42 -0
- constricter/cli/schedule.py +222 -0
- constricter/cli/workers.py +154 -0
- constricter/fix/doubts.py +115 -16
- constricter/fix/fills.py +47 -45
- constricter/fix/fixes.py +60 -20
- constricter/fix/guesses.py +74 -11
- constricter/fix/imports.py +76 -25
- constricter/fix/inference.py +49 -75
- constricter/fix/known.py +79 -5
- constricter/fix/library.py +80 -0
- constricter/fix/members.py +10 -0
- constricter/fix/modules.py +252 -0
- constricter/fix/narrowed.py +164 -0
- constricter/fix/order.py +145 -0
- constricter/fix/overloads.py +394 -0
- constricter/fix/project.py +408 -284
- constricter/fix/returned.py +374 -80
- constricter/fix/stdlib.py +207 -50
- constricter/fix/tables/aliases.json +147 -0
- constricter/fix/tables/attributes.json +675 -0
- constricter/fix/tables/bases.json +888 -0
- constricter/fix/tables/classes.json +1939 -0
- constricter/fix/tables/method_overloads.json +353 -0
- constricter/fix/tables/method_signatures.json +655 -0
- constricter/fix/tables/methods.json +857 -0
- constricter/fix/tables/overloads.json +373 -0
- constricter/fix/tables/returns.json +1328 -0
- constricter/fix/tables/source.json +4 -0
- constricter/fix/tables/type_parameters.json +175 -0
- constricter/offences.py +4 -0
- constricter/rules/annotations.py +55 -1
- constricter/rules/binding.py +241 -0
- constricter/rules/checker.py +259 -333
- constricter/rules/flow.py +2 -0
- constricter/rules/late.py +288 -0
- constricter/rules/narrowing.py +48 -1
- constricter/rules/parsed.py +18 -11
- constricter/rules/redundant.py +2 -8
- constricter/rules/scope.py +97 -223
- constricter/rules/syntax.py +98 -3
- constricter/rules/walked.py +59 -7
- {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/METADATA +9 -8
- python_constricter-0.2.7.dist-info/RECORD +75 -0
- constricter/fix/stdlib.json +0 -15351
- python_constricter-0.2.5.dist-info/RECORD +0 -53
- {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/WHEEL +0 -0
- {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/entry_points.txt +0 -0
- {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py
CHANGED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
"""The garbage collector, as the command runs it: by hand, once per file checked.
|
|
3
|
+
|
|
4
|
+
A check keeps every file's tree (see `parsed`), and allocates a great deal for each file, little of it
|
|
5
|
+
cyclic. Left to itself, the collector looked through the kept trees again and again as the check
|
|
6
|
+
allocated: a third of the standard library's check (34s, 21s without). So while the command runs,
|
|
7
|
+
it's off: what indexing built is frozen (never looked through again), and after each file, only
|
|
8
|
+
what's been allocated since (the young generations) is collected, which bounds what a cycle can hold.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import gc
|
|
13
|
+
from collections.abc import Generator
|
|
14
|
+
from typing import Final
|
|
15
|
+
|
|
16
|
+
_YOUNG: Final = 1 # the generations collected after each file: 0 and 1, never the frozen or the old
|
|
17
|
+
_FREEZES: Final = hasattr(gc, "freeze") # PyPy's collector has no freezing
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@contextlib.contextmanager
|
|
21
|
+
def by_hand() -> Generator[None]:
|
|
22
|
+
"""Turn automatic collection off for the command's run, and back as it was after.
|
|
23
|
+
|
|
24
|
+
Yields:
|
|
25
|
+
Nothing: the run.
|
|
26
|
+
|
|
27
|
+
"""
|
|
28
|
+
was: bool = gc.isenabled()
|
|
29
|
+
gc.disable()
|
|
30
|
+
try:
|
|
31
|
+
yield
|
|
32
|
+
finally:
|
|
33
|
+
if _FREEZES:
|
|
34
|
+
gc.unfreeze()
|
|
35
|
+
if was:
|
|
36
|
+
gc.enable()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def indexed() -> None:
|
|
40
|
+
"""Freeze what indexing built (the kept trees, the index): it lives until the check takes it."""
|
|
41
|
+
if _FREEZES:
|
|
42
|
+
gc.freeze()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def sweep() -> None:
|
|
46
|
+
"""Collect what the file just checked left behind (see the module docstring)."""
|
|
47
|
+
_ = gc.collect(_YOUNG)
|
constricter/cli/command.py
CHANGED
|
@@ -4,25 +4,27 @@
|
|
|
4
4
|
import codecs
|
|
5
5
|
import contextlib
|
|
6
6
|
import difflib
|
|
7
|
+
import importlib
|
|
7
8
|
import io
|
|
8
|
-
import itertools
|
|
9
9
|
import json
|
|
10
10
|
import sys
|
|
11
11
|
import tokenize
|
|
12
|
-
from collections.abc import Callable,
|
|
13
|
-
from
|
|
14
|
-
from dataclasses import dataclass, field, replace
|
|
12
|
+
from collections.abc import Callable, Sequence
|
|
13
|
+
from dataclasses import replace
|
|
15
14
|
from functools import partial
|
|
16
15
|
from pathlib import Path
|
|
17
|
-
from typing import Final, NamedTuple,
|
|
16
|
+
from typing import TYPE_CHECKING, Final, NamedTuple, TextIO, TypeAlias, cast
|
|
18
17
|
|
|
19
18
|
from constricter import notebook
|
|
20
|
-
from constricter.cli import baseline,
|
|
19
|
+
from constricter.cli import baseline, collecting, schedule
|
|
21
20
|
from constricter.cli.options import Mode, Options, Output
|
|
22
21
|
from constricter.cli.paths import STDIN, python_files
|
|
22
|
+
from constricter.cli.protocol import HintError
|
|
23
23
|
from constricter.cli.report import Format, Result, fix_reasons, render, statistics
|
|
24
|
+
from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
|
|
25
|
+
from constricter.cli.workers import Workers
|
|
24
26
|
from constricter.fix import fixes, project
|
|
25
|
-
from constricter.fix.known import Hints, Outside
|
|
27
|
+
from constricter.fix.known import Hints, Outside, Returns
|
|
26
28
|
from constricter.noqa import lines, unsuppressed
|
|
27
29
|
from constricter.offences import (
|
|
28
30
|
DEFAULT_CHECKS,
|
|
@@ -30,8 +32,10 @@ from constricter.offences import (
|
|
|
30
32
|
Edit,
|
|
31
33
|
Offence,
|
|
32
34
|
)
|
|
33
|
-
from constricter.rules import
|
|
34
|
-
|
|
35
|
+
from constricter.rules.checker import Checked, Coverage, annotation_coverage, checked_source
|
|
36
|
+
|
|
37
|
+
if TYPE_CHECKING: # slow to import, and only needed for `--infer-with`
|
|
38
|
+
from constricter.cli import hints
|
|
35
39
|
|
|
36
40
|
EXIT_CLEAN: Final = 0
|
|
37
41
|
EXIT_FOUND: Final = 1
|
|
@@ -75,30 +79,29 @@ def _source(raw: str, name: Path) -> tuple[str, list[notebook.Line]]:
|
|
|
75
79
|
return notebook.parse(raw, str(name)) if name.suffix == notebook.SUFFIX else (raw, [])
|
|
76
80
|
|
|
77
81
|
|
|
78
|
-
def
|
|
82
|
+
def checked_text(
|
|
79
83
|
raw: str,
|
|
80
84
|
name: Path,
|
|
81
85
|
checks: Checks = DEFAULT_CHECKS,
|
|
82
86
|
*,
|
|
83
87
|
outside: Outside | None = None,
|
|
84
|
-
) ->
|
|
85
|
-
"""
|
|
88
|
+
) -> Checked:
|
|
89
|
+
"""Check `raw`, the text of `name`: the offences no `# noqa` suppresses.
|
|
86
90
|
|
|
87
91
|
A notebook's code cells are checked as one module, and each offence placed in its cell.
|
|
88
92
|
Raises `ValueError` for a `.ipynb` that isn't a notebook.
|
|
89
93
|
|
|
90
94
|
Returns:
|
|
91
|
-
The unsuppressed offences, a notebook's placed in their cells
|
|
95
|
+
The unsuppressed offences, a notebook's placed in their cells, and what its functions return
|
|
96
|
+
(see `Checked`).
|
|
92
97
|
|
|
93
98
|
"""
|
|
94
99
|
source: str
|
|
95
100
|
where: list[notebook.Line]
|
|
96
101
|
source, where = _source(raw, name)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
)
|
|
101
|
-
return [_placed(o, where) for o in offences] if where else offences
|
|
102
|
+
checked: Checked = checked_source(source, str(name), checks, outside=outside)
|
|
103
|
+
offences: list[Offence] = unsuppressed(checked.offences, lines(source))
|
|
104
|
+
return checked._replace(offences=[_placed(o, where) for o in offences] if where else offences)
|
|
102
105
|
|
|
103
106
|
|
|
104
107
|
def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
|
|
@@ -111,7 +114,8 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
|
|
|
111
114
|
"""
|
|
112
115
|
line: notebook.Line = where[offence.line - 1]
|
|
113
116
|
placed: Offence = replace(offence, line=line.line, cell=line.cell)
|
|
114
|
-
|
|
117
|
+
# A notebook's cells have no import block.
|
|
118
|
+
if offence.edit is not None and (offence.edit.imports or offence.edit.guarded):
|
|
115
119
|
return replace(placed, edit=None)
|
|
116
120
|
if offence.edit is not None and offence.edit.edit is Edit.DECLARE:
|
|
117
121
|
statement: int = where[offence.edit.span[0] - 1].line
|
|
@@ -176,38 +180,6 @@ def fix_file(path: Path, offences: Sequence[Offence]) -> int:
|
|
|
176
180
|
return count
|
|
177
181
|
|
|
178
182
|
|
|
179
|
-
@dataclass(frozen=True)
|
|
180
|
-
class _CheckRun:
|
|
181
|
-
"""What checking one file found (and fixed, or would fix), in check/fix/diff mode."""
|
|
182
|
-
|
|
183
|
-
results: list[Result] = field(default_factory=list[Result])
|
|
184
|
-
baselined: int = 0
|
|
185
|
-
fixed: int = 0 # --fix: how many offences it fixed
|
|
186
|
-
text: str = "" # --diff: the diff; --fix on standard input: the fixed source
|
|
187
|
-
error: str = ""
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
@dataclass(frozen=True)
|
|
191
|
-
class _BaselineRun:
|
|
192
|
-
"""One file's offences, unfiltered, for --write-baseline."""
|
|
193
|
-
|
|
194
|
-
found: list[Offence] = field(default_factory=list[Offence])
|
|
195
|
-
error: str = ""
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
@dataclass(frozen=True)
|
|
199
|
-
class _CoverageRun:
|
|
200
|
-
"""One file's annotation coverage, for --coverage."""
|
|
201
|
-
|
|
202
|
-
coverage: Coverage | None = None
|
|
203
|
-
error: str = ""
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
_FileRun: TypeAlias = _CheckRun | _BaselineRun | _CoverageRun
|
|
207
|
-
_Check: TypeAlias = Callable[[Path, Outside], _FileRun] # check (or count, or baseline) one file
|
|
208
|
-
# A worker's answers, one per file of its share: what it read (to index), and what it found.
|
|
209
|
-
_Reading: TypeAlias = Future[list[project.Module | None]]
|
|
210
|
-
_Checking: TypeAlias = Future[list[_FileRun]]
|
|
211
183
|
HINT_ROUNDS: Final = 4 # with `--fix --infer-with`: how many times each file is fixed, at most
|
|
212
184
|
|
|
213
185
|
|
|
@@ -227,15 +199,15 @@ def _shown_lines(raw: str, name: Path) -> dict[tuple[int | None, int], str]:
|
|
|
227
199
|
}
|
|
228
200
|
|
|
229
201
|
|
|
230
|
-
def _checked(path: Path, name: Path, checks: Checks, outside: Outside) -> tuple[str,
|
|
202
|
+
def _checked(path: Path, name: Path, checks: Checks, outside: Outside) -> tuple[str, Checked]:
|
|
231
203
|
"""Read `path` and check it as `name`; raises what reading or parsing it does.
|
|
232
204
|
|
|
233
205
|
Returns:
|
|
234
|
-
Its text, and
|
|
206
|
+
Its text, and what checking it found.
|
|
235
207
|
|
|
236
208
|
"""
|
|
237
209
|
raw: str = _read(path)
|
|
238
|
-
return raw,
|
|
210
|
+
return raw, checked_text(raw, name, checks, outside=outside)
|
|
239
211
|
|
|
240
212
|
|
|
241
213
|
def _read_checked(
|
|
@@ -243,20 +215,20 @@ def _read_checked(
|
|
|
243
215
|
name: Path,
|
|
244
216
|
checks: Checks,
|
|
245
217
|
outside: Outside,
|
|
246
|
-
) -> tuple[str,
|
|
218
|
+
) -> tuple[str, Checked, str]:
|
|
247
219
|
"""Read and check `path`, turning a read or parse error into a message instead of raising.
|
|
248
220
|
|
|
249
221
|
Returns:
|
|
250
|
-
Its text and
|
|
222
|
+
Its text and what checking it found (nothing on error), and the error message (empty on success).
|
|
251
223
|
|
|
252
224
|
"""
|
|
253
225
|
raw: str
|
|
254
|
-
|
|
226
|
+
checked: Checked
|
|
255
227
|
try:
|
|
256
|
-
raw,
|
|
228
|
+
raw, checked = _checked(path, name, checks, outside)
|
|
257
229
|
except (OSError, ValueError, SyntaxError) as error: # UnicodeDecodeError is a ValueError
|
|
258
|
-
return "", [], f"{name}: error: {error}"
|
|
259
|
-
return raw,
|
|
230
|
+
return "", Checked([], Returns()), f"{name}: error: {error}"
|
|
231
|
+
return raw, checked, ""
|
|
260
232
|
|
|
261
233
|
|
|
262
234
|
def _results(raw: str, name: Path, offences: Sequence[Offence], options: Options) -> list[Result]:
|
|
@@ -278,7 +250,7 @@ def _results(raw: str, name: Path, offences: Sequence[Offence], options: Options
|
|
|
278
250
|
]
|
|
279
251
|
|
|
280
252
|
|
|
281
|
-
def _check_path(path: Path, outside: Outside, options: Options) ->
|
|
253
|
+
def _check_path(path: Path, outside: Outside, options: Options) -> CheckRun:
|
|
282
254
|
"""Check (and fix, or diff) one file, given what's known of it from outside it.
|
|
283
255
|
|
|
284
256
|
Returns:
|
|
@@ -287,11 +259,24 @@ def _check_path(path: Path, outside: Outside, options: Options) -> _CheckRun:
|
|
|
287
259
|
"""
|
|
288
260
|
name: Path = options.input.name(path)
|
|
289
261
|
raw: str
|
|
290
|
-
|
|
262
|
+
checked: Checked
|
|
291
263
|
error: str
|
|
292
|
-
raw,
|
|
264
|
+
raw, checked, error = _read_checked(path, name, options.checks, outside)
|
|
293
265
|
if error:
|
|
294
|
-
return
|
|
266
|
+
return CheckRun(error=error)
|
|
267
|
+
return replace(_handled(path, (raw, name), checked.offences, options), returned=checked.returned)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], options: Options) -> CheckRun:
|
|
271
|
+
"""Report, fix or diff the offences found in `path` (`text`: its text, and its name).
|
|
272
|
+
|
|
273
|
+
Returns:
|
|
274
|
+
What was found, and fixed.
|
|
275
|
+
|
|
276
|
+
"""
|
|
277
|
+
raw: str
|
|
278
|
+
name: Path
|
|
279
|
+
raw, name = text
|
|
295
280
|
baselined: int
|
|
296
281
|
offences, baselined = options.filter.unbaselined(name, offences)
|
|
297
282
|
results: list[Result] = _results(raw, name, offences, options)
|
|
@@ -300,22 +285,22 @@ def _check_path(path: Path, outside: Outside, options: Options) -> _CheckRun:
|
|
|
300
285
|
r.offence for r in results if r.offence.edit is not None and (unsafe or not r.offence.unsafe)
|
|
301
286
|
]
|
|
302
287
|
if options.mode is Mode.DIFF:
|
|
303
|
-
return
|
|
288
|
+
return CheckRun(text=_diff(raw, name, fixing))
|
|
304
289
|
if options.mode is not Mode.FIX:
|
|
305
|
-
return
|
|
290
|
+
return CheckRun(results, baselined)
|
|
306
291
|
left: list[Result] = [r for r in results if r.offence not in fixing]
|
|
307
292
|
if path == STDIN: # the fixed source goes to stdout
|
|
308
|
-
return
|
|
293
|
+
return CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
|
|
309
294
|
try:
|
|
310
|
-
return
|
|
295
|
+
return CheckRun(left, baselined, fix_file(path, fixing))
|
|
311
296
|
except UnicodeEncodeError as failure: # the file is left as it was, its offences unfixed
|
|
312
297
|
# Its canonical name: PyPy reports `latin1` where CPython says `latin-1`.
|
|
313
298
|
encoding: str = codecs.lookup(failure.encoding).name
|
|
314
299
|
message: str = f"an annotation can't be written in its encoding, {encoding}; left as it was"
|
|
315
|
-
return
|
|
300
|
+
return CheckRun(results, baselined, error=f"{name}: error: {message}")
|
|
316
301
|
|
|
317
302
|
|
|
318
|
-
def _baseline_path(path: Path, outside: Outside, options: Options) ->
|
|
303
|
+
def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
|
|
319
304
|
"""Check one file, unfiltered, for --write-baseline.
|
|
320
305
|
|
|
321
306
|
Returns:
|
|
@@ -323,13 +308,13 @@ def _baseline_path(path: Path, outside: Outside, options: Options) -> _BaselineR
|
|
|
323
308
|
|
|
324
309
|
"""
|
|
325
310
|
name: Path = options.input.name(path)
|
|
326
|
-
|
|
311
|
+
checked: Checked
|
|
327
312
|
error: str
|
|
328
|
-
_,
|
|
329
|
-
return
|
|
313
|
+
_, checked, error = _read_checked(path, name, options.checks, outside)
|
|
314
|
+
return BaselineRun(error=error) if error else BaselineRun(checked.offences, returned=checked.returned)
|
|
330
315
|
|
|
331
316
|
|
|
332
|
-
def _cover_path(path: Path, _outside: Outside, options: Options) ->
|
|
317
|
+
def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
|
|
333
318
|
"""Count one file's typed first bindings.
|
|
334
319
|
|
|
335
320
|
Returns:
|
|
@@ -338,12 +323,12 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> _CoverageRun
|
|
|
338
323
|
"""
|
|
339
324
|
name: Path = options.input.name(path)
|
|
340
325
|
try:
|
|
341
|
-
return
|
|
326
|
+
return CoverageRun(annotation_coverage(_source(_read(path), name)[0], options.checks))
|
|
342
327
|
except (OSError, ValueError, SyntaxError) as error:
|
|
343
|
-
return
|
|
328
|
+
return CoverageRun(error=f"{name}: error: {error}")
|
|
344
329
|
|
|
345
330
|
|
|
346
|
-
def _check_all(options: Options) -> tuple[list[Path], list[
|
|
331
|
+
def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
|
|
347
332
|
"""Check every file (`--jobs` at a time), in order.
|
|
348
333
|
|
|
349
334
|
With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
|
|
@@ -357,7 +342,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
|
|
|
357
342
|
|
|
358
343
|
"""
|
|
359
344
|
paths: list[Path] = list(python_files(options.input.paths, options.input.exclude))
|
|
360
|
-
check: Callable[[Path, Outside],
|
|
345
|
+
check: Callable[[Path, Outside], FileRun]
|
|
361
346
|
if options.mode is Mode.COVERAGE:
|
|
362
347
|
check = partial(_cover_path, options=options)
|
|
363
348
|
elif options.mode is Mode.WRITE_BASELINE:
|
|
@@ -368,20 +353,26 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
|
|
|
368
353
|
if not options.infer_with or options.mode in {Mode.COVERAGE, Mode.WRITE_BASELINE}:
|
|
369
354
|
return names, _checked_all(paths, check, options)[0]
|
|
370
355
|
session: hints.Session
|
|
371
|
-
with
|
|
372
|
-
|
|
356
|
+
# Imported only for `--infer-with`: it's slow to import, for every run's start.
|
|
357
|
+
with cast("type[hints.Session]", importlib.import_module("constricter.cli.hints").Session)(
|
|
358
|
+
options.infer_with,
|
|
359
|
+
Path.cwd(),
|
|
360
|
+
options.jobs,
|
|
361
|
+
options.infer_memory,
|
|
362
|
+
) as session:
|
|
363
|
+
runs: list[FileRun]
|
|
373
364
|
modules: project.Index
|
|
374
365
|
runs, modules = _checked_all(paths, check, options, session)
|
|
375
366
|
# The files the last round changed: only their hints can have changed.
|
|
376
367
|
again: list[int] = [
|
|
377
368
|
index
|
|
378
369
|
for index, run in enumerate(runs)
|
|
379
|
-
if options.mode is Mode.FIX and isinstance(run,
|
|
370
|
+
if options.mode is Mode.FIX and isinstance(run, CheckRun) and run.fixed and paths[index] != STDIN
|
|
380
371
|
]
|
|
381
372
|
for _ in range(HINT_ROUNDS - 1):
|
|
382
373
|
if not again:
|
|
383
374
|
break
|
|
384
|
-
redone: list[
|
|
375
|
+
redone: list[FileRun] = _checked_all(
|
|
385
376
|
[paths[i] for i in again],
|
|
386
377
|
check,
|
|
387
378
|
options,
|
|
@@ -389,20 +380,20 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
|
|
|
389
380
|
modules,
|
|
390
381
|
)[0]
|
|
391
382
|
index: int
|
|
392
|
-
run:
|
|
383
|
+
run: FileRun
|
|
393
384
|
for index, run in zip(again, redone, strict=True):
|
|
394
|
-
runs[index] = _merged(
|
|
395
|
-
again = [index for index, run in zip(again, redone, strict=True) if cast("
|
|
385
|
+
runs[index] = _merged(runs[index], run)
|
|
386
|
+
again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
|
|
396
387
|
return names, runs
|
|
397
388
|
|
|
398
389
|
|
|
399
390
|
def _checked_all(
|
|
400
391
|
paths: Sequence[Path],
|
|
401
|
-
check: Callable[[Path, Outside],
|
|
392
|
+
check: Callable[[Path, Outside], FileRun],
|
|
402
393
|
options: Options,
|
|
403
|
-
session: hints.Session | None = None,
|
|
394
|
+
session: "hints.Session | None" = None,
|
|
404
395
|
modules: project.Index | None = None,
|
|
405
|
-
) -> tuple[list[
|
|
396
|
+
) -> tuple[list[FileRun], project.Index]:
|
|
406
397
|
"""Check `paths` (`--jobs` at a time), with the `session`'s hints, and `modules` (else indexed).
|
|
407
398
|
|
|
408
399
|
Returns:
|
|
@@ -411,151 +402,35 @@ def _checked_all(
|
|
|
411
402
|
"""
|
|
412
403
|
hinted: dict[Path, tuple[Hints, ...]] = {} if session is None else session.hints(_texts(paths))
|
|
413
404
|
coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
|
|
414
|
-
|
|
415
|
-
|
|
405
|
+
stack: contextlib.ExitStack
|
|
406
|
+
with contextlib.ExitStack() as stack:
|
|
407
|
+
pool: Workers | None = None
|
|
408
|
+
if options.jobs != 1 and len(paths) > 1:
|
|
409
|
+
pool = stack.enter_context(Workers(paths, options.jobs))
|
|
410
|
+
if modules is None and pool:
|
|
411
|
+
modules = project.Index({}, []) if coverage else pool.index()
|
|
412
|
+
elif modules is None:
|
|
416
413
|
modules = project.Index({}, []) if coverage else project.index(paths)
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
class _Workers:
|
|
428
|
-
"""Worker processes, each with its own share of the files, which it indexes and then checks.
|
|
429
|
-
|
|
430
|
-
The trees a worker parsed to index its files are still there to check (`parsed.keep`), within its
|
|
431
|
-
part of the budget: sharing the files out anew for the check (a pool's way) would parse them again.
|
|
432
|
-
"""
|
|
433
|
-
|
|
434
|
-
def __init__(self, paths: Sequence[Path], jobs: int) -> None:
|
|
435
|
-
"""Share `paths` out among up to `jobs` workers (see `_shares`)."""
|
|
436
|
-
self.paths: Sequence[Path] = paths
|
|
437
|
-
self.shares: list[list[int]] = _shares(paths, jobs)
|
|
438
|
-
self.stack: contextlib.ExitStack = contextlib.ExitStack()
|
|
439
|
-
self.pools: list[ProcessPoolExecutor] = []
|
|
440
|
-
|
|
441
|
-
def __enter__(self) -> Self:
|
|
442
|
-
"""Start the workers, one process each, each with its part of the budget.
|
|
443
|
-
|
|
444
|
-
Returns:
|
|
445
|
-
Them.
|
|
446
|
-
|
|
447
|
-
"""
|
|
448
|
-
self.pools = [
|
|
449
|
-
self.stack.enter_context(
|
|
450
|
-
ProcessPoolExecutor(max_workers=1, initializer=parsed.budget, initargs=(len(self.shares),)),
|
|
451
|
-
)
|
|
452
|
-
for _ in self.shares
|
|
453
|
-
]
|
|
454
|
-
return self
|
|
455
|
-
|
|
456
|
-
def __exit__(self, *_details: object) -> None:
|
|
457
|
-
"""Stop the workers."""
|
|
458
|
-
self.stack.close()
|
|
459
|
-
|
|
460
|
-
def index(self) -> project.Index:
|
|
461
|
-
"""Index every file, each worker its share.
|
|
462
|
-
|
|
463
|
-
Returns:
|
|
464
|
-
The index, as one process would build it.
|
|
465
|
-
|
|
466
|
-
"""
|
|
467
|
-
read: list[_Reading] = [
|
|
468
|
-
pool.submit(_read_share, [self.paths[at] for at in share])
|
|
469
|
-
for pool, share in zip(self.pools, self.shares, strict=True)
|
|
470
|
-
]
|
|
471
|
-
found: dict[int, project.Module | None] = {}
|
|
472
|
-
share: list[int]
|
|
473
|
-
future: _Reading
|
|
474
|
-
for share, future in zip(self.shares, read, strict=True):
|
|
475
|
-
found.update(zip(share, future.result(), strict=True))
|
|
476
|
-
return project.indexed(found[at] for at in range(len(self.paths))) # in their order, as one at a time
|
|
477
|
-
|
|
478
|
-
def check(self, check: _Check, outside: Sequence[Outside]) -> list[_FileRun]:
|
|
479
|
-
"""Check every file, each worker the share it indexed.
|
|
480
|
-
|
|
481
|
-
Returns:
|
|
482
|
-
What each file found, in their order.
|
|
483
|
-
|
|
484
|
-
"""
|
|
485
|
-
checked: list[_Checking] = [
|
|
486
|
-
pool.submit(_check_share, check, [(self.paths[at], outside[at]) for at in share])
|
|
487
|
-
for pool, share in zip(self.pools, self.shares, strict=True)
|
|
488
|
-
]
|
|
489
|
-
found: dict[int, _FileRun] = {}
|
|
490
|
-
share: list[int]
|
|
491
|
-
future: _Checking
|
|
492
|
-
for share, future in zip(self.shares, checked, strict=True):
|
|
493
|
-
found.update(zip(share, future.result(), strict=True))
|
|
494
|
-
return [found[at] for at in range(len(self.paths))]
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
def _shares(paths: Sequence[Path], workers: int) -> list[list[int]]:
|
|
498
|
-
"""Share the files among the workers, the biggest first to the least loaded: a check's time goes by size.
|
|
499
|
-
|
|
500
|
-
Returns:
|
|
501
|
-
Each worker's files, by their place in `paths` (no more workers than files).
|
|
502
|
-
|
|
503
|
-
"""
|
|
504
|
-
loads: list[int] = [0] * min(workers, len(paths))
|
|
505
|
-
shares: list[list[int]] = [[] for _ in loads]
|
|
506
|
-
sizes: list[int] = [_size(path) for path in paths]
|
|
507
|
-
index: int
|
|
508
|
-
for index in sorted(range(len(paths)), key=lambda at: -sizes[at]):
|
|
509
|
-
least: int = loads.index(min(loads))
|
|
510
|
-
shares[least].append(index)
|
|
511
|
-
loads[least] += sizes[index] or 1
|
|
512
|
-
return [sorted(share) for share in shares if share]
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
def _size(path: Path) -> int:
|
|
516
|
-
"""Find a file's size, to balance the workers by.
|
|
517
|
-
|
|
518
|
-
Returns:
|
|
519
|
-
It, or 0 if it can't be read (checking it reports that).
|
|
520
|
-
|
|
521
|
-
"""
|
|
522
|
-
try:
|
|
523
|
-
return path.stat().st_size
|
|
524
|
-
except OSError:
|
|
525
|
-
return 0
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
def _read_share(paths: Sequence[Path]) -> list[project.Module | None]:
|
|
529
|
-
"""Index a worker's share of the files (see `project.read`).
|
|
530
|
-
|
|
531
|
-
Returns:
|
|
532
|
-
Each file's module, or `None`.
|
|
533
|
-
|
|
534
|
-
"""
|
|
535
|
-
return [project.read(path) for path in paths]
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
def _check_share(
|
|
539
|
-
check: _Check,
|
|
540
|
-
files: Sequence[tuple[Path, Outside]],
|
|
541
|
-
) -> list[_FileRun]:
|
|
542
|
-
"""Check a worker's share of the files.
|
|
543
|
-
|
|
544
|
-
Returns:
|
|
545
|
-
What each found.
|
|
546
|
-
|
|
547
|
-
"""
|
|
548
|
-
return list(itertools.starmap(check, files))
|
|
414
|
+
collecting.indexed()
|
|
415
|
+
return schedule.checked(
|
|
416
|
+
paths,
|
|
417
|
+
check,
|
|
418
|
+
pool,
|
|
419
|
+
(modules, hinted),
|
|
420
|
+
_merged if options.mode is Mode.FIX else None,
|
|
421
|
+
)
|
|
549
422
|
|
|
550
423
|
|
|
551
|
-
def _merged(before:
|
|
552
|
-
"""Join a file's two `--fix` rounds: what's left is the later
|
|
424
|
+
def _merged(before: FileRun, after: FileRun) -> FileRun:
|
|
425
|
+
"""Join a file's two `--fix` rounds' `CheckRun`s: what's left is the later's, what's fixed is both's.
|
|
553
426
|
|
|
554
427
|
Returns:
|
|
555
428
|
The joined run.
|
|
556
429
|
|
|
557
430
|
"""
|
|
558
|
-
|
|
431
|
+
first: CheckRun = cast("CheckRun", before)
|
|
432
|
+
later: CheckRun = cast("CheckRun", after)
|
|
433
|
+
return replace(later, fixed=first.fixed + later.fixed)
|
|
559
434
|
|
|
560
435
|
|
|
561
436
|
def _texts(paths: Sequence[Path]) -> dict[Path, str]:
|
|
@@ -577,35 +452,14 @@ def _texts(paths: Sequence[Path]) -> dict[Path, str]:
|
|
|
577
452
|
return texts
|
|
578
453
|
|
|
579
454
|
|
|
580
|
-
def
|
|
581
|
-
modules: project.Index,
|
|
582
|
-
paths: Sequence[Path],
|
|
583
|
-
hinted: Mapping[Path, tuple[Hints, ...]],
|
|
584
|
-
) -> list[Outside]:
|
|
585
|
-
"""Find what's known of each file from outside it: what it imports from the others, and its hints.
|
|
586
|
-
|
|
587
|
-
Returns:
|
|
588
|
-
Each file's, in order.
|
|
589
|
-
|
|
590
|
-
"""
|
|
591
|
-
found: list[Outside] = []
|
|
592
|
-
path: Path
|
|
593
|
-
for path in paths:
|
|
594
|
-
imported: project.Imported = project.imported(modules, path)
|
|
595
|
-
found.append(
|
|
596
|
-
Outside(imported.calls, imported.classes, hinted.get(path, ()), project.type_vars(modules, path)),
|
|
597
|
-
)
|
|
598
|
-
return found
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
def _report(options: Options, runs: Sequence[_FileRun], files: int) -> int:
|
|
455
|
+
def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
|
|
602
456
|
"""Print the results (`runs` is check/fix/diff mode's: every other mode has its own printing).
|
|
603
457
|
|
|
604
458
|
Returns:
|
|
605
459
|
The exit status.
|
|
606
460
|
|
|
607
461
|
"""
|
|
608
|
-
checked: Sequence[
|
|
462
|
+
checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
|
|
609
463
|
results: list[Result] = [result for run in checked for result in run.results]
|
|
610
464
|
output: Output = options.output
|
|
611
465
|
text: bool = output.fmt in {Format.TEXT, Format.FULL}
|
|
@@ -630,14 +484,14 @@ def _report(options: Options, runs: Sequence[_FileRun], files: int) -> int:
|
|
|
630
484
|
return EXIT_FOUND if errors else EXIT_CLEAN
|
|
631
485
|
|
|
632
486
|
|
|
633
|
-
def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[
|
|
487
|
+
def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun]) -> int:
|
|
634
488
|
"""Print each file's and the total annotation coverage (`runs` is --coverage mode's).
|
|
635
489
|
|
|
636
490
|
Returns:
|
|
637
491
|
The exit status.
|
|
638
492
|
|
|
639
493
|
"""
|
|
640
|
-
covered: Sequence[
|
|
494
|
+
covered: Sequence[CoverageRun] = cast("Sequence[CoverageRun]", runs)
|
|
641
495
|
counted: list[tuple[Path, Coverage]] = [
|
|
642
496
|
(path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
|
|
643
497
|
]
|
|
@@ -673,18 +527,19 @@ def _run(options: Options) -> int:
|
|
|
673
527
|
|
|
674
528
|
"""
|
|
675
529
|
names: list[Path]
|
|
676
|
-
runs: list[
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
530
|
+
runs: list[FileRun]
|
|
531
|
+
with collecting.by_hand():
|
|
532
|
+
try:
|
|
533
|
+
names, runs = _check_all(options)
|
|
534
|
+
except HintError as error:
|
|
535
|
+
_ = sys.stderr.write(f"constricter: error: {error}\n")
|
|
536
|
+
return EXIT_ERROR
|
|
682
537
|
_ = sys.stderr.write("".join(f"{run.error}\n" for run in runs if run.error))
|
|
683
538
|
failed: bool = any(run.error for run in runs)
|
|
684
539
|
status: int
|
|
685
540
|
if options.mode is Mode.WRITE_BASELINE and options.filter.baseline_file is not None:
|
|
686
541
|
file: Path = options.filter.baseline_file
|
|
687
|
-
baselined: Sequence[
|
|
542
|
+
baselined: Sequence[BaselineRun] = cast("Sequence[BaselineRun]", runs)
|
|
688
543
|
found: dict[str, list[Offence]] = {
|
|
689
544
|
baseline.key(n, file): run.found for n, run in zip(names, baselined, strict=True)
|
|
690
545
|
}
|
|
@@ -693,12 +548,12 @@ def _run(options: Options) -> int:
|
|
|
693
548
|
elif options.mode is Mode.COVERAGE:
|
|
694
549
|
status = _coverage(options, names, runs)
|
|
695
550
|
elif options.mode is Mode.DIFF:
|
|
696
|
-
checked: Sequence[
|
|
551
|
+
checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
|
|
697
552
|
diffs: str = "".join(run.text for run in checked)
|
|
698
553
|
_ = sys.stdout.write(diffs)
|
|
699
554
|
status = EXIT_FOUND if diffs else EXIT_CLEAN
|
|
700
555
|
elif options.mode is Mode.FIX and options.input.paths == [STDIN] and runs and not failed:
|
|
701
|
-
fixed:
|
|
556
|
+
fixed: CheckRun = cast("CheckRun", runs[0])
|
|
702
557
|
_ = sys.stdout.write(fixed.text) # standard input, fixed, is the whole output
|
|
703
558
|
status = EXIT_FOUND if any(r.offence.is_error(r.level) for r in fixed.results) else EXIT_CLEAN
|
|
704
559
|
else:
|