python-constricter 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. constricter/__init__.py +1 -1
  2. constricter/cli/collecting.py +47 -0
  3. constricter/cli/command.py +114 -259
  4. constricter/cli/config.py +8 -15
  5. constricter/cli/hints.py +9 -103
  6. constricter/cli/options.py +1 -1
  7. constricter/cli/paths.py +31 -0
  8. constricter/cli/protocol.py +110 -0
  9. constricter/cli/runs.py +42 -0
  10. constricter/cli/schedule.py +222 -0
  11. constricter/cli/workers.py +154 -0
  12. constricter/fix/doubts.py +115 -16
  13. constricter/fix/fills.py +47 -45
  14. constricter/fix/fixes.py +60 -20
  15. constricter/fix/guesses.py +74 -11
  16. constricter/fix/imports.py +76 -25
  17. constricter/fix/inference.py +49 -75
  18. constricter/fix/known.py +79 -5
  19. constricter/fix/library.py +80 -0
  20. constricter/fix/members.py +10 -0
  21. constricter/fix/modules.py +252 -0
  22. constricter/fix/narrowed.py +164 -0
  23. constricter/fix/order.py +145 -0
  24. constricter/fix/overloads.py +394 -0
  25. constricter/fix/project.py +408 -284
  26. constricter/fix/returned.py +374 -80
  27. constricter/fix/stdlib.py +207 -50
  28. constricter/fix/tables/aliases.json +147 -0
  29. constricter/fix/tables/attributes.json +675 -0
  30. constricter/fix/tables/bases.json +888 -0
  31. constricter/fix/tables/classes.json +1939 -0
  32. constricter/fix/tables/method_overloads.json +353 -0
  33. constricter/fix/tables/method_signatures.json +655 -0
  34. constricter/fix/tables/methods.json +857 -0
  35. constricter/fix/tables/overloads.json +373 -0
  36. constricter/fix/tables/returns.json +1328 -0
  37. constricter/fix/tables/source.json +4 -0
  38. constricter/fix/tables/type_parameters.json +175 -0
  39. constricter/offences.py +4 -0
  40. constricter/rules/annotations.py +55 -1
  41. constricter/rules/binding.py +241 -0
  42. constricter/rules/checker.py +259 -333
  43. constricter/rules/flow.py +2 -0
  44. constricter/rules/late.py +288 -0
  45. constricter/rules/narrowing.py +48 -1
  46. constricter/rules/parsed.py +18 -11
  47. constricter/rules/redundant.py +2 -8
  48. constricter/rules/scope.py +97 -223
  49. constricter/rules/syntax.py +98 -3
  50. constricter/rules/walked.py +59 -7
  51. {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/METADATA +9 -8
  52. python_constricter-0.2.7.dist-info/RECORD +75 -0
  53. constricter/fix/stdlib.json +0 -15351
  54. python_constricter-0.2.5.dist-info/RECORD +0 -53
  55. {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/WHEEL +0 -0
  56. {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/entry_points.txt +0 -0
  57. {python_constricter-0.2.5.dist-info → python_constricter-0.2.7.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py CHANGED
@@ -25,7 +25,7 @@ from constricter.offences import (
25
25
  )
26
26
  from constricter.rules.checker import Coverage, annotation_coverage, check_source, check_tree
27
27
 
28
- __version__ = "0.2.5"
28
+ __version__ = "0.2.7"
29
29
  __all__ = [
30
30
  "COMMENT_TYPED_TARGET",
31
31
  "DEFAULT_CHECKS",
@@ -0,0 +1,47 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """The garbage collector, as the command runs it: by hand, once per file checked.
3
+
4
+ A check keeps every file's tree (see `parsed`), and allocates a great deal for each file, little of it
5
+ cyclic. Left to itself, the collector looked through the kept trees again and again as the check
6
+ allocated: a third of the standard library's check (34s, 21s without). So while the command runs,
7
+ it's off: what indexing built is frozen (never looked through again), and after each file, only
8
+ what's been allocated since (the young generations) is collected, which bounds what a cycle can hold.
9
+ """
10
+
11
+ import contextlib
12
+ import gc
13
+ from collections.abc import Generator
14
+ from typing import Final
15
+
16
+ _YOUNG: Final = 1 # the generations collected after each file: 0 and 1, never the frozen or the old
17
+ _FREEZES: Final = hasattr(gc, "freeze") # PyPy's collector has no freezing
18
+
19
+
20
+ @contextlib.contextmanager
21
+ def by_hand() -> Generator[None]:
22
+ """Turn automatic collection off for the command's run, and back as it was after.
23
+
24
+ Yields:
25
+ Nothing: the run.
26
+
27
+ """
28
+ was: bool = gc.isenabled()
29
+ gc.disable()
30
+ try:
31
+ yield
32
+ finally:
33
+ if _FREEZES:
34
+ gc.unfreeze()
35
+ if was:
36
+ gc.enable()
37
+
38
+
39
+ def indexed() -> None:
40
+ """Freeze what indexing built (the kept trees, the index): it lives until the check takes it."""
41
+ if _FREEZES:
42
+ gc.freeze()
43
+
44
+
45
+ def sweep() -> None:
46
+ """Collect what the file just checked left behind (see the module docstring)."""
47
+ _ = gc.collect(_YOUNG)
@@ -4,25 +4,27 @@
4
4
  import codecs
5
5
  import contextlib
6
6
  import difflib
7
+ import importlib
7
8
  import io
8
- import itertools
9
9
  import json
10
10
  import sys
11
11
  import tokenize
12
- from collections.abc import Callable, Mapping, Sequence
13
- from concurrent.futures import Future, ProcessPoolExecutor
14
- from dataclasses import dataclass, field, replace
12
+ from collections.abc import Callable, Sequence
13
+ from dataclasses import replace
15
14
  from functools import partial
16
15
  from pathlib import Path
17
- from typing import Final, NamedTuple, Self, TextIO, TypeAlias, cast
16
+ from typing import TYPE_CHECKING, Final, NamedTuple, TextIO, TypeAlias, cast
18
17
 
19
18
  from constricter import notebook
20
- from constricter.cli import baseline, hints
19
+ from constricter.cli import baseline, collecting, schedule
21
20
  from constricter.cli.options import Mode, Options, Output
22
21
  from constricter.cli.paths import STDIN, python_files
22
+ from constricter.cli.protocol import HintError
23
23
  from constricter.cli.report import Format, Result, fix_reasons, render, statistics
24
+ from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
25
+ from constricter.cli.workers import Workers
24
26
  from constricter.fix import fixes, project
25
- from constricter.fix.known import Hints, Outside
27
+ from constricter.fix.known import Hints, Outside, Returns
26
28
  from constricter.noqa import lines, unsuppressed
27
29
  from constricter.offences import (
28
30
  DEFAULT_CHECKS,
@@ -30,8 +32,10 @@ from constricter.offences import (
30
32
  Edit,
31
33
  Offence,
32
34
  )
33
- from constricter.rules import parsed
34
- from constricter.rules.checker import Coverage, annotation_coverage, check_source
35
+ from constricter.rules.checker import Checked, Coverage, annotation_coverage, checked_source
36
+
37
+ if TYPE_CHECKING: # slow to import, and only needed for `--infer-with`
38
+ from constricter.cli import hints
35
39
 
36
40
  EXIT_CLEAN: Final = 0
37
41
  EXIT_FOUND: Final = 1
@@ -75,30 +79,29 @@ def _source(raw: str, name: Path) -> tuple[str, list[notebook.Line]]:
75
79
  return notebook.parse(raw, str(name)) if name.suffix == notebook.SUFFIX else (raw, [])
76
80
 
77
81
 
78
- def check_text(
82
+ def checked_text(
79
83
  raw: str,
80
84
  name: Path,
81
85
  checks: Checks = DEFAULT_CHECKS,
82
86
  *,
83
87
  outside: Outside | None = None,
84
- ) -> list[Offence]:
85
- """Return the offences in `raw`, the text of `name`, that no `# noqa` suppresses.
88
+ ) -> Checked:
89
+ """Check `raw`, the text of `name`: the offences no `# noqa` suppresses.
86
90
 
87
91
  A notebook's code cells are checked as one module, and each offence placed in its cell.
88
92
  Raises `ValueError` for a `.ipynb` that isn't a notebook.
89
93
 
90
94
  Returns:
91
- The unsuppressed offences, a notebook's placed in their cells.
95
+ The unsuppressed offences, a notebook's placed in their cells, and what its functions return
96
+ (see `Checked`).
92
97
 
93
98
  """
94
99
  source: str
95
100
  where: list[notebook.Line]
96
101
  source, where = _source(raw, name)
97
- offences: list[Offence] = unsuppressed(
98
- check_source(source, str(name), checks, outside=outside),
99
- lines(source),
100
- )
101
- return [_placed(o, where) for o in offences] if where else offences
102
+ checked: Checked = checked_source(source, str(name), checks, outside=outside)
103
+ offences: list[Offence] = unsuppressed(checked.offences, lines(source))
104
+ return checked._replace(offences=[_placed(o, where) for o in offences] if where else offences)
102
105
 
103
106
 
104
107
  def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
@@ -111,7 +114,8 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
111
114
  """
112
115
  line: notebook.Line = where[offence.line - 1]
113
116
  placed: Offence = replace(offence, line=line.line, cell=line.cell)
114
- if offence.edit is not None and offence.edit.imports: # a notebook's cells have no import block
117
+ # A notebook's cells have no import block.
118
+ if offence.edit is not None and (offence.edit.imports or offence.edit.guarded):
115
119
  return replace(placed, edit=None)
116
120
  if offence.edit is not None and offence.edit.edit is Edit.DECLARE:
117
121
  statement: int = where[offence.edit.span[0] - 1].line
@@ -176,38 +180,6 @@ def fix_file(path: Path, offences: Sequence[Offence]) -> int:
176
180
  return count
177
181
 
178
182
 
179
- @dataclass(frozen=True)
180
- class _CheckRun:
181
- """What checking one file found (and fixed, or would fix), in check/fix/diff mode."""
182
-
183
- results: list[Result] = field(default_factory=list[Result])
184
- baselined: int = 0
185
- fixed: int = 0 # --fix: how many offences it fixed
186
- text: str = "" # --diff: the diff; --fix on standard input: the fixed source
187
- error: str = ""
188
-
189
-
190
- @dataclass(frozen=True)
191
- class _BaselineRun:
192
- """One file's offences, unfiltered, for --write-baseline."""
193
-
194
- found: list[Offence] = field(default_factory=list[Offence])
195
- error: str = ""
196
-
197
-
198
- @dataclass(frozen=True)
199
- class _CoverageRun:
200
- """One file's annotation coverage, for --coverage."""
201
-
202
- coverage: Coverage | None = None
203
- error: str = ""
204
-
205
-
206
- _FileRun: TypeAlias = _CheckRun | _BaselineRun | _CoverageRun
207
- _Check: TypeAlias = Callable[[Path, Outside], _FileRun] # check (or count, or baseline) one file
208
- # A worker's answers, one per file of its share: what it read (to index), and what it found.
209
- _Reading: TypeAlias = Future[list[project.Module | None]]
210
- _Checking: TypeAlias = Future[list[_FileRun]]
211
183
  HINT_ROUNDS: Final = 4 # with `--fix --infer-with`: how many times each file is fixed, at most
212
184
 
213
185
 
@@ -227,15 +199,15 @@ def _shown_lines(raw: str, name: Path) -> dict[tuple[int | None, int], str]:
227
199
  }
228
200
 
229
201
 
230
- def _checked(path: Path, name: Path, checks: Checks, outside: Outside) -> tuple[str, list[Offence]]:
202
+ def _checked(path: Path, name: Path, checks: Checks, outside: Outside) -> tuple[str, Checked]:
231
203
  """Read `path` and check it as `name`; raises what reading or parsing it does.
232
204
 
233
205
  Returns:
234
- Its text, and its offences.
206
+ Its text, and what checking it found.
235
207
 
236
208
  """
237
209
  raw: str = _read(path)
238
- return raw, check_text(raw, name, checks, outside=outside)
210
+ return raw, checked_text(raw, name, checks, outside=outside)
239
211
 
240
212
 
241
213
  def _read_checked(
@@ -243,20 +215,20 @@ def _read_checked(
243
215
  name: Path,
244
216
  checks: Checks,
245
217
  outside: Outside,
246
- ) -> tuple[str, list[Offence], str]:
218
+ ) -> tuple[str, Checked, str]:
247
219
  """Read and check `path`, turning a read or parse error into a message instead of raising.
248
220
 
249
221
  Returns:
250
- Its text and offences (empty on error), and the error message (empty on success).
222
+ Its text and what checking it found (nothing on error), and the error message (empty on success).
251
223
 
252
224
  """
253
225
  raw: str
254
- offences: list[Offence]
226
+ checked: Checked
255
227
  try:
256
- raw, offences = _checked(path, name, checks, outside)
228
+ raw, checked = _checked(path, name, checks, outside)
257
229
  except (OSError, ValueError, SyntaxError) as error: # UnicodeDecodeError is a ValueError
258
- return "", [], f"{name}: error: {error}"
259
- return raw, offences, ""
230
+ return "", Checked([], Returns()), f"{name}: error: {error}"
231
+ return raw, checked, ""
260
232
 
261
233
 
262
234
  def _results(raw: str, name: Path, offences: Sequence[Offence], options: Options) -> list[Result]:
@@ -278,7 +250,7 @@ def _results(raw: str, name: Path, offences: Sequence[Offence], options: Options
278
250
  ]
279
251
 
280
252
 
281
- def _check_path(path: Path, outside: Outside, options: Options) -> _CheckRun:
253
+ def _check_path(path: Path, outside: Outside, options: Options) -> CheckRun:
282
254
  """Check (and fix, or diff) one file, given what's known of it from outside it.
283
255
 
284
256
  Returns:
@@ -287,11 +259,24 @@ def _check_path(path: Path, outside: Outside, options: Options) -> _CheckRun:
287
259
  """
288
260
  name: Path = options.input.name(path)
289
261
  raw: str
290
- offences: list[Offence]
262
+ checked: Checked
291
263
  error: str
292
- raw, offences, error = _read_checked(path, name, options.checks, outside)
264
+ raw, checked, error = _read_checked(path, name, options.checks, outside)
293
265
  if error:
294
- return _CheckRun(error=error)
266
+ return CheckRun(error=error)
267
+ return replace(_handled(path, (raw, name), checked.offences, options), returned=checked.returned)
268
+
269
+
270
+ def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], options: Options) -> CheckRun:
271
+ """Report, fix or diff the offences found in `path` (`text`: its text, and its name).
272
+
273
+ Returns:
274
+ What was found, and fixed.
275
+
276
+ """
277
+ raw: str
278
+ name: Path
279
+ raw, name = text
295
280
  baselined: int
296
281
  offences, baselined = options.filter.unbaselined(name, offences)
297
282
  results: list[Result] = _results(raw, name, offences, options)
@@ -300,22 +285,22 @@ def _check_path(path: Path, outside: Outside, options: Options) -> _CheckRun:
300
285
  r.offence for r in results if r.offence.edit is not None and (unsafe or not r.offence.unsafe)
301
286
  ]
302
287
  if options.mode is Mode.DIFF:
303
- return _CheckRun(text=_diff(raw, name, fixing))
288
+ return CheckRun(text=_diff(raw, name, fixing))
304
289
  if options.mode is not Mode.FIX:
305
- return _CheckRun(results, baselined)
290
+ return CheckRun(results, baselined)
306
291
  left: list[Result] = [r for r in results if r.offence not in fixing]
307
292
  if path == STDIN: # the fixed source goes to stdout
308
- return _CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
293
+ return CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
309
294
  try:
310
- return _CheckRun(left, baselined, fix_file(path, fixing))
295
+ return CheckRun(left, baselined, fix_file(path, fixing))
311
296
  except UnicodeEncodeError as failure: # the file is left as it was, its offences unfixed
312
297
  # Its canonical name: PyPy reports `latin1` where CPython says `latin-1`.
313
298
  encoding: str = codecs.lookup(failure.encoding).name
314
299
  message: str = f"an annotation can't be written in its encoding, {encoding}; left as it was"
315
- return _CheckRun(results, baselined, error=f"{name}: error: {message}")
300
+ return CheckRun(results, baselined, error=f"{name}: error: {message}")
316
301
 
317
302
 
318
- def _baseline_path(path: Path, outside: Outside, options: Options) -> _BaselineRun:
303
+ def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
319
304
  """Check one file, unfiltered, for --write-baseline.
320
305
 
321
306
  Returns:
@@ -323,13 +308,13 @@ def _baseline_path(path: Path, outside: Outside, options: Options) -> _BaselineR
323
308
 
324
309
  """
325
310
  name: Path = options.input.name(path)
326
- offences: list[Offence]
311
+ checked: Checked
327
312
  error: str
328
- _, offences, error = _read_checked(path, name, options.checks, outside)
329
- return _BaselineRun(error=error) if error else _BaselineRun(found=offences)
313
+ _, checked, error = _read_checked(path, name, options.checks, outside)
314
+ return BaselineRun(error=error) if error else BaselineRun(checked.offences, returned=checked.returned)
330
315
 
331
316
 
332
- def _cover_path(path: Path, _outside: Outside, options: Options) -> _CoverageRun:
317
+ def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
333
318
  """Count one file's typed first bindings.
334
319
 
335
320
  Returns:
@@ -338,12 +323,12 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> _CoverageRun
338
323
  """
339
324
  name: Path = options.input.name(path)
340
325
  try:
341
- return _CoverageRun(annotation_coverage(_source(_read(path), name)[0], options.checks))
326
+ return CoverageRun(annotation_coverage(_source(_read(path), name)[0], options.checks))
342
327
  except (OSError, ValueError, SyntaxError) as error:
343
- return _CoverageRun(error=f"{name}: error: {error}")
328
+ return CoverageRun(error=f"{name}: error: {error}")
344
329
 
345
330
 
346
- def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
331
+ def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
347
332
  """Check every file (`--jobs` at a time), in order.
348
333
 
349
334
  With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
@@ -357,7 +342,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
357
342
 
358
343
  """
359
344
  paths: list[Path] = list(python_files(options.input.paths, options.input.exclude))
360
- check: Callable[[Path, Outside], _FileRun]
345
+ check: Callable[[Path, Outside], FileRun]
361
346
  if options.mode is Mode.COVERAGE:
362
347
  check = partial(_cover_path, options=options)
363
348
  elif options.mode is Mode.WRITE_BASELINE:
@@ -368,20 +353,26 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
368
353
  if not options.infer_with or options.mode in {Mode.COVERAGE, Mode.WRITE_BASELINE}:
369
354
  return names, _checked_all(paths, check, options)[0]
370
355
  session: hints.Session
371
- with hints.Session(options.infer_with, Path.cwd(), options.jobs, options.infer_memory) as session:
372
- runs: list[_FileRun]
356
+ # Imported only for `--infer-with`: it's slow to import, for every run's start.
357
+ with cast("type[hints.Session]", importlib.import_module("constricter.cli.hints").Session)(
358
+ options.infer_with,
359
+ Path.cwd(),
360
+ options.jobs,
361
+ options.infer_memory,
362
+ ) as session:
363
+ runs: list[FileRun]
373
364
  modules: project.Index
374
365
  runs, modules = _checked_all(paths, check, options, session)
375
366
  # The files the last round changed: only their hints can have changed.
376
367
  again: list[int] = [
377
368
  index
378
369
  for index, run in enumerate(runs)
379
- if options.mode is Mode.FIX and isinstance(run, _CheckRun) and run.fixed and paths[index] != STDIN
370
+ if options.mode is Mode.FIX and isinstance(run, CheckRun) and run.fixed and paths[index] != STDIN
380
371
  ]
381
372
  for _ in range(HINT_ROUNDS - 1):
382
373
  if not again:
383
374
  break
384
- redone: list[_FileRun] = _checked_all(
375
+ redone: list[FileRun] = _checked_all(
385
376
  [paths[i] for i in again],
386
377
  check,
387
378
  options,
@@ -389,20 +380,20 @@ def _check_all(options: Options) -> tuple[list[Path], list[_FileRun]]:
389
380
  modules,
390
381
  )[0]
391
382
  index: int
392
- run: _FileRun
383
+ run: FileRun
393
384
  for index, run in zip(again, redone, strict=True):
394
- runs[index] = _merged(cast("_CheckRun", runs[index]), cast("_CheckRun", run))
395
- again = [index for index, run in zip(again, redone, strict=True) if cast("_CheckRun", run).fixed]
385
+ runs[index] = _merged(runs[index], run)
386
+ again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
396
387
  return names, runs
397
388
 
398
389
 
399
390
  def _checked_all(
400
391
  paths: Sequence[Path],
401
- check: Callable[[Path, Outside], _FileRun],
392
+ check: Callable[[Path, Outside], FileRun],
402
393
  options: Options,
403
- session: hints.Session | None = None,
394
+ session: "hints.Session | None" = None,
404
395
  modules: project.Index | None = None,
405
- ) -> tuple[list[_FileRun], project.Index]:
396
+ ) -> tuple[list[FileRun], project.Index]:
406
397
  """Check `paths` (`--jobs` at a time), with the `session`'s hints, and `modules` (else indexed).
407
398
 
408
399
  Returns:
@@ -411,151 +402,35 @@ def _checked_all(
411
402
  """
412
403
  hinted: dict[Path, tuple[Hints, ...]] = {} if session is None else session.hints(_texts(paths))
413
404
  coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
414
- if options.jobs == 1 or len(paths) <= 1:
415
- if modules is None:
405
+ stack: contextlib.ExitStack
406
+ with contextlib.ExitStack() as stack:
407
+ pool: Workers | None = None
408
+ if options.jobs != 1 and len(paths) > 1:
409
+ pool = stack.enter_context(Workers(paths, options.jobs))
410
+ if modules is None and pool:
411
+ modules = project.Index({}, []) if coverage else pool.index()
412
+ elif modules is None:
416
413
  modules = project.Index({}, []) if coverage else project.index(paths)
417
- return list(
418
- itertools.starmap(check, zip(paths, _outside(modules, paths, hinted), strict=True)),
419
- ), modules
420
- workers: _Workers
421
- with _Workers(paths, options.jobs) as workers:
422
- if modules is None:
423
- modules = project.Index({}, []) if coverage else workers.index()
424
- return workers.check(check, _outside(modules, paths, hinted)), modules
425
-
426
-
427
- class _Workers:
428
- """Worker processes, each with its own share of the files, which it indexes and then checks.
429
-
430
- The trees a worker parsed to index its files are still there to check (`parsed.keep`), within its
431
- part of the budget: sharing the files out anew for the check (a pool's way) would parse them again.
432
- """
433
-
434
- def __init__(self, paths: Sequence[Path], jobs: int) -> None:
435
- """Share `paths` out among up to `jobs` workers (see `_shares`)."""
436
- self.paths: Sequence[Path] = paths
437
- self.shares: list[list[int]] = _shares(paths, jobs)
438
- self.stack: contextlib.ExitStack = contextlib.ExitStack()
439
- self.pools: list[ProcessPoolExecutor] = []
440
-
441
- def __enter__(self) -> Self:
442
- """Start the workers, one process each, each with its part of the budget.
443
-
444
- Returns:
445
- Them.
446
-
447
- """
448
- self.pools = [
449
- self.stack.enter_context(
450
- ProcessPoolExecutor(max_workers=1, initializer=parsed.budget, initargs=(len(self.shares),)),
451
- )
452
- for _ in self.shares
453
- ]
454
- return self
455
-
456
- def __exit__(self, *_details: object) -> None:
457
- """Stop the workers."""
458
- self.stack.close()
459
-
460
- def index(self) -> project.Index:
461
- """Index every file, each worker its share.
462
-
463
- Returns:
464
- The index, as one process would build it.
465
-
466
- """
467
- read: list[_Reading] = [
468
- pool.submit(_read_share, [self.paths[at] for at in share])
469
- for pool, share in zip(self.pools, self.shares, strict=True)
470
- ]
471
- found: dict[int, project.Module | None] = {}
472
- share: list[int]
473
- future: _Reading
474
- for share, future in zip(self.shares, read, strict=True):
475
- found.update(zip(share, future.result(), strict=True))
476
- return project.indexed(found[at] for at in range(len(self.paths))) # in their order, as one at a time
477
-
478
- def check(self, check: _Check, outside: Sequence[Outside]) -> list[_FileRun]:
479
- """Check every file, each worker the share it indexed.
480
-
481
- Returns:
482
- What each file found, in their order.
483
-
484
- """
485
- checked: list[_Checking] = [
486
- pool.submit(_check_share, check, [(self.paths[at], outside[at]) for at in share])
487
- for pool, share in zip(self.pools, self.shares, strict=True)
488
- ]
489
- found: dict[int, _FileRun] = {}
490
- share: list[int]
491
- future: _Checking
492
- for share, future in zip(self.shares, checked, strict=True):
493
- found.update(zip(share, future.result(), strict=True))
494
- return [found[at] for at in range(len(self.paths))]
495
-
496
-
497
- def _shares(paths: Sequence[Path], workers: int) -> list[list[int]]:
498
- """Share the files among the workers, the biggest first to the least loaded: a check's time goes by size.
499
-
500
- Returns:
501
- Each worker's files, by their place in `paths` (no more workers than files).
502
-
503
- """
504
- loads: list[int] = [0] * min(workers, len(paths))
505
- shares: list[list[int]] = [[] for _ in loads]
506
- sizes: list[int] = [_size(path) for path in paths]
507
- index: int
508
- for index in sorted(range(len(paths)), key=lambda at: -sizes[at]):
509
- least: int = loads.index(min(loads))
510
- shares[least].append(index)
511
- loads[least] += sizes[index] or 1
512
- return [sorted(share) for share in shares if share]
513
-
514
-
515
- def _size(path: Path) -> int:
516
- """Find a file's size, to balance the workers by.
517
-
518
- Returns:
519
- It, or 0 if it can't be read (checking it reports that).
520
-
521
- """
522
- try:
523
- return path.stat().st_size
524
- except OSError:
525
- return 0
526
-
527
-
528
- def _read_share(paths: Sequence[Path]) -> list[project.Module | None]:
529
- """Index a worker's share of the files (see `project.read`).
530
-
531
- Returns:
532
- Each file's module, or `None`.
533
-
534
- """
535
- return [project.read(path) for path in paths]
536
-
537
-
538
- def _check_share(
539
- check: _Check,
540
- files: Sequence[tuple[Path, Outside]],
541
- ) -> list[_FileRun]:
542
- """Check a worker's share of the files.
543
-
544
- Returns:
545
- What each found.
546
-
547
- """
548
- return list(itertools.starmap(check, files))
414
+ collecting.indexed()
415
+ return schedule.checked(
416
+ paths,
417
+ check,
418
+ pool,
419
+ (modules, hinted),
420
+ _merged if options.mode is Mode.FIX else None,
421
+ )
549
422
 
550
423
 
551
- def _merged(before: _CheckRun, after: _CheckRun) -> _CheckRun:
552
- """Join a file's two `--fix` rounds: what's left is the later round's, what's fixed is both's.
424
+ def _merged(before: FileRun, after: FileRun) -> FileRun:
425
+ """Join a file's two `--fix` rounds' `CheckRun`s: what's left is the later's, what's fixed is both's.
553
426
 
554
427
  Returns:
555
428
  The joined run.
556
429
 
557
430
  """
558
- return replace(after, fixed=before.fixed + after.fixed)
431
+ first: CheckRun = cast("CheckRun", before)
432
+ later: CheckRun = cast("CheckRun", after)
433
+ return replace(later, fixed=first.fixed + later.fixed)
559
434
 
560
435
 
561
436
  def _texts(paths: Sequence[Path]) -> dict[Path, str]:
@@ -577,35 +452,14 @@ def _texts(paths: Sequence[Path]) -> dict[Path, str]:
577
452
  return texts
578
453
 
579
454
 
580
- def _outside(
581
- modules: project.Index,
582
- paths: Sequence[Path],
583
- hinted: Mapping[Path, tuple[Hints, ...]],
584
- ) -> list[Outside]:
585
- """Find what's known of each file from outside it: what it imports from the others, and its hints.
586
-
587
- Returns:
588
- Each file's, in order.
589
-
590
- """
591
- found: list[Outside] = []
592
- path: Path
593
- for path in paths:
594
- imported: project.Imported = project.imported(modules, path)
595
- found.append(
596
- Outside(imported.calls, imported.classes, hinted.get(path, ()), project.type_vars(modules, path)),
597
- )
598
- return found
599
-
600
-
601
- def _report(options: Options, runs: Sequence[_FileRun], files: int) -> int:
455
+ def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
602
456
  """Print the results (`runs` is check/fix/diff mode's: every other mode has its own printing).
603
457
 
604
458
  Returns:
605
459
  The exit status.
606
460
 
607
461
  """
608
- checked: Sequence[_CheckRun] = cast("Sequence[_CheckRun]", runs)
462
+ checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
609
463
  results: list[Result] = [result for run in checked for result in run.results]
610
464
  output: Output = options.output
611
465
  text: bool = output.fmt in {Format.TEXT, Format.FULL}
@@ -630,14 +484,14 @@ def _report(options: Options, runs: Sequence[_FileRun], files: int) -> int:
630
484
  return EXIT_FOUND if errors else EXIT_CLEAN
631
485
 
632
486
 
633
- def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[_FileRun]) -> int:
487
+ def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun]) -> int:
634
488
  """Print each file's and the total annotation coverage (`runs` is --coverage mode's).
635
489
 
636
490
  Returns:
637
491
  The exit status.
638
492
 
639
493
  """
640
- covered: Sequence[_CoverageRun] = cast("Sequence[_CoverageRun]", runs)
494
+ covered: Sequence[CoverageRun] = cast("Sequence[CoverageRun]", runs)
641
495
  counted: list[tuple[Path, Coverage]] = [
642
496
  (path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
643
497
  ]
@@ -673,18 +527,19 @@ def _run(options: Options) -> int:
673
527
 
674
528
  """
675
529
  names: list[Path]
676
- runs: list[_FileRun]
677
- try:
678
- names, runs = _check_all(options)
679
- except hints.HintError as error:
680
- _ = sys.stderr.write(f"constricter: error: {error}\n")
681
- return EXIT_ERROR
530
+ runs: list[FileRun]
531
+ with collecting.by_hand():
532
+ try:
533
+ names, runs = _check_all(options)
534
+ except HintError as error:
535
+ _ = sys.stderr.write(f"constricter: error: {error}\n")
536
+ return EXIT_ERROR
682
537
  _ = sys.stderr.write("".join(f"{run.error}\n" for run in runs if run.error))
683
538
  failed: bool = any(run.error for run in runs)
684
539
  status: int
685
540
  if options.mode is Mode.WRITE_BASELINE and options.filter.baseline_file is not None:
686
541
  file: Path = options.filter.baseline_file
687
- baselined: Sequence[_BaselineRun] = cast("Sequence[_BaselineRun]", runs)
542
+ baselined: Sequence[BaselineRun] = cast("Sequence[BaselineRun]", runs)
688
543
  found: dict[str, list[Offence]] = {
689
544
  baseline.key(n, file): run.found for n, run in zip(names, baselined, strict=True)
690
545
  }
@@ -693,12 +548,12 @@ def _run(options: Options) -> int:
693
548
  elif options.mode is Mode.COVERAGE:
694
549
  status = _coverage(options, names, runs)
695
550
  elif options.mode is Mode.DIFF:
696
- checked: Sequence[_CheckRun] = cast("Sequence[_CheckRun]", runs)
551
+ checked: Sequence[CheckRun] = cast("Sequence[CheckRun]", runs)
697
552
  diffs: str = "".join(run.text for run in checked)
698
553
  _ = sys.stdout.write(diffs)
699
554
  status = EXIT_FOUND if diffs else EXIT_CLEAN
700
555
  elif options.mode is Mode.FIX and options.input.paths == [STDIN] and runs and not failed:
701
- fixed: _CheckRun = cast("_CheckRun", runs[0])
556
+ fixed: CheckRun = cast("CheckRun", runs[0])
702
557
  _ = sys.stdout.write(fixed.text) # standard input, fixed, is the whole output
703
558
  status = EXIT_FOUND if any(r.offence.is_error(r.level) for r in fixed.results) else EXIT_CLEAN
704
559
  else: