python-constricter 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. constricter/__init__.py +1 -1
  2. constricter/__main__.py +19 -2
  3. constricter/cli/command.py +103 -55
  4. constricter/cli/config.py +28 -1
  5. constricter/cli/hints.py +18 -12
  6. constricter/cli/options.py +29 -1
  7. constricter/cli/protocol.py +25 -5
  8. constricter/cli/remembered.py +52 -0
  9. constricter/cli/report.py +7 -5
  10. constricter/cli/runs.py +49 -2
  11. constricter/cli/schedule.py +221 -68
  12. constricter/cli/traced.py +191 -79
  13. constricter/cli/workers.py +85 -23
  14. constricter/fix/core/fixes.py +51 -7
  15. constricter/fix/core/imports.py +100 -2
  16. constricter/fix/core/inherited.py +117 -17
  17. constricter/fix/core/known.py +177 -49
  18. constricter/fix/core/signatures.py +22 -4
  19. constricter/fix/index/atoms.py +18 -3
  20. constricter/fix/index/awaits.py +90 -0
  21. constricter/fix/index/beyond.py +87 -29
  22. constricter/fix/index/callers.py +2 -1
  23. constricter/fix/index/chained.py +69 -0
  24. constricter/fix/index/classnames.py +17 -0
  25. constricter/fix/index/declared.py +68 -4
  26. constricter/fix/index/decorated.py +3 -1
  27. constricter/fix/index/installed.py +35 -3
  28. constricter/fix/index/linked.py +142 -0
  29. constricter/fix/index/loose.py +98 -0
  30. constricter/fix/index/managed.py +63 -0
  31. constricter/fix/index/modules.py +331 -47
  32. constricter/fix/index/offers.py +4 -1
  33. constricter/fix/index/order.py +10 -3
  34. constricter/fix/index/own_methods.py +67 -0
  35. constricter/fix/index/own_overloads.py +14 -1
  36. constricter/fix/index/own_types.py +161 -0
  37. constricter/fix/index/plain.py +3 -1
  38. constricter/fix/index/project.py +137 -153
  39. constricter/fix/index/sides.py +102 -7
  40. constricter/fix/index/strings.py +46 -0
  41. constricter/fix/index/stubbed.py +51 -12
  42. constricter/fix/index/taken.py +45 -0
  43. constricter/fix/index/tuples.py +54 -28
  44. constricter/fix/libraries/library.py +95 -7
  45. constricter/fix/libraries/opened.py +57 -16
  46. constricter/fix/libraries/overloads.py +32 -6
  47. constricter/fix/libraries/stdlib.py +64 -13
  48. constricter/fix/libraries/walks.py +49 -0
  49. constricter/fix/tables/attributes.json +2 -0
  50. constricter/fix/tables/classes.json +1 -0
  51. constricter/fix/tables/generic_attributes.json +2 -0
  52. constricter/fix/tables/method_overloads.json +14 -6
  53. constricter/fix/tables/method_signatures.json +50 -20
  54. constricter/fix/tables/methods.json +1 -0
  55. constricter/fix/tables/overloads.json +38 -37
  56. constricter/fix/tables/scalars.json +2 -0
  57. constricter/fix/tables/source.json +1 -1
  58. constricter/fix/tables/type_parameters.json +1 -0
  59. constricter/fix/values/aliased.py +9 -1
  60. constricter/fix/values/bodies.py +76 -0
  61. constricter/fix/values/callables.py +321 -0
  62. constricter/fix/values/called.py +10 -12
  63. constricter/fix/values/classvars.py +124 -21
  64. constricter/fix/values/decided.py +22 -2
  65. constricter/fix/values/doubts.py +104 -27
  66. constricter/fix/values/entered.py +192 -34
  67. constricter/fix/values/fills.py +160 -13
  68. constricter/fix/values/guesses.py +51 -19
  69. constricter/fix/values/inference.py +48 -54
  70. constricter/fix/values/members.py +273 -34
  71. constricter/fix/values/narrowed.py +60 -4
  72. constricter/fix/values/operated.py +7 -6
  73. constricter/fix/values/ordered.py +74 -0
  74. constricter/fix/values/returned.py +319 -224
  75. constricter/fix/values/returns.py +44 -1
  76. constricter/fix/values/shapes.py +132 -32
  77. constricter/fix/values/targets.py +42 -15
  78. constricter/fix/values/unchecked.py +162 -0
  79. constricter/offences.py +45 -6
  80. constricter/recording.py +571 -0
  81. constricter/rules/annotations.py +30 -10
  82. constricter/rules/binding.py +229 -37
  83. constricter/rules/calls.py +58 -8
  84. constricter/rules/checker.py +177 -70
  85. constricter/rules/keys.py +79 -0
  86. constricter/rules/late.py +349 -10
  87. constricter/rules/quoted.py +32 -0
  88. constricter/rules/rebinding.py +35 -10
  89. constricter/rules/recorded.py +12 -8
  90. constricter/rules/scope.py +241 -74
  91. constricter/rules/syntax.py +20 -0
  92. constricter/rules/tables.py +18 -4
  93. constricter/rules/widened.py +322 -0
  94. constricter/trace.py +247 -254
  95. {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/METADATA +42 -40
  96. python_constricter-0.3.5.dist-info/RECORD +140 -0
  97. {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/entry_points.txt +1 -1
  98. python_constricter-0.3.3.dist-info/RECORD +0 -122
  99. {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/WHEEL +0 -0
  100. {python_constricter-0.3.3.dist-info → python_constricter-0.3.5.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py CHANGED
@@ -25,7 +25,7 @@ from constricter.offences import (
25
25
  )
26
26
  from constricter.rules.checker import Coverage, annotation_coverage, check_source, check_tree
27
27
 
28
- __version__ = "0.3.3"
28
+ __version__ = "0.3.5"
29
29
  __all__ = [
30
30
  "COMMENT_TYPED_TARGET",
31
31
  "DEFAULT_CHECKS",
constricter/__main__.py CHANGED
@@ -1,9 +1,26 @@
1
1
  # SPDX-License-Identifier: MIT
2
- """`python -m constricter`."""
2
+ """`python -m constricter`, and the `constricter` script."""
3
3
 
4
+ import contextlib
5
+ import os
4
6
  import sys
5
7
 
6
8
  from constricter.cli.command import main
7
9
 
10
+
11
+ def run() -> None:
12
+ """Run the command, and leave with its status at once.
13
+
14
+ Not by the interpreter's own exit, which frees every parsed file and the index an object at a
15
+ time: a fifth of a check of a thousand files. What the command wrote is flushed first; nothing
16
+ else waits on the exit (its workers have ended, its output file is closed).
17
+ """
18
+ status: int = main()
19
+ with contextlib.suppress(OSError, ValueError): # a reader that left, a stream closed already
20
+ _ = sys.stdout.flush()
21
+ _ = sys.stderr.flush()
22
+ os._exit(status)
23
+
24
+
8
25
  if __name__ == "__main__": # not when `--jobs` workers import it
9
- sys.exit(main())
26
+ run()
@@ -21,11 +21,11 @@ from constricter.cli.options import Mode, Options, Output
21
21
  from constricter.cli.paths import STDIN, python_files
22
22
  from constricter.cli.protocol import HintError
23
23
  from constricter.cli.report import Format, Result, fix_reasons, render, statistics
24
- from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
25
- from constricter.cli.workers import Workers
24
+ from constricter.cli.runs import Added, BaselineRun, CheckRun, CoverageRun, FileRun, merged
25
+ from constricter.cli.workers import Workers, kept, started
26
26
  from constricter.fix.core import fixes
27
27
  from constricter.fix.core.known import Callee, Hints, Outside, Returns
28
- from constricter.fix.index import callers, decorated, installed, plain, project
28
+ from constricter.fix.index import callers, decorated, installed, loose, plain, project
29
29
  from constricter.noqa import lines, unsuppressed
30
30
  from constricter.offences import (
31
31
  DEFAULT_CHECKS,
@@ -127,6 +127,7 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
127
127
 
128
128
 
129
129
  # One changed part of a fixed file (the file, or a notebook's cell): its label, old and new lines.
130
+ _Hinted: TypeAlias = dict[Path, tuple[Hints, ...]] # each file's hints, a checker's each
130
131
  _Change: TypeAlias = tuple[str, list[str], list[str]]
131
132
  # One JSON value a coverage report holds, and one file's row of them.
132
133
  _Scalar: TypeAlias = str | int | float
@@ -308,7 +309,29 @@ def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], option
308
309
  return CheckRun(results, baselined, error=f"{name}: error: {message}")
309
310
  if fixed:
310
311
  _ = parsed.take(raw) # the tree of the text it had: no check will ask for it again
311
- return CheckRun(left, baselined, fixed)
312
+ return CheckRun(
313
+ left,
314
+ baselined,
315
+ fixed,
316
+ made=[r._replace(replacements=(), fixed=True) for r in results if r.offence in fixing],
317
+ rounds=(_added(raw, name, fixing),),
318
+ )
319
+
320
+
321
+ def _added(raw: str, name: Path, fixing: Sequence[Offence]) -> Added:
322
+ """Find the lines fixing `fixing` adds to `raw`, the text of `name` (see `fixes.inserted`).
323
+
324
+ Returns:
325
+ Each one's cell (`None` outside a notebook) and the line it goes before; a notebook's cells
326
+ get declarations alone.
327
+
328
+ """
329
+ if name.suffix != notebook.SUFFIX:
330
+ return tuple((None, line) for line in fixes.inserted(lines(raw), fixing))
331
+ declared: list[tuple[int, int]] = [
332
+ (o.cell or 0, o.edit.span[0]) for o in fixing if o.edit is not None and o.edit.edit is Edit.DECLARE
333
+ ]
334
+ return tuple(sorted(declared))
312
335
 
313
336
 
314
337
  def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
@@ -340,6 +363,17 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
340
363
 
341
364
 
342
365
  def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
366
+ """Check every file (see `_check_rounds`), its rounds by one set of worker processes.
367
+
368
+ Returns:
369
+ The names, and what each file found.
370
+
371
+ """
372
+ with kept():
373
+ return _check_rounds(options)
374
+
375
+
376
+ def _check_rounds(options: Options) -> tuple[list[Path], list[FileRun]]:
343
377
  """Check every file (`--jobs` at a time), in order.
344
378
 
345
379
  With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
@@ -401,7 +435,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
401
435
  index: int
402
436
  run: FileRun
403
437
  for index, run in zip(again, redone, strict=True):
404
- runs[index] = _merged(runs[index], run)
438
+ runs[index] = merged(runs[index], run)
405
439
  again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
406
440
  checker: str
407
441
  path: Path
@@ -447,7 +481,7 @@ def _called_again(
447
481
  calling = [
448
482
  at
449
483
  for at in calling
450
- if project.returned(modules, paths[at], {}) != project.returned(before, paths[at], {})
484
+ if loose.returned(modules, paths[at], {}) != loose.returned(before, paths[at], {})
451
485
  ]
452
486
  return runs, _checked_more(paths, runs, calling, (check, options, session), modules)
453
487
 
@@ -473,7 +507,7 @@ def _checked_more(
473
507
  at: int
474
508
  run: FileRun
475
509
  for at, run in zip(which, redone, strict=True):
476
- runs[at] = _merged(runs[at], run) if how[1].mode is Mode.FIX else run
510
+ runs[at] = merged(runs[at], run) if how[1].mode is Mode.FIX else run
477
511
  return modules
478
512
 
479
513
 
@@ -509,44 +543,29 @@ def _checked_all(
509
543
  What each file found, and the index of every file's module.
510
544
 
511
545
  """
512
- hinted: dict[Path, tuple[Hints, ...]] = {} if session is None else session.hints(_texts(paths))
546
+ # Before a worker starts: one forked while the servers' threads run could wait on a lock one holds.
547
+ hinted: _Hinted = {} if session is None else session.hints(_texts(paths))
513
548
  if options.input.trace is not None:
514
549
  hinted = traced.merged(hinted, traced.load(options.input.trace), paths)
515
550
  coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
516
- stack: contextlib.ExitStack
517
- with contextlib.ExitStack() as stack:
518
- pool: Workers | None = None
519
- if options.jobs != 1 and len(paths) > 1:
520
- pool = stack.enter_context(Workers(paths, options.jobs))
521
- if modules is None and pool:
522
- modules = project.Index({}, []) if coverage else pool.index()
523
- elif modules is None:
524
- modules = project.Index({}, []) if coverage else project.index(paths)
525
- collecting.indexed()
526
- if not coverage:
527
- modules = plain.settled(
528
- decorated.passed(installed.with_installed(modules, installed.search_path())),
529
- options.checks.plain_bases,
530
- )
531
- return schedule.checked(
532
- paths,
533
- check,
534
- pool,
535
- (modules, hinted),
536
- _merged if options.mode is Mode.FIX else None,
551
+ pool: Workers | None = started(paths, options.jobs) if options.jobs != 1 and len(paths) > 1 else None
552
+ if modules is None and pool:
553
+ modules = project.Index({}, []) if coverage else pool.index()
554
+ elif modules is None:
555
+ modules = project.Index({}, []) if coverage else project.index(paths)
556
+ collecting.indexed()
557
+ if not coverage:
558
+ modules = plain.settled(
559
+ decorated.passed(installed.with_installed(modules, installed.search_path())),
560
+ options.checks.plain_bases,
537
561
  )
538
-
539
-
540
- def _merged(before: FileRun, after: FileRun) -> FileRun:
541
- """Join a file's two `--fix` rounds' `CheckRun`s: what's left is the later's, what's fixed is both's.
542
-
543
- Returns:
544
- The joined run.
545
-
546
- """
547
- first: CheckRun = cast("CheckRun", before)
548
- later: CheckRun = cast("CheckRun", after)
549
- return replace(later, fixed=first.fixed + later.fixed)
562
+ return schedule.checked(
563
+ paths,
564
+ check,
565
+ pool,
566
+ (modules, hinted),
567
+ merged if options.mode is Mode.FIX else None,
568
+ )
550
569
 
551
570
 
552
571
  def _texts(paths: Sequence[Path]) -> dict[Path, str]:
@@ -579,10 +598,16 @@ def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
579
598
  results: list[Result] = [result for run in checked for result in run.results]
580
599
  output: Output = options.output
581
600
  text: bool = output.fmt in {Format.TEXT, Format.FULL}
601
+ # With `--show-fixes`, the fixes `--fix` made too: after the rest, as text or JSON.
602
+ shown: list[Result] = [*results, *(made for run in checked if output.show_fixes for made in run.made)]
582
603
  line: str
583
- for line in statistics(results) if text and output.statistics else render(output.fmt, results):
604
+ for line in (
605
+ statistics(results)
606
+ if text and output.statistics
607
+ else render(output.fmt, shown if output.fmt is Format.JSON else results)
608
+ ):
584
609
  _ = sys.stdout.write(f"{line}\n")
585
- for line in fix_reasons(results) if text and output.show_fixes else ():
610
+ for line in fix_reasons(shown) if text and output.show_fixes else ():
586
611
  _ = sys.stdout.write(f"{line}\n")
587
612
  errors: int = sum(r.offence.is_error(r.level) for r in results)
588
613
  if text and not output.quiet:
@@ -611,30 +636,53 @@ def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun])
611
636
  counted: list[tuple[Path, Coverage]] = [
612
637
  (path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
613
638
  ]
614
- total: Coverage = Coverage(sum(c.typed for _, c in counted), sum(c.total for _, c in counted))
639
+ total: Coverage = Coverage(
640
+ sum(c.typed for _, c in counted),
641
+ sum(c.total for _, c in counted),
642
+ sum(c.widened for _, c in counted),
643
+ )
615
644
  if options.output.fmt is Format.JSON:
616
645
  report: dict[str, _Scalar | list[_Row]] = {
617
- "typed": total.typed,
618
- "total": total.total,
619
- "percent": round(total.percent, 1),
620
- "files": [
621
- {"path": str(path), "typed": c.typed, "total": c.total, "percent": round(c.percent, 1)}
622
- for path, c in counted
623
- ],
646
+ **_counts(total),
647
+ "files": [{"path": str(path), **_counts(c)} for path, c in counted],
624
648
  }
625
649
  _ = sys.stdout.write(json.dumps(report, indent=2) + "\n")
626
650
  else:
627
651
  path: Path
628
652
  c: Coverage
629
653
  for path, c in counted:
630
- _ = sys.stdout.write(f"{path}: {c.typed}/{c.total} typed ({c.percent:.1f}%)\n")
631
- _ = sys.stdout.write(
632
- f"Total: {total.typed}/{total.total} typed ({total.percent:.1f}%) in {len(counted)} file(s).\n",
633
- )
654
+ _ = sys.stdout.write(f"{path}: {_typed(c)}\n")
655
+ _ = sys.stdout.write(f"Total: {_typed(total)} in {len(counted)} file(s).\n")
634
656
  threshold: float | None = options.output.fail_under
635
657
  return EXIT_FOUND if threshold is not None and total.percent < threshold else EXIT_CLEAN
636
658
 
637
659
 
660
+ def _counts(found: Coverage) -> _Row:
661
+ """Name a coverage's counts, for `--format=json`.
662
+
663
+ Returns:
664
+ Them, by name.
665
+
666
+ """
667
+ return {
668
+ "typed": found.typed,
669
+ "total": found.total,
670
+ "widened": found.widened,
671
+ "percent": round(found.percent, 1),
672
+ }
673
+
674
+
675
+ def _typed(found: Coverage) -> str:
676
+ """Write a coverage as text: `3/4 typed (75.0%)`, then what a widening annotated, if anything.
677
+
678
+ Returns:
679
+ It.
680
+
681
+ """
682
+ wider: str = f", {found.widened} widened" if found.widened else ""
683
+ return f"{found.typed}/{found.total} typed ({found.percent:.1f}%){wider}"
684
+
685
+
638
686
  def _run(options: Options) -> int:
639
687
  """Check, fix, diff, count or baseline, printing what that finds.
640
688
 
constricter/cli/config.py CHANGED
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias, cast
10
10
 
11
11
  from constricter.cli.protocol import SERVERS
12
12
  from constricter.jsonc import is_int
13
- from constricter.offences import FIX_KINDS, LEVELS, MESSAGES
13
+ from constricter.offences import FIX_KINDS, LEVELS, MESSAGES, WIDEN_KINDS
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from datetime import date, datetime, time
@@ -32,6 +32,7 @@ def unknown_codes(codes: Sequence[str]) -> list[str]:
32
32
 
33
33
  DEFAULT_BASELINE: Final = "constricter-baseline.json"
34
34
  _MIN_PYTHON: Final = "min_python"
35
+ _ALL_KINDS: Final = "all" # `fix-widen`'s: every wider type
35
36
  _VERSION: Final = re.compile(r"\d+\.\d+") # `min-python`'s: `3.11`
36
37
  # A lower bound among `requires-python`'s specifiers: `>=3.11`, `~=3.11`, `==3.11.*`, `>3.10`.
37
38
  _LOWER_BOUND: Final = re.compile(r"(?:>=|~=|==|>)\s*(\d+)(?:\.(\d+))?")
@@ -160,6 +161,31 @@ def unknown_fix_kinds(kinds: Sequence[str]) -> list[str]:
160
161
  return [kind for kind in kinds if kind not in FIX_KINDS]
161
162
 
162
163
 
164
+ def unknown_widen_kinds(kinds: Sequence[str]) -> list[str]:
165
+ """Check `kinds` against `WIDEN_KINDS`' ids, and `all`.
166
+
167
+ Returns:
168
+ Those that aren't one.
169
+
170
+ """
171
+ return [kind for kind in kinds if kind not in WIDEN_KINDS | {_ALL_KINDS}]
172
+
173
+
174
+ def widen_kinds(kinds: Sequence[str]) -> list[str]:
175
+ """Read `fix-widen`'s kinds, `all` as every one.
176
+
177
+ Returns:
178
+ Them.
179
+
180
+ """
181
+ return sorted(WIDEN_KINDS) if _ALL_KINDS in kinds else list(kinds)
182
+
183
+
184
+ def _widen_kinds(value: _Toml) -> list[str] | None:
185
+ kinds: list[str] | None = [value] if isinstance(value, str) else _strings(value)
186
+ return None if kinds is None or unknown_widen_kinds(kinds) else widen_kinds(kinds)
187
+
188
+
163
189
  def _fix_kinds(value: _Toml) -> list[str] | None:
164
190
  kinds: list[str] | None = _strings(value)
165
191
  return None if kinds is None or unknown_fix_kinds(kinds) else kinds
@@ -246,6 +272,7 @@ _READERS: dict[str, Callable[[_Toml], Default | None]] = {
246
272
  "fix-select": _fix_kinds,
247
273
  "fix-ignore": _fix_kinds,
248
274
  "unsafe-fix-select": _fix_kinds,
275
+ "fix-widen": _widen_kinds,
249
276
  "fix-plain-bases": _strings,
250
277
  "type-comments": _flag,
251
278
  "all-scopes": _flag,
constricter/cli/hints.py CHANGED
@@ -33,7 +33,7 @@ from functools import partial
33
33
  from pathlib import Path
34
34
  from typing import IO, Final, NamedTuple, Self, TypeAlias, cast
35
35
 
36
- from constricter.cli import edits, guard, protocol
36
+ from constricter.cli import edits, guard, protocol, remembered
37
37
  from constricter.cli.protocol import SERVERS, HintError, HungError, Server
38
38
  from constricter.fix.core.known import Hints, Offered
39
39
 
@@ -47,8 +47,6 @@ _Task: TypeAlias = Callable[[], _Found] # one server's work: the hints of each
47
47
 
48
48
  _BATCH: Final = 8 # files a free server takes at a time: few, so none is left with the slow ones
49
49
  MEMORY: Final = 8 << 30 # the most a checker's servers use together, by default (`--infer-memory`)
50
- _SERVER_MEMORY: Final = 600 << 20 # a server's memory, before the files (basedpyright's, measured)
51
- _MEMORY_PER_BYTE: Final = 130 # and for each byte of them: 1.2 GB for SQLAlchemy's 8 MB, 3.4 for pandas' 21
52
50
  _MEMINFO: Final = Path("/proc/meminfo")
53
51
  _FILES_PER_SERVER: Final = 32 # fewer files than this each don't pay for another server's start
54
52
  # Seconds without a word from a server before it's taken for hung. Not an answer's wait: pandas'
@@ -228,11 +226,10 @@ class Checker:
228
226
  """One checker's servers, and which of them has each file.
229
227
 
230
228
  It has as many as its files need (one per `_FILES_PER_SERVER`), up to the command's `--jobs`, the
231
- checker's own most (`Server.most`: one for a checker that works in parallel itself), and as many
232
- as fit in its memory: `--infer-memory` (never more than is available), or by default `MEMORY`
233
- (never more than half what's available). Each server loads the whole program it checks, which
234
- takes (as measured) about `_SERVER_MEMORY` plus `_MEMORY_PER_BYTE` for each byte of the files.
235
- One server it always has, whatever the memory.
229
+ checker's own most (`Server.most`), and as many as fit in its memory: `--infer-memory` (never
230
+ more than is available), or by default `MEMORY` (never more than half what's available). Each
231
+ server loads the whole program it checks, or its share of it, by its checker's own measure (see
232
+ `Server.fitting`). One server it always has, whatever the memory.
236
233
 
237
234
  The files are handed out a few at a time (`_BATCH`), the biggest first, to whichever server is
238
235
  free: one file's analysis can take far longer than another's, and none waits on a busy one. A
@@ -244,7 +241,8 @@ class Checker:
244
241
  self.name: str = name
245
242
  self.root: Path = root
246
243
  self.command: list[str] = command(name)
247
- self.most: int = max(1, min(servers, SERVERS[name].most))
244
+ self.server: Server = SERVERS[name]
245
+ self.most: int = max(1, min(servers, self.server.most))
248
246
  self.memory: int = budget(memory, available_memory())
249
247
  self.servers: list[Connection] = []
250
248
  self.assigned: dict[Path, int] = {} # each file's server, by its place in `servers`
@@ -258,13 +256,14 @@ class Checker:
258
256
 
259
257
  """
260
258
  size: int = sum(len(text) for text in files.values())
261
- fit: int = max(1, self.memory // (_SERVER_MEMORY + _MEMORY_PER_BYTE * size))
259
+ fit: int = max(1, self.server.fitting(self.memory, size))
262
260
  return max(len(self.servers), min(self.most, -(-len(files) // _FILES_PER_SERVER), fit))
263
261
 
264
262
  def tasks(self, files: Mapping[Path, str]) -> list[_Task]:
265
263
  """Make each server's work on `files`: its own files first, then new ones as it's free for them.
266
264
 
267
- A file abandoned (see `_alone`) isn't asked about again.
265
+ A file abandoned (see `_alone`), in this run or an earlier one (see `remembered`), isn't
266
+ asked about again.
268
267
 
269
268
  Returns:
270
269
  A task per server, giving the hints of every file it was sent.
@@ -274,8 +273,14 @@ class Checker:
274
273
  path: Path
275
274
  for path in files.keys() & self.assigned.keys() - self.abandoned:
276
275
  own.setdefault(self.assigned[path], {})[path] = files[path]
276
+ # One it hung on in an earlier run, unchanged since: left without hints from the start.
277
+ self.abandoned.update(
278
+ path
279
+ for path in files.keys() - self.assigned.keys() - self.abandoned
280
+ if remembered.hung(self.name, self.command, files[path])
281
+ )
277
282
  fresh: list[Path] = sorted(
278
- files.keys() - self.assigned.keys(),
283
+ files.keys() - self.assigned.keys() - self.abandoned,
279
284
  key=lambda new: (-len(files[new]), new),
280
285
  )
281
286
  batches: queue.SimpleQueue[dict[Path, str]] = queue.SimpleQueue()
@@ -352,6 +357,7 @@ class Checker:
352
357
  found.update(server.answers(server.ask({path: text})))
353
358
  except HungError as hung:
354
359
  self.abandoned.add(path)
360
+ remembered.remember(self.name, self.command, text)
355
361
  if len(self.abandoned) > _MOST_ABANDONED:
356
362
  error: str = f"{self.name} hung on {len(self.abandoned)} file(s), each asked about alone"
357
363
  raise HintError(error) from hung
@@ -18,6 +18,8 @@ from constricter.cli.config import (
18
18
  project_root,
19
19
  unknown_codes,
20
20
  unknown_fix_kinds,
21
+ unknown_widen_kinds,
22
+ widen_kinds,
21
23
  )
22
24
  from constricter.cli.explain import explain
23
25
  from constricter.cli.paths import STDIN, excluded
@@ -133,6 +135,24 @@ def _fix_kinds(text: str) -> list[str]:
133
135
  return kinds
134
136
 
135
137
 
138
+ def _widen_kinds(text: str) -> list[str]:
139
+ """Read a comma-separated list of the wider types `--fix` may write (`untyped-parameters`), or `all`.
140
+
141
+ Returns:
142
+ Them.
143
+
144
+ Raises:
145
+ argparse.ArgumentTypeError: One isn't a widening's id.
146
+
147
+ """
148
+ kinds: list[str] = [kind.strip() for kind in text.split(",") if kind.strip()]
149
+ unknown: list[str]
150
+ if unknown := unknown_widen_kinds(kinds):
151
+ message: str = f"no wider type is called {', '.join(unknown)} (see docs/FIXES.md)"
152
+ raise argparse.ArgumentTypeError(message)
153
+ return widen_kinds(kinds)
154
+
155
+
136
156
  def _version(text: str) -> tuple[int, int]:
137
157
  """Read `--min-python`: a Python version (`3.11`).
138
158
 
@@ -333,6 +353,13 @@ def _parser() -> argparse.ArgumentParser:
333
353
  metavar="KINDS",
334
354
  help="treat guesses from these mechanisms (constructor, narrow) as certain",
335
355
  )
356
+ _ = parser.add_argument(
357
+ "--fix-widen",
358
+ type=_widen_kinds,
359
+ default=[],
360
+ metavar="KINDS",
361
+ help="also write these wider types, each marked (untyped-parameters, unions, ...; all: every one)",
362
+ )
336
363
  _ = parser.add_argument(
337
364
  "--fix-plain-bases",
338
365
  type=_listed,
@@ -367,7 +394,7 @@ def _parser() -> argparse.ArgumentParser:
367
394
  _ = parser.add_argument(
368
395
  "--show-fixes",
369
396
  action="store_true",
370
- help="after the report, list each fix and how the value decided it (text)",
397
+ help="after the report, list each fix and how the value decided it; with --fix, those made too",
371
398
  )
372
399
  _ = parser.add_argument(
373
400
  "--statistics",
@@ -594,6 +621,7 @@ class Options:
594
621
  frozenset(cast("list[str]", args.fix_select)),
595
622
  frozenset(cast("list[str]", args.fix_ignore)),
596
623
  frozenset(cast("list[str]", args.unsafe_fix_select)),
624
+ frozenset(cast("list[str]", args.fix_widen)),
597
625
  ),
598
626
  min_python=cast("tuple[int, int] | None", args.min_python),
599
627
  plain_bases=(*PLAIN_BASES, *cast("list[str]", args.fix_plain_bases)),
@@ -33,21 +33,41 @@ class Server(NamedTuple):
33
33
  """A checker's language server: how it's started, and how many are worth running at once.
34
34
 
35
35
  Its executable, the arguments that start it over stdio, and the most of it to run (up to
36
- `MAX_SERVERS`; one for a checker that works in parallel itself).
36
+ `MAX_SERVERS`). And the memory its servers take, as measured: `memory` bytes each before the
37
+ files, and `per_byte` more for each byte of them: each server's again where each holds the
38
+ `whole` program, once among them all where each holds what it's asked about.
37
39
  """
38
40
 
39
41
  executable: str
40
42
  args: tuple[str, ...]
41
43
  most: int
44
+ memory: int
45
+ per_byte: int
46
+ whole: bool = False
47
+
48
+ def fitting(self, budget: int, size: int) -> int:
49
+ """Count the servers that fit in `budget` bytes, for files of `size` bytes together.
50
+
51
+ Returns:
52
+ It: none where one doesn't fit.
53
+
54
+ """
55
+ if self.whole:
56
+ return budget // (self.memory + self.per_byte * size)
57
+ return max(0, budget - self.per_byte * size) // self.memory
42
58
 
43
59
 
44
60
  MAX_SERVERS: Final = 4 # a checker's, at most: each holds its own copy of the program it checks
45
61
  SERVERS: Final = {
46
62
  # One thread each: more servers check more at once (sqlalchemy's hints: 10.8s with one, 7.2s with four).
47
- "basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS),
48
- # Parallel already: more servers only repeat its work (sqlalchemy's: 0.9s with one, 0.7s with four).
49
- "ty": Server("ty", ("server",), 1),
50
- "pyrefly": Server("pyrefly", ("lsp",), 1), # parallel too
63
+ # Each holds the whole program: 1.2 GB for sqlalchemy's 8 MB, 3.4 for pandas's 21.
64
+ "basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS, 600 << 20, 130, whole=True),
65
+ # Parallel itself, and still quicker shared out: a hinted check of pandas takes 36s with one, 25s
66
+ # with two, 22s with three and 21s with four (1.2 GB with one, 1.7 with two, 2.5 with four).
67
+ "ty": Server("ty", ("server",), MAX_SERVERS, 420 << 20, 38),
68
+ # Parallel too: pandas's takes 132s with one, 58s with two and 32s with four (2.8 GB, 3.2, 4.2);
69
+ # the standard library's 39 MB, 4.3 GB with one and 5.6 with four.
70
+ "pyrefly": Server("pyrefly", ("lsp",), MAX_SERVERS, 480 << 20, 120),
51
71
  }
52
72
  _PROBE: Final = "--version" # quick for each; basedpyright-langserver's exits 1 all the same
53
73
  _PROBE_TIMEOUT: Final = 10.0 # seconds it may take
@@ -0,0 +1,52 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """The files a type checker's server hung on, remembered between runs.
3
+
4
+ A server that hangs on a file costs the run its timeout twice, and a restart each time: once with
5
+ the file among others, once with it alone. The file's text, with the checker and its executable as
6
+ it was (its size and when it changed), names an empty file in the cache; while there's one, the
7
+ file is left without hints from the start. A new version of the checker, or a changed file, is
8
+ asked again.
9
+ """
10
+
11
+ import contextlib
12
+ import hashlib
13
+ from collections.abc import Sequence
14
+ from pathlib import Path
15
+
16
+ from constricter.fix.index.installed import cache_directory
17
+
18
+
19
+ def _marker(checker: str, command: Sequence[str], text: str) -> Path:
20
+ """Name what remembers that `checker`, started by `command`, hung on a file holding `text`.
21
+
22
+ Returns:
23
+ Its path in the cache.
24
+
25
+ """
26
+ version: str = ""
27
+ with contextlib.suppress(OSError):
28
+ version = _version(Path(command[0]))
29
+ key: str = "\0".join((checker, command[0], version, text))
30
+ return cache_directory().with_name("hung") / hashlib.sha256(key.encode("utf-8", "replace")).hexdigest()
31
+
32
+
33
+ def _version(executable: Path) -> str: # which build of it: its size and when it changed
34
+ return f"{executable.stat().st_size}:{executable.stat().st_mtime_ns}"
35
+
36
+
37
+ def hung(checker: str, command: Sequence[str], text: str) -> bool:
38
+ """Check whether `checker` hung on a file holding `text`, in an earlier run.
39
+
40
+ Returns:
41
+ Whether it did.
42
+
43
+ """
44
+ return _marker(checker, command, text).exists()
45
+
46
+
47
+ def remember(checker: str, command: Sequence[str], text: str) -> None:
48
+ """Remember that `checker` hung on a file holding `text`; nothing, where the cache can't be written."""
49
+ marker: Path = _marker(checker, command, text)
50
+ with contextlib.suppress(OSError):
51
+ marker.parent.mkdir(parents=True, exist_ok=True)
52
+ marker.touch()
constricter/cli/report.py CHANGED
@@ -40,6 +40,7 @@ class Result(NamedTuple):
40
40
  level: Level
41
41
  source: str = "" # the offending line, for `--format=full`
42
42
  replacements: tuple[Replacement, ...] = () # its fix as text edits, for SARIF and rdjson
43
+ fixed: bool = False # `--fix` made its fix: listed with `--show-fixes`, on the line it was on
43
44
 
44
45
  @property
45
46
  def suggestions(self) -> tuple[Replacement, ...]:
@@ -208,6 +209,7 @@ def _json(results: Sequence[Result]) -> Iterator[str]:
208
209
  "severity": r.severity,
209
210
  "message": r.offence.message,
210
211
  "cell": r.offence.cell,
212
+ "fixed": r.fixed,
211
213
  "fix": None
212
214
  if r.offence.fix is None
213
215
  else {
@@ -360,11 +362,11 @@ def render(fmt: Format, results: Sequence[Result]) -> Iterator[str]:
360
362
 
361
363
 
362
364
  def fix_reasons(results: Sequence[Result]) -> Iterator[str]:
363
- """List the annotation each fixable result would get, and how its value decided it.
365
+ """List the annotation each fixable result would get, or got, and how its value decided it.
364
366
 
365
367
  Yields:
366
- A line each: where, the name, the annotation and its reason; a guess says it needs
367
- `--unsafe-fixes`.
368
+ A line each: where, the name (`fixed`, for a fix `--fix` made), the annotation and its reason;
369
+ a guess left unmade says it needs `--unsafe-fixes`.
368
370
 
369
371
  """
370
372
  r: Result
@@ -372,11 +374,11 @@ def fix_reasons(results: Sequence[Result]) -> Iterator[str]:
372
374
  if r.offence.fix is not None:
373
375
  cell: str = "" if r.offence.cell is None else f"cell {r.offence.cell}:"
374
376
  where: str = f"{r.path}:{cell}{r.offence.line}:{r.offence.col + 1}"
375
- guess: str = " (a guess: --unsafe-fixes)" if r.offence.unsafe else ""
377
+ guess: str = " (a guess: --unsafe-fixes)" if r.offence.unsafe and not r.fixed else ""
376
378
  kinds: str = ", ".join(sorted(r.offence.edit.kinds if r.offence.edit else ()))
377
379
  written: str = f"`{r.offence.fix}`" if r.offence.fix else "drop its annotation"
378
380
  decided: str = f"{written}, from {r.offence.reason} [{kinds}]"
379
- yield f"{where}: fix {r.offence.name!r}: {decided}{guess}"
381
+ yield f"{where}: {'fixed' if r.fixed else 'fix'} {r.offence.name!r}: {decided}{guess}"
380
382
 
381
383
 
382
384
  def statistics(results: Sequence[Result]) -> Iterator[str]: