python-constricter 0.3.2__py3-none-any.whl → 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. constricter/__init__.py +1 -1
  2. constricter/__main__.py +19 -2
  3. constricter/cli/command.py +111 -56
  4. constricter/cli/config.py +30 -2
  5. constricter/cli/hints.py +18 -12
  6. constricter/cli/options.py +57 -3
  7. constricter/cli/protocol.py +25 -5
  8. constricter/cli/remembered.py +52 -0
  9. constricter/cli/report.py +7 -5
  10. constricter/cli/runs.py +49 -2
  11. constricter/cli/schedule.py +241 -68
  12. constricter/cli/traced.py +401 -0
  13. constricter/cli/workers.py +85 -23
  14. constricter/fix/core/asked.py +56 -0
  15. constricter/fix/core/fixes.py +63 -9
  16. constricter/fix/core/imports.py +196 -10
  17. constricter/fix/core/inherited.py +128 -15
  18. constricter/fix/core/known.py +271 -55
  19. constricter/fix/core/signatures.py +29 -6
  20. constricter/fix/index/atoms.py +22 -4
  21. constricter/fix/index/awaits.py +90 -0
  22. constricter/fix/index/beyond.py +148 -0
  23. constricter/fix/index/callers.py +3 -2
  24. constricter/fix/index/chained.py +69 -0
  25. constricter/fix/index/classnames.py +17 -0
  26. constricter/fix/index/declared.py +83 -3
  27. constricter/fix/index/decorated.py +4 -2
  28. constricter/fix/index/fixtures.py +176 -0
  29. constricter/fix/index/installed.py +68 -15
  30. constricter/fix/index/linked.py +142 -0
  31. constricter/fix/index/loose.py +98 -0
  32. constricter/fix/index/managed.py +63 -0
  33. constricter/fix/index/modules.py +368 -46
  34. constricter/fix/index/offers.py +4 -1
  35. constricter/fix/index/order.py +13 -3
  36. constricter/fix/index/own_methods.py +67 -0
  37. constricter/fix/index/own_overloads.py +70 -0
  38. constricter/fix/index/own_types.py +161 -0
  39. constricter/fix/index/plain.py +13 -8
  40. constricter/fix/index/project.py +222 -180
  41. constricter/fix/index/read.py +70 -0
  42. constricter/fix/index/sides.py +102 -7
  43. constricter/fix/index/strings.py +46 -0
  44. constricter/fix/index/stubbed.py +86 -80
  45. constricter/fix/index/taken.py +45 -0
  46. constricter/fix/index/tuples.py +54 -28
  47. constricter/fix/libraries/library.py +208 -6
  48. constricter/fix/libraries/opened.py +88 -15
  49. constricter/fix/libraries/overloads.py +155 -14
  50. constricter/fix/libraries/stdlib.py +278 -24
  51. constricter/fix/libraries/walks.py +49 -0
  52. constricter/fix/tables/attributes.json +25 -0
  53. constricter/fix/tables/awaited.json +69 -0
  54. constricter/fix/tables/awaited_overloads.json +14 -0
  55. constricter/fix/tables/classes.json +3 -1
  56. constricter/fix/tables/elements.json +156 -0
  57. constricter/fix/tables/functions.json +0 -1
  58. constricter/fix/tables/generic_attributes.json +201 -2
  59. constricter/fix/tables/method_overloads.json +146 -99
  60. constricter/fix/tables/method_signatures.json +229 -40
  61. constricter/fix/tables/methods.json +35 -10
  62. constricter/fix/tables/overloads.json +88 -63
  63. constricter/fix/tables/scalars.json +5 -0
  64. constricter/fix/tables/source.json +4 -1
  65. constricter/fix/tables/subscriptable.json +1 -0
  66. constricter/fix/tables/type_parameters.json +3 -0
  67. constricter/fix/tables/variables.json +21 -0
  68. constricter/fix/values/aliased.py +9 -1
  69. constricter/fix/values/bodies.py +76 -0
  70. constricter/fix/values/callables.py +321 -0
  71. constricter/fix/values/called.py +10 -12
  72. constricter/fix/values/classvars.py +216 -40
  73. constricter/fix/values/decided.py +74 -11
  74. constricter/fix/values/displays.py +87 -0
  75. constricter/fix/values/doubts.py +152 -28
  76. constricter/fix/values/entered.py +209 -30
  77. constricter/fix/values/fills.py +208 -30
  78. constricter/fix/values/guesses.py +80 -21
  79. constricter/fix/values/hinted.py +94 -7
  80. constricter/fix/values/inference.py +145 -146
  81. constricter/fix/values/members.py +326 -39
  82. constricter/fix/values/narrowed.py +60 -4
  83. constricter/fix/values/operated.py +200 -0
  84. constricter/fix/values/ordered.py +74 -0
  85. constricter/fix/values/returned.py +487 -226
  86. constricter/fix/values/returns.py +45 -1
  87. constricter/fix/values/shapes.py +267 -52
  88. constricter/fix/values/targets.py +51 -18
  89. constricter/fix/values/unchecked.py +162 -0
  90. constricter/offences.py +48 -6
  91. constricter/recording.py +571 -0
  92. constricter/rules/annotations.py +52 -13
  93. constricter/rules/binding.py +237 -48
  94. constricter/rules/calls.py +222 -26
  95. constricter/rules/checker.py +301 -77
  96. constricter/rules/decorators.py +12 -0
  97. constricter/rules/keys.py +79 -0
  98. constricter/rules/late.py +351 -12
  99. constricter/rules/parsed.py +42 -3
  100. constricter/rules/quoted.py +32 -0
  101. constricter/rules/rebinding.py +38 -11
  102. constricter/rules/recorded.py +53 -10
  103. constricter/rules/scope.py +291 -88
  104. constricter/rules/syntax.py +20 -0
  105. constricter/rules/tables.py +79 -9
  106. constricter/rules/widened.py +322 -0
  107. constricter/trace.py +378 -0
  108. {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/METADATA +42 -38
  109. python_constricter-0.3.5.dist-info/RECORD +140 -0
  110. {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/entry_points.txt +1 -1
  111. python_constricter-0.3.2.dist-info/RECORD +0 -110
  112. {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/WHEEL +0 -0
  113. {python_constricter-0.3.2.dist-info → python_constricter-0.3.5.dist-info}/licenses/LICENSE.md +0 -0
constricter/__init__.py CHANGED
@@ -25,7 +25,7 @@ from constricter.offences import (
25
25
  )
26
26
  from constricter.rules.checker import Coverage, annotation_coverage, check_source, check_tree
27
27
 
28
- __version__ = "0.3.2"
28
+ __version__ = "0.3.5"
29
29
  __all__ = [
30
30
  "COMMENT_TYPED_TARGET",
31
31
  "DEFAULT_CHECKS",
constricter/__main__.py CHANGED
@@ -1,9 +1,26 @@
1
1
  # SPDX-License-Identifier: MIT
2
- """`python -m constricter`."""
2
+ """`python -m constricter`, and the `constricter` script."""
3
3
 
4
+ import contextlib
5
+ import os
4
6
  import sys
5
7
 
6
8
  from constricter.cli.command import main
7
9
 
10
+
11
+ def run() -> None:
12
+ """Run the command, and leave with its status at once.
13
+
14
+ Not by the interpreter's own exit, which frees every parsed file and the index an object at a
15
+ time: a fifth of a check of a thousand files. What the command wrote is flushed first; nothing
16
+ else waits on the exit (its workers have ended, its output file is closed).
17
+ """
18
+ status: int = main()
19
+ with contextlib.suppress(OSError, ValueError): # a reader that left, a stream closed already
20
+ _ = sys.stdout.flush()
21
+ _ = sys.stderr.flush()
22
+ os._exit(status)
23
+
24
+
8
25
  if __name__ == "__main__": # not when `--jobs` workers import it
9
- sys.exit(main())
26
+ run()
@@ -16,16 +16,16 @@ from pathlib import Path
16
16
  from typing import TYPE_CHECKING, Final, NamedTuple, TextIO, TypeAlias, cast
17
17
 
18
18
  from constricter import notebook
19
- from constricter.cli import baseline, collecting, schedule
19
+ from constricter.cli import baseline, collecting, schedule, traced
20
20
  from constricter.cli.options import Mode, Options, Output
21
21
  from constricter.cli.paths import STDIN, python_files
22
22
  from constricter.cli.protocol import HintError
23
23
  from constricter.cli.report import Format, Result, fix_reasons, render, statistics
24
- from constricter.cli.runs import BaselineRun, CheckRun, CoverageRun, FileRun
25
- from constricter.cli.workers import Workers
24
+ from constricter.cli.runs import Added, BaselineRun, CheckRun, CoverageRun, FileRun, merged
25
+ from constricter.cli.workers import Workers, kept, started
26
26
  from constricter.fix.core import fixes
27
27
  from constricter.fix.core.known import Callee, Hints, Outside, Returns
28
- from constricter.fix.index import callers, decorated, installed, plain, project
28
+ from constricter.fix.index import callers, decorated, installed, loose, plain, project
29
29
  from constricter.noqa import lines, unsuppressed
30
30
  from constricter.offences import (
31
31
  DEFAULT_CHECKS,
@@ -33,6 +33,7 @@ from constricter.offences import (
33
33
  Edit,
34
34
  Offence,
35
35
  )
36
+ from constricter.rules import parsed
36
37
  from constricter.rules.checker import Checked, Coverage, annotation_coverage, checked_source
37
38
 
38
39
  if TYPE_CHECKING: # slow to import, and only needed for `--infer-with`
@@ -126,6 +127,7 @@ def _placed(offence: Offence, where: list[notebook.Line]) -> Offence:
126
127
 
127
128
 
128
129
  # One changed part of a fixed file (the file, or a notebook's cell): its label, old and new lines.
130
+ _Hinted: TypeAlias = dict[Path, tuple[Hints, ...]] # each file's hints, a checker's each
129
131
  _Change: TypeAlias = tuple[str, list[str], list[str]]
130
132
  # One JSON value a coverage report holds, and one file's row of them.
131
133
  _Scalar: TypeAlias = str | int | float
@@ -297,13 +299,39 @@ def _handled(path: Path, text: tuple[str, Path], offences: list[Offence], option
297
299
  left: list[Result] = [r for r in results if r.offence not in fixing]
298
300
  if path == STDIN: # the fixed source goes to stdout
299
301
  return CheckRun(left, baselined, len(fixing), _fixed(raw, name, fixing).text)
302
+ fixed: int
300
303
  try:
301
- return CheckRun(left, baselined, fix_file(path, fixing))
304
+ fixed = fix_file(path, fixing)
302
305
  except UnicodeEncodeError as failure: # the file is left as it was, its offences unfixed
303
306
  # Its canonical name: PyPy reports `latin1` where CPython says `latin-1`.
304
307
  encoding: str = codecs.lookup(failure.encoding).name
305
308
  message: str = f"an annotation can't be written in its encoding, {encoding}; left as it was"
306
309
  return CheckRun(results, baselined, error=f"{name}: error: {message}")
310
+ if fixed:
311
+ _ = parsed.take(raw) # the tree of the text it had: no check will ask for it again
312
+ return CheckRun(
313
+ left,
314
+ baselined,
315
+ fixed,
316
+ made=[r._replace(replacements=(), fixed=True) for r in results if r.offence in fixing],
317
+ rounds=(_added(raw, name, fixing),),
318
+ )
319
+
320
+
321
+ def _added(raw: str, name: Path, fixing: Sequence[Offence]) -> Added:
322
+ """Find the lines fixing `fixing` adds to `raw`, the text of `name` (see `fixes.inserted`).
323
+
324
+ Returns:
325
+ Each one's cell (`None` outside a notebook) and the line it goes before; a notebook's cells
326
+ get declarations alone.
327
+
328
+ """
329
+ if name.suffix != notebook.SUFFIX:
330
+ return tuple((None, line) for line in fixes.inserted(lines(raw), fixing))
331
+ declared: list[tuple[int, int]] = [
332
+ (o.cell or 0, o.edit.span[0]) for o in fixing if o.edit is not None and o.edit.edit is Edit.DECLARE
333
+ ]
334
+ return tuple(sorted(declared))
307
335
 
308
336
 
309
337
  def _baseline_path(path: Path, outside: Outside, options: Options) -> BaselineRun:
@@ -335,6 +363,17 @@ def _cover_path(path: Path, _outside: Outside, options: Options) -> CoverageRun:
335
363
 
336
364
 
337
365
  def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
366
+ """Check every file (see `_check_rounds`), its rounds by one set of worker processes.
367
+
368
+ Returns:
369
+ The names, and what each file found.
370
+
371
+ """
372
+ with kept():
373
+ return _check_rounds(options)
374
+
375
+
376
+ def _check_rounds(options: Options) -> tuple[list[Path], list[FileRun]]:
338
377
  """Check every file (`--jobs` at a time), in order.
339
378
 
340
379
  With `--infer-with`, the type checker's server runs throughout: it's asked for every file's
@@ -396,7 +435,7 @@ def _check_all(options: Options) -> tuple[list[Path], list[FileRun]]:
396
435
  index: int
397
436
  run: FileRun
398
437
  for index, run in zip(again, redone, strict=True):
399
- runs[index] = _merged(runs[index], run)
438
+ runs[index] = merged(runs[index], run)
400
439
  again = [index for index, run in zip(again, redone, strict=True) if cast("CheckRun", run).fixed]
401
440
  checker: str
402
441
  path: Path
@@ -442,7 +481,7 @@ def _called_again(
442
481
  calling = [
443
482
  at
444
483
  for at in calling
445
- if project.returned(modules, paths[at], {}) != project.returned(before, paths[at], {})
484
+ if loose.returned(modules, paths[at], {}) != loose.returned(before, paths[at], {})
446
485
  ]
447
486
  return runs, _checked_more(paths, runs, calling, (check, options, session), modules)
448
487
 
@@ -468,7 +507,7 @@ def _checked_more(
468
507
  at: int
469
508
  run: FileRun
470
509
  for at, run in zip(which, redone, strict=True):
471
- runs[at] = _merged(runs[at], run) if how[1].mode is Mode.FIX else run
510
+ runs[at] = merged(runs[at], run) if how[1].mode is Mode.FIX else run
472
511
  return modules
473
512
 
474
513
 
@@ -504,42 +543,29 @@ def _checked_all(
504
543
  What each file found, and the index of every file's module.
505
544
 
506
545
  """
507
- hinted: dict[Path, tuple[Hints, ...]] = {} if session is None else session.hints(_texts(paths))
546
+ # Before a worker starts: one forked while the servers' threads run could wait on a lock one holds.
547
+ hinted: _Hinted = {} if session is None else session.hints(_texts(paths))
548
+ if options.input.trace is not None:
549
+ hinted = traced.merged(hinted, traced.load(options.input.trace), paths)
508
550
  coverage: bool = options.mode is Mode.COVERAGE # needs nothing from the other files
509
- stack: contextlib.ExitStack
510
- with contextlib.ExitStack() as stack:
511
- pool: Workers | None = None
512
- if options.jobs != 1 and len(paths) > 1:
513
- pool = stack.enter_context(Workers(paths, options.jobs))
514
- if modules is None and pool:
515
- modules = project.Index({}, []) if coverage else pool.index()
516
- elif modules is None:
517
- modules = project.Index({}, []) if coverage else project.index(paths)
518
- collecting.indexed()
519
- if not coverage:
520
- modules = plain.settled(
521
- decorated.passed(installed.with_installed(modules, installed.search_path())),
522
- options.checks.plain_bases,
523
- )
524
- return schedule.checked(
525
- paths,
526
- check,
527
- pool,
528
- (modules, hinted),
529
- _merged if options.mode is Mode.FIX else None,
551
+ pool: Workers | None = started(paths, options.jobs) if options.jobs != 1 and len(paths) > 1 else None
552
+ if modules is None and pool:
553
+ modules = project.Index({}, []) if coverage else pool.index()
554
+ elif modules is None:
555
+ modules = project.Index({}, []) if coverage else project.index(paths)
556
+ collecting.indexed()
557
+ if not coverage:
558
+ modules = plain.settled(
559
+ decorated.passed(installed.with_installed(modules, installed.search_path())),
560
+ options.checks.plain_bases,
530
561
  )
531
-
532
-
533
- def _merged(before: FileRun, after: FileRun) -> FileRun:
534
- """Join a file's two `--fix` rounds' `CheckRun`s: what's left is the later's, what's fixed is both's.
535
-
536
- Returns:
537
- The joined run.
538
-
539
- """
540
- first: CheckRun = cast("CheckRun", before)
541
- later: CheckRun = cast("CheckRun", after)
542
- return replace(later, fixed=first.fixed + later.fixed)
562
+ return schedule.checked(
563
+ paths,
564
+ check,
565
+ pool,
566
+ (modules, hinted),
567
+ merged if options.mode is Mode.FIX else None,
568
+ )
543
569
 
544
570
 
545
571
  def _texts(paths: Sequence[Path]) -> dict[Path, str]:
@@ -572,10 +598,16 @@ def _report(options: Options, runs: Sequence[FileRun], files: int) -> int:
572
598
  results: list[Result] = [result for run in checked for result in run.results]
573
599
  output: Output = options.output
574
600
  text: bool = output.fmt in {Format.TEXT, Format.FULL}
601
+ # With `--show-fixes`, the fixes `--fix` made too: after the rest, as text or JSON.
602
+ shown: list[Result] = [*results, *(made for run in checked if output.show_fixes for made in run.made)]
575
603
  line: str
576
- for line in statistics(results) if text and output.statistics else render(output.fmt, results):
604
+ for line in (
605
+ statistics(results)
606
+ if text and output.statistics
607
+ else render(output.fmt, shown if output.fmt is Format.JSON else results)
608
+ ):
577
609
  _ = sys.stdout.write(f"{line}\n")
578
- for line in fix_reasons(results) if text and output.show_fixes else ():
610
+ for line in fix_reasons(shown) if text and output.show_fixes else ():
579
611
  _ = sys.stdout.write(f"{line}\n")
580
612
  errors: int = sum(r.offence.is_error(r.level) for r in results)
581
613
  if text and not output.quiet:
@@ -604,30 +636,53 @@ def _coverage(options: Options, paths: Sequence[Path], runs: Sequence[FileRun])
604
636
  counted: list[tuple[Path, Coverage]] = [
605
637
  (path, run.coverage) for path, run in zip(paths, covered, strict=True) if run.coverage is not None
606
638
  ]
607
- total: Coverage = Coverage(sum(c.typed for _, c in counted), sum(c.total for _, c in counted))
639
+ total: Coverage = Coverage(
640
+ sum(c.typed for _, c in counted),
641
+ sum(c.total for _, c in counted),
642
+ sum(c.widened for _, c in counted),
643
+ )
608
644
  if options.output.fmt is Format.JSON:
609
645
  report: dict[str, _Scalar | list[_Row]] = {
610
- "typed": total.typed,
611
- "total": total.total,
612
- "percent": round(total.percent, 1),
613
- "files": [
614
- {"path": str(path), "typed": c.typed, "total": c.total, "percent": round(c.percent, 1)}
615
- for path, c in counted
616
- ],
646
+ **_counts(total),
647
+ "files": [{"path": str(path), **_counts(c)} for path, c in counted],
617
648
  }
618
649
  _ = sys.stdout.write(json.dumps(report, indent=2) + "\n")
619
650
  else:
620
651
  path: Path
621
652
  c: Coverage
622
653
  for path, c in counted:
623
- _ = sys.stdout.write(f"{path}: {c.typed}/{c.total} typed ({c.percent:.1f}%)\n")
624
- _ = sys.stdout.write(
625
- f"Total: {total.typed}/{total.total} typed ({total.percent:.1f}%) in {len(counted)} file(s).\n",
626
- )
654
+ _ = sys.stdout.write(f"{path}: {_typed(c)}\n")
655
+ _ = sys.stdout.write(f"Total: {_typed(total)} in {len(counted)} file(s).\n")
627
656
  threshold: float | None = options.output.fail_under
628
657
  return EXIT_FOUND if threshold is not None and total.percent < threshold else EXIT_CLEAN
629
658
 
630
659
 
660
+ def _counts(found: Coverage) -> _Row:
661
+ """Name a coverage's counts, for `--format=json`.
662
+
663
+ Returns:
664
+ Them, by name.
665
+
666
+ """
667
+ return {
668
+ "typed": found.typed,
669
+ "total": found.total,
670
+ "widened": found.widened,
671
+ "percent": round(found.percent, 1),
672
+ }
673
+
674
+
675
+ def _typed(found: Coverage) -> str:
676
+ """Write a coverage as text: `3/4 typed (75.0%)`, then what a widening annotated, if anything.
677
+
678
+ Returns:
679
+ It.
680
+
681
+ """
682
+ wider: str = f", {found.widened} widened" if found.widened else ""
683
+ return f"{found.typed}/{found.total} typed ({found.percent:.1f}%){wider}"
684
+
685
+
631
686
  def _run(options: Options) -> int:
632
687
  """Check, fix, diff, count or baseline, printing what that finds.
633
688
 
constricter/cli/config.py CHANGED
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias, cast
10
10
 
11
11
  from constricter.cli.protocol import SERVERS
12
12
  from constricter.jsonc import is_int
13
- from constricter.offences import FIX_KINDS, LEVELS, MESSAGES
13
+ from constricter.offences import FIX_KINDS, LEVELS, MESSAGES, WIDEN_KINDS
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from datetime import date, datetime, time
@@ -32,6 +32,7 @@ def unknown_codes(codes: Sequence[str]) -> list[str]:
32
32
 
33
33
  DEFAULT_BASELINE: Final = "constricter-baseline.json"
34
34
  _MIN_PYTHON: Final = "min_python"
35
+ _ALL_KINDS: Final = "all" # `fix-widen`'s: every wider type
35
36
  _VERSION: Final = re.compile(r"\d+\.\d+") # `min-python`'s: `3.11`
36
37
  # A lower bound among `requires-python`'s specifiers: `>=3.11`, `~=3.11`, `==3.11.*`, `>3.10`.
37
38
  _LOWER_BOUND: Final = re.compile(r"(?:>=|~=|==|>)\s*(\d+)(?:\.(\d+))?")
@@ -160,6 +161,31 @@ def unknown_fix_kinds(kinds: Sequence[str]) -> list[str]:
160
161
  return [kind for kind in kinds if kind not in FIX_KINDS]
161
162
 
162
163
 
164
+ def unknown_widen_kinds(kinds: Sequence[str]) -> list[str]:
165
+ """Check `kinds` against `WIDEN_KINDS`' ids, and `all`.
166
+
167
+ Returns:
168
+ Those that aren't one.
169
+
170
+ """
171
+ return [kind for kind in kinds if kind not in WIDEN_KINDS | {_ALL_KINDS}]
172
+
173
+
174
+ def widen_kinds(kinds: Sequence[str]) -> list[str]:
175
+ """Read `fix-widen`'s kinds, `all` as every one.
176
+
177
+ Returns:
178
+ Them.
179
+
180
+ """
181
+ return sorted(WIDEN_KINDS) if _ALL_KINDS in kinds else list(kinds)
182
+
183
+
184
+ def _widen_kinds(value: _Toml) -> list[str] | None:
185
+ kinds: list[str] | None = [value] if isinstance(value, str) else _strings(value)
186
+ return None if kinds is None or unknown_widen_kinds(kinds) else widen_kinds(kinds)
187
+
188
+
163
189
  def _fix_kinds(value: _Toml) -> list[str] | None:
164
190
  kinds: list[str] | None = _strings(value)
165
191
  return None if kinds is None or unknown_fix_kinds(kinds) else kinds
@@ -223,7 +249,7 @@ def _lists(value: _Toml, read: Callable[[_Toml], list[str] | None]) -> dict[str,
223
249
 
224
250
 
225
251
  def _baseline(value: _Toml, root: Path) -> str | None:
226
- """Read a baseline path, relative to `root` (the pyproject.toml that names it).
252
+ """Read a baseline's or a trace's path, relative to `root` (the pyproject.toml that names it).
227
253
 
228
254
  Returns:
229
255
  The resolved path, or `None` if `value` isn't a non-empty string.
@@ -246,6 +272,7 @@ _READERS: dict[str, Callable[[_Toml], Default | None]] = {
246
272
  "fix-select": _fix_kinds,
247
273
  "fix-ignore": _fix_kinds,
248
274
  "unsafe-fix-select": _fix_kinds,
275
+ "fix-widen": _widen_kinds,
249
276
  "fix-plain-bases": _strings,
250
277
  "type-comments": _flag,
251
278
  "all-scopes": _flag,
@@ -276,6 +303,7 @@ def config_defaults(start: Path) -> dict[str, Default]:
276
303
  readers: dict[str, Callable[[_Toml], Default | None]] = {
277
304
  **_READERS,
278
305
  "baseline": partial(_baseline, root=path.parent),
306
+ "infer-from": partial(_baseline, root=path.parent),
279
307
  }
280
308
  defaults: dict[str, Default] = {}
281
309
  key: str
constricter/cli/hints.py CHANGED
@@ -33,7 +33,7 @@ from functools import partial
33
33
  from pathlib import Path
34
34
  from typing import IO, Final, NamedTuple, Self, TypeAlias, cast
35
35
 
36
- from constricter.cli import edits, guard, protocol
36
+ from constricter.cli import edits, guard, protocol, remembered
37
37
  from constricter.cli.protocol import SERVERS, HintError, HungError, Server
38
38
  from constricter.fix.core.known import Hints, Offered
39
39
 
@@ -47,8 +47,6 @@ _Task: TypeAlias = Callable[[], _Found] # one server's work: the hints of each
47
47
 
48
48
  _BATCH: Final = 8 # files a free server takes at a time: few, so none is left with the slow ones
49
49
  MEMORY: Final = 8 << 30 # the most a checker's servers use together, by default (`--infer-memory`)
50
- _SERVER_MEMORY: Final = 600 << 20 # a server's memory, before the files (basedpyright's, measured)
51
- _MEMORY_PER_BYTE: Final = 130 # and for each byte of them: 1.2 GB for SQLAlchemy's 8 MB, 3.4 for pandas' 21
52
50
  _MEMINFO: Final = Path("/proc/meminfo")
53
51
  _FILES_PER_SERVER: Final = 32 # fewer files than this each don't pay for another server's start
54
52
  # Seconds without a word from a server before it's taken for hung. Not an answer's wait: pandas'
@@ -228,11 +226,10 @@ class Checker:
228
226
  """One checker's servers, and which of them has each file.
229
227
 
230
228
  It has as many as its files need (one per `_FILES_PER_SERVER`), up to the command's `--jobs`, the
231
- checker's own most (`Server.most`: one for a checker that works in parallel itself), and as many
232
- as fit in its memory: `--infer-memory` (never more than is available), or by default `MEMORY`
233
- (never more than half what's available). Each server loads the whole program it checks, which
234
- takes (as measured) about `_SERVER_MEMORY` plus `_MEMORY_PER_BYTE` for each byte of the files.
235
- One server it always has, whatever the memory.
229
+ checker's own most (`Server.most`), and as many as fit in its memory: `--infer-memory` (never
230
+ more than is available), or by default `MEMORY` (never more than half what's available). Each
231
+ server loads the whole program it checks, or its share of it, by its checker's own measure (see
232
+ `Server.fitting`). One server it always has, whatever the memory.
236
233
 
237
234
  The files are handed out a few at a time (`_BATCH`), the biggest first, to whichever server is
238
235
  free: one file's analysis can take far longer than another's, and none waits on a busy one. A
@@ -244,7 +241,8 @@ class Checker:
244
241
  self.name: str = name
245
242
  self.root: Path = root
246
243
  self.command: list[str] = command(name)
247
- self.most: int = max(1, min(servers, SERVERS[name].most))
244
+ self.server: Server = SERVERS[name]
245
+ self.most: int = max(1, min(servers, self.server.most))
248
246
  self.memory: int = budget(memory, available_memory())
249
247
  self.servers: list[Connection] = []
250
248
  self.assigned: dict[Path, int] = {} # each file's server, by its place in `servers`
@@ -258,13 +256,14 @@ class Checker:
258
256
 
259
257
  """
260
258
  size: int = sum(len(text) for text in files.values())
261
- fit: int = max(1, self.memory // (_SERVER_MEMORY + _MEMORY_PER_BYTE * size))
259
+ fit: int = max(1, self.server.fitting(self.memory, size))
262
260
  return max(len(self.servers), min(self.most, -(-len(files) // _FILES_PER_SERVER), fit))
263
261
 
264
262
  def tasks(self, files: Mapping[Path, str]) -> list[_Task]:
265
263
  """Make each server's work on `files`: its own files first, then new ones as it's free for them.
266
264
 
267
- A file abandoned (see `_alone`) isn't asked about again.
265
+ A file abandoned (see `_alone`), in this run or an earlier one (see `remembered`), isn't
266
+ asked about again.
268
267
 
269
268
  Returns:
270
269
  A task per server, giving the hints of every file it was sent.
@@ -274,8 +273,14 @@ class Checker:
274
273
  path: Path
275
274
  for path in files.keys() & self.assigned.keys() - self.abandoned:
276
275
  own.setdefault(self.assigned[path], {})[path] = files[path]
276
+ # One it hung on in an earlier run, unchanged since: left without hints from the start.
277
+ self.abandoned.update(
278
+ path
279
+ for path in files.keys() - self.assigned.keys() - self.abandoned
280
+ if remembered.hung(self.name, self.command, files[path])
281
+ )
277
282
  fresh: list[Path] = sorted(
278
- files.keys() - self.assigned.keys(),
283
+ files.keys() - self.assigned.keys() - self.abandoned,
279
284
  key=lambda new: (-len(files[new]), new),
280
285
  )
281
286
  batches: queue.SimpleQueue[dict[Path, str]] = queue.SimpleQueue()
@@ -352,6 +357,7 @@ class Checker:
352
357
  found.update(server.answers(server.ask({path: text})))
353
358
  except HungError as hung:
354
359
  self.abandoned.add(path)
360
+ remembered.remember(self.name, self.command, text)
355
361
  if len(self.abandoned) > _MOST_ABANDONED:
356
362
  error: str = f"{self.name} hung on {len(self.abandoned)} file(s), each asked about alone"
357
363
  raise HintError(error) from hung
@@ -11,13 +11,15 @@ from pathlib import Path
11
11
  from typing import Final, cast
12
12
 
13
13
  from constricter import __version__
14
- from constricter.cli import baseline
14
+ from constricter.cli import baseline, traced
15
15
  from constricter.cli.config import (
16
16
  DEFAULT_BASELINE,
17
17
  config_defaults,
18
18
  project_root,
19
19
  unknown_codes,
20
20
  unknown_fix_kinds,
21
+ unknown_widen_kinds,
22
+ widen_kinds,
21
23
  )
22
24
  from constricter.cli.explain import explain
23
25
  from constricter.cli.paths import STDIN, excluded
@@ -133,6 +135,24 @@ def _fix_kinds(text: str) -> list[str]:
133
135
  return kinds
134
136
 
135
137
 
138
+ def _widen_kinds(text: str) -> list[str]:
139
+ """Read a comma-separated list of the wider types `--fix` may write (`untyped-parameters`), or `all`.
140
+
141
+ Returns:
142
+ Them.
143
+
144
+ Raises:
145
+ argparse.ArgumentTypeError: One isn't a widening's id.
146
+
147
+ """
148
+ kinds: list[str] = [kind.strip() for kind in text.split(",") if kind.strip()]
149
+ unknown: list[str]
150
+ if unknown := unknown_widen_kinds(kinds):
151
+ message: str = f"no wider type is called {', '.join(unknown)} (see docs/FIXES.md)"
152
+ raise argparse.ArgumentTypeError(message)
153
+ return widen_kinds(kinds)
154
+
155
+
136
156
  def _version(text: str) -> tuple[int, int]:
137
157
  """Read `--min-python`: a Python version (`3.11`).
138
158
 
@@ -333,6 +353,13 @@ def _parser() -> argparse.ArgumentParser:
333
353
  metavar="KINDS",
334
354
  help="treat guesses from these mechanisms (constructor, narrow) as certain",
335
355
  )
356
+ _ = parser.add_argument(
357
+ "--fix-widen",
358
+ type=_widen_kinds,
359
+ default=[],
360
+ metavar="KINDS",
361
+ help="also write these wider types, each marked (untyped-parameters, unions, ...; all: every one)",
362
+ )
336
363
  _ = parser.add_argument(
337
364
  "--fix-plain-bases",
338
365
  type=_listed,
@@ -353,6 +380,12 @@ def _parser() -> argparse.ArgumentParser:
353
380
  metavar="GB",
354
381
  help="with --infer-with: the most memory each checker's servers use together (default: 8)",
355
382
  )
383
+ _ = parser.add_argument(
384
+ "--infer-from",
385
+ type=Path,
386
+ metavar="FILE",
387
+ help="type what --fix can't with the types `python -m constricter.trace` recorded, as guesses",
388
+ )
356
389
  _ = parser.add_argument(
357
390
  "--diff",
358
391
  action="store_true",
@@ -361,7 +394,7 @@ def _parser() -> argparse.ArgumentParser:
361
394
  _ = parser.add_argument(
362
395
  "--show-fixes",
363
396
  action="store_true",
364
- help="after the report, list each fix and how the value decided it (text)",
397
+ help="after the report, list each fix and how the value decided it; with --fix, those made too",
365
398
  )
366
399
  _ = parser.add_argument(
367
400
  "--statistics",
@@ -440,6 +473,7 @@ class Input:
440
473
  paths: list[Path]
441
474
  exclude: list[str]
442
475
  stdin_name: Path = STDIN
476
+ trace: Path | None = None # the trace whose types `--fix` guesses with (`--infer-from`)
443
477
 
444
478
  def name(self, path: Path) -> Path:
445
479
  """Name `path` for reports.
@@ -566,7 +600,12 @@ class Options:
566
600
  parser.error("`-` (standard input) must be the only path")
567
601
  mode: Mode = _mode(parser, args)
568
602
  return cls(
569
- input=Input(paths, cast("list[str]", args.exclude), cast("Path", args.stdin_filename)),
603
+ input=Input(
604
+ paths,
605
+ cast("list[str]", args.exclude),
606
+ cast("Path", args.stdin_filename),
607
+ _trace(parser, cast("Path | None", args.infer_from)),
608
+ ),
570
609
  checks=Checks(
571
610
  type_comments=cast("bool", args.type_comments),
572
611
  all_scopes=cast("bool", args.all_scopes),
@@ -582,6 +621,7 @@ class Options:
582
621
  frozenset(cast("list[str]", args.fix_select)),
583
622
  frozenset(cast("list[str]", args.fix_ignore)),
584
623
  frozenset(cast("list[str]", args.unsafe_fix_select)),
624
+ frozenset(cast("list[str]", args.fix_widen)),
585
625
  ),
586
626
  min_python=cast("tuple[int, int] | None", args.min_python),
587
627
  plain_bases=(*PLAIN_BASES, *cast("list[str]", args.fix_plain_bases)),
@@ -605,6 +645,20 @@ class Options:
605
645
  )
606
646
 
607
647
 
648
+ def _trace(parser: argparse.ArgumentParser, path: Path | None) -> Path | None:
649
+ """Check that `--infer-from`'s file is a trace, read here once for the run.
650
+
651
+ Returns:
652
+ The path.
653
+
654
+ """
655
+ try:
656
+ _ = None if path is None else traced.load(path)
657
+ except ValueError as error:
658
+ parser.error(str(error))
659
+ return path
660
+
661
+
608
662
  def _maximal(args: argparse.Namespace) -> argparse.Namespace:
609
663
  """Apply `--max-fix` (`--max`, every fix, every installed checker) and `--max`.
610
664
 
@@ -33,21 +33,41 @@ class Server(NamedTuple):
33
33
  """A checker's language server: how it's started, and how many are worth running at once.
34
34
 
35
35
  Its executable, the arguments that start it over stdio, and the most of it to run (up to
36
- `MAX_SERVERS`; one for a checker that works in parallel itself).
36
+ `MAX_SERVERS`). And the memory its servers take, as measured: `memory` bytes each before the
37
+ files, and `per_byte` more for each byte of them: each server's again where each holds the
38
+ `whole` program, once among them all where each holds what it's asked about.
37
39
  """
38
40
 
39
41
  executable: str
40
42
  args: tuple[str, ...]
41
43
  most: int
44
+ memory: int
45
+ per_byte: int
46
+ whole: bool = False
47
+
48
+ def fitting(self, budget: int, size: int) -> int:
49
+ """Count the servers that fit in `budget` bytes, for files of `size` bytes together.
50
+
51
+ Returns:
52
+ It: none where one doesn't fit.
53
+
54
+ """
55
+ if self.whole:
56
+ return budget // (self.memory + self.per_byte * size)
57
+ return max(0, budget - self.per_byte * size) // self.memory
42
58
 
43
59
 
44
60
  MAX_SERVERS: Final = 4 # a checker's, at most: each holds its own copy of the program it checks
45
61
  SERVERS: Final = {
46
62
  # One thread each: more servers check more at once (sqlalchemy's hints: 10.8s with one, 7.2s with four).
47
- "basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS),
48
- # Parallel already: more servers only repeat its work (sqlalchemy's: 0.9s with one, 0.7s with four).
49
- "ty": Server("ty", ("server",), 1),
50
- "pyrefly": Server("pyrefly", ("lsp",), 1), # parallel too
63
+ # Each holds the whole program: 1.2 GB for sqlalchemy's 8 MB, 3.4 for pandas's 21.
64
+ "basedpyright": Server("basedpyright-langserver", ("--stdio",), MAX_SERVERS, 600 << 20, 130, whole=True),
65
+ # Parallel itself, and still quicker shared out: a hinted check of pandas takes 36s with one, 25s
66
+ # with two, 22s with three and 21s with four (1.2 GB with one, 1.7 with two, 2.5 with four).
67
+ "ty": Server("ty", ("server",), MAX_SERVERS, 420 << 20, 38),
68
+ # Parallel too: pandas's takes 132s with one, 58s with two and 32s with four (2.8 GB, 3.2, 4.2);
69
+ # the standard library's 39 MB, 4.3 GB with one and 5.6 with four.
70
+ "pyrefly": Server("pyrefly", ("lsp",), MAX_SERVERS, 480 << 20, 120),
51
71
  }
52
72
  _PROBE: Final = "--version" # quick for each; basedpyright-langserver's exits 1 all the same
53
73
  _PROBE_TIMEOUT: Final = 10.0 # seconds it may take