tsc-ts 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +1 -1
  2. package/dist/api/compiler/ast.d.ts +9 -3
  3. package/dist/api/compiler/ast.js +17 -12
  4. package/dist/api/compiler/checker-classes.js +2 -1
  5. package/dist/api/compiler/checker-contextual.d.ts +20 -0
  6. package/dist/api/compiler/checker-contextual.js +111 -1
  7. package/dist/api/compiler/checker-expressions.js +14 -1
  8. package/dist/api/compiler/checker-jsx.js +2 -1
  9. package/dist/api/compiler/checker-type-factory.js +53 -12
  10. package/dist/api/compiler/checker.d.ts +5 -1
  11. package/dist/api/compiler/checker.js +10 -4
  12. package/dist/api/compiler/compile-program.d.ts +5 -0
  13. package/dist/api/compiler/compile-program.js +8 -0
  14. package/dist/api/compiler/expr-record.d.ts +11 -0
  15. package/dist/api/compiler/expr-record.js +53 -0
  16. package/dist/api/compiler/identifier-name.d.ts +9 -0
  17. package/dist/api/compiler/identifier-name.js +10 -0
  18. package/dist/api/compiler/jsdoc.js +4 -2
  19. package/dist/api/compiler/parser.d.ts +2 -1
  20. package/dist/api/compiler/parser.js +10 -8
  21. package/dist/api/compiler/printer/factory.js +3 -2
  22. package/dist/api/compiler/printer/printer.js +3 -2
  23. package/dist/api/compiler/reparser.js +4 -3
  24. package/dist/api/compiler/scanner.d.ts +8 -0
  25. package/dist/api/compiler/scanner.js +58 -42
  26. package/dist/api/compiler/shared-results.d.ts +376 -143
  27. package/dist/api/compiler/shared-results.js +1550 -773
  28. package/dist/api/compiler/transformers/declarations.js +2 -1
  29. package/package.json +9 -9
  30. package/src/compiler/ast.ts +19 -12
  31. package/src/compiler/cgroup.ts +157 -0
  32. package/src/compiler/checker-classes.ts +2 -1
  33. package/src/compiler/checker-contextual.ts +104 -0
  34. package/src/compiler/checker-expressions.ts +16 -1
  35. package/src/compiler/checker-jsx.ts +2 -1
  36. package/src/compiler/checker-type-factory.ts +53 -14
  37. package/src/compiler/checker.ts +11 -4
  38. package/src/compiler/compile-program.ts +8 -0
  39. package/src/compiler/decoder.ts +2 -1
  40. package/src/compiler/doctor-report.ts +133 -0
  41. package/src/compiler/expr-record.ts +59 -0
  42. package/src/compiler/identifier-name.ts +13 -0
  43. package/src/compiler/jsdoc.ts +4 -2
  44. package/src/compiler/parallel-check.ts +353 -146
  45. package/src/compiler/parallel-emit.ts +3 -3
  46. package/src/compiler/parser.ts +10 -8
  47. package/src/compiler/printer/factory.ts +3 -2
  48. package/src/compiler/printer/printer.ts +4 -2
  49. package/src/compiler/reparser.ts +7 -3
  50. package/src/compiler/run-config.ts +1383 -0
  51. package/src/compiler/run-record-read.ts +87 -0
  52. package/src/compiler/run-records.ts +575 -0
  53. package/src/compiler/scanner.ts +57 -41
  54. package/src/compiler/shared-results.ts +1626 -896
  55. package/src/compiler/thread-entry.ts +3 -1
  56. package/src/compiler/transformers/declarations.ts +2 -1
  57. package/src/compiler/tsc-build.ts +64 -6
  58. package/src/compiler/tsc-cli.ts +33 -4
  59. package/src/compiler/tsc-compile.ts +213 -8
  60. package/src/compiler/tsc-watch.ts +25 -4
@@ -50,8 +50,9 @@ import type { Program } from './program.js';
50
50
  import type { SourceFile } from './ast.js';
51
51
  import { lastSymbolIdValue } from './symbol.js';
52
52
  import { shareMergedGlobals } from './checker-symbols.js';
53
+ import { availableCpus, cgroupMemoryLimit } from './cgroup.js';
53
54
  import { readFileSync } from 'node:fs';
54
- import { availableParallelism, totalmem } from 'node:os';
55
+ import { totalmem } from 'node:os';
55
56
  import {
56
57
  forbiddenWords,
57
58
  isForbidden,
@@ -59,10 +60,68 @@ import {
59
60
  numberLogSize,
60
61
  recordTableSize,
61
62
  resultTableSize,
62
- shareThreads,
63
+ shareStatColumns,
64
+ shareStatExport,
65
+ shareStatPosted,
66
+ shareStatSaved,
67
+ shareStatTranslation,
68
+ shareStatTranslated,
63
69
  stringLogSize,
70
+ usedStride,
64
71
  type ThreadSharingData,
65
72
  } from './shared-results.js';
73
+ import {
74
+ autoCheckers,
75
+ Automaticity,
76
+ CheckSignals,
77
+ checkersPolicy,
78
+ decideCheck,
79
+ learns,
80
+ libraryBoundCount,
81
+ maxAutoCheckers,
82
+ minSharingCheckers,
83
+ sharePin,
84
+ sharingCandidate,
85
+ sharingPeakShare,
86
+ Source,
87
+ tsgoCheckers,
88
+ type CheckDecision,
89
+ type CheckFit,
90
+ type RunConfig,
91
+ } from './run-config.js';
92
+ import type { RecordShape } from './run-records.js';
93
+
94
+ // The tests of the scheduler read these here, next to the plans they choose between.
95
+ export { autoCheckers, libraryBoundCount };
96
+
97
+ /**
98
+ * `TSC_TS_SHARE` for a run without a run configuration (the emit threads, which share no program to
99
+ * translate against): `1` shares, `0` does not, and unset shares nothing (`sharePin`).
100
+ */
101
+ const shareThreads = sharePin(process.env);
102
+
103
+ /**
104
+ * TSC_TS_SHARE_STOP_AT_MS: `t` or `t:level` stops a sharing run's sharing `t` milliseconds in, whatever
105
+ * the run's numbers say, to check the switch: `2` (the default) stops posting and translating, `1`
106
+ * stops posting. Testing only.
107
+ */
108
+ function shareStopHook(): { readonly at: number; readonly level: number } | undefined {
109
+ const value = process.env.TSC_TS_SHARE_STOP_AT_MS;
110
+ if (value === undefined || value === '') return undefined;
111
+ const colon = value.indexOf(':');
112
+ const at = Number(colon < 0 ? value : value.substring(0, colon));
113
+ if (!Number.isFinite(at) || at < 0) return undefined;
114
+ return { at, level: colon >= 0 && value.substring(colon + 1) === '1' ? 1 : 2 };
115
+ }
116
+
117
+ const shareStop = shareStopHook();
118
+
119
+ /**
120
+ * The window a run without a configuration uses before the main thread weighs what sharing saved against
121
+ * what it cost (`sharingNumbers`): 300 ms. A run with one uses the larger of 200 ms and a tenth of the
122
+ * check time its record measured (`RunConfig.shareWindowMs`, `shareWindowOf`).
123
+ */
124
+ const shareWindowMs = 300;
66
125
 
67
126
  /** With TSC_TS_CHECKERS_TRACE: the process's resident and peak resident set in MB (Linux), else 0. */
68
127
  export function residentMB(): number[] {
@@ -454,23 +513,9 @@ function checkersEmit(checked: CheckedProgram, emits: boolean): boolean {
454
513
  return emits && parallelCheck.writer !== undefined && options.noEmit !== true && options.listFilesOnly !== true;
455
514
  }
456
515
 
457
- /**
458
- * How the number of checkers is chosen without `--checkers`: `auto` (the default) uses every CPU
459
- * this process may run on (`os.availableParallelism()`, at most `maxAutoCheckers`), and at least
460
- * tsgo's four; fewer than the CPUs, but never fewer than four, when the estimated peak memory of more
461
- * checkers would not fit (`memoryCheckers`). `fixed` (TSC_TS_CHECKERS=fixed) uses tsgo's four.
462
- */
463
- export const checkerPolicy: 'fixed' | 'auto' = process.env.TSC_TS_CHECKERS === 'fixed' ? 'fixed' : 'auto';
464
-
465
- /** tsgo's number of checkers, and the fewest `auto` starts. */
466
- const tsgoCheckers = 4;
467
-
468
- /** The most checkers `auto` starts (measured up to 16 CPUs). */
469
- const maxAutoCheckers = 16;
470
-
471
516
  /**
472
517
  * The number of checker threads for `checkers` (`--checkers`), or 1 to check on this thread. Without
473
- * the option, native builds follow `checkerPolicy` (with `program`, the memory estimate of `auto`
518
+ * the option, native builds follow the checker policy (with `program`, the memory estimate of `auto`
474
519
  * applies); under Node.js, where every thread builds its own copy of the program, tsc-ts checks on
475
520
  * one thread.
476
521
  */
@@ -487,8 +532,8 @@ export function checkerThreads(
487
532
  function defaultCheckers(program: Program | undefined): number {
488
533
  if (!sharesPublishedGraphs) return 1;
489
534
  if (program !== undefined && checksLittle(program)) return 1;
490
- if (checkerPolicy === 'fixed') return tsgoCheckers;
491
- const cpus = Math.min(availableParallelism(), maxAutoCheckers);
535
+ if (checkersPolicy(process.env) === 'fixed') return tsgoCheckers;
536
+ const cpus = Math.min(availableCpus(), maxAutoCheckers);
492
537
  return autoCheckers(cpus, program === undefined ? Number.MAX_SAFE_INTEGER : memoryCheckers(program));
493
538
  }
494
539
 
@@ -514,24 +559,6 @@ function checksLittle(program: Program): boolean {
514
559
  return true;
515
560
  }
516
561
 
517
- /** `auto` with `cpus` CPUs when the memory estimate fits `fit` checkers: tsgo's four at least. */
518
- export function autoCheckers(cpus: number, fit: number): number {
519
- return Math.max(tsgoCheckers, Math.min(cpus, fit));
520
- }
521
-
522
- /**
523
- * The number of threads that parse ahead of a program checked by `checkers` checkers: every CPU this
524
- * process may run on, as tsgo parses on every core whatever its checker count (parse threads hand
525
- * their files to the shared program; what else they allocate is reused by the checker threads), but
526
- * one. The main thread hands out the chunks they load: with a thread on every CPU, it waits for a CPU
527
- * each time a thread returns a chunk, and so does that thread for its next chunk (VS Code's parse took
528
- * 18-57 ms longer with 16 threads on 16 CPUs than with 15).
529
- */
530
- export function parseThreads(checkers: number): number {
531
- const cpus = availableParallelism();
532
- return Math.max(1, Math.min(cpus - 1, Math.max(checkers, Math.min(cpus, maxAutoCheckers))));
533
- }
534
-
535
562
  /**
536
563
  * The program's memory per character of source text, where it can't be measured (`residentBytes`):
537
564
  * 23-28 bytes on the benchmark apps when checking starts (parse threads' arenas included).
@@ -567,9 +594,14 @@ const lastResortShare = 0.95;
567
594
  * bytes, for a program of `programBytes` (`firstCheckerShare`, `checkerBytesPerDeclarationChar`);
568
595
  * at least 1.
569
596
  */
570
- export function checkersFittingMemory(programBytes: number, declarationChars: number, memory: number): number {
571
- const peak = programBytes * (1 + firstCheckerShare);
572
- const perChecker = Math.max(1, declarationChars * checkerBytesPerDeclarationChar);
597
+ export function checkersFittingMemory(
598
+ programBytes: number,
599
+ declarationChars: number,
600
+ memory: number,
601
+ peakShare: number = 1,
602
+ ): number {
603
+ const peak = programBytes * (1 + firstCheckerShare) * peakShare;
604
+ const perChecker = Math.max(1, declarationChars * checkerBytesPerDeclarationChar * peakShare);
573
605
  return Math.max(1, 1 + Math.floor((memoryShare * memory - peak) / perChecker));
574
606
  }
575
607
 
@@ -586,59 +618,6 @@ export function memoryLimit(): number {
586
618
  return limit;
587
619
  }
588
620
 
589
- /**
590
- * The smallest memory limit of this process's cgroup and its ancestors (Linux), or 0 where there is
591
- * none or it is unknown: `memory.max` and `memory.high` under cgroup v2, `memory.limit_in_bytes`
592
- * under v1. `process.constrainedMemory()` does not always see these: libuv reads only the process's
593
- * own cgroup, and scriptc's runtime only the root of the cgroup mount, which is the process's own
594
- * cgroup only inside a cgroup namespace.
595
- */
596
- export function cgroupMemoryLimit(): number {
597
- let membership: string;
598
- try {
599
- membership = readFileSync('/proc/self/cgroup', 'utf8');
600
- } catch {
601
- return 0;
602
- }
603
- let limit = 0;
604
- const lower = (value: number): void => {
605
- if (value > 0 && (limit === 0 || value < limit)) limit = value;
606
- };
607
- for (const line of membership.split('\n')) {
608
- // "0::/path" (v2), "N:memory:/path" or "N:cpu,memory:/path" (v1).
609
- const first = line.indexOf(':');
610
- const second = line.indexOf(':', first + 1);
611
- if (first < 0 || second < 0) continue;
612
- const controllers = line.substring(first + 1, second);
613
- const path = line.substring(second + 1);
614
- if (controllers === '') {
615
- for (let dir = path; ; dir = dir.substring(0, dir.lastIndexOf('/'))) {
616
- const base = '/sys/fs/cgroup' + (dir === '/' ? '' : dir);
617
- lower(cgroupLimitValue(base + '/memory.max'));
618
- lower(cgroupLimitValue(base + '/memory.high'));
619
- if (dir === '/' || dir === '' || dir.lastIndexOf('/') < 0) break;
620
- }
621
- } else if (controllers.split(',').indexOf('memory') >= 0) {
622
- lower(cgroupLimitValue('/sys/fs/cgroup/memory' + (path === '/' ? '' : path) + '/memory.limit_in_bytes'));
623
- }
624
- }
625
- return limit;
626
- }
627
-
628
- /** A cgroup memory limit file's value in bytes, or 0 for `max`, an unreadable file or no number. */
629
- function cgroupLimitValue(file: string): number {
630
- let text: string;
631
- try {
632
- text = readFileSync(file, 'utf8').trim();
633
- } catch {
634
- return 0;
635
- }
636
- if (text === '' || text === 'max') return 0;
637
- const value = Number(text);
638
- // cgroup v1 reports no limit as a page-aligned value near 2^63.
639
- return Number.isFinite(value) && value > 0 && value < 2 ** 62 ? value : 0;
640
- }
641
-
642
621
  /** The program whose text lengths `textLengths` holds. */
643
622
  let lengthsProgram: Program | undefined = undefined;
644
623
  let lengthsOfProgram: number[] = [];
@@ -664,6 +643,27 @@ export function textLengths(program: Program): number[] {
664
643
  * or an estimate from its text where that is unknown.
665
644
  */
666
645
  export function memoryCheckers(program: Program): number {
646
+ const estimate = memoryEstimate(program);
647
+ const fit = checkersFittingMemory(estimate.programBytes, estimate.declarationChars, estimate.memory);
648
+ if (process.env.TSC_TS_CHECKERS_TRACE !== undefined)
649
+ process.stderr.write(
650
+ `checkers: ${estimate.chars} source chars (${estimate.declarationChars} in declaration files), program ${Math.round(estimate.programBytes / 1048576)}MB${estimate.resident > 0 ? '' : ' (estimated)'}, memory ${Math.round(estimate.memory / 1048576)}MB fits ${fit} checkers\n`,
651
+ );
652
+ return fit;
653
+ }
654
+
655
+ /** What the memory fit reads of a program: its text, the declarations in it, and this process's memory. */
656
+ interface MemoryEstimate {
657
+ readonly chars: number;
658
+ readonly declarationChars: number;
659
+ readonly programBytes: number;
660
+ /** Whether `programBytes` is the resident set (`0` where it is an estimate from the text). */
661
+ readonly resident: number;
662
+ readonly memory: number;
663
+ }
664
+
665
+ /** The memory estimate of `program`, of its text (`textLengths`, computed once) and this process's memory. */
666
+ function memoryEstimate(program: Program): MemoryEstimate {
667
667
  let chars = 0;
668
668
  let declarationChars = 0;
669
669
  const files = program.getSourceFiles();
@@ -673,14 +673,24 @@ export function memoryCheckers(program: Program): number {
673
673
  if (files[i]!.isDeclarationFile) declarationChars += length[i]!;
674
674
  }
675
675
  const resident = residentBytes();
676
- const programBytes = resident > 0 ? resident : chars * programBytesPerChar;
677
- const memory = memoryLimit();
678
- const fit = checkersFittingMemory(programBytes, declarationChars, memory);
679
- if (process.env.TSC_TS_CHECKERS_TRACE !== undefined)
680
- process.stderr.write(
681
- `checkers: ${chars} source chars (${declarationChars} in declaration files), program ${Math.round(programBytes / 1048576)}MB${resident > 0 ? '' : ' (estimated)'}, memory ${Math.round(memory / 1048576)}MB fits ${fit} checkers\n`,
682
- );
683
- return fit;
676
+ return {
677
+ chars,
678
+ declarationChars,
679
+ programBytes: resident > 0 ? resident : chars * programBytesPerChar,
680
+ resident,
681
+ memory: memoryLimit(),
682
+ };
683
+ }
684
+
685
+ /**
686
+ * The checkers whose estimated peak memory fits `program` (`checkersFittingMemory`), without sharing and
687
+ * with the peak sharing adds (`sharingPeakShare`), which the sharing rule requires of the count it keeps.
688
+ */
689
+ export function memoryFit(program: Program): CheckFit {
690
+ const estimate = memoryEstimate(program);
691
+ return {
692
+ sharing: checkersFittingMemory(estimate.programBytes, estimate.declarationChars, estimate.memory, sharingPeakShare),
693
+ };
684
694
  }
685
695
 
686
696
  /**
@@ -1464,10 +1474,13 @@ export function planSchedule(
1464
1474
  }
1465
1475
  let all = 0;
1466
1476
  for (const i of order) all += cost[i]!;
1467
- // Files that cost more than a 32nd of a checker's share go first, costliest first: at the end of
1468
- // a range they would make its checker finish last.
1477
+ // Files that cost more than a 128th of the program go first, costliest first: at the end of a range
1478
+ // they would make its checker finish last. A fixed fraction of the program, rather than of a
1479
+ // checker's share, keeps the cut in proportion to how many checkers drain the shared queue: it is a
1480
+ // 32nd of a share at 4 checkers and an eighth at 16, where a range is shorter and a file of the
1481
+ // same size would otherwise sit in it.
1469
1482
  let heavyCount = count;
1470
- while (heavyCount < order.length && cost[order[heavyCount]!]! * count * 32 >= all) heavyCount++;
1483
+ while (heavyCount < order.length && cost[order[heavyCount]!]! * 128 >= all) heavyCount++;
1471
1484
  // The groups go to the shared queue as one entry each, among the costliest files, by total cost.
1472
1485
  const groups = checkGroups(checked, count, groupLoad, costs);
1473
1486
  const grouped = new Set<number>();
@@ -1650,9 +1663,13 @@ function threadSharingOf(
1650
1663
  order: readonly number[],
1651
1664
  units: readonly number[],
1652
1665
  files: number,
1666
+ bounded: boolean,
1653
1667
  ): ThreadSharingData[] {
1654
1668
  const takers = new SharedArrayBuffer(4 * files);
1655
1669
  const forbidden = new SharedArrayBuffer(4 * files * forbiddenWords(count));
1670
+ // The checkers that missed on a key in each file, which a post requires (`ThreadSharing.demanded`). An unbounded
1671
+ // run posts as Phase 1 does and never reads it, so it gets a word rather than a bit per checker per file.
1672
+ const wanted = bounded ? new SharedArrayBuffer(4 * files * forbiddenWords(count)) : new SharedArrayBuffer(4);
1656
1673
  const handoff = new SharedArrayBuffer(4 * (3 + order.length));
1657
1674
  new Int32Array(handoff)[2] = count;
1658
1675
  const taken = new Int32Array(takers);
@@ -1670,17 +1687,23 @@ function threadSharingOf(
1670
1687
  const stringLogs: SharedArrayBuffer[] = [];
1671
1688
  for (let i = 0; i < count; i++) {
1672
1689
  numberLogs.push(new SharedArrayBuffer(4 * numberLogSize));
1673
- recordTables.push(new SharedArrayBuffer(8 * recordTableSize));
1690
+ recordTables.push(new SharedArrayBuffer(4 * recordTableSize));
1674
1691
  stringLogs.push(new SharedArrayBuffer(2 * stringLogSize));
1675
1692
  }
1676
- const used = new SharedArrayBuffer(8 * count);
1693
+ const used = new SharedArrayBuffer(4 * usedStride * count);
1694
+ // The stop switch (0 runs, 1 stops posting, 2 stops posting and translating) and the counters the
1695
+ // main thread reads at its window, a row per checker.
1696
+ const control = new SharedArrayBuffer(4);
1697
+ const stats = new SharedArrayBuffer(8 * shareStatColumns * count);
1677
1698
  const data: ThreadSharingData[] = [];
1678
1699
  for (let i = 0; i < count; i++)
1679
1700
  data.push({
1680
1701
  index: i,
1681
1702
  count,
1703
+ bounded,
1682
1704
  takers,
1683
1705
  forbidden,
1706
+ wanted,
1684
1707
  handoff,
1685
1708
  typeTable,
1686
1709
  returnTable,
@@ -1690,6 +1713,8 @@ function threadSharingOf(
1690
1713
  recordTables,
1691
1714
  stringLogs,
1692
1715
  used,
1716
+ control,
1717
+ stats,
1693
1718
  });
1694
1719
  return data;
1695
1720
  }
@@ -1920,50 +1945,179 @@ export function mainThreadChecks(shared: boolean): boolean {
1920
1945
  }
1921
1946
 
1922
1947
  /**
1923
- * When a checker's own files are a small part of what it resolves (`libraryBoundCheckers`): the least
1924
- * text, in characters, of the libraries the files of a checker's range reach, on average over the
1925
- * ranges, and the least multiple of the text of the range's own files it is.
1948
+ * Whether the run reads the memory fit of sharing (`checkDecisionOf`): the static rule reads it where
1949
+ * the signals point at sharing, and a run the learning rules read may adopt sharing the static rule
1950
+ * leaves off, or explore it, when its samples show the configuration faster (DESIGN 4.3-4.4).
1951
+ *
1952
+ * A pin on sharing chooses sharing without a rule, but where it pins sharing on the learning rules still
1953
+ * compare checker counts with it, and every count they may adopt reads how many fit memory with sharing's
1954
+ * peak: a pinned count that no rule can change, and `TSC_TS_AUTO=0`, read nothing.
1955
+ */
1956
+ export function sharingFitNeeded(config: RunConfig, signals: CheckSignals): boolean {
1957
+ if (config.automaticity === Automaticity.Off) return false;
1958
+ if (config.shareSource === Source.Pinned) {
1959
+ return (
1960
+ config.share &&
1961
+ config.checkersSource !== Source.Pinned &&
1962
+ learns(config.automaticity) &&
1963
+ config.record !== undefined
1964
+ );
1965
+ }
1966
+ if (sharingCandidate(signals)) return true;
1967
+ return learns(config.automaticity) && config.record !== undefined;
1968
+ }
1969
+
1970
+ /**
1971
+ * What `decideCheck` needs of the run besides its signals: whether the checker threads of a shared
1972
+ * program may share results (`startParallelCheck` makes the same test), and, where sharing is possible
1973
+ * and a rule that may choose it reads the estimate, how many checkers fit memory with sharing's peak
1974
+ * (`sharingFitNeeded`). The estimate costs a pass over the program's files and its memory limit, so it
1975
+ * is computed only where a rule reads it: a run without a record pays only where the static rule's
1976
+ * signals point at sharing, and one checker, a program that is not shared, a pinned run or
1977
+ * `TSC_TS_AUTO=0` pay nothing.
1978
+ */
1979
+ function checkDecisionOf(config: RunConfig, signals: CheckSignals, program: Program, shared: boolean): CheckDecision {
1980
+ // `mainThreadChecks` is false whenever the program is shared (`!shared`), as sharing needs.
1981
+ const possible = shared && signals.count > 1 && signals.count <= maxSharingCheckers;
1982
+ const fit = possible && sharingFitNeeded(config, signals) ? memoryFit(program) : undefined;
1983
+ // A run whose record the learning rules read also names the shape its samples describe, the CPUs a
1984
+ // count may use, the memory a configuration may not exceed and the time a control sample ages from.
1985
+ const learning = config.records || config.record !== undefined;
1986
+ return {
1987
+ possible,
1988
+ fit,
1989
+ shape: learning ? recordShapeOf(program, signals) : undefined,
1990
+ cpus: Math.min(availableCpus(), maxAutoCheckers),
1991
+ memoryBytes: memoryLimit(),
1992
+ now: Math.floor(Date.now() / 1000),
1993
+ };
1994
+ }
1995
+
1996
+ /** The program's shape for its record (`RecordShape`), with the shares the sharing rule reads as `F`. */
1997
+ function recordShapeOf(program: Program, signals: CheckSignals): RecordShape {
1998
+ const files = program.getSourceFiles();
1999
+ let chars = 0;
2000
+ let declarationChars = 0;
2001
+ for (const file of files) {
2002
+ chars += file.end;
2003
+ if (file.isDeclarationFile) declarationChars += file.end;
2004
+ }
2005
+ return {
2006
+ files: files.length,
2007
+ chars,
2008
+ declarationChars,
2009
+ checkChars: chars - declarationChars,
2010
+ library: Math.round(signals.library),
2011
+ own: Math.round(signals.own),
2012
+ redundancy: signals.redundancy,
2013
+ fileShare: Math.max(signals.declarationShare, signals.nodeModulesShare),
2014
+ };
2015
+ }
2016
+
2017
+ /** What the counters of `count` checkers say at the sharing window: milliseconds saved and spent. */
2018
+ interface SharingNumbers {
2019
+ /** What taking posted results saved, over the checkers: the computations they did not repeat. */
2020
+ readonly saved: number;
2021
+ /**
2022
+ * What posting cost: exporting and translating results. Translating a result reads its records in
2023
+ * place inside `ThreadSharing.translatePosted`'s timed translation, so `translationMs` holds that
2024
+ * reading too; counting a separate unpacking time would charge the window for it twice.
2025
+ */
2026
+ readonly spent: number;
2027
+ readonly posted: number;
2028
+ readonly translated: number;
2029
+ }
2030
+
2031
+ /**
2032
+ * The counters of `count` checkers' rows of `stats` (`ThreadSharing.writeStats`), summed. The tests read
2033
+ * this, next to the window it feeds.
2034
+ */
2035
+ export function sharingNumbers(stats: Float64Array, count: number): SharingNumbers {
2036
+ let saved = 0;
2037
+ let spent = 0;
2038
+ let posted = 0;
2039
+ let translated = 0;
2040
+ for (let i = 0; i < count; i++) {
2041
+ const row = i * shareStatColumns;
2042
+ saved += stats[row + shareStatSaved]!;
2043
+ spent += stats[row + shareStatExport]! + stats[row + shareStatTranslation]!;
2044
+ posted += stats[row + shareStatPosted]!;
2045
+ translated += stats[row + shareStatTranslated]!;
2046
+ }
2047
+ return { saved, spent, posted, translated };
2048
+ }
2049
+
2050
+ /**
2051
+ * The control a sharing run's numbers ask for: 2 when it saved less than 1.5 times what it spent, 1
2052
+ * when the posts go untaken (under 5% of them, and at least one), else 0. Stopping takes a minute at
2053
+ * worst, when the window's numbers are still too early to tell.
1926
2054
  */
1927
- const libraryBoundText = 6_000_000;
1928
- const libraryBoundShare = 5;
2055
+ export function stopLevel(numbers: SharingNumbers): number {
2056
+ if (numbers.saved < 1.5 * numbers.spent) return 2;
2057
+ if (numbers.posted > 0 && numbers.translated / numbers.posted < 0.05) return 1;
2058
+ return 0;
2059
+ }
1929
2060
 
1930
2061
  /**
1931
- * The number of checkers for `count` checkers whose ranges reach `library` characters of libraries
1932
- * and check `own` characters each, on average: three quarters of them (tsgo's four at least) where
1933
- * every checker spends most of its time resolving the same large libraries (`libraryBoundText`,
1934
- * `libraryBoundShare`), else `count`.
2062
+ * Whether the rules of a run of `config` at `count` checkers read the check signals (`checkSignals`).
2063
+ * Two rules read them:
1935
2064
  *
1936
- * Each checker resolves the declarations its files reach, whatever else it checks. Where they are
1937
- * millions of characters, such as an Effect-based project's, and much more than the checker's own
1938
- * files, a checker costs seconds of CPU and hundreds of MB before its files get cheap, and the last
1939
- * quarter of the checkers (on 16 CPUs of 8 cores, the ones sharing a core with another) repeat that
1940
- * work rather than save time: T3 Code's web and server projects ran 3% and 1% faster with 12
1941
- * checkers than with 16, and used 16% and 8% less memory. Where checkers resolve less, or mostly
1942
- * check their own files, 12 of 16 checkers were slower: Excalidraw (1.4M library and 0.15M own
1943
- * characters per range) by 6%, Sentry (5.7M and 3.0M) and the tsc-ts sources by about 11%.
2065
+ * - the library-bound cut (`decideCheck`), wherever the count is not pinned and is above tsgo's four.
2066
+ * This is the decision a run made before the automatic selection, and `TSC_TS_AUTO=0` keeps it: with
2067
+ * selection off the count is not cut for sharing either, but dropping the signals would leave the
2068
+ * cut nothing to read and start checkers the cut would not;
2069
+ * - the sharing rule, at `minSharingCheckers` or more, where neither the count nor sharing is pinned
2070
+ * (the cut may leave the count sharing was measured at, so the signals are read before it).
2071
+ *
2072
+ * A pinned count and a pinned sharing read neither, and pay nothing for the pass over the ranges.
1944
2073
  */
1945
- export function libraryBoundCount(count: number, library: number, own: number): number {
1946
- return library >= libraryBoundText && library >= libraryBoundShare * own
1947
- ? Math.max(tsgoCheckers, Math.ceil((count * 3) / 4))
1948
- : count;
2074
+ export function signalsNeeded(config: RunConfig, count: number): boolean {
2075
+ const cut = config.checkersSource === Source.Default && count > tsgoCheckers;
2076
+ if (config.automaticity === Automaticity.Off) return cut;
2077
+ return cut || (config.shareSource === Source.Default && count >= minSharingCheckers);
1949
2078
  }
1950
2079
 
1951
2080
  /**
1952
- * The number of checkers for `plan`, made with `count` checkers that `auto` chose
1953
- * (`libraryBoundCount` of what its ranges reach and check).
2081
+ * What `plan`, made with `count` checkers, shows about the cost of checking (`CheckSignals`): the
2082
+ * libraries a range reaches and the files it checks, the share of that a second range reaches too, and
2083
+ * the shares of declaration and node_modules files of the program. Computed where a rule may read them
2084
+ * (`signalsNeeded`) and left at zero elsewhere: a pinned count needs none of them, and the pass over
2085
+ * every range's libraries costs a millisecond on a large program.
1954
2086
  */
1955
- function libraryBoundCheckers(checked: CheckedProgram, plan: CheckPlan, costs: CheckCosts, count: number): number {
1956
- if (checkerPolicy !== 'auto' || checked.program.options.checkers !== undefined || count <= tsgoCheckers) return count;
2087
+ function checkSignals(
2088
+ checked: CheckedProgram,
2089
+ plan: CheckPlan,
2090
+ costs: CheckCosts,
2091
+ count: number,
2092
+ config: RunConfig | undefined = undefined,
2093
+ ): CheckSignals {
2094
+ const ranges = plan.ranges.length >> 1;
2095
+ const signals = new CheckSignals(count, ranges);
1957
2096
  const program = checked.program;
2097
+ // What a rule would read, what the run's record writes (`config.records`), or what a doctor child
2098
+ // reports (`config.measure`), is computed; a pinned count with records off and no report pays nothing.
2099
+ const deciding =
2100
+ config === undefined
2101
+ ? checkersPolicy(process.env) === 'auto' && program.options.checkers === undefined && count > tsgoCheckers
2102
+ : signalsNeeded(config, count);
2103
+ const wanted = deciding || (config !== undefined && (config.records || config.measure));
2104
+ if (!wanted || ranges === 0) return signals;
1958
2105
  const files = program.getSourceFiles();
1959
2106
  const length = textLengths(program);
1960
2107
  const isChecked: boolean[] = [];
1961
- for (let i = 0; i < files.length; i++) isChecked.push(false);
2108
+ let declarationFiles = 0;
2109
+ let nodeModulesFiles = 0;
2110
+ for (let i = 0; i < files.length; i++) {
2111
+ isChecked.push(false);
2112
+ if (files[i]!.isDeclarationFile) declarationFiles++;
2113
+ if (files[i]!.path.indexOf('/node_modules/') >= 0) nodeModulesFiles++;
2114
+ }
1962
2115
  for (const i of costs.order) isChecked[i] = true;
1963
- const ranges = plan.ranges.length >> 1;
1964
- if (ranges === 0) return count;
1965
2116
  let library = 0;
1966
2117
  let own = 0;
2118
+ // The libraries every range reaches together, each counted once (`CheckSignals.redundancy`).
2119
+ const union = new Set<number>();
2120
+ let unionChars = 0;
1967
2121
  for (let r = 0; r < ranges; r++) {
1968
2122
  // The libraries the range's files import, and what they reach through other libraries (as checkCosts counts them).
1969
2123
  const reached = new Set<number>();
@@ -1980,6 +2134,10 @@ function libraryBoundCheckers(checked: CheckedProgram, plan: CheckPlan, costs: C
1980
2134
  while (stack.length > 0) {
1981
2135
  const k = stack.pop()!;
1982
2136
  library += length[k]!;
2137
+ if (!union.has(k)) {
2138
+ union.add(k);
2139
+ unionChars += length[k]!;
2140
+ }
1983
2141
  for (const t of costs.imports[k] ?? [])
1984
2142
  if (
1985
2143
  !reached.has(t) &&
@@ -1992,18 +2150,22 @@ function libraryBoundCheckers(checked: CheckedProgram, plan: CheckPlan, costs: C
1992
2150
  }
1993
2151
  }
1994
2152
  }
1995
- const fewer = libraryBoundCount(count, library / ranges, own / ranges);
2153
+ signals.library = library / ranges;
2154
+ signals.own = own / ranges;
2155
+ signals.redundancy = (library - unionChars) / (library + own);
2156
+ signals.declarationShare = declarationFiles / files.length;
2157
+ signals.nodeModulesShare = nodeModulesFiles / files.length;
2158
+ if (config !== undefined) config.signals = signals;
1996
2159
  if (process.env.TSC_TS_CHECKERS_TRACE !== undefined)
1997
2160
  process.stderr.write(
1998
- `checkers: ranges reach ${Math.round(library / ranges / 1000)}K library chars and check ${Math.round(own / ranges / 1000)}K on average: ${fewer} checkers\n`,
2161
+ `checkers: ranges reach ${Math.round(signals.library / 1000)}K library chars and check ${Math.round(signals.own / 1000)}K on average, ${Math.round(signals.redundancy * 100)}% of it reached twice: ${libraryBoundCount(count, signals.library, signals.own)} checkers\n`,
1999
2162
  );
2000
- return fewer;
2163
+ return signals;
2001
2164
  }
2002
2165
 
2003
2166
  /**
2004
- * Checks `checked` with `checkers` checkers (fewer where `auto` chose them and they would mostly
2005
- * resolve the same libraries: `libraryBoundCheckers`) and calls `done` with the wall time once
2006
- * `checked` holds every file's diagnostics.
2167
+ * Checks `checked` with `checkers` checkers (fewer where the configuration chooses so: `decideCheck`,
2168
+ * run-config.ts) and calls `done` with the wall time once `checked` holds every file's diagnostics.
2007
2169
  */
2008
2170
  /**
2009
2171
  * Shares the program of `checked` with checker threads (scriptc builds): this thread's checker merges the
@@ -2037,7 +2199,8 @@ export function shareWithCheckers(checked: CheckedProgram): void {
2037
2199
  * `emitsTo`, the emit results go only there, by file index, and nothing else the threads find reaches
2038
2200
  * the program: a declaration thread (parallel-emit.ts), while this thread's checker checks. With
2039
2201
  * `declarationsTo`, the same holds for the declaration errors of a `noEmit` program
2040
- * (`checkersReportDeclarations`).
2202
+ * (`checkersReportDeclarations`). `config` is the run's configuration: the count it chose is decided
2203
+ * here, once the program is bound and the plan made, and sharing is what it holds.
2041
2204
  */
2042
2205
  export function startParallelCheck(
2043
2206
  checked: CheckedProgram,
@@ -2048,6 +2211,7 @@ export function startParallelCheck(
2048
2211
  checking: boolean = true,
2049
2212
  emitsTo: (EmitResult | undefined)[] | undefined = undefined,
2050
2213
  declarationsTo: (Diagnostic[] | undefined)[] | undefined = undefined,
2214
+ config: RunConfig | undefined = undefined,
2051
2215
  ): void {
2052
2216
  let count = checkers;
2053
2217
  const start = performance.now();
@@ -2067,7 +2231,12 @@ export function startParallelCheck(
2067
2231
  const orderStart = performance.now();
2068
2232
  const costs = costOrder(checked);
2069
2233
  let plan = planSchedule(checked, count, costs, checking);
2070
- const fewer = libraryBoundCheckers(checked, plan, costs, count);
2234
+ const signals = checkSignals(checked, plan, costs, count, config);
2235
+ if (config !== undefined) config.signals = signals;
2236
+ const fewer =
2237
+ config === undefined
2238
+ ? libraryBoundCount(signals.count, signals.library, signals.own)
2239
+ : decideCheck(config, signals, checkDecisionOf(config, signals, program, shared));
2071
2240
  if (fewer < count) {
2072
2241
  count = fewer;
2073
2242
  plan = planSchedule(checked, count, costs, checking);
@@ -2109,7 +2278,13 @@ export function startParallelCheck(
2109
2278
  const firstThread = mainChecks ? 1 : 0;
2110
2279
  let remaining = count - firstThread;
2111
2280
  let ownDone = false;
2281
+ /** The sharing window's timer, where the threads share results; cleared when they are done. */
2282
+ let shareStopTimer: ReturnType<typeof setTimeout> | undefined = undefined;
2112
2283
  const finish = (): void => {
2284
+ if (shareStopTimer !== undefined) {
2285
+ clearTimeout(shareStopTimer);
2286
+ shareStopTimer = undefined;
2287
+ }
2113
2288
  if (emitsTo !== undefined || declarationsTo !== undefined) {
2114
2289
  mark('emitted on checker threads');
2115
2290
  done(performance.now() - start);
@@ -2188,11 +2363,43 @@ export function startParallelCheck(
2188
2363
  if (remaining === 0 && ownDone) finish();
2189
2364
  };
2190
2365
  const diagnosticFiles = new Map<string, DiagnosticFile>();
2191
- // TSC_TS_SHARE: checker threads share results (shared-results.ts).
2366
+ // Whether checker threads share results (shared-results.ts), which the run's configuration decides;
2367
+ // the emit threads of parallel-emit.ts have no configuration and never share (count 1).
2368
+ const sharesResults = config === undefined ? shareThreads === true : config.share;
2192
2369
  const sharing =
2193
- shareThreads && shared && !mainChecks && count > 1 && count <= maxSharingCheckers
2194
- ? threadSharingOf(count, order, units, files.length)
2370
+ sharesResults && shared && !mainChecks && count > 1 && count <= maxSharingCheckers
2371
+ ? threadSharingOf(count, order, units, files.length, config !== undefined && config.bounded)
2195
2372
  : undefined;
2373
+ if (sharing !== undefined) {
2374
+ // The main thread is idle while the checkers run, so a timer reads their counters at the window and
2375
+ // stops what does not pay (control 1 or 2); every checker computes what it does not take, so the
2376
+ // diagnostics stay the same.
2377
+ const control = new Int32Array(sharing[0]!.control);
2378
+ const stats = new Float64Array(sharing[0]!.stats);
2379
+ const stop = shareStop;
2380
+ const at = stop !== undefined ? stop.at : config !== undefined ? config.shareWindowMs : shareWindowMs;
2381
+ shareStopTimer = setTimeout(() => {
2382
+ const numbers = sharingNumbers(stats, count);
2383
+ if (stop !== undefined) Atomics.store(control, 0, stop.level);
2384
+ else {
2385
+ const level = stopLevel(numbers);
2386
+ if (level === 0) return;
2387
+ Atomics.store(control, 0, level);
2388
+ }
2389
+ // The section says what the window saw, instead of the rule that turned sharing on.
2390
+ if (config !== undefined) {
2391
+ config.shareSource = Source.Measured;
2392
+ config.shareReasons.length = 0;
2393
+ config.shareReasons.push(
2394
+ `stopped after ${at} ms: ${Math.round(numbers.saved)} ms saved for ${Math.round(numbers.spent)} ms spent`,
2395
+ );
2396
+ }
2397
+ if (trace)
2398
+ process.stderr.write(
2399
+ `checkers: stopped sharing after ${at} ms: ${Math.round(numbers.saved)} ms saved for ${Math.round(numbers.spent)} ms spent, ${numbers.translated}/${numbers.posted} posted results taken\n`,
2400
+ );
2401
+ }, at);
2402
+ }
2196
2403
  const sharingStarted: boolean[] = [];
2197
2404
  for (let i = 0; i < count; i++) sharingStarted.push(false);
2198
2405
  const startSharingOf = (i: number): ThreadSharingData => {
@@ -2232,7 +2439,7 @@ export function startParallelCheck(
2232
2439
  });
2233
2440
  worker.on('error', (error: unknown) => {
2234
2441
  // Sharing results (TSC_TS_SHARE) is still checked for such failures, which the main thread hides otherwise.
2235
- if (trace || shareThreads) process.stderr.write(`checkers: thread ${i} failed: ${String(error)}\n`);
2442
+ if (trace || sharesResults) process.stderr.write(`checkers: thread ${i} failed: ${String(error)}\n`);
2236
2443
  threadDone();
2237
2444
  });
2238
2445
  };