tsc-ts 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/compiler/ast.d.ts +9 -3
- package/dist/api/compiler/ast.js +17 -12
- package/dist/api/compiler/checker-classes.js +2 -1
- package/dist/api/compiler/checker-contextual.d.ts +20 -0
- package/dist/api/compiler/checker-contextual.js +111 -1
- package/dist/api/compiler/checker-expressions.js +14 -1
- package/dist/api/compiler/checker-jsx.js +2 -1
- package/dist/api/compiler/checker-type-factory.js +53 -12
- package/dist/api/compiler/checker.d.ts +5 -1
- package/dist/api/compiler/checker.js +10 -4
- package/dist/api/compiler/compile-program.d.ts +5 -0
- package/dist/api/compiler/compile-program.js +8 -0
- package/dist/api/compiler/expr-record.d.ts +11 -0
- package/dist/api/compiler/expr-record.js +53 -0
- package/dist/api/compiler/identifier-name.d.ts +9 -0
- package/dist/api/compiler/identifier-name.js +10 -0
- package/dist/api/compiler/jsdoc.js +4 -2
- package/dist/api/compiler/parser.d.ts +2 -1
- package/dist/api/compiler/parser.js +10 -8
- package/dist/api/compiler/printer/factory.js +3 -2
- package/dist/api/compiler/printer/printer.js +3 -2
- package/dist/api/compiler/reparser.js +4 -3
- package/dist/api/compiler/scanner.d.ts +8 -0
- package/dist/api/compiler/scanner.js +58 -42
- package/dist/api/compiler/shared-results.d.ts +376 -143
- package/dist/api/compiler/shared-results.js +1550 -773
- package/dist/api/compiler/transformers/declarations.js +2 -1
- package/package.json +9 -9
- package/src/compiler/ast.ts +19 -12
- package/src/compiler/cgroup.ts +157 -0
- package/src/compiler/checker-classes.ts +2 -1
- package/src/compiler/checker-contextual.ts +104 -0
- package/src/compiler/checker-expressions.ts +16 -1
- package/src/compiler/checker-jsx.ts +2 -1
- package/src/compiler/checker-type-factory.ts +53 -14
- package/src/compiler/checker.ts +11 -4
- package/src/compiler/compile-program.ts +8 -0
- package/src/compiler/decoder.ts +2 -1
- package/src/compiler/doctor-report.ts +133 -0
- package/src/compiler/expr-record.ts +59 -0
- package/src/compiler/identifier-name.ts +13 -0
- package/src/compiler/jsdoc.ts +4 -2
- package/src/compiler/parallel-check.ts +353 -146
- package/src/compiler/parallel-emit.ts +3 -3
- package/src/compiler/parser.ts +10 -8
- package/src/compiler/printer/factory.ts +3 -2
- package/src/compiler/printer/printer.ts +4 -2
- package/src/compiler/reparser.ts +7 -3
- package/src/compiler/run-config.ts +1383 -0
- package/src/compiler/run-record-read.ts +87 -0
- package/src/compiler/run-records.ts +575 -0
- package/src/compiler/scanner.ts +57 -41
- package/src/compiler/shared-results.ts +1626 -896
- package/src/compiler/thread-entry.ts +3 -1
- package/src/compiler/transformers/declarations.ts +2 -1
- package/src/compiler/tsc-build.ts +64 -6
- package/src/compiler/tsc-cli.ts +33 -4
- package/src/compiler/tsc-compile.ts +213 -8
- package/src/compiler/tsc-watch.ts +25 -4
|
@@ -50,8 +50,9 @@ import type { Program } from './program.js';
|
|
|
50
50
|
import type { SourceFile } from './ast.js';
|
|
51
51
|
import { lastSymbolIdValue } from './symbol.js';
|
|
52
52
|
import { shareMergedGlobals } from './checker-symbols.js';
|
|
53
|
+
import { availableCpus, cgroupMemoryLimit } from './cgroup.js';
|
|
53
54
|
import { readFileSync } from 'node:fs';
|
|
54
|
-
import {
|
|
55
|
+
import { totalmem } from 'node:os';
|
|
55
56
|
import {
|
|
56
57
|
forbiddenWords,
|
|
57
58
|
isForbidden,
|
|
@@ -59,10 +60,68 @@ import {
|
|
|
59
60
|
numberLogSize,
|
|
60
61
|
recordTableSize,
|
|
61
62
|
resultTableSize,
|
|
62
|
-
|
|
63
|
+
shareStatColumns,
|
|
64
|
+
shareStatExport,
|
|
65
|
+
shareStatPosted,
|
|
66
|
+
shareStatSaved,
|
|
67
|
+
shareStatTranslation,
|
|
68
|
+
shareStatTranslated,
|
|
63
69
|
stringLogSize,
|
|
70
|
+
usedStride,
|
|
64
71
|
type ThreadSharingData,
|
|
65
72
|
} from './shared-results.js';
|
|
73
|
+
import {
|
|
74
|
+
autoCheckers,
|
|
75
|
+
Automaticity,
|
|
76
|
+
CheckSignals,
|
|
77
|
+
checkersPolicy,
|
|
78
|
+
decideCheck,
|
|
79
|
+
learns,
|
|
80
|
+
libraryBoundCount,
|
|
81
|
+
maxAutoCheckers,
|
|
82
|
+
minSharingCheckers,
|
|
83
|
+
sharePin,
|
|
84
|
+
sharingCandidate,
|
|
85
|
+
sharingPeakShare,
|
|
86
|
+
Source,
|
|
87
|
+
tsgoCheckers,
|
|
88
|
+
type CheckDecision,
|
|
89
|
+
type CheckFit,
|
|
90
|
+
type RunConfig,
|
|
91
|
+
} from './run-config.js';
|
|
92
|
+
import type { RecordShape } from './run-records.js';
|
|
93
|
+
|
|
94
|
+
// The tests of the scheduler read these here, next to the plans they choose between.
|
|
95
|
+
export { autoCheckers, libraryBoundCount };
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* `TSC_TS_SHARE` for a run without a run configuration (the emit threads, which share no program to
|
|
99
|
+
* translate against): `1` shares, `0` does not, and unset shares nothing (`sharePin`).
|
|
100
|
+
*/
|
|
101
|
+
const shareThreads = sharePin(process.env);
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* TSC_TS_SHARE_STOP_AT_MS: `t` or `t:level` stops a sharing run's sharing `t` milliseconds in, whatever
|
|
105
|
+
* the run's numbers say, to check the switch: `2` (the default) stops posting and translating, `1`
|
|
106
|
+
* stops posting. Testing only.
|
|
107
|
+
*/
|
|
108
|
+
function shareStopHook(): { readonly at: number; readonly level: number } | undefined {
|
|
109
|
+
const value = process.env.TSC_TS_SHARE_STOP_AT_MS;
|
|
110
|
+
if (value === undefined || value === '') return undefined;
|
|
111
|
+
const colon = value.indexOf(':');
|
|
112
|
+
const at = Number(colon < 0 ? value : value.substring(0, colon));
|
|
113
|
+
if (!Number.isFinite(at) || at < 0) return undefined;
|
|
114
|
+
return { at, level: colon >= 0 && value.substring(colon + 1) === '1' ? 1 : 2 };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const shareStop = shareStopHook();
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The window a run without a configuration uses before the main thread weighs what sharing saved against
|
|
121
|
+
* what it cost (`sharingNumbers`): 300 ms. A run with one uses the larger of 200 ms and a tenth of the
|
|
122
|
+
* check time its record measured (`RunConfig.shareWindowMs`, `shareWindowOf`).
|
|
123
|
+
*/
|
|
124
|
+
const shareWindowMs = 300;
|
|
66
125
|
|
|
67
126
|
/** With TSC_TS_CHECKERS_TRACE: the process's resident and peak resident set in MB (Linux), else 0. */
|
|
68
127
|
export function residentMB(): number[] {
|
|
@@ -454,23 +513,9 @@ function checkersEmit(checked: CheckedProgram, emits: boolean): boolean {
|
|
|
454
513
|
return emits && parallelCheck.writer !== undefined && options.noEmit !== true && options.listFilesOnly !== true;
|
|
455
514
|
}
|
|
456
515
|
|
|
457
|
-
/**
|
|
458
|
-
* How the number of checkers is chosen without `--checkers`: `auto` (the default) uses every CPU
|
|
459
|
-
* this process may run on (`os.availableParallelism()`, at most `maxAutoCheckers`), and at least
|
|
460
|
-
* tsgo's four; fewer than the CPUs, but never fewer than four, when the estimated peak memory of more
|
|
461
|
-
* checkers would not fit (`memoryCheckers`). `fixed` (TSC_TS_CHECKERS=fixed) uses tsgo's four.
|
|
462
|
-
*/
|
|
463
|
-
export const checkerPolicy: 'fixed' | 'auto' = process.env.TSC_TS_CHECKERS === 'fixed' ? 'fixed' : 'auto';
|
|
464
|
-
|
|
465
|
-
/** tsgo's number of checkers, and the fewest `auto` starts. */
|
|
466
|
-
const tsgoCheckers = 4;
|
|
467
|
-
|
|
468
|
-
/** The most checkers `auto` starts (measured up to 16 CPUs). */
|
|
469
|
-
const maxAutoCheckers = 16;
|
|
470
|
-
|
|
471
516
|
/**
|
|
472
517
|
* The number of checker threads for `checkers` (`--checkers`), or 1 to check on this thread. Without
|
|
473
|
-
* the option, native builds follow
|
|
518
|
+
* the option, native builds follow the checker policy (with `program`, the memory estimate of `auto`
|
|
474
519
|
* applies); under Node.js, where every thread builds its own copy of the program, tsc-ts checks on
|
|
475
520
|
* one thread.
|
|
476
521
|
*/
|
|
@@ -487,8 +532,8 @@ export function checkerThreads(
|
|
|
487
532
|
function defaultCheckers(program: Program | undefined): number {
|
|
488
533
|
if (!sharesPublishedGraphs) return 1;
|
|
489
534
|
if (program !== undefined && checksLittle(program)) return 1;
|
|
490
|
-
if (
|
|
491
|
-
const cpus = Math.min(
|
|
535
|
+
if (checkersPolicy(process.env) === 'fixed') return tsgoCheckers;
|
|
536
|
+
const cpus = Math.min(availableCpus(), maxAutoCheckers);
|
|
492
537
|
return autoCheckers(cpus, program === undefined ? Number.MAX_SAFE_INTEGER : memoryCheckers(program));
|
|
493
538
|
}
|
|
494
539
|
|
|
@@ -514,24 +559,6 @@ function checksLittle(program: Program): boolean {
|
|
|
514
559
|
return true;
|
|
515
560
|
}
|
|
516
561
|
|
|
517
|
-
/** `auto` with `cpus` CPUs when the memory estimate fits `fit` checkers: tsgo's four at least. */
|
|
518
|
-
export function autoCheckers(cpus: number, fit: number): number {
|
|
519
|
-
return Math.max(tsgoCheckers, Math.min(cpus, fit));
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
/**
|
|
523
|
-
* The number of threads that parse ahead of a program checked by `checkers` checkers: every CPU this
|
|
524
|
-
* process may run on, as tsgo parses on every core whatever its checker count (parse threads hand
|
|
525
|
-
* their files to the shared program; what else they allocate is reused by the checker threads), but
|
|
526
|
-
* one. The main thread hands out the chunks they load: with a thread on every CPU, it waits for a CPU
|
|
527
|
-
* each time a thread returns a chunk, and so does that thread for its next chunk (VS Code's parse took
|
|
528
|
-
* 18-57 ms longer with 16 threads on 16 CPUs than with 15).
|
|
529
|
-
*/
|
|
530
|
-
export function parseThreads(checkers: number): number {
|
|
531
|
-
const cpus = availableParallelism();
|
|
532
|
-
return Math.max(1, Math.min(cpus - 1, Math.max(checkers, Math.min(cpus, maxAutoCheckers))));
|
|
533
|
-
}
|
|
534
|
-
|
|
535
562
|
/**
|
|
536
563
|
* The program's memory per character of source text, where it can't be measured (`residentBytes`):
|
|
537
564
|
* 23-28 bytes on the benchmark apps when checking starts (parse threads' arenas included).
|
|
@@ -567,9 +594,14 @@ const lastResortShare = 0.95;
|
|
|
567
594
|
* bytes, for a program of `programBytes` (`firstCheckerShare`, `checkerBytesPerDeclarationChar`);
|
|
568
595
|
* at least 1.
|
|
569
596
|
*/
|
|
570
|
-
export function checkersFittingMemory(
|
|
571
|
-
|
|
572
|
-
|
|
597
|
+
export function checkersFittingMemory(
|
|
598
|
+
programBytes: number,
|
|
599
|
+
declarationChars: number,
|
|
600
|
+
memory: number,
|
|
601
|
+
peakShare: number = 1,
|
|
602
|
+
): number {
|
|
603
|
+
const peak = programBytes * (1 + firstCheckerShare) * peakShare;
|
|
604
|
+
const perChecker = Math.max(1, declarationChars * checkerBytesPerDeclarationChar * peakShare);
|
|
573
605
|
return Math.max(1, 1 + Math.floor((memoryShare * memory - peak) / perChecker));
|
|
574
606
|
}
|
|
575
607
|
|
|
@@ -586,59 +618,6 @@ export function memoryLimit(): number {
|
|
|
586
618
|
return limit;
|
|
587
619
|
}
|
|
588
620
|
|
|
589
|
-
/**
|
|
590
|
-
* The smallest memory limit of this process's cgroup and its ancestors (Linux), or 0 where there is
|
|
591
|
-
* none or it is unknown: `memory.max` and `memory.high` under cgroup v2, `memory.limit_in_bytes`
|
|
592
|
-
* under v1. `process.constrainedMemory()` does not always see these: libuv reads only the process's
|
|
593
|
-
* own cgroup, and scriptc's runtime only the root of the cgroup mount, which is the process's own
|
|
594
|
-
* cgroup only inside a cgroup namespace.
|
|
595
|
-
*/
|
|
596
|
-
export function cgroupMemoryLimit(): number {
|
|
597
|
-
let membership: string;
|
|
598
|
-
try {
|
|
599
|
-
membership = readFileSync('/proc/self/cgroup', 'utf8');
|
|
600
|
-
} catch {
|
|
601
|
-
return 0;
|
|
602
|
-
}
|
|
603
|
-
let limit = 0;
|
|
604
|
-
const lower = (value: number): void => {
|
|
605
|
-
if (value > 0 && (limit === 0 || value < limit)) limit = value;
|
|
606
|
-
};
|
|
607
|
-
for (const line of membership.split('\n')) {
|
|
608
|
-
// "0::/path" (v2), "N:memory:/path" or "N:cpu,memory:/path" (v1).
|
|
609
|
-
const first = line.indexOf(':');
|
|
610
|
-
const second = line.indexOf(':', first + 1);
|
|
611
|
-
if (first < 0 || second < 0) continue;
|
|
612
|
-
const controllers = line.substring(first + 1, second);
|
|
613
|
-
const path = line.substring(second + 1);
|
|
614
|
-
if (controllers === '') {
|
|
615
|
-
for (let dir = path; ; dir = dir.substring(0, dir.lastIndexOf('/'))) {
|
|
616
|
-
const base = '/sys/fs/cgroup' + (dir === '/' ? '' : dir);
|
|
617
|
-
lower(cgroupLimitValue(base + '/memory.max'));
|
|
618
|
-
lower(cgroupLimitValue(base + '/memory.high'));
|
|
619
|
-
if (dir === '/' || dir === '' || dir.lastIndexOf('/') < 0) break;
|
|
620
|
-
}
|
|
621
|
-
} else if (controllers.split(',').indexOf('memory') >= 0) {
|
|
622
|
-
lower(cgroupLimitValue('/sys/fs/cgroup/memory' + (path === '/' ? '' : path) + '/memory.limit_in_bytes'));
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
return limit;
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
/** A cgroup memory limit file's value in bytes, or 0 for `max`, an unreadable file or no number. */
|
|
629
|
-
function cgroupLimitValue(file: string): number {
|
|
630
|
-
let text: string;
|
|
631
|
-
try {
|
|
632
|
-
text = readFileSync(file, 'utf8').trim();
|
|
633
|
-
} catch {
|
|
634
|
-
return 0;
|
|
635
|
-
}
|
|
636
|
-
if (text === '' || text === 'max') return 0;
|
|
637
|
-
const value = Number(text);
|
|
638
|
-
// cgroup v1 reports no limit as a page-aligned value near 2^63.
|
|
639
|
-
return Number.isFinite(value) && value > 0 && value < 2 ** 62 ? value : 0;
|
|
640
|
-
}
|
|
641
|
-
|
|
642
621
|
/** The program whose text lengths `textLengths` holds. */
|
|
643
622
|
let lengthsProgram: Program | undefined = undefined;
|
|
644
623
|
let lengthsOfProgram: number[] = [];
|
|
@@ -664,6 +643,27 @@ export function textLengths(program: Program): number[] {
|
|
|
664
643
|
* or an estimate from its text where that is unknown.
|
|
665
644
|
*/
|
|
666
645
|
export function memoryCheckers(program: Program): number {
|
|
646
|
+
const estimate = memoryEstimate(program);
|
|
647
|
+
const fit = checkersFittingMemory(estimate.programBytes, estimate.declarationChars, estimate.memory);
|
|
648
|
+
if (process.env.TSC_TS_CHECKERS_TRACE !== undefined)
|
|
649
|
+
process.stderr.write(
|
|
650
|
+
`checkers: ${estimate.chars} source chars (${estimate.declarationChars} in declaration files), program ${Math.round(estimate.programBytes / 1048576)}MB${estimate.resident > 0 ? '' : ' (estimated)'}, memory ${Math.round(estimate.memory / 1048576)}MB fits ${fit} checkers\n`,
|
|
651
|
+
);
|
|
652
|
+
return fit;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/** What the memory fit reads of a program: its text, the declarations in it, and this process's memory. */
|
|
656
|
+
interface MemoryEstimate {
|
|
657
|
+
readonly chars: number;
|
|
658
|
+
readonly declarationChars: number;
|
|
659
|
+
readonly programBytes: number;
|
|
660
|
+
/** Whether `programBytes` is the resident set (`0` where it is an estimate from the text). */
|
|
661
|
+
readonly resident: number;
|
|
662
|
+
readonly memory: number;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/** The memory estimate of `program`, of its text (`textLengths`, computed once) and this process's memory. */
|
|
666
|
+
function memoryEstimate(program: Program): MemoryEstimate {
|
|
667
667
|
let chars = 0;
|
|
668
668
|
let declarationChars = 0;
|
|
669
669
|
const files = program.getSourceFiles();
|
|
@@ -673,14 +673,24 @@ export function memoryCheckers(program: Program): number {
|
|
|
673
673
|
if (files[i]!.isDeclarationFile) declarationChars += length[i]!;
|
|
674
674
|
}
|
|
675
675
|
const resident = residentBytes();
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
676
|
+
return {
|
|
677
|
+
chars,
|
|
678
|
+
declarationChars,
|
|
679
|
+
programBytes: resident > 0 ? resident : chars * programBytesPerChar,
|
|
680
|
+
resident,
|
|
681
|
+
memory: memoryLimit(),
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* The checkers whose estimated peak memory fits `program` (`checkersFittingMemory`), without sharing and
|
|
687
|
+
* with the peak sharing adds (`sharingPeakShare`), which the sharing rule requires of the count it keeps.
|
|
688
|
+
*/
|
|
689
|
+
export function memoryFit(program: Program): CheckFit {
|
|
690
|
+
const estimate = memoryEstimate(program);
|
|
691
|
+
return {
|
|
692
|
+
sharing: checkersFittingMemory(estimate.programBytes, estimate.declarationChars, estimate.memory, sharingPeakShare),
|
|
693
|
+
};
|
|
684
694
|
}
|
|
685
695
|
|
|
686
696
|
/**
|
|
@@ -1464,10 +1474,13 @@ export function planSchedule(
|
|
|
1464
1474
|
}
|
|
1465
1475
|
let all = 0;
|
|
1466
1476
|
for (const i of order) all += cost[i]!;
|
|
1467
|
-
// Files that cost more than a
|
|
1468
|
-
//
|
|
1477
|
+
// Files that cost more than a 128th of the program go first, costliest first: at the end of a range
|
|
1478
|
+
// they would make its checker finish last. A fixed fraction of the program, rather than of a
|
|
1479
|
+
// checker's share, keeps the cut in proportion to how many checkers drain the shared queue: it is a
|
|
1480
|
+
// 32nd of a share at 4 checkers and an eighth at 16, where a range is shorter and a file of the
|
|
1481
|
+
// same size would otherwise sit in it.
|
|
1469
1482
|
let heavyCount = count;
|
|
1470
|
-
while (heavyCount < order.length && cost[order[heavyCount]!]! *
|
|
1483
|
+
while (heavyCount < order.length && cost[order[heavyCount]!]! * 128 >= all) heavyCount++;
|
|
1471
1484
|
// The groups go to the shared queue as one entry each, among the costliest files, by total cost.
|
|
1472
1485
|
const groups = checkGroups(checked, count, groupLoad, costs);
|
|
1473
1486
|
const grouped = new Set<number>();
|
|
@@ -1650,9 +1663,13 @@ function threadSharingOf(
|
|
|
1650
1663
|
order: readonly number[],
|
|
1651
1664
|
units: readonly number[],
|
|
1652
1665
|
files: number,
|
|
1666
|
+
bounded: boolean,
|
|
1653
1667
|
): ThreadSharingData[] {
|
|
1654
1668
|
const takers = new SharedArrayBuffer(4 * files);
|
|
1655
1669
|
const forbidden = new SharedArrayBuffer(4 * files * forbiddenWords(count));
|
|
1670
|
+
// The checkers that missed on a key in each file, which a post requires (`ThreadSharing.demanded`). An unbounded
|
|
1671
|
+
// run posts as Phase 1 does and never reads it, so it gets a word rather than a bit per checker per file.
|
|
1672
|
+
const wanted = bounded ? new SharedArrayBuffer(4 * files * forbiddenWords(count)) : new SharedArrayBuffer(4);
|
|
1656
1673
|
const handoff = new SharedArrayBuffer(4 * (3 + order.length));
|
|
1657
1674
|
new Int32Array(handoff)[2] = count;
|
|
1658
1675
|
const taken = new Int32Array(takers);
|
|
@@ -1670,17 +1687,23 @@ function threadSharingOf(
|
|
|
1670
1687
|
const stringLogs: SharedArrayBuffer[] = [];
|
|
1671
1688
|
for (let i = 0; i < count; i++) {
|
|
1672
1689
|
numberLogs.push(new SharedArrayBuffer(4 * numberLogSize));
|
|
1673
|
-
recordTables.push(new SharedArrayBuffer(
|
|
1690
|
+
recordTables.push(new SharedArrayBuffer(4 * recordTableSize));
|
|
1674
1691
|
stringLogs.push(new SharedArrayBuffer(2 * stringLogSize));
|
|
1675
1692
|
}
|
|
1676
|
-
const used = new SharedArrayBuffer(
|
|
1693
|
+
const used = new SharedArrayBuffer(4 * usedStride * count);
|
|
1694
|
+
// The stop switch (0 runs, 1 stops posting, 2 stops posting and translating) and the counters the
|
|
1695
|
+
// main thread reads at its window, a row per checker.
|
|
1696
|
+
const control = new SharedArrayBuffer(4);
|
|
1697
|
+
const stats = new SharedArrayBuffer(8 * shareStatColumns * count);
|
|
1677
1698
|
const data: ThreadSharingData[] = [];
|
|
1678
1699
|
for (let i = 0; i < count; i++)
|
|
1679
1700
|
data.push({
|
|
1680
1701
|
index: i,
|
|
1681
1702
|
count,
|
|
1703
|
+
bounded,
|
|
1682
1704
|
takers,
|
|
1683
1705
|
forbidden,
|
|
1706
|
+
wanted,
|
|
1684
1707
|
handoff,
|
|
1685
1708
|
typeTable,
|
|
1686
1709
|
returnTable,
|
|
@@ -1690,6 +1713,8 @@ function threadSharingOf(
|
|
|
1690
1713
|
recordTables,
|
|
1691
1714
|
stringLogs,
|
|
1692
1715
|
used,
|
|
1716
|
+
control,
|
|
1717
|
+
stats,
|
|
1693
1718
|
});
|
|
1694
1719
|
return data;
|
|
1695
1720
|
}
|
|
@@ -1920,50 +1945,179 @@ export function mainThreadChecks(shared: boolean): boolean {
|
|
|
1920
1945
|
}
|
|
1921
1946
|
|
|
1922
1947
|
/**
|
|
1923
|
-
*
|
|
1924
|
-
*
|
|
1925
|
-
*
|
|
1948
|
+
* Whether the run reads the memory fit of sharing (`checkDecisionOf`): the static rule reads it where
|
|
1949
|
+
* the signals point at sharing, and a run the learning rules read may adopt sharing the static rule
|
|
1950
|
+
* leaves off, or explore it, when its samples show the configuration faster (DESIGN 4.3-4.4).
|
|
1951
|
+
*
|
|
1952
|
+
* A pin on sharing chooses sharing without a rule, but where it pins sharing on the learning rules still
|
|
1953
|
+
* compare checker counts with it, and every count they may adopt reads how many fit memory with sharing's
|
|
1954
|
+
* peak: a pinned count that no rule can change, and `TSC_TS_AUTO=0`, read nothing.
|
|
1955
|
+
*/
|
|
1956
|
+
export function sharingFitNeeded(config: RunConfig, signals: CheckSignals): boolean {
|
|
1957
|
+
if (config.automaticity === Automaticity.Off) return false;
|
|
1958
|
+
if (config.shareSource === Source.Pinned) {
|
|
1959
|
+
return (
|
|
1960
|
+
config.share &&
|
|
1961
|
+
config.checkersSource !== Source.Pinned &&
|
|
1962
|
+
learns(config.automaticity) &&
|
|
1963
|
+
config.record !== undefined
|
|
1964
|
+
);
|
|
1965
|
+
}
|
|
1966
|
+
if (sharingCandidate(signals)) return true;
|
|
1967
|
+
return learns(config.automaticity) && config.record !== undefined;
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
/**
|
|
1971
|
+
* What `decideCheck` needs of the run besides its signals: whether the checker threads of a shared
|
|
1972
|
+
* program may share results (`startParallelCheck` makes the same test), and, where sharing is possible
|
|
1973
|
+
* and a rule that may choose it reads the estimate, how many checkers fit memory with sharing's peak
|
|
1974
|
+
* (`sharingFitNeeded`). The estimate costs a pass over the program's files and its memory limit, so it
|
|
1975
|
+
* is computed only where a rule reads it: a run without a record pays only where the static rule's
|
|
1976
|
+
* signals point at sharing, and one checker, a program that is not shared, a pinned run or
|
|
1977
|
+
* `TSC_TS_AUTO=0` pay nothing.
|
|
1978
|
+
*/
|
|
1979
|
+
function checkDecisionOf(config: RunConfig, signals: CheckSignals, program: Program, shared: boolean): CheckDecision {
|
|
1980
|
+
// `mainThreadChecks` is false whenever the program is shared (`!shared`), as sharing needs.
|
|
1981
|
+
const possible = shared && signals.count > 1 && signals.count <= maxSharingCheckers;
|
|
1982
|
+
const fit = possible && sharingFitNeeded(config, signals) ? memoryFit(program) : undefined;
|
|
1983
|
+
// A run whose record the learning rules read also names the shape its samples describe, the CPUs a
|
|
1984
|
+
// count may use, the memory a configuration may not exceed and the time a control sample ages from.
|
|
1985
|
+
const learning = config.records || config.record !== undefined;
|
|
1986
|
+
return {
|
|
1987
|
+
possible,
|
|
1988
|
+
fit,
|
|
1989
|
+
shape: learning ? recordShapeOf(program, signals) : undefined,
|
|
1990
|
+
cpus: Math.min(availableCpus(), maxAutoCheckers),
|
|
1991
|
+
memoryBytes: memoryLimit(),
|
|
1992
|
+
now: Math.floor(Date.now() / 1000),
|
|
1993
|
+
};
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
/** The program's shape for its record (`RecordShape`), with the shares the sharing rule reads as `F`. */
|
|
1997
|
+
function recordShapeOf(program: Program, signals: CheckSignals): RecordShape {
|
|
1998
|
+
const files = program.getSourceFiles();
|
|
1999
|
+
let chars = 0;
|
|
2000
|
+
let declarationChars = 0;
|
|
2001
|
+
for (const file of files) {
|
|
2002
|
+
chars += file.end;
|
|
2003
|
+
if (file.isDeclarationFile) declarationChars += file.end;
|
|
2004
|
+
}
|
|
2005
|
+
return {
|
|
2006
|
+
files: files.length,
|
|
2007
|
+
chars,
|
|
2008
|
+
declarationChars,
|
|
2009
|
+
checkChars: chars - declarationChars,
|
|
2010
|
+
library: Math.round(signals.library),
|
|
2011
|
+
own: Math.round(signals.own),
|
|
2012
|
+
redundancy: signals.redundancy,
|
|
2013
|
+
fileShare: Math.max(signals.declarationShare, signals.nodeModulesShare),
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
/** What the counters of `count` checkers say at the sharing window: milliseconds saved and spent. */
|
|
2018
|
+
interface SharingNumbers {
|
|
2019
|
+
/** What taking posted results saved, over the checkers: the computations they did not repeat. */
|
|
2020
|
+
readonly saved: number;
|
|
2021
|
+
/**
|
|
2022
|
+
* What posting cost: exporting and translating results. Translating a result reads its records in
|
|
2023
|
+
* place inside `ThreadSharing.translatePosted`'s timed translation, so `translationMs` holds that
|
|
2024
|
+
* reading too; counting a separate unpacking time would charge the window for it twice.
|
|
2025
|
+
*/
|
|
2026
|
+
readonly spent: number;
|
|
2027
|
+
readonly posted: number;
|
|
2028
|
+
readonly translated: number;
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
/**
|
|
2032
|
+
* The counters of `count` checkers' rows of `stats` (`ThreadSharing.writeStats`), summed. The tests read
|
|
2033
|
+
* this, next to the window it feeds.
|
|
2034
|
+
*/
|
|
2035
|
+
export function sharingNumbers(stats: Float64Array, count: number): SharingNumbers {
|
|
2036
|
+
let saved = 0;
|
|
2037
|
+
let spent = 0;
|
|
2038
|
+
let posted = 0;
|
|
2039
|
+
let translated = 0;
|
|
2040
|
+
for (let i = 0; i < count; i++) {
|
|
2041
|
+
const row = i * shareStatColumns;
|
|
2042
|
+
saved += stats[row + shareStatSaved]!;
|
|
2043
|
+
spent += stats[row + shareStatExport]! + stats[row + shareStatTranslation]!;
|
|
2044
|
+
posted += stats[row + shareStatPosted]!;
|
|
2045
|
+
translated += stats[row + shareStatTranslated]!;
|
|
2046
|
+
}
|
|
2047
|
+
return { saved, spent, posted, translated };
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
/**
|
|
2051
|
+
* The control a sharing run's numbers ask for: 2 when it saved less than 1.5 times what it spent, 1
|
|
2052
|
+
* when the posts go untaken (under 5% of them, and at least one), else 0. Stopping takes a minute at
|
|
2053
|
+
* worst, when the window's numbers are still too early to tell.
|
|
1926
2054
|
*/
|
|
1927
|
-
|
|
1928
|
-
|
|
2055
|
+
export function stopLevel(numbers: SharingNumbers): number {
|
|
2056
|
+
if (numbers.saved < 1.5 * numbers.spent) return 2;
|
|
2057
|
+
if (numbers.posted > 0 && numbers.translated / numbers.posted < 0.05) return 1;
|
|
2058
|
+
return 0;
|
|
2059
|
+
}
|
|
1929
2060
|
|
|
1930
2061
|
/**
|
|
1931
|
-
*
|
|
1932
|
-
*
|
|
1933
|
-
* every checker spends most of its time resolving the same large libraries (`libraryBoundText`,
|
|
1934
|
-
* `libraryBoundShare`), else `count`.
|
|
2062
|
+
* Whether the rules of a run of `config` at `count` checkers read the check signals (`checkSignals`).
|
|
2063
|
+
* Two rules read them:
|
|
1935
2064
|
*
|
|
1936
|
-
*
|
|
1937
|
-
*
|
|
1938
|
-
*
|
|
1939
|
-
*
|
|
1940
|
-
*
|
|
1941
|
-
*
|
|
1942
|
-
*
|
|
1943
|
-
*
|
|
2065
|
+
* - the library-bound cut (`decideCheck`), wherever the count is not pinned and is above tsgo's four.
|
|
2066
|
+
* This is the decision a run made before the automatic selection, and `TSC_TS_AUTO=0` keeps it: with
|
|
2067
|
+
* selection off the count is not cut for sharing either, but dropping the signals would leave the
|
|
2068
|
+
* cut nothing to read and start checkers the cut would not;
|
|
2069
|
+
* - the sharing rule, at `minSharingCheckers` or more, where neither the count nor sharing is pinned
|
|
2070
|
+
* (the cut may leave the count sharing was measured at, so the signals are read before it).
|
|
2071
|
+
*
|
|
2072
|
+
* A pinned count and a pinned sharing read neither, and pay nothing for the pass over the ranges.
|
|
1944
2073
|
*/
|
|
1945
|
-
export function
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
2074
|
+
export function signalsNeeded(config: RunConfig, count: number): boolean {
|
|
2075
|
+
const cut = config.checkersSource === Source.Default && count > tsgoCheckers;
|
|
2076
|
+
if (config.automaticity === Automaticity.Off) return cut;
|
|
2077
|
+
return cut || (config.shareSource === Source.Default && count >= minSharingCheckers);
|
|
1949
2078
|
}
|
|
1950
2079
|
|
|
1951
2080
|
/**
|
|
1952
|
-
*
|
|
1953
|
-
*
|
|
2081
|
+
* What `plan`, made with `count` checkers, shows about the cost of checking (`CheckSignals`): the
|
|
2082
|
+
* libraries a range reaches and the files it checks, the share of that a second range reaches too, and
|
|
2083
|
+
* the shares of declaration and node_modules files of the program. Computed where a rule may read them
|
|
2084
|
+
* (`signalsNeeded`) and left at zero elsewhere: a pinned count needs none of them, and the pass over
|
|
2085
|
+
* every range's libraries costs a millisecond on a large program.
|
|
1954
2086
|
*/
|
|
1955
|
-
function
|
|
1956
|
-
|
|
2087
|
+
function checkSignals(
|
|
2088
|
+
checked: CheckedProgram,
|
|
2089
|
+
plan: CheckPlan,
|
|
2090
|
+
costs: CheckCosts,
|
|
2091
|
+
count: number,
|
|
2092
|
+
config: RunConfig | undefined = undefined,
|
|
2093
|
+
): CheckSignals {
|
|
2094
|
+
const ranges = plan.ranges.length >> 1;
|
|
2095
|
+
const signals = new CheckSignals(count, ranges);
|
|
1957
2096
|
const program = checked.program;
|
|
2097
|
+
// What a rule would read, what the run's record writes (`config.records`), or what a doctor child
|
|
2098
|
+
// reports (`config.measure`), is computed; a pinned count with records off and no report pays nothing.
|
|
2099
|
+
const deciding =
|
|
2100
|
+
config === undefined
|
|
2101
|
+
? checkersPolicy(process.env) === 'auto' && program.options.checkers === undefined && count > tsgoCheckers
|
|
2102
|
+
: signalsNeeded(config, count);
|
|
2103
|
+
const wanted = deciding || (config !== undefined && (config.records || config.measure));
|
|
2104
|
+
if (!wanted || ranges === 0) return signals;
|
|
1958
2105
|
const files = program.getSourceFiles();
|
|
1959
2106
|
const length = textLengths(program);
|
|
1960
2107
|
const isChecked: boolean[] = [];
|
|
1961
|
-
|
|
2108
|
+
let declarationFiles = 0;
|
|
2109
|
+
let nodeModulesFiles = 0;
|
|
2110
|
+
for (let i = 0; i < files.length; i++) {
|
|
2111
|
+
isChecked.push(false);
|
|
2112
|
+
if (files[i]!.isDeclarationFile) declarationFiles++;
|
|
2113
|
+
if (files[i]!.path.indexOf('/node_modules/') >= 0) nodeModulesFiles++;
|
|
2114
|
+
}
|
|
1962
2115
|
for (const i of costs.order) isChecked[i] = true;
|
|
1963
|
-
const ranges = plan.ranges.length >> 1;
|
|
1964
|
-
if (ranges === 0) return count;
|
|
1965
2116
|
let library = 0;
|
|
1966
2117
|
let own = 0;
|
|
2118
|
+
// The libraries every range reaches together, each counted once (`CheckSignals.redundancy`).
|
|
2119
|
+
const union = new Set<number>();
|
|
2120
|
+
let unionChars = 0;
|
|
1967
2121
|
for (let r = 0; r < ranges; r++) {
|
|
1968
2122
|
// The libraries the range's files import, and what they reach through other libraries (as checkCosts counts them).
|
|
1969
2123
|
const reached = new Set<number>();
|
|
@@ -1980,6 +2134,10 @@ function libraryBoundCheckers(checked: CheckedProgram, plan: CheckPlan, costs: C
|
|
|
1980
2134
|
while (stack.length > 0) {
|
|
1981
2135
|
const k = stack.pop()!;
|
|
1982
2136
|
library += length[k]!;
|
|
2137
|
+
if (!union.has(k)) {
|
|
2138
|
+
union.add(k);
|
|
2139
|
+
unionChars += length[k]!;
|
|
2140
|
+
}
|
|
1983
2141
|
for (const t of costs.imports[k] ?? [])
|
|
1984
2142
|
if (
|
|
1985
2143
|
!reached.has(t) &&
|
|
@@ -1992,18 +2150,22 @@ function libraryBoundCheckers(checked: CheckedProgram, plan: CheckPlan, costs: C
|
|
|
1992
2150
|
}
|
|
1993
2151
|
}
|
|
1994
2152
|
}
|
|
1995
|
-
|
|
2153
|
+
signals.library = library / ranges;
|
|
2154
|
+
signals.own = own / ranges;
|
|
2155
|
+
signals.redundancy = (library - unionChars) / (library + own);
|
|
2156
|
+
signals.declarationShare = declarationFiles / files.length;
|
|
2157
|
+
signals.nodeModulesShare = nodeModulesFiles / files.length;
|
|
2158
|
+
if (config !== undefined) config.signals = signals;
|
|
1996
2159
|
if (process.env.TSC_TS_CHECKERS_TRACE !== undefined)
|
|
1997
2160
|
process.stderr.write(
|
|
1998
|
-
`checkers: ranges reach ${Math.round(library /
|
|
2161
|
+
`checkers: ranges reach ${Math.round(signals.library / 1000)}K library chars and check ${Math.round(signals.own / 1000)}K on average, ${Math.round(signals.redundancy * 100)}% of it reached twice: ${libraryBoundCount(count, signals.library, signals.own)} checkers\n`,
|
|
1999
2162
|
);
|
|
2000
|
-
return
|
|
2163
|
+
return signals;
|
|
2001
2164
|
}
|
|
2002
2165
|
|
|
2003
2166
|
/**
|
|
2004
|
-
* Checks `checked` with `checkers` checkers (fewer where
|
|
2005
|
-
*
|
|
2006
|
-
* `checked` holds every file's diagnostics.
|
|
2167
|
+
* Checks `checked` with `checkers` checkers (fewer where the configuration chooses so: `decideCheck`,
|
|
2168
|
+
* run-config.ts) and calls `done` with the wall time once `checked` holds every file's diagnostics.
|
|
2007
2169
|
*/
|
|
2008
2170
|
/**
|
|
2009
2171
|
* Shares the program of `checked` with checker threads (scriptc builds): this thread's checker merges the
|
|
@@ -2037,7 +2199,8 @@ export function shareWithCheckers(checked: CheckedProgram): void {
|
|
|
2037
2199
|
* `emitsTo`, the emit results go only there, by file index, and nothing else the threads find reaches
|
|
2038
2200
|
* the program: a declaration thread (parallel-emit.ts), while this thread's checker checks. With
|
|
2039
2201
|
* `declarationsTo`, the same holds for the declaration errors of a `noEmit` program
|
|
2040
|
-
* (`checkersReportDeclarations`).
|
|
2202
|
+
* (`checkersReportDeclarations`). `config` is the run's configuration: the count it chose is decided
|
|
2203
|
+
* here, once the program is bound and the plan made, and sharing is what it holds.
|
|
2041
2204
|
*/
|
|
2042
2205
|
export function startParallelCheck(
|
|
2043
2206
|
checked: CheckedProgram,
|
|
@@ -2048,6 +2211,7 @@ export function startParallelCheck(
|
|
|
2048
2211
|
checking: boolean = true,
|
|
2049
2212
|
emitsTo: (EmitResult | undefined)[] | undefined = undefined,
|
|
2050
2213
|
declarationsTo: (Diagnostic[] | undefined)[] | undefined = undefined,
|
|
2214
|
+
config: RunConfig | undefined = undefined,
|
|
2051
2215
|
): void {
|
|
2052
2216
|
let count = checkers;
|
|
2053
2217
|
const start = performance.now();
|
|
@@ -2067,7 +2231,12 @@ export function startParallelCheck(
|
|
|
2067
2231
|
const orderStart = performance.now();
|
|
2068
2232
|
const costs = costOrder(checked);
|
|
2069
2233
|
let plan = planSchedule(checked, count, costs, checking);
|
|
2070
|
-
const
|
|
2234
|
+
const signals = checkSignals(checked, plan, costs, count, config);
|
|
2235
|
+
if (config !== undefined) config.signals = signals;
|
|
2236
|
+
const fewer =
|
|
2237
|
+
config === undefined
|
|
2238
|
+
? libraryBoundCount(signals.count, signals.library, signals.own)
|
|
2239
|
+
: decideCheck(config, signals, checkDecisionOf(config, signals, program, shared));
|
|
2071
2240
|
if (fewer < count) {
|
|
2072
2241
|
count = fewer;
|
|
2073
2242
|
plan = planSchedule(checked, count, costs, checking);
|
|
@@ -2109,7 +2278,13 @@ export function startParallelCheck(
|
|
|
2109
2278
|
const firstThread = mainChecks ? 1 : 0;
|
|
2110
2279
|
let remaining = count - firstThread;
|
|
2111
2280
|
let ownDone = false;
|
|
2281
|
+
/** The sharing window's timer, where the threads share results; cleared when they are done. */
|
|
2282
|
+
let shareStopTimer: ReturnType<typeof setTimeout> | undefined = undefined;
|
|
2112
2283
|
const finish = (): void => {
|
|
2284
|
+
if (shareStopTimer !== undefined) {
|
|
2285
|
+
clearTimeout(shareStopTimer);
|
|
2286
|
+
shareStopTimer = undefined;
|
|
2287
|
+
}
|
|
2113
2288
|
if (emitsTo !== undefined || declarationsTo !== undefined) {
|
|
2114
2289
|
mark('emitted on checker threads');
|
|
2115
2290
|
done(performance.now() - start);
|
|
@@ -2188,11 +2363,43 @@ export function startParallelCheck(
|
|
|
2188
2363
|
if (remaining === 0 && ownDone) finish();
|
|
2189
2364
|
};
|
|
2190
2365
|
const diagnosticFiles = new Map<string, DiagnosticFile>();
|
|
2191
|
-
//
|
|
2366
|
+
// Whether checker threads share results (shared-results.ts), which the run's configuration decides;
|
|
2367
|
+
// the emit threads of parallel-emit.ts have no configuration and never share (count 1).
|
|
2368
|
+
const sharesResults = config === undefined ? shareThreads === true : config.share;
|
|
2192
2369
|
const sharing =
|
|
2193
|
-
|
|
2194
|
-
? threadSharingOf(count, order, units, files.length)
|
|
2370
|
+
sharesResults && shared && !mainChecks && count > 1 && count <= maxSharingCheckers
|
|
2371
|
+
? threadSharingOf(count, order, units, files.length, config !== undefined && config.bounded)
|
|
2195
2372
|
: undefined;
|
|
2373
|
+
if (sharing !== undefined) {
|
|
2374
|
+
// The main thread is idle while the checkers run, so a timer reads their counters at the window and
|
|
2375
|
+
// stops what does not pay (control 1 or 2); every checker computes what it does not take, so the
|
|
2376
|
+
// diagnostics stay the same.
|
|
2377
|
+
const control = new Int32Array(sharing[0]!.control);
|
|
2378
|
+
const stats = new Float64Array(sharing[0]!.stats);
|
|
2379
|
+
const stop = shareStop;
|
|
2380
|
+
const at = stop !== undefined ? stop.at : config !== undefined ? config.shareWindowMs : shareWindowMs;
|
|
2381
|
+
shareStopTimer = setTimeout(() => {
|
|
2382
|
+
const numbers = sharingNumbers(stats, count);
|
|
2383
|
+
if (stop !== undefined) Atomics.store(control, 0, stop.level);
|
|
2384
|
+
else {
|
|
2385
|
+
const level = stopLevel(numbers);
|
|
2386
|
+
if (level === 0) return;
|
|
2387
|
+
Atomics.store(control, 0, level);
|
|
2388
|
+
}
|
|
2389
|
+
// The section says what the window saw, instead of the rule that turned sharing on.
|
|
2390
|
+
if (config !== undefined) {
|
|
2391
|
+
config.shareSource = Source.Measured;
|
|
2392
|
+
config.shareReasons.length = 0;
|
|
2393
|
+
config.shareReasons.push(
|
|
2394
|
+
`stopped after ${at} ms: ${Math.round(numbers.saved)} ms saved for ${Math.round(numbers.spent)} ms spent`,
|
|
2395
|
+
);
|
|
2396
|
+
}
|
|
2397
|
+
if (trace)
|
|
2398
|
+
process.stderr.write(
|
|
2399
|
+
`checkers: stopped sharing after ${at} ms: ${Math.round(numbers.saved)} ms saved for ${Math.round(numbers.spent)} ms spent, ${numbers.translated}/${numbers.posted} posted results taken\n`,
|
|
2400
|
+
);
|
|
2401
|
+
}, at);
|
|
2402
|
+
}
|
|
2196
2403
|
const sharingStarted: boolean[] = [];
|
|
2197
2404
|
for (let i = 0; i < count; i++) sharingStarted.push(false);
|
|
2198
2405
|
const startSharingOf = (i: number): ThreadSharingData => {
|
|
@@ -2232,7 +2439,7 @@ export function startParallelCheck(
|
|
|
2232
2439
|
});
|
|
2233
2440
|
worker.on('error', (error: unknown) => {
|
|
2234
2441
|
// Sharing results (TSC_TS_SHARE) is still checked for such failures, which the main thread hides otherwise.
|
|
2235
|
-
if (trace ||
|
|
2442
|
+
if (trace || sharesResults) process.stderr.write(`checkers: thread ${i} failed: ${String(error)}\n`);
|
|
2236
2443
|
threadDone();
|
|
2237
2444
|
});
|
|
2238
2445
|
};
|