tsc-ts 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/compiler/ast.d.ts +2 -0
- package/dist/api/compiler/ast.js +3 -0
- package/dist/api/compiler/checker-calls.js +5 -1
- package/dist/api/compiler/checker-grammar.js +3 -2
- package/dist/api/compiler/checker-inference.js +8 -2
- package/dist/api/compiler/checker-members.js +2 -1
- package/dist/api/compiler/checker-type-factory.js +8 -2
- package/dist/api/compiler/checker.d.ts +11 -0
- package/dist/api/compiler/checker.js +40 -0
- package/dist/api/compiler/compile-program.d.ts +26 -2
- package/dist/api/compiler/compile-program.js +100 -7
- package/dist/api/compiler/diagnostics.d.ts +22 -0
- package/dist/api/compiler/diagnostics.js +86 -4
- package/dist/api/compiler/file-loader.d.ts +11 -1
- package/dist/api/compiler/file-loader.js +63 -7
- package/dist/api/compiler/parallel-parse.d.ts +3 -2
- package/dist/api/compiler/parallel-parse.js +17 -5
- package/dist/api/compiler/printer/emit-context.d.ts +2 -0
- package/dist/api/compiler/printer/emit-context.js +4 -0
- package/dist/api/compiler/printer/printer.d.ts +48 -2
- package/dist/api/compiler/printer/printer.js +238 -14
- package/dist/api/compiler/program-checker.d.ts +4 -0
- package/dist/api/compiler/program-checker.js +4 -0
- package/dist/api/compiler/program.d.ts +2 -6
- package/dist/api/compiler/program.js +5 -30
- package/dist/api/compiler/relations.js +10 -0
- package/dist/api/compiler/symbol.d.ts +1 -1
- package/dist/api/compiler/symbol.js +8 -10
- package/package.json +9 -9
- package/src/compiler/ast.ts +3 -0
- package/src/compiler/checker-calls.ts +3 -1
- package/src/compiler/checker-grammar.ts +3 -2
- package/src/compiler/checker-inference.ts +7 -2
- package/src/compiler/checker-members.ts +3 -1
- package/src/compiler/checker-type-factory.ts +4 -2
- package/src/compiler/checker.ts +37 -0
- package/src/compiler/compile-program.ts +121 -7
- package/src/compiler/diagnostics.ts +84 -3
- package/src/compiler/file-loader.ts +58 -6
- package/src/compiler/incremental.ts +80 -13
- package/src/compiler/parallel-check.ts +129 -44
- package/src/compiler/parallel-emit.ts +320 -0
- package/src/compiler/parallel-parse.ts +16 -5
- package/src/compiler/parse-thread.ts +3 -0
- package/src/compiler/printer/emit-context.ts +5 -0
- package/src/compiler/printer/printer.ts +231 -16
- package/src/compiler/program-checker.ts +4 -0
- package/src/compiler/program.ts +5 -25
- package/src/compiler/relations.ts +7 -0
- package/src/compiler/symbol.ts +15 -10
- package/src/compiler/thread-entry.ts +45 -2
- package/src/compiler/tsc-build.ts +63 -9
- package/src/compiler/tsc-cli.ts +5 -0
- package/src/compiler/tsc-compile.ts +278 -3
|
@@ -33,7 +33,14 @@ import {
|
|
|
33
33
|
type OutputWriter,
|
|
34
34
|
ProgramEmitter,
|
|
35
35
|
} from './compile-program.js';
|
|
36
|
-
import {
|
|
36
|
+
import {
|
|
37
|
+
Diagnostic,
|
|
38
|
+
DiagnosticFile,
|
|
39
|
+
type DiagnosticFormatOptions,
|
|
40
|
+
formatDiagnostic,
|
|
41
|
+
formattedReports,
|
|
42
|
+
RepopulateInfo,
|
|
43
|
+
} from './diagnostics.js';
|
|
37
44
|
import { EmitOnly, EmitResult } from './emitter.js';
|
|
38
45
|
import { getEmitDeclarations, isIncremental } from './options.js';
|
|
39
46
|
import { DeclarationSignatureWriter } from './incremental.js';
|
|
@@ -144,10 +151,13 @@ export const recycleFiles: number = Math.max(0, Number(process.env.TSC_TS_RECYCL
|
|
|
144
151
|
* text (index, -1 when the program has the file), pos, end, code, category, message,
|
|
145
152
|
* skippedOnNoEmit, the argument count and indices, its repopulate kind (0 for none) followed by the
|
|
146
153
|
* module reference (index), mode and package name (index) of other kinds, then its message chain and
|
|
147
|
-
* related information as lists.
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
154
|
+
* related information as lists. A diagnostic of a reported list (the files' diagnostic and declaration
|
|
155
|
+
* diagnostic lists, the global diagnostics and an emit result's diagnostics) is followed by its text in
|
|
156
|
+
* the main thread's report format (`Diagnostic.formatted`, an index, -1 for none). An emit result is
|
|
157
|
+
* emitSkipped, the emitted files, errors and unsupported files as lists of indices, then its
|
|
158
|
+
* diagnostics and its declaration signatures as a list of indices. `files` and `nums` are typed
|
|
159
|
+
* arrays, which postMessage copies as one block of memory each; it copies an array of numbers element
|
|
160
|
+
* by element.
|
|
151
161
|
*/
|
|
152
162
|
export type PackedResult = {
|
|
153
163
|
readonly index: number;
|
|
@@ -167,7 +177,11 @@ class ResultPacker {
|
|
|
167
177
|
readonly strs: string[] = [];
|
|
168
178
|
private readonly indices = new Map<string, number>();
|
|
169
179
|
|
|
170
|
-
constructor(
|
|
180
|
+
constructor(
|
|
181
|
+
private readonly program: Program,
|
|
182
|
+
/** The main thread's report format, which this thread formats the reported diagnostics in. */
|
|
183
|
+
private readonly format: DiagnosticFormatOptions | undefined,
|
|
184
|
+
) {}
|
|
171
185
|
|
|
172
186
|
private str(text: string): number {
|
|
173
187
|
let k = this.indices.get(text);
|
|
@@ -184,6 +198,21 @@ class ResultPacker {
|
|
|
184
198
|
for (const d of diagnostics) this.diagnostic(d);
|
|
185
199
|
}
|
|
186
200
|
|
|
201
|
+
/** A list the main thread reports: each diagnostic with its text in the report format. */
|
|
202
|
+
reported(diagnostics: readonly Diagnostic[]): void {
|
|
203
|
+
const nums = this.nums;
|
|
204
|
+
const format = this.format;
|
|
205
|
+
nums.push(diagnostics.length);
|
|
206
|
+
for (const d of diagnostics) {
|
|
207
|
+
this.diagnostic(d);
|
|
208
|
+
if (format === undefined) nums.push(-1);
|
|
209
|
+
else {
|
|
210
|
+
nums.push(this.strs.length);
|
|
211
|
+
this.strs.push(formatDiagnostic(d, format));
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
187
216
|
private strList(texts: readonly string[]): void {
|
|
188
217
|
this.nums.push(texts.length);
|
|
189
218
|
for (const text of texts) this.nums.push(this.str(text));
|
|
@@ -194,7 +223,7 @@ class ResultPacker {
|
|
|
194
223
|
this.strList(result.emittedFiles);
|
|
195
224
|
this.strList(result.errors);
|
|
196
225
|
this.strList(result.unsupported);
|
|
197
|
-
this.
|
|
226
|
+
this.reported(result.diagnostics);
|
|
198
227
|
this.strList(result.declarationSignatures);
|
|
199
228
|
this.strList(result.heldOutputs);
|
|
200
229
|
}
|
|
@@ -226,11 +255,11 @@ class ResultPacker {
|
|
|
226
255
|
|
|
227
256
|
/** Packs `result` (on the checker thread) for `postMessage`. */
|
|
228
257
|
export function packResult(result: CheckerResult, program: Program): PackedResult {
|
|
229
|
-
const packer = new ResultPacker(program);
|
|
230
|
-
for (const list of result.diagnostics) packer.
|
|
258
|
+
const packer = new ResultPacker(program, formattedReports.format);
|
|
259
|
+
for (const list of result.diagnostics) packer.reported(list);
|
|
231
260
|
packer.nums.push(result.declarations.length);
|
|
232
|
-
for (const list of result.declarations) packer.
|
|
233
|
-
packer.
|
|
261
|
+
for (const list of result.declarations) packer.reported(list);
|
|
262
|
+
packer.reported(result.globals);
|
|
234
263
|
packer.nums.push(result.emits.length);
|
|
235
264
|
for (const emit of result.emits) packer.emit(emit);
|
|
236
265
|
return {
|
|
@@ -272,6 +301,19 @@ class ResultUnpacker {
|
|
|
272
301
|
return result;
|
|
273
302
|
}
|
|
274
303
|
|
|
304
|
+
/** A list `ResultPacker.reported` packed. */
|
|
305
|
+
reported(): Diagnostic[] {
|
|
306
|
+
const n = this.next();
|
|
307
|
+
const result: Diagnostic[] = [];
|
|
308
|
+
for (let k = 0; k < n; k++) {
|
|
309
|
+
const d = this.diagnostic();
|
|
310
|
+
const text = this.next();
|
|
311
|
+
if (text >= 0) d.formatted = this.packed.strs[text]!;
|
|
312
|
+
result.push(d);
|
|
313
|
+
}
|
|
314
|
+
return result;
|
|
315
|
+
}
|
|
316
|
+
|
|
275
317
|
private diagnostic(): Diagnostic {
|
|
276
318
|
const strs = this.packed.strs;
|
|
277
319
|
const name = this.next();
|
|
@@ -325,7 +367,7 @@ class ResultUnpacker {
|
|
|
325
367
|
this.strList(result.emittedFiles);
|
|
326
368
|
this.strList(result.errors);
|
|
327
369
|
this.strList(result.unsupported);
|
|
328
|
-
for (const d of this.
|
|
370
|
+
for (const d of this.reported()) result.diagnostics.push(d);
|
|
329
371
|
this.strList(result.declarationSignatures);
|
|
330
372
|
this.strList(result.heldOutputs);
|
|
331
373
|
return result;
|
|
@@ -339,11 +381,11 @@ export function unpackResult(
|
|
|
339
381
|
): CheckerResult {
|
|
340
382
|
const unpacker = new ResultUnpacker(packed, program, files);
|
|
341
383
|
const diagnostics: Diagnostic[][] = [];
|
|
342
|
-
for (let k = 0; k < packed.files.length; k++) diagnostics.push(unpacker.
|
|
384
|
+
for (let k = 0; k < packed.files.length; k++) diagnostics.push(unpacker.reported());
|
|
343
385
|
const declarations: Diagnostic[][] = [];
|
|
344
386
|
const nd = unpacker.count();
|
|
345
|
-
for (let k = 0; k < nd; k++) declarations.push(unpacker.
|
|
346
|
-
const globals = unpacker.
|
|
387
|
+
for (let k = 0; k < nd; k++) declarations.push(unpacker.reported());
|
|
388
|
+
const globals = unpacker.reported();
|
|
347
389
|
const emits: EmitResult[] = [];
|
|
348
390
|
const ne = unpacker.count();
|
|
349
391
|
for (let k = 0; k < ne; k++) emits.push(unpacker.emit());
|
|
@@ -650,8 +692,8 @@ class CheckCosts {
|
|
|
650
692
|
readonly libraries: number[][],
|
|
651
693
|
/** What resolving a library costs a checker the first time (`libraryFirstUseShare` of the text it reaches), by index. */
|
|
652
694
|
readonly libraryCost: number[],
|
|
653
|
-
/** The program files (indices) each file
|
|
654
|
-
readonly imports: (number[]
|
|
695
|
+
/** The program files (indices) each file imports (`ProcessedFiles.addedFiles`), by index. */
|
|
696
|
+
readonly imports: readonly (readonly number[])[],
|
|
655
697
|
) {}
|
|
656
698
|
}
|
|
657
699
|
|
|
@@ -696,34 +738,18 @@ function checkCosts(checked: CheckedProgram, order: number[]): CheckCosts {
|
|
|
696
738
|
}
|
|
697
739
|
const length = textLengths(program);
|
|
698
740
|
for (const i of order) cost[i] = length[i]!;
|
|
699
|
-
const indexOf = new Map<SourceFile, number>();
|
|
700
|
-
for (let i = 0; i < files.length; i++) indexOf.set(files[i]!, i);
|
|
701
|
-
// A file resolved by its own file name is that file (its path is the path of that name).
|
|
702
|
-
const byFileName = new Map<string, SourceFile>();
|
|
703
|
-
for (const file of files) byFileName.set(file.fileName, file);
|
|
704
741
|
const checkedFile: boolean[] = [];
|
|
705
742
|
for (let i = 0; i < files.length; i++) checkedFile.push(false);
|
|
706
743
|
for (const i of order) checkedFile[i] = true;
|
|
707
744
|
// Libraries: unchecked modules that are not default libraries (globals every checker has).
|
|
708
745
|
const isLibrary = (i: number): boolean =>
|
|
709
746
|
!checkedFile[i]! && !program.isSourceFileDefaultLibrary(files[i]!.path) && isExternalModuleFile(files[i]!);
|
|
710
|
-
|
|
711
|
-
const
|
|
712
|
-
let result = edges[i];
|
|
713
|
-
if (result === undefined) {
|
|
714
|
-
result = [];
|
|
715
|
-
for (const target of program.getResolvedFilesOf(files[i]!, byFileName)) {
|
|
716
|
-
const k = indexOf.get(target);
|
|
717
|
-
if (k !== undefined && k !== i) result.push(k);
|
|
718
|
-
}
|
|
719
|
-
edges[i] = result;
|
|
720
|
-
}
|
|
721
|
-
return result;
|
|
722
|
-
};
|
|
747
|
+
// The files each file imports: those it added to the program (`ProcessedFiles.addedFiles`).
|
|
748
|
+
const edges = program.addedFiles();
|
|
723
749
|
const importers = new Map<number, number>();
|
|
724
750
|
for (const i of order) {
|
|
725
751
|
const libraries: number[] = [];
|
|
726
|
-
for (const k of
|
|
752
|
+
for (const k of edges[i]!) if (isLibrary(k) && libraries.indexOf(k) < 0) libraries.push(k);
|
|
727
753
|
librariesOf[i] = libraries;
|
|
728
754
|
for (const k of libraries) importers.set(k, (importers.get(k) ?? 0) + 1);
|
|
729
755
|
}
|
|
@@ -738,7 +764,7 @@ function checkCosts(checked: CheckedProgram, order: number[]): CheckCosts {
|
|
|
738
764
|
while (stack.length > 0) {
|
|
739
765
|
const k = stack.pop()!;
|
|
740
766
|
total += length[k]!;
|
|
741
|
-
for (const t of
|
|
767
|
+
for (const t of edges[k]!)
|
|
742
768
|
if (!seen.has(t) && isLibrary(t)) {
|
|
743
769
|
seen.add(t);
|
|
744
770
|
stack.push(t);
|
|
@@ -1642,20 +1668,34 @@ export function checkFromQueue(
|
|
|
1642
1668
|
} else if (writer !== undefined) {
|
|
1643
1669
|
const emitStart = timeline ? performance.now() : 0;
|
|
1644
1670
|
const kind = (emitKinds === undefined ? EmitOnly.All : emitKinds[i]!) as EmitOnly;
|
|
1671
|
+
// Global diagnostics found while emitting are not reported: the program collects them
|
|
1672
|
+
// before it emits.
|
|
1673
|
+
const checker = checked.checker;
|
|
1645
1674
|
if (kind < 0) emits.push(new EmitResult());
|
|
1646
1675
|
else if (signatures !== undefined) {
|
|
1647
1676
|
signatures.file = files[i]!;
|
|
1648
|
-
const result = emitter!.emit([files[i]!], kind, signatures);
|
|
1677
|
+
const result = checker.withoutGlobalDiagnostics(() => emitter!.emit([files[i]!], kind, signatures));
|
|
1649
1678
|
for (const s of signatures.take()) result.declarationSignatures.push(s);
|
|
1650
1679
|
if (holding !== undefined) for (const o of holding.take()) result.heldOutputs.push(o);
|
|
1651
1680
|
emits.push(result);
|
|
1652
1681
|
} else {
|
|
1653
|
-
const result = emitter!.emit([files[i]!], kind, writer);
|
|
1682
|
+
const result = checker.withoutGlobalDiagnostics(() => emitter!.emit([files[i]!], kind, writer));
|
|
1654
1683
|
if (holding !== undefined) for (const o of holding.take()) result.heldOutputs.push(o);
|
|
1655
1684
|
emits.push(result);
|
|
1656
1685
|
}
|
|
1657
1686
|
if (timeline) markAt(emitStart, `emit ${(performance.now() - emitStart).toFixed(1)} ${files[i]!.fileName}`);
|
|
1658
|
-
} else if (emitter !== undefined)
|
|
1687
|
+
} else if (emitter !== undefined)
|
|
1688
|
+
declarations.push(
|
|
1689
|
+
checked.checker.withoutGlobalDiagnostics(() => declarationDiagnosticsOfFile(emitter, files[i]!)),
|
|
1690
|
+
);
|
|
1691
|
+
// What emitting found and forgot can stay cached in this checker (a missing global type, an
|
|
1692
|
+
// instantiation too deep), where checking a later file would report it: a new checker continues
|
|
1693
|
+
// the schedule, as at the file limit.
|
|
1694
|
+
if (checked.checker.forgottenGlobalDiagnostics() > 0) {
|
|
1695
|
+
more = true;
|
|
1696
|
+
left = reservedBy(index).splice(0);
|
|
1697
|
+
break;
|
|
1698
|
+
}
|
|
1659
1699
|
}
|
|
1660
1700
|
mark(`checked ${done.length} files, ${unitsDone.length} ranges`);
|
|
1661
1701
|
// The main thread reports its own global diagnostics (CheckedProgram.getGlobalDiagnostics).
|
|
@@ -1828,7 +1868,11 @@ export function shareWithCheckers(checked: CheckedProgram): void {
|
|
|
1828
1868
|
/**
|
|
1829
1869
|
* Checks `checked` on `checkers` threads and calls `done` once they are done. With `emits`, each thread
|
|
1830
1870
|
* emits the files it checks (`emitKinds` limits what). With `checking` false, the threads only emit:
|
|
1831
|
-
* for a program whose report has no semantic diagnostics, so that no file is checked in full.
|
|
1871
|
+
* for a program whose report has no semantic diagnostics, so that no file is checked in full. With
|
|
1872
|
+
* `emitsTo`, the emit results go only there, by file index, and nothing else the threads find reaches
|
|
1873
|
+
* the program: a declaration thread (parallel-emit.ts), while this thread's checker checks. With
|
|
1874
|
+
* `declarationsTo`, the same holds for the declaration errors of a `noEmit` program
|
|
1875
|
+
* (`checkersReportDeclarations`).
|
|
1832
1876
|
*/
|
|
1833
1877
|
export function startParallelCheck(
|
|
1834
1878
|
checked: CheckedProgram,
|
|
@@ -1837,6 +1881,8 @@ export function startParallelCheck(
|
|
|
1837
1881
|
emits: boolean = true,
|
|
1838
1882
|
emitKinds: number[] | undefined = undefined,
|
|
1839
1883
|
checking: boolean = true,
|
|
1884
|
+
emitsTo: (EmitResult | undefined)[] | undefined = undefined,
|
|
1885
|
+
declarationsTo: (Diagnostic[] | undefined)[] | undefined = undefined,
|
|
1840
1886
|
): void {
|
|
1841
1887
|
let count = checkers;
|
|
1842
1888
|
const start = performance.now();
|
|
@@ -1899,6 +1945,11 @@ export function startParallelCheck(
|
|
|
1899
1945
|
let remaining = count - firstThread;
|
|
1900
1946
|
let ownDone = false;
|
|
1901
1947
|
const finish = (): void => {
|
|
1948
|
+
if (emitsTo !== undefined || declarationsTo !== undefined) {
|
|
1949
|
+
mark('emitted on checker threads');
|
|
1950
|
+
done(performance.now() - start);
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1902
1953
|
if (checking) {
|
|
1903
1954
|
joinRanges(checked, units, perUnit, perFile);
|
|
1904
1955
|
// A thread that failed may have taken files it never checked: check them here.
|
|
@@ -1929,6 +1980,22 @@ export function startParallelCheck(
|
|
|
1929
1980
|
done(performance.now() - start);
|
|
1930
1981
|
};
|
|
1931
1982
|
const record = (result: CheckerResult): void => {
|
|
1983
|
+
if (declarationsTo !== undefined) {
|
|
1984
|
+
for (let k = 0; k < result.declarations.length; k++) declarationsTo[result.files[k]!] = result.declarations[k]!;
|
|
1985
|
+
if (trace)
|
|
1986
|
+
process.stderr.write(
|
|
1987
|
+
`checkers: emit-only thread ${result.index} found the declaration errors of ${result.declarations.length} files in ${result.checkTime.toFixed(1)}ms, done at ${(performance.now() - start).toFixed(1)}ms rss ${result.stats[7] ?? '?'}MB\n`,
|
|
1988
|
+
);
|
|
1989
|
+
return;
|
|
1990
|
+
}
|
|
1991
|
+
if (emitsTo !== undefined) {
|
|
1992
|
+
for (let k = 0; k < result.emits.length; k++) emitsTo[result.files[k]!] = result.emits[k]!;
|
|
1993
|
+
if (trace)
|
|
1994
|
+
process.stderr.write(
|
|
1995
|
+
`checkers: emit-only thread ${result.index} emitted ${result.emits.length} files in ${result.checkTime.toFixed(1)}ms, done at ${(performance.now() - start).toFixed(1)}ms rss ${result.stats[7] ?? '?'}MB\n`,
|
|
1996
|
+
);
|
|
1997
|
+
return;
|
|
1998
|
+
}
|
|
1932
1999
|
for (let k = 0; k < result.files.length; k++) perFile[result.files[k]!] = result.diagnostics[k]!;
|
|
1933
2000
|
for (let k = 0; k < result.declarations.length; k++)
|
|
1934
2001
|
perFileDeclarations[result.files[k]!] = result.declarations[k]!;
|
|
@@ -1991,10 +2058,28 @@ export function startParallelCheck(
|
|
|
1991
2058
|
mark('reporting checker created');
|
|
1992
2059
|
const emittedHere =
|
|
1993
2060
|
!mainChecks && emitKinds === undefined ? emitUncheckedFiles(checked, order, emits) : new Map<number, EmitResult>();
|
|
1994
|
-
if (mainChecks)
|
|
1995
|
-
|
|
1996
|
-
|
|
2061
|
+
if (mainChecks) {
|
|
2062
|
+
const own = checkFromQueue(
|
|
2063
|
+
checked,
|
|
2064
|
+
0,
|
|
2065
|
+
new Int32Array(counter),
|
|
2066
|
+
order,
|
|
2067
|
+
epoch,
|
|
2068
|
+
true,
|
|
2069
|
+
0,
|
|
2070
|
+
[],
|
|
2071
|
+
emits,
|
|
2072
|
+
emitKinds,
|
|
2073
|
+
checking,
|
|
2074
|
+
units,
|
|
1997
2075
|
);
|
|
2076
|
+
record(own);
|
|
2077
|
+
// A thread continues where this checker stopped (`checkFromQueue`).
|
|
2078
|
+
if (own.more) {
|
|
2079
|
+
remaining++;
|
|
2080
|
+
spawn(0, own.reserved);
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
1998
2083
|
ownDone = true;
|
|
1999
2084
|
if (remaining === 0) finish();
|
|
2000
2085
|
}
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Emitting a program's JavaScript on threads while the main thread checks it (`--checkers 1` in
|
|
3
|
+
* scriptc builds).
|
|
4
|
+
*
|
|
5
|
+
* JavaScript emit needs nothing of a checker but how decorator metadata serializes a type: import
|
|
6
|
+
* elision, export names and const enum values come from syntax, the binder's symbols and the other
|
|
7
|
+
* files of the program (`SyntacticEmitResolver`, `ReferenceResolver`). So emit threads create no
|
|
8
|
+
* checker. They receive the published program, emit with a `JavaScriptEmitter` the files the main
|
|
9
|
+
* thread hands out through a shared counter, write the outputs and send back their emit results.
|
|
10
|
+
* The main thread checks meanwhile, and so does the declaration thread, a checker thread that only
|
|
11
|
+
* emits declarations (`startDeclarationThread`): `--checkers` counts the checkers that check. Then the
|
|
12
|
+
* main thread records how its checker serializes the types of the decorator metadata of the files that
|
|
13
|
+
* have it (`recordMetadataKinds`), which a second wave of emit threads emits with those answers as plain
|
|
14
|
+
* data (`startMetadataEmit`). Which thread emits a file doesn't change its output, and the main
|
|
15
|
+
* thread's checker finds every diagnostic but those of declaration emit.
|
|
16
|
+
*/
|
|
17
|
+
import type { Worker } from 'node:worker_threads';
|
|
18
|
+
import { availableParallelism } from 'node:os';
|
|
19
|
+
import { publish } from '@scriptc/threads';
|
|
20
|
+
import { lastNodeIdValue, type SourceFile } from './ast.js';
|
|
21
|
+
import { JavaScriptEmitter, type OutputWriter } from './compile-program.js';
|
|
22
|
+
import type { Diagnostic, DiagnosticFile } from './diagnostics.js';
|
|
23
|
+
import { EmitOnly, EmitResult, sourceFileMayBeEmitted } from './emitter.js';
|
|
24
|
+
import {
|
|
25
|
+
CheckerResult,
|
|
26
|
+
packResult,
|
|
27
|
+
packsResults,
|
|
28
|
+
type PackedResult,
|
|
29
|
+
startParallelCheck,
|
|
30
|
+
unpackResult,
|
|
31
|
+
} from './parallel-check.js';
|
|
32
|
+
import type { CheckedProgram } from './program-checker.js';
|
|
33
|
+
import type { Program } from './program.js';
|
|
34
|
+
import { lastSymbolIdValue } from './symbol.js';
|
|
35
|
+
import { mark } from './timeline.js';
|
|
36
|
+
import { SubtreeFacts, SyntaxFacts } from './transformers/syntax-facts.js';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Starts emit thread `index` (main.ts owns the worker entry point) on the shared, published
|
|
40
|
+
* `program`: it emits the files of `files` at the positions it takes from `counter`, with the decorator
|
|
41
|
+
* metadata `kinds` the main thread recorded (`startMetadataEmit`); ids continue above `nodeIds` and
|
|
42
|
+
* `symbolIds`.
|
|
43
|
+
*/
|
|
44
|
+
export type EmitSpawn = (
|
|
45
|
+
program: Program,
|
|
46
|
+
index: number,
|
|
47
|
+
counter: SharedArrayBuffer,
|
|
48
|
+
files: number[],
|
|
49
|
+
kinds: number[],
|
|
50
|
+
nodeIds: number,
|
|
51
|
+
symbolIds: number,
|
|
52
|
+
) => Worker;
|
|
53
|
+
|
|
54
|
+
export class ParallelEmit {
|
|
55
|
+
/** Set where emit threads can be started. */
|
|
56
|
+
spawn: EmitSpawn | undefined = undefined;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export const parallelEmit = new ParallelEmit();
|
|
60
|
+
|
|
61
|
+
/** The most emit threads: four emitted VS Code's JavaScript as fast as 15 did while one thread checked it. */
|
|
62
|
+
const maxEmitThreads = 4;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The least text of the files to emit, in characters, for emit threads: starting them and sharing
|
|
66
|
+
* the program takes about as long as emitting this much.
|
|
67
|
+
*/
|
|
68
|
+
const littleEmitText = 50_000;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The number of threads to emit the JavaScript of `program` on, or 0 to emit it on this thread:
|
|
72
|
+
* where threads can start and the program emits enough, one fewer than the CPUs, at most
|
|
73
|
+
* `maxEmitThreads`. `emitKinds` are what the emit of each file includes (`emitsJavaScript`).
|
|
74
|
+
*/
|
|
75
|
+
export function emitThreads(
|
|
76
|
+
program: Program,
|
|
77
|
+
singleThreaded: boolean | undefined,
|
|
78
|
+
emitKinds: readonly number[] | undefined = undefined,
|
|
79
|
+
): number {
|
|
80
|
+
if (parallelEmit.spawn === undefined || singleThreaded === true) return 0;
|
|
81
|
+
const cpus = availableParallelism();
|
|
82
|
+
if (cpus < 2) return 0;
|
|
83
|
+
const files = program.getSourceFiles();
|
|
84
|
+
let text = 0;
|
|
85
|
+
for (let i = 0; i < files.length; i++)
|
|
86
|
+
if (emitsJavaScript(program, emitKinds, i)) {
|
|
87
|
+
text += files[i]!.end;
|
|
88
|
+
if (text >= littleEmitText) return Math.min(cpus - 1, maxEmitThreads);
|
|
89
|
+
}
|
|
90
|
+
return 0;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Whether the emit of file `index` of `program` includes its JavaScript. `emitKinds` are what the emit of
|
|
95
|
+
* each file includes, by file index: an `EmitOnly`, or -1 for nothing, as an incremental program's pending
|
|
96
|
+
* emit selects (`IncrementalProgram.emitKindsOnCheckers`); `undefined` where every file that may be emitted
|
|
97
|
+
* is emitted in full.
|
|
98
|
+
*/
|
|
99
|
+
function emitsJavaScript(program: Program, emitKinds: readonly number[] | undefined, index: number): boolean {
|
|
100
|
+
if (emitKinds === undefined) return sourceFileMayBeEmitted(program.getSourceFiles()[index]!, program.options);
|
|
101
|
+
const kind = emitKinds[index]!;
|
|
102
|
+
return kind === EmitOnly.All || kind === EmitOnly.Js;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Whether the JavaScript of `file` needs decorator metadata kinds from the checker (`emitDecoratorMetadata`
|
|
107
|
+
* asks it how the types of decorated declarations serialize): a file with decorators is emitted by the second
|
|
108
|
+
* wave of emit threads, with the kinds the main thread's checker recorded after checking (`startMetadataEmit`).
|
|
109
|
+
*/
|
|
110
|
+
function needsMetadataKinds(file: SourceFile, program: Program): boolean {
|
|
111
|
+
return program.options.emitDecoratorMetadata === true && new SyntaxFacts().has(file, SubtreeFacts.ContainsDecorators);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Emits the JavaScript of the files of `checked` whose emit includes it (`emitKinds`) on `threads` threads
|
|
116
|
+
* and calls `done` once they are emitted; the program must be shared (`shareWithCheckers`). The emit results
|
|
117
|
+
* go to `checked.emittedInParallel`, which the program's emit (`PlainProgram`, `IncrementalProgram`) completes
|
|
118
|
+
* on this thread. Returns at once, with the indexes of the files left for `startMetadataEmit`: the results
|
|
119
|
+
* arrive on this thread's event loop.
|
|
120
|
+
*/
|
|
121
|
+
export function startParallelEmit(
|
|
122
|
+
checked: CheckedProgram,
|
|
123
|
+
threads: number,
|
|
124
|
+
done: () => void,
|
|
125
|
+
emitKinds: readonly number[] | undefined = undefined,
|
|
126
|
+
): number[] {
|
|
127
|
+
const program = checked.program;
|
|
128
|
+
const files = program.getSourceFiles();
|
|
129
|
+
const order: number[] = [];
|
|
130
|
+
const metadata: number[] = [];
|
|
131
|
+
for (let i = 0; i < files.length; i++) {
|
|
132
|
+
const file = files[i]!;
|
|
133
|
+
if (!emitsJavaScript(program, emitKinds, i)) continue;
|
|
134
|
+
if (needsMetadataKinds(file, program)) metadata.push(i);
|
|
135
|
+
else order.push(i);
|
|
136
|
+
}
|
|
137
|
+
const perFileEmits: (EmitResult | undefined)[] = [];
|
|
138
|
+
for (let i = 0; i < files.length; i++) perFileEmits.push(undefined);
|
|
139
|
+
checked.emittedInParallel = perFileEmits;
|
|
140
|
+
checked.javaScriptEmittedOnThreads = true;
|
|
141
|
+
spawnEmitThreads(checked, threads, order, [], done);
|
|
142
|
+
return metadata;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Emits the JavaScript of `files` (indexes, from `startParallelEmit`), whose decorator metadata asks the
|
|
147
|
+
* checker, on `threads` threads with the kinds that this thread's checker gave (`recordMetadataKinds`), and
|
|
148
|
+
* calls `done` once they are emitted. Called once this thread has checked the program, so that asking for the
|
|
149
|
+
* kinds doesn't change what checking finds.
|
|
150
|
+
*/
|
|
151
|
+
export function startMetadataEmit(
|
|
152
|
+
checked: CheckedProgram,
|
|
153
|
+
threads: number,
|
|
154
|
+
files: number[],
|
|
155
|
+
kinds: number[],
|
|
156
|
+
done: () => void,
|
|
157
|
+
): void {
|
|
158
|
+
spawnEmitThreads(checked, Math.min(threads, files.length), files, kinds, done);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function spawnEmitThreads(
|
|
162
|
+
checked: CheckedProgram,
|
|
163
|
+
threads: number,
|
|
164
|
+
files: number[],
|
|
165
|
+
kinds: number[],
|
|
166
|
+
done: () => void,
|
|
167
|
+
): void {
|
|
168
|
+
const program = checked.program;
|
|
169
|
+
const sourceFiles = program.getSourceFiles();
|
|
170
|
+
const perFileEmits = checked.emittedInParallel!;
|
|
171
|
+
const order = files.slice();
|
|
172
|
+
// The longest files first, so that the threads finish together.
|
|
173
|
+
order.sort((a, b) => sourceFiles[b]!.end - sourceFiles[a]!.end || a - b);
|
|
174
|
+
publish(order);
|
|
175
|
+
publish(kinds);
|
|
176
|
+
mark(`planned ${order.length} files to emit`);
|
|
177
|
+
const trace = process.env.TSC_TS_CHECKERS_TRACE !== undefined;
|
|
178
|
+
const start = performance.now();
|
|
179
|
+
const counter = new SharedArrayBuffer(4);
|
|
180
|
+
const diagnosticFiles = new Map<string, DiagnosticFile>();
|
|
181
|
+
let remaining = threads;
|
|
182
|
+
if (remaining === 0) done();
|
|
183
|
+
for (let i = 0; i < threads; i++) {
|
|
184
|
+
const worker = parallelEmit.spawn!(program, i, counter, order, kinds, lastNodeIdValue(), lastSymbolIdValue());
|
|
185
|
+
let ended = false;
|
|
186
|
+
const end = (): void => {
|
|
187
|
+
if (ended) return;
|
|
188
|
+
ended = true;
|
|
189
|
+
remaining--;
|
|
190
|
+
if (remaining === 0) done();
|
|
191
|
+
};
|
|
192
|
+
worker.on('message', (message: unknown) => {
|
|
193
|
+
mark(`result from e${i}`);
|
|
194
|
+
const result = packsResults
|
|
195
|
+
? unpackResult(message as PackedResult, program, diagnosticFiles)
|
|
196
|
+
: publish((message as { result: CheckerResult }).result);
|
|
197
|
+
for (let k = 0; k < result.files.length; k++) perFileEmits[result.files[k]!] = result.emits[k]!;
|
|
198
|
+
if (trace)
|
|
199
|
+
process.stderr.write(
|
|
200
|
+
`emit threads: thread ${i} emitted ${result.files.length} files${kinds.length > 0 ? ` with metadata kinds (${result.stats[0] ?? 0} missed a kind)` : ''} in ${result.checkTime.toFixed(1)}ms, done at ${(performance.now() - start).toFixed(1)}ms\n`,
|
|
201
|
+
);
|
|
202
|
+
end();
|
|
203
|
+
});
|
|
204
|
+
// The files a failed thread took and did not send back are emitted on this thread (`PlainProgram.emit`).
|
|
205
|
+
worker.on('error', (error: unknown) => {
|
|
206
|
+
if (trace) process.stderr.write(`emit threads: thread ${i} failed: ${String(error)}\n`);
|
|
207
|
+
end();
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
mark('emit threads spawned');
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** The most checkers that a declaration thread makes faster (`declarationThreadPays`). */
|
|
214
|
+
const maxDeclarationThreadCheckers = 8;
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Whether a declaration thread (`startDeclarationThread`) shortens a compile with `checkers` checkers: it
|
|
218
|
+
* takes the declaration emit off the checkers, but a checker that only emits declarations takes about as long
|
|
219
|
+
* as the whole check of VS Code or typeorm at 16 checkers. Measured with declarations on both: 6-16% faster
|
|
220
|
+
* from 1 to 8 checkers, about even at 12, and 32-38% slower at 16. It also needs a CPU of its own.
|
|
221
|
+
*/
|
|
222
|
+
export function declarationThreadPays(checkers: number): boolean {
|
|
223
|
+
return checkers <= maxDeclarationThreadCheckers && checkers < availableParallelism();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Starts the declaration thread: one checker thread that only emits, while the program is checked on this
|
|
228
|
+
* thread (`--checkers 1`) or on checker threads that emit only JavaScript, and calls `done` once it is done. It
|
|
229
|
+
* emits the declarations of every file of `checked` whose emit includes them (`emitKinds`), and also the
|
|
230
|
+
* JavaScript of `metadata` (file indexes, from `startParallelEmit`), whose decorator metadata asks a checker. Its
|
|
231
|
+
* checker checks only what emit asks; it reports nothing but the emit results, which it writes and which the
|
|
232
|
+
* returned array receives by file index (undefined for a file it didn't emit, which the program's emit then
|
|
233
|
+
* emits here). Declaration files and decorator metadata are the same with any checker that emits them.
|
|
234
|
+
*/
|
|
235
|
+
export function startDeclarationThread(
|
|
236
|
+
checked: CheckedProgram,
|
|
237
|
+
metadata: readonly number[],
|
|
238
|
+
done: () => void,
|
|
239
|
+
emitKinds: readonly number[] | undefined = undefined,
|
|
240
|
+
): (EmitResult | undefined)[] {
|
|
241
|
+
const program = checked.program;
|
|
242
|
+
const options = program.options;
|
|
243
|
+
const files = program.getSourceFiles();
|
|
244
|
+
const kinds: number[] = [];
|
|
245
|
+
const results: (EmitResult | undefined)[] = [];
|
|
246
|
+
for (let i = 0; i < files.length; i++) {
|
|
247
|
+
let declarations: boolean;
|
|
248
|
+
if (emitKinds === undefined) declarations = sourceFileMayBeEmitted(files[i]!, options);
|
|
249
|
+
else declarations = emitKinds[i] === EmitOnly.All || emitKinds[i] === EmitOnly.Dts;
|
|
250
|
+
kinds.push(declarations ? EmitOnly.Dts : -1);
|
|
251
|
+
results.push(undefined);
|
|
252
|
+
}
|
|
253
|
+
for (const i of metadata) kinds[i] = emitKinds === undefined ? EmitOnly.All : emitKinds[i]!;
|
|
254
|
+
startParallelCheck(checked, 1, () => done(), true, kinds, false, results);
|
|
255
|
+
mark('declaration thread spawned');
|
|
256
|
+
return results;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Starts the declaration thread of a `noEmit` program that reports the errors of declaration emit
|
|
261
|
+
* (`checkersReportDeclarations`): one checker thread that finds the declaration errors of every file of
|
|
262
|
+
* `checked` while the program is checked on this thread, and calls `done` once it is done. Its checker
|
|
263
|
+
* checks only what declaration emit asks; it reports nothing but the declaration errors, which `done`
|
|
264
|
+
* receives by file index (undefined for a file it didn't reach, whose errors
|
|
265
|
+
* `PlainProgram.getDeclarationDiagnostics` then finds here).
|
|
266
|
+
*/
|
|
267
|
+
export function startDeclarationErrorThread(
|
|
268
|
+
checked: CheckedProgram,
|
|
269
|
+
done: (declarations: (Diagnostic[] | undefined)[]) => void,
|
|
270
|
+
): void {
|
|
271
|
+
const results: (Diagnostic[] | undefined)[] = [];
|
|
272
|
+
for (let i = 0; i < checked.program.getSourceFiles().length; i++) results.push(undefined);
|
|
273
|
+
startParallelCheck(checked, 1, () => done(results), true, undefined, false, undefined, results);
|
|
274
|
+
mark('declaration thread spawned');
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* On emit thread `index`: emits the JavaScript of the files of `order` at the positions it takes
|
|
279
|
+
* from `counter` until none are left, writing it with `writer`; `kinds` are the decorator metadata kinds
|
|
280
|
+
* the main thread recorded for them (`recordMetadataKinds`), if any. `stats` has the number of files that
|
|
281
|
+
* missed a kind.
|
|
282
|
+
*/
|
|
283
|
+
export function emitFromQueue(
|
|
284
|
+
program: Program,
|
|
285
|
+
index: number,
|
|
286
|
+
counter: Int32Array,
|
|
287
|
+
order: readonly number[],
|
|
288
|
+
kinds: readonly number[],
|
|
289
|
+
writer: OutputWriter,
|
|
290
|
+
): CheckerResult {
|
|
291
|
+
const start = performance.now();
|
|
292
|
+
const files = program.getSourceFiles();
|
|
293
|
+
const emitter = new JavaScriptEmitter(program, kinds);
|
|
294
|
+
mark('emitter ready');
|
|
295
|
+
const done: number[] = [];
|
|
296
|
+
const diagnostics: Diagnostic[][] = [];
|
|
297
|
+
const emits: EmitResult[] = [];
|
|
298
|
+
let missed = 0;
|
|
299
|
+
for (;;) {
|
|
300
|
+
const position = Atomics.add(counter, 0, 1);
|
|
301
|
+
if (position >= order.length) break;
|
|
302
|
+
const i = order[position]!;
|
|
303
|
+
const result = emitter.emit(files[i]!, writer);
|
|
304
|
+
// A file whose emit missed a kind isn't sent back: the main thread emits it again, with its checker.
|
|
305
|
+
if (result === undefined) {
|
|
306
|
+
missed++;
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
done.push(i);
|
|
310
|
+
diagnostics.push([]);
|
|
311
|
+
emits.push(result);
|
|
312
|
+
}
|
|
313
|
+
mark(`emitted ${done.length} files`);
|
|
314
|
+
return new CheckerResult(index, done, diagnostics, [], [], performance.now() - start, [missed], false, [], emits);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** What an emit thread posts back: `result`, packed for `postMessage` where results are packed. */
|
|
318
|
+
export function postedEmitResult(result: CheckerResult, program: Program): PackedResult | { result: CheckerResult } {
|
|
319
|
+
return packsResults ? packResult(result, program) : { result: publish(result) };
|
|
320
|
+
}
|