testguard-cli 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +21 -5
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +21 -0
- package/spec/lib/validate.mjs +3 -0
- package/spec/schemas/evidence.schema.json +3 -2
- package/src/cli.mjs +1 -1
- package/src/probe/probe.mjs +1 -1
- package/src/probe/runners/discovery.mjs +12 -1
- package/src/probe/runners/index.mjs +3 -1
- package/src/probe/runners/node/entry.mjs +11 -0
- package/src/probe/runners/node/report.mjs +66 -0
- package/src/probe/runners/node/runner.mjs +42 -0
- package/src/probe/runners/node-test.mjs +130 -0
- package/src/probe/runners/shared.mjs +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.18.1] - 2026-10-01
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.18.0`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- test: consolidate native runtime acceptance checks
|
|
17
|
+
- perf: isolate native admission mutation defenders
|
|
18
|
+
- perf: size bounded report buffers to observed files
|
|
19
|
+
- chore(homebrew): sha256 for v0.18.0 (#194)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
- Runner report and discovery-config reads reserve memory for the observed
|
|
25
|
+
file size instead of the maximum allowed size, retaining the hard byte
|
|
26
|
+
ceiling and checking file growth before parsing.
|
|
27
|
+
- Native-runner admission self-probes use focused defenders, preserving the
|
|
28
|
+
fault recipes and full runner acceptance coverage while avoiding unrelated
|
|
29
|
+
timeout and process-cleanup checks for each admission confirmation.
|
|
30
|
+
|
|
31
|
+
## [0.18.0] - 2026-10-01
|
|
32
|
+
|
|
33
|
+
Automated weekly release — everything merged since `v0.17.0`.
|
|
34
|
+
|
|
35
|
+
### Changed
|
|
36
|
+
|
|
37
|
+
- Merge branch 'main' into dependabot/github_actions/all-actions-d5b8684899
|
|
38
|
+
- feat: native Node probes with measured startup savings (#192)
|
|
39
|
+
- chore(homebrew): sha256 for v0.17.0 (#191)
|
|
40
|
+
- chore(deps): bump actions/cache
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
### Added
|
|
44
|
+
|
|
45
|
+
- Explicit `--runner node-test` support for plain JavaScript projects using
|
|
46
|
+
Node's built-in test framework, with native entry-file discovery, fresh
|
|
47
|
+
process confirmations and bounded structured reports. No runner is installed
|
|
48
|
+
and existing automatic runner selection is unchanged.
|
|
49
|
+
|
|
10
50
|
## [0.17.0] - 2026-10-01
|
|
11
51
|
|
|
12
52
|
Automated weekly release — everything merged since `v0.16.0`.
|
package/README.md
CHANGED
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.18.1` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.18.1/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -302,7 +302,23 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
302
302
|
version, its own bin script — and only then from an executable on PATH,
|
|
303
303
|
which the evidence records as `runner.source: "path"`; `npx` is never
|
|
304
304
|
asked, because its cache answers for packages a project does not have.
|
|
305
|
-
|
|
305
|
+
**Node's built-in framework** uses explicit `--runner node-test` with plain
|
|
306
|
+
JavaScript test files and default Node collection. It uses the Node executable
|
|
307
|
+
running TestGuard, records `runner.source: "builtin"` and its actual version,
|
|
308
|
+
and installs nothing. Discovery loads test modules with test bodies filtered
|
|
309
|
+
out, so top-level code can run. Custom loaders, `NODE_OPTIONS`, `NODE_PATH`
|
|
310
|
+
and module-mocking configurations are outside this initial adapter's scope.
|
|
311
|
+
`auto` keeps its existing runner order. Each confirmation uses fresh isolated
|
|
312
|
+
processes and the same worker, timeout and confirmation rules.
|
|
313
|
+
|
|
314
|
+
To reproduce the narrow native-runner comparison, run
|
|
315
|
+
`node .github/scripts/benchmark-node-runner.mjs --out /absolute/receipt-dir`
|
|
316
|
+
from this repository after installing its pinned development dependencies.
|
|
317
|
+
It compares 17 identical classifier inputs and four existing faults through
|
|
318
|
+
complete probes, with three confirmations, fresh worker receipts and
|
|
319
|
+
reversed execution order. Results describe that workload and runtime only.
|
|
320
|
+
|
|
321
|
+
Other frameworks go through `--runner-cmd`. Each runner is proven against
|
|
306
322
|
its own copy of the known-answer fixture.
|
|
307
323
|
|
|
308
324
|
**Python** runs under `pytest` when the project's interpreter can import
|
|
@@ -769,12 +785,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
769
785
|
# GitHub Actions
|
|
770
786
|
- uses: actions/checkout@v4
|
|
771
787
|
with: { fetch-depth: 0 }
|
|
772
|
-
- uses: raccioly/testguard@v0.
|
|
788
|
+
- uses: raccioly/testguard@v0.18.1
|
|
773
789
|
with: { command: gate }
|
|
774
790
|
|
|
775
791
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
776
792
|
include:
|
|
777
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
793
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.18.1/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
778
794
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
779
795
|
```
|
|
780
796
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.1",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/spec/GATE-SEMANTICS.md
CHANGED
|
@@ -311,6 +311,27 @@ Rules that follow from the table:
|
|
|
311
311
|
competing owned runners or a declared defender absent from every manifest
|
|
312
312
|
are precondition failures, never heuristic routing decisions.
|
|
313
313
|
|
|
314
|
+
The explicit `node-test` adapter uses Node's native collection with test
|
|
315
|
+
bodies excluded by a never-matching name pattern. Collection still loads
|
|
316
|
+
test modules and may run their top-level code; a load error, incomplete
|
|
317
|
+
collection or budget expiry refuses discovery. An injected child preload
|
|
318
|
+
records each actual entry file before project evaluation, so imported test
|
|
319
|
+
declarations cannot masquerade as additional entry files. No filename
|
|
320
|
+
fallback or inferred npm-script configuration is permitted. This adapter
|
|
321
|
+
supports plain JavaScript under Node's default collection, without custom
|
|
322
|
+
loaders, `NODE_OPTIONS` or `NODE_PATH`; it is never selected by `auto`.
|
|
323
|
+
Each confirmation uses fresh, isolated Node test processes under the
|
|
324
|
+
existing worker ceiling. Reports count executed test results, not individual
|
|
325
|
+
assertion calls: suite and file-container summaries are not extra tests,
|
|
326
|
+
and skipped/TODO results cannot establish a green defender by themselves.
|
|
327
|
+
A test-body failure counts as a rejection; a load error, hook failure,
|
|
328
|
+
cancellation or timeout never counts as an assertion kill. Malformed or
|
|
329
|
+
unfinished reports are load errors. Evidence names `node-test`, records
|
|
330
|
+
`source: builtin` and the actual Node version, and binds the executable and
|
|
331
|
+
reporter strategy to its discovery hash. Another framework's evidence
|
|
332
|
+
cannot supply its confirmations. Native module-mocking configurations are
|
|
333
|
+
outside this adapter's initial admission boundary.
|
|
334
|
+
|
|
314
335
|
Static string literals are conservative dependency candidates, not proof
|
|
315
336
|
of a filesystem dependency. Missing candidates and ordinary directories
|
|
316
337
|
(including `/` in URL bases and string separators) are ignored. An existing
|
package/spec/lib/validate.mjs
CHANGED
|
@@ -183,6 +183,9 @@ const semantic = {
|
|
|
183
183
|
|
|
184
184
|
evidence(doc) {
|
|
185
185
|
const errors = [];
|
|
186
|
+
if (doc.run.runner?.source === 'builtin' && (doc.run.runner.name !== 'node-test' || !/^(?:[2-9]\d|[1-9]\d{2,})\.\d+\.\d+(?:[-+].+)?$/.test(doc.run.runner.version ?? ''))) {
|
|
187
|
+
errors.push({ path: '/run/runner', message: 'builtin source requires node-test and its actual Node version (20 or newer)' });
|
|
188
|
+
}
|
|
186
189
|
if (doc.run.measurements) {
|
|
187
190
|
const m = doc.run.measurements;
|
|
188
191
|
if (Math.abs(m.elapsedMs - m.runnerMs - m.overheadMs) > 0.001) errors.push({ path: '/run/measurements', message: 'elapsedMs must equal runnerMs plus overheadMs' });
|
|
@@ -81,10 +81,11 @@
|
|
|
81
81
|
"maxLength": 64
|
|
82
82
|
},
|
|
83
83
|
"source": {
|
|
84
|
-
"description": "`project`: the runner package resolved from the project
|
|
84
|
+
"description": "`project`: the runner package resolved from the project. `path`: an executable on PATH ran instead. `builtin`: the explicitly selected node-test engine uses the Node executable running TestGuard; its version is required.",
|
|
85
85
|
"enum": [
|
|
86
86
|
"project",
|
|
87
|
-
"path"
|
|
87
|
+
"path",
|
|
88
|
+
"builtin"
|
|
88
89
|
]
|
|
89
90
|
}
|
|
90
91
|
},
|
package/src/cli.mjs
CHANGED
|
@@ -64,7 +64,7 @@ probe
|
|
|
64
64
|
append-only lines everywhere else, so a redirected log and CI see progress
|
|
65
65
|
instead of twenty silent minutes. Always on stderr; ndjson also streams verdicts
|
|
66
66
|
--confirm below 3 is PROVISIONAL: verdicts print with "?", evidence goes to .testguard/evidence-provisional.json
|
|
67
|
-
--runner <name> vitest | jest | playwright | python | pytest | unittest | auto
|
|
67
|
+
--runner <name> vitest | jest | playwright | python | pytest | unittest | node-test | auto
|
|
68
68
|
(default: auto — first of vitest, jest, python that resolves. A defender under playwright's testDir
|
|
69
69
|
always runs under playwright, and a .py defender always under python, whatever the project runner.
|
|
70
70
|
python picks pytest when the interpreter can import it and stdlib unittest otherwise; pytest and
|
package/src/probe/probe.mjs
CHANGED
|
@@ -403,7 +403,7 @@ export async function probe({
|
|
|
403
403
|
startedAt,
|
|
404
404
|
finishedAt: new Date().toISOString(),
|
|
405
405
|
repo: { head, dirty, ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
|
|
406
|
-
runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
|
|
406
|
+
runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}), ...(runnerSource === 'builtin' ? { source: runnerSource } : {}) },
|
|
407
407
|
...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
|
|
408
408
|
confirmRuns,
|
|
409
409
|
measurements,
|
|
@@ -130,12 +130,21 @@ const readBoundedFile = (path, maxBytes, label, { optional = false } = {}) => {
|
|
|
130
130
|
const stat = fstatSync(descriptor);
|
|
131
131
|
if (!stat.isFile()) throw new DiscoveryError(`${label} is not a regular file`);
|
|
132
132
|
if (stat.size > maxBytes) throw new DiscoveryError(`${label} exceeded ${maxBytes} bytes`);
|
|
133
|
-
|
|
133
|
+
// Reserve the observed file size, not the allowance for the largest report.
|
|
134
|
+
// The extra byte detects growth; a full buffer grows within the same hard
|
|
135
|
+
// ceiling so a concurrently appended suffix can never be silently dropped.
|
|
136
|
+
let bytes = Buffer.alloc(stat.size + 1);
|
|
134
137
|
let length = 0;
|
|
135
138
|
while (length < bytes.length) {
|
|
136
139
|
const read = readSync(descriptor, bytes, length, bytes.length - length, null);
|
|
137
140
|
if (read === 0) break;
|
|
138
141
|
length += read;
|
|
142
|
+
if (length > maxBytes) throw new DiscoveryError(`${label} exceeded ${maxBytes} bytes`);
|
|
143
|
+
if (length === bytes.length) {
|
|
144
|
+
const grown = Buffer.alloc(Math.min(maxBytes + 1, Math.max(4096, bytes.length * 2)));
|
|
145
|
+
bytes.copy(grown, 0, 0, length);
|
|
146
|
+
bytes = grown;
|
|
147
|
+
}
|
|
139
148
|
}
|
|
140
149
|
if (length > maxBytes) throw new DiscoveryError(`${label} exceeded ${maxBytes} bytes`);
|
|
141
150
|
return bytes.subarray(0, length);
|
|
@@ -156,6 +165,7 @@ export function runDiscoveryProcess({
|
|
|
156
165
|
timeoutMs = DISCOVERY_TIMEOUT_MS,
|
|
157
166
|
maxOutputBytes = MAX_DISCOVERY_OUTPUT_BYTES,
|
|
158
167
|
env = {},
|
|
168
|
+
cleanupOnClose = false,
|
|
159
169
|
}) {
|
|
160
170
|
validateArgv(argv);
|
|
161
171
|
if (typeof projectDir !== 'string' || projectDir.length === 0 || projectDir.includes('\0')) throw new DiscoveryError('projectDir must be a non-empty path');
|
|
@@ -223,6 +233,7 @@ export function runDiscoveryProcess({
|
|
|
223
233
|
|
|
224
234
|
child.on('close', (code, signal) => {
|
|
225
235
|
clearTimeout(timer);
|
|
236
|
+
if (cleanupOnClose) terminateProcessTree(child);
|
|
226
237
|
if (settled) return;
|
|
227
238
|
if (terminalError) {
|
|
228
239
|
reject(terminalError);
|
|
@@ -2,6 +2,7 @@ import * as vitest from './vitest.mjs';
|
|
|
2
2
|
import * as jest from './jest.mjs';
|
|
3
3
|
import * as playwright from './playwright.mjs';
|
|
4
4
|
import * as pythonRunner from './python.mjs';
|
|
5
|
+
import * as nodeTest from './node-test.mjs';
|
|
5
6
|
import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs } from './discovery.mjs';
|
|
6
7
|
|
|
7
8
|
export const RUNNERS = {
|
|
@@ -14,9 +15,10 @@ export const RUNNERS = {
|
|
|
14
15
|
// evidence means.
|
|
15
16
|
pytest: pythonRunner.pinned('pytest'),
|
|
16
17
|
unittest: pythonRunner.pinned('unittest'),
|
|
18
|
+
'node-test': nodeTest,
|
|
17
19
|
};
|
|
18
20
|
|
|
19
|
-
export const RUNNER_NAMES = ['vitest', 'jest', 'playwright', 'python', 'pytest', 'unittest'];
|
|
21
|
+
export const RUNNER_NAMES = ['vitest', 'jest', 'playwright', 'python', 'pytest', 'unittest', 'node-test'];
|
|
20
22
|
|
|
21
23
|
/**
|
|
22
24
|
* Runners that own files rather than projects: a defender under Playwright's
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { appendFileSync, statSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
// Node sets this only in its isolated test workers. Record the entry before
|
|
4
|
+
// evaluating project code; a declaration imported from a helper is not an entry.
|
|
5
|
+
if (process.env.NODE_TEST_CONTEXT?.startsWith('child')) {
|
|
6
|
+
const receipt = JSON.stringify({ file: process.argv[1], pid: process.pid, version: process.versions.node });
|
|
7
|
+
if (Buffer.byteLength(receipt) > 8192 || statSync(process.env.TESTGUARD_NODE_ENTRIES).size + Buffer.byteLength(receipt) > 16 * 1024 * 1024) {
|
|
8
|
+
throw new Error('node-test entry receipts exceeded their byte bound');
|
|
9
|
+
}
|
|
10
|
+
appendFileSync(process.env.TESTGUARD_NODE_ENTRIES, `${receipt},`);
|
|
11
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { isAbsolute, relative, resolve, sep } from 'node:path';
|
|
2
|
+
|
|
3
|
+
const MAX_RESULTS = 100_000;
|
|
4
|
+
const MAX_REPORT_BYTES = 16 * 1024 * 1024;
|
|
5
|
+
const boundedText = (value) => String(value ?? '').slice(0, 1000);
|
|
6
|
+
|
|
7
|
+
/** Keep only structured identity, never arbitrary error objects or assertion values. */
|
|
8
|
+
const errorIdentity = (error) => error == null ? undefined : {
|
|
9
|
+
failureType: boundedText(error.failureType),
|
|
10
|
+
code: boundedText(error.code),
|
|
11
|
+
message: boundedText(error.message ?? error),
|
|
12
|
+
fileExit: Object.hasOwn(error, 'exitCode') || Object.hasOwn(error, 'signal'),
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/** Pure report accumulator. File/suite summaries never count as assertion kills. */
|
|
16
|
+
export function createReport({ projectDir, version, collect = false }) {
|
|
17
|
+
const cases = [];
|
|
18
|
+
const parents = [];
|
|
19
|
+
const pending = new Map();
|
|
20
|
+
let bytes = 0;
|
|
21
|
+
let failed = false;
|
|
22
|
+
let diagnostic = '';
|
|
23
|
+
const add = (item) => {
|
|
24
|
+
bytes += Buffer.byteLength(JSON.stringify(item));
|
|
25
|
+
if (cases.length >= MAX_RESULTS || bytes > MAX_REPORT_BYTES) throw new Error('node-test results exceeded their report bound');
|
|
26
|
+
cases.push(item);
|
|
27
|
+
};
|
|
28
|
+
const accept = ({ type, data }) => {
|
|
29
|
+
if (['test:stderr', 'test:diagnostic'].includes(type)) {
|
|
30
|
+
diagnostic = `${diagnostic}\n${String(data?.message ?? '')}`.slice(-65536);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (!['test:start', 'test:pass', 'test:fail'].includes(type)) return;
|
|
34
|
+
if (!data || typeof data.name !== 'string' || !Number.isSafeInteger(data.nesting) || data.nesting < 0) throw new Error('malformed node-test event identity');
|
|
35
|
+
const file = typeof data.file === 'string' ? resolve(projectDir, data.file) : undefined;
|
|
36
|
+
const names = parents;
|
|
37
|
+
const key = JSON.stringify([file, data.nesting, data.name]);
|
|
38
|
+
if (type === 'test:start') {
|
|
39
|
+
if (pending.size >= MAX_RESULTS || data.nesting > 1000) throw new Error('node-test pending results exceeded their bound');
|
|
40
|
+
pending.set(key, (pending.get(key) ?? 0) + 1);
|
|
41
|
+
names.length = data.nesting;
|
|
42
|
+
names[data.nesting] = data.name;
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (!pending.has(key)) throw new Error('node-test finished a result without its start');
|
|
46
|
+
if (pending.get(key) === 1) pending.delete(key);
|
|
47
|
+
else pending.set(key, pending.get(key) - 1);
|
|
48
|
+
const error = errorIdentity(data.details?.error);
|
|
49
|
+
const skipped = ['skip', 'todo', 'expectFailure'].some((key) => Object.hasOwn(data, key) && data[key] !== false);
|
|
50
|
+
if (type === 'test:fail' && !skipped) failed = true;
|
|
51
|
+
const container = data.nesting === 0 && (isAbsolute(data.name) || file && resolve(projectDir, data.name) === file)
|
|
52
|
+
&& (data.line === undefined || data.line === 1) && (data.column === undefined || data.column === 1);
|
|
53
|
+
const suite = data.details?.type === 'suite' || error?.failureType === 'subtestsFailed';
|
|
54
|
+
const name = [...names.slice(0, data.nesting), data.name].join(' > ');
|
|
55
|
+
const rel = file ? relative(projectDir, file).split(sep).join('/') : '';
|
|
56
|
+
// Refuse ambiguous/escaped attribution rather than invent a killer id.
|
|
57
|
+
const outside = rel === '..' || rel.startsWith('../') || isAbsolute(rel);
|
|
58
|
+
if (!container && (!rel || outside || !name || name.length > 1000)) throw new Error(outside ? 'node-test result attribution is outside the selected project' : 'node-test result has unavailable or unsupported attribution');
|
|
59
|
+
add({ file: rel, name, container, suite, skipped, passed: type === 'test:pass', ...(error ? { error } : {}) });
|
|
60
|
+
};
|
|
61
|
+
const finish = (entries) => {
|
|
62
|
+
if (pending.size) throw new Error('node-test stopped before every started result finished');
|
|
63
|
+
return { schemaVersion: 1, engine: 'node-test', version, collect, complete: true, failed, entries, cases, diagnostic };
|
|
64
|
+
};
|
|
65
|
+
return { accept, finish };
|
|
66
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { appendFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { readBoundedJsonFile } from '../discovery.mjs';
|
|
3
|
+
import { createReport } from './report.mjs';
|
|
4
|
+
|
|
5
|
+
// The tool may itself be called from a Node test worker. This new controller
|
|
6
|
+
// must initialize as a parent, rather than Node's nested-run no-op mode.
|
|
7
|
+
delete process.env.NODE_TEST_CONTEXT;
|
|
8
|
+
const { run } = await import('node:test');
|
|
9
|
+
|
|
10
|
+
const collect = process.env.TESTGUARD_NODE_COLLECT === '1';
|
|
11
|
+
const workers = Number(process.env.TESTGUARD_NODE_WORKERS);
|
|
12
|
+
const files = process.argv.slice(2);
|
|
13
|
+
if (!Number.isSafeInteger(workers) || workers < 1) throw new Error('invalid node-test worker ceiling');
|
|
14
|
+
const report = createReport({ projectDir: process.cwd(), version: process.versions.node, collect });
|
|
15
|
+
// run() on the Node20 floor uses argv for default native collection. The
|
|
16
|
+
// launcher is not a project test entry. Explicit file runs keep their paths.
|
|
17
|
+
process.argv = [process.execPath];
|
|
18
|
+
// The floor predates run({testNamePatterns}) and --test-concurrency. Put the
|
|
19
|
+
// name filter only in CHILD argv, keeping native file containers collectible.
|
|
20
|
+
if (collect) process.execArgv.push('--test-name-pattern=(?!)');
|
|
21
|
+
const cancellation = new AbortController();
|
|
22
|
+
const stream = run({ signal: cancellation.signal, concurrency: workers, ...(files.length ? { files } : {}), ...(collect ? { testNamePatterns: ['(?!)'] } : {}) });
|
|
23
|
+
try {
|
|
24
|
+
for await (const event of stream) report.accept(event);
|
|
25
|
+
appendFileSync(process.env.TESTGUARD_NODE_ENTRIES, 'null]}');
|
|
26
|
+
const entries = readBoundedJsonFile(process.env.TESTGUARD_NODE_ENTRIES).entries;
|
|
27
|
+
if (!Array.isArray(entries) || entries.at(-1) !== null) throw new Error('incomplete node-test entry receipts');
|
|
28
|
+
entries.pop();
|
|
29
|
+
const doc = report.finish(entries);
|
|
30
|
+
const text = JSON.stringify(doc);
|
|
31
|
+
if (Buffer.byteLength(text) > 16 * 1024 * 1024) throw new Error('node-test report exceeded its byte bound');
|
|
32
|
+
if (collect) process.stdout.write(text);
|
|
33
|
+
else writeFileSync(process.env.TESTGUARD_NODE_REPORT, text, { mode: 0o600 });
|
|
34
|
+
if (doc.failed) process.exitCode = 1;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
cancellation.abort();
|
|
37
|
+
stream.destroy();
|
|
38
|
+
// The Node20 floor's test harness can swallow an uncaught controller error.
|
|
39
|
+
// Retain bounded diagnostics while leaving the incomplete report unwritten.
|
|
40
|
+
process.stderr.write(`node-test reporter failed: ${String(error.message ?? error).slice(0, 1000)}\n`);
|
|
41
|
+
process.exitCode = 1;
|
|
42
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { closeSync, mkdtempSync, openSync, readFileSync, readSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { dirname, join, resolve } from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { runProcess } from './shared.mjs';
|
|
7
|
+
import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs, normalizeDiscoveredFiles, runDiscoveryProcess } from './discovery.mjs';
|
|
8
|
+
|
|
9
|
+
export const name = 'node-test';
|
|
10
|
+
export const testGlobs = []; // Native collection is authoritative; never guess from globs.
|
|
11
|
+
const HERE = join(dirname(fileURLToPath(import.meta.url)), 'node');
|
|
12
|
+
const RUNNER = join(HERE, 'runner.mjs');
|
|
13
|
+
const ENTRY = join(HERE, 'entry.mjs');
|
|
14
|
+
const VERSION = process.versions.node;
|
|
15
|
+
const strategyHash = () => {
|
|
16
|
+
const digest = createHash('sha256');
|
|
17
|
+
const descriptor = openSync(process.execPath, 'r');
|
|
18
|
+
try {
|
|
19
|
+
const bytes = Buffer.alloc(256 * 1024);
|
|
20
|
+
let length;
|
|
21
|
+
while ((length = readSync(descriptor, bytes, 0, bytes.length, null)) > 0) digest.update(bytes.subarray(0, length));
|
|
22
|
+
} finally { closeSync(descriptor); }
|
|
23
|
+
for (const file of [fileURLToPath(import.meta.url), ...['runner.mjs', 'entry.mjs', 'report.mjs'].map((file) => join(HERE, file)), ...['shared.mjs', 'discovery.mjs', 'lifecycle.mjs'].map((file) => join(HERE, '..', file))]) digest.update(readFileSync(file));
|
|
24
|
+
return digest.digest('hex');
|
|
25
|
+
};
|
|
26
|
+
const admissionError = () => Number(VERSION.split('.')[0]) < 20 ? 'node-test requires Node20 or newer'
|
|
27
|
+
: process.env.NODE_OPTIONS || process.env.NODE_PATH ? 'node-test supports default JavaScript loading; NODE_OPTIONS and NODE_PATH must be unset'
|
|
28
|
+
: undefined;
|
|
29
|
+
export const check = async () => admissionError()
|
|
30
|
+
? { ok: false, message: admissionError() }
|
|
31
|
+
: { ok: true, version: VERSION, source: 'builtin' };
|
|
32
|
+
export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';
|
|
33
|
+
|
|
34
|
+
const receipts = () => {
|
|
35
|
+
const dir = mkdtempSync(join(tmpdir(), 'testguard-node-'));
|
|
36
|
+
const file = join(dir, 'entries.json');
|
|
37
|
+
writeFileSync(file, '{"entries":[', { mode: 0o600 });
|
|
38
|
+
return { dir, file };
|
|
39
|
+
};
|
|
40
|
+
const environment = (file, collect, workers) => ({
|
|
41
|
+
TESTGUARD_NODE_ENTRIES: file,
|
|
42
|
+
TESTGUARD_NODE_COLLECT: collect ? '1' : '0',
|
|
43
|
+
TESTGUARD_NODE_WORKERS: String(workers),
|
|
44
|
+
TESTGUARD_NODE_REPORT: '{out}',
|
|
45
|
+
NODE_TEST_CONTEXT: '',
|
|
46
|
+
NODE_DISABLE_COMPILE_CACHE: '1',
|
|
47
|
+
});
|
|
48
|
+
export const argvFor = (projectDir, files) => [process.execPath, `--import=${ENTRY}`, RUNNER, ...files.map((file) => resolve(realpathSync(projectDir), file))];
|
|
49
|
+
|
|
50
|
+
/** Reject unsupported/incomplete envelopes before deriving any verdict inputs. */
|
|
51
|
+
export function validateReport(report) {
|
|
52
|
+
if (report?.schemaVersion !== 1 || report.engine !== name || report.complete !== true
|
|
53
|
+
|| report.version !== VERSION || typeof report.collect !== 'boolean' || typeof report.failed !== 'boolean'
|
|
54
|
+
|| !Array.isArray(report.entries) || report.entries.length > 100_000
|
|
55
|
+
|| !Array.isArray(report.cases) || report.cases.length > 100_000
|
|
56
|
+
|| report.diagnostic !== undefined && (typeof report.diagnostic !== 'string' || report.diagnostic.length > 65536)) throw new DiscoveryError('malformed or incomplete node-test report');
|
|
57
|
+
const pids = new Set();
|
|
58
|
+
for (const entry of report.entries) {
|
|
59
|
+
if (!entry || typeof entry.file !== 'string' || !entry.file || entry.file.includes('\0') || !Number.isSafeInteger(entry.pid) || entry.pid <= 0 || pids.has(entry.pid) || entry.version !== report.version) throw new DiscoveryError('malformed node-test worker receipt');
|
|
60
|
+
pids.add(entry.pid);
|
|
61
|
+
}
|
|
62
|
+
for (const t of report.cases) {
|
|
63
|
+
if (!t || typeof t.file !== 'string' || typeof t.name !== 'string'
|
|
64
|
+
|| ['container', 'suite', 'skipped', 'passed'].some((k) => typeof t[k] !== 'boolean')
|
|
65
|
+
|| t.error !== undefined && (typeof t.error !== 'object' || t.error === null || typeof t.error.failureType !== 'string' || typeof t.error.code !== 'string' || typeof t.error.message !== 'string' || typeof t.error.fileExit !== 'boolean')) throw new DiscoveryError('malformed node-test result');
|
|
66
|
+
if (t.passed && t.error || !t.container && (!t.file || t.file.split('/').some((part) => part === '..' || part === '') || t.file.includes('\\') || t.file.startsWith('/') || !t.name)) throw new DiscoveryError('inconsistent node-test result identity');
|
|
67
|
+
}
|
|
68
|
+
if (report.failed !== report.cases.some((t) => !t.passed && !t.skipped)) throw new DiscoveryError('inconsistent node-test failure summary');
|
|
69
|
+
return report;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function parseReport(report, durationMs) {
|
|
73
|
+
validateReport(report);
|
|
74
|
+
if (report.collect) throw new DiscoveryError('collection is not a test run');
|
|
75
|
+
const tests = report.cases.filter((t) => !t.container && !t.suite && !t.skipped);
|
|
76
|
+
const failed = tests.filter((t) => !t.passed);
|
|
77
|
+
const timeout = report.cases.filter((t) => !t.skipped && t.error?.failureType === 'testTimeoutFailure');
|
|
78
|
+
const loadErrors = report.cases.filter((t) => !t.skipped && !t.passed
|
|
79
|
+
&& (t.container || t.suite ? t.error?.failureType !== 'subtestsFailed' : !['testCodeFailure', 'testTimeoutFailure'].includes(t.error?.failureType) || t.error?.fileExit));
|
|
80
|
+
if (!tests.length || !report.entries.length) loadErrors.push({ error: { message: 'node-test executed no test bodies' } });
|
|
81
|
+
const assertionFailures = failed.filter((t) => t.error?.failureType === 'testCodeFailure' && !t.error.fileExit).length;
|
|
82
|
+
const names = new Map();
|
|
83
|
+
for (const t of tests) { const key = `${t.file}::${t.name}`; names.set(key, (names.get(key) ?? 0) + 1); }
|
|
84
|
+
const seen = new Map();
|
|
85
|
+
const failedTests = [];
|
|
86
|
+
for (const t of tests) {
|
|
87
|
+
const key = `${t.file}::${t.name}`;
|
|
88
|
+
const ordinal = (seen.get(key) ?? 0) + 1; seen.set(key, ordinal);
|
|
89
|
+
if (!t.passed) failedTests.push(names.get(key) > 1 ? `${key} [case ${ordinal}]` : key);
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
run: { outcome: loadErrors.length ? 'error' : report.failed || failed.length ? 'fail' : 'pass', tests: { total: tests.length, passed: tests.length - failed.length, failed: failed.length }, assertionFailures: loadErrors.length ? 0 : assertionFailures, durationMs },
|
|
93
|
+
timeouts: timeout.length,
|
|
94
|
+
loadMessage: loadErrors.length ? report.diagnostic?.split(/\r?\n/).find((line) => /(?:\bError:|SyntaxError:|cannot find|unexpected token)/i.test(line))?.trim().slice(0, 1000) ?? loadErrors[0]?.error?.message : undefined,
|
|
95
|
+
failedTests,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export async function discoverTests({ projectDir, version, timeoutMs, maxOutputBytes } = {}) {
|
|
100
|
+
if (admissionError()) throw new DiscoveryError(admissionError());
|
|
101
|
+
const before = hashDiscoveryConfigs(projectDir);
|
|
102
|
+
const spool = receipts();
|
|
103
|
+
try {
|
|
104
|
+
const result = await runDiscoveryProcess({ projectDir, argv: argvFor(projectDir, []), timeoutMs, maxOutputBytes, cleanupOnClose: true, env: environment(spool.file, true, 1) });
|
|
105
|
+
let report;
|
|
106
|
+
try { report = validateReport(JSON.parse(result.stdout)); } catch (error) { throw new DiscoveryError(`node-test collection report: ${error.message}`); }
|
|
107
|
+
if (!report.collect || report.failed) throw new DiscoveryError('node-test collection did not complete successfully');
|
|
108
|
+
const files = normalizeDiscoveredFiles(projectDir, report.entries.map((entry) => entry.file));
|
|
109
|
+
if (files.length !== report.entries.length) throw new DiscoveryError('node-test collection executed a duplicate entry');
|
|
110
|
+
const configFiles = hashDiscoveryConfigs(projectDir);
|
|
111
|
+
if (JSON.stringify(configFiles) !== JSON.stringify(before)) throw new DiscoveryError('node-test discovery config changed while modules were being collected');
|
|
112
|
+
const strategy = strategyHash();
|
|
113
|
+
return createDiscoveryManifest({ runner: name, version, files, configFiles, source: `native-node-entry-v1:${strategy}` });
|
|
114
|
+
} finally { rmSync(spool.dir, { recursive: true, force: true }); }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export async function run(opts) {
|
|
118
|
+
if (admissionError()) throw new DiscoveryError(admissionError());
|
|
119
|
+
if (!opts.files?.length) throw new DiscoveryError('node-test requires explicit selected test files');
|
|
120
|
+
const files = normalizeDiscoveredFiles(opts.projectDir, opts.files);
|
|
121
|
+
const spool = receipts();
|
|
122
|
+
try {
|
|
123
|
+
return await runProcess({ ...opts, files, cleanupOnClose: true, env: environment(spool.file, false, opts.serial ? 1 : opts.workers ?? 1), argv: (selected) => argvFor(opts.projectDir, selected), parse: (report, durationMs) => {
|
|
124
|
+
validateReport(report);
|
|
125
|
+
const entries = normalizeDiscoveredFiles(opts.projectDir, report.entries.map((entry) => entry.file));
|
|
126
|
+
if (entries.length !== report.entries.length || JSON.stringify(entries) !== JSON.stringify(files)) throw new DiscoveryError('node-test did not execute every selected entry exactly');
|
|
127
|
+
return parseReport(report, durationMs);
|
|
128
|
+
} });
|
|
129
|
+
} finally { rmSync(spool.dir, { recursive: true, force: true }); }
|
|
130
|
+
}
|
|
@@ -241,7 +241,7 @@ export function parseReport(report, durationMs) {
|
|
|
241
241
|
* runner's command line; `command` (tests) or `commandTemplate` (--runner-cmd)
|
|
242
242
|
* override it.
|
|
243
243
|
*/
|
|
244
|
-
export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv, env = {}, parse = parseReport }) {
|
|
244
|
+
export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv, env = {}, parse = parseReport, cleanupOnClose = false }) {
|
|
245
245
|
assertNotCancelled();
|
|
246
246
|
const outFile = join(tmpdir(), `testguard-run-${randomBytes(6).toString('hex')}.json`);
|
|
247
247
|
const [cmd, ...args] = command
|
|
@@ -268,6 +268,7 @@ export function runProcess({ projectDir, files, budgetMs = 120_000, command, com
|
|
|
268
268
|
|
|
269
269
|
child.on('close', () => {
|
|
270
270
|
clearTimeout(timer);
|
|
271
|
+
if (cleanupOnClose) terminateProcessTree(child);
|
|
271
272
|
const durationMs = Date.now() - started;
|
|
272
273
|
let result;
|
|
273
274
|
if (killed) {
|