supercov 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -22
- package/docs/agent-loop.md +8 -13
- package/docs/assertions.md +161 -75
- package/docs/cli.md +111 -59
- package/docs/coverage-model.md +5 -6
- package/docs/evidence.md +0 -11
- package/docs/getting-started.md +1 -1
- package/docs/performance.md +6 -14
- package/docs/quality.md +1 -1
- package/docs/security.md +43 -12
- package/docs/supported-suites.md +44 -48
- package/docs/troubleshooting.md +12 -5
- package/docs/workspace-isolation.md +23 -2
- package/package.json +36 -21
- package/runtime/javascript/atomic.mjs +46 -2
- package/runtime/javascript/ava.mjs +82 -0
- package/runtime/javascript/bootstrap.cjs +43 -0
- package/runtime/javascript/jest.cjs +52 -15
- package/runtime/javascript/jest.config.mjs +9 -1
- package/runtime/javascript/jestReporter.mjs +5 -6
- package/runtime/javascript/launchSupervisor.mjs +127 -15
- package/runtime/javascript/mocha.mjs +180 -0
- package/runtime/javascript/nodeTest.mjs +57 -3
- package/runtime/javascript/playwright.mjs +241 -32
- package/runtime/javascript/provenance.mjs +1 -1
- package/runtime/javascript/register.mjs +290 -2
- package/runtime/javascript/resolve-loader.mjs +40 -1
- package/runtime/javascript/runnerEvidence.mjs +3 -5
- package/runtime/javascript/runtime.mjs +551 -61
- package/runtime/javascript/tap.mjs +130 -0
- package/runtime/javascript/transport.mjs +7 -2
- package/runtime/javascript/vitest.mjs +5 -8
- package/runtime/javascript/vitestReporter.mjs +3 -10
- package/docs/assertion-agent.md +0 -150
- package/docs/assertion-evidence.md +0 -103
- package/docs/assertion-maps.md +0 -316
- package/schemas/assertions.schema.json +0 -276
|
@@ -27,6 +27,19 @@ snapshots, calls a service, or changes a database, wrapping it does not remove
|
|
|
27
27
|
that behavior. The isolation guarantee applies to Supercov's instrumentation
|
|
28
28
|
and evidence work.
|
|
29
29
|
|
|
30
|
+
## Tools that judge the code
|
|
31
|
+
|
|
32
|
+
Some tools a test command runs read or measure the code instead of running it,
|
|
33
|
+
and the instrumented copy is not what you wrote:
|
|
34
|
+
|
|
35
|
+
- Linters, formatters and `tsc --noEmit` (ESLint, Prettier, standard, xo and
|
|
36
|
+
the like) read each file as you wrote it, and don't see `.supercov`.
|
|
37
|
+
- Coverage tools the command runs itself (tap, c8, nyc, Jest's and Vitest's
|
|
38
|
+
`--coverage`) still collect and report coverage. They measure the
|
|
39
|
+
instrumented copy, and report close to what they report without Supercov,
|
|
40
|
+
but not always exactly, so their thresholds are not checked, and the run
|
|
41
|
+
prints a line saying so. Supercov's own report has the run's coverage.
|
|
42
|
+
|
|
30
43
|
Files the wrapped command creates or changes inside the isolated workspace are
|
|
31
44
|
synced back to the project after the run, so `supercov -- npm test -- -u`
|
|
32
45
|
updates snapshots in the repository exactly as `npm test -- -u` would. Two
|
|
@@ -87,8 +100,16 @@ project lock, and does not scan for similarly named directories.
|
|
|
87
100
|
## Containers and remote workspaces
|
|
88
101
|
|
|
89
102
|
When a suite launches a container or VM from a mounted workspace, Supercov uses
|
|
90
|
-
the isolated workspace as the source presented to that environment.
|
|
91
|
-
|
|
103
|
+
the isolated workspace as the source presented to that environment. Where it
|
|
104
|
+
sees the launch call -- a command and its environment passed as an `env`,
|
|
105
|
+
`environment` or `envs` option, as a plain map after the command, or to a
|
|
106
|
+
builder method such as Testcontainers' `withEnvironment` -- it hands the guest
|
|
107
|
+
its settings with paths moved to the guest's mount, and the guest's coverage is
|
|
108
|
+
credited to the test that launched it. A guest started any other way, with none
|
|
109
|
+
of those settings, still runs the instrumented code: it reads the run's
|
|
110
|
+
settings from the mounted workspace itself, and its coverage counts for the run
|
|
111
|
+
as background coverage, credited to no test. The run summary's `Guests` line
|
|
112
|
+
says how many processes did that.
|
|
92
113
|
|
|
93
114
|
Dependencies stay out of the instrumented copy. The root `node_modules` is
|
|
94
115
|
linked entry by entry to the project's own, so an environment that mounts the
|
package/package.json
CHANGED
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "supercov",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "Coverage, security and code quality for coding agents",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"coverage",
|
|
7
|
+
"code-coverage",
|
|
8
|
+
"test-coverage",
|
|
9
|
+
"branch-coverage",
|
|
10
|
+
"mcdc",
|
|
11
|
+
"vitest",
|
|
12
|
+
"jest",
|
|
13
|
+
"playwright",
|
|
14
|
+
"pytest",
|
|
15
|
+
"security",
|
|
16
|
+
"code-quality",
|
|
17
|
+
"coding-agents",
|
|
18
|
+
"claude-code",
|
|
19
|
+
"codex",
|
|
20
|
+
"cli"
|
|
21
|
+
],
|
|
5
22
|
"license": "MIT",
|
|
6
23
|
"repository": {
|
|
7
24
|
"type": "git",
|
|
@@ -20,9 +37,6 @@
|
|
|
20
37
|
"docs/assertions.md",
|
|
21
38
|
"docs/quality.md",
|
|
22
39
|
"docs/security.md",
|
|
23
|
-
"docs/assertion-evidence.md",
|
|
24
|
-
"docs/assertion-maps.md",
|
|
25
|
-
"docs/assertion-agent.md",
|
|
26
40
|
"docs/troubleshooting.md",
|
|
27
41
|
"docs/cli.md",
|
|
28
42
|
"docs/coverage-model.md",
|
|
@@ -31,7 +45,6 @@
|
|
|
31
45
|
"docs/verification.md",
|
|
32
46
|
"docs/performance.md",
|
|
33
47
|
"docs/workspace-isolation.md",
|
|
34
|
-
"schemas",
|
|
35
48
|
"README.md"
|
|
36
49
|
],
|
|
37
50
|
"engines": {
|
|
@@ -52,9 +65,6 @@
|
|
|
52
65
|
"build": "cargo build -p supercov",
|
|
53
66
|
"sync:rust-assets": "node scripts/sync-rust-package-assets.mjs",
|
|
54
67
|
"test:rust-assets": "node scripts/sync-rust-package-assets.mjs --check",
|
|
55
|
-
"check:rustc-backend-spike": "(cd spikes/rustc-backend && RUSTC_BOOTSTRAP=1 RUSTUP_TOOLCHAIN=1.95.0 cargo fmt --check && RUSTC_BOOTSTRAP=1 RUSTUP_TOOLCHAIN=1.95.0 cargo clippy --all-targets -- -D warnings)",
|
|
56
|
-
"test:rustc-backend-spike": "node scripts/rustc-backend-spike.mjs",
|
|
57
|
-
"test:rust-compiler-spikes": "cargo build -p supercov && node scripts/rust-libtest-companion-spike.mjs && node scripts/rust-async-attribution-spike.mjs && node scripts/rust-subprocess-attribution-spike.mjs && node scripts/rust-custom-harness-spike.mjs && node scripts/rust-libtest-builder-lifecycle-spike.mjs",
|
|
58
68
|
"test": "cargo test --workspace",
|
|
59
69
|
"test:react": "cargo build -p supercov && node scripts/rust-vitest-projects-integration.mjs && npm --prefix examples/react-verification ci && npm --prefix examples/react-verification run demo && node scripts/react-hydration-compatibility.mjs",
|
|
60
70
|
"test:release-tooling": "node --test tests/release/*.test.mjs",
|
|
@@ -67,8 +77,11 @@
|
|
|
67
77
|
"test:html-report": "cargo build -p supercov && node scripts/html-report-integration.mjs",
|
|
68
78
|
"test:engine-contract": "cargo build -p supercov && node scripts/engine-contract.mjs",
|
|
69
79
|
"test:agent": "cargo build -p supercov && node scripts/agent-query-eval.mjs",
|
|
80
|
+
"test:agent-commands": "cargo build -p supercov && node scripts/check-agent-commands.mjs",
|
|
81
|
+
"eval:skill": "claude plugin eval plugins/supercov --scaffold --trust-plugin --no-publish --model claude-sonnet-5 --judge-model claude-haiku-4-5 --threshold 0.66 --tag behavior trigger --allow-tools Bash Edit Write --max-cost-usd 15",
|
|
82
|
+
"eval:skill-triggers": "claude plugin eval plugins/supercov --scaffold --trust-plugin --no-publish --model claude-sonnet-5 --judge-model claude-haiku-4-5 --threshold 0.66 --tag trigger --ablation none --max-cost-usd 5",
|
|
70
83
|
"test:engine": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && cargo test --workspace && npm run test:runtime && npm run test:rust-assets && npm run test:engine-integrations",
|
|
71
|
-
"test:engine-integrations": "cargo build -p supercov && node scripts/html-report-integration.mjs && node scripts/rust-process-supervision.mjs && node scripts/rust-direct-node-integration.mjs && node scripts/rust-public-run-integration.mjs && node scripts/rust-embedded-runtime-integration.mjs && node scripts/rust-direct-vitest-integration.mjs && node scripts/rust-direct-jest-integration.mjs && node scripts/rust-direct-playwright-integration.mjs && node scripts/rust-custom-browser-playwright-integration.mjs && node scripts/rust-generic-esbuild-integration.mjs && node scripts/rust-generic-tsc-integration.mjs && node scripts/rust-generic-build-matrix.mjs && node scripts/rust-vite-playwright-integration.mjs",
|
|
84
|
+
"test:engine-integrations": "cargo build -p supercov && node scripts/html-report-integration.mjs && node scripts/rust-process-supervision.mjs && node scripts/rust-direct-node-integration.mjs && node scripts/rust-public-run-integration.mjs && node scripts/rust-embedded-runtime-integration.mjs && node scripts/rust-direct-vitest-integration.mjs && node scripts/rust-direct-jest-integration.mjs && node scripts/rust-direct-playwright-integration.mjs && node scripts/rust-custom-browser-playwright-integration.mjs && node scripts/rust-classic-browser-integration.mjs && node scripts/rust-top-packages-integration.mjs && node scripts/rust-generic-esbuild-integration.mjs && node scripts/rust-generic-tsc-integration.mjs && node scripts/rust-generic-build-matrix.mjs && node scripts/rust-vite-playwright-integration.mjs",
|
|
72
85
|
"test:platform": "cargo test --workspace && cargo build -p supercov && node scripts/rust-process-supervision.mjs && node scripts/workspace-crash-integration.mjs",
|
|
73
86
|
"test:native-package": "cargo build --release -p supercov && node scripts/native-package-integration.mjs && node scripts/native-release-set-integration.mjs",
|
|
74
87
|
"test:pypi-wheel": "node scripts/pypi-wheel-integration.mjs",
|
|
@@ -81,7 +94,7 @@
|
|
|
81
94
|
"test:test262": "cargo build --release -p supercov && node scripts/test262-equivalence.mjs",
|
|
82
95
|
"benchmark:check": "cargo build --release -p supercov && node scripts/rust-transform-benchmark.mjs",
|
|
83
96
|
"benchmark:python": "cargo build -p supercov && node scripts/python-benchmark.mjs",
|
|
84
|
-
"check": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && npm run test && npm run test:release-tooling && npm run test:runtime && npm run test:rust-assets && npm run test:
|
|
97
|
+
"check": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && npm run test && npm run test:release-tooling && npm run test:runtime && npm run test:rust-assets && npm run test:agent-commands && npm run test:assertions && npm run test:sweep-target && npm run test:skip-guard && npm run test:windows-tls && node scripts/package-preflight.mjs && node scripts/verify-binstall-metadata.mjs --offline",
|
|
85
98
|
"release:check": "cargo clean && node scripts/sweep-target.mjs && npm run check && npm run test:engine && npm run test:go && npm run test:jvm && npm run test:launcher && npm run test:fixture && npm run test:watchdog && npm run test:engine-contract && npm run test:agent && npm run test:child-attribution && npm run test:host-loader && npm run test:python && npm run test:ruby-coverage && npm run test:rust-public-cargo && npm run test:packed-npx && npm run test:clang-mcdc && npm run benchmark:check",
|
|
86
99
|
"release:bump": "node scripts/bump-version.mjs",
|
|
87
100
|
"sweep": "node scripts/sweep-target.mjs",
|
|
@@ -95,9 +108,7 @@
|
|
|
95
108
|
"test:go": "cargo test --no-fail-fast -p supercov-engine --test go_frontend --test go_run",
|
|
96
109
|
"test:jvm": "cargo test --no-fail-fast -p supercov-engine --test jvm_frontend --test jvm_run -- --include-ignored --test-threads=1",
|
|
97
110
|
"test:rust-public-cargo": "cargo build -p supercov && node scripts/rust-public-cargo-integration.mjs",
|
|
98
|
-
"test:
|
|
99
|
-
"test:assertion-maps:js": "cargo build -p supercov && node scripts/assertion-map-schema.mjs --check && node scripts/assertion-map-js-integration.mjs && node scripts/assertion-dogfood-integration.mjs",
|
|
100
|
-
"sync:assertion-schema": "cargo build -p supercov && node scripts/assertion-map-schema.mjs",
|
|
111
|
+
"test:assertions": "cargo build -p supercov && node scripts/assertions-integration.mjs",
|
|
101
112
|
"docs:sync": "node scripts/sync-docs.mjs",
|
|
102
113
|
"docs:check": "node scripts/sync-docs.mjs --check",
|
|
103
114
|
"test:launcher": "go vet ./cmd/... && go test ./cmd/...",
|
|
@@ -107,14 +118,14 @@
|
|
|
107
118
|
"test:windows-tls": "node scripts/windows-tls-test.mjs"
|
|
108
119
|
},
|
|
109
120
|
"optionalDependencies": {
|
|
110
|
-
"@supercov/cli-darwin-arm64": "
|
|
111
|
-
"@supercov/cli-darwin-x64": "
|
|
112
|
-
"@supercov/cli-linux-arm64-gnu": "
|
|
113
|
-
"@supercov/cli-linux-arm64-musl": "
|
|
114
|
-
"@supercov/cli-linux-x64-gnu": "
|
|
115
|
-
"@supercov/cli-linux-x64-musl": "
|
|
116
|
-
"@supercov/cli-win32-arm64": "
|
|
117
|
-
"@supercov/cli-win32-x64": "
|
|
121
|
+
"@supercov/cli-darwin-arm64": "3.0.0",
|
|
122
|
+
"@supercov/cli-darwin-x64": "3.0.0",
|
|
123
|
+
"@supercov/cli-linux-arm64-gnu": "3.0.0",
|
|
124
|
+
"@supercov/cli-linux-arm64-musl": "3.0.0",
|
|
125
|
+
"@supercov/cli-linux-x64-gnu": "3.0.0",
|
|
126
|
+
"@supercov/cli-linux-x64-musl": "3.0.0",
|
|
127
|
+
"@supercov/cli-win32-arm64": "3.0.0",
|
|
128
|
+
"@supercov/cli-win32-x64": "3.0.0"
|
|
118
129
|
},
|
|
119
130
|
"peerDependencies": {
|
|
120
131
|
"@playwright/test": ">=1.55.0",
|
|
@@ -137,9 +148,13 @@
|
|
|
137
148
|
"@swc/core": "^1.16.1",
|
|
138
149
|
"@types/node": "^24.0.0",
|
|
139
150
|
"@vitest/browser-playwright": "4.1.11",
|
|
151
|
+
"@vitest/coverage-v8": "4.1.11",
|
|
152
|
+
"ava": "8.0.1",
|
|
140
153
|
"esbuild": "^0.28.2",
|
|
141
154
|
"expect": "30.4.1",
|
|
142
155
|
"jest": "30.5.1",
|
|
156
|
+
"libtap": "1.4.1",
|
|
157
|
+
"mocha": "12.0.3",
|
|
143
158
|
"next": "^16.3.2",
|
|
144
159
|
"react": "^19.2.4",
|
|
145
160
|
"react-dom": "^19.2.4",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, mkdirSync, openSync, closeSync, fsyncSync, renameSync, rmSync, writeFileSync, } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, openSync, closeSync, fsyncSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { dirname } from "node:path";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
import { Buffer } from "node:buffer";
|
|
4
5
|
function fsyncDirectory(path) {
|
|
5
6
|
try {
|
|
6
7
|
const directory = openSync(path, "r");
|
|
@@ -64,3 +65,46 @@ export function atomicRenameSync(source, destination) {
|
|
|
64
65
|
renameSync(source, destination);
|
|
65
66
|
fsyncDirectory(dirname(destination));
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Append one test's evidence to this writer's own journal.
|
|
70
|
+
*
|
|
71
|
+
* A file per test cost a directory, a temporary file, two fsyncs and a rename:
|
|
72
|
+
* jshttp/cookie's 63,740 tests had written 26,600 of them after seven
|
|
73
|
+
* minutes, where the suite alone takes two seconds. An append is one write the
|
|
74
|
+
* kernel keeps when the process is killed, which is the durability the rest of
|
|
75
|
+
* the workspace has; set SUPERCOV_DURABLE_EVIDENCE_EACH_TEST=1 to fsync each
|
|
76
|
+
* record as well. The reader already takes `*.mcdc.jsonl` journals, one record
|
|
77
|
+
* per line, as Playwright writes them.
|
|
78
|
+
*
|
|
79
|
+
* A journal is named by a token, never by the pid alone: worker threads share
|
|
80
|
+
* a pid, and VMs restored from one snapshot share the token too, so a journal
|
|
81
|
+
* that grew by anything but this writer's own appends is left to its other
|
|
82
|
+
* writer and a new one is started.
|
|
83
|
+
*/
|
|
84
|
+
const journals = new Map();
|
|
85
|
+
export function appendEvidenceRecord(directory, prefix, payload) {
|
|
86
|
+
const line = `${JSON.stringify(payload)}\n`;
|
|
87
|
+
let journal = journals.get(prefix);
|
|
88
|
+
if (journal) {
|
|
89
|
+
let size = -1;
|
|
90
|
+
try {
|
|
91
|
+
size = statSync(journal.path).size;
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// A missing journal is started again below.
|
|
95
|
+
}
|
|
96
|
+
if (size !== journal.size)
|
|
97
|
+
journal = undefined;
|
|
98
|
+
}
|
|
99
|
+
if (!journal) {
|
|
100
|
+
const writer = (process.env.SUPERCOV_EXECUTION_LOG_SHARD ?? `pid-${process.pid}`)
|
|
101
|
+
.replace(/[^A-Za-z0-9_-]/g, "_");
|
|
102
|
+
journal = { path: resolve(directory, `${prefix}-${writer}-${randomUUID()}.mcdc.jsonl`), size: 0 };
|
|
103
|
+
journals.set(prefix, journal);
|
|
104
|
+
}
|
|
105
|
+
const append = process.env.SUPERCOV_DURABLE_EVIDENCE_EACH_TEST === "1"
|
|
106
|
+
? appendJsonLineDurableSync
|
|
107
|
+
: appendJsonLineSync;
|
|
108
|
+
append(journal.path, line);
|
|
109
|
+
journal.size += Buffer.byteLength(line);
|
|
110
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Supercov AVA adapter. The preload imports it in each AVA worker -- a
|
|
2
|
+
// worker thread, or a child process under --no-worker-threads -- and it
|
|
3
|
+
// patches AVA's own Runner, so a suite runs exactly as written. AVA runs a
|
|
4
|
+
// file's tests concurrently, so a test's coverage cannot be told apart by
|
|
5
|
+
// time: each test, with its beforeEach and afterEach hooks, runs inside a
|
|
6
|
+
// coverage carrier of its own, as node:test subtests do, and what ran under
|
|
7
|
+
// it is written with the test's outcome once it settles. Loading the test
|
|
8
|
+
// file and before/after hooks run outside any test and count for the run.
|
|
9
|
+
// It computes no coverage.
|
|
10
|
+
import { realpathSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { pathToFileURL } from "node:url";
|
|
13
|
+
import { beginBufferedServerEvidence, flushBufferedServerEvidence, takeNodeAssertionPhases, withCoverageCarrier, } from "./runtime.mjs";
|
|
14
|
+
import { runnerExecutionScope, writeRunnerEvidence } from "./runnerEvidence.mjs";
|
|
15
|
+
|
|
16
|
+
// The installed ava package the entrypoint belongs to.
|
|
17
|
+
function avaRoot(entrypoint) {
|
|
18
|
+
let real;
|
|
19
|
+
try {
|
|
20
|
+
real = realpathSync(entrypoint).replaceAll("\\", "/");
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const at = real.lastIndexOf("/node_modules/ava/");
|
|
26
|
+
return at < 0 ? undefined : real.slice(0, at + "/node_modules/ava".length);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function adaptAva(entrypoint) {
|
|
30
|
+
const evidenceDirectory = process.env.SUPERCOV_EVIDENCE_DIR;
|
|
31
|
+
const root = avaRoot(entrypoint);
|
|
32
|
+
if (!evidenceDirectory || !root)
|
|
33
|
+
return;
|
|
34
|
+
let loaded;
|
|
35
|
+
try {
|
|
36
|
+
loaded = await import(pathToFileURL(join(root, "lib", "runner.js")).href);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const Runner = loaded.default ?? loaded.Runner;
|
|
42
|
+
const runTest = Runner?.prototype?.runTest;
|
|
43
|
+
if (typeof runTest !== "function" || runTest.supercovAva)
|
|
44
|
+
return;
|
|
45
|
+
Runner.prototype.runTest = function supercovAvaRunTest(task, contextRef) {
|
|
46
|
+
// AVA refuses two tests with one title in a file, so the title names
|
|
47
|
+
// the test.
|
|
48
|
+
const identity = { runner: "ava", file: this.file, name: task.title };
|
|
49
|
+
const scope = runnerExecutionScope(identity);
|
|
50
|
+
beginBufferedServerEvidence(scope);
|
|
51
|
+
const finish = (status) => writeRunnerEvidence(identity, status, scope, evidenceDirectory, takeNodeAssertionPhases(scope), flushBufferedServerEvidence(scope));
|
|
52
|
+
let settled;
|
|
53
|
+
try {
|
|
54
|
+
settled = Promise.resolve(withCoverageCarrier({ version: 1, scope }, () => Reflect.apply(runTest, this, [task, contextRef])));
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
finish("failed");
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
return settled.then((passed) => {
|
|
61
|
+
finish(passed ? "passed" : "failed");
|
|
62
|
+
return passed;
|
|
63
|
+
}, (error) => {
|
|
64
|
+
finish("failed");
|
|
65
|
+
throw error;
|
|
66
|
+
});
|
|
67
|
+
};
|
|
68
|
+
Runner.prototype.runTest.supercovAva = true;
|
|
69
|
+
// A skipped or todo test never reaches runTest; AVA announces it.
|
|
70
|
+
const start = Runner.prototype.start;
|
|
71
|
+
if (typeof start === "function")
|
|
72
|
+
Runner.prototype.start = function supercovAvaStart(...args) {
|
|
73
|
+
// AVA's emitter hands a listener { data: state }.
|
|
74
|
+
this.on("stateChange", ({ data: state } = {}) => {
|
|
75
|
+
if (state?.type !== "selected-test" || !(state.skip || state.todo))
|
|
76
|
+
return;
|
|
77
|
+
const identity = { runner: "ava", file: this.file, name: state.title };
|
|
78
|
+
writeRunnerEvidence(identity, "skipped", runnerExecutionScope(identity), evidenceDirectory);
|
|
79
|
+
});
|
|
80
|
+
return Reflect.apply(start, this, args);
|
|
81
|
+
};
|
|
82
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Instrumented sources load this before anything else. Under the preload it
|
|
3
|
+
// finds the runtime already installed and does nothing. A process that runs
|
|
4
|
+
// the workspace from somewhere else -- a container or VM the workspace is
|
|
5
|
+
// mounted into, started by an SDK that passed on none of Supercov's settings,
|
|
6
|
+
// or a worker the preload never reached -- has no runtime, and without one the
|
|
7
|
+
// first probe would throw. Such a process installs the runtime itself, with
|
|
8
|
+
// the run's settings read from beside this file and their paths moved to
|
|
9
|
+
// wherever the workspace is seen from here, so its coverage lands in the
|
|
10
|
+
// run's own evidence directory.
|
|
11
|
+
const installed = globalThis.__SUPERCOV_DIRECT_RUNTIME__ ?? process.__SUPERCOV_DIRECT_RUNTIME__;
|
|
12
|
+
if (installed) {
|
|
13
|
+
globalThis.__SUPERCOV_DIRECT_RUNTIME__ ??= installed;
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
try {
|
|
17
|
+
const fs = require("node:fs");
|
|
18
|
+
const path = require("node:path");
|
|
19
|
+
// <workspace>/.supercov/node_modules/bootstrap.cjs
|
|
20
|
+
const root = path.resolve(__dirname, "..", "..");
|
|
21
|
+
const settings = JSON.parse(fs.readFileSync(path.join(__dirname, "run.json"), "utf8"));
|
|
22
|
+
const moved = (value) => typeof value === "string" && settings.root !== root
|
|
23
|
+
? value.split(settings.root).join(root)
|
|
24
|
+
: value;
|
|
25
|
+
// Settings carried over from the host name the host's paths, which do
|
|
26
|
+
// not exist here; replace them all rather than keep a mixture.
|
|
27
|
+
const foreign = process.env.SUPERCOV_PROJECT_ROOT !== root;
|
|
28
|
+
for (const [name, value] of Object.entries(settings.environment)) {
|
|
29
|
+
if (foreign || process.env[name] === undefined)
|
|
30
|
+
process.env[name] = moved(value);
|
|
31
|
+
}
|
|
32
|
+
process.env.SUPERCOV_PROJECT_ROOT = root;
|
|
33
|
+
process.env.SUPERCOV_EXECUTION_LOG_SHARD ??= `bootstrap-${process.pid}-${Date.now()}`;
|
|
34
|
+
require("./runtime.mjs");
|
|
35
|
+
// Counted in the run's summary: coverage from here belongs to no test.
|
|
36
|
+
require("./launchSupervisor.mjs").recordExecution({ event: "guest-process", guestRoot: root });
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// Without its settings or its runtime nothing can be recorded here,
|
|
40
|
+
// and the first probe fails as it did before this file existed.
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
module.exports = globalThis.__SUPERCOV_DIRECT_RUNTIME__;
|
|
@@ -9,9 +9,10 @@
|
|
|
9
9
|
//
|
|
10
10
|
// CommonJS on purpose: Jest's module system loads setup files without the
|
|
11
11
|
// ESM flag, and the runtime it needs is already on the sandbox global.
|
|
12
|
-
const { createHash } = require("node:crypto");
|
|
13
|
-
const { mkdirSync,
|
|
12
|
+
const { createHash, randomUUID } = require("node:crypto");
|
|
13
|
+
const { appendFileSync, closeSync, fsyncSync, mkdirSync, openSync, statSync, writeFileSync } = require("node:fs");
|
|
14
14
|
const { relative, resolve, sep } = require("node:path");
|
|
15
|
+
const { Buffer } = require("node:buffer");
|
|
15
16
|
|
|
16
17
|
// jest-environment-node exposes the outer process's own globals to the sandbox
|
|
17
18
|
// through getters, so this is the same runtime instance the instrumented
|
|
@@ -48,7 +49,11 @@ function currentDeclaration() {
|
|
|
48
49
|
const occurrence = names.get(name) ?? 0;
|
|
49
50
|
names.set(name, occurrence + 1);
|
|
50
51
|
if (child === state.currentlyRunningTest)
|
|
51
|
-
found = {
|
|
52
|
+
found = {
|
|
53
|
+
occurrence,
|
|
54
|
+
retry: Math.max((child.invocations ?? 1) - 1, 0),
|
|
55
|
+
path: [...parents, child.name],
|
|
56
|
+
};
|
|
52
57
|
}
|
|
53
58
|
}
|
|
54
59
|
}
|
|
@@ -71,13 +76,43 @@ function provenance(testFile) {
|
|
|
71
76
|
? { runner: "jest", kind, source: "path" }
|
|
72
77
|
: { runner: "jest", kind: "unit", source: "runner-default" };
|
|
73
78
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
// One journal per writer, as atomic.mjs's appendEvidenceRecord keeps it: a
|
|
80
|
+
// file per test cost a directory, a file and a rename each. Jest gives every
|
|
81
|
+
// test file a fresh module registry, so a worker starts one journal per file.
|
|
82
|
+
let journal;
|
|
83
|
+
function writeEvidence(payload) {
|
|
84
|
+
const line = `${JSON.stringify(payload)}\n`;
|
|
85
|
+
if (journal) {
|
|
86
|
+
let size = -1;
|
|
87
|
+
try {
|
|
88
|
+
size = statSync(journal.path).size;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// A missing journal is started again below.
|
|
92
|
+
}
|
|
93
|
+
if (size !== journal.size)
|
|
94
|
+
journal = undefined;
|
|
95
|
+
}
|
|
96
|
+
if (!journal) {
|
|
97
|
+
const directory = resolve(process.cwd(), evidenceDirectory);
|
|
98
|
+
mkdirSync(directory, { recursive: true });
|
|
99
|
+
const writer = (process.env.SUPERCOV_EXECUTION_LOG_SHARD ?? `pid-${process.pid}`).replace(/[^A-Za-z0-9_-]/g, "_");
|
|
100
|
+
journal = { path: resolve(directory, `jest-worker-${writer}-${randomUUID()}.mcdc.jsonl`), size: 0 };
|
|
101
|
+
}
|
|
102
|
+
if (process.env.SUPERCOV_DURABLE_EVIDENCE_EACH_TEST === "1") {
|
|
103
|
+
const descriptor = openSync(journal.path, "a", 0o600);
|
|
104
|
+
try {
|
|
105
|
+
writeFileSync(descriptor, line);
|
|
106
|
+
fsyncSync(descriptor);
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
closeSync(descriptor);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
appendFileSync(journal.path, line, { mode: 0o600 });
|
|
114
|
+
}
|
|
115
|
+
journal.size += Buffer.byteLength(line);
|
|
81
116
|
}
|
|
82
117
|
|
|
83
118
|
if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof afterAll === "function") {
|
|
@@ -96,7 +131,7 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
96
131
|
emittedSetupFiles.add(testFile);
|
|
97
132
|
const setupSnapshot = runtime.coverageSnapshot();
|
|
98
133
|
if (setupSnapshot.hits.length || setupSnapshot.decisions.length) {
|
|
99
|
-
writeEvidence(
|
|
134
|
+
writeEvidence({
|
|
100
135
|
testId: `jest:${digest(testFile)}:setup`,
|
|
101
136
|
test: `${testFile} > module setup`,
|
|
102
137
|
testFile,
|
|
@@ -126,7 +161,9 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
126
161
|
retry,
|
|
127
162
|
attemptId: `${testKey}-${retry}`,
|
|
128
163
|
};
|
|
129
|
-
|
|
164
|
+
// Named as the reporter names it; the id keeps Jest's full name.
|
|
165
|
+
const name = declaration?.path?.join(" > ") ?? fullName;
|
|
166
|
+
active = { scope, testFile, fullName, name, retry };
|
|
130
167
|
runtime.activateCoverageScope(scope);
|
|
131
168
|
runtime.resetCoverage(testId);
|
|
132
169
|
});
|
|
@@ -139,12 +176,12 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
139
176
|
if (!current)
|
|
140
177
|
return;
|
|
141
178
|
const { scope } = current;
|
|
142
|
-
writeEvidence(
|
|
179
|
+
writeEvidence({
|
|
143
180
|
testId: scope.testId,
|
|
144
181
|
scope,
|
|
145
|
-
test: current.
|
|
182
|
+
test: current.name,
|
|
146
183
|
testFile: current.testFile,
|
|
147
|
-
title: current.
|
|
184
|
+
title: current.name,
|
|
148
185
|
retry: current.retry,
|
|
149
186
|
// The reporter's record carries the outcome.
|
|
150
187
|
status: "unknown",
|
|
@@ -26,8 +26,16 @@ export default async function supercovJestConfig() {
|
|
|
26
26
|
// configuration is not read and Jest's defaults apply.
|
|
27
27
|
process.emitWarning(`[supercov] the project's Jest configuration was not read (${error?.message ?? error}); Jest's defaults apply`);
|
|
28
28
|
}
|
|
29
|
+
// Jest's coverage measures the instrumented copy, probes included, and a
|
|
30
|
+
// suite at 100% read 83.33% of branches: its coverageThreshold failed
|
|
31
|
+
// passing runs. Jest still collects and reports coverage.
|
|
32
|
+
const { coverageThreshold, ...own } = config;
|
|
33
|
+
const collecting = own.collectCoverage === true ||
|
|
34
|
+
process.argv.some((argument) => /^--(?:coverage|collectCoverage|collect-coverage)(?:=true)?$/.test(argument));
|
|
35
|
+
if (coverageThreshold && collecting)
|
|
36
|
+
process.__SUPERCOV_SKIPPED_COVERAGE_THRESHOLDS__?.("Jest");
|
|
29
37
|
return {
|
|
30
|
-
...
|
|
38
|
+
...own,
|
|
31
39
|
// Concurrent tests in one file would share the worker's scope.
|
|
32
40
|
maxConcurrency: 1,
|
|
33
41
|
setupFiles: [here("jestRuntime.cjs"), ...asList(config.setupFiles)],
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { mkdirSync } from "node:fs";
|
|
3
2
|
import { relative, resolve, sep } from "node:path";
|
|
4
3
|
import { inferTestProvenance } from "./provenance.mjs";
|
|
5
|
-
import {
|
|
4
|
+
import { appendEvidenceRecord } from "./atomic.mjs";
|
|
6
5
|
function digest(value) {
|
|
7
6
|
return createHash("sha256").update(value).digest("hex").slice(0, 24);
|
|
8
7
|
}
|
|
@@ -57,7 +56,9 @@ export default class SupercovJestReporter {
|
|
|
57
56
|
const record = (attempt, attemptStatus, flaky) => {
|
|
58
57
|
const payload = {
|
|
59
58
|
testId,
|
|
60
|
-
|
|
59
|
+
// Named by its describe blocks, `small > charges`, as every
|
|
60
|
+
// JavaScript runner is; the id keeps Jest's own full name.
|
|
61
|
+
test: [...(result.ancestorTitles ?? []), result.title].join(" > "),
|
|
61
62
|
testFile,
|
|
62
63
|
title: result.title,
|
|
63
64
|
retry: attempt,
|
|
@@ -69,9 +70,7 @@ export default class SupercovJestReporter {
|
|
|
69
70
|
browser: [],
|
|
70
71
|
server: [],
|
|
71
72
|
};
|
|
72
|
-
|
|
73
|
-
mkdirSync(directory, { recursive: true });
|
|
74
|
-
atomicWriteFileSync(resolve(directory, "mcdc.json"), `${JSON.stringify(payload)}\n`);
|
|
73
|
+
appendEvidenceRecord(resolve(process.cwd(), evidenceDirectory), "jest-status", payload);
|
|
75
74
|
};
|
|
76
75
|
// `jest.retryTimes` reports one result after the last attempt; every
|
|
77
76
|
// attempt before it failed, or there would have been no retry. The
|