supercov 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +201 -0
- package/bin/supercov.js +26 -0
- package/package.json +57 -0
- package/src/analyze.ts +993 -0
- package/src/cli.ts +336 -0
- package/src/instrumenter.ts +1254 -0
- package/src/integrity.ts +187 -0
- package/src/playwright.ts +1009 -0
- package/src/playwrightReporter.ts +55 -0
- package/src/project.ts +149 -0
- package/src/provenance.ts +69 -0
- package/src/query.ts +1354 -0
- package/src/queueAdapters.ts +104 -0
- package/src/register.mjs +123 -0
- package/src/reporter.ts +431 -0
- package/src/resolve-loader.mjs +45 -0
- package/src/runtime.ts +656 -0
- package/src/transport.ts +132 -0
- package/src/types.ts +412 -0
- package/src/vitePlugin.ts +121 -0
- package/src/vitest.ts +101 -0
- package/src/vitestReporter.ts +72 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { CoverageCarrier } from "./types.ts";
|
|
2
|
+
import {
|
|
3
|
+
bindCoverageContext,
|
|
4
|
+
coverageCarrier,
|
|
5
|
+
withCoverageCarrier,
|
|
6
|
+
} from "./runtime.ts";
|
|
7
|
+
import {
|
|
8
|
+
decodeCoverageCarrier,
|
|
9
|
+
encodeCoverageCarrier,
|
|
10
|
+
} from "./transport.ts";
|
|
11
|
+
|
|
12
|
+
export const COVERAGE_JOB_FIELD = "__supercov";
|
|
13
|
+
|
|
14
|
+
type RecordValue = Record<string, unknown>;
|
|
15
|
+
|
|
16
|
+
function record(value: unknown): RecordValue | undefined {
|
|
17
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
18
|
+
? (value as RecordValue)
|
|
19
|
+
: undefined;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function injectCoverageCarrier<T>(
|
|
23
|
+
payload: T,
|
|
24
|
+
carrier = coverageCarrier(),
|
|
25
|
+
): T {
|
|
26
|
+
const object = record(payload);
|
|
27
|
+
if (!object) return payload;
|
|
28
|
+
return {
|
|
29
|
+
...object,
|
|
30
|
+
[COVERAGE_JOB_FIELD]: encodeCoverageCarrier(carrier),
|
|
31
|
+
} as T;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function extractCoverageCarrier(
|
|
35
|
+
payload: unknown,
|
|
36
|
+
): CoverageCarrier | undefined {
|
|
37
|
+
const encoded = record(payload)?.[COVERAGE_JOB_FIELD];
|
|
38
|
+
return decodeCoverageCarrier(
|
|
39
|
+
typeof encoded === "string" ? encoded : undefined,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function wrapQueuePublisher<
|
|
44
|
+
T extends (...args: never[]) => unknown,
|
|
45
|
+
>(publisher: T, payloadIndex = 0): T {
|
|
46
|
+
return function coverageQueuePublisher(
|
|
47
|
+
this: unknown,
|
|
48
|
+
...args: Parameters<T>
|
|
49
|
+
): ReturnType<T> {
|
|
50
|
+
const scoped = [...args] as unknown[];
|
|
51
|
+
scoped[payloadIndex] = injectCoverageCarrier(scoped[payloadIndex]);
|
|
52
|
+
return Reflect.apply(publisher, this, scoped) as ReturnType<T>;
|
|
53
|
+
} as T;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function wrapQueueProcessor<
|
|
57
|
+
T extends (...args: never[]) => unknown,
|
|
58
|
+
>(processor: T, payload: (args: Parameters<T>) => unknown): T {
|
|
59
|
+
return function coverageQueueProcessor(
|
|
60
|
+
this: unknown,
|
|
61
|
+
...args: Parameters<T>
|
|
62
|
+
): ReturnType<T> {
|
|
63
|
+
const carrier = extractCoverageCarrier(payload(args));
|
|
64
|
+
return withCoverageCarrier(carrier, () =>
|
|
65
|
+
Reflect.apply(processor, this, args),
|
|
66
|
+
) as ReturnType<T>;
|
|
67
|
+
} as T;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** BullMQ and Bee-Queue jobs expose user data through `job.data`. */
|
|
71
|
+
export function wrapBullProcessor<T extends (...args: never[]) => unknown>(
|
|
72
|
+
processor: T,
|
|
73
|
+
): T {
|
|
74
|
+
return wrapQueueProcessor(processor, (args) =>
|
|
75
|
+
record(args[0])?.["data"],
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** pg-boss handlers receive either one job or a batch whose jobs have `data`. */
|
|
80
|
+
export function wrapPgBossProcessor<T extends (...args: never[]) => unknown>(
|
|
81
|
+
processor: T,
|
|
82
|
+
): T {
|
|
83
|
+
return wrapQueueProcessor(processor, (args) => {
|
|
84
|
+
const first = args[0] as unknown;
|
|
85
|
+
const job = Array.isArray(first) ? first[0] : first;
|
|
86
|
+
return record(job)?.["data"];
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Agenda stores user data in `job.attrs.data`. */
|
|
91
|
+
export function wrapAgendaProcessor<T extends (...args: never[]) => unknown>(
|
|
92
|
+
processor: T,
|
|
93
|
+
): T {
|
|
94
|
+
return wrapQueueProcessor(processor, (args) =>
|
|
95
|
+
record(record(args[0])?.["attrs"])?.["data"],
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Capture context for in-process scheduler callbacks without changing payloads. */
|
|
100
|
+
export function wrapScheduledCallback<
|
|
101
|
+
T extends (...args: never[]) => unknown,
|
|
102
|
+
>(callback: T): T {
|
|
103
|
+
return bindCoverageContext(callback);
|
|
104
|
+
}
|
package/src/register.mjs
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import Module, { register } from "node:module";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
|
|
4
|
+
register(new URL("./resolve-loader.mjs", import.meta.url));
|
|
5
|
+
|
|
6
|
+
// NODE_OPTIONS reaches commands launched through npm scripts. When that child
|
|
7
|
+
// is Vitest, replace its config with our generated merging config before the
|
|
8
|
+
// CLI parses argv. This is what makes `supercov -- npm test` work
|
|
9
|
+
// without editing package scripts, Vitest configs, setup files, or test imports.
|
|
10
|
+
const generatedVitestConfig =
|
|
11
|
+
process.env.SUPERCOV_GENERATED_VITEST_CONFIG;
|
|
12
|
+
const generatedPlaywrightConfig =
|
|
13
|
+
process.env.SUPERCOV_GENERATED_PLAYWRIGHT_CONFIG;
|
|
14
|
+
const entrypoint = process.argv[1]?.replaceAll("\\", "/") ?? "";
|
|
15
|
+
const playwrightTarget = process.env.SUPERCOV_PLAYWRIGHT_MODULE;
|
|
16
|
+
|
|
17
|
+
if (process.env.SUPERCOV_DEBUG === "1") {
|
|
18
|
+
console.error("[supercov] preload", { entrypoint });
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// The Essential runner launches Playwright inside an isolated VM where the
|
|
22
|
+
// host's absolute NODE_OPTIONS imports do not exist. Its generated Playwright
|
|
23
|
+
// config already installs the adapter, so stop only that inherited preload at
|
|
24
|
+
// the host runner boundary. This still lets an earlier Vitest npm script in the
|
|
25
|
+
// same outer command use the generic hook.
|
|
26
|
+
if (
|
|
27
|
+
playwrightTarget === "@essential-apps/shopify-test-admin" &&
|
|
28
|
+
/\/shopify-test-runner\/src\/scripts\/runOffline(?:Pool)?\.[cm]?[jt]s$/.test(
|
|
29
|
+
entrypoint,
|
|
30
|
+
)
|
|
31
|
+
) {
|
|
32
|
+
delete process.env.NODE_OPTIONS;
|
|
33
|
+
delete process.env.SUPERCOV_CJS_INTERCEPT;
|
|
34
|
+
}
|
|
35
|
+
if (generatedVitestConfig && /\/vitest(?:\.m?js)?$/.test(entrypoint)) {
|
|
36
|
+
// Worker processes inherit this marker. In particular, do not eagerly load
|
|
37
|
+
// Playwright's expect implementation in a Vitest worker: both runners use
|
|
38
|
+
// the Jest matcher registry symbol and intentionally cannot coexist there.
|
|
39
|
+
process.env.SUPERCOV_INSIDE_VITEST = "1";
|
|
40
|
+
let originalConfig;
|
|
41
|
+
for (let index = 2; index < process.argv.length; index += 1) {
|
|
42
|
+
const argument = process.argv[index];
|
|
43
|
+
if (argument === "--config" || argument === "-c") {
|
|
44
|
+
const configured = process.argv[index + 1];
|
|
45
|
+
const resolvedConfig = configured
|
|
46
|
+
? resolve(process.cwd(), configured)
|
|
47
|
+
: undefined;
|
|
48
|
+
if (resolvedConfig && resolvedConfig !== resolve(generatedVitestConfig)) {
|
|
49
|
+
originalConfig = resolvedConfig;
|
|
50
|
+
}
|
|
51
|
+
process.argv.splice(index, configured ? 2 : 1);
|
|
52
|
+
index -= 1;
|
|
53
|
+
} else if (argument?.startsWith("--config=")) {
|
|
54
|
+
const resolvedConfig = resolve(
|
|
55
|
+
process.cwd(),
|
|
56
|
+
argument.slice("--config=".length),
|
|
57
|
+
);
|
|
58
|
+
if (resolvedConfig !== resolve(generatedVitestConfig)) {
|
|
59
|
+
originalConfig = resolvedConfig;
|
|
60
|
+
}
|
|
61
|
+
process.argv.splice(index, 1);
|
|
62
|
+
index -= 1;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (originalConfig) {
|
|
66
|
+
process.env.SUPERCOV_ORIGINAL_VITEST_CONFIG = originalConfig;
|
|
67
|
+
}
|
|
68
|
+
process.argv.push("--config", generatedVitestConfig);
|
|
69
|
+
if (process.env.SUPERCOV_DEBUG === "1") {
|
|
70
|
+
console.error("[supercov] Vitest argv configured", {
|
|
71
|
+
entrypoint,
|
|
72
|
+
originalConfig,
|
|
73
|
+
generatedVitestConfig,
|
|
74
|
+
argv: process.argv.slice(2),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (
|
|
80
|
+
generatedPlaywrightConfig &&
|
|
81
|
+
/\/(?:node_modules\/\.bin\/playwright|node_modules\/(?:@playwright\/test|playwright)\/(?:cli\.js|.*\/program\.js))$/.test(
|
|
82
|
+
entrypoint,
|
|
83
|
+
)
|
|
84
|
+
) {
|
|
85
|
+
for (let index = 2; index < process.argv.length; index += 1) {
|
|
86
|
+
const argument = process.argv[index];
|
|
87
|
+
if (argument === "--config" || argument === "-c") {
|
|
88
|
+
process.argv.splice(index, process.argv[index + 1] ? 2 : 1);
|
|
89
|
+
index -= 1;
|
|
90
|
+
} else if (argument?.startsWith("--config=")) {
|
|
91
|
+
process.argv.splice(index, 1);
|
|
92
|
+
index -= 1;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
process.argv.push("--config", generatedPlaywrightConfig);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (
|
|
99
|
+
process.env.SUPERCOV_CJS_INTERCEPT === "1" &&
|
|
100
|
+
process.env.SUPERCOV_INSIDE_VITEST !== "1" &&
|
|
101
|
+
process.env.VITEST !== "true"
|
|
102
|
+
) {
|
|
103
|
+
const target = playwrightTarget ?? "@playwright/test";
|
|
104
|
+
const projectRoot = process.env.SUPERCOV_PROJECT_ROOT;
|
|
105
|
+
const wrapper = await import(new URL("./playwright.ts", import.meta.url));
|
|
106
|
+
const originalLoad = Module._load;
|
|
107
|
+
|
|
108
|
+
Module._load = function supercovLoad(request, parent, isMain) {
|
|
109
|
+
const parentFile = parent?.filename?.replaceAll("\\", "/");
|
|
110
|
+
const normalizedRoot = projectRoot
|
|
111
|
+
?.replaceAll("\\", "/")
|
|
112
|
+
.replace(/\/$/, "");
|
|
113
|
+
const belongsToProject =
|
|
114
|
+
Boolean(parentFile) &&
|
|
115
|
+
!parentFile.includes("/node_modules/") &&
|
|
116
|
+
!parentFile.includes("/.supercov/") &&
|
|
117
|
+
(normalizedRoot
|
|
118
|
+
? parentFile.startsWith(`${normalizedRoot}/`)
|
|
119
|
+
: parentFile.includes("/tests/"));
|
|
120
|
+
if (request === target && belongsToProject) return wrapper;
|
|
121
|
+
return originalLoad.call(this, request, parent, isMain);
|
|
122
|
+
};
|
|
123
|
+
}
|
package/src/reporter.ts
ADDED
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
import {
|
|
2
|
+
existsSync,
|
|
3
|
+
mkdirSync,
|
|
4
|
+
readFileSync,
|
|
5
|
+
readdirSync,
|
|
6
|
+
statSync,
|
|
7
|
+
writeFileSync,
|
|
8
|
+
} from "node:fs";
|
|
9
|
+
import { basename, dirname, resolve } from "node:path";
|
|
10
|
+
import { gzipSync } from "node:zlib";
|
|
11
|
+
import type { FullConfig, Reporter } from "@playwright/test/reporter";
|
|
12
|
+
import { createMcdcReport } from "./analyze.ts";
|
|
13
|
+
import { backgroundEvidenceDirectory } from "./transport.ts";
|
|
14
|
+
import type {
|
|
15
|
+
CoverageManifest,
|
|
16
|
+
McdcCoverageView,
|
|
17
|
+
McdcRawTestResult,
|
|
18
|
+
McdcReport,
|
|
19
|
+
McdcVector,
|
|
20
|
+
CoverageServerRecord,
|
|
21
|
+
CoverageRunIntegrity,
|
|
22
|
+
} from "./types.ts";
|
|
23
|
+
|
|
24
|
+
function findFiles(root: string, name: string): string[] {
|
|
25
|
+
if (!existsSync(root)) return [];
|
|
26
|
+
const found: string[] = [];
|
|
27
|
+
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
28
|
+
const path = resolve(root, entry.name);
|
|
29
|
+
if (entry.isDirectory()) found.push(...findFiles(path, name));
|
|
30
|
+
else if (entry.name === name) found.push(path);
|
|
31
|
+
}
|
|
32
|
+
return found;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function readBackgroundEvidence(runId: string): McdcRawTestResult | undefined {
|
|
36
|
+
const directory = backgroundEvidenceDirectory(runId);
|
|
37
|
+
if (!existsSync(directory)) return undefined;
|
|
38
|
+
const records = readdirSync(directory, { withFileTypes: true }).flatMap(
|
|
39
|
+
(entry) => {
|
|
40
|
+
if (!entry.isFile() || !entry.name.endsWith(".jsonl")) return [];
|
|
41
|
+
return readFileSync(resolve(directory, entry.name), "utf8")
|
|
42
|
+
.split("\n")
|
|
43
|
+
.filter(Boolean)
|
|
44
|
+
.flatMap((line) => {
|
|
45
|
+
try {
|
|
46
|
+
return [JSON.parse(line) as CoverageServerRecord];
|
|
47
|
+
} catch {
|
|
48
|
+
return [];
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
},
|
|
52
|
+
);
|
|
53
|
+
if (records.length === 0) return undefined;
|
|
54
|
+
return {
|
|
55
|
+
testId: `background:${runId}`,
|
|
56
|
+
test: "Background / unattributed",
|
|
57
|
+
title: "Background / unattributed",
|
|
58
|
+
status: "unknown",
|
|
59
|
+
provenance: {
|
|
60
|
+
runner: "background",
|
|
61
|
+
kind: "background",
|
|
62
|
+
source: "explicit",
|
|
63
|
+
},
|
|
64
|
+
role: "background",
|
|
65
|
+
browser: [],
|
|
66
|
+
server: records,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function formatVector(vector: McdcVector): string {
|
|
71
|
+
return `${vector.values.map((value) => (value === null ? "–" : value ? "T" : "F")).join(" ")} → ${
|
|
72
|
+
vector.outcome ? "T" : "F"
|
|
73
|
+
}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function escapeHtml(value: unknown): string {
|
|
77
|
+
return String(value)
|
|
78
|
+
.replaceAll("&", "&")
|
|
79
|
+
.replaceAll("<", "<")
|
|
80
|
+
.replaceAll(">", ">")
|
|
81
|
+
.replaceAll('"', """);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function renderHtml(
|
|
85
|
+
report: McdcCoverageView,
|
|
86
|
+
filter: "all" | "passed" | "failed",
|
|
87
|
+
runValid?: boolean,
|
|
88
|
+
): string {
|
|
89
|
+
const testNames = new Map(
|
|
90
|
+
report.tests.map((test) => [test.id, test.name] as const),
|
|
91
|
+
);
|
|
92
|
+
const phaseNames = new Map(
|
|
93
|
+
report.phases.map(
|
|
94
|
+
(phase) =>
|
|
95
|
+
[
|
|
96
|
+
phase.id,
|
|
97
|
+
`${phase.operation}${phase.source ? ` at ${phase.source}` : ""}`,
|
|
98
|
+
] as const,
|
|
99
|
+
),
|
|
100
|
+
);
|
|
101
|
+
const formatTests = (tests: string[]): string =>
|
|
102
|
+
tests.map((test) => escapeHtml(testNames.get(test) ?? test)).join(", ");
|
|
103
|
+
const decisionRows = report.decisions
|
|
104
|
+
.map((decision) => {
|
|
105
|
+
const conditionRows = decision.conditions
|
|
106
|
+
.map(
|
|
107
|
+
(condition) => `
|
|
108
|
+
<li class="${condition.covered ? "covered" : "missing"}">
|
|
109
|
+
C${condition.index + 1}: <code>${escapeHtml(condition.source)}</code>
|
|
110
|
+
${condition.assertionCovered ? '<strong class="covered">assertion-linked</strong>' : '<small>execution-only</small>'}
|
|
111
|
+
— ${
|
|
112
|
+
condition.covered
|
|
113
|
+
? `covered by ${condition
|
|
114
|
+
.witness!.map(
|
|
115
|
+
(vector, index) =>
|
|
116
|
+
`${formatVector(vector)} [${formatTests(condition.witnessTests?.[index] ?? [])}]`,
|
|
117
|
+
)
|
|
118
|
+
.join(" / ")}`
|
|
119
|
+
: "missing independence pair"
|
|
120
|
+
}
|
|
121
|
+
</li>`,
|
|
122
|
+
)
|
|
123
|
+
.join("");
|
|
124
|
+
const vectorRows = decision.vectorObservations
|
|
125
|
+
.map(
|
|
126
|
+
(observation) =>
|
|
127
|
+
`<li><code>${escapeHtml(formatVector(observation.vector))}</code> — ${formatTests(observation.tests)} <small>confidence: ${escapeHtml(observation.confidence?.level ?? "unknown")}</small>${observation.phases?.length ? `<br><small>${observation.phases.map((phase) => escapeHtml(phaseNames.get(phase) ?? phase)).join(" · ")}</small>` : ""}</li>`,
|
|
128
|
+
)
|
|
129
|
+
.join("");
|
|
130
|
+
return `
|
|
131
|
+
<section>
|
|
132
|
+
<h2>${escapeHtml(decision.meta.file)}:${decision.meta.line}:${decision.meta.column}</h2>
|
|
133
|
+
<pre>${escapeHtml(decision.meta.source)}</pre>
|
|
134
|
+
<p>${decision.vectors.length} distinct vector(s); ${decision.covered ? "fully covered" : "incomplete"}; confidence: <strong>${escapeHtml(decision.confidence?.level ?? "unknown")}</strong></p>
|
|
135
|
+
<ul>${conditionRows}</ul>
|
|
136
|
+
<details><summary>Vector-to-test attribution</summary><ul>${vectorRows || "<li>None</li>"}</ul></details>
|
|
137
|
+
</section>`;
|
|
138
|
+
})
|
|
139
|
+
.join("");
|
|
140
|
+
const branchRows = report.branches
|
|
141
|
+
.map(
|
|
142
|
+
(branch) => `
|
|
143
|
+
<section>
|
|
144
|
+
<h2>${escapeHtml(branch.meta.file)}:${branch.meta.line}:${branch.meta.column}</h2>
|
|
145
|
+
<p><strong>${escapeHtml(branch.meta.kind)}</strong></p>
|
|
146
|
+
<pre>${escapeHtml(branch.meta.source)}</pre>
|
|
147
|
+
<ul>${branch.alternatives
|
|
148
|
+
.map(
|
|
149
|
+
(alternative) => `
|
|
150
|
+
<li class="${alternative.covered ? "covered" : "missing"}">
|
|
151
|
+
${escapeHtml(alternative.label)} — ${alternative.covered ? `covered by ${formatTests(alternative.tests)}` : "not observed"} <small>confidence: ${escapeHtml(alternative.confidence?.level ?? "unexecuted")}</small>
|
|
152
|
+
</li>`,
|
|
153
|
+
)
|
|
154
|
+
.join("")}</ul>
|
|
155
|
+
</section>`,
|
|
156
|
+
)
|
|
157
|
+
.join("");
|
|
158
|
+
const testRows = report.tests
|
|
159
|
+
.map(
|
|
160
|
+
(test) => `
|
|
161
|
+
<section>
|
|
162
|
+
<h2>${escapeHtml(test.name)}</h2>
|
|
163
|
+
<p><small>${escapeHtml(test.provenance.kind)}/${escapeHtml(test.provenance.runner)}${test.role !== "test" ? ` — ${escapeHtml(test.role)} scope` : ""} — outcome: <strong>${escapeHtml(test.outcome)}</strong>${test.attempts.length ? ` (${test.attempts.map((attempt) => `retry ${attempt.retry}: ${attempt.status}`).join(", ")})` : ""}</small></p>
|
|
164
|
+
<p>${test.lines.length} source line(s), ${test.hits.length} point/alternative hit(s), ${test.decisions.reduce((total, decision) => total + decision.vectors.length, 0)} decision vector(s)</p>
|
|
165
|
+
<details><summary>Covered source lines</summary><p>${
|
|
166
|
+
test.lines.length > 0
|
|
167
|
+
? test.lines
|
|
168
|
+
.map(
|
|
169
|
+
(line) =>
|
|
170
|
+
`<code>${escapeHtml(line.file)}:${line.line}</code>`,
|
|
171
|
+
)
|
|
172
|
+
.join(" · ")
|
|
173
|
+
: "None"
|
|
174
|
+
}</p></details>
|
|
175
|
+
</section>`,
|
|
176
|
+
)
|
|
177
|
+
.join("");
|
|
178
|
+
const phaseRows = report.phases
|
|
179
|
+
.map(
|
|
180
|
+
(phase) => `
|
|
181
|
+
<section>
|
|
182
|
+
<h2><span class="${phase.kind === "assertion" ? "covered" : ""}">${escapeHtml(phase.kind)}</span> — ${escapeHtml(phase.operation)}</h2>
|
|
183
|
+
<p>${phase.source ? `<code>${escapeHtml(phase.source)}</code><br>` : ""}${escapeHtml(testNames.get(phase.test) ?? phase.test)}</p>
|
|
184
|
+
${phase.causedByPhaseId ? `<p>Observes the result of: <strong>${escapeHtml(phaseNames.get(phase.causedByPhaseId) ?? phase.causedByPhaseId)}</strong></p>` : ""}
|
|
185
|
+
<p>${phase.lines.length} source line(s), ${phase.decisions.reduce((total, decision) => total + decision.vectors.length, 0)} decision vector(s), ${phase.browserEvents} browser and ${phase.serverEvents} server event(s)<br><small>Browser: ${phase.explicitBrowserEvents} explicit / ${phase.inferredBrowserEvents} fallback. Server: ${phase.explicitServerEvents} explicit / ${phase.inferredServerEvents} fallback.</small></p>
|
|
186
|
+
<details><summary>Attributed source lines</summary><p>${
|
|
187
|
+
phase.lines.length > 0
|
|
188
|
+
? phase.lines
|
|
189
|
+
.map(
|
|
190
|
+
(line) =>
|
|
191
|
+
`<code>${escapeHtml(line.file)}:${line.line}</code>`,
|
|
192
|
+
)
|
|
193
|
+
.join(" · ")
|
|
194
|
+
: "None"
|
|
195
|
+
}</p></details>
|
|
196
|
+
</section>`,
|
|
197
|
+
)
|
|
198
|
+
.join("");
|
|
199
|
+
const uncoveredPoints = report.points
|
|
200
|
+
.filter((point) => !point.covered)
|
|
201
|
+
.map(
|
|
202
|
+
(point) => `
|
|
203
|
+
<li><strong>${escapeHtml(point.meta.kind)}</strong> ${escapeHtml(point.meta.file)}:${point.meta.line}:${point.meta.column}
|
|
204
|
+
— <code>${escapeHtml(point.meta.label ?? point.meta.source.slice(0, 160))}</code></li>`,
|
|
205
|
+
)
|
|
206
|
+
.join("");
|
|
207
|
+
const summary = report.summary;
|
|
208
|
+
const verifiedComplete = runValid === true && summary.coverageComplete;
|
|
209
|
+
const verdict =
|
|
210
|
+
filter === "all"
|
|
211
|
+
? summary.coverageComplete
|
|
212
|
+
? "OBSERVED COMPLETE"
|
|
213
|
+
: "OBSERVED INCOMPLETE"
|
|
214
|
+
: filter === "failed"
|
|
215
|
+
? "DIAGNOSTIC"
|
|
216
|
+
: runValid === false
|
|
217
|
+
? "INVALID"
|
|
218
|
+
: verifiedComplete
|
|
219
|
+
? "COMPLETE"
|
|
220
|
+
: "INCOMPLETE";
|
|
221
|
+
const verdictComplete = filter === "all"
|
|
222
|
+
? summary.coverageComplete
|
|
223
|
+
: filter === "passed" && verifiedComplete;
|
|
224
|
+
return `<!doctype html>
|
|
225
|
+
<html><head><meta charset="utf-8"><title>Essential SEO coverage completeness</title>
|
|
226
|
+
<style>
|
|
227
|
+
body{font:15px/1.45 system-ui,sans-serif;max-width:1100px;margin:40px auto;padding:0 20px;color:#202124}
|
|
228
|
+
header{padding:22px;border-radius:12px;background:#f4f6f8}section{border-top:1px solid #ddd;padding:18px 0}
|
|
229
|
+
pre,code{font-family:ui-monospace,SFMono-Regular,monospace}pre{white-space:pre-wrap;background:#f7f7f7;padding:12px}
|
|
230
|
+
.covered{color:#176b36}.missing{color:#a12622}li{margin:7px 0}
|
|
231
|
+
.metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:10px}.metric{background:white;padding:10px;border-radius:8px}
|
|
232
|
+
</style></head><body>
|
|
233
|
+
<nav><strong>Filter:</strong> ${filter === "all" ? "all attempts" : '<a href="report.html">all attempts</a>'} · ${filter === "passed" ? "passed" : '<a href="report-passed.html">passed</a>'} · ${filter === "failed" ? "failed" : '<a href="report-failed.html">failed</a>'}</nav>
|
|
234
|
+
<header><h1>${filter === "all" ? "Observed" : filter === "passed" ? "Passed-only verified" : "Failed-attempt"} coverage report</h1>
|
|
235
|
+
${filter === "all" ? "<p>Diagnostic aggregate: includes execution from every attempt.</p>" : filter === "passed" ? "<p>Verified filter: includes only successful attempts belonging to ultimately passing tests. Failed retry attempts are excluded.</p>" : "<p>Diagnostic filter: includes only failed attempts, including failed retries of flaky tests.</p>"}
|
|
236
|
+
<p class="${verdictComplete ? "covered" : "missing"}"><strong>${verdict}</strong> — ${filter === "passed" && runValid === false ? "the test command failed, so this run cannot establish completeness even if its passing tests cover every obligation" : filter === "failed" ? "this evidence explains what failing attempts executed and never establishes completeness" : `assuming test expectations are correct, ${summary.coverageComplete ? "all obligations in the measured model were exercised" : summary.completenessBlocked ? "a discovered construct cannot yet receive a truthful denominator" : "uncovered obligations remain in the measured model"}`}.</p>
|
|
237
|
+
<div class="metrics">
|
|
238
|
+
${[
|
|
239
|
+
["Lines", summary.lines],
|
|
240
|
+
["Statements", summary.statements],
|
|
241
|
+
["Functions", summary.functions],
|
|
242
|
+
["Branches", summary.branches],
|
|
243
|
+
["Decision outcomes", summary.decisionOutcomes],
|
|
244
|
+
["Condition outcomes", summary.conditionOutcomes],
|
|
245
|
+
["Value selections", summary.valueSelections],
|
|
246
|
+
]
|
|
247
|
+
.map(([label, metric]) => {
|
|
248
|
+
const value = metric as McdcReport["summary"]["lines"];
|
|
249
|
+
return `<div class="metric"><strong>${label}</strong><br>${value.percentage}% (${value.covered}/${value.total})</div>`;
|
|
250
|
+
})
|
|
251
|
+
.join("")}
|
|
252
|
+
<div class="metric"><strong>Masking MC/DC</strong><br>${summary.conditionCoveragePct}% (${summary.coveredConditions}/${summary.conditions})</div>
|
|
253
|
+
</div></header>
|
|
254
|
+
<section><h1>What this verdict means</h1>
|
|
255
|
+
<p>${escapeHtml(report.model.completenessMeaning)}</p>
|
|
256
|
+
<details><summary>Measured obligations</summary><ul>${report.model.measured.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul></details>
|
|
257
|
+
<details><summary>Not measured</summary><ul>${report.model.notMeasured.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul></details>
|
|
258
|
+
${report.limitations?.length ? `<details open><summary>Completeness blockers discovered in this source</summary><ul>${report.limitations.map((item) => `<li><code>${escapeHtml(item.file)}:${item.line}:${item.column}</code> — ${escapeHtml(item.reason)}<br><code>${escapeHtml(item.source)}</code></li>`).join("")}</ul></details>` : ""}
|
|
259
|
+
</section>
|
|
260
|
+
<h1>Per-test attribution</h1>
|
|
261
|
+
<p>Each source hit and decision vector is attributed to its runner, semantic test level, and individual test. Runner setup/import work and background work are separate scopes. Confidence distinguishes execution-only evidence, actions, and action/request chains ending in a passed assertion.</p>
|
|
262
|
+
${testRows || "<p>No tests were collected.</p>"}
|
|
263
|
+
<h1>Action and assertion attribution</h1>
|
|
264
|
+
<p>Coverage events are assigned to automatically instrumented Playwright actions and assertions. Browser requests carry an explicit phase ID into Remix loaders/actions, where async context preserves it through awaited server work. Events outside a traced request are visibly counted as timing fallbacks.</p>
|
|
265
|
+
${phaseRows || "<p>No instrumented actions or assertions were collected.</p>"}
|
|
266
|
+
<h1>Uncovered statements and functions</h1><ul>${uncoveredPoints || '<li class="covered">None</li>'}</ul>
|
|
267
|
+
<h1>Control decisions</h1>${decisionRows}
|
|
268
|
+
<h1>Value and switch alternatives</h1>${branchRows || "<p>None</p>"}
|
|
269
|
+
</body></html>`;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function rawTestId(raw: McdcRawTestResult): string {
|
|
273
|
+
return raw.testId ?? raw.test;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Keep only the successful final attempt of ultimately passing tests. Status
|
|
278
|
+
* records and coverage records are separate for some runners, so eligibility
|
|
279
|
+
* is resolved per (stable test ID, retry) before evidence is filtered.
|
|
280
|
+
*/
|
|
281
|
+
export function passingCoverageResults(
|
|
282
|
+
rawResults: McdcRawTestResult[],
|
|
283
|
+
): McdcRawTestResult[] {
|
|
284
|
+
const attemptsByTest = new Map<
|
|
285
|
+
string,
|
|
286
|
+
Map<number, { statuses: Set<string>; expectsFailure: boolean }>
|
|
287
|
+
>();
|
|
288
|
+
for (const raw of rawResults) {
|
|
289
|
+
const retry = raw.retry ?? 0;
|
|
290
|
+
const attempts = attemptsByTest.get(rawTestId(raw)) ?? new Map();
|
|
291
|
+
const attempt = attempts.get(retry) ?? {
|
|
292
|
+
statuses: new Set<string>(),
|
|
293
|
+
expectsFailure: false,
|
|
294
|
+
};
|
|
295
|
+
if (raw.status) attempt.statuses.add(raw.status);
|
|
296
|
+
attempt.expectsFailure ||= raw.expectedStatus === "failed";
|
|
297
|
+
attempts.set(retry, attempt);
|
|
298
|
+
attemptsByTest.set(rawTestId(raw), attempts);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const accepted = new Set<string>();
|
|
302
|
+
for (const [testId, attempts] of attemptsByTest) {
|
|
303
|
+
const retry = Math.max(...attempts.keys());
|
|
304
|
+
const terminal = attempts.get(retry)!;
|
|
305
|
+
if (terminal.statuses.has("passed") && !terminal.expectsFailure) {
|
|
306
|
+
accepted.add(`${testId}\0${retry}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return rawResults.filter((raw) =>
|
|
310
|
+
accepted.has(`${rawTestId(raw)}\0${raw.retry ?? 0}`),
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Coverage executed by attempts whose actual runner status is failed. */
|
|
315
|
+
export function failedCoverageResults(
|
|
316
|
+
rawResults: McdcRawTestResult[],
|
|
317
|
+
): McdcRawTestResult[] {
|
|
318
|
+
const failedAttempts = new Set(
|
|
319
|
+
rawResults
|
|
320
|
+
.filter((raw) => raw.status === "failed")
|
|
321
|
+
.map((raw) => `${rawTestId(raw)}\0${raw.retry ?? 0}`),
|
|
322
|
+
);
|
|
323
|
+
return rawResults.filter((raw) =>
|
|
324
|
+
failedAttempts.has(`${rawTestId(raw)}\0${raw.retry ?? 0}`),
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
export function writeMcdcReport(
|
|
329
|
+
outputDir: string,
|
|
330
|
+
runId: string,
|
|
331
|
+
minimumArtifactMtimeMs = 0,
|
|
332
|
+
configuredManifestPath?: string,
|
|
333
|
+
testExitCode?: number | null,
|
|
334
|
+
integrity?: CoverageRunIntegrity,
|
|
335
|
+
): McdcReport {
|
|
336
|
+
const manifestPath = configuredManifestPath
|
|
337
|
+
? resolve(configuredManifestPath)
|
|
338
|
+
: resolve(
|
|
339
|
+
process.cwd(),
|
|
340
|
+
process.env["SUPERCOV_MANIFEST"] ??
|
|
341
|
+
".supercov/mcdc-manifest.json",
|
|
342
|
+
);
|
|
343
|
+
if (!existsSync(manifestPath)) {
|
|
344
|
+
throw new Error(`Coverage manifest was not found at ${manifestPath}`);
|
|
345
|
+
}
|
|
346
|
+
const manifest = JSON.parse(
|
|
347
|
+
readFileSync(manifestPath, "utf8"),
|
|
348
|
+
) as CoverageManifest;
|
|
349
|
+
const rawResults = findFiles(outputDir, "mcdc.json")
|
|
350
|
+
.filter((path) => statSync(path).mtimeMs >= minimumArtifactMtimeMs)
|
|
351
|
+
.map((path) => JSON.parse(readFileSync(path, "utf8")) as McdcRawTestResult);
|
|
352
|
+
const background = readBackgroundEvidence(runId);
|
|
353
|
+
if (background) rawResults.push(background);
|
|
354
|
+
const incompatibleScope = rawResults.find(
|
|
355
|
+
(raw) => raw.scope && raw.scope.runId !== runId,
|
|
356
|
+
);
|
|
357
|
+
if (incompatibleScope) {
|
|
358
|
+
throw new Error(
|
|
359
|
+
`Coverage evidence for run ${incompatibleScope.scope!.runId} cannot be used in run ${runId}`,
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
if (rawResults.length === 0) {
|
|
363
|
+
throw new Error(`No coverage evidence was collected under ${outputDir}`);
|
|
364
|
+
}
|
|
365
|
+
const report = createMcdcReport(manifest, rawResults);
|
|
366
|
+
const passed = createMcdcReport(manifest, passingCoverageResults(rawResults));
|
|
367
|
+
const failed = createMcdcReport(manifest, failedCoverageResults(rawResults));
|
|
368
|
+
report.filters = { passed, failed };
|
|
369
|
+
if (integrity) {
|
|
370
|
+
report.integrity = integrity;
|
|
371
|
+
passed.integrity = integrity;
|
|
372
|
+
failed.integrity = integrity;
|
|
373
|
+
}
|
|
374
|
+
if (testExitCode !== undefined) {
|
|
375
|
+
report.execution = { testExitCode, valid: testExitCode === 0 };
|
|
376
|
+
}
|
|
377
|
+
const storedRunDirectory = resolve(
|
|
378
|
+
process.cwd(),
|
|
379
|
+
".supercov/runs",
|
|
380
|
+
runId,
|
|
381
|
+
);
|
|
382
|
+
mkdirSync(storedRunDirectory, { recursive: true });
|
|
383
|
+
const htmlPath = resolve(storedRunDirectory, "report.html");
|
|
384
|
+
const serializedReport = `${JSON.stringify(report, null, 2)}\n`;
|
|
385
|
+
writeFileSync(
|
|
386
|
+
resolve(storedRunDirectory, "report.json.gz"),
|
|
387
|
+
gzipSync(serializedReport, { level: 9 }),
|
|
388
|
+
);
|
|
389
|
+
writeFileSync(
|
|
390
|
+
htmlPath,
|
|
391
|
+
renderHtml(report, "all", report.execution?.valid),
|
|
392
|
+
);
|
|
393
|
+
writeFileSync(
|
|
394
|
+
resolve(storedRunDirectory, "report-passed.html"),
|
|
395
|
+
renderHtml(passed, "passed", report.execution?.valid),
|
|
396
|
+
);
|
|
397
|
+
writeFileSync(
|
|
398
|
+
resolve(storedRunDirectory, "report-failed.html"),
|
|
399
|
+
renderHtml(failed, "failed", report.execution?.valid),
|
|
400
|
+
);
|
|
401
|
+
|
|
402
|
+
const summary = report.summary;
|
|
403
|
+
console.log(
|
|
404
|
+
`[coverage] lines ${summary.lines.percentage}%, statements ${summary.statements.percentage}%, ` +
|
|
405
|
+
`functions ${summary.functions.percentage}%, branches ${summary.branches.percentage}%, ` +
|
|
406
|
+
`MC/DC ${summary.conditionCoveragePct}%`,
|
|
407
|
+
);
|
|
408
|
+
console.log(
|
|
409
|
+
`[coverage] verdict: ${summary.coverageComplete ? "COMPLETE" : "INCOMPLETE"}`,
|
|
410
|
+
);
|
|
411
|
+
console.log(
|
|
412
|
+
`[coverage] passed only: lines ${passed.summary.lines.percentage}%, branches ${passed.summary.branches.percentage}%, MC/DC ${passed.summary.conditionCoveragePct}%`,
|
|
413
|
+
);
|
|
414
|
+
console.log(`[coverage] report: ${htmlPath}`);
|
|
415
|
+
return report;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
export default class McdcReporter implements Reporter {
|
|
419
|
+
private outputDir = "";
|
|
420
|
+
|
|
421
|
+
onBegin(config: FullConfig): void {
|
|
422
|
+
this.outputDir = config.projects[0]?.outputDir ?? "";
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
onEnd(): void {
|
|
426
|
+
const runId =
|
|
427
|
+
process.env["TEST_OFFLINE_RESULTS_RUN_ID"] ??
|
|
428
|
+
basename(dirname(this.outputDir));
|
|
429
|
+
writeMcdcReport(this.outputDir, runId);
|
|
430
|
+
}
|
|
431
|
+
}
|