suitevitals 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +169 -2
- package/dist/adapters/dispatch.js +19 -0
- package/dist/adapters/junit.js +176 -0
- package/dist/adapters/mochawesome.js +78 -0
- package/dist/adapters/playwright-blob.js +146 -0
- package/dist/artifacts/extract.js +93 -0
- package/dist/artifacts/fetch.js +167 -0
- package/dist/cli.js +55 -0
- package/dist/collect/config.js +155 -0
- package/dist/collect/filter.js +45 -0
- package/dist/collect/matrix.js +69 -0
- package/dist/collect/parse.js +89 -0
- package/dist/collect/rerun.js +86 -0
- package/dist/collect/watermark.js +28 -0
- package/dist/collect/window.js +28 -0
- package/dist/colors.js +35 -0
- package/dist/commands/collect.js +466 -0
- package/dist/commands/compact.js +72 -0
- package/dist/commands/dashboard.js +116 -0
- package/dist/commands/doctor.js +110 -0
- package/dist/commands/flags.js +21 -0
- package/dist/commands/prune.js +87 -0
- package/dist/commands/repo-flags.js +23 -0
- package/dist/commands/report.js +138 -0
- package/dist/doctor/diagnose.js +171 -0
- package/dist/events/error.js +94 -0
- package/dist/events/event.js +0 -0
- package/dist/github/artifacts.js +72 -0
- package/dist/github/auth.js +47 -0
- package/dist/github/client.js +62 -0
- package/dist/github/contents.js +74 -0
- package/dist/github/repo.js +26 -0
- package/dist/github/runs.js +129 -0
- package/dist/report/aggregate.js +0 -0
- package/dist/report/dashboard.js +187 -0
- package/dist/report/rank.js +46 -0
- package/dist/report/read.js +95 -0
- package/dist/report/render.js +77 -0
- package/dist/report/score.js +50 -0
- package/dist/state/lock.js +134 -0
- package/dist/state/state.js +48 -0
- package/dist/store/compact.js +119 -0
- package/dist/store/compress.js +33 -0
- package/dist/store/dedup.js +0 -0
- package/dist/store/durable.js +40 -0
- package/dist/store/ndjson.js +125 -0
- package/dist/store/partition.js +16 -0
- package/dist/store/prune.js +51 -0
- package/package.json +62 -4
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, join, normalize, sep } from "node:path";
|
|
3
|
+
import { unzipSync } from "fflate";
|
|
4
|
+
/** Default ceiling (MB) on a single archive's *decompressed* size — a zip-bomb / OOM guard (F1). */
|
|
5
|
+
export const DEFAULT_MAX_EXTRACT_MB = 500;
|
|
6
|
+
/** Thrown when an archive's declared uncompressed size exceeds the extraction budget. */
|
|
7
|
+
export class ArchiveTooLargeError extends Error {
|
|
8
|
+
name = "ArchiveTooLargeError";
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Extracts a zip archive's bytes into `destDir`, returning the absolute paths of the written
|
|
12
|
+
* files. Directory entries are skipped. Entry paths are validated to stay within `destDir` so a
|
|
13
|
+
* malicious archive cannot escape via `../` (zip-slip). The summed declared uncompressed size is
|
|
14
|
+
* capped at `maxBytes` and checked *before* each entry is inflated, so a zip bomb is rejected
|
|
15
|
+
* (with {@link ArchiveTooLargeError}) rather than exhausting memory (F1).
|
|
16
|
+
*/
|
|
17
|
+
export function extractZip(zip, destDir, maxBytes = DEFAULT_MAX_EXTRACT_MB * 1024 * 1024) {
|
|
18
|
+
return extractWithBudget(zip, destDir, { used: 0, max: maxBytes });
|
|
19
|
+
}
|
|
20
|
+
async function extractWithBudget(zip, destDir, budget) {
|
|
21
|
+
const entries = unzipSync(zip, { filter: budgetFilter(budget) });
|
|
22
|
+
const written = [];
|
|
23
|
+
for (const [name, bytes] of Object.entries(entries)) {
|
|
24
|
+
// Directory entries are emitted with empty contents and a trailing slash.
|
|
25
|
+
if (name.endsWith("/") || bytes.length === 0) {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const target = safeJoin(destDir, name);
|
|
29
|
+
await mkdir(dirname(target), { recursive: true });
|
|
30
|
+
await writeFile(target, bytes);
|
|
31
|
+
written.push(target);
|
|
32
|
+
}
|
|
33
|
+
return written;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Like {@link extractZip}, but also extracts nested zip archives up to `maxDepth` levels,
|
|
37
|
+
* returning the leaf (non-zip) file paths. Playwright blob reports are double-zipped — the
|
|
38
|
+
* artifact zip holds a `report-*.zip` which holds `report.jsonl` — so one nested level is
|
|
39
|
+
* needed to surface the report. A nested entry that fails to unzip is kept as-is; a nested
|
|
40
|
+
* entry that blows the extraction budget rethrows, since the whole artifact is hostile.
|
|
41
|
+
*/
|
|
42
|
+
export function extractZipNested(zip, destDir, maxDepth = 2, maxBytes = DEFAULT_MAX_EXTRACT_MB * 1024 * 1024) {
|
|
43
|
+
return extractNested(zip, destDir, maxDepth, { used: 0, max: maxBytes });
|
|
44
|
+
}
|
|
45
|
+
async function extractNested(zip, destDir, maxDepth, budget) {
|
|
46
|
+
const written = await extractWithBudget(zip, destDir, budget);
|
|
47
|
+
if (maxDepth <= 1) {
|
|
48
|
+
return written;
|
|
49
|
+
}
|
|
50
|
+
const leaves = [];
|
|
51
|
+
for (const file of written) {
|
|
52
|
+
if (!file.toLowerCase().endsWith(".zip")) {
|
|
53
|
+
leaves.push(file);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
const bytes = await readFile(file);
|
|
58
|
+
leaves.push(...(await extractNested(new Uint8Array(bytes), `${file}.extracted`, maxDepth - 1, budget)));
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
if (error instanceof ArchiveTooLargeError) {
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
// Not a readable zip; keep the file so nothing is silently lost.
|
|
65
|
+
leaves.push(file);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return leaves;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* A fflate entry filter that accumulates each entry's *declared* uncompressed size and throws
|
|
72
|
+
* once the shared total exceeds `budget.max` — before the offending entry is inflated, so a zip
|
|
73
|
+
* bomb never allocates its payload. (A crafted header could understate its size; the declared
|
|
74
|
+
* size catches every realistic bomb and all accidental-OOM cases.)
|
|
75
|
+
*/
|
|
76
|
+
function budgetFilter(budget) {
|
|
77
|
+
return (file) => {
|
|
78
|
+
budget.used += file.originalSize;
|
|
79
|
+
if (budget.used > budget.max) {
|
|
80
|
+
throw new ArchiveTooLargeError(`Archive expands past the ${Math.round(budget.max / (1024 * 1024))} MB extraction budget; refusing to inflate.`);
|
|
81
|
+
}
|
|
82
|
+
return true;
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** Joins `name` onto `base`, throwing if the result escapes `base` (zip-slip guard). */
|
|
86
|
+
function safeJoin(base, name) {
|
|
87
|
+
const target = normalize(join(base, name));
|
|
88
|
+
const root = normalize(base.endsWith(sep) ? base : base + sep);
|
|
89
|
+
if (target !== normalize(base) && !target.startsWith(root)) {
|
|
90
|
+
throw new Error(`Refusing to extract entry outside the destination: ${name}`);
|
|
91
|
+
}
|
|
92
|
+
return target;
|
|
93
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import pLimit from "p-limit";
|
|
3
|
+
import { terminalSafe } from "../colors.js";
|
|
4
|
+
import { partitionArtifacts } from "../github/artifacts.js";
|
|
5
|
+
import { attemptForArtifact } from "../github/runs.js";
|
|
6
|
+
import { ArchiveTooLargeError, DEFAULT_MAX_EXTRACT_MB, extractZipNested } from "./extract.js";
|
|
7
|
+
/** Default number of runs fetched in parallel; balances throughput against rate limits. */
|
|
8
|
+
export const DEFAULT_CONCURRENCY = 5;
|
|
9
|
+
/** Default size ceiling (MB) for tier-3 sniffing, so failure videos/traces never dominate (D9). */
|
|
10
|
+
export const DEFAULT_MAX_ARTIFACT_MB = 100;
|
|
11
|
+
function emptySummary() {
|
|
12
|
+
return {
|
|
13
|
+
runsScanned: 0,
|
|
14
|
+
artifactsSeen: 0,
|
|
15
|
+
reportArtifacts: 0,
|
|
16
|
+
filesExtracted: 0,
|
|
17
|
+
expiredArtifacts: 0,
|
|
18
|
+
runsWithExpired: 0,
|
|
19
|
+
sniffedArtifacts: 0,
|
|
20
|
+
oversizedArtifacts: 0,
|
|
21
|
+
failedArtifacts: 0,
|
|
22
|
+
skippedArtifacts: 0,
|
|
23
|
+
excludedArtifacts: 0,
|
|
24
|
+
collectedArtifacts: 0,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* For each run: list artifacts (D9 tier 1), partition them, download the report-glob
|
|
29
|
+
* matches (tier 2), and extract each into `workdir/<run_id>/<artifact_id>/`. Expired
|
|
30
|
+
* artifacts are counted, never silently dropped.
|
|
31
|
+
*
|
|
32
|
+
* Runs are fetched up to `concurrency` at a time (p-limit), since a backfill is dozens
|
|
33
|
+
* of independent, network-bound runs — the transport (client.ts) absorbs any rate-limit
|
|
34
|
+
* pushback with backoff. Each run is processed into an isolated {@link RunOutcome} so the
|
|
35
|
+
* parallel tasks never race on shared state; outcomes are then folded together in input
|
|
36
|
+
* order, keeping the summary and `reports[]` deterministic regardless of finish order.
|
|
37
|
+
*/
|
|
38
|
+
export async function fetchReportArtifacts(fetcher, runs, workdir, options = {}) {
|
|
39
|
+
const { dryRun = false, onProgress, concurrency = DEFAULT_CONCURRENCY, maxArtifactMb = DEFAULT_MAX_ARTIFACT_MB, maxExtractMb = DEFAULT_MAX_EXTRACT_MB, excludeArtifacts = [], collectedAttempt = () => 0, } = options;
|
|
40
|
+
const limit = pLimit(Math.max(1, concurrency));
|
|
41
|
+
const maxArtifactBytes = maxArtifactMb * 1024 * 1024;
|
|
42
|
+
const maxExtractBytes = maxExtractMb * 1024 * 1024;
|
|
43
|
+
let completed = 0;
|
|
44
|
+
const outcomes = await Promise.all(runs.map((run) => limit(async () => {
|
|
45
|
+
const outcome = await fetchRun(fetcher, run, workdir, dryRun, maxArtifactBytes, maxExtractBytes, excludeArtifacts, collectedAttempt(run));
|
|
46
|
+
completed++;
|
|
47
|
+
onProgress?.(completed, runs.length);
|
|
48
|
+
return outcome;
|
|
49
|
+
})));
|
|
50
|
+
return foldOutcomes(outcomes);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Fetches and extracts one run's artifacts in isolation (no shared state). Tier 2 downloads the
|
|
54
|
+
* report-name matches under the extraction budget. Tier 3 (D9): only when a run has *no* name
|
|
55
|
+
* match, the remaining artifacts under the size cap are downloaded and extracted too, so a report
|
|
56
|
+
* uploaded under an unexpected name is still found — the content sniff in the parse step discards
|
|
57
|
+
* anything that is not a report. Oversized artifacts are counted and reported, never silently
|
|
58
|
+
* dropped.
|
|
59
|
+
*/
|
|
60
|
+
async function fetchRun(fetcher, run, workdir, dryRun, maxArtifactBytes, maxExtractBytes, excludeArtifacts, collectedAttempt) {
|
|
61
|
+
const artifacts = await fetcher.list(run.run_id);
|
|
62
|
+
const partition = partitionArtifacts(artifacts, excludeArtifacts);
|
|
63
|
+
let oversized = 0;
|
|
64
|
+
// Tier 2: a report match whose *compressed* size already exceeds the decompressed-size budget
|
|
65
|
+
// could only fail that budget anyway — after an unbounded in-memory download. Skip it (and
|
|
66
|
+
// surface it) before downloading, so a huge "playwright-report" of videos can never OOM the
|
|
67
|
+
// process (F1).
|
|
68
|
+
const toDownload = [];
|
|
69
|
+
for (const artifact of partition.reports) {
|
|
70
|
+
if (artifact.size_bytes > maxExtractBytes) {
|
|
71
|
+
oversized++;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
toDownload.push(artifact);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// Tier 3 runs only when no report-name match was found for this run.
|
|
78
|
+
const sniffRun = partition.reports.length === 0;
|
|
79
|
+
const toSniff = [];
|
|
80
|
+
if (sniffRun) {
|
|
81
|
+
for (const artifact of partition.skipped) {
|
|
82
|
+
if (artifact.size_bytes > maxArtifactBytes) {
|
|
83
|
+
oversized++;
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
toSniff.push(artifact);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const files = [];
|
|
91
|
+
let failed = 0;
|
|
92
|
+
const collectedIds = new Set();
|
|
93
|
+
if (!dryRun) {
|
|
94
|
+
const candidates = [...toDownload, ...toSniff];
|
|
95
|
+
const attemptStarts = run.run_attempt > 1 && candidates.length > 0 ? await fetcher.attemptStarts(run) : [];
|
|
96
|
+
for (const artifact of candidates) {
|
|
97
|
+
const attempt = attemptStarts.length > 0 && artifact.created_at
|
|
98
|
+
? attemptForArtifact(artifact.created_at, attemptStarts)
|
|
99
|
+
: run.run_attempt;
|
|
100
|
+
if (attempt <= collectedAttempt) {
|
|
101
|
+
collectedIds.add(artifact.id);
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
const zip = await fetcher.download(artifact.id);
|
|
105
|
+
const dest = join(workdir, String(run.run_id), String(artifact.id));
|
|
106
|
+
try {
|
|
107
|
+
const paths = await extractZipNested(zip, dest, 2, maxExtractBytes);
|
|
108
|
+
files.push(...paths.map((path) => ({ path, artifact: artifact.name, attempt })));
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
// A zip bomb (declared size past the extraction budget) is counted as oversized and
|
|
112
|
+
// skipped — surfaced in the summary, never allowed to OOM the whole collection (F1).
|
|
113
|
+
if (error instanceof ArchiveTooLargeError) {
|
|
114
|
+
oversized++;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
// Any other extraction failure — corrupt zip bytes, an entry escaping the destination
|
|
118
|
+
// (zip-slip), an unwritable entry name — is deterministic content, so retrying cannot
|
|
119
|
+
// help; it costs this one artifact with a warning, never the whole collect (F1).
|
|
120
|
+
// Download errors, by contrast, still propagate: they are transient transport failures,
|
|
121
|
+
// and aborting before the watermark advances is what makes them safe to retry (D12).
|
|
122
|
+
failed++;
|
|
123
|
+
console.warn(terminalSafe(`Skipping artifact "${artifact.name}" (#${artifact.id}) of run ${run.run_id}: ` +
|
|
124
|
+
`${error.message}`));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
run,
|
|
130
|
+
artifactsSeen: artifacts.length,
|
|
131
|
+
reportArtifacts: toDownload.filter((artifact) => !collectedIds.has(artifact.id)).length,
|
|
132
|
+
expiredArtifacts: partition.expired.length,
|
|
133
|
+
sniffedArtifacts: toSniff.filter((artifact) => !collectedIds.has(artifact.id)).length,
|
|
134
|
+
oversizedArtifacts: oversized,
|
|
135
|
+
failedArtifacts: failed,
|
|
136
|
+
// When tier-3 ran, the skipped bucket was examined (sniffed/oversized); otherwise it is left alone.
|
|
137
|
+
skippedArtifacts: sniffRun ? 0 : partition.skipped.length,
|
|
138
|
+
excludedArtifacts: partition.excluded.length,
|
|
139
|
+
collectedArtifacts: collectedIds.size,
|
|
140
|
+
files,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
/** Folds per-run outcomes into one summary + report list, in input order. */
|
|
144
|
+
function foldOutcomes(outcomes) {
|
|
145
|
+
const summary = emptySummary();
|
|
146
|
+
const reports = [];
|
|
147
|
+
for (const outcome of outcomes) {
|
|
148
|
+
summary.runsScanned++;
|
|
149
|
+
summary.artifactsSeen += outcome.artifactsSeen;
|
|
150
|
+
summary.reportArtifacts += outcome.reportArtifacts;
|
|
151
|
+
summary.expiredArtifacts += outcome.expiredArtifacts;
|
|
152
|
+
summary.sniffedArtifacts += outcome.sniffedArtifacts;
|
|
153
|
+
summary.oversizedArtifacts += outcome.oversizedArtifacts;
|
|
154
|
+
summary.failedArtifacts += outcome.failedArtifacts;
|
|
155
|
+
summary.skippedArtifacts += outcome.skippedArtifacts;
|
|
156
|
+
if (outcome.expiredArtifacts > 0) {
|
|
157
|
+
summary.runsWithExpired++;
|
|
158
|
+
}
|
|
159
|
+
summary.excludedArtifacts += outcome.excludedArtifacts;
|
|
160
|
+
summary.collectedArtifacts += outcome.collectedArtifacts;
|
|
161
|
+
summary.filesExtracted += outcome.files.length;
|
|
162
|
+
if (outcome.files.length > 0) {
|
|
163
|
+
reports.push({ run: outcome.run, files: outcome.files });
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return { summary, reports };
|
|
167
|
+
}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { defineCommand, runMain } from "citty";
|
|
4
|
+
import { colorSupported, createPalette, terminalSafe } from "./colors.js";
|
|
5
|
+
import { collectCommand } from "./commands/collect.js";
|
|
6
|
+
import { compactCommand } from "./commands/compact.js";
|
|
7
|
+
import { dashboardCommand } from "./commands/dashboard.js";
|
|
8
|
+
import { doctorCommand } from "./commands/doctor.js";
|
|
9
|
+
import { pruneCommand } from "./commands/prune.js";
|
|
10
|
+
import { reportCommand } from "./commands/report.js";
|
|
11
|
+
const { version } = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
12
|
+
/**
|
|
13
|
+
* Prints a command's failure as one `Error: …` line and exits 1. `SUITEVITALS_DEBUG=1` prints
|
|
14
|
+
* the full error instead.
|
|
15
|
+
*/
|
|
16
|
+
function withCleanErrors(command) {
|
|
17
|
+
const { run } = command;
|
|
18
|
+
if (!run) {
|
|
19
|
+
return command;
|
|
20
|
+
}
|
|
21
|
+
return {
|
|
22
|
+
...command,
|
|
23
|
+
run: async (context) => {
|
|
24
|
+
try {
|
|
25
|
+
return await run(context);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (process.env.SUITEVITALS_DEBUG) {
|
|
29
|
+
console.error(error);
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33
|
+
console.error(createPalette(colorSupported).bad(`Error: ${terminalSafe(message)}`));
|
|
34
|
+
}
|
|
35
|
+
process.exitCode = 1;
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
const main = defineCommand({
|
|
41
|
+
meta: {
|
|
42
|
+
name: "suitevitals",
|
|
43
|
+
version,
|
|
44
|
+
description: "The vital signs of your test suite — flaky-test detection from CI artifacts.",
|
|
45
|
+
},
|
|
46
|
+
subCommands: {
|
|
47
|
+
collect: withCleanErrors(collectCommand),
|
|
48
|
+
report: withCleanErrors(reportCommand),
|
|
49
|
+
dashboard: withCleanErrors(dashboardCommand),
|
|
50
|
+
compact: withCleanErrors(compactCommand),
|
|
51
|
+
prune: withCleanErrors(pruneCommand),
|
|
52
|
+
doctor: withCleanErrors(doctorCommand),
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
runMain(main);
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { parseRepoRef, repoSlug } from "../github/repo.js";
|
|
3
|
+
import { compileMatrixPattern } from "./matrix.js";
|
|
4
|
+
/**
|
|
5
|
+
* Reads and validates a JSON config file. The shape is `{ hoursBack?, concurrency?, repos[] }`,
|
|
6
|
+
* where each `repos[]` entry is `"owner/name"` or `{ repo, hoursBack?, concurrency? }`. Validation
|
|
7
|
+
* is strict so a typo surfaces immediately rather than silently collecting the wrong repos.
|
|
8
|
+
*/
|
|
9
|
+
export async function loadConfig(path) {
|
|
10
|
+
const raw = await readFile(path, "utf8");
|
|
11
|
+
return parseConfig(raw, path);
|
|
12
|
+
}
|
|
13
|
+
/** Pure parse+validate of config JSON, separated from I/O for testing. */
|
|
14
|
+
export function parseConfig(raw, source = "config") {
|
|
15
|
+
let data;
|
|
16
|
+
try {
|
|
17
|
+
data = JSON.parse(raw);
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
throw new Error(`Invalid config ${source}: not valid JSON (${error.message}).`);
|
|
21
|
+
}
|
|
22
|
+
if (!isObject(data)) {
|
|
23
|
+
throw new Error(`Invalid config ${source}: expected a JSON object.`);
|
|
24
|
+
}
|
|
25
|
+
const repos = data.repos;
|
|
26
|
+
if (!Array.isArray(repos)) {
|
|
27
|
+
throw new Error(`Invalid config ${source}: "repos" must be an array.`);
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
hoursBack: optionalHoursBack(data.hoursBack, `${source} hoursBack`),
|
|
31
|
+
concurrency: optionalConcurrency(data.concurrency, `${source} concurrency`),
|
|
32
|
+
workflow: optionalString(data.workflow, `${source} workflow`),
|
|
33
|
+
excludeTriggers: optionalStringArray(data.excludeTriggers, `${source} excludeTriggers`),
|
|
34
|
+
matrixFromArtifact: optionalPattern(data.matrixFromArtifact, `${source} matrixFromArtifact`),
|
|
35
|
+
excludeArtifacts: optionalStringArray(data.excludeArtifacts, `${source} excludeArtifacts`),
|
|
36
|
+
repos: repos.map((entry, i) => parseEntry(entry, `${source} repos[${i}]`)),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function parseEntry(entry, where) {
|
|
40
|
+
if (typeof entry === "string") {
|
|
41
|
+
return entry;
|
|
42
|
+
}
|
|
43
|
+
if (!isObject(entry) || typeof entry.repo !== "string") {
|
|
44
|
+
throw new Error(`Invalid config ${where}: expected "owner/name" or an object with "repo".`);
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
repo: entry.repo,
|
|
48
|
+
hoursBack: optionalHoursBack(entry.hoursBack, `${where} hoursBack`),
|
|
49
|
+
concurrency: optionalConcurrency(entry.concurrency, `${where} concurrency`),
|
|
50
|
+
workflow: optionalString(entry.workflow, `${where} workflow`),
|
|
51
|
+
excludeTriggers: optionalStringArray(entry.excludeTriggers, `${where} excludeTriggers`),
|
|
52
|
+
matrixFromArtifact: optionalPattern(entry.matrixFromArtifact, `${where} matrixFromArtifact`),
|
|
53
|
+
excludeArtifacts: optionalStringArray(entry.excludeArtifacts, `${where} excludeArtifacts`),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Merges CLI repos and config repos into the final target list. Precedence per field is
|
|
58
|
+
* most-specific-wins: a per-repo config value beats the CLI flag, which beats the config-level
|
|
59
|
+
* default, which beats the built-in. Config repos come first (in file order), then any
|
|
60
|
+
* CLI-only repos; a repo named in both is de-duplicated to its config entry. Throws if the
|
|
61
|
+
* merge yields no repos.
|
|
62
|
+
*/
|
|
63
|
+
export function resolveTargets(input) {
|
|
64
|
+
const { cliRepos, cliHoursBack, cliConcurrency, cliWorkflow, cliExcludeTriggers, cliMatrixFromArtifact, cliExcludeArtifacts, config, defaultConcurrency, } = input;
|
|
65
|
+
const targets = [];
|
|
66
|
+
const seen = new Set();
|
|
67
|
+
const add = (repo, perRepo) => {
|
|
68
|
+
const ref = parseRepoRef(repo);
|
|
69
|
+
const slug = repoSlug(ref);
|
|
70
|
+
if (seen.has(slug)) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
seen.add(slug);
|
|
74
|
+
targets.push({
|
|
75
|
+
ref,
|
|
76
|
+
slug,
|
|
77
|
+
hoursBack: perRepo.hoursBack ?? cliHoursBack ?? config?.hoursBack,
|
|
78
|
+
concurrency: perRepo.concurrency ?? cliConcurrency ?? config?.concurrency ?? defaultConcurrency,
|
|
79
|
+
workflow: perRepo.workflow ?? cliWorkflow ?? config?.workflow,
|
|
80
|
+
excludeTriggers: perRepo.excludeTriggers ?? cliExcludeTriggers ?? config?.excludeTriggers,
|
|
81
|
+
matrixFromArtifact: perRepo.matrixFromArtifact ?? cliMatrixFromArtifact ?? config?.matrixFromArtifact,
|
|
82
|
+
excludeArtifacts: perRepo.excludeArtifacts ?? cliExcludeArtifacts ?? config?.excludeArtifacts,
|
|
83
|
+
});
|
|
84
|
+
};
|
|
85
|
+
for (const entry of config?.repos ?? []) {
|
|
86
|
+
if (typeof entry === "string") {
|
|
87
|
+
add(entry, {});
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
const { repo, ...perRepo } = entry;
|
|
91
|
+
add(repo, perRepo);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
for (const repo of cliRepos) {
|
|
95
|
+
add(repo, {});
|
|
96
|
+
}
|
|
97
|
+
if (targets.length === 0) {
|
|
98
|
+
throw new Error("No repositories to collect: pass --repo or list repos in --config.");
|
|
99
|
+
}
|
|
100
|
+
return targets;
|
|
101
|
+
}
|
|
102
|
+
function isObject(value) {
|
|
103
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
104
|
+
}
|
|
105
|
+
function optionalHoursBack(value, where) {
|
|
106
|
+
if (value === undefined) {
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
110
|
+
throw new Error(`Invalid ${where}: expected a positive number.`);
|
|
111
|
+
}
|
|
112
|
+
return value;
|
|
113
|
+
}
|
|
114
|
+
function optionalConcurrency(value, where) {
|
|
115
|
+
if (value === undefined) {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
|
|
119
|
+
throw new Error(`Invalid ${where}: expected a positive integer.`);
|
|
120
|
+
}
|
|
121
|
+
return value;
|
|
122
|
+
}
|
|
123
|
+
export function optionalString(value, where) {
|
|
124
|
+
if (value === undefined) {
|
|
125
|
+
return undefined;
|
|
126
|
+
}
|
|
127
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
128
|
+
throw new Error(`Invalid ${where}: expected a non-empty string.`);
|
|
129
|
+
}
|
|
130
|
+
return value.trim();
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Validates a matrix pattern at config-parse time by compiling it, so a bad regex fails
|
|
134
|
+
* immediately with the offending key named — rather than at the first artifact, mid-collect,
|
|
135
|
+
* after the network work is already done.
|
|
136
|
+
*/
|
|
137
|
+
export function optionalPattern(value, where) {
|
|
138
|
+
if (optionalString(value, where) === undefined) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
const pattern = value;
|
|
142
|
+
compileMatrixPattern(pattern, where);
|
|
143
|
+
return pattern;
|
|
144
|
+
}
|
|
145
|
+
function optionalStringArray(value, where) {
|
|
146
|
+
if (value === undefined) {
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
if (!Array.isArray(value) ||
|
|
150
|
+
value.length === 0 ||
|
|
151
|
+
value.some((item) => typeof item !== "string" || item.trim().length === 0)) {
|
|
152
|
+
throw new Error(`Invalid ${where}: expected an array of non-empty strings.`);
|
|
153
|
+
}
|
|
154
|
+
return value.map((item) => item.trim());
|
|
155
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
function escapeRegExp(text) {
|
|
2
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3
|
+
}
|
|
4
|
+
/** Case-insensitive glob match where `*` matches any span; every other character is literal. */
|
|
5
|
+
export function matchesGlob(pattern, value) {
|
|
6
|
+
const regex = new RegExp(`^${pattern.split("*").map(escapeRegExp).join(".*")}$`, "i");
|
|
7
|
+
return regex.test(value);
|
|
8
|
+
}
|
|
9
|
+
/** Partitions listed runs by the filter. An empty filter includes everything. */
|
|
10
|
+
export function filterRuns(runs, filter) {
|
|
11
|
+
const excludeTriggers = new Set(filter.excludeTriggers ?? []);
|
|
12
|
+
const included = [];
|
|
13
|
+
const excluded = [];
|
|
14
|
+
for (const run of runs) {
|
|
15
|
+
const workflowOk = filter.workflow === undefined || matchesGlob(filter.workflow, run.workflow);
|
|
16
|
+
const triggerOk = !excludeTriggers.has(run.trigger);
|
|
17
|
+
(workflowOk && triggerOk ? included : excluded).push(run);
|
|
18
|
+
}
|
|
19
|
+
return { included, excluded };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Caps how many completed runs one invocation processes (`--max-runs`, a safety valve for huge
|
|
23
|
+
* first backfills). The *oldest* runs go first: the watermark then advances only past what was
|
|
24
|
+
* actually processed, so the deferred newer runs are simply re-listed — and collected — next
|
|
25
|
+
* time. Zero data loss, bounded work per invocation.
|
|
26
|
+
*/
|
|
27
|
+
export function capRuns(completed, maxRuns) {
|
|
28
|
+
if (maxRuns === undefined || completed.length <= maxRuns) {
|
|
29
|
+
return { processed: completed, deferred: 0 };
|
|
30
|
+
}
|
|
31
|
+
const oldestFirst = [...completed].sort((a, b) => (a.created_at < b.created_at ? -1 : 1));
|
|
32
|
+
return { processed: oldestFirst.slice(0, maxRuns), deferred: completed.length - maxRuns };
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The runs the watermark may advance past: the processed completed runs plus the policy-excluded
|
|
36
|
+
* runs. When `--max-runs` deferred newer completed runs, only excluded runs at or below the
|
|
37
|
+
* processed ceiling count — the watermark must never pass a deferred (still uncollected) run.
|
|
38
|
+
*/
|
|
39
|
+
export function watermarkDisposal(processed, excluded, deferred) {
|
|
40
|
+
if (deferred === 0) {
|
|
41
|
+
return [...processed, ...excluded];
|
|
42
|
+
}
|
|
43
|
+
const ceiling = processed.reduce((max, run) => (run.created_at > max ? run.created_at : max), "");
|
|
44
|
+
return [...processed, ...excluded.filter((run) => run.created_at <= ceiling)];
|
|
45
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deriving a matrix cell from the artifact an extracted report came from.
|
|
3
|
+
*
|
|
4
|
+
* Playwright blob reports carry their own matrix (the project name), so the adapter sets it
|
|
5
|
+
* per test and this never applies. JUnit and mochawesome carry none — but a CI matrix job
|
|
6
|
+
* uploads its results under its own artifact name (`cypress-results-ui-firefox-desktop`), so
|
|
7
|
+
* the name is the cell.
|
|
8
|
+
*
|
|
9
|
+
* This matters beyond labelling. The dedup key is `(run_id, run_attempt, id, matrix)`, so an
|
|
10
|
+
* empty matrix makes every cell's result for one test share a key: the first written wins and
|
|
11
|
+
* the rest are dropped as duplicates. A test failing only on Firefox is then undetectable —
|
|
12
|
+
* the Chrome pass and the Firefox failure collapse into one arbitrary survivor.
|
|
13
|
+
*
|
|
14
|
+
* The matrix is never part of test identity (D5), so populating it re-keys dedup but never
|
|
15
|
+
* `id` — history stays attached to the same test.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Compiles a `matrixFromArtifact` pattern. The pattern must declare at least one **named**
|
|
19
|
+
* group — the group names become the matrix dimensions, so an unnamed pattern could not
|
|
20
|
+
* produce a cell and is far more likely a mistake than an intent.
|
|
21
|
+
*
|
|
22
|
+
* @throws if the pattern is not valid regex, or declares no named groups.
|
|
23
|
+
*/
|
|
24
|
+
export function compileMatrixPattern(pattern, where = "matrixFromArtifact") {
|
|
25
|
+
let regex;
|
|
26
|
+
let groups;
|
|
27
|
+
try {
|
|
28
|
+
regex = new RegExp(pattern);
|
|
29
|
+
groups = new RegExp(`${pattern}|`).exec("")?.groups;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
throw new Error(`Invalid ${where}: not a valid regular expression (${error.message}).`);
|
|
33
|
+
}
|
|
34
|
+
if (!groups || Object.keys(groups).length === 0) {
|
|
35
|
+
throw new Error(`Invalid ${where}: the pattern must declare at least one named group, ` +
|
|
36
|
+
`e.g. "results-(?<browser>[^-]+)" — group names become the matrix dimensions.`);
|
|
37
|
+
}
|
|
38
|
+
return regex;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Resolves the matrix cell for a report file, given the artifact it was extracted from.
|
|
42
|
+
*
|
|
43
|
+
* With a pattern: the named groups that participated in the match become the dimensions. A
|
|
44
|
+
* non-matching artifact falls back to the opaque form rather than being dropped — a report is
|
|
45
|
+
* evidence, and silently discarding it because a pattern was mistyped is the failure mode this
|
|
46
|
+
* whole module exists to prevent.
|
|
47
|
+
*
|
|
48
|
+
* Without a pattern: `{ artifact: <name> }`. Verbose, but it is a fact rather than an
|
|
49
|
+
* inference, and it keeps distinct cells distinct under the dedup key.
|
|
50
|
+
*/
|
|
51
|
+
export function matrixForArtifact(artifact, pattern) {
|
|
52
|
+
if (pattern) {
|
|
53
|
+
const groups = pattern.exec(artifact)?.groups;
|
|
54
|
+
if (groups) {
|
|
55
|
+
const cell = {};
|
|
56
|
+
for (const [key, value] of Object.entries(groups)) {
|
|
57
|
+
// An optional group that did not participate is absent, not empty — omit it so two
|
|
58
|
+
// artifacts differing only in an unmatched optional group share one cell.
|
|
59
|
+
if (value !== undefined) {
|
|
60
|
+
cell[key] = value;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (Object.keys(cell).length > 0) {
|
|
64
|
+
return cell;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { artifact };
|
|
69
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { extname } from "node:path";
|
|
3
|
+
import { detectAndParse } from "../adapters/dispatch.js";
|
|
4
|
+
import { terminalSafe } from "../colors.js";
|
|
5
|
+
import { buildEvent } from "../events/event.js";
|
|
6
|
+
import { matrixForArtifact } from "./matrix.js";
|
|
7
|
+
/** Extensions worth reading as report candidates: JUnit XML, Playwright JSONL, mochawesome JSON. */
|
|
8
|
+
const CANDIDATE_EXTENSIONS = new Set([".xml", ".jsonl", ".json"]);
|
|
9
|
+
function emptySummary() {
|
|
10
|
+
return {
|
|
11
|
+
filesScanned: 0,
|
|
12
|
+
filesParsed: 0,
|
|
13
|
+
filesFailed: 0,
|
|
14
|
+
eventsBuilt: 0,
|
|
15
|
+
passed: 0,
|
|
16
|
+
failed: 0,
|
|
17
|
+
skipped: 0,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Reads each run's extracted report files, dispatches them to the matching adapter,
|
|
22
|
+
* and enriches the parsed tests into events. Non-report files (HTML, assets) are
|
|
23
|
+
* skipped by extension, then by the adapter sniff.
|
|
24
|
+
*/
|
|
25
|
+
export async function parseRunReports(reports, repoSlug, options = {}) {
|
|
26
|
+
const collectedAt = (options.now ?? new Date()).toISOString();
|
|
27
|
+
const events = [];
|
|
28
|
+
const summary = emptySummary();
|
|
29
|
+
for (const report of reports) {
|
|
30
|
+
for (const { path: file, artifact, attempt } of report.files) {
|
|
31
|
+
if (!CANDIDATE_EXTENSIONS.has(extname(file).toLowerCase())) {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
summary.filesScanned++;
|
|
35
|
+
let detected;
|
|
36
|
+
try {
|
|
37
|
+
const content = await readFile(file, "utf8");
|
|
38
|
+
detected = detectAndParse(content, { fullErrors: options.fullErrors });
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
// A truncated upload or a hostile document (invalid JSON past the sniff, pathological
|
|
42
|
+
// XML, absurd nesting) is deterministic content — retrying yields the same bytes — so
|
|
43
|
+
// it costs this one file, with a warning, never the whole collect (F1).
|
|
44
|
+
summary.filesFailed++;
|
|
45
|
+
console.warn(terminalSafe(`Skipping unparseable report file ${file}: ${error.message}`));
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (!detected) {
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
summary.filesParsed++;
|
|
52
|
+
for (const test of detected.tests) {
|
|
53
|
+
events.push(buildEvent(test, {
|
|
54
|
+
framework: detected.framework,
|
|
55
|
+
run: runContext(repoSlug, report.run, attempt),
|
|
56
|
+
matrix: matrixForArtifact(artifact, options.matrixPattern),
|
|
57
|
+
collected_at: collectedAt,
|
|
58
|
+
}));
|
|
59
|
+
summary.eventsBuilt++;
|
|
60
|
+
countStatus(summary, test.status);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return { events, summary };
|
|
65
|
+
}
|
|
66
|
+
function countStatus(summary, status) {
|
|
67
|
+
if (status === "passed") {
|
|
68
|
+
summary.passed++;
|
|
69
|
+
}
|
|
70
|
+
else if (status === "failed") {
|
|
71
|
+
summary.failed++;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
summary.skipped++;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function runContext(repo, run, attempt) {
|
|
78
|
+
return {
|
|
79
|
+
provider: "github",
|
|
80
|
+
repo,
|
|
81
|
+
run_id: run.run_id,
|
|
82
|
+
run_attempt: attempt,
|
|
83
|
+
workflow: run.workflow,
|
|
84
|
+
branch: run.branch,
|
|
85
|
+
sha: run.sha,
|
|
86
|
+
trigger: run.trigger,
|
|
87
|
+
started_at: run.created_at,
|
|
88
|
+
};
|
|
89
|
+
}
|