qa-engineer 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +50 -26
- package/README.md +40 -32
- package/package.json +12 -20
- package/packages/engine/bin/qa-engine.mjs +336 -0
- package/packages/engine/lib/analysis/branding.mjs +187 -0
- package/packages/engine/lib/analysis/context.mjs +294 -0
- package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
- package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
- package/packages/engine/lib/analysis/discovery.mjs +220 -0
- package/packages/engine/lib/analysis/evidence.mjs +116 -0
- package/packages/engine/lib/analysis/har.mjs +90 -0
- package/packages/engine/lib/analysis/junit.mjs +126 -0
- package/packages/engine/lib/analysis/redaction.mjs +127 -0
- package/packages/engine/lib/analysis/report-html.mjs +763 -0
- package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
- package/packages/engine/lib/analysis/xml.mjs +153 -0
- package/packages/engine/lib/analysis/zip.mjs +107 -0
- package/packages/engine/lib/diagnostics/engine.mjs +165 -0
- package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
- package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
- package/packages/engine/lib/diagnostics/repair.mjs +73 -0
- package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
- package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
- package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
- package/packages/engine/lib/frameworks/playwright.mjs +158 -0
- package/packages/engine/package.json +14 -0
- package/packages/installer/lib/agents/registry.mjs +46 -0
- package/packages/installer/lib/commands/doctor.mjs +7 -16
- package/packages/installer/lib/commands/install.mjs +59 -4
- package/packages/installer/lib/core/bundle.mjs +69 -92
- package/packages/installer/lib/core/config.mjs +1 -1
- package/packages/installer/lib/core/lockfile.mjs +1 -1
- package/packages/installer/lib/core/manifest.mjs +48 -54
- package/packages/installer/lib/core/validate-install.mjs +12 -19
- package/packages/installer/package.json +1 -1
- package/shared/tooling/qa-tool.mjs +125 -0
- package/skills/qa-api/SKILL.md +5 -5
- package/skills/qa-api/references/deterministic-tooling.md +34 -30
- package/skills/qa-api/references/evidence-and-reporting.md +29 -4
- package/skills/qa-api/scripts/qa-tool.mjs +126 -0
- package/skills/qa-audit/SKILL.md +5 -5
- package/skills/qa-audit/references/deterministic-tooling.md +34 -30
- package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
- package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
- package/skills/qa-debug/SKILL.md +6 -6
- package/skills/qa-debug/references/deterministic-tooling.md +34 -30
- package/skills/qa-debug/references/diagnostic-engine.md +3 -3
- package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
- package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
- package/skills/qa-explore/SKILL.md +22 -5
- package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
- package/skills/qa-explore/references/deterministic-tooling.md +132 -0
- package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
- package/skills/qa-explore/references/report-pipeline.md +89 -8
- package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
- package/skills/qa-fix/SKILL.md +4 -4
- package/skills/qa-fix/references/deterministic-tooling.md +34 -30
- package/skills/qa-fix/references/diagnostic-engine.md +3 -3
- package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
- package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
- package/skills/qa-flaky/SKILL.md +4 -4
- package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
- package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
- package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
- package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/SKILL.md +4 -4
- package/skills/qa-init/references/deterministic-tooling.md +34 -30
- package/skills/qa-init/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/scripts/qa-tool.mjs +126 -0
- package/skills/qa-report/SKILL.md +8 -7
- package/skills/qa-report/references/deterministic-tooling.md +34 -30
- package/skills/qa-report/references/diagnostic-engine.md +3 -3
- package/skills/qa-report/references/evidence-and-reporting.md +29 -4
- package/skills/qa-report/scripts/qa-tool.mjs +126 -0
- package/skills/qa-review/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/SKILL.md +6 -6
- package/skills/qa-run/references/deterministic-tooling.md +34 -30
- package/skills/qa-run/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/scripts/qa-tool.mjs +126 -0
- package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
- package/shared/analysis/lib/qa_analysis/branding.py +0 -175
- package/shared/analysis/lib/qa_analysis/cli.py +0 -129
- package/shared/analysis/lib/qa_analysis/context.py +0 -233
- package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
- package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
- package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
- package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
- package/shared/analysis/lib/qa_analysis/har.py +0 -58
- package/shared/analysis/lib/qa_analysis/junit.py +0 -80
- package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
- package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
- package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
- package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
- package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
- package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
- package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
- package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
- package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
- package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
- package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
- package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
- package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
- package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
- package/shared/tooling/qa_tool.py +0 -127
- /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
- /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
// The diff guard: a deterministic protection layer against unsafe "fixes".
|
|
2
|
+
//
|
|
3
|
+
// Given a unified diff of test code, it flags changes that make a suite pass
|
|
4
|
+
// without proving the software works. Each flag explains why it is unsafe. It
|
|
5
|
+
// never edits; it only judges.
|
|
6
|
+
//
|
|
7
|
+
// The guard has two jobs, and they pull against each other:
|
|
8
|
+
//
|
|
9
|
+
// 1. **Catch every way a suite is made to lie.** Deleting or weakening
|
|
10
|
+
// assertions, skipping, forcing a pass, excluding specs from the run, making
|
|
11
|
+
// the test command always exit zero, swallowing failures, inflating timeouts,
|
|
12
|
+
// deleting test files.
|
|
13
|
+
// 2. **Not cry wolf on real repairs.** Healing a stale locator *is* the job of
|
|
14
|
+
// `/qa-fix`, and it necessarily rewrites an assertion line. A guard that flags
|
|
15
|
+
// every legitimate repair as `high` trains the agent — and the human — to
|
|
16
|
+
// override it, which is worse than having no guard.
|
|
17
|
+
//
|
|
18
|
+
// Job 2 is why assertions are compared by **strength** rather than by presence. An
|
|
19
|
+
// assertion replaced by one that is at least as strong, keeping the same expected
|
|
20
|
+
// values, is a modification worth confirming (`low`) — not a removal (`high`). An
|
|
21
|
+
// assertion replaced by a weaker one, or one that drops the expected value, is
|
|
22
|
+
// `weakened-assertion` (`high`), because that is how a test quietly stops checking
|
|
23
|
+
// anything.
|
|
24
|
+
|
|
25
|
+
// --- assertion recognition ---------------------------------------------------
|
|
26
|
+
|
|
27
|
+
const ASSERTION = /\b(expect|assert|should|toBe|toEqual|toHaveText|toBeVisible|toContain|assertThat)\b/i;
|
|
28
|
+
|
|
29
|
+
// Matchers that pin a specific value or state. Replacing one of these with a
|
|
30
|
+
// weaker matcher means the test no longer checks what it used to.
|
|
31
|
+
const STRONG_MATCHERS = new RegExp(
|
|
32
|
+
'\\b(toBe|toEqual|toStrictEqual|toHaveText|toContainText|toHaveValue|toHaveURL|' +
|
|
33
|
+
'toHaveTitle|toHaveCount|toHaveLength|toHaveAttribute|toHaveClass|toHaveScreenshot|' +
|
|
34
|
+
'toMatch|toMatchObject|toMatchSnapshot|toBeVisible|toBeChecked|toBeEnabled|toBeDisabled|' +
|
|
35
|
+
'toBeEmpty|toBeGreaterThan|toBeLessThan|toBeCloseTo|' +
|
|
36
|
+
'assertEqual|assertEquals|assertIn|assertTrue|assertFalse|isEqualTo|containsExactly)\\b',
|
|
37
|
+
'i',
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
// Matchers that only prove something exists or is loosely truthy.
|
|
41
|
+
const WEAK_MATCHERS = new RegExp(
|
|
42
|
+
'\\b(toBeDefined|toBeUndefined|toBeTruthy|toBeFalsy|toBeNull|toBeNaN|toBeAttached|' +
|
|
43
|
+
'toBeOk|assertIsNotNone|assertIsNone|isNotNull|isNotEmpty|notNull|toBeInstanceOf)\\b',
|
|
44
|
+
'i',
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
// A soft assertion does not stop the test at the point of failure; swapping a hard
|
|
48
|
+
// assertion for a soft one weakens it.
|
|
49
|
+
const SOFT_ASSERTION = /\b(expect\.soft|softAssert|assertSoftly)\b/i;
|
|
50
|
+
|
|
51
|
+
// String and numeric literals — the expected values an assertion pins down.
|
|
52
|
+
const STRING_LITERAL = /(['"])((?:(?!\1).){2,})\1/g;
|
|
53
|
+
const NUMBER_LITERAL = /(?<![\w.])\d+(?:\.\d+)?(?![\w.])/g;
|
|
54
|
+
|
|
55
|
+
const IDENTIFIER = /[A-Za-z_][A-Za-z0-9_]*/g;
|
|
56
|
+
|
|
57
|
+
// --- unsafe-change patterns --------------------------------------------------
|
|
58
|
+
|
|
59
|
+
const SKIP_ADDED = new RegExp(
|
|
60
|
+
'(\\.skip\\b|\\.fixme\\b|\\bxit\\b|\\bxdescribe\\b|test\\.skip|it\\.skip|describe\\.skip|' +
|
|
61
|
+
'@pytest\\.mark\\.skip|@Ignore\\b|@Disabled\\b|\\.only\\b|this\\.skip\\(|t\\.Skip\\()',
|
|
62
|
+
'i',
|
|
63
|
+
);
|
|
64
|
+
const FORCED_PASS = new RegExp(
|
|
65
|
+
'(assert\\s+True\\b|expect\\(true\\)\\.toBe\\(true\\)|expect\\(1\\)\\.toBe\\(1\\)|' +
|
|
66
|
+
'return\\s*;?\\s*//\\s*pass|assert\\s+1\\s*==\\s*1|pass\\s*#\\s*(?:todo|skip|pass))',
|
|
67
|
+
'i',
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
// A bare early return inside a test body ends the test before it verifies anything
|
|
71
|
+
// — the quietest way to fake a pass. Only value-free returns count, so
|
|
72
|
+
// `return page.click(...)` and `return await expect(...)` stay clean.
|
|
73
|
+
const EARLY_RETURN = /^\s*(?:if\s*\(.*?\)\s*\{?\s*)?return\s*;?\s*\}?\s*$/;
|
|
74
|
+
|
|
75
|
+
// Excluding specs from the run drops coverage without touching a test file.
|
|
76
|
+
const SUITE_EXCLUSION = new RegExp(
|
|
77
|
+
'(testIgnore|testPathIgnorePatterns|excludeSpecPattern|' +
|
|
78
|
+
'exclude\\s*[:=]|ignorePatterns|--grep-invert|--ignore-pattern|' +
|
|
79
|
+
'specs?\\s*:\\s*\\[\\s*\\]|testMatch\\s*[:=]\\s*\\[\\s*\\])',
|
|
80
|
+
'i',
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
// Making the test command exit zero regardless of the result.
|
|
84
|
+
const FORCED_PASS_COMMAND = new RegExp(
|
|
85
|
+
'(\\|\\|\\s*true\\b|\\|\\|\\s*exit\\s+0\\b|;\\s*exit\\s+0\\b|--passWithNoTests\\b|' +
|
|
86
|
+
'--pass-with-no-tests\\b|continue-on-error\\s*:\\s*true|set\\s+\\+e\\b|' +
|
|
87
|
+
'-DskipTests\\b|--exit-zero\\b|\\|\\|\\s*:)',
|
|
88
|
+
'i',
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
// A catch block that discards the failure.
|
|
92
|
+
const SWALLOWED_FAILURE = /(catch\s*(?:\([^)]*\))?\s*\{\s*\}|except\s*[\w.]*\s*:\s*pass\b)/i;
|
|
93
|
+
const CATCH_ADDED = /(\bcatch\s*[({]|^\s*except\b)/i;
|
|
94
|
+
|
|
95
|
+
const TIMEOUT = /(timeout|setTimeout|wait_?for|implicitly_wait|setDefaultTimeout)\D{0,20}?(\d{3,})/i;
|
|
96
|
+
// Deliberately excludes `expect(`: an expectation is an assertion, not a wait, and
|
|
97
|
+
// including it made this rule unreachable.
|
|
98
|
+
const WAIT = /(waitFor\w*|wait_?for\w*|\.wait\(|awaitVisible|implicitly_wait)/i;
|
|
99
|
+
const LOCATOR = /(getBy\w+|locator|find_element|\$\(|css=|xpath=|By\.\w+)/i;
|
|
100
|
+
const EMPTY_BODY = /(async\s+)?(\([^)]*\)\s*=>\s*\{\s*\}|function[^{]*\{\s*\}|def\s+test\w*\([^)]*\):\s*pass)/i;
|
|
101
|
+
const RETRY = /(retries?\s*[:=]\s*(\d+)|\.retry\((\d+)\))/i;
|
|
102
|
+
|
|
103
|
+
const TEST_FILE = /(\.spec\.|\.test\.|_test\.|test_|\/tests?\/|\/e2e\/|\.feature$|Test\.java$|Tests\.cs$)/i;
|
|
104
|
+
|
|
105
|
+
export const MASS_DELETION_THRESHOLD = 15;
|
|
106
|
+
|
|
107
|
+
// How much token overlap makes an added assertion the counterpart of a removed
|
|
108
|
+
// one. Tuned so a locator rewrite on the same expectation pairs up, while two
|
|
109
|
+
// unrelated assertions in the same file do not.
|
|
110
|
+
const COUNTERPART_THRESHOLD = 0.34;
|
|
111
|
+
|
|
112
|
+
/** Yield `[file, sign, text]` for each changed line. Sign is '+' or '-'. */
|
|
113
|
+
function parseDiff(diffText) {
|
|
114
|
+
const changes = [];
|
|
115
|
+
let current = 'unknown';
|
|
116
|
+
for (const line of String(diffText).split('\n')) {
|
|
117
|
+
if (line.startsWith('+++ ') || line.startsWith('--- ')) {
|
|
118
|
+
const candidate = line.slice(4).trim();
|
|
119
|
+
if (candidate !== '/dev/null' && candidate !== '') {
|
|
120
|
+
current = candidate.replace(/^[ab]\//, '');
|
|
121
|
+
}
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (line.startsWith('@@')) continue;
|
|
125
|
+
if (line.startsWith('+') && !line.startsWith('+++')) changes.push([current, '+', line.slice(1)]);
|
|
126
|
+
else if (line.startsWith('-') && !line.startsWith('---')) changes.push([current, '-', line.slice(1)]);
|
|
127
|
+
}
|
|
128
|
+
return changes;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Files whose new side is /dev/null — deleted outright. */
|
|
132
|
+
function deletedFiles(diffText) {
|
|
133
|
+
const deleted = new Set();
|
|
134
|
+
let pending = null;
|
|
135
|
+
for (const line of String(diffText).split('\n')) {
|
|
136
|
+
if (line.startsWith('--- ')) {
|
|
137
|
+
const candidate = line.slice(4).trim();
|
|
138
|
+
pending = candidate !== '/dev/null' ? candidate.replace(/^[ab]\//, '') : null;
|
|
139
|
+
} else if (line.startsWith('+++ ')) {
|
|
140
|
+
if (line.slice(4).trim() === '/dev/null' && pending) deleted.add(pending);
|
|
141
|
+
pending = null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return deleted;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function tokens(text) {
|
|
148
|
+
return new Set([...String(text).matchAll(IDENTIFIER)].map((m) => m[0].toLowerCase()));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The part of an assertion that carries its expected value.
|
|
153
|
+
*
|
|
154
|
+
* `expect(page.locator('#total')).toHaveText('42')` has two literals, and only one
|
|
155
|
+
* of them is an expectation: `'#total'` is the *subject* — which element to look at
|
|
156
|
+
* — and `'42'` is what the test claims about it. Scanning the whole line treated the
|
|
157
|
+
* selector as an expected value, so the canonical `/qa-fix` repair
|
|
158
|
+
*
|
|
159
|
+
* -expect(page.locator('#total')).toHaveText('42')
|
|
160
|
+
* +expect(page.getByTestId('total')).toHaveText('42')
|
|
161
|
+
*
|
|
162
|
+
* was reported as `weakened-assertion` at `high` severity for "dropping" the
|
|
163
|
+
* selector, even though the assertion is unchanged. That is a false positive on the
|
|
164
|
+
* exact case this module's second job exists to protect.
|
|
165
|
+
*/
|
|
166
|
+
function expectedPart(text) {
|
|
167
|
+
let earliest = null;
|
|
168
|
+
for (const pattern of [STRONG_MATCHERS, WEAK_MATCHERS]) {
|
|
169
|
+
const match = pattern.exec(text);
|
|
170
|
+
if (match && (earliest === null || match.index < earliest)) earliest = match.index;
|
|
171
|
+
}
|
|
172
|
+
return earliest === null ? text : text.slice(earliest);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function literals(text) {
|
|
176
|
+
const scope = expectedPart(String(text));
|
|
177
|
+
const strings = new Set([...scope.matchAll(STRING_LITERAL)].map((m) => m[2]));
|
|
178
|
+
const numbers = new Set([...scope.matchAll(NUMBER_LITERAL)].map((m) => m[0]));
|
|
179
|
+
return { strings, numbers };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** 3 = pins a value or state, 2 = unclassified assertion, 1 = existence only. */
|
|
183
|
+
function strength(text) {
|
|
184
|
+
if (SOFT_ASSERTION.test(text)) return 1;
|
|
185
|
+
if (WEAK_MATCHERS.test(text) && !STRONG_MATCHERS.test(text)) return 1;
|
|
186
|
+
if (STRONG_MATCHERS.test(text)) return 3;
|
|
187
|
+
return 2;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** The added assertion line most plausibly replacing this removed one. */
|
|
191
|
+
function counterpart(removedText, candidates) {
|
|
192
|
+
const removedTokens = tokens(removedText);
|
|
193
|
+
if (removedTokens.size === 0) return null;
|
|
194
|
+
let best = null;
|
|
195
|
+
let bestScore = 0;
|
|
196
|
+
for (const text of candidates) {
|
|
197
|
+
const candidateTokens = tokens(text);
|
|
198
|
+
let overlap = 0;
|
|
199
|
+
for (const token of removedTokens) if (candidateTokens.has(token)) overlap += 1;
|
|
200
|
+
const score = overlap / Math.max(removedTokens.size, 1);
|
|
201
|
+
if (score > bestScore) {
|
|
202
|
+
best = text;
|
|
203
|
+
bestScore = score;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return bestScore >= COUNTERPART_THRESHOLD ? best : null;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Analyze a unified diff.
|
|
211
|
+
*
|
|
212
|
+
* Returns `[{rule, severity, file, why, sample}]`. An empty list means no unsafe
|
|
213
|
+
* change was detected — which is not a guarantee of correctness, only of safety.
|
|
214
|
+
*/
|
|
215
|
+
export function checkDiff(diffText) {
|
|
216
|
+
const issues = [];
|
|
217
|
+
const changes = parseDiff(diffText);
|
|
218
|
+
const removed = changes.filter(([, sign]) => sign === '-').map(([file, , text]) => [file, text]);
|
|
219
|
+
const added = changes.filter(([, sign]) => sign === '+').map(([file, , text]) => [file, text]);
|
|
220
|
+
const deleted = deletedFiles(diffText);
|
|
221
|
+
|
|
222
|
+
const flag = (rule, severity, file, why, sample) => {
|
|
223
|
+
issues.push({ rule, severity, file, why, sample: String(sample).trim().slice(0, 200) });
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
const addedByFile = new Map();
|
|
227
|
+
for (const [file, text] of added) {
|
|
228
|
+
if (!addedByFile.has(file)) addedByFile.set(file, []);
|
|
229
|
+
addedByFile.get(file).push(text);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// --- removed or weakened assertions ---------------------------------------
|
|
233
|
+
for (const [file, text] of removed) {
|
|
234
|
+
if (ASSERTION.test(text)) {
|
|
235
|
+
const sameFileAdded = addedByFile.get(file) ?? [];
|
|
236
|
+
// An identical line re-added is a move, not a change.
|
|
237
|
+
if (sameFileAdded.some((line) => line.trim() === text.trim())) continue;
|
|
238
|
+
|
|
239
|
+
const candidates = sameFileAdded.filter((line) => ASSERTION.test(line));
|
|
240
|
+
const replacement = counterpart(text, candidates);
|
|
241
|
+
|
|
242
|
+
if (replacement === null) {
|
|
243
|
+
if (deleted.has(file)) continue; // reported once as a deleted test file, below
|
|
244
|
+
flag('removed-assertion', 'high', file,
|
|
245
|
+
'An assertion or expectation was removed with nothing replacing it; ' +
|
|
246
|
+
'a test that no longer asserts proves nothing.', text);
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const oldStrength = strength(text);
|
|
251
|
+
const newStrength = strength(replacement);
|
|
252
|
+
const before = literals(text);
|
|
253
|
+
const after = literals(replacement);
|
|
254
|
+
const droppedStrings = [...before.strings].filter((value) => !after.strings.has(value));
|
|
255
|
+
const droppedNumbers = [...before.numbers].filter((value) => !after.numbers.has(value));
|
|
256
|
+
|
|
257
|
+
if (newStrength < oldStrength) {
|
|
258
|
+
flag('weakened-assertion', 'high', file,
|
|
259
|
+
'An assertion was replaced by a weaker one (existence or truthiness ' +
|
|
260
|
+
'instead of a specific value or state); the test can now pass without ' +
|
|
261
|
+
'the behaviour being correct.', replacement);
|
|
262
|
+
} else if (droppedStrings.length > 0 || droppedNumbers.length > 0) {
|
|
263
|
+
// A set union, as the Python implementation used: the literal "42" is
|
|
264
|
+
// matched both as a string and as a number, and reporting it twice would
|
|
265
|
+
// read as two dropped values when only one was dropped.
|
|
266
|
+
const dropped = [...new Set([...droppedStrings, ...droppedNumbers])].sort();
|
|
267
|
+
flag('weakened-assertion', 'high', file,
|
|
268
|
+
'An assertion kept its matcher but dropped the expected value(s) ' +
|
|
269
|
+
`${formatList(dropped)}; it no longer pins down what it used to.`, replacement);
|
|
270
|
+
} else {
|
|
271
|
+
flag('assertion-modified', 'low', file,
|
|
272
|
+
'An assertion was rewritten with equal or greater strength and the same ' +
|
|
273
|
+
'expected values — consistent with a legitimate repair. Confirm it still ' +
|
|
274
|
+
'targets the same behaviour.', replacement);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (WAIT.test(text) && !ASSERTION.test(text)) {
|
|
279
|
+
flag('removed-wait', 'medium', file,
|
|
280
|
+
'A wait/synchronization was removed, which can mask a real failure or ' +
|
|
281
|
+
'introduce flakiness.', text);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// --- added skips, forced passes, early returns, empty bodies --------------
|
|
286
|
+
for (const [file, text] of added) {
|
|
287
|
+
if (SKIP_ADDED.test(text)) {
|
|
288
|
+
flag('added-skip-or-only', 'high', file,
|
|
289
|
+
'A skip/ignore/only marker was added; skipping or narrowing hides failures ' +
|
|
290
|
+
'instead of fixing them.', text);
|
|
291
|
+
}
|
|
292
|
+
if (FORCED_PASS.test(text)) {
|
|
293
|
+
flag('forced-pass', 'high', file,
|
|
294
|
+
'A tautological or forced-pass assertion was added; it makes the test pass ' +
|
|
295
|
+
'without checking behavior.', text);
|
|
296
|
+
}
|
|
297
|
+
if (EMPTY_BODY.test(text)) {
|
|
298
|
+
flag('empty-test-body', 'high', file,
|
|
299
|
+
'A test body was emptied; an empty test passes vacuously.', text);
|
|
300
|
+
}
|
|
301
|
+
if (EARLY_RETURN.test(text) && TEST_FILE.test(file)) {
|
|
302
|
+
flag('conditional-skip', 'high', file,
|
|
303
|
+
'An unconditional or condition-guarded early return was added to a test; ' +
|
|
304
|
+
'the test exits before verifying anything, which is a skip in disguise.', text);
|
|
305
|
+
}
|
|
306
|
+
if (SUITE_EXCLUSION.test(text)) {
|
|
307
|
+
flag('suite-exclusion', 'high', file,
|
|
308
|
+
'Specs were excluded from the run (ignore pattern, exclusion list, or empty ' +
|
|
309
|
+
'spec list); coverage is dropped without any test file changing.', text);
|
|
310
|
+
}
|
|
311
|
+
if (FORCED_PASS_COMMAND.test(text)) {
|
|
312
|
+
flag('forced-pass-command', 'high', file,
|
|
313
|
+
'The test command was made to succeed regardless of the result ' +
|
|
314
|
+
'(`|| true`, `exit 0`, `--passWithNoTests`, `continue-on-error`); the suite ' +
|
|
315
|
+
'can no longer fail a pipeline.', text);
|
|
316
|
+
}
|
|
317
|
+
if (SWALLOWED_FAILURE.test(text)) {
|
|
318
|
+
flag('swallowed-failure', 'high', file,
|
|
319
|
+
'A failure is caught and discarded; an assertion inside a swallowed catch ' +
|
|
320
|
+
'cannot fail the test.', text);
|
|
321
|
+
} else if (CATCH_ADDED.test(text) && TEST_FILE.test(file)) {
|
|
322
|
+
flag('added-error-handling', 'medium', file,
|
|
323
|
+
'Error handling was added inside a test; confirm the failure still surfaces ' +
|
|
324
|
+
'rather than being caught and logged.', text);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// --- deleted test files ---------------------------------------------------
|
|
329
|
+
for (const file of [...deleted].sort()) {
|
|
330
|
+
if (TEST_FILE.test(file)) {
|
|
331
|
+
flag('test-file-deleted', 'high', file,
|
|
332
|
+
'A test file was deleted outright; every case it held stops running, ' +
|
|
333
|
+
"regardless of the file's size.", file);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// --- timeout inflation and unsafe retry increases -------------------------
|
|
338
|
+
flagNumericInflation(TIMEOUT, removed, added, 'timeout-inflation', 'medium',
|
|
339
|
+
'A timeout was increased substantially, which can paper over a real slowness or hang.', flag);
|
|
340
|
+
flagNumericInflation(RETRY, removed, added, 'unsafe-retry-increase', 'medium',
|
|
341
|
+
'Retry count was increased, which can convert a real failure into an intermittent pass.', flag);
|
|
342
|
+
|
|
343
|
+
// --- suspicious locator changes ------------------------------------------
|
|
344
|
+
// Last removed locator per file wins, matching the dict comprehension this
|
|
345
|
+
// replaces — the most recent removal is the one an addition is compared against.
|
|
346
|
+
const removedLocators = new Map();
|
|
347
|
+
for (const [file, text] of removed) {
|
|
348
|
+
if (LOCATOR.test(text)) removedLocators.set(file, text);
|
|
349
|
+
}
|
|
350
|
+
for (const [file, text] of added) {
|
|
351
|
+
if (
|
|
352
|
+
LOCATOR.test(text) &&
|
|
353
|
+
removedLocators.has(file) &&
|
|
354
|
+
text.trim() !== removedLocators.get(file).trim()
|
|
355
|
+
) {
|
|
356
|
+
flag('suspicious-locator-change', 'low', file,
|
|
357
|
+
'A locator was changed; confirm it targets the same element and was not ' +
|
|
358
|
+
'loosened to pass.', text);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// --- mass deletion per file ----------------------------------------------
|
|
363
|
+
const perFile = new Map();
|
|
364
|
+
for (const [file] of removed) perFile.set(file, (perFile.get(file) ?? 0) + 1);
|
|
365
|
+
for (const [file, count] of perFile) {
|
|
366
|
+
if (count >= MASS_DELETION_THRESHOLD && !deleted.has(file)) {
|
|
367
|
+
flag('mass-deletion', 'high', file,
|
|
368
|
+
`${count} lines were deleted from a test file; large deletions can silently drop coverage.`,
|
|
369
|
+
`${count} lines removed`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
return issues;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Python renders a list of dropped values with `repr()`, which single-quotes each
|
|
378
|
+
* one. The message text is part of what a reviewer reads, so it is reproduced
|
|
379
|
+
* rather than approximated.
|
|
380
|
+
*/
|
|
381
|
+
function formatList(values) {
|
|
382
|
+
return `[${values.map((value) => `'${value}'`).join(', ')}]`;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Compare the number each rule's own pattern captured, before and after.
|
|
387
|
+
*
|
|
388
|
+
* A shared `\d{2,}` scan of the line — two or more digits — made
|
|
389
|
+
* `unsafe-retry-increase` unreachable for every realistic retry count, because
|
|
390
|
+
* `retries: 1` to `retries: 5` is all single digits: a declared safety rule that
|
|
391
|
+
* could not fire below ten. On a line carrying more than one number
|
|
392
|
+
* (`timeout: 5000, port: 8080`) it also compared the wrong one.
|
|
393
|
+
*/
|
|
394
|
+
function flagNumericInflation(pattern, removed, added, rule, severity, why, flag) {
|
|
395
|
+
const global = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
|
|
396
|
+
const maxNumber = (text) => {
|
|
397
|
+
const values = [];
|
|
398
|
+
for (const match of String(text).matchAll(global)) {
|
|
399
|
+
for (const group of match.slice(1)) {
|
|
400
|
+
if (group !== undefined && /^\d+$/.test(group)) values.push(Number.parseInt(group, 10));
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return values.length > 0 ? Math.max(...values) : null;
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
const removedByFile = new Map();
|
|
407
|
+
for (const [file, text] of removed) {
|
|
408
|
+
if (!pattern.test(text)) continue;
|
|
409
|
+
const value = maxNumber(text);
|
|
410
|
+
if (value === null) continue;
|
|
411
|
+
if (!removedByFile.has(file)) removedByFile.set(file, []);
|
|
412
|
+
removedByFile.get(file).push(value);
|
|
413
|
+
}
|
|
414
|
+
for (const [file, text] of added) {
|
|
415
|
+
if (!pattern.test(text)) continue;
|
|
416
|
+
const newValue = maxNumber(text);
|
|
417
|
+
if (newValue === null) continue;
|
|
418
|
+
for (const oldValue of removedByFile.get(file) ?? []) {
|
|
419
|
+
if (newValue >= oldValue * 2 && newValue > oldValue) {
|
|
420
|
+
flag(rule, severity, file, why, text);
|
|
421
|
+
break;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// Deterministic artifact discovery.
|
|
2
|
+
//
|
|
3
|
+
// Locates the artifacts a run produced — automatically by convention or from
|
|
4
|
+
// explicit paths — and classifies each as present, partial, or corrupted. It
|
|
5
|
+
// handles multiple runs, parallel workers, and sharded output (many result files),
|
|
6
|
+
// and it reports what is missing rather than inventing it.
|
|
7
|
+
//
|
|
8
|
+
// Two things Python's standard library provided that are written out here:
|
|
9
|
+
//
|
|
10
|
+
// - `glob(..., recursive=True)`. Node's `fs.globSync` only arrived in Node 22 and
|
|
11
|
+
// the pack supports 18.17, so the walk is explicit. It reproduces glob's rules
|
|
12
|
+
// that matter here: `**` spans zero or more directories, and an entry whose name
|
|
13
|
+
// begins with a dot is not matched by a wildcard.
|
|
14
|
+
// - `zipfile.is_zipfile`, which locates the end-of-central-directory record
|
|
15
|
+
// rather than trusting the first four bytes. A trace that has been truncated
|
|
16
|
+
// still starts with `PK\x03\x04`, so magic bytes alone would call a broken
|
|
17
|
+
// archive intact — the exact misreport this classification exists to prevent.
|
|
18
|
+
|
|
19
|
+
import fs from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
|
|
22
|
+
import { artifact } from './evidence.mjs';
|
|
23
|
+
|
|
24
|
+
// Convention globs for known artifact types, relative to a run root. Order is
|
|
25
|
+
// significant: the first type whose pattern matches a path claims it.
|
|
26
|
+
const PATTERNS = [
|
|
27
|
+
['junit', ['**/results.xml', '**/junit*.xml', '**/*junit*.xml']],
|
|
28
|
+
['report', ['**/results.json', '**/report.json']],
|
|
29
|
+
['html-report', ['**/playwright-report/index.html', '**/*-report/index.html']],
|
|
30
|
+
['trace', ['**/trace.zip', '**/*-trace.zip']],
|
|
31
|
+
['har', ['**/*.har']],
|
|
32
|
+
['video', ['**/*.webm', '**/*.mp4']],
|
|
33
|
+
['screenshot', ['**/*.png', '**/*-actual.png']],
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
const EXTENSIONS = {
|
|
37
|
+
junit: ['.xml'],
|
|
38
|
+
report: ['.json'],
|
|
39
|
+
trace: ['.zip'],
|
|
40
|
+
har: ['.har'],
|
|
41
|
+
video: ['.webm', '.mp4'],
|
|
42
|
+
screenshot: ['.png'],
|
|
43
|
+
'html-report': ['.html'],
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** Classify an artifact as present, partial (empty), corrupted, or missing. */
|
|
47
|
+
export function integrity(artifactType, filePath) {
|
|
48
|
+
let stats;
|
|
49
|
+
try {
|
|
50
|
+
stats = fs.statSync(filePath);
|
|
51
|
+
} catch {
|
|
52
|
+
return 'missing';
|
|
53
|
+
}
|
|
54
|
+
if (stats.size === 0) return 'partial';
|
|
55
|
+
|
|
56
|
+
// Structural spot-checks for formats we can cheaply verify.
|
|
57
|
+
if (artifactType === 'trace') {
|
|
58
|
+
return isZipFile(filePath) ? 'present' : 'corrupted';
|
|
59
|
+
}
|
|
60
|
+
if (artifactType === 'report' || artifactType === 'har') {
|
|
61
|
+
try {
|
|
62
|
+
JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
63
|
+
} catch {
|
|
64
|
+
return 'corrupted';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return 'present';
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Discover artifacts under `root`, or validate explicit paths.
|
|
72
|
+
*
|
|
73
|
+
* Explicit paths that do not exist are reported as missing; convention discovery
|
|
74
|
+
* never reports missing, because absence is simply "not found".
|
|
75
|
+
*/
|
|
76
|
+
export function discover({ root = '.', explicit = null, framework = 'unknown' } = {}) {
|
|
77
|
+
const present = [];
|
|
78
|
+
const partial = [];
|
|
79
|
+
const corrupted = [];
|
|
80
|
+
const missing = [];
|
|
81
|
+
|
|
82
|
+
const record = (artifactType, filePath) => {
|
|
83
|
+
const state = integrity(artifactType, filePath);
|
|
84
|
+
const entry = artifact({
|
|
85
|
+
type: artifactType,
|
|
86
|
+
location: path.isAbsolute(filePath) ? path.relative(root, filePath) : filePath,
|
|
87
|
+
framework,
|
|
88
|
+
present: state === 'present',
|
|
89
|
+
});
|
|
90
|
+
if (state === 'present') present.push(entry);
|
|
91
|
+
else if (state === 'partial') partial.push(entry);
|
|
92
|
+
else if (state === 'corrupted') corrupted.push(entry);
|
|
93
|
+
else missing.push(filePath);
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
if (explicit && explicit.length > 0) {
|
|
97
|
+
for (const candidate of explicit) {
|
|
98
|
+
if (!fs.existsSync(candidate)) {
|
|
99
|
+
missing.push(candidate);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const matched = PATTERNS.find(([type]) => matchesType(type, candidate))?.[0] ?? 'attachment';
|
|
103
|
+
record(matched, candidate);
|
|
104
|
+
}
|
|
105
|
+
} else {
|
|
106
|
+
const files = walk(root);
|
|
107
|
+
const seen = new Set();
|
|
108
|
+
for (const [artifactType, patterns] of PATTERNS) {
|
|
109
|
+
for (const pattern of patterns) {
|
|
110
|
+
const matches = files.filter((file) => matchesPattern(pattern, file.relative)).map((f) => f.joined);
|
|
111
|
+
for (const filePath of matches.sort()) {
|
|
112
|
+
if (seen.has(filePath)) continue;
|
|
113
|
+
seen.add(filePath);
|
|
114
|
+
record(artifactType, filePath);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return { present, partial, corrupted, missing };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Every file under `root`, with the path spelled the way `glob` would spell it. */
|
|
124
|
+
function walk(root) {
|
|
125
|
+
const found = [];
|
|
126
|
+
const visit = (dir, relativeParts) => {
|
|
127
|
+
let entries;
|
|
128
|
+
try {
|
|
129
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
130
|
+
} catch {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
for (const entry of entries) {
|
|
134
|
+
// A wildcard does not match a leading dot, so such entries are invisible to
|
|
135
|
+
// convention discovery — the same rule `glob` applies.
|
|
136
|
+
if (entry.name.startsWith('.')) continue;
|
|
137
|
+
const next = [...relativeParts, entry.name];
|
|
138
|
+
const full = path.join(dir, entry.name);
|
|
139
|
+
if (entry.isDirectory()) visit(full, next);
|
|
140
|
+
else if (entry.isFile()) found.push({ relative: next, joined: joinLikePython(root, next) });
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
visit(root, []);
|
|
144
|
+
return found;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* `os.path.join(root, relative)`, whose output the discovered `location` carries.
|
|
149
|
+
*
|
|
150
|
+
* Node's `path.join('.', 'a/b')` normalizes the leading `./` away and Python's
|
|
151
|
+
* does not, so a bare `path.join` would report a different location string for
|
|
152
|
+
* every artifact found under the default root.
|
|
153
|
+
*/
|
|
154
|
+
function joinLikePython(root, parts) {
|
|
155
|
+
const tail = parts.join('/');
|
|
156
|
+
if (root === '') return tail;
|
|
157
|
+
return root.endsWith('/') ? `${root}${tail}` : `${root}/${tail}`;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Does a `**`-prefixed pattern match this relative path? */
|
|
161
|
+
function matchesPattern(pattern, relativeParts) {
|
|
162
|
+
const tail = pattern.startsWith('**/') ? pattern.slice(3) : pattern;
|
|
163
|
+
const wanted = tail.split('/');
|
|
164
|
+
if (wanted.length > relativeParts.length) return false;
|
|
165
|
+
// `**` spans zero or more directories, so the tail is matched against the last
|
|
166
|
+
// segments of the path.
|
|
167
|
+
const actual = relativeParts.slice(relativeParts.length - wanted.length);
|
|
168
|
+
return wanted.every((segment, index) => segmentRegExp(segment).test(actual[index]));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const segmentCache = new Map();
|
|
172
|
+
|
|
173
|
+
function segmentRegExp(segment) {
|
|
174
|
+
let cached = segmentCache.get(segment);
|
|
175
|
+
if (!cached) {
|
|
176
|
+
const source = segment
|
|
177
|
+
.split('*')
|
|
178
|
+
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
|
179
|
+
.join('[^/]*');
|
|
180
|
+
cached = new RegExp(`^${source}$`);
|
|
181
|
+
segmentCache.set(segment, cached);
|
|
182
|
+
}
|
|
183
|
+
return cached;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function matchesType(artifactType, filePath) {
|
|
187
|
+
const lower = filePath.toLowerCase();
|
|
188
|
+
return (EXTENSIONS[artifactType] ?? []).some((extension) => lower.endsWith(extension));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Is this a readable ZIP archive?
|
|
193
|
+
*
|
|
194
|
+
* Locates the end-of-central-directory record the way `zipfile.is_zipfile` does,
|
|
195
|
+
* searching the tail of the file (the record is last, and may be followed by up to
|
|
196
|
+
* 64 KiB of comment). Reading the header alone would accept a truncated archive.
|
|
197
|
+
*/
|
|
198
|
+
export function isZipFile(filePath) {
|
|
199
|
+
const EOCD = Buffer.from([0x50, 0x4b, 0x05, 0x06]); // "PK\x05\x06"
|
|
200
|
+
let handle;
|
|
201
|
+
try {
|
|
202
|
+
handle = fs.openSync(filePath, 'r');
|
|
203
|
+
const { size } = fs.fstatSync(handle);
|
|
204
|
+
if (size < 22) return false;
|
|
205
|
+
const window = Math.min(size, 22 + 0xffff);
|
|
206
|
+
const buffer = Buffer.alloc(window);
|
|
207
|
+
fs.readSync(handle, buffer, 0, window, size - window);
|
|
208
|
+
return buffer.lastIndexOf(EOCD) !== -1;
|
|
209
|
+
} catch {
|
|
210
|
+
return false;
|
|
211
|
+
} finally {
|
|
212
|
+
if (handle !== undefined) {
|
|
213
|
+
try {
|
|
214
|
+
fs.closeSync(handle);
|
|
215
|
+
} catch {
|
|
216
|
+
/* already gone */
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|