qa-engineer 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/COMPATIBILITY.md +50 -26
  2. package/README.md +40 -32
  3. package/package.json +12 -20
  4. package/packages/engine/bin/qa-engine.mjs +336 -0
  5. package/packages/engine/lib/analysis/branding.mjs +187 -0
  6. package/packages/engine/lib/analysis/context.mjs +294 -0
  7. package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
  8. package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
  9. package/packages/engine/lib/analysis/discovery.mjs +220 -0
  10. package/packages/engine/lib/analysis/evidence.mjs +116 -0
  11. package/packages/engine/lib/analysis/har.mjs +90 -0
  12. package/packages/engine/lib/analysis/junit.mjs +126 -0
  13. package/packages/engine/lib/analysis/redaction.mjs +127 -0
  14. package/packages/engine/lib/analysis/report-html.mjs +763 -0
  15. package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
  16. package/packages/engine/lib/analysis/xml.mjs +153 -0
  17. package/packages/engine/lib/analysis/zip.mjs +107 -0
  18. package/packages/engine/lib/diagnostics/engine.mjs +165 -0
  19. package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
  20. package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
  21. package/packages/engine/lib/diagnostics/repair.mjs +73 -0
  22. package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
  23. package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
  24. package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
  25. package/packages/engine/lib/frameworks/playwright.mjs +158 -0
  26. package/packages/engine/package.json +14 -0
  27. package/packages/installer/lib/agents/registry.mjs +46 -0
  28. package/packages/installer/lib/commands/doctor.mjs +7 -16
  29. package/packages/installer/lib/commands/install.mjs +59 -4
  30. package/packages/installer/lib/core/bundle.mjs +69 -92
  31. package/packages/installer/lib/core/config.mjs +1 -1
  32. package/packages/installer/lib/core/lockfile.mjs +1 -1
  33. package/packages/installer/lib/core/manifest.mjs +48 -54
  34. package/packages/installer/lib/core/validate-install.mjs +12 -19
  35. package/packages/installer/package.json +1 -1
  36. package/shared/tooling/qa-tool.mjs +125 -0
  37. package/skills/qa-api/SKILL.md +5 -5
  38. package/skills/qa-api/references/deterministic-tooling.md +34 -30
  39. package/skills/qa-api/references/evidence-and-reporting.md +29 -4
  40. package/skills/qa-api/scripts/qa-tool.mjs +126 -0
  41. package/skills/qa-audit/SKILL.md +5 -5
  42. package/skills/qa-audit/references/deterministic-tooling.md +34 -30
  43. package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
  44. package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
  45. package/skills/qa-debug/SKILL.md +6 -6
  46. package/skills/qa-debug/references/deterministic-tooling.md +34 -30
  47. package/skills/qa-debug/references/diagnostic-engine.md +3 -3
  48. package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
  49. package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
  50. package/skills/qa-explore/SKILL.md +22 -5
  51. package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
  52. package/skills/qa-explore/references/deterministic-tooling.md +132 -0
  53. package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
  54. package/skills/qa-explore/references/report-pipeline.md +89 -8
  55. package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
  56. package/skills/qa-fix/SKILL.md +4 -4
  57. package/skills/qa-fix/references/deterministic-tooling.md +34 -30
  58. package/skills/qa-fix/references/diagnostic-engine.md +3 -3
  59. package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
  60. package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
  61. package/skills/qa-flaky/SKILL.md +4 -4
  62. package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
  63. package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
  64. package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
  65. package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
  66. package/skills/qa-init/SKILL.md +4 -4
  67. package/skills/qa-init/references/deterministic-tooling.md +34 -30
  68. package/skills/qa-init/references/evidence-and-reporting.md +29 -4
  69. package/skills/qa-init/scripts/qa-tool.mjs +126 -0
  70. package/skills/qa-report/SKILL.md +8 -7
  71. package/skills/qa-report/references/deterministic-tooling.md +34 -30
  72. package/skills/qa-report/references/diagnostic-engine.md +3 -3
  73. package/skills/qa-report/references/evidence-and-reporting.md +29 -4
  74. package/skills/qa-report/scripts/qa-tool.mjs +126 -0
  75. package/skills/qa-review/references/evidence-and-reporting.md +29 -4
  76. package/skills/qa-run/SKILL.md +6 -6
  77. package/skills/qa-run/references/deterministic-tooling.md +34 -30
  78. package/skills/qa-run/references/evidence-and-reporting.md +29 -4
  79. package/skills/qa-run/scripts/qa-tool.mjs +126 -0
  80. package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
  81. package/shared/analysis/lib/qa_analysis/branding.py +0 -175
  82. package/shared/analysis/lib/qa_analysis/cli.py +0 -129
  83. package/shared/analysis/lib/qa_analysis/context.py +0 -233
  84. package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
  85. package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
  86. package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
  87. package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
  88. package/shared/analysis/lib/qa_analysis/har.py +0 -58
  89. package/shared/analysis/lib/qa_analysis/junit.py +0 -80
  90. package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
  91. package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
  92. package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
  93. package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
  94. package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
  95. package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
  96. package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
  97. package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
  98. package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
  99. package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
  100. package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
  101. package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
  102. package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
  103. package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
  104. package/shared/tooling/qa_tool.py +0 -127
  105. /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
  106. /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
  107. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
  108. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
  109. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
@@ -1,327 +0,0 @@
1
- """The diff guard: a deterministic protection layer against unsafe "fixes".
2
-
3
- Given a unified diff of test code, it flags changes that make a suite pass
4
- without proving the software works. Each flag explains why it is unsafe. It
5
- never edits; it only judges.
6
-
7
- The guard has two jobs, and they pull against each other:
8
-
9
- 1. **Catch every way a suite is made to lie.** Deleting or weakening
10
- assertions, skipping, forcing a pass, excluding specs from the run, making the
11
- test command always exit zero, swallowing failures, inflating timeouts,
12
- deleting test files.
13
- 2. **Not cry wolf on real repairs.** Healing a stale locator *is* the job of
14
- `/qa-fix`, and it necessarily rewrites an assertion line. A guard that flags
15
- every legitimate repair as `high` trains the agent — and the human — to
16
- override it, which is worse than having no guard.
17
-
18
- Job 2 is why assertions are compared by **strength** rather than by presence. An
19
- assertion replaced by one that is at least as strong, keeping the same expected
20
- values, is a modification worth confirming (`low`) — not a removal (`high`). An
21
- assertion replaced by a weaker one, or one that drops the expected value, is
22
- `weakened-assertion` (`high`), because that is how a test quietly stops checking
23
- anything.
24
- """
25
-
26
- import re
27
-
28
- # --- assertion recognition ---------------------------------------------------
29
-
30
- _ASSERTION = re.compile(r"(?i)\b(expect|assert|should|toBe|toEqual|toHaveText|toBeVisible|toContain|assertThat)\b")
31
-
32
- # Matchers that pin a specific value or state. Replacing one of these with a
33
- # weaker matcher means the test no longer checks what it used to.
34
- _STRONG_MATCHERS = re.compile(
35
- r"(?i)\b(toBe|toEqual|toStrictEqual|toHaveText|toContainText|toHaveValue|toHaveURL|"
36
- r"toHaveTitle|toHaveCount|toHaveLength|toHaveAttribute|toHaveClass|toHaveScreenshot|"
37
- r"toMatch|toMatchObject|toMatchSnapshot|toBeVisible|toBeChecked|toBeEnabled|toBeDisabled|"
38
- r"toBeEmpty|toBeGreaterThan|toBeLessThan|toBeCloseTo|"
39
- r"assertEqual|assertEquals|assertIn|assertTrue|assertFalse|isEqualTo|containsExactly)\b"
40
- )
41
-
42
- # Matchers that only prove something exists or is loosely truthy.
43
- _WEAK_MATCHERS = re.compile(
44
- r"(?i)\b(toBeDefined|toBeUndefined|toBeTruthy|toBeFalsy|toBeNull|toBeNaN|toBeAttached|"
45
- r"toBeOk|assertIsNotNone|assertIsNone|isNotNull|isNotEmpty|notNull|toBeInstanceOf)\b"
46
- )
47
-
48
- # A soft assertion does not stop the test at the point of failure; swapping a
49
- # hard assertion for a soft one weakens it.
50
- _SOFT_ASSERTION = re.compile(r"(?i)\b(expect\.soft|softAssert|assertSoftly)\b")
51
-
52
- # String and numeric literals — the expected values an assertion pins down.
53
- _STRING_LITERAL = re.compile(r"""(['"])((?:(?!\1).){2,})\1""")
54
- _NUMBER_LITERAL = re.compile(r"(?<![\w.])\d+(?:\.\d+)?(?![\w.])")
55
-
56
- _IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
57
-
58
- # --- unsafe-change patterns --------------------------------------------------
59
-
60
- _SKIP_ADDED = re.compile(
61
- r"(?i)(\.skip\b|\.fixme\b|\bxit\b|\bxdescribe\b|test\.skip|it\.skip|describe\.skip|"
62
- r"@pytest\.mark\.skip|@Ignore\b|@Disabled\b|\.only\b|this\.skip\(|t\.Skip\()"
63
- )
64
- _FORCED_PASS = re.compile(
65
- r"(?i)(assert\s+True\b|expect\(true\)\.toBe\(true\)|expect\(1\)\.toBe\(1\)|"
66
- r"return\s*;?\s*//\s*pass|assert\s+1\s*==\s*1|pass\s*#\s*(?:todo|skip|pass))"
67
- )
68
-
69
- # A bare early return inside a test body ends the test before it verifies
70
- # anything — the quietest way to fake a pass. Only value-free returns count, so
71
- # `return page.click(...)` and `return await expect(...)` stay clean.
72
- _EARLY_RETURN = re.compile(r"^\s*(?:if\s*\(.*?\)\s*\{?\s*)?return\s*;?\s*\}?\s*$")
73
-
74
- # Excluding specs from the run drops coverage without touching a test file.
75
- _SUITE_EXCLUSION = re.compile(
76
- r"(?i)(testIgnore|testPathIgnorePatterns|excludeSpecPattern|"
77
- r"exclude\s*[:=]|ignorePatterns|--grep-invert|--ignore-pattern|"
78
- r"specs?\s*:\s*\[\s*\]|testMatch\s*[:=]\s*\[\s*\])"
79
- )
80
-
81
- # Making the test command exit zero regardless of the result.
82
- _FORCED_PASS_COMMAND = re.compile(
83
- r"(?i)(\|\|\s*true\b|\|\|\s*exit\s+0\b|;\s*exit\s+0\b|--passWithNoTests\b|"
84
- r"--pass-with-no-tests\b|continue-on-error\s*:\s*true|set\s+\+e\b|"
85
- r"-DskipTests\b|--exit-zero\b|\|\|\s*:)"
86
- )
87
-
88
- # A catch block that discards the failure.
89
- _SWALLOWED_FAILURE = re.compile(r"(?i)(catch\s*(?:\([^)]*\))?\s*\{\s*\}|except\s*[\w.]*\s*:\s*pass\b)")
90
- _CATCH_ADDED = re.compile(r"(?i)(\bcatch\s*[({]|^\s*except\b)")
91
-
92
- _TIMEOUT = re.compile(r"(?i)(timeout|setTimeout|wait_?for|implicitly_wait|setDefaultTimeout)\D{0,20}?(\d{3,})")
93
- # Deliberately excludes `expect(`: an expectation is an assertion, not a wait,
94
- # and including it made this rule unreachable.
95
- _WAIT = re.compile(r"(?i)(waitFor\w*|wait_?for\w*|\.wait\(|awaitVisible|implicitly_wait)")
96
- _LOCATOR = re.compile(r"""(?i)(getBy\w+|locator|find_element|\$\(|css=|xpath=|By\.\w+)""")
97
- _EMPTY_BODY = re.compile(r"(?i)(async\s+)?(\([^)]*\)\s*=>\s*\{\s*\}|function[^{]*\{\s*\}|def\s+test\w*\([^)]*\):\s*pass)")
98
- _RETRY = re.compile(r"(?i)(retries?\s*[:=]\s*(\d+)|\.retry\((\d+)\))")
99
-
100
- _TEST_FILE = re.compile(r"(?i)(\.spec\.|\.test\.|_test\.|test_|/tests?/|/e2e/|\.feature$|Test\.java$|Tests\.cs$)")
101
-
102
- MASS_DELETION_THRESHOLD = 15
103
-
104
- # How much token overlap makes an added assertion the counterpart of a removed
105
- # one. Tuned so a locator rewrite on the same expectation pairs up, while two
106
- # unrelated assertions in the same file do not.
107
- _COUNTERPART_THRESHOLD = 0.34
108
-
109
-
110
- def _parse_diff(diff_text):
111
- """Yield (hunk_file, sign, text) for each changed line. sign is '+' or '-'."""
112
- current = "unknown"
113
- for line in diff_text.splitlines():
114
- if line.startswith("+++ ") or line.startswith("--- "):
115
- path = line[4:].strip()
116
- if path not in ("/dev/null", ""):
117
- current = re.sub(r"^[ab]/", "", path)
118
- continue
119
- if line.startswith("@@"):
120
- continue
121
- if line.startswith("+") and not line.startswith("+++"):
122
- yield (current, "+", line[1:])
123
- elif line.startswith("-") and not line.startswith("---"):
124
- yield (current, "-", line[1:])
125
-
126
-
127
- def _deleted_files(diff_text):
128
- """Files whose new side is /dev/null — deleted outright."""
129
- deleted, pending = set(), None
130
- for line in diff_text.splitlines():
131
- if line.startswith("--- "):
132
- path = line[4:].strip()
133
- pending = re.sub(r"^[ab]/", "", path) if path != "/dev/null" else None
134
- elif line.startswith("+++ "):
135
- if line[4:].strip() == "/dev/null" and pending:
136
- deleted.add(pending)
137
- pending = None
138
- return deleted
139
-
140
-
141
- def _tokens(text):
142
- return {t.lower() for t in _IDENTIFIER.findall(text)}
143
-
144
-
145
- def _literals(text):
146
- strings = {m.group(2) for m in _STRING_LITERAL.finditer(text)}
147
- numbers = set(_NUMBER_LITERAL.findall(text))
148
- return strings, numbers
149
-
150
-
151
- def _strength(text):
152
- """3 = pins a value/state, 2 = unclassified assertion, 1 = existence only."""
153
- if _SOFT_ASSERTION.search(text):
154
- return 1
155
- if _WEAK_MATCHERS.search(text) and not _STRONG_MATCHERS.search(text):
156
- return 1
157
- if _STRONG_MATCHERS.search(text):
158
- return 3
159
- return 2
160
-
161
-
162
- def _counterpart(removed_text, candidates):
163
- """The added assertion line most plausibly replacing this removed one."""
164
- removed_tokens = _tokens(removed_text)
165
- if not removed_tokens:
166
- return None
167
- best, best_score = None, 0.0
168
- for text in candidates:
169
- overlap = removed_tokens & _tokens(text)
170
- score = len(overlap) / max(len(removed_tokens), 1)
171
- if score > best_score:
172
- best, best_score = text, score
173
- return best if best_score >= _COUNTERPART_THRESHOLD else None
174
-
175
-
176
- def check_diff(diff_text):
177
- """Analyze a unified diff. Returns a list of issue dicts:
178
- {rule, severity, file, why, sample}. An empty list means no unsafe change
179
- was detected (which is not a guarantee of correctness, only of safety)."""
180
- issues = []
181
- removed = [(f, t) for f, s, t in _parse_diff(diff_text) if s == "-"]
182
- added = [(f, t) for f, s, t in _parse_diff(diff_text) if s == "+"]
183
- deleted_files = _deleted_files(diff_text)
184
-
185
- def flag(rule, severity, file, why, sample):
186
- issues.append({"rule": rule, "severity": severity, "file": file,
187
- "why": why, "sample": str(sample).strip()[:200]})
188
-
189
- added_by_file = {}
190
- for file, text in added:
191
- added_by_file.setdefault(file, []).append(text)
192
-
193
- # --- removed or weakened assertions --------------------------------------
194
- for file, text in removed:
195
- if _ASSERTION.search(text):
196
- same_file_added = added_by_file.get(file, [])
197
- # An identical line re-added is a move, not a change.
198
- if any(a.strip() == text.strip() for a in same_file_added):
199
- continue
200
-
201
- assertion_candidates = [a for a in same_file_added if _ASSERTION.search(a)]
202
- replacement = _counterpart(text, assertion_candidates)
203
-
204
- if replacement is None:
205
- if file in deleted_files:
206
- continue # reported once as a deleted test file, below
207
- flag("removed-assertion", "high", file,
208
- "An assertion or expectation was removed with nothing replacing it; "
209
- "a test that no longer asserts proves nothing.", text)
210
- continue
211
-
212
- old_strength, new_strength = _strength(text), _strength(replacement)
213
- old_strings, old_numbers = _literals(text)
214
- new_strings, new_numbers = _literals(replacement)
215
- dropped_strings = old_strings - new_strings
216
- dropped_numbers = old_numbers - new_numbers
217
-
218
- if new_strength < old_strength:
219
- flag("weakened-assertion", "high", file,
220
- "An assertion was replaced by a weaker one (existence or truthiness "
221
- "instead of a specific value or state); the test can now pass without "
222
- "the behaviour being correct.", replacement)
223
- elif dropped_strings or dropped_numbers:
224
- dropped = sorted(dropped_strings | dropped_numbers)
225
- flag("weakened-assertion", "high", file,
226
- "An assertion kept its matcher but dropped the expected value(s) "
227
- f"{dropped}; it no longer pins down what it used to.", replacement)
228
- else:
229
- flag("assertion-modified", "low", file,
230
- "An assertion was rewritten with equal or greater strength and the same "
231
- "expected values — consistent with a legitimate repair. Confirm it still "
232
- "targets the same behaviour.", replacement)
233
-
234
- if _WAIT.search(text) and not _ASSERTION.search(text):
235
- flag("removed-wait", "medium", file,
236
- "A wait/synchronization was removed, which can mask a real failure or "
237
- "introduce flakiness.", text)
238
-
239
- # --- added skips, forced passes, early returns, empty bodies -------------
240
- for file, text in added:
241
- if _SKIP_ADDED.search(text):
242
- flag("added-skip-or-only", "high", file,
243
- "A skip/ignore/only marker was added; skipping or narrowing hides failures "
244
- "instead of fixing them.", text)
245
- if _FORCED_PASS.search(text):
246
- flag("forced-pass", "high", file,
247
- "A tautological or forced-pass assertion was added; it makes the test pass "
248
- "without checking behavior.", text)
249
- if _EMPTY_BODY.search(text):
250
- flag("empty-test-body", "high", file,
251
- "A test body was emptied; an empty test passes vacuously.", text)
252
- if _EARLY_RETURN.match(text) and _TEST_FILE.search(file):
253
- flag("conditional-skip", "high", file,
254
- "An unconditional or condition-guarded early return was added to a test; "
255
- "the test exits before verifying anything, which is a skip in disguise.", text)
256
- if _SUITE_EXCLUSION.search(text):
257
- flag("suite-exclusion", "high", file,
258
- "Specs were excluded from the run (ignore pattern, exclusion list, or empty "
259
- "spec list); coverage is dropped without any test file changing.", text)
260
- if _FORCED_PASS_COMMAND.search(text):
261
- flag("forced-pass-command", "high", file,
262
- "The test command was made to succeed regardless of the result "
263
- "(`|| true`, `exit 0`, `--passWithNoTests`, `continue-on-error`); the suite "
264
- "can no longer fail a pipeline.", text)
265
- if _SWALLOWED_FAILURE.search(text):
266
- flag("swallowed-failure", "high", file,
267
- "A failure is caught and discarded; an assertion inside a swallowed catch "
268
- "cannot fail the test.", text)
269
- elif _CATCH_ADDED.search(text) and _TEST_FILE.search(file):
270
- flag("added-error-handling", "medium", file,
271
- "Error handling was added inside a test; confirm the failure still surfaces "
272
- "rather than being caught and logged.", text)
273
-
274
- # --- deleted test files --------------------------------------------------
275
- for file in sorted(deleted_files):
276
- if _TEST_FILE.search(file):
277
- flag("test-file-deleted", "high", file,
278
- "A test file was deleted outright; every case it held stops running, "
279
- "regardless of the file's size.", file)
280
-
281
- # --- timeout inflation and unsafe retry increases -------------------------
282
- _flag_numeric_inflation(_TIMEOUT, removed, added, "timeout-inflation", "medium",
283
- "A timeout was increased substantially, which can paper over a real slowness or hang.", flag)
284
- _flag_numeric_inflation(_RETRY, removed, added, "unsafe-retry-increase", "medium",
285
- "Retry count was increased, which can convert a real failure into an intermittent pass.", flag)
286
-
287
- # --- suspicious locator changes ------------------------------------------
288
- removed_locators = {f: t for f, t in removed if _LOCATOR.search(t)}
289
- for file, text in added:
290
- if _LOCATOR.search(text) and file in removed_locators and text.strip() != removed_locators[file].strip():
291
- flag("suspicious-locator-change", "low", file,
292
- "A locator was changed; confirm it targets the same element and was not "
293
- "loosened to pass.", text)
294
-
295
- # --- mass deletion per file ----------------------------------------------
296
- per_file = {}
297
- for file, _ in removed:
298
- per_file[file] = per_file.get(file, 0) + 1
299
- for file, count in per_file.items():
300
- if count >= MASS_DELETION_THRESHOLD and file not in deleted_files:
301
- flag("mass-deletion", "high", file,
302
- f"{count} lines were deleted from a test file; large deletions can silently drop coverage.",
303
- f"{count} lines removed")
304
-
305
- return issues
306
-
307
-
308
- def _flag_numeric_inflation(pattern, removed, added, rule, severity, why, flag):
309
- def max_number(text):
310
- nums = [int(n) for n in re.findall(r"\d{2,}", text)]
311
- return max(nums) if nums else None
312
-
313
- removed_by_file = {}
314
- for file, text in removed:
315
- if pattern.search(text):
316
- value = max_number(text)
317
- if value is not None:
318
- removed_by_file.setdefault(file, []).append(value)
319
- for file, text in added:
320
- if pattern.search(text):
321
- new_value = max_number(text)
322
- if new_value is None:
323
- continue
324
- for old_value in removed_by_file.get(file, []):
325
- if new_value >= old_value * 2 and new_value > old_value:
326
- flag(rule, severity, file, why, text)
327
- break
@@ -1,113 +0,0 @@
1
- """Deterministic artifact discovery.
2
-
3
- Locates the artifacts a run produced — automatically by convention or from
4
- explicit paths — and classifies each as present, partial, or corrupted. It
5
- handles multiple runs, parallel workers, and sharded output (many result files),
6
- and it reports what is missing rather than inventing it.
7
- """
8
-
9
- import json
10
- import os
11
- import zipfile
12
- from glob import glob
13
-
14
- from .evidence import Artifact
15
-
16
- # Convention globs for known artifact types, relative to a run root.
17
- _PATTERNS = [
18
- ("junit", ["**/results.xml", "**/junit*.xml", "**/*junit*.xml"]),
19
- ("report", ["**/results.json", "**/report.json"]),
20
- ("html-report", ["**/playwright-report/index.html", "**/*-report/index.html"]),
21
- ("trace", ["**/trace.zip", "**/*-trace.zip"]),
22
- ("har", ["**/*.har"]),
23
- ("video", ["**/*.webm", "**/*.mp4"]),
24
- ("screenshot", ["**/*.png", "**/*-actual.png"]),
25
- ]
26
-
27
-
28
- def _integrity(artifact_type, path):
29
- """Classify an artifact as present, partial (empty), or corrupted."""
30
- try:
31
- size = os.path.getsize(path)
32
- except OSError:
33
- return "missing"
34
- if size == 0:
35
- return "partial"
36
- # Structural spot-checks for formats we can cheaply verify.
37
- if artifact_type == "trace":
38
- if not zipfile.is_zipfile(path):
39
- return "corrupted"
40
- elif artifact_type in ("report", "har"):
41
- try:
42
- with open(path, "r", encoding="utf-8") as handle:
43
- json.load(handle)
44
- except (json.JSONDecodeError, OSError, UnicodeDecodeError):
45
- return "corrupted"
46
- return "present"
47
-
48
-
49
- def discover(root=".", explicit=None, framework="unknown"):
50
- """Discover artifacts under root, or validate explicit paths.
51
-
52
- Returns {present: [Artifact...], partial: [...], corrupted: [...],
53
- missing: [paths]}. Explicit paths that do not exist are reported as missing;
54
- convention discovery never reports missing (absence is simply not found).
55
- """
56
- present, partial, corrupted, missing = [], [], [], []
57
-
58
- def record(artifact_type, path):
59
- state = _integrity(artifact_type, path)
60
- artifact = Artifact(type=artifact_type, location=os.path.relpath(path, root)
61
- if os.path.isabs(path) else path, framework=framework,
62
- present=(state == "present"))
63
- if state == "present":
64
- present.append(artifact)
65
- elif state == "partial":
66
- partial.append(artifact)
67
- elif state == "corrupted":
68
- corrupted.append(artifact)
69
- else:
70
- missing.append(path)
71
-
72
- if explicit:
73
- for path in explicit:
74
- if not os.path.exists(path):
75
- missing.append(path)
76
- continue
77
- matched = next((t for t, _ in _PATTERNS if _matches_type(t, path)), "attachment")
78
- record(matched, path)
79
- else:
80
- seen = set()
81
- for artifact_type, patterns in _PATTERNS:
82
- for pattern in patterns:
83
- for path in sorted(glob(os.path.join(root, pattern), recursive=True)):
84
- if path in seen:
85
- continue
86
- seen.add(path)
87
- record(artifact_type, path)
88
-
89
- return {
90
- "present": present,
91
- "partial": partial,
92
- "corrupted": corrupted,
93
- "missing": missing,
94
- }
95
-
96
-
97
- def _matches_type(artifact_type, path):
98
- lower = path.lower()
99
- if artifact_type == "junit":
100
- return lower.endswith(".xml")
101
- if artifact_type == "report":
102
- return lower.endswith(".json")
103
- if artifact_type == "trace":
104
- return lower.endswith(".zip")
105
- if artifact_type == "har":
106
- return lower.endswith(".har")
107
- if artifact_type == "video":
108
- return lower.endswith((".webm", ".mp4"))
109
- if artifact_type == "screenshot":
110
- return lower.endswith(".png")
111
- if artifact_type == "html-report":
112
- return lower.endswith(".html")
113
- return False
@@ -1,128 +0,0 @@
1
- """The evidence and finding model shared by every analyzer.
2
-
3
- Every finding an analyzer produces carries the same structure, so downstream
4
- skills (qa-debug, qa-report, qa-fix) consume one shape regardless of which
5
- analyzer or framework produced it. Text fields are redacted at construction.
6
- """
7
-
8
- from dataclasses import dataclass, field, asdict
9
- from datetime import datetime, timezone
10
-
11
- from .redaction import redact_text
12
-
13
- EVIDENCE_TYPES = {
14
- "trace", "har", "junit", "report", "console", "network", "stdout",
15
- "stderr", "screenshot", "video", "log", "file", "diff",
16
- }
17
-
18
-
19
- def utc_now():
20
- """ISO 8601 UTC timestamp. Isolated so tests can monkeypatch it."""
21
- return datetime.now(timezone.utc).isoformat()
22
-
23
-
24
- @dataclass
25
- class Evidence:
26
- """One observation supporting a finding. Excerpts are redacted."""
27
- type: str
28
- description: str
29
- source: str
30
- excerpt: str = ""
31
-
32
- def __post_init__(self):
33
- if self.type not in EVIDENCE_TYPES:
34
- raise ValueError(f"unknown evidence type: {self.type}")
35
- self.excerpt = redact_text(self.excerpt)
36
-
37
- def to_dict(self):
38
- data = {"type": self.type, "description": self.description, "source": self.source}
39
- if self.excerpt:
40
- data["excerpt"] = self.excerpt
41
- return data
42
-
43
-
44
- @dataclass
45
- class Finding:
46
- """A single diagnostic conclusion, traceable to a specific artifact.
47
-
48
- Carries everything the evidence model requires: the artifact and location
49
- it came from, when, why, the supporting evidence, a calibrated confidence,
50
- the affected tests, related artifacts, and recommended actions.
51
- """
52
- classification: str
53
- reason: str
54
- artifact: str
55
- location: str
56
- confidence: float = None
57
- timestamp: str = field(default_factory=utc_now)
58
- evidence: list = field(default_factory=list)
59
- affected_tests: list = field(default_factory=list)
60
- related_artifacts: list = field(default_factory=list)
61
- recommendations: list = field(default_factory=list)
62
-
63
- def to_dict(self):
64
- data = {
65
- "classification": self.classification,
66
- "reason": self.reason,
67
- "artifact": self.artifact,
68
- "location": self.location,
69
- "timestamp": self.timestamp,
70
- "evidence": [e.to_dict() if isinstance(e, Evidence) else e for e in self.evidence],
71
- "affectedTests": self.affected_tests,
72
- "relatedArtifacts": self.related_artifacts,
73
- "recommendations": self.recommendations,
74
- }
75
- if self.confidence is not None:
76
- data["confidence"] = self.confidence
77
- return data
78
-
79
-
80
- @dataclass
81
- class AnalyzerOutput:
82
- """The envelope an analyzer emits: findings plus the artifacts it examined.
83
-
84
- A downstream skill wraps this in its own output contract; on its own it is
85
- the deterministic, machine-readable result of one analysis.
86
- """
87
- analyzer: str
88
- findings: list = field(default_factory=list)
89
- artifacts: list = field(default_factory=list)
90
- warnings: list = field(default_factory=list)
91
- generated_at: str = field(default_factory=utc_now)
92
-
93
- def to_dict(self):
94
- return {
95
- "analyzer": self.analyzer,
96
- "generatedAt": self.generated_at,
97
- "findings": [f.to_dict() if isinstance(f, Finding) else f for f in self.findings],
98
- "artifacts": [a.to_dict() if hasattr(a, "to_dict") else a for a in self.artifacts],
99
- "warnings": self.warnings,
100
- }
101
-
102
-
103
- @dataclass
104
- class Artifact:
105
- """A discovered artifact, in the common model shared with the execution engine."""
106
- type: str
107
- location: str
108
- framework: str = "unknown"
109
- ownership: str = "qa-analysis"
110
- timestamp: str = field(default_factory=utc_now)
111
- media_type: str = ""
112
- test_ref: str = ""
113
- present: bool = True
114
-
115
- def to_dict(self):
116
- data = {
117
- "type": self.type,
118
- "location": self.location,
119
- "framework": self.framework,
120
- "timestamp": self.timestamp,
121
- "ownership": self.ownership,
122
- "present": self.present,
123
- }
124
- if self.media_type:
125
- data["mediaType"] = self.media_type
126
- if self.test_ref:
127
- data["testRef"] = self.test_ref
128
- return data
@@ -1,58 +0,0 @@
1
- """HAR (HTTP Archive) parser.
2
-
3
- Framework-agnostic: a HAR is a standard JSON format, whoever produced it.
4
- Extracts request outcomes, flags failures and slow calls, and redacts headers
5
- and credentialed URLs before anything is exposed. A malformed HAR raises.
6
- """
7
-
8
- import json
9
-
10
- from .redaction import redact_text, redact_headers
11
- from .junit import MalformedArtifact
12
-
13
-
14
- def parse_har(path, slow_ms=1000):
15
- """Parse a HAR file into a redacted network summary.
16
-
17
- Returns {entries, failures, slow, redacted: True}, where each entry is
18
- {method, url, status, durationMs, headersRedacted}. Failures are entries
19
- with status >= 400 or status 0 (no response). Raises MalformedArtifact on
20
- unreadable input.
21
- """
22
- try:
23
- with open(path, "r", encoding="utf-8") as handle:
24
- data = json.load(handle)
25
- except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
26
- raise MalformedArtifact(f"could not parse HAR at {path}: {exc}") from exc
27
-
28
- try:
29
- raw_entries = data["log"]["entries"]
30
- except (KeyError, TypeError) as exc:
31
- raise MalformedArtifact(f"not a HAR document at {path}") from exc
32
-
33
- entries = []
34
- for item in raw_entries:
35
- request = item.get("request", {})
36
- response = item.get("response", {})
37
- status = _int(response.get("status"))
38
- entry = {
39
- "method": request.get("method", ""),
40
- # Redaction strips any credentials embedded in the URL.
41
- "url": redact_text(request.get("url", "")),
42
- "status": status,
43
- "durationMs": int(round(float(item.get("time", 0) or 0))),
44
- "requestHeaders": redact_headers(request.get("headers", [])),
45
- "responseHeaders": redact_headers(response.get("headers", [])),
46
- }
47
- entries.append(entry)
48
-
49
- failures = [e for e in entries if e["status"] == 0 or e["status"] >= 400]
50
- slow = [e for e in entries if e["durationMs"] >= slow_ms]
51
- return {"entries": entries, "failures": failures, "slow": slow, "redacted": True}
52
-
53
-
54
- def _int(value, default=0):
55
- try:
56
- return int(value)
57
- except (TypeError, ValueError):
58
- return default