qa-engineer 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +26 -12
- package/README.md +20 -30
- package/package.json +11 -20
- package/packages/engine/bin/qa-engine.mjs +336 -0
- package/packages/engine/lib/analysis/branding.mjs +187 -0
- package/packages/engine/lib/analysis/context.mjs +294 -0
- package/packages/engine/lib/analysis/contracts.mjs +149 -0
- package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
- package/packages/engine/lib/analysis/discovery.mjs +220 -0
- package/packages/engine/lib/analysis/evidence.mjs +116 -0
- package/packages/engine/lib/analysis/har.mjs +90 -0
- package/packages/engine/lib/analysis/junit.mjs +126 -0
- package/packages/engine/lib/analysis/redaction.mjs +127 -0
- package/packages/engine/lib/analysis/report-html.mjs +763 -0
- package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
- package/packages/engine/lib/analysis/xml.mjs +153 -0
- package/packages/engine/lib/analysis/zip.mjs +107 -0
- package/packages/engine/lib/diagnostics/engine.mjs +165 -0
- package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
- package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
- package/packages/engine/lib/diagnostics/repair.mjs +73 -0
- package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
- package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
- package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
- package/packages/engine/lib/frameworks/playwright.mjs +158 -0
- package/packages/engine/package.json +14 -0
- package/packages/installer/lib/commands/doctor.mjs +7 -16
- package/packages/installer/lib/commands/install.mjs +28 -0
- package/packages/installer/lib/core/bundle.mjs +69 -92
- package/packages/installer/lib/core/manifest.mjs +48 -57
- package/packages/installer/lib/core/validate-install.mjs +12 -19
- package/packages/installer/package.json +1 -1
- package/shared/tooling/qa-tool.mjs +125 -0
- package/skills/qa-api/SKILL.md +5 -5
- package/skills/qa-api/references/deterministic-tooling.md +34 -32
- package/skills/qa-api/references/evidence-and-reporting.md +5 -5
- package/skills/qa-api/scripts/qa-tool.mjs +126 -0
- package/skills/qa-audit/SKILL.md +5 -5
- package/skills/qa-audit/references/deterministic-tooling.md +34 -32
- package/skills/qa-audit/references/evidence-and-reporting.md +5 -5
- package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
- package/skills/qa-debug/SKILL.md +6 -6
- package/skills/qa-debug/references/deterministic-tooling.md +34 -32
- package/skills/qa-debug/references/diagnostic-engine.md +3 -3
- package/skills/qa-debug/references/evidence-and-reporting.md +5 -5
- package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
- package/skills/qa-explore/SKILL.md +6 -6
- package/skills/qa-explore/references/deterministic-tooling.md +34 -32
- package/skills/qa-explore/references/evidence-and-reporting.md +5 -5
- package/skills/qa-explore/references/report-pipeline.md +3 -3
- package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
- package/skills/qa-fix/SKILL.md +4 -4
- package/skills/qa-fix/references/deterministic-tooling.md +34 -32
- package/skills/qa-fix/references/diagnostic-engine.md +3 -3
- package/skills/qa-fix/references/evidence-and-reporting.md +5 -5
- package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
- package/skills/qa-flaky/SKILL.md +4 -4
- package/skills/qa-flaky/references/deterministic-tooling.md +34 -32
- package/skills/qa-flaky/references/evidence-and-reporting.md +5 -5
- package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
- package/skills/qa-generate/references/evidence-and-reporting.md +5 -5
- package/skills/qa-init/SKILL.md +4 -4
- package/skills/qa-init/references/deterministic-tooling.md +34 -32
- package/skills/qa-init/references/evidence-and-reporting.md +5 -5
- package/skills/qa-init/scripts/qa-tool.mjs +126 -0
- package/skills/qa-report/SKILL.md +7 -7
- package/skills/qa-report/references/deterministic-tooling.md +34 -32
- package/skills/qa-report/references/diagnostic-engine.md +3 -3
- package/skills/qa-report/references/evidence-and-reporting.md +5 -5
- package/skills/qa-report/scripts/qa-tool.mjs +126 -0
- package/skills/qa-review/references/evidence-and-reporting.md +5 -5
- package/skills/qa-run/SKILL.md +6 -6
- package/skills/qa-run/references/deterministic-tooling.md +34 -32
- package/skills/qa-run/references/evidence-and-reporting.md +5 -5
- package/skills/qa-run/scripts/qa-tool.mjs +126 -0
- package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
- package/shared/analysis/lib/qa_analysis/branding.py +0 -175
- package/shared/analysis/lib/qa_analysis/cli.py +0 -144
- package/shared/analysis/lib/qa_analysis/context.py +0 -233
- package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
- package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
- package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
- package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
- package/shared/analysis/lib/qa_analysis/har.py +0 -87
- package/shared/analysis/lib/qa_analysis/junit.py +0 -104
- package/shared/analysis/lib/qa_analysis/redaction.py +0 -107
- package/shared/analysis/lib/qa_analysis/report_html.py +0 -781
- package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
- package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
- package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
- package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
- package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
- package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
- package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
- package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
- package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
- package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
- package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
- package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
- package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
- package/shared/tooling/qa_tool.py +0 -127
- /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
- /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
package/COMPATIBILITY.md
CHANGED
|
@@ -9,7 +9,8 @@ This document records which AI coding agents the pack targets, how each one disc
|
|
|
9
9
|
| Pack version | Released | Discovery paths verified against agent docs | Live agent runs |
|
|
10
10
|
| --- | --- | --- | --- |
|
|
11
11
|
| 0.9.0, 0.9.1 | 2026-07-28 | Not systematically re-read for the release | Claude Code, Cursor (manual, by the maintainer) |
|
|
12
|
-
| 0.9.2 | 2026-07-28 | Yes — every host's own documentation, re-read on 2026-07-28 (sources in the matrix below) |
|
|
12
|
+
| 0.9.2 | 2026-07-28 | Yes — every host's own documentation, re-read on 2026-07-28 (sources in the matrix below) | **Do not use.** The tarball omitted the engine; every command failed on a missing module |
|
|
13
|
+
| 0.9.3 | 2026-07-28 | Yes, as 0.9.2 | Claude Code, Cursor (manual, by the maintainer) |
|
|
13
14
|
|
|
14
15
|
"Verified" in the third column means the discovery paths in the matrix below were read off each host's own documentation on that date and are asserted against the installer by `packages/installer/test/hosts.test.mjs`, which fails if the installer writes to a path that host does not document reading. It does **not** mean the pack was executed inside every host: that is the Tier 1 bar, and only Claude Code and Cursor have had a real run.
|
|
15
16
|
|
|
@@ -55,8 +56,14 @@ The tier column in the matrix above is the **planned** tier — the level each a
|
|
|
55
56
|
|
|
56
57
|
| Requirement | Status | Used for |
|
|
57
58
|
| --- | --- | --- |
|
|
58
|
-
|
|
|
59
|
-
|
|
59
|
+
| Node.js 18.17+ | The only requirement | Everything: `npx qa-engineer` and the CLI, and the deterministic engine the skills run (`packages/engine/`) |
|
|
60
|
+
|
|
61
|
+
**One runtime, no dependencies.** Until 0.9.2 the engine was Python, which meant a
|
|
62
|
+
second runtime users had to install for tooling they had installed with Node — and
|
|
63
|
+
when it was missing, skills fell back to model guesswork and said nothing.
|
|
64
|
+
[ADR-0012](docs/architecture/ADR-0012-node-engine.md) records why that changed and
|
|
65
|
+
how the migration was verified. The engine still takes no third-party dependencies:
|
|
66
|
+
the XML and ZIP readers it needs are written out rather than depended on.
|
|
60
67
|
|
|
61
68
|
## Operating systems
|
|
62
69
|
|
|
@@ -66,15 +73,22 @@ The tier column in the matrix above is the **planned** tier — the level each a
|
|
|
66
73
|
| macOS | Expected to work; not covered by CI |
|
|
67
74
|
| Windows | Expected to work; **not yet verified end to end** |
|
|
68
75
|
|
|
69
|
-
The installer uses Node's platform-independent path handling, and the skills
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
The installer uses Node's platform-independent path handling, and the skills invoke
|
|
77
|
+
the engine through a committed launcher (`scripts/qa-tool.mjs`) that needs no shell
|
|
78
|
+
features — the command shape is identical in bash, zsh, PowerShell, and cmd.exe,
|
|
79
|
+
with no platform difference at all.
|
|
80
|
+
|
|
81
|
+
The Windows row said "expected" for a specific reason that no longer applies: the
|
|
82
|
+
engine was Python, and `python3` is not on PATH on Windows by default, so every
|
|
83
|
+
deterministic call there failed and each skill quietly fell back to guesswork. The
|
|
84
|
+
engine is now the same Node that ran the install. The row still says "expected"
|
|
85
|
+
rather than "supported" because nobody has run the pack end to end on Windows —
|
|
86
|
+
that is an unverified claim, not a known gap.
|
|
87
|
+
|
|
88
|
+
Requirements are declared per skill via the specification's `compatibility`
|
|
89
|
+
frontmatter field, and the pack degrades gracefully: skills describe a manual
|
|
90
|
+
fallback whenever a runtime or optional integration (such as an MCP server) is
|
|
91
|
+
unavailable.
|
|
78
92
|
|
|
79
93
|
## Reporting compatibility problems
|
|
80
94
|
|
package/README.md
CHANGED
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
> Teach your AI coding assistant to work like a senior QA engineer — and stop it from telling you tests pass when they don't.
|
|
4
4
|
|
|
5
|
-
[](CHANGELOG.md)
|
|
6
6
|
[](docs/release/v0.9-release-checklist.md)
|
|
7
7
|
[](LICENSE)
|
|
8
|
-
[](#step-1--check-your-
|
|
9
|
-
[](#step-1--check-your-prerequisites)
|
|
8
|
+
[](#step-1--check-your-prerequisite)
|
|
10
9
|
[](https://agentskills.io)
|
|
11
10
|
[](https://github.com/abisheik88/qa-engineer/actions/workflows/ci.yml)
|
|
12
|
-
[](#how-this-is-verified)
|
|
13
12
|
|
|
14
13
|
**This page is everything you need.** Install, first run, every command, and what to do when something breaks — no other document required.
|
|
15
14
|
|
|
@@ -19,7 +18,7 @@
|
|
|
19
18
|
|
|
20
19
|
1. [What this is, in plain language](#what-this-is-in-plain-language)
|
|
21
20
|
2. [Is this for me?](#is-this-for-me)
|
|
22
|
-
3. [Step 1 — Check your prerequisites](#step-1--check-your-
|
|
21
|
+
3. [Step 1 — Check your prerequisites](#step-1--check-your-prerequisite)
|
|
23
22
|
4. [Step 2 — Install it](#step-2--install-it)
|
|
24
23
|
5. [Step 3 — Confirm it worked](#step-3--confirm-it-worked)
|
|
25
24
|
6. [Step 4 — Your first real task](#step-4--your-first-real-task)
|
|
@@ -67,21 +66,26 @@ You do not need to learn a new language, framework, or config file. You install
|
|
|
67
66
|
|
|
68
67
|
**Works best with Playwright** today. Selenium, Cypress, and WebdriverIO are detected and their results are understood, but running and generating tests live is currently Playwright-only. See [framework support](#framework-support).
|
|
69
68
|
|
|
70
|
-
## Step 1 — Check your
|
|
69
|
+
## Step 1 — Check your prerequisite
|
|
71
70
|
|
|
72
|
-
You need
|
|
71
|
+
You need one thing. Run this to check — copy and paste it exactly:
|
|
73
72
|
|
|
74
73
|
```bash
|
|
75
74
|
node --version
|
|
76
|
-
python3 --version
|
|
77
75
|
```
|
|
78
76
|
|
|
79
|
-
You want **Node 18.18 or newer
|
|
77
|
+
You want **Node 18.18 or newer**. If it prints a high enough version, skip ahead to
|
|
78
|
+
[Step 2](#step-2--install-it).
|
|
79
|
+
|
|
80
|
+
That is the whole list. There is nothing to `pip install`, no second language, and no
|
|
81
|
+
dependencies: the tools that read your test results are dependency-free JavaScript
|
|
82
|
+
that runs on the same Node you just checked.
|
|
80
83
|
|
|
81
84
|
<details>
|
|
82
85
|
<summary><b>"command not found" — how to install Node.js</b></summary>
|
|
83
86
|
|
|
84
|
-
Node.js gives you the `npx` command used to install this pack
|
|
87
|
+
Node.js gives you the `npx` command used to install this pack, and it runs the
|
|
88
|
+
analysis tools afterwards.
|
|
85
89
|
|
|
86
90
|
- **macOS** — `brew install node` (needs [Homebrew](https://brew.sh)), or download from [nodejs.org](https://nodejs.org).
|
|
87
91
|
- **Windows** — download the LTS installer from [nodejs.org](https://nodejs.org) and run it.
|
|
@@ -91,19 +95,6 @@ Then re-run `node --version`. Take the **LTS** version if offered a choice.
|
|
|
91
95
|
|
|
92
96
|
</details>
|
|
93
97
|
|
|
94
|
-
<details>
|
|
95
|
-
<summary><b>"command not found" — how to install Python</b></summary>
|
|
96
|
-
|
|
97
|
-
Python runs the analysis tools that read your test results. It uses only Python's built-in library — there is nothing extra to install, no `pip install` step.
|
|
98
|
-
|
|
99
|
-
- **macOS** — `brew install python3`, or download from [python.org](https://python.org).
|
|
100
|
-
- **Windows** — install from [python.org](https://python.org) and **tick "Add Python to PATH"** during setup. Then use `python --version` if `python3` is not found.
|
|
101
|
-
- **Linux** — `sudo apt install python3` (Debian/Ubuntu).
|
|
102
|
-
|
|
103
|
-
**Can you skip Python?** Yes, but you shouldn't. Without it the skills fall back to the AI reading files by eye, and they will mark their results *degraded* to tell you so. The whole point of this project is that the numbers come from tools, not guesses.
|
|
104
|
-
|
|
105
|
-
</details>
|
|
106
|
-
|
|
107
98
|
You also need **an AI coding assistant** open on your project — Claude Code, Cursor, GitHub Copilot, Codex CLI, OpenCode, Gemini CLI, Antigravity, or Kimi.
|
|
108
99
|
|
|
109
100
|
## Step 2 — Install it
|
|
@@ -165,7 +156,7 @@ Real output from a healthy install:
|
|
|
165
156
|
✓ [PASS] skills: 13 skill(s) installed
|
|
166
157
|
✓ [PASS] contracts: 12 contract schema(s) present
|
|
167
158
|
✓ [PASS] engine: deterministic engine bundled under .agents/skills/qa-init/scripts/lib
|
|
168
|
-
✓ [PASS]
|
|
159
|
+
✓ [PASS] engine-runs: bundled engine runs cleanly
|
|
169
160
|
✓ [PASS] node: Node v24.18.0
|
|
170
161
|
✓ self-test PASSED
|
|
171
162
|
```
|
|
@@ -347,10 +338,9 @@ npx qa-engineer install --yes --force --project .
|
|
|
347
338
|
|
|
348
339
|
### A skill says the engine is missing, or results are "degraded"
|
|
349
340
|
|
|
350
|
-
The
|
|
341
|
+
The deterministic engine isn't reachable. Fix in this order:
|
|
351
342
|
|
|
352
343
|
```bash
|
|
353
|
-
python3 --version # is Python installed at all?
|
|
354
344
|
npx qa-engineer repair --project . # reinstall the bundled tools
|
|
355
345
|
npx qa-engineer doctor --project . # should now say "bundled engine runs cleanly"
|
|
356
346
|
```
|
|
@@ -409,7 +399,7 @@ The core idea in one line: **tools produce the facts, the AI explains them.**
|
|
|
409
399
|
│ raw output, exit code
|
|
410
400
|
▼
|
|
411
401
|
┌───────────────────────────────────┐
|
|
412
|
-
│ Bundled
|
|
402
|
+
│ Bundled Node engine │ count the results, classify the failure,
|
|
413
403
|
│ (installed inside the skill) │ redact secrets — no guessing
|
|
414
404
|
└───────────────────────────────────┘
|
|
415
405
|
│ facts
|
|
@@ -459,11 +449,11 @@ Everything below is reproducible from a clone with the command beside it.
|
|
|
459
449
|
|
|
460
450
|
| Evidence | Command |
|
|
461
451
|
| --- | --- |
|
|
462
|
-
|
|
|
463
|
-
|
|
|
452
|
+
| 82 engine tests: analysis, diagnostics, frameworks, seams | `node --test packages/engine/test/*.test.mjs` |
|
|
453
|
+
| The recorded corpus, proven against a second implementation | `node --test packages/engine/test/corpus.test.mjs` |
|
|
464
454
|
| 50 installer tests — including security and repeated-use stress | `npm test` |
|
|
465
455
|
| 21 evaluation cases, including deliberately dishonest outputs the scorer must reject | `npm run validate:evals` |
|
|
466
|
-
| 4 real AI-produced results, scored | `
|
|
456
|
+
| 4 real AI-produced results, scored | `node tests/evals/run-live.mjs --captures claude-opus-5` |
|
|
467
457
|
| 17 repository checks — including "documentation matches implementation" | `npm run validate:skills` … |
|
|
468
458
|
|
|
469
459
|
## Going deeper
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "qa-engineer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "A vendor-neutral Agent Skills pack that teaches AI coding agents to work like senior QA automation engineers.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -29,43 +29,34 @@
|
|
|
29
29
|
"packages/installer/schemas",
|
|
30
30
|
"packages/installer/package.json",
|
|
31
31
|
"packages/installer/README.md",
|
|
32
|
+
"packages/engine/lib",
|
|
33
|
+
"packages/engine/bin",
|
|
34
|
+
"packages/engine/package.json",
|
|
32
35
|
"skills",
|
|
33
|
-
"shared/analysis/lib/qa_analysis",
|
|
34
|
-
"shared/analysis/schemas",
|
|
35
|
-
"shared/diagnostics/lib/qa_diagnostics",
|
|
36
|
-
"shared/diagnostics/schemas",
|
|
37
|
-
"shared/tooling/qa_tool.py",
|
|
38
36
|
"shared/frameworks/registry.json",
|
|
39
37
|
"shared/frameworks/registry.schema.json",
|
|
40
38
|
"shared/frameworks/registry.mjs",
|
|
41
|
-
"shared/frameworks/playwright/lib/playwright_analysis.py",
|
|
42
|
-
"shared/frameworks/selenium/lib/selenium_analysis.py",
|
|
43
|
-
"shared/frameworks/cypress/lib/cypress_analysis.py",
|
|
44
|
-
"shared/frameworks/webdriverio/lib/webdriverio_analysis.py",
|
|
45
39
|
"COMPATIBILITY.md",
|
|
46
|
-
"!**/
|
|
47
|
-
"
|
|
48
|
-
"
|
|
40
|
+
"!**/tests",
|
|
41
|
+
"shared/tooling/qa-tool.mjs",
|
|
42
|
+
"!packages/engine/test"
|
|
49
43
|
],
|
|
50
44
|
"scripts": {
|
|
51
|
-
"eval:live": "
|
|
52
|
-
"prepublishOnly": "npm run validate:release && npm run validate:doc-claims && npm run validate:docs-commands && npm test && npm run validate:
|
|
45
|
+
"eval:live": "node tests/evals/run-live.mjs --baseline tests/evals/baselines/reference.json",
|
|
46
|
+
"prepublishOnly": "npm run validate:release && npm run validate:doc-claims && npm run validate:docs-commands && npm test && npm run validate:evals",
|
|
53
47
|
"qa": "node packages/installer/bin/qa.mjs",
|
|
54
48
|
"release:checksums": "node scripts/release/release-notes.mjs --checksums",
|
|
55
49
|
"release:notes": "node scripts/release/release-notes.mjs",
|
|
56
50
|
"test": "node --test packages/installer/test/*.test.mjs packages/engine/test/*.test.mjs",
|
|
57
51
|
"validate:architecture": "node scripts/check-architecture-fitness.mjs",
|
|
58
52
|
"validate:branding": "node scripts/check-branding.mjs",
|
|
59
|
-
"validate:bundle": "python3 scripts/bundle_python.py --check",
|
|
60
|
-
"validate:context": "cd shared/analysis/lib && python3 -m qa_analysis.cli context --path tests/fixtures/valid-context.md",
|
|
61
53
|
"validate:doc-claims": "node scripts/check-doc-claims.mjs",
|
|
62
54
|
"validate:docs-commands": "node scripts/check-docs-commands.mjs",
|
|
63
|
-
"validate:engine
|
|
64
|
-
"validate:evals": "
|
|
55
|
+
"validate:engine": "node --test packages/engine/test/*.test.mjs",
|
|
56
|
+
"validate:evals": "node tests/evals/run-evals.mjs",
|
|
65
57
|
"validate:keywords": "node scripts/check-keywords.mjs",
|
|
66
58
|
"validate:knowledge": "node scripts/check-knowledge.mjs",
|
|
67
59
|
"validate:matrix": "node scripts/check-capability-matrix.mjs",
|
|
68
|
-
"validate:python": "python3 shared/analysis/lib/run_tests.py",
|
|
69
60
|
"validate:registry": "node scripts/check-framework-registry.mjs",
|
|
70
61
|
"validate:release": "node scripts/release/validate-release.mjs",
|
|
71
62
|
"validate:release-notes": "node scripts/release/release-notes.mjs --check",
|
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The deterministic engine's command line — how a skill reaches it.
|
|
3
|
+
//
|
|
4
|
+
// One entry point for all three tool groups, replacing three Python CLIs and a
|
|
5
|
+
// launcher. Every subcommand writes JSON to stdout, so an agent never has to invent
|
|
6
|
+
// glue code, and the shapes are checked rather than assumed.
|
|
7
|
+
//
|
|
8
|
+
// qa-engine analysis <subcommand> [args]
|
|
9
|
+
// qa-engine diagnostics <subcommand> [args]
|
|
10
|
+
// qa-engine playwright <subcommand> [args]
|
|
11
|
+
//
|
|
12
|
+
// Exit codes are the contract, not an implementation detail. A skill's fallback
|
|
13
|
+
// logic reads them:
|
|
14
|
+
//
|
|
15
|
+
// 0 success
|
|
16
|
+
// 1 a document was read fine but failed its contract (`validate`, `context`)
|
|
17
|
+
// 2 unreadable input, malformed JSON, a failed seam contract, or bad usage
|
|
18
|
+
//
|
|
19
|
+
// Nothing here reasons: it reads files, calls the engine, prints the result. That
|
|
20
|
+
// separation is the whole architecture — deterministic code owns the facts, the
|
|
21
|
+
// model owns the explanation.
|
|
22
|
+
|
|
23
|
+
import fs from 'node:fs';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
|
|
26
|
+
import * as junit from '../lib/analysis/junit.mjs';
|
|
27
|
+
import * as har from '../lib/analysis/har.mjs';
|
|
28
|
+
import * as discovery from '../lib/analysis/discovery.mjs';
|
|
29
|
+
import * as diffGuard from '../lib/analysis/diff-guard.mjs';
|
|
30
|
+
import * as redaction from '../lib/analysis/redaction.mjs';
|
|
31
|
+
import * as contracts from '../lib/analysis/contracts.mjs';
|
|
32
|
+
import * as taxonomy from '../lib/analysis/taxonomy.mjs';
|
|
33
|
+
import * as contextModule from '../lib/analysis/context.mjs';
|
|
34
|
+
import * as branding from '../lib/analysis/branding.mjs';
|
|
35
|
+
import * as reportHtml from '../lib/analysis/report-html.mjs';
|
|
36
|
+
import * as diagnostics from '../lib/diagnostics/engine.mjs';
|
|
37
|
+
import {
|
|
38
|
+
InternalContractError, validateAnalysisResult, validateExecutionResultMin, validateDiagnosis,
|
|
39
|
+
} from '../lib/diagnostics/internal-contracts.mjs';
|
|
40
|
+
import * as playwright from '../lib/frameworks/playwright.mjs';
|
|
41
|
+
import * as junitFrameworks from '../lib/frameworks/junit-frameworks.mjs';
|
|
42
|
+
|
|
43
|
+
const USAGE = `usage: qa-engine <tool> <subcommand> [args]
|
|
44
|
+
|
|
45
|
+
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
46
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
47
|
+
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
48
|
+
playwright normalize a Playwright report or summarize a trace
|
|
49
|
+
|
|
50
|
+
analysis subcommands
|
|
51
|
+
junit <report.xml> normalized {tests, executed}
|
|
52
|
+
har <file.har> [--slow-ms N] redacted network summary
|
|
53
|
+
discover [--root DIR] [--path P ...] artifacts found, by state
|
|
54
|
+
diff-guard <diff-file> {issues, safe} — safe:false blocks a change
|
|
55
|
+
redact <file> the file's text with secrets masked
|
|
56
|
+
validate <instance.json> <schema.json> {valid, errors}; exit 1 when invalid
|
|
57
|
+
classify "<message>" [--http-status N] {classification, confidence, reason}
|
|
58
|
+
context [--root DIR] [--path P] parsed, schema-checked project context
|
|
59
|
+
report-html <result.json> [--out FILE] a self-contained HTML report
|
|
60
|
+
branding [--format html|markdown|text] the exact footer bytes for a report
|
|
61
|
+
|
|
62
|
+
diagnostics subcommands
|
|
63
|
+
diagnose --execution-result P [--analysis-result P]
|
|
64
|
+
plan-repairs --diagnosis P
|
|
65
|
+
summarize --execution-result P --diagnosis P
|
|
66
|
+
report --execution-result P [--analysis-result P] all three in one call
|
|
67
|
+
|
|
68
|
+
playwright subcommands
|
|
69
|
+
report <results.json> the same shape as analysis junit
|
|
70
|
+
trace <trace.zip> actions, errors, and a classification
|
|
71
|
+
|
|
72
|
+
examples
|
|
73
|
+
qa-engine analysis junit test-results/results.xml
|
|
74
|
+
qa-engine analysis report-html qa-artifacts/explore-result.json --out report.html
|
|
75
|
+
qa-engine diagnostics report --execution-result qa-artifacts/run.json
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
function emit(value) {
|
|
79
|
+
process.stdout.write(`${JSON.stringify(value, sortedReplacer(), 2)}\n`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Keys are emitted in sorted order, matching the Python CLIs' `sort_keys=True`.
|
|
84
|
+
* The output is read by people and diffed in CI, so a stable order is part of it.
|
|
85
|
+
*/
|
|
86
|
+
function sortedReplacer() {
|
|
87
|
+
return function replacer(key, value) {
|
|
88
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) return value;
|
|
89
|
+
return Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function fail(error, detail) {
|
|
94
|
+
emit({ error, detail });
|
|
95
|
+
return 2;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function readJson(file) {
|
|
99
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Flags, parsed the way the Python argparse CLIs accepted them. */
|
|
103
|
+
function parseFlags(argv, { repeatable = [] } = {}) {
|
|
104
|
+
const flags = {};
|
|
105
|
+
const positional = [];
|
|
106
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
107
|
+
const token = argv[index];
|
|
108
|
+
if (!token.startsWith('--')) {
|
|
109
|
+
positional.push(token);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
const [name, inline] = token.slice(2).split('=', 2);
|
|
113
|
+
const camel = name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
114
|
+
const value = inline !== undefined ? inline : argv[++index];
|
|
115
|
+
if (repeatable.includes(camel)) {
|
|
116
|
+
flags[camel] = [...(flags[camel] ?? []), value];
|
|
117
|
+
} else {
|
|
118
|
+
flags[camel] = value;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return { flags, positional };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Which flags may appear more than once, per subcommand. `discover` accepts a list
|
|
125
|
+
// of explicit paths; `context` names one file. Treating `--path` as repeatable
|
|
126
|
+
// everywhere handed `context` an array and it failed on "path must be a string".
|
|
127
|
+
const REPEATABLE = { discover: ['path'] };
|
|
128
|
+
|
|
129
|
+
function analysis(argv) {
|
|
130
|
+
const [subcommand, ...rest] = argv;
|
|
131
|
+
const { flags, positional } = parseFlags(rest, { repeatable: REPEATABLE[subcommand] ?? [] });
|
|
132
|
+
|
|
133
|
+
switch (subcommand) {
|
|
134
|
+
case 'junit':
|
|
135
|
+
requirePositional(positional, 1, 'junit <report.xml>');
|
|
136
|
+
emit(junit.parseJUnit(positional[0]));
|
|
137
|
+
return 0;
|
|
138
|
+
|
|
139
|
+
case 'har':
|
|
140
|
+
requirePositional(positional, 1, 'har <file.har>');
|
|
141
|
+
emit(har.parseHar(positional[0], { slowMs: Number(flags.slowMs ?? 1000) }));
|
|
142
|
+
return 0;
|
|
143
|
+
|
|
144
|
+
case 'discover':
|
|
145
|
+
emit(discovery.discover({ root: flags.root ?? '.', explicit: flags.path ?? null }));
|
|
146
|
+
return 0;
|
|
147
|
+
|
|
148
|
+
case 'diff-guard': {
|
|
149
|
+
requirePositional(positional, 1, 'diff-guard <diff-file>');
|
|
150
|
+
const issues = diffGuard.checkDiff(fs.readFileSync(positional[0], 'utf8'));
|
|
151
|
+
emit({ issues, safe: !issues.some((issue) => issue.severity === 'high') });
|
|
152
|
+
return 0;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
case 'redact':
|
|
156
|
+
requirePositional(positional, 1, 'redact <file>');
|
|
157
|
+
process.stdout.write(redaction.redactText(fs.readFileSync(positional[0], 'utf8')));
|
|
158
|
+
return 0;
|
|
159
|
+
|
|
160
|
+
case 'validate': {
|
|
161
|
+
requirePositional(positional, 2, 'validate <instance.json> <schema.json>');
|
|
162
|
+
const errors = contracts.validate(readJson(positional[0]), readJson(positional[1]));
|
|
163
|
+
emit({ valid: errors.length === 0, errors });
|
|
164
|
+
return errors.length === 0 ? 0 : 1;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
case 'classify': {
|
|
168
|
+
requirePositional(positional, 1, 'classify "<message>"');
|
|
169
|
+
const status = flags.httpStatus === undefined ? null : Number(flags.httpStatus);
|
|
170
|
+
emit(taxonomy.classify(positional[0], status));
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
case 'context': {
|
|
175
|
+
const file = flags.path ?? path.join(flags.root ?? '.', '.qa', 'context.md');
|
|
176
|
+
const schema = contextSchema();
|
|
177
|
+
const parsed = contextModule.parseFile(file, { schema });
|
|
178
|
+
emit({
|
|
179
|
+
path: file,
|
|
180
|
+
context: parsed.context,
|
|
181
|
+
valid: parsed.valid,
|
|
182
|
+
errors: parsed.errors,
|
|
183
|
+
schemaChecked: schema !== null,
|
|
184
|
+
});
|
|
185
|
+
return parsed.valid ? 0 : 1;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
case 'report-html': {
|
|
189
|
+
requirePositional(positional, 1, 'report-html <result.json>');
|
|
190
|
+
const document = reportHtml.renderFile(positional[0], { title: flags.title ?? null });
|
|
191
|
+
if (flags.out) {
|
|
192
|
+
fs.writeFileSync(flags.out, document);
|
|
193
|
+
emit({ written: flags.out, bytes: document.length });
|
|
194
|
+
} else {
|
|
195
|
+
process.stdout.write(document);
|
|
196
|
+
}
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
case 'branding':
|
|
201
|
+
// Written to stdout verbatim, not as JSON: the caller embeds these exact
|
|
202
|
+
// bytes in a rendered report.
|
|
203
|
+
if (flags.metadata !== undefined) emit(branding.metadata());
|
|
204
|
+
else process.stdout.write(branding.footer(flags.format ?? 'text'));
|
|
205
|
+
return 0;
|
|
206
|
+
|
|
207
|
+
default:
|
|
208
|
+
throw new UsageError(`unknown analysis subcommand: ${subcommand ?? '(none)'}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function diagnosticsCommand(argv) {
|
|
213
|
+
const [subcommand, ...rest] = argv;
|
|
214
|
+
const { flags } = parseFlags(rest);
|
|
215
|
+
|
|
216
|
+
const execution = () => {
|
|
217
|
+
if (!flags.executionResult) throw new UsageError('--execution-result is required');
|
|
218
|
+
return validateExecutionResultMin(readJson(flags.executionResult));
|
|
219
|
+
};
|
|
220
|
+
const analysisResult = () =>
|
|
221
|
+
flags.analysisResult ? validateAnalysisResult(readJson(flags.analysisResult)) : null;
|
|
222
|
+
// A diagnosis handed back in is held to the same seam contract it was emitted
|
|
223
|
+
// under, so a hand-edited file is refused rather than half-understood.
|
|
224
|
+
const diagnosisInput = () => {
|
|
225
|
+
if (!flags.diagnosis) throw new UsageError('--diagnosis is required');
|
|
226
|
+
return validateDiagnosis(readJson(flags.diagnosis));
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
switch (subcommand) {
|
|
230
|
+
case 'diagnose':
|
|
231
|
+
emit(diagnostics.diagnose(execution(), { analysisResult: analysisResult() }));
|
|
232
|
+
return 0;
|
|
233
|
+
|
|
234
|
+
case 'plan-repairs':
|
|
235
|
+
// Wrapped in {plans}, matching the shape skills already parse. A bare array
|
|
236
|
+
// would be tidier and would break every caller.
|
|
237
|
+
emit({ plans: diagnostics.planRepairs(diagnosisInput()) });
|
|
238
|
+
return 0;
|
|
239
|
+
|
|
240
|
+
case 'summarize': {
|
|
241
|
+
const result = execution();
|
|
242
|
+
emit(diagnostics.summarize(result, diagnosisInput()));
|
|
243
|
+
return 0;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
case 'report': {
|
|
247
|
+
// The one-shot path: diagnose, plan, and summarize in a single call, so the
|
|
248
|
+
// common case is one command instead of three.
|
|
249
|
+
const result = execution();
|
|
250
|
+
const diagnosis = diagnostics.diagnose(result, { analysisResult: analysisResult() });
|
|
251
|
+
emit({
|
|
252
|
+
diagnosis,
|
|
253
|
+
plans: diagnostics.planRepairs(diagnosis),
|
|
254
|
+
summary: diagnostics.summarize(result, diagnosis),
|
|
255
|
+
});
|
|
256
|
+
return 0;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
default:
|
|
260
|
+
throw new UsageError(`unknown diagnostics subcommand: ${subcommand ?? '(none)'}`);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function playwrightCommand(argv) {
|
|
265
|
+
const [subcommand, ...rest] = argv;
|
|
266
|
+
const { positional } = parseFlags(rest);
|
|
267
|
+
if (subcommand === 'report') {
|
|
268
|
+
requirePositional(positional, 1, 'report <results.json>');
|
|
269
|
+
emit(playwright.parseReport(positional[0]));
|
|
270
|
+
return 0;
|
|
271
|
+
}
|
|
272
|
+
if (subcommand === 'trace') {
|
|
273
|
+
requirePositional(positional, 1, 'trace <trace.zip>');
|
|
274
|
+
emit(playwright.analyzeTrace(positional[0]));
|
|
275
|
+
return 0;
|
|
276
|
+
}
|
|
277
|
+
throw new UsageError(`unknown playwright subcommand: ${subcommand ?? '(none)'}`);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
class UsageError extends Error {}
|
|
281
|
+
|
|
282
|
+
function requirePositional(positional, count, shape) {
|
|
283
|
+
if (positional.length < count) throw new UsageError(`usage: qa-engine analysis ${shape}`);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** The context contract, in whichever layout the engine is running from. */
|
|
287
|
+
function contextSchema() {
|
|
288
|
+
const candidates = [new URL('../lib/analysis/schemas/context.schema.json', import.meta.url)];
|
|
289
|
+
for (const candidate of candidates) {
|
|
290
|
+
const file = candidate.pathname;
|
|
291
|
+
if (fs.existsSync(file)) return readJson(file);
|
|
292
|
+
}
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export function main(argv = process.argv.slice(2)) {
|
|
297
|
+
if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {
|
|
298
|
+
process.stdout.write(USAGE);
|
|
299
|
+
return argv.length === 0 ? 2 : 0;
|
|
300
|
+
}
|
|
301
|
+
if (argv[0] === '--version') {
|
|
302
|
+
process.stdout.write(`${engineVersion()}\n`);
|
|
303
|
+
return 0;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const [tool, ...rest] = argv;
|
|
307
|
+
try {
|
|
308
|
+
if (tool === 'analysis') return analysis(rest);
|
|
309
|
+
if (tool === 'diagnostics') return diagnosticsCommand(rest);
|
|
310
|
+
if (tool === 'playwright') return playwrightCommand(rest);
|
|
311
|
+
process.stderr.write(USAGE);
|
|
312
|
+
return fail('usage-error', `unknown tool: ${tool}`);
|
|
313
|
+
} catch (error) {
|
|
314
|
+
if (error instanceof junit.MalformedArtifact) return fail('malformed-artifact', error.message);
|
|
315
|
+
if (error instanceof contextModule.MalformedContext) return fail('malformed-context', error.message);
|
|
316
|
+
if (error instanceof reportHtml.ReportError) return fail('report-error', error.message);
|
|
317
|
+
if (error instanceof branding.BrandingError) return fail('branding-error', error.message);
|
|
318
|
+
if (error instanceof InternalContractError) return fail('invalid-payload', error.message);
|
|
319
|
+
if (error instanceof UsageError) return fail('usage-error', error.message);
|
|
320
|
+
if (error instanceof SyntaxError) return fail('io-error', error.message);
|
|
321
|
+
if (error.code === 'ENOENT' || error.code === 'EACCES' || error.code === 'EISDIR') {
|
|
322
|
+
return fail('io-error', error.message);
|
|
323
|
+
}
|
|
324
|
+
throw error;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function engineVersion() {
|
|
329
|
+
try {
|
|
330
|
+
return readJson(new URL('../package.json', import.meta.url).pathname).version;
|
|
331
|
+
} catch {
|
|
332
|
+
return '0.0.0';
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
process.exitCode = main();
|