qa-engineer 0.9.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +26 -12
- package/README.md +182 -84
- package/package.json +11 -20
- package/packages/engine/bin/qa-engine.mjs +560 -0
- package/packages/engine/lib/analysis/branding.mjs +187 -0
- package/packages/engine/lib/analysis/context.mjs +294 -0
- package/packages/engine/lib/analysis/contracts.mjs +149 -0
- package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
- package/packages/engine/lib/analysis/discovery.mjs +220 -0
- package/packages/engine/lib/analysis/evidence.mjs +116 -0
- package/packages/engine/lib/analysis/har.mjs +124 -0
- package/packages/engine/lib/analysis/junit.mjs +126 -0
- package/packages/engine/lib/analysis/network.mjs +237 -0
- package/packages/engine/lib/analysis/redaction.mjs +127 -0
- package/packages/engine/lib/analysis/report-html.mjs +76 -0
- package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
- package/packages/engine/lib/analysis/xml.mjs +153 -0
- package/packages/engine/lib/analysis/zip.mjs +107 -0
- package/packages/engine/lib/artifacts/manager.mjs +453 -0
- package/packages/engine/lib/artifacts/mime.mjs +109 -0
- package/packages/engine/lib/artifacts/zip-write.mjs +143 -0
- package/packages/engine/lib/diagnostics/engine.mjs +165 -0
- package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
- package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
- package/packages/engine/lib/diagnostics/repair.mjs +73 -0
- package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
- package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
- package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
- package/packages/engine/lib/frameworks/playwright.mjs +158 -0
- package/packages/engine/lib/report/components/charts.mjs +424 -0
- package/packages/engine/lib/report/components/evidence.mjs +207 -0
- package/packages/engine/lib/report/components/findings.mjs +258 -0
- package/packages/engine/lib/report/components/nav.mjs +99 -0
- package/packages/engine/lib/report/components/primitives.mjs +246 -0
- package/packages/engine/lib/report/components/runtime.mjs +246 -0
- package/packages/engine/lib/report/components/timeline.mjs +65 -0
- package/packages/engine/lib/report/core/model.mjs +270 -0
- package/packages/engine/lib/report/core/normalize.mjs +226 -0
- package/packages/engine/lib/report/core/sections.mjs +978 -0
- package/packages/engine/lib/report/export/bundle.mjs +293 -0
- package/packages/engine/lib/report/export/html.mjs +183 -0
- package/packages/engine/lib/report/export/machine.mjs +290 -0
- package/packages/engine/lib/report/export/markdown.mjs +323 -0
- package/packages/engine/lib/report/schemas/qa-report.schema.json +555 -0
- package/packages/engine/lib/report/theme/css.mjs +529 -0
- package/packages/engine/lib/report/theme/tokens.mjs +137 -0
- package/packages/engine/lib/report/version.mjs +78 -0
- package/packages/engine/package.json +14 -0
- package/packages/installer/lib/agents/targets.mjs +90 -0
- package/packages/installer/lib/agents/user-level.mjs +80 -0
- package/packages/installer/lib/cli/flags.mjs +20 -2
- package/packages/installer/lib/commands/doctor.mjs +13 -20
- package/packages/installer/lib/commands/install.mjs +160 -91
- package/packages/installer/lib/commands/repair.mjs +10 -6
- package/packages/installer/lib/commands/self-test.mjs +4 -3
- package/packages/installer/lib/commands/uninstall.mjs +14 -8
- package/packages/installer/lib/commands/update.mjs +9 -4
- package/packages/installer/lib/commands/verify.mjs +13 -12
- package/packages/installer/lib/constants.mjs +13 -0
- package/packages/installer/lib/core/bundle.mjs +69 -92
- package/packages/installer/lib/core/conflict.mjs +5 -4
- package/packages/installer/lib/core/fs-safe.mjs +146 -6
- package/packages/installer/lib/core/integrity.mjs +59 -0
- package/packages/installer/lib/core/lockfile.mjs +19 -3
- package/packages/installer/lib/core/manifest.mjs +48 -57
- package/packages/installer/lib/core/plan.mjs +213 -0
- package/packages/installer/lib/core/qa-home.mjs +145 -0
- package/packages/installer/lib/core/scope.mjs +274 -0
- package/packages/installer/lib/core/validate-install.mjs +49 -31
- package/packages/installer/package.json +1 -1
- package/packages/installer/schemas/qa-lock.schema.json +119 -21
- package/shared/tooling/qa-tool.mjs +161 -0
- package/skills/qa-api/SKILL.md +5 -5
- package/skills/qa-api/references/deterministic-tooling.md +34 -32
- package/skills/qa-api/references/evidence-and-reporting.md +5 -5
- package/skills/qa-api/scripts/qa-tool.mjs +162 -0
- package/skills/qa-audit/SKILL.md +5 -5
- package/skills/qa-audit/references/deterministic-tooling.md +34 -32
- package/skills/qa-audit/references/evidence-and-reporting.md +5 -5
- package/skills/qa-audit/scripts/qa-tool.mjs +162 -0
- package/skills/qa-debug/SKILL.md +6 -6
- package/skills/qa-debug/references/deterministic-tooling.md +34 -32
- package/skills/qa-debug/references/diagnostic-engine.md +3 -3
- package/skills/qa-debug/references/evidence-and-reporting.md +5 -5
- package/skills/qa-debug/scripts/qa-tool.mjs +162 -0
- package/skills/qa-explore/SKILL.md +31 -15
- package/skills/qa-explore/contracts/explore-result.schema.json +517 -11
- package/skills/qa-explore/references/api-replay.md +40 -1
- package/skills/qa-explore/references/deterministic-tooling.md +34 -32
- package/skills/qa-explore/references/evidence-and-reporting.md +5 -5
- package/skills/qa-explore/references/report-pipeline.md +266 -96
- package/skills/qa-explore/scripts/qa-tool.mjs +162 -0
- package/skills/qa-fix/SKILL.md +4 -4
- package/skills/qa-fix/references/deterministic-tooling.md +34 -32
- package/skills/qa-fix/references/diagnostic-engine.md +3 -3
- package/skills/qa-fix/references/evidence-and-reporting.md +5 -5
- package/skills/qa-fix/scripts/qa-tool.mjs +162 -0
- package/skills/qa-flaky/SKILL.md +4 -4
- package/skills/qa-flaky/references/deterministic-tooling.md +34 -32
- package/skills/qa-flaky/references/evidence-and-reporting.md +5 -5
- package/skills/qa-flaky/scripts/qa-tool.mjs +162 -0
- package/skills/qa-generate/references/evidence-and-reporting.md +5 -5
- package/skills/qa-init/SKILL.md +4 -4
- package/skills/qa-init/references/deterministic-tooling.md +34 -32
- package/skills/qa-init/references/evidence-and-reporting.md +5 -5
- package/skills/qa-init/scripts/qa-tool.mjs +162 -0
- package/skills/qa-report/SKILL.md +7 -7
- package/skills/qa-report/references/deterministic-tooling.md +34 -32
- package/skills/qa-report/references/diagnostic-engine.md +3 -3
- package/skills/qa-report/references/evidence-and-reporting.md +5 -5
- package/skills/qa-report/scripts/qa-tool.mjs +162 -0
- package/skills/qa-review/references/evidence-and-reporting.md +5 -5
- package/skills/qa-run/SKILL.md +6 -6
- package/skills/qa-run/references/deterministic-tooling.md +34 -32
- package/skills/qa-run/references/evidence-and-reporting.md +5 -5
- package/skills/qa-run/scripts/qa-tool.mjs +162 -0
- package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
- package/shared/analysis/lib/qa_analysis/branding.py +0 -175
- package/shared/analysis/lib/qa_analysis/cli.py +0 -144
- package/shared/analysis/lib/qa_analysis/context.py +0 -233
- package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
- package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
- package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
- package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
- package/shared/analysis/lib/qa_analysis/har.py +0 -87
- package/shared/analysis/lib/qa_analysis/junit.py +0 -104
- package/shared/analysis/lib/qa_analysis/redaction.py +0 -107
- package/shared/analysis/lib/qa_analysis/report_html.py +0 -781
- package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
- package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
- package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
- package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
- package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
- package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
- package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
- package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
- package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
- package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
- package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
- package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
- package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
- package/shared/tooling/qa_tool.py +0 -127
- /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
- /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
package/COMPATIBILITY.md
CHANGED
|
@@ -9,7 +9,8 @@ This document records which AI coding agents the pack targets, how each one disc
|
|
|
9
9
|
| Pack version | Released | Discovery paths verified against agent docs | Live agent runs |
|
|
10
10
|
| --- | --- | --- | --- |
|
|
11
11
|
| 0.9.0, 0.9.1 | 2026-07-28 | Not systematically re-read for the release | Claude Code, Cursor (manual, by the maintainer) |
|
|
12
|
-
| 0.9.2 | 2026-07-28 | Yes — every host's own documentation, re-read on 2026-07-28 (sources in the matrix below) |
|
|
12
|
+
| 0.9.2 | 2026-07-28 | Yes — every host's own documentation, re-read on 2026-07-28 (sources in the matrix below) | **Do not use.** The tarball omitted the engine; every command failed on a missing module |
|
|
13
|
+
| 0.9.3 | 2026-07-28 | Yes, as 0.9.2 | Claude Code, Cursor (manual, by the maintainer) |
|
|
13
14
|
|
|
14
15
|
"Verified" in the third column means the discovery paths in the matrix below were read off each host's own documentation on that date and are asserted against the installer by `packages/installer/test/hosts.test.mjs`, which fails if the installer writes to a path that host does not document reading. It does **not** mean the pack was executed inside every host: that is the Tier 1 bar, and only Claude Code and Cursor have had a real run.
|
|
15
16
|
|
|
@@ -55,8 +56,14 @@ The tier column in the matrix above is the **planned** tier — the level each a
|
|
|
55
56
|
|
|
56
57
|
| Requirement | Status | Used for |
|
|
57
58
|
| --- | --- | --- |
|
|
58
|
-
|
|
|
59
|
-
|
|
59
|
+
| Node.js 18.17+ | The only requirement | Everything: `npx qa-engineer` and the CLI, and the deterministic engine the skills run (`packages/engine/`) |
|
|
60
|
+
|
|
61
|
+
**One runtime, no dependencies.** Until 0.9.2 the engine was Python, which meant a
|
|
62
|
+
second runtime users had to install for tooling they had installed with Node — and
|
|
63
|
+
when it was missing, skills fell back to model guesswork and said nothing.
|
|
64
|
+
[ADR-0012](docs/architecture/ADR-0012-node-engine.md) records why that changed and
|
|
65
|
+
how the migration was verified. The engine still takes no third-party dependencies:
|
|
66
|
+
the XML and ZIP readers it needs are written out rather than depended on.
|
|
60
67
|
|
|
61
68
|
## Operating systems
|
|
62
69
|
|
|
@@ -66,15 +73,22 @@ The tier column in the matrix above is the **planned** tier — the level each a
|
|
|
66
73
|
| macOS | Expected to work; not covered by CI |
|
|
67
74
|
| Windows | Expected to work; **not yet verified end to end** |
|
|
68
75
|
|
|
69
|
-
The installer uses Node's platform-independent path handling, and the skills
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
The installer uses Node's platform-independent path handling, and the skills invoke
|
|
77
|
+
the engine through a committed launcher (`scripts/qa-tool.mjs`) that needs no shell
|
|
78
|
+
features — the command shape is identical in bash, zsh, PowerShell, and cmd.exe,
|
|
79
|
+
with no platform difference at all.
|
|
80
|
+
|
|
81
|
+
The Windows row said "expected" for a specific reason that no longer applies: the
|
|
82
|
+
engine was Python, and `python3` is not on PATH on Windows by default, so every
|
|
83
|
+
deterministic call there failed and each skill quietly fell back to guesswork. The
|
|
84
|
+
engine is now the same Node that ran the install. The row still says "expected"
|
|
85
|
+
rather than "supported" because nobody has run the pack end to end on Windows —
|
|
86
|
+
that is an unverified claim, not a known gap.
|
|
87
|
+
|
|
88
|
+
Requirements are declared per skill via the specification's `compatibility`
|
|
89
|
+
frontmatter field, and the pack degrades gracefully: skills describe a manual
|
|
90
|
+
fallback whenever a runtime or optional integration (such as an MCP server) is
|
|
91
|
+
unavailable.
|
|
78
92
|
|
|
79
93
|
## Reporting compatibility problems
|
|
80
94
|
|
package/README.md
CHANGED
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
> Teach your AI coding assistant to work like a senior QA engineer — and stop it from telling you tests pass when they don't.
|
|
4
4
|
|
|
5
|
-
[](CHANGELOG.md)
|
|
6
6
|
[](docs/release/v0.9-release-checklist.md)
|
|
7
7
|
[](LICENSE)
|
|
8
|
-
[](#step-1--check-your-
|
|
9
|
-
[](#step-1--check-your-prerequisites)
|
|
8
|
+
[](#step-1--check-your-prerequisite)
|
|
10
9
|
[](https://agentskills.io)
|
|
11
10
|
[](https://github.com/abisheik88/qa-engineer/actions/workflows/ci.yml)
|
|
12
|
-
[](#how-this-is-verified)
|
|
13
12
|
|
|
14
13
|
**This page is everything you need.** Install, first run, every command, and what to do when something breaks — no other document required.
|
|
15
14
|
|
|
@@ -19,7 +18,7 @@
|
|
|
19
18
|
|
|
20
19
|
1. [What this is, in plain language](#what-this-is-in-plain-language)
|
|
21
20
|
2. [Is this for me?](#is-this-for-me)
|
|
22
|
-
3. [Step 1 — Check your prerequisites](#step-1--check-your-
|
|
21
|
+
3. [Step 1 — Check your prerequisites](#step-1--check-your-prerequisite)
|
|
23
22
|
4. [Step 2 — Install it](#step-2--install-it)
|
|
24
23
|
5. [Step 3 — Confirm it worked](#step-3--confirm-it-worked)
|
|
25
24
|
6. [Step 4 — Your first real task](#step-4--your-first-real-task)
|
|
@@ -46,13 +45,13 @@ If you use an **AI coding assistant** — Claude Code, Cursor, GitHub Copilot, C
|
|
|
46
45
|
|
|
47
46
|
That last one is the dangerous part. A green test suite that checks nothing is worse than a red one, because you stop looking.
|
|
48
47
|
|
|
49
|
-
**This project installs
|
|
48
|
+
**This project installs thirteen QA skills** — twelve commands plus a worked example — that your AI assistant reads and follows. Think of it as handing your assistant a senior QA engineer's playbook, plus a set of tools it must actually run — so its answers come from real measurements instead of guesses.
|
|
50
49
|
|
|
51
50
|
<details>
|
|
52
51
|
<summary><b>New to this? Three terms explained</b></summary>
|
|
53
52
|
|
|
54
53
|
- **AI coding assistant / agent** — a tool where you chat with an AI inside your codebase, and it can read and edit files. Claude Code, Cursor, and GitHub Copilot are examples.
|
|
55
|
-
- **Skill** — a Markdown file with instructions your assistant reads when a task matches. It's like a checklist the AI follows. This project installs
|
|
54
|
+
- **Skill** — a Markdown file with instructions your assistant reads when a task matches. It's like a checklist the AI follows. This project installs thirteen of them — twelve commands you use, plus one worked example.
|
|
56
55
|
- **Slash command** — how you trigger a skill: you type `/qa-run` in your assistant's chat. If your assistant doesn't support slash commands, plain English works too ("run my tests and report the result").
|
|
57
56
|
|
|
58
57
|
You do not need to learn a new language, framework, or config file. You install once and then talk to your assistant normally.
|
|
@@ -67,21 +66,26 @@ You do not need to learn a new language, framework, or config file. You install
|
|
|
67
66
|
|
|
68
67
|
**Works best with Playwright** today. Selenium, Cypress, and WebdriverIO are detected and their results are understood, but running and generating tests live is currently Playwright-only. See [framework support](#framework-support).
|
|
69
68
|
|
|
70
|
-
## Step 1 — Check your
|
|
69
|
+
## Step 1 — Check your prerequisite
|
|
71
70
|
|
|
72
|
-
You need
|
|
71
|
+
You need one thing. Run this to check — copy and paste it exactly:
|
|
73
72
|
|
|
74
73
|
```bash
|
|
75
74
|
node --version
|
|
76
|
-
python3 --version
|
|
77
75
|
```
|
|
78
76
|
|
|
79
|
-
You want **Node 18.18 or newer
|
|
77
|
+
You want **Node 18.18 or newer**. If it prints a high enough version, skip ahead to
|
|
78
|
+
[Step 2](#step-2--install-it).
|
|
79
|
+
|
|
80
|
+
That is the whole list. There is nothing to `pip install`, no second language, and no
|
|
81
|
+
dependencies: the tools that read your test results are dependency-free JavaScript
|
|
82
|
+
that runs on the same Node you just checked.
|
|
80
83
|
|
|
81
84
|
<details>
|
|
82
85
|
<summary><b>"command not found" — how to install Node.js</b></summary>
|
|
83
86
|
|
|
84
|
-
Node.js gives you the `
|
|
87
|
+
Node.js gives you the `npm` command used to install this pack, and it runs the
|
|
88
|
+
analysis tools afterwards.
|
|
85
89
|
|
|
86
90
|
- **macOS** — `brew install node` (needs [Homebrew](https://brew.sh)), or download from [nodejs.org](https://nodejs.org).
|
|
87
91
|
- **Windows** — download the LTS installer from [nodejs.org](https://nodejs.org) and run it.
|
|
@@ -91,86 +95,137 @@ Then re-run `node --version`. Take the **LTS** version if offered a choice.
|
|
|
91
95
|
|
|
92
96
|
</details>
|
|
93
97
|
|
|
94
|
-
<details>
|
|
95
|
-
<summary><b>"command not found" — how to install Python</b></summary>
|
|
96
|
-
|
|
97
|
-
Python runs the analysis tools that read your test results. It uses only Python's built-in library — there is nothing extra to install, no `pip install` step.
|
|
98
|
-
|
|
99
|
-
- **macOS** — `brew install python3`, or download from [python.org](https://python.org).
|
|
100
|
-
- **Windows** — install from [python.org](https://python.org) and **tick "Add Python to PATH"** during setup. Then use `python --version` if `python3` is not found.
|
|
101
|
-
- **Linux** — `sudo apt install python3` (Debian/Ubuntu).
|
|
102
|
-
|
|
103
|
-
**Can you skip Python?** Yes, but you shouldn't. Without it the skills fall back to the AI reading files by eye, and they will mark their results *degraded* to tell you so. The whole point of this project is that the numbers come from tools, not guesses.
|
|
104
|
-
|
|
105
|
-
</details>
|
|
106
|
-
|
|
107
98
|
You also need **an AI coding assistant** open on your project — Claude Code, Cursor, GitHub Copilot, Codex CLI, OpenCode, Gemini CLI, Antigravity, or Kimi.
|
|
108
99
|
|
|
109
100
|
## Step 2 — Install it
|
|
110
101
|
|
|
111
|
-
|
|
102
|
+
**Install once for your whole machine.** Every project then works with no further setup:
|
|
112
103
|
|
|
113
104
|
```bash
|
|
114
|
-
|
|
105
|
+
npm install -g qa-engineer
|
|
106
|
+
qa-engineer install --global
|
|
115
107
|
```
|
|
116
108
|
|
|
117
|
-
That's the
|
|
109
|
+
That's it. Open any project, and the QA commands are already there.
|
|
118
110
|
|
|
119
|
-
|
|
120
|
-
| --- | --- |
|
|
121
|
-
| `npx` | Comes with Node. Downloads and runs a tool without installing it permanently. |
|
|
122
|
-
| `qa-engineer` | The name of this package. |
|
|
123
|
-
| `--yes` | Don't ask me questions, just use sensible defaults. |
|
|
124
|
-
| `--project .` | Install into the current folder (`.` means "here"). |
|
|
111
|
+
What those two lines do, one at a time:
|
|
125
112
|
|
|
126
|
-
|
|
113
|
+
| Line | What it does |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `npm install -g qa-engineer` | Puts the `qa-engineer` command on your system, so you can type it anywhere. `-g` means "global" — available everywhere, not just in one folder. |
|
|
116
|
+
| `qa-engineer install --global` | Installs the QA skills your assistant reads, once, for every project on this machine. |
|
|
127
117
|
|
|
128
|
-
|
|
118
|
+
Here is what the second command prints:
|
|
129
119
|
|
|
130
120
|
```text
|
|
131
|
-
›
|
|
132
|
-
› agents:
|
|
121
|
+
› scope: global (/Users/you/.qa-engineer)
|
|
122
|
+
› agents: claude-code, antigravity
|
|
133
123
|
› skills: 13
|
|
124
|
+
› shared engine: /Users/you/.qa-engineer/engine
|
|
134
125
|
› ██████████████████ 100% Configuration complete
|
|
135
|
-
✓ installed
|
|
126
|
+
✓ installed 214 file(s) and 26 link(s); lockfile /Users/you/.qa-engineer/qa-lock.json
|
|
127
|
+
→ in Claude Code: type /qa-explore, or just describe the task — skills auto-activate
|
|
136
128
|
```
|
|
137
129
|
|
|
130
|
+
**Everything lives in one folder, `~/.qa-engineer/`.** Nothing is scattered around your
|
|
131
|
+
home directory, so removing it later is one clean step.
|
|
132
|
+
|
|
133
|
+
> **On "not installed for cursor, codex, …"** — you may see a note listing assistants
|
|
134
|
+
> that were skipped, each with a reason. That is expected and honest: those assistants
|
|
135
|
+
> only look for skills *inside a project*, so there is no machine-wide folder to put
|
|
136
|
+
> them in. For those, use a project install (below). Claude Code and Antigravity do have
|
|
137
|
+
> a machine-wide folder, so they are set up for you.
|
|
138
|
+
|
|
138
139
|
<details>
|
|
139
|
-
<summary><b>
|
|
140
|
+
<summary><b>Installing from a clone instead</b></summary>
|
|
140
141
|
|
|
141
|
-
|
|
142
|
+
To run the newest code, or to work on the pack itself, install from a clone rather than
|
|
143
|
+
from npm:
|
|
142
144
|
|
|
143
145
|
```bash
|
|
144
|
-
git clone
|
|
146
|
+
git clone https://github.com/abisheik88/qa-engineer.git
|
|
145
147
|
cd qa-engineer
|
|
146
148
|
npm install
|
|
147
|
-
npm
|
|
149
|
+
npm install -g . # puts `qa-engineer` on your PATH
|
|
150
|
+
qa-engineer install --global
|
|
148
151
|
```
|
|
149
152
|
|
|
150
|
-
|
|
153
|
+
Everything else in this README works exactly the same afterwards. Use `npm link` in
|
|
154
|
+
place of `npm install -g .` if you are editing the pack — it symlinks, so your changes
|
|
155
|
+
take effect without reinstalling.
|
|
156
|
+
|
|
157
|
+
</details>
|
|
158
|
+
|
|
159
|
+
### Three ways to install — pick one
|
|
160
|
+
|
|
161
|
+
Most people want the first. The other two exist for real situations, not as clutter.
|
|
162
|
+
|
|
163
|
+
| Command | Installs | Choose it when |
|
|
164
|
+
| --- | --- | --- |
|
|
165
|
+
| `qa-engineer install --global` | Once per machine, in `~/.qa-engineer/` | **The usual choice.** You want every project to just work. |
|
|
166
|
+
| `qa-engineer install --workspace` | Once at the top of a monorepo | You have many packages in one repository and want them to share an install. |
|
|
167
|
+
| `qa-engineer install --project .` | Into the current project only | The install should be committed to the repository and shared with your team. |
|
|
168
|
+
|
|
169
|
+
**They can coexist.** If a project has its own install, that one is used; otherwise the
|
|
170
|
+
machine-wide one is. This is the same rule you already rely on: a package installed in
|
|
171
|
+
your project beats the same package installed globally. So a repository can pin the
|
|
172
|
+
version it wants, and nothing else on your machine changes.
|
|
173
|
+
|
|
174
|
+
<details>
|
|
175
|
+
<summary><b>What a project install is for</b></summary>
|
|
176
|
+
|
|
177
|
+
A project install is **self-contained**: the analysis engine is copied inside each
|
|
178
|
+
skill, so the repository works on a machine where nothing else is installed. That costs
|
|
179
|
+
more disk, and it is exactly what you want when the install is committed to git and a
|
|
180
|
+
colleague clones the repository.
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
cd /path/to/your-project
|
|
184
|
+
qa-engineer install --project .
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
A machine-wide install shares one copy of the engine instead — about 2 MB total,
|
|
188
|
+
compared with 13 MB per project — which is why it is the default recommendation.
|
|
189
|
+
|
|
190
|
+
</details>
|
|
191
|
+
|
|
192
|
+
<details>
|
|
193
|
+
<summary><b>Extra options you may want later</b></summary>
|
|
194
|
+
|
|
195
|
+
| Option | What it does |
|
|
196
|
+
| --- | --- |
|
|
197
|
+
| `--all-agents` | Set up every assistant with a machine-wide skills folder, not only the ones detected here. |
|
|
198
|
+
| `--yes` | Don't ask any questions; use sensible defaults. Useful in scripts and CI. |
|
|
199
|
+
| `--dry-run` | Show what would change and write nothing. |
|
|
200
|
+
| `--force` | Overwrite files a previous install doesn't own. Backs them up first. |
|
|
201
|
+
| `QA_ENGINEER_HOME` | An environment variable that moves `~/.qa-engineer/` somewhere else entirely. |
|
|
151
202
|
|
|
152
203
|
</details>
|
|
153
204
|
|
|
154
205
|
## Step 3 — Confirm it worked
|
|
155
206
|
|
|
156
207
|
```bash
|
|
157
|
-
|
|
208
|
+
qa-engineer self-test --global
|
|
158
209
|
```
|
|
159
210
|
|
|
160
|
-
Real output from a healthy install:
|
|
211
|
+
Real output from a healthy machine-wide install:
|
|
161
212
|
|
|
162
213
|
```text
|
|
163
|
-
✓ [PASS] lockfile: qa-lock.json present (
|
|
214
|
+
✓ [PASS] lockfile: .qa-engineer/qa-lock.json present (240 files, pack 0.10.0)
|
|
164
215
|
✓ [PASS] integrity: all installed files match lockfile hashes
|
|
165
216
|
✓ [PASS] skills: 13 skill(s) installed
|
|
166
217
|
✓ [PASS] contracts: 12 contract schema(s) present
|
|
167
|
-
✓ [PASS] engine: deterministic engine
|
|
168
|
-
✓ [PASS]
|
|
218
|
+
✓ [PASS] engine: deterministic engine shared at .qa-engineer/engine
|
|
219
|
+
✓ [PASS] engine-runs: shared engine runs (node 24.18.0)
|
|
169
220
|
✓ [PASS] node: Node v24.18.0
|
|
170
221
|
✓ self-test PASSED
|
|
171
222
|
```
|
|
172
223
|
|
|
173
|
-
Every line `[PASS]`? You're done installing. If anything says `[FAIL]`, jump to
|
|
224
|
+
Every line `[PASS]`? You're done installing. If anything says `[FAIL]`, jump to
|
|
225
|
+
[When something goes wrong](#when-something-goes-wrong).
|
|
226
|
+
|
|
227
|
+
> Checking a **project** install instead? Drop the `--global`: run
|
|
228
|
+
> `qa-engineer self-test` from inside the project, and it finds the right one on its own.
|
|
174
229
|
|
|
175
230
|
## Step 4 — Your first real task
|
|
176
231
|
|
|
@@ -232,29 +287,51 @@ That's the core loop: **`/qa-init` once, then `/qa-run` and `/qa-debug` whenever
|
|
|
232
287
|
|
|
233
288
|
## What got installed
|
|
234
289
|
|
|
235
|
-
|
|
290
|
+
**With a machine-wide install**, everything lives in one folder:
|
|
291
|
+
|
|
292
|
+
```text
|
|
293
|
+
~/.qa-engineer/
|
|
294
|
+
├── engine/ ← the analysis tools, one shared copy
|
|
295
|
+
├── skills/ ← the thirteen skills your assistant reads
|
|
296
|
+
└── qa-lock.json ← every installed file and its checksum
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
Your assistant is pointed at those skills by a shortcut in its own folder
|
|
300
|
+
(`~/.claude/skills/` for Claude Code). **Your projects get nothing at all** until you
|
|
301
|
+
actually run a command — at which point you'll see:
|
|
236
302
|
|
|
237
303
|
```text
|
|
238
304
|
your-app/
|
|
239
|
-
├── .
|
|
240
|
-
|
|
241
|
-
│ ├── qa-debug/
|
|
242
|
-
│ └── …
|
|
243
|
-
├── .claude/skills/ ← same files again, only if you use Claude Code
|
|
244
|
-
├── qa-lock.json ← a list of every installed file and its checksum
|
|
245
|
-
└── (later) .qa/context.md and qa-artifacts/ ← created when you run the commands
|
|
305
|
+
├── .qa/context.md ← what this project is, written by /qa-init
|
|
306
|
+
└── qa-artifacts/ ← reports and evidence from the commands you run
|
|
246
307
|
```
|
|
247
308
|
|
|
248
|
-
|
|
309
|
+
<details>
|
|
310
|
+
<summary><b>What a project install puts in your repository instead</b></summary>
|
|
249
311
|
|
|
250
|
-
|
|
312
|
+
```text
|
|
313
|
+
your-app/
|
|
314
|
+
├── .agents/skills/ ← the thirteen skills your assistant reads
|
|
315
|
+
├── .claude/skills/ ← the same skills again, only if you use Claude Code
|
|
316
|
+
├── qa-lock.json ← every installed file and its checksum
|
|
317
|
+
└── (later) .qa/context.md and qa-artifacts/
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
Worth adding to your `.gitignore` if you don't want the outputs committed:
|
|
251
321
|
|
|
252
322
|
```gitignore
|
|
253
323
|
qa-artifacts/
|
|
254
324
|
.qa/backups/
|
|
255
325
|
```
|
|
256
326
|
|
|
257
|
-
Most teams **do** commit `.agents/skills/` and `qa-lock.json`, so everyone
|
|
327
|
+
Most teams **do** commit `.agents/skills/` and `qa-lock.json`, so everyone gets the same
|
|
328
|
+
skills from a single `git clone`.
|
|
329
|
+
|
|
330
|
+
</details>
|
|
331
|
+
|
|
332
|
+
**Nothing else is touched.** Your source code, your tests, and your config are left
|
|
333
|
+
exactly as they were. `qa-lock.json` records precisely what was written, so `uninstall`
|
|
334
|
+
removes that and nothing more.
|
|
258
335
|
|
|
259
336
|
## How you type a command in your assistant
|
|
260
337
|
|
|
@@ -323,36 +400,35 @@ reaches `/qa-explore` in all of them. Full details and the source for each path:
|
|
|
323
400
|
Start here. It fixes most problems:
|
|
324
401
|
|
|
325
402
|
```bash
|
|
326
|
-
|
|
403
|
+
qa-engineer doctor
|
|
327
404
|
```
|
|
328
405
|
|
|
329
406
|
`doctor` checks your environment and your install and prints a hint for anything it finds. Items marked `warn` with an "optional" hint are fine to ignore.
|
|
330
407
|
|
|
331
408
|
### The commands don't appear in my assistant
|
|
332
409
|
|
|
333
|
-
1. **
|
|
410
|
+
1. **Is it actually installed?** Run `qa-engineer self-test`. If it reports a missing lockfile, the install did not land where you think — re-run `qa-engineer install --global`.
|
|
334
411
|
2. **Restart your assistant.** Most only look for new skills at startup.
|
|
335
412
|
3. **Try plain English.** Instead of `/qa-run`, ask *"run my tests and report the result"*. Slash-command support varies by assistant; the skills work either way.
|
|
336
|
-
4. **Check your assistant reads the standard path.** Most read `.agents/skills/`. Claude Code uses `.claude/skills/`, which the installer also writes when it detects Claude. To force it: `
|
|
413
|
+
4. **Check your assistant reads the standard path.** Most read `.agents/skills/`. Claude Code uses `.claude/skills/`, which the installer also writes when it detects Claude. To force it: `qa-engineer install --global --agent claude-code`
|
|
337
414
|
|
|
338
415
|
### "refusing to overwrite N file(s) not owned by a previous install"
|
|
339
416
|
|
|
340
417
|
You already have a file where the pack wants to write one. It will not silently overwrite your work. Either move your file, or overwrite deliberately:
|
|
341
418
|
|
|
342
419
|
```bash
|
|
343
|
-
|
|
420
|
+
qa-engineer install --force
|
|
344
421
|
```
|
|
345
422
|
|
|
346
423
|
`--force` backs everything up to `.qa/backups/<timestamp>/` first.
|
|
347
424
|
|
|
348
425
|
### A skill says the engine is missing, or results are "degraded"
|
|
349
426
|
|
|
350
|
-
The
|
|
427
|
+
The deterministic engine isn't reachable. Fix in this order:
|
|
351
428
|
|
|
352
429
|
```bash
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
npx qa-engineer doctor --project . # should now say "bundled engine runs cleanly"
|
|
430
|
+
qa-engineer repair # reinstall the analysis tools
|
|
431
|
+
qa-engineer doctor # should now report the engine runs
|
|
356
432
|
```
|
|
357
433
|
|
|
358
434
|
"Degraded" is not a bug — it is the skill telling you it could not run a tool and therefore trusts its own answer less. That's the honest behaviour.
|
|
@@ -362,7 +438,7 @@ npx qa-engineer doctor --project . # should now say "bundled engine runs
|
|
|
362
438
|
Someone edited an installed skill file. Restore them:
|
|
363
439
|
|
|
364
440
|
```bash
|
|
365
|
-
|
|
441
|
+
qa-engineer repair
|
|
366
442
|
```
|
|
367
443
|
|
|
368
444
|
If the edit was deliberate, note that `update` will overwrite it again — keep customisations outside `.agents/skills/`.
|
|
@@ -374,7 +450,7 @@ That is the exact failure this project exists to prevent, and it has three defen
|
|
|
374
450
|
### Still stuck
|
|
375
451
|
|
|
376
452
|
```bash
|
|
377
|
-
|
|
453
|
+
qa-engineer doctor --json
|
|
378
454
|
```
|
|
379
455
|
|
|
380
456
|
Open an issue and paste that output. It answers most questions before they're asked.
|
|
@@ -382,12 +458,34 @@ Open an issue and paste that output. It answers most questions before they're as
|
|
|
382
458
|
## Updating and uninstalling
|
|
383
459
|
|
|
384
460
|
```bash
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
461
|
+
qa-engineer update # refresh to the current version
|
|
462
|
+
qa-engineer verify # check nothing was corrupted
|
|
463
|
+
qa-engineer doctor # diagnose problems and suggest fixes
|
|
464
|
+
qa-engineer repair # reinstall anything missing or damaged
|
|
465
|
+
qa-engineer uninstall # remove everything it installed
|
|
388
466
|
```
|
|
389
467
|
|
|
390
|
-
|
|
468
|
+
**You don't have to say which install you mean.** Each command looks for a project
|
|
469
|
+
install where you're standing, and falls back to the machine-wide one. Add `--global` or
|
|
470
|
+
`--project .` when you want to be explicit.
|
|
471
|
+
|
|
472
|
+
`uninstall` removes exactly the files listed in `qa-lock.json`, backs each one up first,
|
|
473
|
+
and leaves everything else alone. If you edited an installed file it stops and tells you
|
|
474
|
+
rather than destroying your change; `--force` proceeds anyway. Add `--dry-run` to any
|
|
475
|
+
command to see what it *would* do without doing it.
|
|
476
|
+
|
|
477
|
+
<details>
|
|
478
|
+
<summary><b>Removing it completely</b></summary>
|
|
479
|
+
|
|
480
|
+
```bash
|
|
481
|
+
qa-engineer uninstall --global # the machine-wide install
|
|
482
|
+
npm uninstall -g qa-engineer # the qa-engineer command itself
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
Run `qa-engineer uninstall --project .` inside any repository that has its own install
|
|
486
|
+
first — those are separate and are not removed by the global uninstall.
|
|
487
|
+
|
|
488
|
+
</details>
|
|
391
489
|
|
|
392
490
|
## How it works
|
|
393
491
|
|
|
@@ -409,7 +507,7 @@ The core idea in one line: **tools produce the facts, the AI explains them.**
|
|
|
409
507
|
│ raw output, exit code
|
|
410
508
|
▼
|
|
411
509
|
┌───────────────────────────────────┐
|
|
412
|
-
│ Bundled
|
|
510
|
+
│ Bundled Node engine │ count the results, classify the failure,
|
|
413
511
|
│ (installed inside the skill) │ redact secrets — no guessing
|
|
414
512
|
└───────────────────────────────────┘
|
|
415
513
|
│ facts
|
|
@@ -449,7 +547,7 @@ Stated plainly, so nothing surprises you:
|
|
|
449
547
|
- **It is not a test runner.** It drives Playwright; it doesn't replace it.
|
|
450
548
|
- **It doesn't run your tests in CI by itself.** The skills need an AI assistant to read them. The installer works fine in CI, and `verify` makes a good pipeline check.
|
|
451
549
|
- **It can't stop an assistant that ignores it.** The defences make dishonest answers *fail loudly* rather than pass quietly — they can't force a model to read the skill.
|
|
452
|
-
- **It has not been benchmarked across AI models.** The tooling is thoroughly tested (
|
|
550
|
+
- **It has not been benchmarked across AI models.** The tooling is thoroughly tested (245 automated tests, 27 evaluation cases). How faithfully each assistant follows the skills is measured for one model, in one session, and [documented as such](docs/release/v1-excellence-audit.md). If you need a published accuracy number before adopting a tool, this one doesn't have it yet.
|
|
453
551
|
- **It sends nothing anywhere.** No telemetry, no network calls, no accounts.
|
|
454
552
|
- **It is version 0.9.0** — a public preview. Solid and heavily tested, still pre-1.0.
|
|
455
553
|
|
|
@@ -459,11 +557,11 @@ Everything below is reproducible from a clone with the command beside it.
|
|
|
459
557
|
|
|
460
558
|
| Evidence | Command |
|
|
461
559
|
| --- | --- |
|
|
462
|
-
|
|
|
463
|
-
|
|
|
464
|
-
|
|
|
465
|
-
|
|
|
466
|
-
| 4 real AI-produced results, scored | `
|
|
560
|
+
| 151 engine tests: analysis, diagnostics, reporting, artifacts, frameworks, seams | `node --test packages/engine/test/*.test.mjs` |
|
|
561
|
+
| The recorded corpus, proven against a second implementation | `node --test packages/engine/test/corpus.test.mjs` |
|
|
562
|
+
| 94 installer tests — installation scopes, security, and repeated-use stress | `npm test` |
|
|
563
|
+
| 27 evaluation cases, including deliberately dishonest outputs the scorer must reject | `npm run validate:evals` |
|
|
564
|
+
| 4 real AI-produced results, scored | `node tests/evals/run-live.mjs --captures claude-opus-5` |
|
|
467
565
|
| 17 repository checks — including "documentation matches implementation" | `npm run validate:skills` … |
|
|
468
566
|
|
|
469
567
|
## Going deeper
|
|
@@ -480,7 +578,7 @@ Nothing below is required to use the pack.
|
|
|
480
578
|
| [docs/report-format.md](docs/report-format.md) | Consuming the JSON output from your own tooling |
|
|
481
579
|
| [docs/installation/](docs/installation/README.md) | Per-assistant installation guides |
|
|
482
580
|
| [COMPATIBILITY.md](COMPATIBILITY.md) | Supported assistants, runtimes, and frameworks |
|
|
483
|
-
| [docs/architecture/README.md](docs/architecture/README.md) |
|
|
581
|
+
| [docs/architecture/README.md](docs/architecture/README.md) | 17 decision records explaining every major choice |
|
|
484
582
|
| [docs/release/v0.9-release-checklist.md](docs/release/v0.9-release-checklist.md) | Exactly what was verified for this release, and what wasn't |
|
|
485
583
|
| [CHANGELOG.md](CHANGELOG.md) | What changed, including what regressed |
|
|
486
584
|
|
|
@@ -512,7 +610,7 @@ Start with [CONTRIBUTING.md](CONTRIBUTING.md). Everyone is bound by the [Code of
|
|
|
512
610
|
|
|
513
611
|
| Need | Where |
|
|
514
612
|
| --- | --- |
|
|
515
|
-
| Something is broken | Run `
|
|
613
|
+
| Something is broken | Run `qa-engineer doctor`, then read [When something goes wrong](#when-something-goes-wrong) |
|
|
516
614
|
| A question | GitHub Discussions — see [SUPPORT.md](SUPPORT.md) |
|
|
517
615
|
| A bug | GitHub Issues, with your `doctor --json` output |
|
|
518
616
|
| A security problem | **Privately** — see [SECURITY.md](SECURITY.md). Never a public issue. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "qa-engineer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "A vendor-neutral Agent Skills pack that teaches AI coding agents to work like senior QA automation engineers.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -29,43 +29,34 @@
|
|
|
29
29
|
"packages/installer/schemas",
|
|
30
30
|
"packages/installer/package.json",
|
|
31
31
|
"packages/installer/README.md",
|
|
32
|
+
"packages/engine/lib",
|
|
33
|
+
"packages/engine/bin",
|
|
34
|
+
"packages/engine/package.json",
|
|
32
35
|
"skills",
|
|
33
|
-
"shared/analysis/lib/qa_analysis",
|
|
34
|
-
"shared/analysis/schemas",
|
|
35
|
-
"shared/diagnostics/lib/qa_diagnostics",
|
|
36
|
-
"shared/diagnostics/schemas",
|
|
37
|
-
"shared/tooling/qa_tool.py",
|
|
38
36
|
"shared/frameworks/registry.json",
|
|
39
37
|
"shared/frameworks/registry.schema.json",
|
|
40
38
|
"shared/frameworks/registry.mjs",
|
|
41
|
-
"shared/frameworks/playwright/lib/playwright_analysis.py",
|
|
42
|
-
"shared/frameworks/selenium/lib/selenium_analysis.py",
|
|
43
|
-
"shared/frameworks/cypress/lib/cypress_analysis.py",
|
|
44
|
-
"shared/frameworks/webdriverio/lib/webdriverio_analysis.py",
|
|
45
39
|
"COMPATIBILITY.md",
|
|
46
|
-
"!**/
|
|
47
|
-
"
|
|
48
|
-
"
|
|
40
|
+
"!**/tests",
|
|
41
|
+
"shared/tooling/qa-tool.mjs",
|
|
42
|
+
"!packages/engine/test"
|
|
49
43
|
],
|
|
50
44
|
"scripts": {
|
|
51
|
-
"eval:live": "
|
|
52
|
-
"prepublishOnly": "npm run validate:release && npm run validate:doc-claims && npm run validate:docs-commands && npm test && npm run validate:
|
|
45
|
+
"eval:live": "node tests/evals/run-live.mjs --baseline tests/evals/baselines/reference.json",
|
|
46
|
+
"prepublishOnly": "npm run validate:release && npm run validate:doc-claims && npm run validate:docs-commands && npm test && npm run validate:evals",
|
|
53
47
|
"qa": "node packages/installer/bin/qa.mjs",
|
|
54
48
|
"release:checksums": "node scripts/release/release-notes.mjs --checksums",
|
|
55
49
|
"release:notes": "node scripts/release/release-notes.mjs",
|
|
56
50
|
"test": "node --test packages/installer/test/*.test.mjs packages/engine/test/*.test.mjs",
|
|
57
51
|
"validate:architecture": "node scripts/check-architecture-fitness.mjs",
|
|
58
52
|
"validate:branding": "node scripts/check-branding.mjs",
|
|
59
|
-
"validate:bundle": "python3 scripts/bundle_python.py --check",
|
|
60
|
-
"validate:context": "cd shared/analysis/lib && python3 -m qa_analysis.cli context --path tests/fixtures/valid-context.md",
|
|
61
53
|
"validate:doc-claims": "node scripts/check-doc-claims.mjs",
|
|
62
54
|
"validate:docs-commands": "node scripts/check-docs-commands.mjs",
|
|
63
|
-
"validate:engine
|
|
64
|
-
"validate:evals": "
|
|
55
|
+
"validate:engine": "node --test packages/engine/test/*.test.mjs",
|
|
56
|
+
"validate:evals": "node tests/evals/run-evals.mjs",
|
|
65
57
|
"validate:keywords": "node scripts/check-keywords.mjs",
|
|
66
58
|
"validate:knowledge": "node scripts/check-knowledge.mjs",
|
|
67
59
|
"validate:matrix": "node scripts/check-capability-matrix.mjs",
|
|
68
|
-
"validate:python": "python3 shared/analysis/lib/run_tests.py",
|
|
69
60
|
"validate:registry": "node scripts/check-framework-registry.mjs",
|
|
70
61
|
"validate:release": "node scripts/release/validate-release.mjs",
|
|
71
62
|
"validate:release-notes": "node scripts/release/release-notes.mjs --check",
|