qa-engineer 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -59
- package/package.json +1 -1
- package/packages/engine/bin/qa-engine.mjs +232 -8
- package/packages/engine/lib/analysis/har.mjs +34 -0
- package/packages/engine/lib/analysis/network.mjs +237 -0
- package/packages/engine/lib/analysis/report-html.mjs +47 -734
- package/packages/engine/lib/artifacts/manager.mjs +453 -0
- package/packages/engine/lib/artifacts/mime.mjs +109 -0
- package/packages/engine/lib/artifacts/zip-write.mjs +143 -0
- package/packages/engine/lib/report/components/charts.mjs +424 -0
- package/packages/engine/lib/report/components/evidence.mjs +207 -0
- package/packages/engine/lib/report/components/findings.mjs +258 -0
- package/packages/engine/lib/report/components/nav.mjs +99 -0
- package/packages/engine/lib/report/components/primitives.mjs +246 -0
- package/packages/engine/lib/report/components/runtime.mjs +246 -0
- package/packages/engine/lib/report/components/timeline.mjs +65 -0
- package/packages/engine/lib/report/core/model.mjs +270 -0
- package/packages/engine/lib/report/core/normalize.mjs +226 -0
- package/packages/engine/lib/report/core/sections.mjs +978 -0
- package/packages/engine/lib/report/export/bundle.mjs +293 -0
- package/packages/engine/lib/report/export/html.mjs +183 -0
- package/packages/engine/lib/report/export/machine.mjs +290 -0
- package/packages/engine/lib/report/export/markdown.mjs +323 -0
- package/packages/engine/lib/report/schemas/qa-report.schema.json +555 -0
- package/packages/engine/lib/report/theme/css.mjs +529 -0
- package/packages/engine/lib/report/theme/tokens.mjs +137 -0
- package/packages/engine/lib/report/version.mjs +78 -0
- package/packages/engine/package.json +2 -2
- package/packages/installer/lib/agents/targets.mjs +90 -0
- package/packages/installer/lib/agents/user-level.mjs +80 -0
- package/packages/installer/lib/cli/flags.mjs +20 -2
- package/packages/installer/lib/commands/doctor.mjs +6 -4
- package/packages/installer/lib/commands/install.mjs +134 -93
- package/packages/installer/lib/commands/repair.mjs +10 -6
- package/packages/installer/lib/commands/self-test.mjs +4 -3
- package/packages/installer/lib/commands/uninstall.mjs +14 -8
- package/packages/installer/lib/commands/update.mjs +9 -4
- package/packages/installer/lib/commands/verify.mjs +13 -12
- package/packages/installer/lib/constants.mjs +13 -0
- package/packages/installer/lib/core/conflict.mjs +5 -4
- package/packages/installer/lib/core/fs-safe.mjs +146 -6
- package/packages/installer/lib/core/integrity.mjs +59 -0
- package/packages/installer/lib/core/lockfile.mjs +19 -3
- package/packages/installer/lib/core/plan.mjs +213 -0
- package/packages/installer/lib/core/qa-home.mjs +145 -0
- package/packages/installer/lib/core/scope.mjs +274 -0
- package/packages/installer/lib/core/validate-install.mjs +37 -12
- package/packages/installer/package.json +1 -1
- package/packages/installer/schemas/qa-lock.schema.json +119 -21
- package/shared/tooling/qa-tool.mjs +41 -5
- package/skills/qa-api/scripts/qa-tool.mjs +41 -5
- package/skills/qa-audit/scripts/qa-tool.mjs +41 -5
- package/skills/qa-debug/scripts/qa-tool.mjs +41 -5
- package/skills/qa-explore/SKILL.md +26 -10
- package/skills/qa-explore/contracts/explore-result.schema.json +517 -11
- package/skills/qa-explore/references/api-replay.md +40 -1
- package/skills/qa-explore/references/report-pipeline.md +265 -95
- package/skills/qa-explore/scripts/qa-tool.mjs +41 -5
- package/skills/qa-fix/scripts/qa-tool.mjs +41 -5
- package/skills/qa-flaky/scripts/qa-tool.mjs +41 -5
- package/skills/qa-init/scripts/qa-tool.mjs +41 -5
- package/skills/qa-report/scripts/qa-tool.mjs +41 -5
- package/skills/qa-run/scripts/qa-tool.mjs +41 -5
|
@@ -1,150 +1,320 @@
|
|
|
1
1
|
# Report pipeline
|
|
2
2
|
|
|
3
|
-
How `/qa-explore` turns findings into
|
|
3
|
+
How `/qa-explore` turns findings into a JSON artifact and every rendering of it.
|
|
4
|
+
|
|
5
|
+
## The one rule
|
|
6
|
+
|
|
7
|
+
**You produce structured data. The pack produces every document.**
|
|
8
|
+
|
|
9
|
+
The JSON is written first, validated, and then rendered. HTML, Markdown, SARIF, JUnit,
|
|
10
|
+
and CSV are all produced *from* it by the engine. **You never write HTML, CSS, or any
|
|
11
|
+
styling** — there is no fallback that allows it.
|
|
12
|
+
|
|
13
|
+
```text
|
|
14
|
+
explore-result.json ──validate──► artifacts verify ──► report-html
|
|
15
|
+
(source of truth) (does the evidence report-export
|
|
16
|
+
actually exist?) (md/sarif/junit/csv)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
That is not a style preference. Producing prose first and JSON afterwards is how the
|
|
20
|
+
first live run shipped a report whose findings each had `actual`, `expected`, and
|
|
21
|
+
`fixDirection` in the artifact and one flattened sentence in the HTML. The renderer
|
|
22
|
+
cannot drop a field it is required to render; a person retyping can drop four.
|
|
23
|
+
|
|
24
|
+
## Why you have no say in how it looks
|
|
25
|
+
|
|
26
|
+
A qa-engineer report must be the same document whoever produced it — Claude Code,
|
|
27
|
+
Cursor, Codex, Gemini, or an agent that does not exist yet. That is only true if no
|
|
28
|
+
producer can influence presentation, so the contract gives you no way to: there is no
|
|
29
|
+
field for a colour, a class, a font, or a template, and `additionalProperties: false`
|
|
30
|
+
means you cannot add one.
|
|
31
|
+
|
|
32
|
+
Record which agent you are in `metadata.producer` when you use the canonical schema. It
|
|
33
|
+
is displayed in the appendix and nothing branches on it.
|
|
34
|
+
|
|
35
|
+
If `report-html` is unreachable, **stop and say the engine is missing**. Do not
|
|
36
|
+
hand-write a substitute — an improvised report is the failure this design removes.
|
|
37
|
+
|
|
38
|
+
## Which schema to write
|
|
39
|
+
|
|
40
|
+
| Situation | Contract |
|
|
41
|
+
| --- | --- |
|
|
42
|
+
| A `/qa-explore` run in this pack | `qa-explore/explore-result` (`contracts/explore-result.schema.json`) |
|
|
43
|
+
| Any other producer, or a new integration | `qa-engineer/qa-report` schema 2.0 |
|
|
44
|
+
|
|
45
|
+
Fetch the canonical schema when you need it:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-schema --out qa-report.schema.json
|
|
49
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-versions # schema/theme/renderer
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Both render through the same code to the same document. The canonical form groups
|
|
53
|
+
fields by audience (`metadata`, `summary`, `coverage`, `issues`, `artifacts`,
|
|
54
|
+
`performance`, `security`, `recommendations`); the exploratory form is flat. Nothing is
|
|
55
|
+
lost either way.
|
|
4
56
|
|
|
5
57
|
## Artifact paths
|
|
6
58
|
|
|
7
|
-
For run id `R
|
|
59
|
+
For run id `R`, everything lives in one folder so the whole thing moves as a unit:
|
|
60
|
+
|
|
61
|
+
```text
|
|
62
|
+
qa-artifacts/explore-R/
|
|
63
|
+
├── explore-result.json the artifact — written first, validated, source of truth
|
|
64
|
+
├── explore-report.html rendered
|
|
65
|
+
├── explore-report.md rendered
|
|
66
|
+
├── screenshots/ proof images
|
|
67
|
+
├── network/ request captures
|
|
68
|
+
├── console/ console logs
|
|
69
|
+
└── dom/ DOM snapshots
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Register every artifact
|
|
73
|
+
|
|
74
|
+
Write an `artifacts[]` entry for every file the run produces, and point evidence at it
|
|
75
|
+
by `artifactId`:
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"artifacts": [
|
|
80
|
+
{
|
|
81
|
+
"id": "shot-login-dupe",
|
|
82
|
+
"kind": "screenshot",
|
|
83
|
+
"path": "screenshots/login-duplicate.png",
|
|
84
|
+
"label": "Two auth requests for one submit",
|
|
85
|
+
"capturedAt": "2026-07-29T09:02:11Z"
|
|
86
|
+
}
|
|
87
|
+
],
|
|
88
|
+
"findings": [
|
|
89
|
+
{
|
|
90
|
+
"id": "EXP-1",
|
|
91
|
+
"evidence": [
|
|
92
|
+
{ "type": "screenshot", "artifactId": "shot-login-dupe",
|
|
93
|
+
"caption": "The stored session blob in DevTools" }
|
|
94
|
+
]
|
|
95
|
+
}
|
|
96
|
+
]
|
|
97
|
+
}
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Why the indirection is worth it: the path then exists in exactly one place. Two
|
|
101
|
+
findings citing the same screenshot cannot disagree about where it is, the renderer
|
|
102
|
+
knows its size and type, and a typo produces a *named missing artifact* rather than a
|
|
103
|
+
broken image.
|
|
8
104
|
|
|
9
|
-
|
|
10
|
-
- `qa-artifacts/explore-R/explore-report.html`
|
|
11
|
-
- `qa-artifacts/explore-R/explore-result.json`
|
|
12
|
-
- `qa-artifacts/explore-R/screenshots/…`
|
|
105
|
+
### Paths are relative to the result JSON
|
|
13
106
|
|
|
14
|
-
|
|
107
|
+
`screenshots/login-duplicate.png`, not `qa-artifacts/explore-R/screenshots/…` and not
|
|
108
|
+
an absolute path. The resolver will still find a project-root-relative path by walking
|
|
109
|
+
up from the run folder — but it is a fallback for a mistake, not the contract.
|
|
15
110
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
111
|
+
### Before/after pairs
|
|
112
|
+
|
|
113
|
+
Set `compares` on the *after* image and both render side by side under one caption:
|
|
114
|
+
|
|
115
|
+
```json
|
|
116
|
+
{ "id": "shot-holiday-after", "kind": "screenshot",
|
|
117
|
+
"path": "screenshots/holiday-after.png", "compares": "shot-holiday-before" }
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Verify the evidence before rendering
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs artifacts verify \
|
|
124
|
+
qa-artifacts/explore-R/explore-result.json
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Exit `0` means every file a finding points at is on disk and non-empty. Exit `1` lists
|
|
128
|
+
what is missing and why. **Run this before rendering and before declaring the run
|
|
129
|
+
complete.** A zero-byte screenshot counts as missing, because a failed capture renders
|
|
130
|
+
exactly as badly as an absent one.
|
|
131
|
+
|
|
132
|
+
If something is genuinely gone, do not delete the evidence entry to make the check
|
|
133
|
+
pass. Leave it: the report states the absence, names the file, and gives the reason —
|
|
134
|
+
which is honest, and tells whoever re-runs the pass what to fix.
|
|
135
|
+
|
|
136
|
+
## Write the report for a reader who has never seen the product
|
|
137
|
+
|
|
138
|
+
The report is forwarded. A founder, a designer, and the developer who owns one finding
|
|
139
|
+
will each open it without the conversation that produced it.
|
|
19
140
|
|
|
20
141
|
- **Name things, do not point at them.** "the Sign in button on /login", not "the
|
|
21
142
|
button". A reader cannot see your screen.
|
|
22
143
|
- **No unexplained jargon.** IDOR, CLS, and *soft assertion* need a clause of
|
|
23
144
|
explanation or a plainer word.
|
|
24
|
-
- **State the consequence.**
|
|
25
|
-
|
|
26
|
-
twice.
|
|
145
|
+
- **State the consequence.** Say that a duplicate request can trip rate limiting and
|
|
146
|
+
charge twice — a reader who does not know the product cannot infer it.
|
|
27
147
|
|
|
28
|
-
The renderer supplies the standing furniture
|
|
29
|
-
|
|
30
|
-
supply what only this run knows
|
|
31
|
-
`actual` and `expected` read as complete sentences.
|
|
148
|
+
The renderer supplies the standing furniture: what an exploratory QA pass is, what each
|
|
149
|
+
dimension means, what the severity labels claim, what could not be run, the severity
|
|
150
|
+
legend, and the artifact index. You supply what only this run knows.
|
|
32
151
|
|
|
33
|
-
##
|
|
152
|
+
## The fields that carry the report
|
|
153
|
+
|
|
154
|
+
| Field | What good looks like |
|
|
155
|
+
| --- | --- |
|
|
156
|
+
| `actual` | "Two identical `POST /graphql` requests are sent; the second returns 401 and the form clears" — not "Login fails" |
|
|
157
|
+
| `expected` | "One request per submit; the button is disabled while in flight" — not "Should not fail" |
|
|
158
|
+
| `repro` / `steps` | Numbered, retypable, starting from a URL — not "Double-click login" |
|
|
159
|
+
| `fixDirection` | "Disable submit on the first click and re-enable it in the request's finally block" — not "Fix the handler" |
|
|
160
|
+
| `businessImpact` | What it costs in money, trust, compliance, or support load. Not a restatement of the defect |
|
|
161
|
+
| `rootCause` | Only when the run actually established one. `chain` is the causal steps in order |
|
|
162
|
+
| `regressionRisk` | The level, plus what QA should re-run once it is fixed |
|
|
163
|
+
|
|
164
|
+
`businessImpact` and `rootCause` are what turn "button not working" into something a
|
|
165
|
+
manager can prioritise and an engineer can start on. Write them when the run supports
|
|
166
|
+
them, and leave them out when it does not — an invented root cause is worse than none.
|
|
34
167
|
|
|
35
|
-
|
|
36
|
-
not there, and without it the report opens on a wall of findings.
|
|
168
|
+
## Scope: what you did, and what you left alone
|
|
37
169
|
|
|
38
170
|
| Field | What goes in it |
|
|
39
171
|
| --- | --- |
|
|
40
|
-
| `objective` | One or two sentences: which feature you exercised
|
|
41
|
-
| `covered` | The concrete things you
|
|
42
|
-
| `notCovered` | Every boundary of the run, **each with its reason
|
|
172
|
+
| `objective` | One or two sentences: which feature you exercised, what you were establishing |
|
|
173
|
+
| `covered` | The concrete things you touched — screens, controls, flows, request paths |
|
|
174
|
+
| `notCovered` | Every boundary of the run, **each with its reason** |
|
|
43
175
|
|
|
44
|
-
`notCovered`
|
|
45
|
-
|
|
46
|
-
|
|
176
|
+
`notCovered` protects the reader. A login report that does not say "signing in
|
|
177
|
+
successfully was never tested, because a QA run must not enter real credentials"
|
|
178
|
+
invites the conclusion that login works. State:
|
|
47
179
|
|
|
48
180
|
- what needed access or credentials you did not have, and why you did not ask
|
|
49
|
-
- what a
|
|
50
|
-
|
|
181
|
+
- what a development environment cannot tell you (security headers, bundle size, real
|
|
182
|
+
latency) and where it must be re-checked
|
|
51
183
|
- what would have caused damage to test (rate limits, lockout, destructive actions)
|
|
52
184
|
- what was in scope but blocked, naming the case id
|
|
53
185
|
|
|
54
|
-
Blocked cases and unrun dimensions are added by the renderer
|
|
55
|
-
|
|
186
|
+
Blocked cases and unrun dimensions are added by the renderer. Write only the boundaries
|
|
187
|
+
it cannot know.
|
|
188
|
+
|
|
189
|
+
## The executive block
|
|
190
|
+
|
|
191
|
+
Fill `executive` on any run a non-engineer will read. It is the only part a CEO reads.
|
|
192
|
+
|
|
193
|
+
```json
|
|
194
|
+
{
|
|
195
|
+
"executive": {
|
|
196
|
+
"verdict": "do-not-ship",
|
|
197
|
+
"headline": "Two security findings block release; everything else is fixable within a day.",
|
|
198
|
+
"health": "The console is functionally sound. The blockers are exposure, not feature gaps.",
|
|
199
|
+
"risks": ["The session token is in localStorage, readable by any XSS on the origin."],
|
|
200
|
+
"recommendedAction": "Fix the two security findings, then re-run this pass.",
|
|
201
|
+
"estimatedFixHours": { "low": 6, "high": 8 },
|
|
202
|
+
"confidence": 0.86
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
```
|
|
56
206
|
|
|
57
|
-
|
|
207
|
+
`verdict` must agree with the findings. A `ship` verdict over a critical finding is a
|
|
208
|
+
contradiction the reader will catch, and it costs the whole report its credibility.
|
|
209
|
+
When the run did not establish enough to judge, say `insufficient-data`.
|
|
58
210
|
|
|
59
|
-
`
|
|
60
|
-
|
|
61
|
-
drift, and the reader gets the weaker one.
|
|
211
|
+
`confidence` is about *this run*, not about the product: how much of what is reported
|
|
212
|
+
was reproduced rather than observed once.
|
|
62
213
|
|
|
63
|
-
##
|
|
214
|
+
## Scores
|
|
64
215
|
|
|
65
|
-
|
|
66
|
-
|
|
216
|
+
Only report a score for something the run measured. There is no accessibility score
|
|
217
|
+
without an accessibility pass. The renderer derives `overall` from the severity counts
|
|
218
|
+
using a published formula and labels it as derived; everything else must be real.
|
|
67
219
|
|
|
68
|
-
|
|
69
|
-
| --- | --- |
|
|
70
|
-
| URL | … |
|
|
71
|
-
| Date | … |
|
|
72
|
-
| Tester | AI-assisted QA (qa-explore) |
|
|
73
|
-
| Adapter | … |
|
|
74
|
-
| Verdict | N critical, N high, … — <one line> |
|
|
75
|
-
| Version | v1.0 |
|
|
76
|
-
|
|
77
|
-
## Evidence index
|
|
78
|
-
| File | Shows | Findings |
|
|
79
|
-
|
|
80
|
-
## Executive summary
|
|
81
|
-
## Findings
|
|
82
|
-
### EXP-1 · critical — title
|
|
83
|
-
- Repro / Actual / Expected / Fix
|
|
84
|
-
- Evidence: an image reference to the screenshot stored beside the report, such as screenshots/finding-01.png
|
|
85
|
-
## Performance
|
|
86
|
-
## Security summary
|
|
87
|
-
## Database validation <!-- omit if not in scope -->
|
|
88
|
-
## Test case coverage <!-- omit if none -->
|
|
89
|
-
## What works well
|
|
90
|
-
## Suggested fix order
|
|
91
|
-
## Changelog
|
|
92
|
-
```
|
|
220
|
+
## Rendering
|
|
93
221
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
## JSON
|
|
97
|
-
|
|
98
|
-
Write `explore-result.json` matching `contracts/explore-result.schema.json`. Validate before completion:
|
|
222
|
+
**The bundle is the canonical output.** It is a folder on the user's disk that contains
|
|
223
|
+
the entire report and nothing outside it.
|
|
99
224
|
|
|
100
225
|
```bash
|
|
101
|
-
node <SKILL_DIR>/scripts/qa-tool.mjs analysis
|
|
226
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-bundle \
|
|
102
227
|
qa-artifacts/explore-R/explore-result.json \
|
|
103
|
-
|
|
228
|
+
--out qa-artifacts/explore-R/report --zip
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
```text
|
|
232
|
+
qa-artifacts/explore-R/report/
|
|
233
|
+
├── index.html open this in any browser, offline
|
|
234
|
+
├── report.json the artifact it was rendered from
|
|
235
|
+
├── report.md the same content for tickets
|
|
236
|
+
├── manifest.json every file, with its SHA-256
|
|
237
|
+
└── assets/
|
|
238
|
+
├── css/report.css
|
|
239
|
+
├── js/report.js
|
|
240
|
+
└── screenshots/ videos/ traces/ network/ dom/ console/ logs/
|
|
104
241
|
```
|
|
105
242
|
|
|
106
|
-
|
|
243
|
+
Every link is verified to resolve inside the folder *after* it is written; a reference
|
|
244
|
+
that escapes fails the command rather than shipping a report with a hole in it. `--zip`
|
|
245
|
+
also emits `report.zip` — one file to send a stakeholder.
|
|
246
|
+
|
|
247
|
+
Point the user at `index.html`. **Never** make a hosted preview the primary output: a
|
|
248
|
+
platform URL expires, and a QA report has to still open in six months. If you publish
|
|
249
|
+
one as a convenience, say plainly that it is a convenience and that the folder is the
|
|
250
|
+
real deliverable.
|
|
107
251
|
|
|
108
|
-
|
|
109
|
-
throwaway md→html script:
|
|
252
|
+
### Single-file and other formats
|
|
110
253
|
|
|
111
254
|
```bash
|
|
255
|
+
# One HTML file with images inlined — for a lone email attachment
|
|
112
256
|
node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-html \
|
|
113
|
-
qa-artifacts/explore-R/explore-result.json \
|
|
257
|
+
qa-artifacts/explore-R/explore-result.json --embed \
|
|
114
258
|
--out qa-artifacts/explore-R/explore-report.html
|
|
259
|
+
|
|
260
|
+
# Markdown on its own
|
|
261
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-export \
|
|
262
|
+
qa-artifacts/explore-R/explore-result.json \
|
|
263
|
+
--format markdown --out qa-artifacts/explore-R/explore-report.md
|
|
115
264
|
```
|
|
116
265
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
the attribution footer come with it.
|
|
266
|
+
Add `--embed` to inline every image as a data URI. Use it when the report will be
|
|
267
|
+
mailed or uploaded on its own, without the folder beside it. It costs about a third
|
|
268
|
+
more bytes than the images.
|
|
269
|
+
|
|
270
|
+
### Modes
|
|
123
271
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
272
|
+
`--mode` selects the audience. All four share one model and one stylesheet, so they
|
|
273
|
+
cannot disagree about a number and an embedded report looks identical to a standalone
|
|
274
|
+
one.
|
|
127
275
|
|
|
128
|
-
|
|
|
276
|
+
| Mode | Sections | Use it for |
|
|
129
277
|
| --- | --- | --- |
|
|
130
|
-
| `
|
|
131
|
-
| `
|
|
132
|
-
| `
|
|
133
|
-
| `
|
|
278
|
+
| `full` | everything | the default; the canonical artifact |
|
|
279
|
+
| `executive` | overview, summary, coverage, recommendations, appendix | a report going to leadership or a client |
|
|
280
|
+
| `developer` | overview, findings, performance, API, security, a11y, console, screenshots, artifacts | the engineer doing the work |
|
|
281
|
+
| `artifact` | everything, without `<html>`/`<head>`/`<body>` | embedding in a host page — Claude Artifacts, a wiki, a dashboard |
|
|
282
|
+
|
|
283
|
+
### Other formats
|
|
134
284
|
|
|
135
|
-
|
|
136
|
-
|
|
285
|
+
| Format | For |
|
|
286
|
+
| --- | --- |
|
|
287
|
+
| `sarif` | GitHub's Security tab, and any code-scanning viewer |
|
|
288
|
+
| `junit` | A CI dashboard, beside the unit tests |
|
|
289
|
+
| `csv` | The spreadsheet a QA lead already tracks the release in |
|
|
290
|
+
| `json` | The artifact itself, re-emitted with sorted keys for a clean diff |
|
|
291
|
+
| `bundle` | A manifest of every file a shareable archive must contain, with hashes |
|
|
137
292
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
293
|
+
There is no PDF exporter, deliberately: the HTML report has a print stylesheet that
|
|
294
|
+
forces every finding open, drops the navigation and the glass, and prints links with
|
|
295
|
+
their targets. **Print → Save as PDF** produces a better document than a bundled PDF
|
|
296
|
+
library would, with nothing to install.
|
|
141
297
|
|
|
142
298
|
## Versioning
|
|
143
299
|
|
|
144
300
|
- `v1.0` initial report.
|
|
145
|
-
- Feedback rounds bump minor (`v1.1`, …): add measurements, evidence, changelog notes
|
|
146
|
-
-
|
|
301
|
+
- Feedback rounds bump minor (`v1.1`, …): add measurements, evidence, changelog notes.
|
|
302
|
+
- **Never renumber finding ids.** A ticket already cites `EXP-3`.
|
|
303
|
+
- Recount `severityCounts` after every add or remove.
|
|
304
|
+
|
|
305
|
+
## Redaction
|
|
306
|
+
|
|
307
|
+
For any excerpt that could carry a token or a credential:
|
|
308
|
+
|
|
309
|
+
```bash
|
|
310
|
+
node <SKILL_DIR>/scripts/qa-tool.mjs analysis redact <file>
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The renderer escapes markup; it does not redact secrets. Redact before the text enters
|
|
314
|
+
the JSON, not after.
|
|
147
315
|
|
|
148
316
|
## Conversation output
|
|
149
317
|
|
|
150
|
-
After writing files, reply with: verdict line, top
|
|
318
|
+
After writing the files, reply with: the verdict line, the top three to five findings
|
|
319
|
+
with their ids, the paths to the HTML/MD/JSON, and an offer to iterate. Do not paste
|
|
320
|
+
base64 images into chat.
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|