qa-engineer 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -59
- package/package.json +1 -1
- package/packages/engine/bin/qa-engine.mjs +232 -8
- package/packages/engine/lib/analysis/har.mjs +34 -0
- package/packages/engine/lib/analysis/network.mjs +237 -0
- package/packages/engine/lib/analysis/report-html.mjs +47 -734
- package/packages/engine/lib/artifacts/manager.mjs +453 -0
- package/packages/engine/lib/artifacts/mime.mjs +109 -0
- package/packages/engine/lib/artifacts/zip-write.mjs +143 -0
- package/packages/engine/lib/report/components/charts.mjs +424 -0
- package/packages/engine/lib/report/components/evidence.mjs +207 -0
- package/packages/engine/lib/report/components/findings.mjs +258 -0
- package/packages/engine/lib/report/components/nav.mjs +99 -0
- package/packages/engine/lib/report/components/primitives.mjs +246 -0
- package/packages/engine/lib/report/components/runtime.mjs +246 -0
- package/packages/engine/lib/report/components/timeline.mjs +65 -0
- package/packages/engine/lib/report/core/model.mjs +270 -0
- package/packages/engine/lib/report/core/normalize.mjs +226 -0
- package/packages/engine/lib/report/core/sections.mjs +978 -0
- package/packages/engine/lib/report/export/bundle.mjs +293 -0
- package/packages/engine/lib/report/export/html.mjs +183 -0
- package/packages/engine/lib/report/export/machine.mjs +290 -0
- package/packages/engine/lib/report/export/markdown.mjs +323 -0
- package/packages/engine/lib/report/schemas/qa-report.schema.json +555 -0
- package/packages/engine/lib/report/theme/css.mjs +529 -0
- package/packages/engine/lib/report/theme/tokens.mjs +137 -0
- package/packages/engine/lib/report/version.mjs +78 -0
- package/packages/engine/package.json +2 -2
- package/packages/installer/lib/agents/targets.mjs +90 -0
- package/packages/installer/lib/agents/user-level.mjs +80 -0
- package/packages/installer/lib/cli/flags.mjs +20 -2
- package/packages/installer/lib/commands/doctor.mjs +6 -4
- package/packages/installer/lib/commands/install.mjs +134 -93
- package/packages/installer/lib/commands/repair.mjs +10 -6
- package/packages/installer/lib/commands/self-test.mjs +4 -3
- package/packages/installer/lib/commands/uninstall.mjs +14 -8
- package/packages/installer/lib/commands/update.mjs +9 -4
- package/packages/installer/lib/commands/verify.mjs +13 -12
- package/packages/installer/lib/constants.mjs +13 -0
- package/packages/installer/lib/core/conflict.mjs +5 -4
- package/packages/installer/lib/core/fs-safe.mjs +146 -6
- package/packages/installer/lib/core/integrity.mjs +59 -0
- package/packages/installer/lib/core/lockfile.mjs +19 -3
- package/packages/installer/lib/core/plan.mjs +213 -0
- package/packages/installer/lib/core/qa-home.mjs +145 -0
- package/packages/installer/lib/core/scope.mjs +274 -0
- package/packages/installer/lib/core/validate-install.mjs +37 -12
- package/packages/installer/package.json +1 -1
- package/packages/installer/schemas/qa-lock.schema.json +119 -21
- package/shared/tooling/qa-tool.mjs +41 -5
- package/skills/qa-api/scripts/qa-tool.mjs +41 -5
- package/skills/qa-audit/scripts/qa-tool.mjs +41 -5
- package/skills/qa-debug/SKILL.md +3 -0
- package/skills/qa-debug/references/failure-handoff.md +44 -0
- package/skills/qa-debug/scripts/qa-tool.mjs +41 -5
- package/skills/qa-explore/SKILL.md +26 -10
- package/skills/qa-explore/contracts/explore-result.schema.json +517 -11
- package/skills/qa-explore/references/api-replay.md +40 -1
- package/skills/qa-explore/references/report-pipeline.md +265 -95
- package/skills/qa-explore/scripts/qa-tool.mjs +41 -5
- package/skills/qa-fix/scripts/qa-tool.mjs +41 -5
- package/skills/qa-flaky/scripts/qa-tool.mjs +41 -5
- package/skills/qa-init/scripts/qa-tool.mjs +41 -5
- package/skills/qa-report/scripts/qa-tool.mjs +41 -5
- package/skills/qa-run/README.md +7 -0
- package/skills/qa-run/SKILL.md +12 -7
- package/skills/qa-run/contracts/execution-result.schema.json +77 -0
- package/skills/qa-run/examples/execute-playwright.md +42 -10
- package/skills/qa-run/examples/plan-a-run.md +5 -3
- package/skills/qa-run/references/artifact-collector.md +2 -1
- package/skills/qa-run/references/command-builder.md +18 -1
- package/skills/qa-run/references/execution-strategy.md +2 -0
- package/skills/qa-run/references/failure-handoff.md +44 -0
- package/skills/qa-run/references/playwright-artifacts.md +3 -1
- package/skills/qa-run/references/playwright-execution.md +14 -1
- package/skills/qa-run/scripts/qa-tool.mjs +41 -5
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|
package/skills/qa-debug/SKILL.md
CHANGED
|
@@ -21,6 +21,8 @@ Investigate a failure the way an experienced QA engineer would: gather the evide
|
|
|
21
21
|
|
|
22
22
|
Do not propose or make code changes here — that is `/qa-fix`, which consumes this skill's output. Do not run tests (`/qa-run`) or generate them (`/qa-generate`). This skill explains a failure; it does not repair it.
|
|
23
23
|
|
|
24
|
+
This skill is often entered **automatically**: a red `/qa-run` hands its validated execution result straight here rather than asking the user to type the next command. That changes nothing about the investigation — the input is the same artifact a human would have passed — but it does bind this skill to the far end of the handoff: the chain stops here. Do not dispatch onward, and do not re-run the suite to gather more evidence; when the evidence is insufficient, say what is missing and name the command that would capture it.
|
|
25
|
+
|
|
24
26
|
## Inputs
|
|
25
27
|
|
|
26
28
|
- The user's request, which follows in the conversation: the failing test, run, or artifact.
|
|
@@ -35,6 +37,7 @@ Do not propose or make code changes here — that is `/qa-fix`, which consumes t
|
|
|
35
37
|
| When | Load |
|
|
36
38
|
| --- | --- |
|
|
37
39
|
| Following the investigation end to end | [references/investigation-workflow.md](references/investigation-workflow.md) |
|
|
40
|
+
| Being entered automatically from a red run, and where the chain stops | [references/failure-handoff.md](references/failure-handoff.md) |
|
|
38
41
|
| Classifying the root cause | [references/root-cause-analysis.md](references/root-cause-analysis.md), [references/failure-taxonomy.md](references/failure-taxonomy.md) |
|
|
39
42
|
| Reconstructing the sequence of events | [references/timeline-builder.md](references/timeline-builder.md) |
|
|
40
43
|
| Assigning severity, priority, and owner | [references/finding-prioritization.md](references/finding-prioritization.md) |
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
<!-- synced-from: shared/execution/failure-handoff.md — do not edit; edit the source and run: node scripts/sync-shared.mjs --write -->
|
|
2
|
+
# Failure Handoff
|
|
3
|
+
|
|
4
|
+
What an execution skill does the moment a run goes red: it hands the completed result to the diagnostic skill **automatically**, by command name, and shows the diagnosis alongside the run. A red run whose only output is "1 failed — try `/qa-debug`" makes the reader do the one step they were always going to do, and it makes them do it without the run's own knowledge of what was captured and where.
|
|
5
|
+
|
|
6
|
+
This is the pack's single, deliberately narrow exception to composition-by-recommendation: everywhere else a skill ends by *naming* the next command. The exception exists because a failure is the one outcome where the next step is not a choice.
|
|
7
|
+
|
|
8
|
+
## When it fires
|
|
9
|
+
|
|
10
|
+
| Run status | Handoff |
|
|
11
|
+
| --- | --- |
|
|
12
|
+
| `failed` | Automatic: dispatch the diagnostic command with the execution result |
|
|
13
|
+
| `errored` | Automatic: the run itself broke, and *why* is the whole question |
|
|
14
|
+
| `passed` with flaky tests | No dispatch. The run is green and nobody is blocked; recommend the flakiness command instead |
|
|
15
|
+
| `passed`, `no-tests-run` | No dispatch. There is nothing to diagnose |
|
|
16
|
+
| `blocked` | No dispatch. No run was attempted, so there is no failure — the block itself is the finding |
|
|
17
|
+
|
|
18
|
+
## The rules that keep it bounded
|
|
19
|
+
|
|
20
|
+
1. **One hop, forward only.** The diagnostic skill may not dispatch onward and may not re-enter execution. A run that diagnoses that re-runs that diagnoses is a loop no user asked for; the chain is exactly two links and then it stops with a recommendation.
|
|
21
|
+
2. **The successor must not mutate.** Diagnosis reads artifacts and explains. Anything that edits a file, a test, or a config stays a recommendation the user approves — automation is for *learning* why the run failed, never for acting on the answer.
|
|
22
|
+
3. **Dispatch by name, never by path.** The handoff invokes the diagnostic command; it does not load the sibling skill's files, which are not guaranteed to exist at runtime.
|
|
23
|
+
4. **The artifact goes first.** The execution result is written and validated against its contract *before* the handoff, and the handoff passes its path. The successor consumes a contract-conformant artifact, exactly as it would from a human invocation — there is no privileged in-memory channel between the two, so the automatic path and the manual path produce the same diagnosis.
|
|
24
|
+
5. **Announce it, and let the user decline.** The run says it is diagnosing before it does. An explicit "no debug", "just run it", or a request for the run only suppresses the handoff, and the suppression is recorded rather than silently obeyed.
|
|
25
|
+
6. **Degrade honestly.** The diagnostic command may be unavailable — not installed, or its engine missing. The handoff is then recorded as unavailable with the reason, and the recommendation to run it stands. A failed handoff never changes the run's status: the run's own numbers are already final.
|
|
26
|
+
7. **The run's verdict is never revised.** The diagnosis explains a failure; it cannot argue it away. Nothing the successor concludes edits the execution result — a "the test is just flaky" diagnosis leaves a `failed` run `failed`.
|
|
27
|
+
|
|
28
|
+
## What is recorded
|
|
29
|
+
|
|
30
|
+
The execution result carries the handoff as data, so the automatic step is auditable rather than something that happened in the transcript:
|
|
31
|
+
|
|
32
|
+
| Field | Meaning |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| `skill` | The successor, by command name |
|
|
35
|
+
| `command` | The exact command dispatched, with the artifact path |
|
|
36
|
+
| `artifact` | The execution result handed over |
|
|
37
|
+
| `status` | `dispatched`, `skipped` (the user declined), or `unavailable` (it could not run) |
|
|
38
|
+
| `reason` | Required for `skipped` and `unavailable` — why the diagnosis is not there |
|
|
39
|
+
|
|
40
|
+
A red result that records no handoff at all is incomplete: the reader cannot tell whether the diagnosis was declined, impossible, or forgotten.
|
|
41
|
+
|
|
42
|
+
## Why the failure evidence floor comes first
|
|
43
|
+
|
|
44
|
+
The handoff is only worth as much as what the run captured. The diagnostic layer reads screenshots, traces, and logs; a run that failed with none of them produces a diagnosis of "insufficient evidence, re-run with tracing" — the exact round trip the automatic handoff exists to remove. That is why the floor (in the command-builder module) is a floor and not a strategy preference: the two features are one feature, seen from each end.
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// `node qa-tool.mjs …`, which needs none.
|
|
30
30
|
|
|
31
31
|
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
32
33
|
import path from 'node:path';
|
|
33
34
|
import { spawnSync } from 'node:child_process';
|
|
34
35
|
import { fileURLToPath } from 'node:url';
|
|
@@ -38,7 +39,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
38
39
|
const USAGE = `usage: node qa-tool.mjs <tool> <subcommand> [args]
|
|
39
40
|
|
|
40
41
|
analysis parse artifacts, classify errors, validate contracts, diff-guard,
|
|
41
|
-
|
|
42
|
+
read .qa/context.md, render an HTML report, print the footer
|
|
42
43
|
diagnostics root cause, timeline, priority, repair plans, release readiness
|
|
43
44
|
playwright normalize a Playwright report or summarize a trace
|
|
44
45
|
|
|
@@ -53,15 +54,26 @@ examples:
|
|
|
53
54
|
/**
|
|
54
55
|
* Where the engine is, and how we found it.
|
|
55
56
|
*
|
|
56
|
-
* Ordered by cost: a bundled copy needs no resolution and no network, a
|
|
57
|
-
* node_modules copy needs no network, and npx needs both
|
|
58
|
-
* *which* one answered matters when a skill degrades — "the
|
|
59
|
-
* "the engine is being fetched" are different problems.
|
|
57
|
+
* Ordered by cost: a bundled copy needs no resolution and no network, a shared copy
|
|
58
|
+
* needs one stat per ancestor, a node_modules copy needs no network, and npx needs both
|
|
59
|
+
* on first use. Reporting *which* one answered matters when a skill degrades — "the
|
|
60
|
+
* engine is missing" and "the engine is being fetched" are different problems.
|
|
61
|
+
*
|
|
62
|
+
* The shared lookup is what lets one skill directory serve a project, a workspace, and a
|
|
63
|
+
* machine-wide install without knowing which installed it. Walking up for
|
|
64
|
+
* `.qa-engineer/engine` finds a workspace install from a skill inside the repository,
|
|
65
|
+
* and finds a global install from a skill linked into an agent's user-level directory —
|
|
66
|
+
* `~/.claude/skills/qa-explore` walks up to `~`, where `~/.qa-engineer/engine` is.
|
|
60
67
|
*/
|
|
61
68
|
function resolveEngine() {
|
|
62
69
|
const bundled = path.join(here, 'lib', 'bin', 'qa-engine.mjs');
|
|
63
70
|
if (fs.existsSync(bundled)) return { kind: 'bundled', command: [process.execPath, bundled] };
|
|
64
71
|
|
|
72
|
+
for (const root of candidateSharedRoots()) {
|
|
73
|
+
const shared = path.join(root, 'bin', 'qa-engine.mjs');
|
|
74
|
+
if (fs.existsSync(shared)) return { kind: 'shared', command: [process.execPath, shared] };
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
for (const base of candidateModuleRoots()) {
|
|
66
78
|
const installed = path.join(base, 'qa-engineer', 'packages', 'engine', 'bin', 'qa-engine.mjs');
|
|
67
79
|
if (fs.existsSync(installed)) {
|
|
@@ -75,6 +87,30 @@ function resolveEngine() {
|
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
|
90
|
+
/** Shared engine directories worth checking, most specific first. */
|
|
91
|
+
function candidateSharedRoots() {
|
|
92
|
+
const roots = [];
|
|
93
|
+
|
|
94
|
+
// An explicit home wins over anything discovered, so a user who moved the install can
|
|
95
|
+
// rely on it rather than on whatever the walk happens to find first.
|
|
96
|
+
const override = process.env.QA_ENGINEER_HOME;
|
|
97
|
+
if (override && override.trim()) roots.push(path.join(path.resolve(override.trim()), 'engine'));
|
|
98
|
+
|
|
99
|
+
let dir = here;
|
|
100
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
101
|
+
roots.push(path.join(dir, '.qa-engineer', 'engine'));
|
|
102
|
+
const parent = path.dirname(dir);
|
|
103
|
+
if (parent === dir) break;
|
|
104
|
+
dir = parent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The default machine home, for the case where the skill lives outside it entirely.
|
|
108
|
+
const home = os.homedir();
|
|
109
|
+
if (home) roots.push(path.join(home, '.qa-engineer', 'engine'));
|
|
110
|
+
|
|
111
|
+
return roots;
|
|
112
|
+
}
|
|
113
|
+
|
|
78
114
|
/** node_modules directories worth checking, nearest first. */
|
|
79
115
|
function candidateModuleRoots() {
|
|
80
116
|
const roots = [];
|
|
@@ -58,12 +58,18 @@ Load only what the situation requires:
|
|
|
58
58
|
1. **Intake.** Confirm URL. Parse attached test cases into a numbered checklist ([test-case-intake.md](references/test-case-intake.md)). Note known bugs as hypotheses to validate. Create a run id and artifact directory `qa-artifacts/explore-<run-id>/`.
|
|
59
59
|
2. **Session.** Select a browser adapter ([browser-adapters.md](references/browser-adapters.md)). Navigate to the URL. If a login wall appears, **stop** and ask the user to sign in; never type credentials or OTPs. Baseline console errors and resource entries.
|
|
60
60
|
3. **Functional.** Execute attached cases first, then surface-by-surface exploration per [pipeline.md](references/pipeline.md). After every action: DOM-verify, then capture evidence on failure or notable finding ([evidence-capture.md](references/evidence-capture.md)).
|
|
61
|
-
4. **API audit.**
|
|
61
|
+
4. **API audit.** Capture a HAR, then run `analysis network` on it — **never count requests, time them, or spot duplicates by eye.** Replay exact app URLs in-page where a call needs confirming. The parser says what happened; you decide which of its flags matter to this product and write those as findings ([api-replay.md](references/api-replay.md)).
|
|
62
62
|
5. **Performance.** Measure payloads, cold vs warm, long tasks / vitals signals under controlled conditions ([performance.md](references/performance.md)).
|
|
63
63
|
6. **Security (client).** Run the client-side pass only — token storage, PII in URLs/payloads, error leakage, headers, optional read-only IDOR probe when in scope ([security.md](references/security.md)). No destructive tests.
|
|
64
64
|
7. **UI / UX.** Check empty/loading/error states, consistency, mobile viewport spot-check; optional persona lens if the user named a role.
|
|
65
65
|
8. **Optional DB.** Only if the user provided access: capture UI values with timestamps; query; separate data vs presentation bugs. Skip entirely otherwise and note "DB validation not in scope".
|
|
66
|
-
9. **Report.** Assign stable IDs and severities ([finding-taxonomy.md](references/finding-taxonomy.md)). Every finding must include proof.
|
|
66
|
+
9. **Report.** Assign stable IDs and severities ([finding-taxonomy.md](references/finding-taxonomy.md)). Every finding must include proof. Then, in this order ([report-pipeline.md](references/report-pipeline.md)):
|
|
67
|
+
1. Write `explore-result.json` — `artifacts[]` for every captured file, `scope` (what you set out to check, what you touched, and every boundary of the run with its reason), `executive` when a non-engineer will read it, and findings written for a reader who has never seen the product.
|
|
68
|
+
2. **Validate** it against the contract.
|
|
69
|
+
3. **Verify the artifacts** — `artifacts verify` must pass before anything is rendered. A missing or zero-byte file is reported, never quietly dropped.
|
|
70
|
+
4. **Build the bundle** with `report-bundle` — a portable folder that opens offline, with every link verified. That folder is the deliverable; point the user at its `index.html`. Never type HTML.
|
|
71
|
+
|
|
72
|
+
Include "what works well" and a prioritized fix order. Only report a score for a dimension the run actually measured.
|
|
67
73
|
10. **Iterate.** On user feedback: validate live, add evidence, bump report version, never renumber IDs. After three stuck browser attempts on the same blocker, stop and escalate with findings.
|
|
68
74
|
|
|
69
75
|
## Guardrails
|
|
@@ -84,21 +90,31 @@ Invoke the bundled engine through its launcher, as documented in [references/det
|
|
|
84
90
|
| Tool | Invocation | Output | Fallback |
|
|
85
91
|
| --- | --- | --- | --- |
|
|
86
92
|
| Contract self-check | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis validate <explore-result.json> <SKILL_DIR>/contracts/explore-result.schema.json` | `{valid, errors}` — run this before rendering | None: an invalid result is not a report |
|
|
87
|
-
|
|
|
93
|
+
| Artifact check | `node <SKILL_DIR>/scripts/qa-tool.mjs artifacts verify <explore-result.json>` | `{ok, stats, missing}`; exit 1 when a file a finding points at is absent or empty. Run it **before** rendering | List the missing files in the report yourself; never delete the evidence entry to make it pass |
|
|
94
|
+
| Portable report bundle | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-bundle <explore-result.json> --out report --zip` | **The canonical output.** A folder — `index.html` + `assets/` — that opens offline in any browser, with every link verified to resolve inside it, plus a `.zip` to send. | **None.** Report that the engine is missing and stop |
|
|
95
|
+
| Single-file HTML | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-html <explore-result.json> --embed --out explore-report.html` | The complete self-contained report — nav, search, charts, timeline, expandable findings, evidence, attribution. `--embed` inlines images; `--mode full\|executive\|developer\|artifact` picks the audience | **None.** Report that the engine is missing and stop — a hand-written report is the failure this design removes |
|
|
96
|
+
| Other renderings | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-export <explore-result.json> --format <markdown\|sarif\|junit\|csv\|json\|bundle> --out <file>` | The same validated result in the format the destination reads | Write the Markdown by hand; skip the machine formats |
|
|
97
|
+
| Canonical schema | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis report-schema` · `… analysis report-versions` | The producer-neutral contract any agent writes to, and the schema/theme/renderer versions in force | Use the skill's own `contracts/explore-result.schema.json` |
|
|
98
|
+
| Network analysis | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis network <capture.har> [--slow-ms N]` | The report's `network` block, counted and flagged by the parser: totals, per-endpoint timing and size, and an `issue` on each — `failed`, `slow`, `polling`, `duplicate`, `n-plus-one`, `large-payload`, `uncached`. Drop it straight into the result | Read `performance.getEntriesByType('resource')` and say in the report that counts were observed rather than parsed |
|
|
88
99
|
| Secret redaction | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis redact <file>` | The file with credentials and tokens masked, for evidence excerpts | Redact by hand before the excerpt is written |
|
|
89
100
|
| Failure classification | `node <SKILL_DIR>/scripts/qa-tool.mjs analysis classify "<error message>"` | `{classification, confidence, reason}` for a console or network error | Classify per [finding-taxonomy.md](references/finding-taxonomy.md) |
|
|
90
101
|
|
|
91
102
|
A missing `qa-tool.mjs` means the engine is not installed; run `qa doctor`.
|
|
92
103
|
|
|
93
|
-
**
|
|
104
|
+
**You produce structured data; the pack produces every document.** The result JSON is the source of truth and every format is rendered from it. You never write HTML, CSS, or styling of any kind — the contract has no field to carry a presentation hint, which is what makes a report from this skill identical whichever agent ran it. Hand-typing the page is how the first live run silently dropped `actual`, `expected`, and `fixDirection` from every finding while the JSON held all three.
|
|
105
|
+
|
|
106
|
+
**Register artifacts; do not inline paths.** Every captured file gets an `artifacts[]` entry, and evidence points at it by `artifactId`. Paths are relative to the result JSON. That is what makes `artifacts verify` meaningful and what stopped the second live run's screenshots from rendering as broken images.
|
|
94
107
|
|
|
95
108
|
## Output
|
|
96
109
|
|
|
97
|
-
Write under `qa-artifacts/explore-<run-id
|
|
110
|
+
Write under `qa-artifacts/explore-<run-id>/`, so the whole run folder moves as a unit:
|
|
111
|
+
|
|
112
|
+
- `explore-result.json` — machine-readable result conforming to [contracts/explore-result.schema.json](contracts/explore-result.schema.json); written first, and the source of every rendering below
|
|
113
|
+
- `report/` — **the deliverable.** A portable bundle written by `report-bundle`: `index.html`, `report.json`, `report.md`, `manifest.json`, and `assets/`. It opens offline in any browser and every link is verified to resolve inside it. `--zip` adds `report.zip` for sending.
|
|
114
|
+
- `screenshots/`, `network/`, `console/`, `dom/` — proof, registered in `artifacts[]` and referenced by `artifactId`; the bundle copies these into its own `assets/` tree
|
|
115
|
+
|
|
116
|
+
Point the user at `report/index.html`. A hosted preview — a Claude Artifact, a Cursor preview, any cloud viewer — is an optional convenience that expires; it must never be presented as the deliverable.
|
|
98
117
|
|
|
99
|
-
- `
|
|
100
|
-
- `explore-result.json` — machine-readable result conforming to [contracts/explore-result.schema.json](contracts/explore-result.schema.json); written first, and the source of the two renderings below
|
|
101
|
-
- `explore-report.html` — the report a person reads, **rendered** from the JSON by `report-html`
|
|
102
|
-
- `explore-report.md` — the same content as Markdown, with the rendered attribution footer appended
|
|
118
|
+
On request, also `report-html --embed` for one self-contained file, or `report-export` to SARIF (code-scanning viewers), JUnit (CI), or CSV (a tracking sheet). There is no PDF writer: the HTML has a print stylesheet that forces findings open and prints link targets, so **Print → Save as PDF** is the supported path.
|
|
103
119
|
|
|
104
|
-
Validate the JSON
|
|
120
|
+
Validate the JSON and run `artifacts verify` before rendering, and again before declaring completion. Present a short prose verdict (severity counts + top findings) in the conversation, and point to the artifact paths.
|