qualiow-exploratory-testing 1.3.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +26 -0
- package/.claude-plugin/plugin.json +22 -0
- package/.env.example +27 -0
- package/CHANGELOG.md +259 -0
- package/README.md +472 -204
- package/agents/qa-diff-indexer-agent.md +81 -0
- package/agents/qa-gather-agent.md +41 -0
- package/agents/qa-page-mapper-agent.md +74 -0
- package/agents/qa-reporting-agent.md +103 -0
- package/{scripts → bin}/doctor-mobile.sh +57 -18
- package/bin/mcli +15 -9
- package/bin/mobile-cli.mjs +574 -286
- package/bin/qualiow +30 -0
- package/{scripts → bin}/setup-mobile.sh +56 -19
- package/bin/wadb +10 -4
- package/bin/wk-ios +29 -17
- package/bin/wkeval.mjs +7 -1
- package/data/domains/_default.yml +1 -1
- package/data/knowledge/changelog.yml +30 -12
- package/data/knowledge/manifest.yml +355 -127
- package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
- package/data/knowledge/releases/v0.1.0/release.yml +8 -1
- package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
- package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
- package/data/knowledge/releases/v0.6.0/release.yml +30 -0
- package/data/security/SECURITY-POLICY.md +273 -0
- package/data/targets/_default.yml +1 -1
- package/data/targets/_example-api-only.yml +80 -0
- package/data/targets/_example-mobile-emulation.yml +2 -2
- package/data/targets/_example-native-mobile.yml +2 -2
- package/data/targets/_example-sim-android-chrome.yml +1 -1
- package/data/targets/_example-sim-ios-safari.yml +1 -1
- package/data/templates/api-probe-matrix.md +3 -3
- package/data/templates/bug-report.md +29 -151
- package/data/templates/coverage-map.md +9 -6
- package/data/templates/expected-behaviour.md +3 -2
- package/data/templates/session-charter.md +3 -0
- package/data/templates/session-report.md +57 -80
- package/dist/cli/index.js +2286 -701
- package/dist/index.d.ts +637 -81
- package/dist/index.js +2058 -228
- package/docs/BACKEND-VERIFICATION.md +39 -9
- package/docs/GETTING-STARTED.md +300 -121
- package/docs/MOBILE-SETUP.md +149 -18
- package/hooks/hooks.json +27 -0
- package/hooks/scripts/read-guard.mjs +95 -0
- package/hooks/scripts/secret-patterns.d.mts +8 -0
- package/hooks/scripts/secret-patterns.mjs +136 -0
- package/hooks/scripts/write-guard.mjs +87 -0
- package/package.json +33 -18
- package/skills/qa-explore/SKILL.md +32 -25
- package/skills/qa-explore/phases/00-setup.md +76 -42
- package/skills/qa-explore/phases/01-auth.md +32 -15
- package/skills/qa-explore/phases/02-charter.md +16 -5
- package/skills/qa-explore/phases/03-discovery.md +93 -27
- package/skills/qa-explore/phases/04-journeys.md +2 -2
- package/skills/qa-explore/phases/05-features.md +39 -24
- package/skills/qa-explore/phases/06-edge-cases.md +24 -13
- package/skills/qa-explore/phases/07-reporting.md +102 -142
- package/skills/qa-explore/references/context-integration.md +1 -1
- package/skills/qa-explore/references/delegation-rules.md +82 -0
- package/skills/qa-explore/references/output-contract.md +192 -0
- package/skills/qa-explore/references/paths.md +108 -0
- package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
- package/skills/qa-explore/references/security-rules.md +70 -25
- package/skills/qa-explore/references/session-rules.md +2 -1
- package/skills/qa-explore/references/severity-guide.md +31 -4
- package/skills/qa-explore-cleanup/SKILL.md +51 -15
- package/skills/qa-explore-feedback/SKILL.md +23 -5
- package/skills/qa-explore-mobile/SKILL.md +86 -64
- package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
- package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
- package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
- package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
- package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
- package/skills/qa-explore-mobile/phases/05-features.md +9 -9
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
- package/skills/qa-explore-mobile/phases/07-reporting.md +108 -60
- package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
- package/skills/qa-explore-quick/SKILL.md +71 -26
- package/skills/qa-explore-report/SKILL.md +65 -35
- package/skills/qa-gather/SKILL.md +44 -8
- package/skills/qa-knowledge-add/SKILL.md +25 -9
- package/skills/qa-knowledge-list/SKILL.md +37 -40
- package/skills/qa-target-setup/SKILL.md +63 -41
- package/skills/qa-verify-backend/SKILL.md +31 -19
- package/skills/qa-verify-backend/phases/00-setup.md +57 -12
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +2 -2
- package/skills/qa-verify-backend/phases/02-static-review.md +11 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +5 -3
- package/skills/qa-verify-backend/phases/03b-api-verification.md +24 -9
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +18 -2
- package/skills/qa-verify-backend/phases/05-reporting.md +72 -12
- package/skills/qa-verify-backend/references/api-probes.md +28 -1
- package/skills/qa-verify-backend/references/payload-verification.md +1 -1
- package/skills/qa-verify-backend/references/release-readiness.md +1 -1
- package/skills/qa-verify-backend/references/safety-rules.md +13 -8
- package/data/domains/_default.md +0 -181
- package/data/domains/ecommerce.md +0 -192
- package/data/domains/fintech.md +0 -200
- package/data/domains/identity.md +0 -88
- package/data/domains/marketing.md +0 -194
- package/data/domains/saas.md +0 -197
- package/dist/cli/index.d.ts +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/index.js.map +0 -1
|
@@ -38,10 +38,19 @@ implements it.
|
|
|
38
38
|
### 2. Create a target config
|
|
39
39
|
|
|
40
40
|
```bash
|
|
41
|
-
cp data/targets/_example-backend.yml data/targets/local-my-service.yml
|
|
41
|
+
cp data/targets/_example-backend.yml data/targets/local-my-service.yml # service behind a UI
|
|
42
|
+
cp data/targets/_example-api-only.yml data/targets/local-my-api.yml # HTTP API, no UI
|
|
42
43
|
```
|
|
43
44
|
|
|
45
|
+
Both templates ship with the package, so the `cp` works from an installed copy as well as
|
|
46
|
+
from a checkout.
|
|
47
|
+
|
|
44
48
|
Fill in `base_url`, `environment.kind`, the `backend.resources` names, and `source.branch`.
|
|
49
|
+
For an API-only service most of that is empty: what matters is the `api:` block — its
|
|
50
|
+
`auth` mode, the env var holding the key, and the two allowlists. The `domain:` field names a
|
|
51
|
+
profile in `data/domains/` (`_default.yml`, `fintech.yml`, `identity.yml`, `saas.yml`,
|
|
52
|
+
`ecommerce.yml`, `marketing.yml`) — YAML only since 2.0.0 — which supplies the risk ranking
|
|
53
|
+
and integrity checks the session applies.
|
|
45
54
|
|
|
46
55
|
Two rules:
|
|
47
56
|
|
|
@@ -57,10 +66,14 @@ Two rules:
|
|
|
57
66
|
/qa-verify-backend --target local-my-service --context output/context/TICKET-123-context.md
|
|
58
67
|
```
|
|
59
68
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
69
|
+
All four flags:
|
|
70
|
+
|
|
71
|
+
| Flag | Effect |
|
|
72
|
+
|------|--------|
|
|
73
|
+
| `--static-only` | Branch review only — skip the live, API and end-to-end lanes. Use when you have no cloud credentials |
|
|
74
|
+
| `--api-only` | Skip the cloud lane and verify the HTTP surface |
|
|
75
|
+
| `--no-e2e` | Run everything except the write-path phase |
|
|
76
|
+
| `--parity <target-id>` | Run the same API matrix in a second environment and compare the two column by column |
|
|
64
77
|
|
|
65
78
|
### 4. Explore the UI, if the ticket has one
|
|
66
79
|
|
|
@@ -109,6 +122,20 @@ playwright-cli -s=my-env open https://service.example.com/ --headed --profile .a
|
|
|
109
122
|
playwright-cli -s=my-env eval "$(cat probes/search-cases.js)" --raw
|
|
110
123
|
```
|
|
111
124
|
|
|
125
|
+
**When the service has no UI**, none of that applies: there is no page to run inside. Declare
|
|
126
|
+
`api.auth: api-key-env` with `token_env` naming the variable and `header_name` naming the
|
|
127
|
+
header, and the lane runs out of browser, reading the key from the environment at run time.
|
|
128
|
+
Two obligations come with it. Prove the credential is doing something before the first probe —
|
|
129
|
+
one call without the header must be rejected, one with it must succeed, both recorded — or a
|
|
130
|
+
`401` rendered as JSON will read like a passing case all session. And skip the differential
|
|
131
|
+
pass with the reason stated: with no second surface every finding is "API only", which is not
|
|
132
|
+
a downgrade but the whole contract, since there is no client guard in front of the endpoint
|
|
133
|
+
for anyone.
|
|
134
|
+
|
|
135
|
+
State-changing calls are limited to `api.write_allowlist` and belong to phase 4. Where a
|
|
136
|
+
service offers a workspace reset, it stays deliberate: it is the one call that erases the
|
|
137
|
+
evidence for everything already observed.
|
|
138
|
+
|
|
112
139
|
Cases come from the ACs plus the standard families — length boundaries, tokenisation,
|
|
113
140
|
metacharacters of whatever query language sits underneath, the four different kinds of
|
|
114
141
|
nothing (`""`, `null`, whitespace-only, field absent), enum values valid and invalid,
|
|
@@ -137,7 +164,7 @@ And the payload is held next to the screen, because a correct response can still
|
|
|
137
164
|
user as a wrong number — a formatter that guesses what a value is, a unit applied twice,
|
|
138
165
|
rounding that crosses a threshold, a truncated figure shown as a total. That defect is
|
|
139
166
|
invisible from either surface alone, and it usually belongs to a different change than the one
|
|
140
|
-
under test. See
|
|
167
|
+
under test. See `skills/qa-verify-backend/references/payload-verification.md`.
|
|
141
168
|
|
|
142
169
|
## Verdicts
|
|
143
170
|
|
|
@@ -176,7 +203,7 @@ Each AC is routed to the channel that can actually falsify it
|
|
|
176
203
|
## Safety
|
|
177
204
|
|
|
178
205
|
The full policy is in
|
|
179
|
-
|
|
206
|
+
`skills/qa-verify-backend/references/safety-rules.md`. The parts worth knowing before
|
|
180
207
|
your first session:
|
|
181
208
|
|
|
182
209
|
- **Production hard stop.** If the account, profile, resource name or URL contains `prod`,
|
|
@@ -224,13 +251,16 @@ Knowledge base entries loaded by the skill (`data/knowledge/`):
|
|
|
224
251
|
| `technique-expected-behaviour-specification` | The session found behaviour no AC covers |
|
|
225
252
|
| `technique-config-surface-verification` | The change under test *is* the configuration mechanism |
|
|
226
253
|
| `technique-release-readiness-verification` | The ask is "is this release good to go" |
|
|
254
|
+
| `technique-exactly-once-verification` | Money moves, or any write that must land exactly once — assert the balance delta, not the status field, and attack redelivery and identity collision |
|
|
255
|
+
| `technique-async-callback-contracts` | A webhook or callback endpoint: what it owes its caller, and what it does with an event it cannot match |
|
|
227
256
|
|
|
228
|
-
Browse them with `/qa-knowledge-list`.
|
|
257
|
+
Browse them with `/qa-knowledge-list`. The knowledge base ships 29 entries across releases
|
|
258
|
+
v0.1.0–v0.6.0.
|
|
229
259
|
|
|
230
260
|
## Verifying a release rather than a ticket
|
|
231
261
|
|
|
232
262
|
When the ask is "is this release good to go" instead of "does this ticket meet its ACs", follow
|
|
233
|
-
|
|
263
|
+
`skills/qa-verify-backend/references/release-readiness.md`. It changes the shape of
|
|
234
264
|
the session:
|
|
235
265
|
|
|
236
266
|
- **The build table comes first, for everything.** Per ticket, is the commit it depends on
|
package/docs/GETTING-STARTED.md
CHANGED
|
@@ -2,123 +2,279 @@
|
|
|
2
2
|
|
|
3
3
|
Step-by-step guide to running your first AI-powered exploratory testing session.
|
|
4
4
|
|
|
5
|
-
## 1.
|
|
5
|
+
## 1. Install
|
|
6
6
|
|
|
7
7
|
### System requirements
|
|
8
8
|
|
|
9
|
-
- Node.js >=
|
|
10
|
-
-
|
|
11
|
-
-
|
|
9
|
+
- **Node.js >= 22.4** (`node --version`)
|
|
10
|
+
- **Claude Code**, installed and configured — the skills run as slash commands inside it
|
|
11
|
+
- `playwright-cli` arrives as a dependency of this package. There is nothing else to install
|
|
12
|
+
for the web path.
|
|
12
13
|
|
|
13
14
|
### Install the package
|
|
14
15
|
|
|
15
16
|
```bash
|
|
16
|
-
npm install -g
|
|
17
|
+
npm install -g qualiow-exploratory-testing
|
|
18
|
+
qualiow --version # prints the installed package version, e.g. 2.1.0
|
|
17
19
|
```
|
|
18
20
|
|
|
19
|
-
###
|
|
21
|
+
### Initialise a project
|
|
20
22
|
|
|
21
|
-
|
|
23
|
+
```bash
|
|
24
|
+
cd my-project
|
|
25
|
+
qualiow init
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
That writes:
|
|
29
|
+
|
|
30
|
+
| What | Where |
|
|
31
|
+
|------|-------|
|
|
32
|
+
| the 11 skills | `.claude/skills/qa-*/` |
|
|
33
|
+
| the four sub-agents | `.claude/agents/` |
|
|
34
|
+
| knowledge base, domain profiles, templates, security policy | `data/knowledge/`, `data/domains/`, `data/templates/`, `data/security/` |
|
|
35
|
+
| the `_default` target (add `--include-examples` for the `_example-*` templates and `testers-ai.yml`) | `data/targets/` |
|
|
36
|
+
| the mobile driver plus `setup-mobile.sh` and `doctor-mobile.sh`, executable | `qa/bin/` |
|
|
37
|
+
| the credential template | `qa/.env.example` |
|
|
38
|
+
| empty session/bug/context trees and the auth directory | `output/sessions/`, `output/bugs/`, `output/context/`, `.auth/` |
|
|
39
|
+
| a `# Qualiow` block appended to `.gitignore` | `.gitignore` |
|
|
40
|
+
|
|
41
|
+
Re-running `qualiow init` changes nothing unless you pass `--force`. Use `--dry-run` to see
|
|
42
|
+
the plan first.
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
cp qa/.env.example qa/.env # credentials go here, never in a YAML file
|
|
46
|
+
npx playwright-cli install --skills # optional: the official Playwright skill alongside
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Optionally, install the guard hooks
|
|
22
50
|
|
|
23
51
|
```bash
|
|
24
|
-
|
|
52
|
+
qualiow init --hooks
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Optional, and only for npm projects — a plugin install already ships the same two hooks and
|
|
56
|
+
has them on by default, so do not enable both in one project. `--hooks` copies the guard
|
|
57
|
+
scripts to `qa/hooks/` and merges into `.claude/settings.json` two `PreToolUse` entries and
|
|
58
|
+
three `permissions.allow` rules. The merge is by exact string: running it again changes
|
|
59
|
+
nothing, and hooks you already had are kept.
|
|
60
|
+
|
|
61
|
+
What the guards do: `read-guard.mjs` denies a whole-file `Read` of a qualiow file over
|
|
62
|
+
`QUALIOW_READ_MAX_LINES` lines (default 300) — the knowledge manifest, a knowledge release
|
|
63
|
+
file, a session phase file or a raw snapshot — and names the cheaper route in the refusal; a
|
|
64
|
+
`Read` with `offset` or `limit` passes. `write-guard.mjs` refuses a `Write` or `Edit` under
|
|
65
|
+
`output/` whose content matches the redaction list, so a secret never reaches disk. Neither
|
|
66
|
+
one looks at anything else in your project. `QUALIOW_HOOKS=off` disables both.
|
|
67
|
+
|
|
68
|
+
To write it by hand instead, this is the whole of it:
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{
|
|
72
|
+
"hooks": {
|
|
73
|
+
"PreToolUse": [
|
|
74
|
+
{
|
|
75
|
+
"matcher": "Read",
|
|
76
|
+
"hooks": [
|
|
77
|
+
{ "type": "command", "command": "node \"$CLAUDE_PROJECT_DIR/qa/hooks/read-guard.mjs\"" }
|
|
78
|
+
]
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"matcher": "Write|Edit|MultiEdit",
|
|
82
|
+
"hooks": [
|
|
83
|
+
{ "type": "command", "command": "node \"$CLAUDE_PROJECT_DIR/qa/hooks/write-guard.mjs\"" }
|
|
84
|
+
]
|
|
85
|
+
}
|
|
86
|
+
]
|
|
87
|
+
},
|
|
88
|
+
"permissions": {
|
|
89
|
+
"allow": [
|
|
90
|
+
"Bash(playwright-cli:*)",
|
|
91
|
+
"Bash(npx playwright-cli:*)",
|
|
92
|
+
"Bash(qualiow:*)"
|
|
93
|
+
]
|
|
94
|
+
},
|
|
95
|
+
"env": {
|
|
96
|
+
"QUALIOW_READ_MAX_LINES": "300"
|
|
97
|
+
}
|
|
98
|
+
}
|
|
25
99
|
```
|
|
26
100
|
|
|
27
|
-
###
|
|
101
|
+
### Or install it as a Claude Code plugin
|
|
102
|
+
|
|
103
|
+
The repository is its own Claude Code marketplace, so an install is two commands:
|
|
28
104
|
|
|
29
105
|
```bash
|
|
30
|
-
qualiow
|
|
106
|
+
claude plugin marketplace add willcoliveira/qualiow-exploratory-testing-skills
|
|
107
|
+
claude plugin install qualiow@qualiow
|
|
31
108
|
```
|
|
32
109
|
|
|
33
|
-
|
|
110
|
+
Skills then appear namespaced — `/qualiow:qa-explore`, `/qualiow:qa-verify-backend` — and the
|
|
111
|
+
data files resolve from `${CLAUDE_PLUGIN_ROOT}/data/` unless the project has its own `data/`
|
|
112
|
+
or `qa/` directory, which wins. Pointing Claude Code at a local checkout
|
|
113
|
+
(`claude --plugin-dir /path/to/qualiow-exploratory-testing-skills`) is the development path
|
|
114
|
+
and behaves the same, except that `bin/` is not put on `PATH`, so the helpers are called by
|
|
115
|
+
their full path (`${CLAUDE_PLUGIN_ROOT}/bin/mcli`).
|
|
34
116
|
|
|
35
|
-
|
|
117
|
+
A plugin install has no `dist/`, so the `qualiow` CLI runs through the `bin/qualiow` launcher
|
|
118
|
+
shim: it uses the local build when the checkout has one, and otherwise fetches the published
|
|
119
|
+
npm package with `npx`, pinned to the version in `.claude-plugin/plugin.json`. The first run
|
|
120
|
+
of that path needs network. A plain git checkout is described in the project README.
|
|
36
121
|
|
|
37
|
-
|
|
122
|
+
## 2. Run your first session
|
|
38
123
|
|
|
39
|
-
|
|
40
|
-
|------|-----|--------|
|
|
41
|
-
| SauceDemo | https://www.saucedemo.com | ecommerce |
|
|
42
|
-
| ParaBank | https://parabank.parasoft.com/parabank/index.htm | fintech |
|
|
43
|
-
| DemoQA | https://demoqa.com | general |
|
|
124
|
+
### Option A: explore a public site
|
|
44
125
|
|
|
45
|
-
|
|
126
|
+
Point `/qa-explore` at any public site. A good first target is
|
|
127
|
+
<https://testers.ai/testing/>, which ships as the `testers-ai` target config.
|
|
46
128
|
|
|
47
129
|
```bash
|
|
48
|
-
/qa-explore https://
|
|
130
|
+
/qa-explore https://testers.ai/testing/
|
|
49
131
|
```
|
|
50
132
|
|
|
51
133
|
Claude will:
|
|
52
|
-
1. Launch a browser via Playwright CLI
|
|
53
|
-
2. Read the knowledge base for applicable heuristics
|
|
54
|
-
3. Explore the site across 3-4 phases (discovery, functional, edge cases, synthesis)
|
|
55
|
-
4. File individual bug reports in `output/sessions/<session-id>/bugs/`
|
|
56
|
-
5. Write a session report at `output/sessions/<session-id>/session-report.md`
|
|
57
|
-
|
|
58
|
-
### Option B: Quick check on a specific page
|
|
59
134
|
|
|
60
|
-
|
|
135
|
+
1. Resolve the target config and credentials, and create the session directory
|
|
136
|
+
2. Authenticate if the target requires it
|
|
137
|
+
3. Write a charter — what is being tested, why, and what "done" looks like
|
|
138
|
+
4. Load the heuristics that apply to this domain with `qualiow kb digest --for explore
|
|
139
|
+
--domain <domain>`, rather than reading the manifest and the entry files whole
|
|
140
|
+
5. Explore across eight phases, saving findings to disk between each
|
|
141
|
+
6. File one bug report per finding in `bugs/`
|
|
142
|
+
7. Write `phase-7-notes.md` — the executive summary, coverage map, recommendations and
|
|
143
|
+
reflection — then hand the assembly of `session-report.md` and the
|
|
144
|
+
`qualiow session finalize <session-dir>` call to the `qa-reporting-agent` sub-agent
|
|
145
|
+
|
|
146
|
+
The eight phases and their share of the 45-minute cap:
|
|
147
|
+
|
|
148
|
+
| # | Phase | Minutes | What happens |
|
|
149
|
+
|---|-------|---------|--------------|
|
|
150
|
+
| 0 | Setup | 2 | Resolve config, create session dir, open the browser session |
|
|
151
|
+
| 1 | Auth | 2 | Log in (storage state or adaptive login) |
|
|
152
|
+
| 2 | Charter | 5 | Understand the business, risk-rank features P0–P3 |
|
|
153
|
+
| 3 | Discovery | 6 | Map the app: pages, forms, roles, console and network health |
|
|
154
|
+
| 4 | Journeys | 10 | End-to-end user journeys with data-integrity checks |
|
|
155
|
+
| 5 | Features | 10 | Deep testing of the highest-risk features |
|
|
156
|
+
| 6 | Edge cases | 6 | Boundaries, negative input, security, "what is missing?" |
|
|
157
|
+
| 7 | Reporting | 4 | Reflection, coverage map, session report, cleanup |
|
|
158
|
+
|
|
159
|
+
Three points in that run are not done by the session itself. At setup the knowledge base
|
|
160
|
+
arrives as a `qualiow kb digest` call rather than as the manifest and the entry files. In
|
|
161
|
+
discovery, a page whose raw accessibility tree runs past 300 lines is saved to
|
|
162
|
+
`snapshots/` and handed to the `qa-page-mapper-agent` sub-agent, which returns the forms,
|
|
163
|
+
navigation, controls and error text as a short map — the session works from that instead of
|
|
164
|
+
the tree, and keeps driving the page itself. At the end, the session writes its judgements
|
|
165
|
+
into `phase-7-notes.md` and the `qa-reporting-agent` sub-agent assembles `session-report.md`
|
|
166
|
+
around them and runs `finalize`; the bug reports, `stats.json` and every word of the summary
|
|
167
|
+
and reflection are still the session's own.
|
|
168
|
+
|
|
169
|
+
### Option B: quick check on one page
|
|
61
170
|
|
|
62
171
|
```bash
|
|
63
172
|
/qa-explore-quick https://app.example.com/checkout
|
|
64
173
|
```
|
|
65
174
|
|
|
66
|
-
|
|
175
|
+
A 15-minute focused session — no full site mapping. It produces the same artefacts as a full
|
|
176
|
+
session, so reports, feedback and cleanup all work on it.
|
|
177
|
+
|
|
178
|
+
### Option C: give it the requirements first
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
/qa-gather
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Point it at a ticket, a PR, a design doc, a URL or pasted text. It runs in the
|
|
185
|
+
`qa-gather-agent` sub-agent, which sees only the invocation, so put the paths, the URLs or the
|
|
186
|
+
pasted text in the same message — it cannot come back with a question, and anything a source
|
|
187
|
+
leaves unsaid is marked `[GAP]` or `[ASSUMPTION]`. It writes
|
|
188
|
+
`output/context/<TICKET>-context.md`, which you then pass to a session:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
/qa-explore --target my-app --context output/context/TICKET-123-context.md
|
|
192
|
+
```
|
|
67
193
|
|
|
68
|
-
## 3.
|
|
194
|
+
## 3. Understand the output
|
|
69
195
|
|
|
70
|
-
|
|
196
|
+
Every session — web, quick, mobile or backend — lands in
|
|
197
|
+
`output/sessions/<YYYY-MM-DD-HHmm>-<kind>-<slug>/`, where `kind` is `explore`, `quick`,
|
|
198
|
+
`mobile` or `backend`:
|
|
71
199
|
|
|
72
200
|
```
|
|
73
|
-
output/sessions
|
|
74
|
-
charter.md # what
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
phase-3-
|
|
78
|
-
|
|
201
|
+
output/sessions/2026-09-08-1813-explore-parabank/
|
|
202
|
+
charter.md # what is being tested and why
|
|
203
|
+
session-log.md # real-time findings, with the WHY not just the WHAT
|
|
204
|
+
progress.json
|
|
205
|
+
phase-3-discovery.md # phase artefacts — the file number is the phase number
|
|
206
|
+
phase-4-journeys.md
|
|
207
|
+
phase-5-features.md
|
|
208
|
+
phase-6-edge-cases.md
|
|
209
|
+
phase-7-notes.md # the session's own summary, coverage map, recommendations
|
|
79
210
|
bugs/
|
|
80
|
-
BUG-001.md #
|
|
211
|
+
BUG-001.md # one bug = one report
|
|
81
212
|
BUG-002.md
|
|
82
|
-
...
|
|
83
213
|
screenshots/
|
|
84
|
-
|
|
85
|
-
|
|
214
|
+
BUG-001.png
|
|
215
|
+
snapshots/ # raw page snapshots — working files, never part of the report
|
|
216
|
+
videos/
|
|
217
|
+
session-report.md # the deliverable
|
|
218
|
+
stats.json # machine-readable session metrics
|
|
86
219
|
```
|
|
87
220
|
|
|
221
|
+
Plus, at the top level:
|
|
222
|
+
|
|
223
|
+
- `output/sessions/INDEX.md` — one row per session
|
|
224
|
+
(`| Date | Kind | Target | Bugs | Duration | Status | Report |`)
|
|
225
|
+
- `output/bugs/all-bugs.md` — every bug across every session
|
|
226
|
+
|
|
227
|
+
Both are written by `qualiow session finalize`, which the session runs as its last step. It
|
|
228
|
+
checks the session against the output contract first — a confidentiality header on every
|
|
229
|
+
markdown file, `stats.json` in the metrics shape, no unredacted secret anywhere — and refuses
|
|
230
|
+
to append rows until the violations it lists are fixed. Run it by hand at any time:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
qualiow session finalize latest --check # report violations, write nothing
|
|
234
|
+
qualiow session finalize latest --redact # rewrite files that still carry a secret
|
|
235
|
+
qualiow session finalize latest # validate, then append the rows
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
Finalizing twice changes nothing — the rows are appended once.
|
|
239
|
+
|
|
88
240
|
### Bug report structure
|
|
89
241
|
|
|
90
|
-
Each
|
|
242
|
+
Each report opens with the confidentiality blockquote, then:
|
|
91
243
|
|
|
92
|
-
- **Title**
|
|
93
|
-
- **Severity**
|
|
94
|
-
|
|
95
|
-
- **Expected
|
|
96
|
-
- **
|
|
97
|
-
- **
|
|
244
|
+
- **Title** — `# BUG-NNN: [Component] fails [Condition] causing [Impact]`
|
|
245
|
+
- **Severity** — Critical, High, Medium or Low (conservative by design) — plus Priority,
|
|
246
|
+
Component, URL, Environment and reproduction rate
|
|
247
|
+
- **Summary**, **Expected Behavior**, **Actual Behavior**
|
|
248
|
+
- **Steps to Reproduce** — numbered, specific enough for someone else to follow
|
|
249
|
+
- **Business Impact** — revenue, trust, regulatory, data and scale. Mandatory
|
|
250
|
+
- **Evidence** — screenshot, video, logs, console errors, network failures
|
|
251
|
+
- **Recommended Fix Priority**
|
|
98
252
|
|
|
99
253
|
### Session report structure
|
|
100
254
|
|
|
101
|
-
|
|
255
|
+
- Session metadata and executive summary
|
|
256
|
+
- Summary stats and the bug table
|
|
257
|
+
- **Coverage map** (`| Area | Risk | Status | Bugs | Notes |`) — including what was *not* tested
|
|
258
|
+
- Observations that are not bugs but are worth discussing
|
|
259
|
+
- Recommendations, reflection, session stats
|
|
102
260
|
|
|
103
|
-
|
|
104
|
-
- Bug table with severity distribution
|
|
105
|
-
- Coverage map showing which areas were tested
|
|
106
|
-
- Observations (not bugs, but worth discussing)
|
|
107
|
-
- Recommendations for follow-up testing
|
|
261
|
+
## 4. Configure a target
|
|
108
262
|
|
|
109
|
-
|
|
263
|
+
For repeated testing of the same application, save a target config.
|
|
110
264
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
### Generate a target config interactively
|
|
265
|
+
### Interactively
|
|
114
266
|
|
|
115
267
|
```bash
|
|
116
268
|
/qa-target-setup
|
|
117
269
|
```
|
|
118
270
|
|
|
119
|
-
|
|
271
|
+
It navigates to the URL, detects the auth requirement, walks you through the login flow, and
|
|
272
|
+
writes `qa/target.yml` — the project-local target. Pass `--shared` to write
|
|
273
|
+
`data/targets/<id>.yml` instead, for a config the whole team uses.
|
|
274
|
+
|
|
275
|
+
### Or by hand
|
|
120
276
|
|
|
121
|
-
|
|
277
|
+
`qa/target.yml`:
|
|
122
278
|
|
|
123
279
|
```yaml
|
|
124
280
|
id: my-app
|
|
@@ -147,93 +303,103 @@ scope:
|
|
|
147
303
|
- /api/*
|
|
148
304
|
```
|
|
149
305
|
|
|
150
|
-
|
|
306
|
+
Validate it before you use it:
|
|
151
307
|
|
|
152
308
|
```bash
|
|
153
|
-
|
|
309
|
+
qualiow validate --all
|
|
154
310
|
```
|
|
155
311
|
|
|
312
|
+
### Resolution order
|
|
313
|
+
|
|
314
|
+
```
|
|
315
|
+
--target <name> → qa/target.yml → data/targets/_default.yml
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
Domains come from `data/domains/*.yml` (`_default`, `ecommerce`, `fintech`, `identity`,
|
|
319
|
+
`marketing`, `saas`), and drive the risk ranking, completeness checklist and data-integrity
|
|
320
|
+
checks a session applies.
|
|
321
|
+
|
|
156
322
|
### Authentication strategies
|
|
157
323
|
|
|
158
324
|
| Strategy | When to use | Setup |
|
|
159
325
|
|----------|-------------|-------|
|
|
160
326
|
| `none` | Public sites, no login required | Default |
|
|
161
|
-
| `storage_state` |
|
|
162
|
-
| `credentials` | Username/password login |
|
|
163
|
-
| `token` | API token or bearer auth |
|
|
327
|
+
| `storage_state` | Cookie/session auth | Log in once by hand, then `playwright-cli state-save .auth/<id>.json` |
|
|
328
|
+
| `credentials` | Username/password login | Values in `qa/.env`, referenced by variable name |
|
|
329
|
+
| `token` | API token or bearer auth | Value in `qa/.env`, referenced by variable name |
|
|
164
330
|
|
|
165
|
-
|
|
331
|
+
### Credentials
|
|
166
332
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
MY_APP_USERNAME=testuser
|
|
170
|
-
MY_APP_PASSWORD=testpass123
|
|
171
|
-
```
|
|
172
|
-
|
|
173
|
-
## 5. How to Add Knowledge
|
|
174
|
-
|
|
175
|
-
The knowledge base drives Claude's testing strategy. Add domain-specific heuristics, techniques, or checklists.
|
|
176
|
-
|
|
177
|
-
### Add knowledge interactively
|
|
333
|
+
Secrets live in `qa/.env` (gitignored), falling back to `.env`. YAML files reference them **by
|
|
334
|
+
variable name only** — never by value. The names the shipped configs use:
|
|
178
335
|
|
|
179
336
|
```bash
|
|
180
|
-
|
|
337
|
+
# qa/.env
|
|
338
|
+
QA_USER=testuser
|
|
339
|
+
QA_PASS=...
|
|
340
|
+
QA_TOKEN=...
|
|
341
|
+
|
|
342
|
+
# /qa-verify-backend
|
|
343
|
+
QA_AWS_PROFILE=... # read-only profile. Never point this at production
|
|
344
|
+
QA_AWS_REGION=eu-west-1
|
|
345
|
+
QA_API_TOKEN=... # only for a target declaring api.auth: bearer-env
|
|
346
|
+
EXAMPLE_API_KEY=... # a no-UI service names its own variable; see _example-api-only.yml
|
|
181
347
|
```
|
|
182
348
|
|
|
183
|
-
|
|
349
|
+
## 5. Add knowledge
|
|
350
|
+
|
|
351
|
+
The knowledge base drives Claude's testing strategy: 29 entries across releases v0.1.0–v0.6.0,
|
|
352
|
+
indexed by `data/knowledge/manifest.yml`.
|
|
184
353
|
|
|
185
354
|
```bash
|
|
186
|
-
/qa-knowledge-
|
|
355
|
+
/qa-knowledge-add # add a heuristic, technique, checklist or reference
|
|
356
|
+
/qa-knowledge-list # browse what is there, by type, domain or tag
|
|
187
357
|
```
|
|
188
358
|
|
|
189
|
-
### Knowledge types
|
|
190
|
-
|
|
191
359
|
| Type | Purpose | Example |
|
|
192
360
|
|------|---------|---------|
|
|
193
361
|
| `heuristic` | Mental model or framework for exploration | SFDIPOT, FEW HICCUPPS |
|
|
194
|
-
| `technique` |
|
|
362
|
+
| `technique` | A specific testing method | boundary testing, authenticated API probing |
|
|
195
363
|
| `checklist` | Verification list for a specific area | WCAG accessibility checks |
|
|
196
364
|
| `reference` | Background material | exploratory testing type definitions |
|
|
197
365
|
|
|
198
|
-
|
|
366
|
+
Entries are YAML files in `data/knowledge/releases/<version>/entries/`. After editing by hand,
|
|
367
|
+
run `qualiow kb sync` to regenerate the manifest and `qualiow kb check` to verify it.
|
|
199
368
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
369
|
+
A session does not read the manifest or the entry files. It calls `qualiow kb digest`, which
|
|
370
|
+
prints the entries that apply — the always-load set plus whatever the domain, the tags and the
|
|
371
|
+
skill select — as a few lines each:
|
|
203
372
|
|
|
204
373
|
```bash
|
|
205
|
-
qualiow
|
|
374
|
+
qualiow kb digest --for explore --domain fintech # what a session loads at setup
|
|
375
|
+
qualiow kb digest --for backend # /qa-verify-backend's set
|
|
376
|
+
qualiow kb digest --tag security data-integrity # one or more tags
|
|
377
|
+
qualiow kb digest --entry heuristic-test-tours # one entry, in full
|
|
206
378
|
```
|
|
207
379
|
|
|
208
|
-
|
|
380
|
+
`--for` is driven by the optional `loading_strategy.by_skill` block in the manifest, so you can
|
|
381
|
+
decide which entries a skill always gets without touching the skill itself.
|
|
209
382
|
|
|
210
|
-
|
|
383
|
+
## 6. CLI commands
|
|
211
384
|
|
|
212
385
|
```bash
|
|
213
|
-
#
|
|
214
|
-
qualiow
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
qualiow list knowledge
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
qualiow
|
|
221
|
-
|
|
222
|
-
#
|
|
223
|
-
qualiow list domains
|
|
386
|
+
qualiow init --dry-run # preview what init would write
|
|
387
|
+
qualiow init --hooks # add the read/write guards to .claude/settings.json
|
|
388
|
+
qualiow validate --all # targets + qa/target.yml + domains + knowledge base
|
|
389
|
+
qualiow list sessions # also: knowledge | targets | domains
|
|
390
|
+
qualiow list knowledge --stats # also: --domain --tag --type --entry <id> --changelog
|
|
391
|
+
qualiow report -s latest -f html -o report.html
|
|
392
|
+
qualiow kb check
|
|
393
|
+
qualiow kb digest --for explore # the knowledge a session loads
|
|
394
|
+
qualiow session finalize latest # validate a finished session and index it
|
|
395
|
+
qualiow session prune --older-than 30 # dry-run; add --yes to remove
|
|
224
396
|
```
|
|
225
397
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
qualiow report
|
|
230
|
-
```
|
|
398
|
+
`qualiow explore [url]` is **pre-flight only**: it validates the inputs, creates the session
|
|
399
|
+
directory, and prints the `/qa-explore … --session <dir>` command to run inside Claude Code.
|
|
400
|
+
It does not drive a browser itself.
|
|
231
401
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
```bash
|
|
235
|
-
/qa-explore-report output/sessions/2026-03-28-parabank
|
|
236
|
-
```
|
|
402
|
+
There is no `qualiow gather` — it was removed in 2.0.0. Use the `/qa-gather` skill.
|
|
237
403
|
|
|
238
404
|
### Export session data programmatically
|
|
239
405
|
|
|
@@ -242,24 +408,37 @@ import {
|
|
|
242
408
|
generateHtmlReport,
|
|
243
409
|
generateJsonReport,
|
|
244
410
|
generateJiraExport,
|
|
245
|
-
} from '
|
|
411
|
+
} from 'qualiow-exploratory-testing';
|
|
412
|
+
|
|
413
|
+
const session = 'output/sessions/2026-09-08-1813-explore-parabank';
|
|
246
414
|
|
|
247
415
|
// HTML report (standalone, dark-mode, print-friendly)
|
|
248
|
-
const html = await generateHtmlReport(
|
|
416
|
+
const html = await generateHtmlReport(session);
|
|
249
417
|
|
|
250
418
|
// JSON report (structured data for dashboards and CI)
|
|
251
|
-
const json = await generateJsonReport(
|
|
419
|
+
const json = await generateJsonReport(session);
|
|
252
420
|
|
|
253
421
|
// Jira CSV (bulk import into Jira)
|
|
254
|
-
const csv = await generateJiraExport(
|
|
422
|
+
const csv = await generateJiraExport(session);
|
|
255
423
|
```
|
|
256
424
|
|
|
257
|
-
|
|
425
|
+
Every export carries the confidentiality header and runs through the same redaction list the
|
|
426
|
+
skills apply.
|
|
427
|
+
|
|
428
|
+
## 7. Beyond the desktop browser
|
|
429
|
+
|
|
430
|
+
- **Mobile** — `/qa-explore-mobile` runs a session on an iOS Simulator or Android Emulator,
|
|
431
|
+
against a native app or a web app in the real device browser. The machine needs the mobile
|
|
432
|
+
toolchain first: run `qa/bin/setup-mobile.sh`, then `qa/bin/doctor-mobile.sh` until it says
|
|
433
|
+
READY. Full guide: **MOBILE-SETUP.md**.
|
|
434
|
+
- **Backend, API and infrastructure** — `/qa-verify-backend` verifies acceptance criteria that
|
|
435
|
+
never reach a screen and produces an AC traceability matrix. Full guide:
|
|
436
|
+
**BACKEND-VERIFICATION.md**.
|
|
258
437
|
|
|
259
|
-
|
|
438
|
+
## What to do next
|
|
260
439
|
|
|
261
|
-
1. Review the session report and bug reports
|
|
262
|
-
2.
|
|
263
|
-
3. Configure a target for your own application
|
|
264
|
-
4. Add domain-specific knowledge
|
|
440
|
+
1. Review the session report and the bug reports
|
|
441
|
+
2. Run `/qa-explore-feedback` to mark false positives or note bugs the session missed
|
|
442
|
+
3. Configure a target for your own application with `/qa-target-setup`
|
|
443
|
+
4. Add domain-specific knowledge so the next session starts smarter
|
|
265
444
|
5. Run `/qa-explore-cleanup` to archive completed sessions
|