qualiow-exploratory-testing 1.3.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +26 -0
- package/.claude-plugin/plugin.json +22 -0
- package/.env.example +27 -0
- package/CHANGELOG.md +259 -0
- package/README.md +472 -204
- package/agents/qa-diff-indexer-agent.md +81 -0
- package/agents/qa-gather-agent.md +41 -0
- package/agents/qa-page-mapper-agent.md +74 -0
- package/agents/qa-reporting-agent.md +103 -0
- package/{scripts → bin}/doctor-mobile.sh +57 -18
- package/bin/mcli +15 -9
- package/bin/mobile-cli.mjs +574 -286
- package/bin/qualiow +30 -0
- package/{scripts → bin}/setup-mobile.sh +56 -19
- package/bin/wadb +10 -4
- package/bin/wk-ios +29 -17
- package/bin/wkeval.mjs +7 -1
- package/data/domains/_default.yml +1 -1
- package/data/knowledge/changelog.yml +30 -12
- package/data/knowledge/manifest.yml +355 -127
- package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
- package/data/knowledge/releases/v0.1.0/release.yml +8 -1
- package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
- package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
- package/data/knowledge/releases/v0.6.0/release.yml +30 -0
- package/data/security/SECURITY-POLICY.md +273 -0
- package/data/targets/_default.yml +1 -1
- package/data/targets/_example-api-only.yml +80 -0
- package/data/targets/_example-mobile-emulation.yml +2 -2
- package/data/targets/_example-native-mobile.yml +2 -2
- package/data/targets/_example-sim-android-chrome.yml +1 -1
- package/data/targets/_example-sim-ios-safari.yml +1 -1
- package/data/templates/api-probe-matrix.md +3 -3
- package/data/templates/bug-report.md +29 -151
- package/data/templates/coverage-map.md +9 -6
- package/data/templates/expected-behaviour.md +3 -2
- package/data/templates/session-charter.md +3 -0
- package/data/templates/session-report.md +57 -80
- package/dist/cli/index.js +2286 -701
- package/dist/index.d.ts +637 -81
- package/dist/index.js +2058 -228
- package/docs/BACKEND-VERIFICATION.md +39 -9
- package/docs/GETTING-STARTED.md +300 -121
- package/docs/MOBILE-SETUP.md +149 -18
- package/hooks/hooks.json +27 -0
- package/hooks/scripts/read-guard.mjs +95 -0
- package/hooks/scripts/secret-patterns.d.mts +8 -0
- package/hooks/scripts/secret-patterns.mjs +136 -0
- package/hooks/scripts/write-guard.mjs +87 -0
- package/package.json +33 -18
- package/skills/qa-explore/SKILL.md +32 -25
- package/skills/qa-explore/phases/00-setup.md +76 -42
- package/skills/qa-explore/phases/01-auth.md +32 -15
- package/skills/qa-explore/phases/02-charter.md +16 -5
- package/skills/qa-explore/phases/03-discovery.md +93 -27
- package/skills/qa-explore/phases/04-journeys.md +2 -2
- package/skills/qa-explore/phases/05-features.md +39 -24
- package/skills/qa-explore/phases/06-edge-cases.md +24 -13
- package/skills/qa-explore/phases/07-reporting.md +102 -142
- package/skills/qa-explore/references/context-integration.md +1 -1
- package/skills/qa-explore/references/delegation-rules.md +82 -0
- package/skills/qa-explore/references/output-contract.md +192 -0
- package/skills/qa-explore/references/paths.md +108 -0
- package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
- package/skills/qa-explore/references/security-rules.md +70 -25
- package/skills/qa-explore/references/session-rules.md +2 -1
- package/skills/qa-explore/references/severity-guide.md +31 -4
- package/skills/qa-explore-cleanup/SKILL.md +51 -15
- package/skills/qa-explore-feedback/SKILL.md +23 -5
- package/skills/qa-explore-mobile/SKILL.md +86 -64
- package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
- package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
- package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
- package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
- package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
- package/skills/qa-explore-mobile/phases/05-features.md +9 -9
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
- package/skills/qa-explore-mobile/phases/07-reporting.md +108 -60
- package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
- package/skills/qa-explore-quick/SKILL.md +71 -26
- package/skills/qa-explore-report/SKILL.md +65 -35
- package/skills/qa-gather/SKILL.md +44 -8
- package/skills/qa-knowledge-add/SKILL.md +25 -9
- package/skills/qa-knowledge-list/SKILL.md +37 -40
- package/skills/qa-target-setup/SKILL.md +63 -41
- package/skills/qa-verify-backend/SKILL.md +31 -19
- package/skills/qa-verify-backend/phases/00-setup.md +57 -12
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +2 -2
- package/skills/qa-verify-backend/phases/02-static-review.md +11 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +5 -3
- package/skills/qa-verify-backend/phases/03b-api-verification.md +24 -9
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +18 -2
- package/skills/qa-verify-backend/phases/05-reporting.md +72 -12
- package/skills/qa-verify-backend/references/api-probes.md +28 -1
- package/skills/qa-verify-backend/references/payload-verification.md +1 -1
- package/skills/qa-verify-backend/references/release-readiness.md +1 -1
- package/skills/qa-verify-backend/references/safety-rules.md +13 -8
- package/data/domains/_default.md +0 -181
- package/data/domains/ecommerce.md +0 -192
- package/data/domains/fintech.md +0 -200
- package/data/domains/identity.md +0 -88
- package/data/domains/marketing.md +0 -194
- package/data/domains/saas.md +0 -197
- package/dist/cli/index.d.ts +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/index.js.map +0 -1
|
@@ -1,63 +1,129 @@
|
|
|
1
|
-
# Phase 3: Discovery & Mapping (
|
|
1
|
+
# Phase 3: Discovery & Mapping (6 min)
|
|
2
2
|
|
|
3
|
-
**Goal:** Map the app structure and refine your risk ranking.
|
|
3
|
+
**Goal:** Map the app structure and refine your risk ranking. Every command carries `-s=<sid>` (omitted here).
|
|
4
4
|
|
|
5
5
|
## Site Mapping
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
8
|
playwright-cli goto <start_url>
|
|
9
|
-
playwright-cli snapshot
|
|
9
|
+
playwright-cli snapshot --depth=3
|
|
10
10
|
playwright-cli console error
|
|
11
|
-
playwright-cli
|
|
11
|
+
playwright-cli requests
|
|
12
12
|
```
|
|
13
13
|
|
|
14
14
|
For each page:
|
|
15
|
-
1. `playwright-cli snapshot
|
|
16
|
-
2. Click through navigation to discover all sections
|
|
15
|
+
1. `playwright-cli snapshot --depth=3` — note structure, forms, interactive elements.
|
|
16
|
+
2. Click through navigation to discover all sections.
|
|
17
17
|
3. Log: `[DISCOVERED] <url> -- <description> -- Risk: <P0/P1/P2/P3>`
|
|
18
|
-
4. Check `playwright-cli console error` on each page
|
|
18
|
+
4. Check `playwright-cli console error` and `playwright-cli requests` on each page.
|
|
19
|
+
|
|
20
|
+
## Snapshot Policy
|
|
21
|
+
|
|
22
|
+
A full accessibility tree is the single largest thing this session can read. On discovery
|
|
23
|
+
pages you take the shape, not the tree:
|
|
24
|
+
|
|
25
|
+
- `playwright-cli snapshot --depth=3` — the default. Enough to see sections, forms and nav.
|
|
26
|
+
- `playwright-cli snapshot <ref>` — zoom into one region once `--depth=3` shows you where to look.
|
|
27
|
+
- `playwright-cli find "<text>"` (or `find --regex "<re>"`) — locate a label, message or
|
|
28
|
+
control by its text, without a tree at all.
|
|
29
|
+
|
|
30
|
+
When you genuinely need the full tree of a page, **write it to disk and size it first —
|
|
31
|
+
never read it straight into the session**:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
playwright-cli --raw snapshot > output/sessions/<session-dir>/snapshots/<page>.yml
|
|
35
|
+
wc -l output/sessions/<session-dir>/snapshots/<page>.yml
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
- **At most 300 lines** — `Read` the file.
|
|
39
|
+
- **Over 300 lines** — invoke the `qa-page-mapper-agent` sub-agent
|
|
40
|
+
(`qualiow:qa-page-mapper-agent` under a plugin install) with the file path and the page
|
|
41
|
+
URL, and work from the map it returns: forms (fields, which are required, the submit ref),
|
|
42
|
+
nav (text → ref), interactive controls, visible error and empty-state text, hidden and
|
|
43
|
+
disabled counts. It returns structure only — no test ideas, no findings. If sub-agents are
|
|
44
|
+
unavailable, do the step yourself as in 2.1.0: read the snapshot in windows (`Grep` for the
|
|
45
|
+
region, then `Read` with `offset`/`limit`).
|
|
46
|
+
|
|
47
|
+
`snapshots/` is working material, not session evidence: it is never shipped in the report and
|
|
48
|
+
never read back by the reporting step.
|
|
49
|
+
|
|
50
|
+
Before/after diff — both files go into `snapshots/` the same way:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
playwright-cli --raw snapshot > output/sessions/<session-dir>/snapshots/<page>-before.yml
|
|
54
|
+
# ...act...
|
|
55
|
+
playwright-cli --raw snapshot > output/sessions/<session-dir>/snapshots/<page>-after.yml
|
|
56
|
+
diff output/sessions/<session-dir>/snapshots/<page>-before.yml output/sessions/<session-dir>/snapshots/<page>-after.yml
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The diff is what you read — it shows exactly what the action changed.
|
|
60
|
+
|
|
61
|
+
Network helper: `playwright-cli requests --filter="/api/"` — only the calls that matter;
|
|
62
|
+
`playwright-cli request <n>` for headers and body of one of them.
|
|
63
|
+
|
|
64
|
+
The interaction phases (4-6) are unchanged: snapshot before every interaction, and keep the
|
|
65
|
+
refs you act on in this session.
|
|
19
66
|
|
|
20
67
|
## Scope Enforcement
|
|
21
68
|
|
|
22
|
-
Stay within the target domain. Do not follow links to external sites unless they are part of
|
|
69
|
+
Stay within the target domain. Do not follow links to external sites unless they are part of
|
|
70
|
+
the application's workflow (OAuth redirects, payment gateways).
|
|
23
71
|
|
|
24
72
|
## What's MISSING Check
|
|
25
73
|
|
|
26
|
-
**Critical during discovery
|
|
27
|
-
- Read the domain completeness checklist from
|
|
28
|
-
- Compare what the app HAS
|
|
29
|
-
- E-commerce missing payment method? Log it now.
|
|
74
|
+
**Critical during discovery — look for what's MISSING:**
|
|
75
|
+
- Read the domain completeness checklist from `<data>/domains/<domain>.yml` (`completeness_checklist`)
|
|
76
|
+
- Compare what the app HAS against what it SHOULD have
|
|
77
|
+
- E-commerce missing a payment method? Log it now.
|
|
30
78
|
- Banking app missing transaction history? Flag it immediately.
|
|
31
79
|
- No error messages on empty states? Note it.
|
|
32
80
|
|
|
33
81
|
## Console & Network Checks
|
|
34
82
|
|
|
35
83
|
On every page during discovery:
|
|
36
|
-
- `playwright-cli console error`
|
|
37
|
-
- `playwright-cli
|
|
38
|
-
- Watch for mixed
|
|
84
|
+
- `playwright-cli console error` — JavaScript errors
|
|
85
|
+
- `playwright-cli requests` — failed requests (4xx, 5xx); `requests --static` to include assets
|
|
86
|
+
- Watch for mixed-content warnings (HTTP resources on HTTPS pages)
|
|
87
|
+
|
|
88
|
+
### Cross-navigation observation via `run-code` (Playwright 1.59+)
|
|
39
89
|
|
|
40
|
-
|
|
90
|
+
The CLI's `console` and `requests` show only the tail since the last navigation. To reach
|
|
91
|
+
messages from *before* it, run Playwright's retrieval methods inside the session with
|
|
92
|
+
`run-code` (the callback receives the live `page`) and pass `filter: 'all'` — **without it
|
|
93
|
+
they default to `since-navigation` and return exactly the same window as the CLI**. The
|
|
94
|
+
buffers are capped: 200 console messages, 200 page errors, 100 requests.
|
|
41
95
|
|
|
42
|
-
|
|
96
|
+
```bash
|
|
97
|
+
# every console message still buffered, across navigations
|
|
98
|
+
playwright-cli run-code "async page => (await page.consoleMessages({ filter: 'all' })).map(m => m.type() + ': ' + m.text()).join('\n')"
|
|
43
99
|
|
|
44
|
-
|
|
45
|
-
-
|
|
46
|
-
|
|
100
|
+
# uncaught exceptions only — more reliable than filtering console output
|
|
101
|
+
playwright-cli run-code "async page => (await page.pageErrors({ filter: 'all' })).map(e => e.message).join('\n')"
|
|
102
|
+
|
|
103
|
+
# failures and error statuses among the last 100 requests (page.requests() takes no filter)
|
|
104
|
+
playwright-cli run-code "async page => { const out = []; for (const r of await page.requests()) { const res = await r.response(); const f = r.failure(); if (f) out.push('FAILED ' + f.errorText + ' ' + r.url()); else if (res && res.status() >= 400) out.push(res.status() + ' ' + r.url()); } return out.join('\n'); }"
|
|
105
|
+
```
|
|
47
106
|
|
|
48
|
-
These are
|
|
107
|
+
These are optional; the bash commands above remain the primary path.
|
|
49
108
|
|
|
50
|
-
### Service Workers (Playwright 1.
|
|
109
|
+
### Service Workers (Playwright 1.61+)
|
|
51
110
|
|
|
52
|
-
|
|
111
|
+
Service-worker requests are reported and routable at the **BrowserContext** level from 1.61.
|
|
112
|
+
`playwright-cli route` maps to `page.route()`, which per the API docs still does **not**
|
|
113
|
+
intercept service-worker traffic. So if `playwright-cli requests` shows calls you cannot
|
|
114
|
+
account for on the main document, assume a service worker is serving them: flag the area as
|
|
115
|
+
high-risk, and note in the report that mocking it needs service workers blocked
|
|
116
|
+
(`serviceWorkers: 'block'`), not `route`.
|
|
53
117
|
|
|
54
118
|
## After Discovery
|
|
55
119
|
|
|
56
|
-
1. Write findings to `phase-
|
|
120
|
+
1. Write findings to `phase-3-discovery.md` (confidentiality header first)
|
|
57
121
|
2. **Update your risk ranking** if discovery changed your understanding
|
|
58
|
-
3. Carry forward: site
|
|
122
|
+
3. Carry forward: site-map summary, risk ranking, list of what's missing
|
|
59
123
|
|
|
60
|
-
**Log your reasoning:** "I expected a payment
|
|
124
|
+
**Log your reasoning:** "I expected a payment-method selector in checkout because this is
|
|
125
|
+
e-commerce, but it's missing. I'll flag this as a medium bug."
|
|
61
126
|
|
|
62
|
-
**Update progress:**
|
|
127
|
+
**Update progress:** set the discovery phase complete in `progress.json` with the
|
|
128
|
+
`pages_explored` count. Append to `session-log.md`:
|
|
63
129
|
`[<timestamp>] [PHASE] Discovery complete — <N> pages mapped, <N> console errors, <N> missing checklist items, <N> bugs so far`
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Phase 4: End-to-End User Journeys (
|
|
1
|
+
# Phase 4: End-to-End User Journeys (10 min)
|
|
2
2
|
|
|
3
3
|
**Goal:** Test the critical user journeys you defined in the charter -- end to end, crossing multiple features.
|
|
4
4
|
|
|
@@ -43,7 +43,7 @@ Do things change? Are order IDs unique? Are timestamps correct?
|
|
|
43
43
|
|
|
44
44
|
## After Journeys
|
|
45
45
|
|
|
46
|
-
1. Write findings to `phase-
|
|
46
|
+
1. Write findings to `phase-4-journeys.md` (confidentiality header first)
|
|
47
47
|
2. Include data integrity results
|
|
48
48
|
3. Note any cross-feature inconsistencies
|
|
49
49
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
# Phase 5: Deep Feature Testing (
|
|
1
|
+
# Phase 5: Deep Feature Testing (10 min)
|
|
2
2
|
|
|
3
|
-
**Goal:** Go deep on P0 and P1 features. Spend time proportional to risk.
|
|
3
|
+
**Goal:** Go deep on P0 and P1 features. Spend time proportional to risk. Every command carries `-s=<sid>` (omitted here).
|
|
4
4
|
|
|
5
|
-
Focus on the features your risk ranking identified as P0/P1. Apply SFDIPOT, but only the
|
|
5
|
+
Focus on the features your risk ranking identified as P0/P1. Apply SFDIPOT, but only the
|
|
6
|
+
dimensions you selected in the charter.
|
|
6
7
|
|
|
7
8
|
## For Each P0/P1 Feature
|
|
8
9
|
|
|
@@ -21,55 +22,69 @@ playwright-cli fill <ref> "a" # Minimum
|
|
|
21
22
|
playwright-cli fill <ref> "<1000+ chars>" # Maximum
|
|
22
23
|
```
|
|
23
24
|
|
|
24
|
-
### Business Logic Stress (from knowledge base)
|
|
25
|
+
### Business Logic Stress (from the knowledge base)
|
|
25
26
|
|
|
26
|
-
- **Wrong state order:**
|
|
27
|
-
- **Parameter tampering:**
|
|
27
|
+
- **Wrong state order:** skip a step in a multi-step flow; navigate directly to confirmation
|
|
28
|
+
- **Parameter tampering:** change values that should be server-controlled
|
|
28
29
|
```bash
|
|
30
|
+
playwright-cli fill <ref> "-100" # preferred: types, so framework handlers fire
|
|
29
31
|
playwright-cli eval "document.querySelector('input[name=amount]').value = '-100'"
|
|
30
32
|
```
|
|
31
|
-
|
|
32
|
-
-
|
|
33
|
+
The `eval` form assigns `.value` without dispatching `input`/`change`, so a
|
|
34
|
+
React/Vue-controlled field reverts on the next render — prefer `fill`, and tamper at the
|
|
35
|
+
network layer (`route`) when the value is genuinely server-controlled.
|
|
36
|
+
- **Replay:** complete an action, then try to complete it again immediately
|
|
37
|
+
- **Self-referential:** transfer from account A to account A; apply a discount code twice
|
|
33
38
|
|
|
34
39
|
### Negative Space Check
|
|
35
40
|
|
|
36
41
|
- For EACH form, ask: "What fields should be here that aren't?"
|
|
37
42
|
- For EACH flow, ask: "What steps are missing?"
|
|
38
|
-
- Compare against domain checklist
|
|
43
|
+
- Compare against the domain checklist (`<data>/domains/<domain>.yml`)
|
|
39
44
|
|
|
40
45
|
### Consistency Oracles (FEW HICCUPPS)
|
|
41
46
|
|
|
42
47
|
After testing each feature, explicitly ask:
|
|
43
|
-
- **Users:** "Would a real user expect this
|
|
48
|
+
- **Users:** "Would a real user expect this behaviour?"
|
|
44
49
|
- **Claims:** "Does this match what the app promises?"
|
|
45
50
|
- **Product:** "Is this consistent with how other features in the app work?"
|
|
46
51
|
- **Standards:** "Does this meet industry standards for this domain?"
|
|
47
52
|
|
|
48
|
-
If you notice an inconsistency (
|
|
53
|
+
If you notice an inconsistency ("Bill Pay validates but Transfer doesn't"), **follow that
|
|
54
|
+
thread**; don't move on. The inconsistency is a signal.
|
|
49
55
|
|
|
50
|
-
###
|
|
56
|
+
### Stable locators for the bug report
|
|
51
57
|
|
|
52
|
-
|
|
58
|
+
A reproduction step must name the element unambiguously. When a ref is ambiguous:
|
|
53
59
|
|
|
54
|
-
|
|
55
|
-
-
|
|
60
|
+
```bash
|
|
61
|
+
playwright-cli generate-locator <ref> # a Playwright locator for the element behind the ref
|
|
62
|
+
playwright-cli highlight <ref> # confirm visually which element you mean, then
|
|
63
|
+
playwright-cli highlight --hide
|
|
64
|
+
```
|
|
56
65
|
|
|
57
|
-
|
|
66
|
+
Paste the generated locator into the bug's steps. `page.pickLocator()` (Playwright 1.59)
|
|
67
|
+
exists for the same purpose but is **human-only**: it opens an interactive picker in a headed
|
|
68
|
+
browser and waits for a click, so it is not usable from an agent session.
|
|
58
69
|
|
|
59
70
|
### Test Agents Handoff (optional, Playwright 1.56+)
|
|
60
71
|
|
|
61
|
-
When feature testing produces a **reproducible** bug (you can trigger it twice in a row,
|
|
72
|
+
When feature testing produces a **reproducible** bug (you can trigger it twice in a row,
|
|
73
|
+
same steps, same result), you may hand it off after the session to the Playwright Test
|
|
74
|
+
Agents **generator** to convert the reproduction into a regression test:
|
|
62
75
|
|
|
63
|
-
1. Finish
|
|
64
|
-
2. If
|
|
65
|
-
3. The generator produces a `tests/repro
|
|
66
|
-
4. The **healer** agent
|
|
76
|
+
1. Finish the bug report with exact steps (phase 7).
|
|
77
|
+
2. If Step 6 of setup bootstrapped Test Agents, pass the bug report to the generator agent.
|
|
78
|
+
3. The generator produces a `tests/repro-BUG-NNN.spec.ts` you can commit alongside the bug report.
|
|
79
|
+
4. The **healer** agent later repairs the test if it breaks for non-bug reasons.
|
|
67
80
|
|
|
68
|
-
Full workflow in
|
|
81
|
+
Full workflow in `${CLAUDE_SKILL_DIR}/references/playwright-agents-integration.md`. This
|
|
82
|
+
handoff is **opt-in**; it does not replace the bug report and is skipped entirely when
|
|
83
|
+
`@playwright/test` is not installed.
|
|
69
84
|
|
|
70
85
|
## After Feature Testing
|
|
71
86
|
|
|
72
|
-
Write
|
|
87
|
+
Write `phase-5-features.md` (confidentiality header first). Include the reasoning for each test.
|
|
73
88
|
|
|
74
|
-
**Update progress:**
|
|
89
|
+
**Update progress:** set the features phase complete in `progress.json`. Append to `session-log.md`:
|
|
75
90
|
`[<timestamp>] [PHASE] Features complete — <N> P0 features tested, <N> P1 features tested, <N> inconsistencies found, <N> bugs so far`
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
# Phase 6: Edge Cases, Security & Negative Testing (
|
|
1
|
+
# Phase 6: Edge Cases, Security & Negative Testing (6 min)
|
|
2
2
|
|
|
3
|
-
**Goal:** Saboteur tour
|
|
3
|
+
**Goal:** Saboteur tour: actively try to break things. Every command carries `-s=<sid>` (omitted here). In read-only mode, fill but never submit.
|
|
4
4
|
|
|
5
5
|
## Input Attacks (from error-guessing knowledge)
|
|
6
6
|
|
|
@@ -17,15 +17,21 @@ playwright-cli fill <ref> " "
|
|
|
17
17
|
playwright-cli fill <ref> "A repeated 5000 times..."
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
+
After an XSS payload, check whether it executed: `playwright-cli console error` plus
|
|
21
|
+
`playwright-cli run-code "async page => (await page.pageErrors()).length"`, and look for a
|
|
22
|
+
dialog (`dialog-dismiss` if one opened).
|
|
23
|
+
|
|
20
24
|
## Race Conditions (if applicable)
|
|
21
25
|
|
|
22
26
|
```bash
|
|
23
|
-
# Open a second session
|
|
24
|
-
playwright-cli -s
|
|
25
|
-
playwright-cli -s
|
|
26
|
-
# Set up identical action in both sessions
|
|
27
|
+
# Open a second, separately isolated session alongside the main one
|
|
28
|
+
playwright-cli -s=<sid>-race open <url>
|
|
29
|
+
playwright-cli -s=<sid>-race state-load .auth/<target>.json
|
|
30
|
+
# Set up the identical action in both sessions
|
|
27
31
|
# Submit both as close to simultaneously as possible
|
|
28
|
-
# Check for double-processing
|
|
32
|
+
# Check for double-processing, then close the second session:
|
|
33
|
+
playwright-cli -s=<sid>-race close
|
|
34
|
+
playwright-cli -s=<sid>-race delete-data
|
|
29
35
|
```
|
|
30
36
|
|
|
31
37
|
## State Manipulation
|
|
@@ -33,14 +39,15 @@ playwright-cli -s=race state-load .auth/<target>.json
|
|
|
33
39
|
- Navigate directly to pages that require prior steps
|
|
34
40
|
- Use browser back/forward mid-flow
|
|
35
41
|
- Double-click submit buttons
|
|
36
|
-
- Simulate network errors: `playwright-cli route "**/*" --status=500`
|
|
42
|
+
- Simulate network errors: `playwright-cli route "**/*" --status=500` (then `playwright-cli unroute` to restore)
|
|
43
|
+
- Simulate offline: `playwright-cli network-state-set offline` / `online`
|
|
37
44
|
|
|
38
45
|
## Security
|
|
39
46
|
|
|
40
47
|
- Check for session tokens in URLs
|
|
41
48
|
- Try accessing admin/internal pages directly
|
|
42
|
-
- Check
|
|
43
|
-
- Test brute force: 5+ failed login attempts
|
|
49
|
+
- Check whether authenticated endpoints work without auth (`playwright-cli -s=<sid>-anon open <api-url>` in a fresh session, then close and `delete-data` it)
|
|
50
|
+
- Test brute force: 5+ failed login attempts; check for lockout
|
|
44
51
|
|
|
45
52
|
## Accessibility (quick check)
|
|
46
53
|
|
|
@@ -54,7 +61,11 @@ playwright-cli snapshot
|
|
|
54
61
|
playwright-cli resize 1280 720 # restore
|
|
55
62
|
```
|
|
56
63
|
|
|
57
|
-
|
|
64
|
+
The CLI `snapshot` **is** an aria snapshot: roles, names and states as assistive technology
|
|
65
|
+
sees them, so missing names and duplicate landmarks show up directly. For the raw YAML aria
|
|
66
|
+
tree of a region, `playwright-cli run-code "async page => page.locator('main').ariaSnapshot()"`.
|
|
67
|
+
Note: Playwright **removed** `page.accessibility` in 1.57; anything still calling it will
|
|
68
|
+
throw.
|
|
58
69
|
|
|
59
70
|
## Empty States
|
|
60
71
|
|
|
@@ -65,7 +76,7 @@ playwright-cli resize 1280 720 # restore
|
|
|
65
76
|
|
|
66
77
|
## After Edge Cases
|
|
67
78
|
|
|
68
|
-
Write
|
|
79
|
+
Write `phase-6-edge-cases.md` (confidentiality header first).
|
|
69
80
|
|
|
70
|
-
**Update progress:**
|
|
81
|
+
**Update progress:** set the edge_cases phase complete in `progress.json`. Append to `session-log.md`:
|
|
71
82
|
`[<timestamp>] [PHASE] Edge cases complete — <N> input attacks, <N> security checks, <N> a11y checks, <N> bugs so far`
|
|
@@ -1,27 +1,43 @@
|
|
|
1
|
-
# Phase 7: Reflection & Reporting (
|
|
1
|
+
# Phase 7: Reflection & Reporting (4 min)
|
|
2
|
+
|
|
3
|
+
Every command carries `-s=<sid>` (omitted here). The browser stays open until the evidence is captured; it is closed in the last step.
|
|
2
4
|
|
|
3
5
|
## Stop Recording
|
|
4
6
|
|
|
5
7
|
```bash
|
|
8
|
+
playwright-cli video-chapter "Reporting"
|
|
9
|
+
playwright-cli video-stop
|
|
6
10
|
playwright-cli tracing-stop
|
|
7
|
-
playwright-cli close
|
|
8
11
|
```
|
|
9
12
|
|
|
10
|
-
### Trace
|
|
13
|
+
### Trace analysis (Playwright 1.59+, optional)
|
|
14
|
+
|
|
15
|
+
`tracing-stop` writes a trace under `traces/` (or the path it prints). To inspect it from
|
|
16
|
+
the command line: `npx playwright trace open <trace>` then `npx playwright trace actions --errors-only`,
|
|
17
|
+
`npx playwright trace requests --failed`, `npx playwright trace console`, and `npx playwright trace close`
|
|
18
|
+
(check `npx playwright trace --help` for the exact subcommands of the installed version).
|
|
19
|
+
Copy the trace into `output/sessions/<session-dir>/` if it is evidence for a bug.
|
|
20
|
+
|
|
21
|
+
### Annotated video for Critical / High bugs (optional)
|
|
11
22
|
|
|
12
|
-
|
|
23
|
+
The session recording from phase 2 already has chapters. For a polished reproduction of a
|
|
24
|
+
Critical or High bug, record a second clip with action callouts:
|
|
13
25
|
|
|
14
|
-
|
|
15
|
-
-
|
|
16
|
-
-
|
|
17
|
-
|
|
26
|
+
```bash
|
|
27
|
+
playwright-cli video-start output/sessions/<session-dir>/videos/BUG-NNN.webm
|
|
28
|
+
playwright-cli video-show-actions
|
|
29
|
+
# ...reproduce the bug...
|
|
30
|
+
playwright-cli video-hide-actions
|
|
31
|
+
playwright-cli video-stop
|
|
32
|
+
```
|
|
18
33
|
|
|
19
|
-
|
|
34
|
+
Or write a scripted "hero" reproduction with `page.screencast.start()` / `showChapter()` /
|
|
35
|
+
`showOverlay()` (Playwright 1.59) and run it with `playwright-cli run-code --filename=<script.js>`.
|
|
36
|
+
A screenshot remains sufficient for Medium / Low.
|
|
20
37
|
|
|
21
38
|
## Reflect Before Writing
|
|
22
39
|
|
|
23
|
-
|
|
24
|
-
1. **What was the biggest risk I found?** (not the most bugs -- the biggest RISK)
|
|
40
|
+
1. **What was the biggest risk I found?** (not the most bugs; the biggest RISK)
|
|
25
41
|
2. **What would I tell the CEO in one sentence?**
|
|
26
42
|
3. **What did I NOT test that still worries me?**
|
|
27
43
|
4. **Which heuristic was most useful? Which was useless?**
|
|
@@ -29,145 +45,89 @@ Before writing the report, answer these questions:
|
|
|
29
45
|
|
|
30
46
|
## Write Bug Reports
|
|
31
47
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
- **Executive summary** (3 sentences max -- what was tested, what was found, what's the risk)
|
|
71
|
-
- Business context and charter summary
|
|
72
|
-
- Bug summary table (severity, component, business impact)
|
|
73
|
-
- Data integrity results
|
|
74
|
-
- What's MISSING (negative space findings)
|
|
75
|
-
- Coverage map with risk ranking
|
|
76
|
-
- Cross-feature journey results
|
|
77
|
-
- Heuristic effectiveness (which worked, which didn't)
|
|
78
|
-
- Areas NOT tested and why
|
|
79
|
-
- Recommendations for next session
|
|
80
|
-
- Reflection answers
|
|
81
|
-
|
|
82
|
-
## Coverage Map
|
|
83
|
-
|
|
84
|
-
Document what was tested and what was not:
|
|
85
|
-
|
|
86
|
-
| Feature | Risk | Tested? | Findings | Notes |
|
|
87
|
-
|---------|------|---------|----------|-------|
|
|
88
|
-
| [feature] | P0 | Yes | 2 bugs | [notes] |
|
|
89
|
-
| [feature] | P1 | Partial | 1 bug | [notes] |
|
|
90
|
-
| [feature] | P2 | No | -- | Time ran out |
|
|
48
|
+
One file per bug, `bugs/BUG-NNN.md`, in exactly the format of
|
|
49
|
+
`${CLAUDE_SKILL_DIR}/references/output-contract.md` (the template
|
|
50
|
+
`<data>/templates/bug-report.md` is the same format with guidance):
|
|
51
|
+
|
|
52
|
+
- the confidentiality header as the **first two lines**, verbatim:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
> CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
|
|
56
|
+
> and application details. Do not share outside your organization without review.
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
- then `# BUG-NNN: [Component] fails [Condition] causing [Impact]`
|
|
60
|
+
- `**Severity:**` (per `severity-guide.md`; when in doubt go lower), `**Priority:**`,
|
|
61
|
+
`**Component:**`, `**URL:**`, `**Environment:** Playwright CLI, Chromium, <viewport>`,
|
|
62
|
+
`**Reproduction rate:**`
|
|
63
|
+
- `## Summary`, `## Expected Behavior`, `## Actual Behavior`, `## Steps to Reproduce`
|
|
64
|
+
- `## Business Impact` — MANDATORY; answer at least one of Revenue / Trust / Regulatory /
|
|
65
|
+
Data / Scale, write "none identified" for the rest
|
|
66
|
+
- `## Evidence` — `Screenshot: screenshots/BUG-NNN.png` (take it with
|
|
67
|
+
`playwright-cli screenshot --filename=output/sessions/<session-dir>/screenshots/BUG-NNN.png`),
|
|
68
|
+
console errors, network failures (`playwright-cli request <n>` output, redacted)
|
|
69
|
+
- `## Recommended Fix Priority`
|
|
70
|
+
|
|
71
|
+
Redact per `security-rules.md` before writing.
|
|
72
|
+
|
|
73
|
+
## Write phase-7-notes.md
|
|
74
|
+
|
|
75
|
+
Your judgement, in your own words, in `output/sessions/<session-dir>/phase-7-notes.md`. The
|
|
76
|
+
confidentiality header first, then exactly these sections — the report is assembled from
|
|
77
|
+
them, so a section you leave out is a section nobody can write for you:
|
|
78
|
+
|
|
79
|
+
- `## Executive Summary` — 3 sentences: what was tested, what was found, the biggest risk
|
|
80
|
+
- `## Coverage Map` — rows for `| Area | Risk | Status | Bugs | Notes |`
|
|
81
|
+
(Status: tested / partial / not-tested / code-verified-only)
|
|
82
|
+
- `## Observations` — including what's MISSING and the data-integrity results
|
|
83
|
+
- `## Areas Not Tested` — each with its reason
|
|
84
|
+
- `## Recommendations` — for the next session
|
|
85
|
+
- `## Reflection` — the five answers above
|
|
91
86
|
|
|
92
87
|
## Session Stats
|
|
93
88
|
|
|
94
|
-
Write `output/sessions/<session-dir>/stats.json`
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
"checklist_verified": "<count checked>",
|
|
117
|
-
"checklist_percentage": "<calculated>",
|
|
118
|
-
"data_integrity_checks": "<count>",
|
|
119
|
-
"data_integrity_passed": "<count>",
|
|
120
|
-
"personas_tested": ["<list>"],
|
|
121
|
-
"purchase_flows_tested": ["<list>"]
|
|
122
|
-
},
|
|
123
|
-
"evidence": {
|
|
124
|
-
"screenshots": "<count>",
|
|
125
|
-
"console_errors_found": "<count>",
|
|
126
|
-
"network_failures_found": "<count>"
|
|
127
|
-
},
|
|
128
|
-
"areas_not_tested": ["<list with reasons>"],
|
|
129
|
-
"blocked_by": "<if something blocked testing, note it>"
|
|
130
|
-
}
|
|
131
|
-
```
|
|
132
|
-
|
|
133
|
-
Also append a **stats summary** at the bottom of `session-report.md`:
|
|
134
|
-
|
|
135
|
-
```markdown
|
|
136
|
-
---
|
|
137
|
-
|
|
138
|
-
## Session Stats
|
|
89
|
+
Write `output/sessions/<session-dir>/stats.json` exactly as `output-contract.md` defines it
|
|
90
|
+
(`session_id`, `kind: "explore"`, `target`, `date`, `duration_min`, `bugs_found`,
|
|
91
|
+
`severity_counts`, `pages_explored`, plus the optional `domain`, `started_at`,
|
|
92
|
+
`completed_at`, `phases_completed`, `total_phases: 8`, `coverage`, `evidence`,
|
|
93
|
+
`areas_not_tested`, `blocked_by`). No other top-level keys. The `## Session Stats` table in
|
|
94
|
+
the report is rendered from this file.
|
|
95
|
+
|
|
96
|
+
## Assemble and Finalize
|
|
97
|
+
|
|
98
|
+
The bugs, the stats and the notes are yours; assembling them into the report is not. Invoke
|
|
99
|
+
the `qa-reporting-agent` sub-agent (`qualiow:qa-reporting-agent` under a plugin install) with
|
|
100
|
+
the session directory and `kind: explore`. It reads `charter.md`, `stats.json`, `bugs/*.md`,
|
|
101
|
+
the phase files in windows and `phase-7-notes.md`, writes `session-report.md` in the format
|
|
102
|
+
of `${CLAUDE_SKILL_DIR}/references/output-contract.md` — copying your Executive Summary,
|
|
103
|
+
Recommendations and Reflection verbatim — and then runs `qualiow session finalize`, which
|
|
104
|
+
validates `stats.json` against the strict schema, checks the confidentiality header on every
|
|
105
|
+
artefact, scans the directory against the redaction list, appends the session row to
|
|
106
|
+
`output/sessions/INDEX.md` and one row per bug to `output/bugs/all-bugs.md`, records the
|
|
107
|
+
session metrics and sets `progress.json` to `complete`. It is idempotent.
|
|
108
|
+
|
|
109
|
+
Read its return (at most 8 lines). If it reports a missing notes section or a violation it
|
|
110
|
+
could not fix, fix that yourself and re-run the check:
|
|
139
111
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
| Phases completed | <N>/8 |
|
|
144
|
-
| Bugs found | <N> (Critical: <N>, High: <N>, Medium: <N>, Low: <N>) |
|
|
145
|
-
| Pages explored | <N> |
|
|
146
|
-
| Screenshots taken | <N> |
|
|
147
|
-
| Console errors found | <N> |
|
|
112
|
+
```bash
|
|
113
|
+
qualiow session finalize output/sessions/<session-dir> --check
|
|
114
|
+
```
|
|
148
115
|
|
|
149
|
-
|
|
116
|
+
`--check` writes nothing and prints a numbered list of violations naming each file. Resolve
|
|
117
|
+
the `qualiow` prefix per `${CLAUDE_SKILL_DIR}/references/paths.md`; if the CLI is unavailable,
|
|
118
|
+
write the index rows by hand in the column order defined there.
|
|
150
119
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
| <area> | P0 | Yes | Deep | <N> | <notes> |
|
|
154
|
-
| <area> | P1 | Partial | Shallow | <N> | <blocked by...> |
|
|
155
|
-
| <area> | P2 | No | — | — | Time ran out |
|
|
120
|
+
If sub-agents are unavailable, do the step yourself as in 2.1.0: write `session-report.md`
|
|
121
|
+
from the notes in the contract's format and run `qualiow session finalize output/sessions/<session-dir>`.
|
|
156
122
|
|
|
157
|
-
|
|
158
|
-
<N>/<N> items verified (<X>%)
|
|
123
|
+
## Close the Browser Session
|
|
159
124
|
|
|
160
|
-
|
|
161
|
-
<N>/<N> checks passed
|
|
125
|
+
Last, once every screenshot, trace and video is on disk:
|
|
162
126
|
|
|
163
|
-
|
|
164
|
-
-
|
|
127
|
+
```bash
|
|
128
|
+
playwright-cli close
|
|
129
|
+
playwright-cli delete-data
|
|
165
130
|
```
|
|
166
131
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
Set status to "complete", current_phase to "reporting", all phases to their final status.
|
|
170
|
-
|
|
171
|
-
## Update Indexes
|
|
172
|
-
|
|
173
|
-
Append to `output/sessions/INDEX.md` and `output/bugs/all-bugs.md`.
|
|
132
|
+
(both with `-s=<sid>`; also close and `delete-data` any extra session such as `-s=<sid>-race`).
|
|
133
|
+
Append to `session-log.md`: `[<timestamp>] [PHASE] Reporting complete — <N> bugs, session closed`.
|