qualiow-exploratory-testing 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/.claude-plugin/plugin.json +22 -0
  2. package/.env.example +27 -0
  3. package/CHANGELOG.md +194 -0
  4. package/README.md +428 -179
  5. package/agents/qa-gather-agent.md +24 -0
  6. package/{scripts → bin}/doctor-mobile.sh +57 -18
  7. package/bin/mcli +15 -9
  8. package/bin/mobile-cli.mjs +574 -286
  9. package/{scripts → bin}/setup-mobile.sh +56 -19
  10. package/bin/wadb +10 -4
  11. package/bin/wk-ios +29 -17
  12. package/bin/wkeval.mjs +7 -1
  13. package/data/domains/_default.yml +1 -1
  14. package/data/domains/identity.yml +138 -0
  15. package/data/knowledge/changelog.yml +104 -0
  16. package/data/knowledge/learned-patterns.md +172 -86
  17. package/data/knowledge/manifest.yml +481 -70
  18. package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
  19. package/data/knowledge/releases/v0.1.0/release.yml +8 -1
  20. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  21. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  22. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  23. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  24. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  25. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  26. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  28. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  29. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  30. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  31. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  34. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  35. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  36. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  37. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  38. package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
  39. package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
  40. package/data/knowledge/releases/v0.6.0/release.yml +30 -0
  41. package/data/security/SECURITY-POLICY.md +273 -0
  42. package/data/targets/_default.yml +1 -1
  43. package/data/targets/_example-api-only.yml +80 -0
  44. package/data/targets/_example-backend.yml +123 -0
  45. package/data/targets/_example-mobile-emulation.yml +2 -2
  46. package/data/targets/_example-native-mobile.yml +2 -2
  47. package/data/targets/_example-sim-android-chrome.yml +1 -1
  48. package/data/targets/_example-sim-ios-safari.yml +1 -1
  49. package/data/templates/api-probe-matrix.md +99 -0
  50. package/data/templates/bug-report.md +29 -151
  51. package/data/templates/coverage-map.md +23 -6
  52. package/data/templates/expected-behaviour.md +94 -0
  53. package/data/templates/session-charter.md +3 -0
  54. package/data/templates/session-report.md +57 -80
  55. package/dist/cli/index.js +1500 -739
  56. package/dist/index.d.ts +673 -75
  57. package/dist/index.js +1664 -270
  58. package/docs/BACKEND-VERIFICATION.md +307 -0
  59. package/docs/GETTING-STARTED.md +188 -128
  60. package/docs/MOBILE-SETUP.md +149 -18
  61. package/package.json +32 -16
  62. package/skills/qa-explore/SKILL.md +31 -25
  63. package/skills/qa-explore/phases/00-setup.md +67 -41
  64. package/skills/qa-explore/phases/01-auth.md +32 -15
  65. package/skills/qa-explore/phases/02-charter.md +16 -5
  66. package/skills/qa-explore/phases/03-discovery.md +50 -26
  67. package/skills/qa-explore/phases/04-journeys.md +2 -2
  68. package/skills/qa-explore/phases/05-features.md +39 -24
  69. package/skills/qa-explore/phases/06-edge-cases.md +24 -13
  70. package/skills/qa-explore/phases/07-reporting.md +79 -140
  71. package/skills/qa-explore/references/context-integration.md +1 -1
  72. package/skills/qa-explore/references/output-contract.md +187 -0
  73. package/skills/qa-explore/references/paths.md +84 -0
  74. package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
  75. package/skills/qa-explore/references/security-rules.md +66 -25
  76. package/skills/qa-explore/references/severity-guide.md +31 -4
  77. package/skills/qa-explore-cleanup/SKILL.md +17 -6
  78. package/skills/qa-explore-feedback/SKILL.md +18 -5
  79. package/skills/qa-explore-mobile/SKILL.md +86 -64
  80. package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
  81. package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
  82. package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
  83. package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
  84. package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
  85. package/skills/qa-explore-mobile/phases/05-features.md +9 -9
  86. package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
  87. package/skills/qa-explore-mobile/phases/07-reporting.md +82 -56
  88. package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
  89. package/skills/qa-explore-quick/SKILL.md +60 -25
  90. package/skills/qa-explore-report/SKILL.md +40 -34
  91. package/skills/qa-gather/SKILL.md +17 -2
  92. package/skills/qa-knowledge-add/SKILL.md +20 -9
  93. package/skills/qa-knowledge-list/SKILL.md +12 -3
  94. package/skills/qa-target-setup/SKILL.md +63 -41
  95. package/skills/qa-verify-backend/SKILL.md +172 -0
  96. package/skills/qa-verify-backend/phases/00-setup.md +142 -0
  97. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  98. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  99. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  100. package/skills/qa-verify-backend/phases/03b-api-verification.md +170 -0
  101. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +82 -0
  102. package/skills/qa-verify-backend/phases/05-reporting.md +163 -0
  103. package/skills/qa-verify-backend/references/api-probes.md +168 -0
  104. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  105. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  106. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  107. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  108. package/skills/qa-verify-backend/references/safety-rules.md +95 -0
  109. package/dist/cli/index.d.ts +0 -1
  110. package/dist/cli/index.js.map +0 -1
  111. package/dist/index.js.map +0 -1
@@ -1,8 +1,9 @@
1
- # Phase 5: Deep Feature Testing (~25% of time)
1
+ # Phase 5: Deep Feature Testing (10 min)
2
2
 
3
- **Goal:** Go deep on P0 and P1 features. Spend time proportional to risk.
3
+ **Goal:** Go deep on P0 and P1 features. Spend time proportional to risk. Every command carries `-s=<sid>` (omitted here).
4
4
 
5
- Focus on the features your risk ranking identified as P0/P1. Apply SFDIPOT, but only the dimensions you selected in the charter.
5
+ Focus on the features your risk ranking identified as P0/P1. Apply SFDIPOT, but only the
6
+ dimensions you selected in the charter.
6
7
 
7
8
  ## For Each P0/P1 Feature
8
9
 
@@ -21,55 +22,69 @@ playwright-cli fill <ref> "a" # Minimum
21
22
  playwright-cli fill <ref> "<1000+ chars>" # Maximum
22
23
  ```
23
24
 
24
- ### Business Logic Stress (from knowledge base)
25
+ ### Business Logic Stress (from the knowledge base)
25
26
 
26
- - **Wrong state order:** Skip a step in a multi-step flow -- navigate directly to confirmation
27
- - **Parameter tampering:** Change values that should be server-controlled
27
+ - **Wrong state order:** skip a step in a multi-step flow; navigate directly to confirmation
28
+ - **Parameter tampering:** change values that should be server-controlled
28
29
  ```bash
30
+ playwright-cli fill <ref> "-100" # preferred: types, so framework handlers fire
29
31
  playwright-cli eval "document.querySelector('input[name=amount]').value = '-100'"
30
32
  ```
31
- - **Replay:** Complete an action, then try to complete it again immediately
32
- - **Self-referential:** Transfer from account A to account A. Apply discount code twice.
33
+ The `eval` form assigns `.value` without dispatching `input`/`change`, so a
34
+ React/Vue-controlled field reverts on the next render — prefer `fill`, and tamper at the
35
+ network layer (`route`) when the value is genuinely server-controlled.
36
+ - **Replay:** complete an action, then try to complete it again immediately
37
+ - **Self-referential:** transfer from account A to account A; apply a discount code twice
33
38
 
34
39
  ### Negative Space Check
35
40
 
36
41
  - For EACH form, ask: "What fields should be here that aren't?"
37
42
  - For EACH flow, ask: "What steps are missing?"
38
- - Compare against domain checklist
43
+ - Compare against the domain checklist (`<data>/domains/<domain>.yml`)
39
44
 
40
45
  ### Consistency Oracles (FEW HICCUPPS)
41
46
 
42
47
  After testing each feature, explicitly ask:
43
- - **Users:** "Would a real user expect this behavior?"
48
+ - **Users:** "Would a real user expect this behaviour?"
44
49
  - **Claims:** "Does this match what the app promises?"
45
50
  - **Product:** "Is this consistent with how other features in the app work?"
46
51
  - **Standards:** "Does this meet industry standards for this domain?"
47
52
 
48
- If you notice an inconsistency (e.g., "Bill Pay validates but Transfer doesn't"), **follow that thread** -- don't move on. The inconsistency is a signal.
53
+ If you notice an inconsistency ("Bill Pay validates but Transfer doesn't"), **follow that
54
+ thread**; don't move on. The inconsistency is a signal.
49
55
 
50
- ### Locator Stabilization Helpers (Playwright 1.59+, optional)
56
+ ### Stable locators for the bug report
51
57
 
52
- When testing a feature produces flaky snapshots because a locator isn't uniquely addressable, the following 1.59 helpers can harden the repro before you file a bug. They require the `@playwright/test` peer dep.
58
+ A reproduction step must name the element unambiguously. When a ref is ambiguous:
53
59
 
54
- - `page.pickLocator()` — interactive locator picker. Use this when your current locator matches multiple elements and you need to find the one the user actually sees.
55
- - `locator.normalize()` — returns a cleaned-up, canonical form of a locator. Use before copying a locator into a bug report so the reproduction is portable across runs.
60
+ ```bash
61
+ playwright-cli generate-locator <ref> # a Playwright locator for the element behind the ref
62
+ playwright-cli highlight <ref> # confirm visually which element you mean, then
63
+ playwright-cli highlight --hide
64
+ ```
56
65
 
57
- These are diagnostic aids, not required steps. The bash `playwright-cli` flow above is the primary path.
66
+ Paste the generated locator into the bug's steps. `page.pickLocator()` (Playwright 1.59)
67
+ exists for the same purpose but is **human-only**: it opens an interactive picker in a headed
68
+ browser and waits for a click, so it is not usable from an agent session.
58
69
 
59
70
  ### Test Agents Handoff (optional, Playwright 1.56+)
60
71
 
61
- When feature testing produces a **reproducible** bug (you can trigger it twice in a row, same steps, same result), you may hand off to the Playwright Test Agents **generator** to convert the reproduction into a regression test case:
72
+ When feature testing produces a **reproducible** bug (you can trigger it twice in a row,
73
+ same steps, same result), you may hand it off after the session to the Playwright Test
74
+ Agents **generator** to convert the reproduction into a regression test:
62
75
 
63
- 1. Finish writing the bug report with exact steps (see phase 07).
64
- 2. If the session's Step 6 bootstrapped Test Agents, pass the repro steps to the generator agent.
65
- 3. The generator produces a `tests/repro-<bug-id>.spec.ts` you can commit alongside the bug report.
66
- 4. The **healer** agent can later be used by the team if the test becomes flaky.
76
+ 1. Finish the bug report with exact steps (phase 7).
77
+ 2. If Step 6 of setup bootstrapped Test Agents, pass the bug report to the generator agent.
78
+ 3. The generator produces a `tests/repro-BUG-NNN.spec.ts` you can commit alongside the bug report.
79
+ 4. The **healer** agent later repairs the test if it breaks for non-bug reasons.
67
80
 
68
- Full workflow in `../references/playwright-agents-integration.md`. This handoff is **opt-in** — it does not replace the bug report and is skipped entirely when `@playwright/test` is not installed.
81
+ Full workflow in `${CLAUDE_SKILL_DIR}/references/playwright-agents-integration.md`. This
82
+ handoff is **opt-in**; it does not replace the bug report and is skipped entirely when
83
+ `@playwright/test` is not installed.
69
84
 
70
85
  ## After Feature Testing
71
86
 
72
- Write to `phase-3-feature-testing.md`. Include reasoning for each test.
87
+ Write `phase-5-features.md` (confidentiality header first). Include the reasoning for each test.
73
88
 
74
- **Update progress:** Set features phase complete in progress.json. Append to session-log.md:
89
+ **Update progress:** set the features phase complete in `progress.json`. Append to `session-log.md`:
75
90
  `[<timestamp>] [PHASE] Features complete — <N> P0 features tested, <N> P1 features tested, <N> inconsistencies found, <N> bugs so far`
@@ -1,6 +1,6 @@
1
- # Phase 6: Edge Cases, Security & Negative Testing (~20% of time)
1
+ # Phase 6: Edge Cases, Security & Negative Testing (6 min)
2
2
 
3
- **Goal:** Saboteur tour -- actively try to break things.
3
+ **Goal:** Saboteur tour: actively try to break things. Every command carries `-s=<sid>` (omitted here). In read-only mode, fill but never submit.
4
4
 
5
5
  ## Input Attacks (from error-guessing knowledge)
6
6
 
@@ -17,15 +17,21 @@ playwright-cli fill <ref> " "
17
17
  playwright-cli fill <ref> "A repeated 5000 times..."
18
18
  ```
19
19
 
20
+ After an XSS payload, check whether it executed: `playwright-cli console error` plus
21
+ `playwright-cli run-code "async page => (await page.pageErrors()).length"`, and look for a
22
+ dialog (`dialog-dismiss` if one opened).
23
+
20
24
  ## Race Conditions (if applicable)
21
25
 
22
26
  ```bash
23
- # Open a second session
24
- playwright-cli -s=race open <url>
25
- playwright-cli -s=race state-load .auth/<target>.json
26
- # Set up identical action in both sessions
27
+ # Open a second, separately isolated session alongside the main one
28
+ playwright-cli -s=<sid>-race open <url>
29
+ playwright-cli -s=<sid>-race state-load .auth/<target>.json
30
+ # Set up the identical action in both sessions
27
31
  # Submit both as close to simultaneously as possible
28
- # Check for double-processing
32
+ # Check for double-processing, then close the second session:
33
+ playwright-cli -s=<sid>-race close
34
+ playwright-cli -s=<sid>-race delete-data
29
35
  ```
30
36
 
31
37
  ## State Manipulation
@@ -33,14 +39,15 @@ playwright-cli -s=race state-load .auth/<target>.json
33
39
  - Navigate directly to pages that require prior steps
34
40
  - Use browser back/forward mid-flow
35
41
  - Double-click submit buttons
36
- - Simulate network errors: `playwright-cli route "**/*" --status=500`
42
+ - Simulate network errors: `playwright-cli route "**/*" --status=500` (then `playwright-cli unroute` to restore)
43
+ - Simulate offline: `playwright-cli network-state-set offline` / `online`
37
44
 
38
45
  ## Security
39
46
 
40
47
  - Check for session tokens in URLs
41
48
  - Try accessing admin/internal pages directly
42
- - Check if authenticated endpoints work without auth
43
- - Test brute force: 5+ failed login attempts -- check for lockout
49
+ - Check whether authenticated endpoints work without auth (`playwright-cli -s=<sid>-anon open <api-url>` in a fresh session, then close and `delete-data` it)
50
+ - Test brute force: 5+ failed login attempts; check for lockout
44
51
 
45
52
  ## Accessibility (quick check)
46
53
 
@@ -54,7 +61,11 @@ playwright-cli snapshot
54
61
  playwright-cli resize 1280 720 # restore
55
62
  ```
56
63
 
57
- > **Note on `page.accessibility` (Playwright 1.57 deprecation):** Upstream Playwright deprecated the `page.accessibility` snapshot API in 1.57. The replacement, available as of Playwright 1.59, is `page.ariaSnapshot()` — an aria-tree-based accessibility snapshot that is both more accurate and directly usable as a locator base. If a session upgrades to deeper a11y checks than the quick-pass above, prefer `page.ariaSnapshot()` over `page.accessibility`. The CLI checks above are unaffected and remain the primary path. See `../references/playwright-agents-integration.md` for a feature cross-reference.
64
+ The CLI `snapshot` **is** an aria snapshot: roles, names and states as assistive technology
65
+ sees them, so missing names and duplicate landmarks show up directly. For the raw YAML aria
66
+ tree of a region, `playwright-cli run-code "async page => page.locator('main').ariaSnapshot()"`.
67
+ Note: Playwright **removed** `page.accessibility` in 1.57; anything still calling it will
68
+ throw.
58
69
 
59
70
  ## Empty States
60
71
 
@@ -65,7 +76,7 @@ playwright-cli resize 1280 720 # restore
65
76
 
66
77
  ## After Edge Cases
67
78
 
68
- Write to `phase-4-edge-cases.md`.
79
+ Write `phase-6-edge-cases.md` (confidentiality header first).
69
80
 
70
- **Update progress:** Set edge_cases phase complete in progress.json. Append to session-log.md:
81
+ **Update progress:** set the edge_cases phase complete in `progress.json`. Append to `session-log.md`:
71
82
  `[<timestamp>] [PHASE] Edge cases complete — <N> input attacks, <N> security checks, <N> a11y checks, <N> bugs so far`
@@ -1,27 +1,43 @@
1
- # Phase 7: Reflection & Reporting (~15% of time)
1
+ # Phase 7: Reflection & Reporting (4 min)
2
+
3
+ Every command carries `-s=<sid>` (omitted here). The browser stays open until the evidence is captured; it is closed in the last step.
2
4
 
3
5
  ## Stop Recording
4
6
 
5
7
  ```bash
8
+ playwright-cli video-chapter "Reporting"
9
+ playwright-cli video-stop
6
10
  playwright-cli tracing-stop
7
- playwright-cli close
8
11
  ```
9
12
 
10
- ### Trace Analysis (Playwright 1.58–1.59, optional)
13
+ ### Trace analysis (Playwright 1.59+, optional)
14
+
15
+ `tracing-stop` writes a trace under `traces/` (or the path it prints). To inspect it from
16
+ the command line: `npx playwright trace open <trace>` then `npx playwright trace actions --errors-only`,
17
+ `npx playwright trace requests --failed`, `npx playwright trace console`, and `npx playwright trace close`
18
+ (check `npx playwright trace --help` for the exact subcommands of the installed version).
19
+ Copy the trace into `output/sessions/<session-dir>/` if it is evidence for a bug.
20
+
21
+ ### Annotated video for Critical / High bugs (optional)
11
22
 
12
- Once tracing is stopped, the captured trace can be explored from the command line or the HTML reporter:
23
+ The session recording from phase 2 already has chapters. For a polished reproduction of a
24
+ Critical or High bug, record a second clip with action callouts:
13
25
 
14
- - **`npx playwright trace <trace.zip>`** *(Playwright 1.59)* — opens the trace from the CLI without launching the full UI reporter. Useful for agent-driven debugging of a specific step or failure.
15
- - **HTML reporter "Speedboard" timeline** *(Playwright 1.58)* — when the session generated a Playwright HTML report, the "Speedboard" tab shows a timeline visualization of steps useful for spotting latency anomalies that correlate with the bugs you filed.
16
- - **Trace Viewer themes** *(Playwright 1.58)* — the Trace Viewer now honors system theme; no action required, just mentioned here so the reviewer knows the option exists.
17
- - **`page.screencast()`** *(Playwright 1.59, optional)* — if the session wants to attach an annotated video of the reproduction to the bug report, `page.screencast()` captures it with action annotations and chapter overlays. Treat it as premium evidence for Critical / High bugs; a screenshot remains sufficient for Medium / Low.
26
+ ```bash
27
+ playwright-cli video-start output/sessions/<session-dir>/videos/BUG-NNN.webm
28
+ playwright-cli video-show-actions
29
+ # ...reproduce the bug...
30
+ playwright-cli video-hide-actions
31
+ playwright-cli video-stop
32
+ ```
18
33
 
19
- These helpers are additive — the existing `screenshots/bug-NNN.png` workflow below is unchanged.
34
+ Or write a scripted "hero" reproduction with `page.screencast.start()` / `showChapter()` /
35
+ `showOverlay()` (Playwright 1.59) and run it with `playwright-cli run-code --filename=<script.js>`.
36
+ A screenshot remains sufficient for Medium / Low.
20
37
 
21
38
  ## Reflect Before Writing
22
39
 
23
- Before writing the report, answer these questions:
24
- 1. **What was the biggest risk I found?** (not the most bugs -- the biggest RISK)
40
+ 1. **What was the biggest risk I found?** (not the most bugs; the biggest RISK)
25
41
  2. **What would I tell the CEO in one sentence?**
26
42
  3. **What did I NOT test that still worries me?**
27
43
  4. **Which heuristic was most useful? Which was useless?**
@@ -29,145 +45,68 @@ Before writing the report, answer these questions:
29
45
 
30
46
  ## Write Bug Reports
31
47
 
32
- For each bug, write `bugs/BUG-NNN.md`:
33
-
34
- ```markdown
35
- ## [Component] fails [Condition] causing [Impact]
36
-
37
- **Severity:** Critical | High | Medium | Low
38
- **URL:** [exact URL]
39
- **Environment:** Playwright CLI, Chromium, [viewport]
40
-
41
- ### Expected Behavior
42
- [What should happen -- from the USER's perspective]
43
-
44
- ### Actual Behavior
45
- [What actually happens -- be specific, include error messages]
46
-
47
- ### Steps to Reproduce
48
- 1. [Exact steps -- a developer should be able to reproduce this]
49
-
50
- ### Business Impact
51
- [THIS IS MANDATORY -- answer at least one:]
52
- - **Revenue impact:** [Does this prevent purchases? Cause incorrect charges?]
53
- - **Trust impact:** [Would a user lose confidence? Would they call support?]
54
- - **Regulatory risk:** [Does this violate SOX, PCI, GDPR, WCAG, etc.?]
55
- - **Data risk:** [Is data corrupted, lost, or exposed?]
56
- - **Scale:** [How many users are affected? Under what conditions?]
57
-
58
- ### Evidence
59
- - Screenshot: `screenshots/bug-NNN.png`
60
- - Console errors: [if any]
61
- - Network failures: [if any]
62
-
63
- ### Recommended Fix Priority
64
- [Why this should be fixed before/after other bugs]
65
- ```
66
-
67
- ## Write Session Report
68
-
69
- `session-report.md` with:
70
- - **Executive summary** (3 sentences max -- what was tested, what was found, what's the risk)
71
- - Business context and charter summary
72
- - Bug summary table (severity, component, business impact)
73
- - Data integrity results
74
- - What's MISSING (negative space findings)
75
- - Coverage map with risk ranking
76
- - Cross-feature journey results
77
- - Heuristic effectiveness (which worked, which didn't)
78
- - Areas NOT tested and why
79
- - Recommendations for next session
80
- - Reflection answers
81
-
82
- ## Coverage Map
83
-
84
- Document what was tested and what was not:
85
-
86
- | Feature | Risk | Tested? | Findings | Notes |
87
- |---------|------|---------|----------|-------|
88
- | [feature] | P0 | Yes | 2 bugs | [notes] |
89
- | [feature] | P1 | Partial | 1 bug | [notes] |
90
- | [feature] | P2 | No | -- | Time ran out |
48
+ One file per bug, `bugs/BUG-NNN.md`, in exactly the format of
49
+ `${CLAUDE_SKILL_DIR}/references/output-contract.md` (the template
50
+ `<data>/templates/bug-report.md` is the same format with guidance):
51
+
52
+ - the confidentiality header as the **first two lines**, verbatim:
53
+
54
+ ```
55
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
56
+ > and application details. Do not share outside your organization without review.
57
+ ```
58
+
59
+ - then `# BUG-NNN: [Component] fails [Condition] causing [Impact]`
60
+ - `**Severity:**` (per `severity-guide.md`; when in doubt go lower), `**Priority:**`,
61
+ `**Component:**`, `**URL:**`, `**Environment:** Playwright CLI, Chromium, <viewport>`,
62
+ `**Reproduction rate:**`
63
+ - `## Summary`, `## Expected Behavior`, `## Actual Behavior`, `## Steps to Reproduce`
64
+ - `## Business Impact` — MANDATORY; answer at least one of Revenue / Trust / Regulatory /
65
+ Data / Scale, write "none identified" for the rest
66
+ - `## Evidence` — `Screenshot: screenshots/BUG-NNN.png` (take it with
67
+ `playwright-cli screenshot --filename=output/sessions/<session-dir>/screenshots/BUG-NNN.png`),
68
+ console errors, network failures (`playwright-cli request <n>` output, redacted)
69
+ - `## Recommended Fix Priority`
70
+
71
+ Redact per `security-rules.md` before writing.
72
+
73
+ ## Write the Session Report
74
+
75
+ `session-report.md` in the format of `output-contract.md`: header, `# Session Report — <target>`,
76
+ `## Session Metadata`, `## Executive Summary` (3 sentences: what was tested, what was found,
77
+ the biggest risk), `## Summary Stats`, `## Coverage Map` with the header
78
+ `| Area | Risk | Status | Bugs | Notes |` (Status: tested / partial / not-tested /
79
+ code-verified-only), `## Bugs Found`, `## Observations` (including what's MISSING and the
80
+ data-integrity results), `## Areas Not Tested` with reasons, `## Recommendations` for the
81
+ next session, `## Reflection` (the five answers above), `## Session Stats`.
91
82
 
92
83
  ## Session Stats
93
84
 
94
- Write `output/sessions/<session-dir>/stats.json` with session metrics:
95
-
96
- ```json
97
- {
98
- "session_id": "<session-dir>",
99
- "target": "<target_url>",
100
- "domain": "<domain>",
101
- "started_at": "<from progress.json>",
102
- "completed_at": "<ISO timestamp now>",
103
- "duration_min": "<calculated>",
104
- "phases_completed": "<count of phases with status=complete>",
105
- "total_phases": 8,
106
- "bugs": {
107
- "total": "<count>",
108
- "critical": "<count>",
109
- "high": "<count>",
110
- "medium": "<count>",
111
- "low": "<count>"
112
- },
113
- "coverage": {
114
- "pages_explored": "<count>",
115
- "checklist_total": "<from domain config>",
116
- "checklist_verified": "<count checked>",
117
- "checklist_percentage": "<calculated>",
118
- "data_integrity_checks": "<count>",
119
- "data_integrity_passed": "<count>",
120
- "personas_tested": ["<list>"],
121
- "purchase_flows_tested": ["<list>"]
122
- },
123
- "evidence": {
124
- "screenshots": "<count>",
125
- "console_errors_found": "<count>",
126
- "network_failures_found": "<count>"
127
- },
128
- "areas_not_tested": ["<list with reasons>"],
129
- "blocked_by": "<if something blocked testing, note it>"
130
- }
131
- ```
132
-
133
- Also append a **stats summary** at the bottom of `session-report.md`:
85
+ Write `output/sessions/<session-dir>/stats.json` exactly as `output-contract.md` defines it
86
+ (`session_id`, `kind: "explore"`, `target`, `date`, `duration_min`, `bugs_found`,
87
+ `severity_counts`, `pages_explored`, plus the optional `domain`, `started_at`,
88
+ `completed_at`, `phases_completed`, `total_phases: 8`, `coverage`, `evidence`,
89
+ `areas_not_tested`, `blocked_by`). No other top-level keys.
134
90
 
135
- ```markdown
136
- ---
91
+ Append the same numbers as the `## Session Stats` table at the end of `session-report.md`.
137
92
 
138
- ## Session Stats
93
+ ## Update progress.json — Final
139
94
 
140
- | Metric | Value |
141
- |--------|-------|
142
- | Duration | <X> min |
143
- | Phases completed | <N>/8 |
144
- | Bugs found | <N> (Critical: <N>, High: <N>, Medium: <N>, Low: <N>) |
145
- | Pages explored | <N> |
146
- | Screenshots taken | <N> |
147
- | Console errors found | <N> |
95
+ Set `status` to `complete`, `current_phase` to `reporting`, every phase to its final status.
148
96
 
149
- ### Coverage
97
+ ## Update Indexes
150
98
 
151
- | Area | Risk | Tested | Depth | Bugs | Notes |
152
- |------|------|--------|-------|------|-------|
153
- | <area> | P0 | Yes | Deep | <N> | <notes> |
154
- | <area> | P1 | Partial | Shallow | <N> | <blocked by...> |
155
- | <area> | P2 | No | — | — | Time ran out |
99
+ Append the session row to `output/sessions/INDEX.md` and one row per bug to
100
+ `output/bugs/all-bugs.md`, in the column order defined in `paths.md`.
156
101
 
157
- ### Checklist Coverage
158
- <N>/<N> items verified (<X>%)
102
+ ## Close the Browser Session
159
103
 
160
- ### Data Integrity
161
- <N>/<N> checks passed
104
+ Last, once every screenshot, trace and video is on disk:
162
105
 
163
- ### Not Tested (and why)
164
- - <area>: <reason>
106
+ ```bash
107
+ playwright-cli close
108
+ playwright-cli delete-data
165
109
  ```
166
110
 
167
- ## Update progress.json — Final
168
-
169
- Set status to "complete", current_phase to "reporting", all phases to their final status.
170
-
171
- ## Update Indexes
172
-
173
- Append to `output/sessions/INDEX.md` and `output/bugs/all-bugs.md`.
111
+ (both with `-s=<sid>`; also close and `delete-data` any extra session such as `-s=<sid>-race`).
112
+ Append to `session-log.md`: `[<timestamp>] [PHASE] Reporting complete — <N> bugs, session closed`.
@@ -4,7 +4,7 @@ How the `--context` parameter changes the session.
4
4
 
5
5
  ## Mode 1: Context File (`--context <file>`)
6
6
 
7
- Read the file. It may be a `tool-qa-workflow` output (structured markdown with Jira data, Confluence specs, GitLab MR diffs, Figma design notes) or any document describing what to test.
7
+ Read the file. It may be a `/qa-gather` context file (structured markdown with ticket data, specs, MR diffs, design notes) or any document describing what to test.
8
8
 
9
9
  Extract and use:
10
10
 
@@ -0,0 +1,187 @@
1
+ # Output Contract
2
+
3
+ One format for every session kind (explore, quick, mobile, backend). The templates in
4
+ `<data>/templates/` follow it, `qualiow report` parses it, and the feedback and cleanup
5
+ skills read it. A session that deviates from it is invisible to all of them.
6
+
7
+ ## Confidentiality header
8
+
9
+ The first two lines of **every** markdown artefact (charter, phase files, bug reports,
10
+ session report, coverage map, AC matrix, expected-behaviour spec):
11
+
12
+ ```
13
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
14
+ > and application details. Do not share outside your organization without review.
15
+ ```
16
+
17
+ ## Session files
18
+
19
+ | File | Written by |
20
+ |---|---|
21
+ | `charter.md` | phase 2 |
22
+ | `session-log.md` | every phase, append-only |
23
+ | `progress.json` | phase 0, updated after every phase |
24
+ | `phase-3-discovery.md` … `phase-6-edge-cases.md` | phases 3–6 (file number = phase number) |
25
+ | `screenshots/BUG-NNN.png`, other `screenshots/*.png` | as taken |
26
+ | `videos/*.webm` (web) / `videos/*.mp4` (mobile) | recording |
27
+ | `bugs/BUG-NNN.md` | phase 7 |
28
+ | `session-report.md` | phase 7 |
29
+ | `stats.json` | phase 7 |
30
+ | mobile: `logs/BUG-NNN.log` · backend: `ac-matrix.md`, `evidence/`, `probes/`, `expected-behaviour.md` | as produced |
31
+
32
+ The coverage map lives inside `session-report.md`. There is no separate session-level
33
+ `coverage-map.md`; `/qa-explore-report` can export one on request.
34
+
35
+ ## Bug report — `bugs/BUG-NNN.md`
36
+
37
+ ```markdown
38
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
39
+ > and application details. Do not share outside your organization without review.
40
+
41
+ # BUG-NNN: [Component] fails [Condition] causing [Impact]
42
+
43
+ **Severity:** Critical | High | Medium | Low
44
+ **Priority:** P0 | P1 | P2 | P3
45
+ **Component:** <Component>
46
+ **URL:** <exact URL — mobile: screen name or deep link — backend: endpoint or resource>
47
+ **Environment:** <browser + viewport — mobile: mode / platform / device / OS — backend: env kind + build>
48
+ **Reproduction rate:** Always | Intermittent (~X%) | Once
49
+
50
+ ## Summary
51
+ <two sentences: what breaks and why it matters>
52
+
53
+ ## Expected Behavior
54
+ <from the user's perspective>
55
+
56
+ ## Actual Behavior
57
+ <be specific; quote error messages verbatim>
58
+
59
+ ## Steps to Reproduce
60
+ 1. <exact steps a developer can follow>
61
+
62
+ ## Business Impact
63
+ - **Revenue impact:** <or "none identified">
64
+ - **Trust impact:** …
65
+ - **Regulatory risk:** …
66
+ - **Data risk:** …
67
+ - **Scale:** <who is affected, under what conditions>
68
+
69
+ ## Evidence
70
+ - Screenshot: `screenshots/BUG-NNN.png`
71
+ - Video: `videos/<file>` (if recorded)
72
+ - Log: `logs/BUG-NNN.log` (mobile) / `evidence/<file>` (backend)
73
+ - Console errors: <or "none">
74
+ - Network failures: <or "none">
75
+
76
+ ## Recommended Fix Priority
77
+ <why this priority, relative to the other findings>
78
+ ```
79
+
80
+ Rules: numbered from `BUG-001` per session; one bug per file; Business Impact is mandatory
81
+ (answer at least one bullet, write "none identified" for the rest); severity per
82
+ `severity-guide.md`, and when in doubt go one level lower.
83
+
84
+ ## Session report — `session-report.md`
85
+
86
+ ```markdown
87
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
88
+ > and application details. Do not share outside your organization without review.
89
+
90
+ # Session Report — <target>
91
+
92
+ ## Session Metadata
93
+ | Field | Value |
94
+ |---|---|
95
+ | Session ID | <session-dir> |
96
+ | Kind | explore / quick / mobile / backend |
97
+ | Target | <target id or name> |
98
+ | URL | <base url> |
99
+ | Date | <YYYY-MM-DD> |
100
+ | Duration | <N> min |
101
+ | Charter | charter.md |
102
+
103
+ ## Executive Summary
104
+ <three sentences: what was tested, what was found, the biggest risk>
105
+
106
+ ## Summary Stats
107
+ | Metric | Count |
108
+ |---|---|
109
+ | Pages explored | |
110
+ | Actions performed | |
111
+ | Bugs — Critical | |
112
+ | Bugs — High | |
113
+ | Bugs — Medium | |
114
+ | Bugs — Low | |
115
+ | **Total bugs** | |
116
+
117
+ ## Coverage Map
118
+ | Area | Risk | Status | Bugs | Notes |
119
+ |---|---|---|---|---|
120
+ | <area> | P0 | tested | 2 | |
121
+ | <area> | P1 | partial | 1 | keyboard covered submit |
122
+ | <area> | P2 | not-tested | — | time ran out |
123
+
124
+ ## Bugs Found
125
+ | # | ID | Title | Severity | Report |
126
+ |---|---|---|---|---|
127
+ | 1 | BUG-001 | <title> | High | bugs/BUG-001.md |
128
+
129
+ ## Observations
130
+ ## Areas Not Tested
131
+ | Area | Reason |
132
+ |---|---|
133
+
134
+ ## Recommendations
135
+ ## Reflection
136
+ ## Session Stats
137
+ | Metric | Value |
138
+ |---|---|
139
+ ```
140
+
141
+ Coverage `Status` ∈ `tested` · `partial` · `not-tested` · `code-verified-only` (believed
142
+ correct from reading source, never observed: this is UNVERIFIABLE, not a pass, and must not
143
+ be skimmed as green).
144
+
145
+ Mobile appends `## Mobile Context` and `## Deferred Tests`; backend inserts
146
+ `## AC Matrix Summary` (all six verdicts counted) after the executive summary.
147
+
148
+ ## `stats.json`
149
+
150
+ ```json
151
+ {
152
+ "session_id": "<session-dir>",
153
+ "kind": "explore",
154
+ "target": "<target id or url>",
155
+ "date": "<YYYY-MM-DD>",
156
+ "duration_min": 42,
157
+ "bugs_found": 5,
158
+ "severity_counts": { "critical": 1, "high": 2, "medium": 2, "low": 0 },
159
+ "pages_explored": 14,
160
+ "domain": "ecommerce",
161
+ "started_at": "<ISO timestamp>",
162
+ "completed_at": "<ISO timestamp>",
163
+ "phases_completed": 8,
164
+ "total_phases": 8,
165
+ "coverage": { "checklist_total": 22, "checklist_verified": 15, "data_integrity_checks": 6, "data_integrity_passed": 5 },
166
+ "evidence": { "screenshots": 9, "console_errors_found": 2, "network_failures_found": 1 },
167
+ "areas_not_tested": ["settings — time ran out"],
168
+ "blocked_by": null
169
+ }
170
+ ```
171
+
172
+ The first eight keys are required; the rest are optional. No other top-level keys (the
173
+ schema is strict). Mobile puts its device block under `coverage.mobile`; backend puts the
174
+ verdict counts under `coverage.verdicts`.
175
+
176
+ ## Index rows
177
+
178
+ Every session ends by appending one row to `output/sessions/INDEX.md` and one row per bug
179
+ to `output/bugs/all-bugs.md`. The column order (defined once in `paths.md`) is:
180
+
181
+ ```
182
+ | Date | Kind | Target | Bugs | Duration | Status | Report |
183
+ | ID | Session | Title | Severity | Status | Report |
184
+ ```
185
+
186
+ A session that skips these rows is invisible to `/qa-explore-report`,
187
+ `/qa-explore-feedback`, `/qa-explore-cleanup` and `qualiow list sessions`.