qualiow-exploratory-testing 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/.claude-plugin/plugin.json +22 -0
  2. package/.env.example +27 -0
  3. package/CHANGELOG.md +194 -0
  4. package/README.md +362 -205
  5. package/agents/qa-gather-agent.md +24 -0
  6. package/{scripts → bin}/doctor-mobile.sh +57 -18
  7. package/bin/mcli +15 -9
  8. package/bin/mobile-cli.mjs +574 -286
  9. package/{scripts → bin}/setup-mobile.sh +56 -19
  10. package/bin/wadb +10 -4
  11. package/bin/wk-ios +29 -17
  12. package/bin/wkeval.mjs +7 -1
  13. package/data/domains/_default.yml +1 -1
  14. package/data/knowledge/changelog.yml +30 -12
  15. package/data/knowledge/manifest.yml +333 -127
  16. package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
  17. package/data/knowledge/releases/v0.1.0/release.yml +8 -1
  18. package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
  19. package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
  20. package/data/knowledge/releases/v0.6.0/release.yml +30 -0
  21. package/data/security/SECURITY-POLICY.md +273 -0
  22. package/data/targets/_default.yml +1 -1
  23. package/data/targets/_example-api-only.yml +80 -0
  24. package/data/targets/_example-mobile-emulation.yml +2 -2
  25. package/data/targets/_example-native-mobile.yml +2 -2
  26. package/data/targets/_example-sim-android-chrome.yml +1 -1
  27. package/data/targets/_example-sim-ios-safari.yml +1 -1
  28. package/data/templates/api-probe-matrix.md +3 -3
  29. package/data/templates/bug-report.md +29 -151
  30. package/data/templates/coverage-map.md +9 -6
  31. package/data/templates/expected-behaviour.md +3 -2
  32. package/data/templates/session-charter.md +3 -0
  33. package/data/templates/session-report.md +57 -80
  34. package/dist/cli/index.js +1452 -700
  35. package/dist/index.d.ts +589 -81
  36. package/dist/index.js +1612 -227
  37. package/docs/BACKEND-VERIFICATION.md +39 -9
  38. package/docs/GETTING-STARTED.md +188 -128
  39. package/docs/MOBILE-SETUP.md +149 -18
  40. package/package.json +32 -18
  41. package/skills/qa-explore/SKILL.md +31 -25
  42. package/skills/qa-explore/phases/00-setup.md +67 -41
  43. package/skills/qa-explore/phases/01-auth.md +32 -15
  44. package/skills/qa-explore/phases/02-charter.md +16 -5
  45. package/skills/qa-explore/phases/03-discovery.md +50 -26
  46. package/skills/qa-explore/phases/04-journeys.md +2 -2
  47. package/skills/qa-explore/phases/05-features.md +39 -24
  48. package/skills/qa-explore/phases/06-edge-cases.md +24 -13
  49. package/skills/qa-explore/phases/07-reporting.md +79 -140
  50. package/skills/qa-explore/references/context-integration.md +1 -1
  51. package/skills/qa-explore/references/output-contract.md +187 -0
  52. package/skills/qa-explore/references/paths.md +84 -0
  53. package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
  54. package/skills/qa-explore/references/security-rules.md +66 -25
  55. package/skills/qa-explore/references/severity-guide.md +31 -4
  56. package/skills/qa-explore-cleanup/SKILL.md +17 -6
  57. package/skills/qa-explore-feedback/SKILL.md +18 -5
  58. package/skills/qa-explore-mobile/SKILL.md +86 -64
  59. package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
  60. package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
  61. package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
  62. package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
  63. package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
  64. package/skills/qa-explore-mobile/phases/05-features.md +9 -9
  65. package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
  66. package/skills/qa-explore-mobile/phases/07-reporting.md +82 -56
  67. package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
  68. package/skills/qa-explore-quick/SKILL.md +60 -25
  69. package/skills/qa-explore-report/SKILL.md +40 -34
  70. package/skills/qa-gather/SKILL.md +17 -2
  71. package/skills/qa-knowledge-add/SKILL.md +20 -9
  72. package/skills/qa-knowledge-list/SKILL.md +12 -3
  73. package/skills/qa-target-setup/SKILL.md +63 -41
  74. package/skills/qa-verify-backend/SKILL.md +26 -17
  75. package/skills/qa-verify-backend/phases/00-setup.md +57 -12
  76. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +2 -2
  77. package/skills/qa-verify-backend/phases/03-live-verification.md +3 -3
  78. package/skills/qa-verify-backend/phases/03b-api-verification.md +22 -9
  79. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +18 -2
  80. package/skills/qa-verify-backend/phases/05-reporting.md +38 -10
  81. package/skills/qa-verify-backend/references/api-probes.md +28 -1
  82. package/skills/qa-verify-backend/references/payload-verification.md +1 -1
  83. package/skills/qa-verify-backend/references/release-readiness.md +1 -1
  84. package/skills/qa-verify-backend/references/safety-rules.md +13 -8
  85. package/data/domains/_default.md +0 -181
  86. package/data/domains/ecommerce.md +0 -192
  87. package/data/domains/fintech.md +0 -200
  88. package/data/domains/identity.md +0 -88
  89. package/data/domains/marketing.md +0 -194
  90. package/data/domains/saas.md +0 -197
  91. package/dist/cli/index.d.ts +0 -1
  92. package/dist/cli/index.js.map +0 -1
  93. package/dist/index.js.map +0 -1
@@ -4,60 +4,72 @@ description: >
4
4
  Interactive setup for a new target application. Navigates to the URL, detects auth requirements,
5
5
  walks through login flow creation, auto-detects scope, and saves target config.
6
6
  Use when user says: "setup target", "add target", "configure app", "new target".
7
- allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Read, Write, Glob, Grep
7
+ argument-hint: "<url> [--name <id>] [--shared]"
8
+ allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(npx playwright:*), Read, Write, Glob, Grep
8
9
  ---
9
10
 
10
11
  # Target Application Setup
11
12
 
12
- Interactive wizard to create a new target config in `data/targets/`.
13
+ Interactive wizard that writes a target config. By default it writes the **project-local**
14
+ `qa/target.yml` (the file every session skill reads when no `--target` is given; see
15
+ `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`). With `--shared`, it writes
16
+ `data/targets/<name>.yml` instead, for a target the whole team shares by name. Keep internal
17
+ hostnames in `qa/target.yml` or in `data/targets/local-*.yml` (both gitignored patterns).
18
+
19
+ Security: `${CLAUDE_SKILL_DIR}/../qa-explore/references/security-rules.md` applies (the
20
+ production rule decides the `safety:` block; never write a credential value into YAML).
21
+ The fields written here are what the session skills read to fill `**Environment:**` and the
22
+ session metadata in `${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md`, so
23
+ `id`, `environment.kind` and `base_url` must be the real ones.
13
24
 
14
25
  ## Flow
15
26
 
16
27
  ### 1. Gather Basic Info
17
28
 
18
- Ask user for:
19
- - **Target name** (slug for filename, e.g., "company-staging")
20
- - **Base URL** (e.g., "https://staging.company.com")
21
- - **Domain type** (ecommerce, saas, fintech, marketing, or auto-detect)
29
+ Ask the user for:
30
+ - **Target name** (slug, e.g. `company-staging`; becomes `id:` and, with `--shared`, the filename)
31
+ - **Base URL** (e.g. `https://staging.company.com`)
32
+ - **Domain type** (`ecommerce`, `saas`, `fintech`, `marketing`, `identity`, or auto-detect against `<data>/domains/*.yml`)
33
+
34
+ Apply the production rule to the hostname now and tell the user what it decided.
22
35
 
23
36
  ### 2. Navigate and Analyze
24
37
 
38
+ Use the session id `-s=setup-<HHmm>-<name>` on every command (omitted below):
39
+
25
40
  ```bash
26
41
  playwright-cli open <base_url>
27
42
  playwright-cli snapshot
28
43
  ```
29
44
 
30
45
  From the snapshot, detect:
31
- - Is this a login page? (look for password fields, "Sign in" buttons)
46
+ - Is this a login page? (password fields, "Sign in" buttons)
32
47
  - What's the main navigation structure?
33
- - What kind of application is this? (helps auto-detect domain)
48
+ - What kind of application is this? (helps auto-detect the domain)
34
49
 
35
50
  ### 3. Handle Authentication
36
51
 
37
- **If no login detected:**
38
- - Set `auth.strategy: none`
39
- - Skip to step 4
40
-
41
- **If login detected:**
52
+ **If no login detected:** set `auth.strategy: none` and skip to step 4.
42
53
 
43
- Ask user which auth strategy to use:
54
+ **If login detected**, ask which strategy to use:
44
55
 
45
56
  **Option A: Storage State (recommended)**
46
- 1. Ask user to provide env var names for credentials
47
- 2. Login via playwright-cli:
57
+ 1. Ask the user for the env var **names** for username and password (values go in `qa/.env`).
58
+ 2. Log in via playwright-cli, reading the values from `qa/.env` (never echo them):
48
59
  ```bash
49
- playwright-cli snapshot # Find form fields
50
- playwright-cli fill <email_ref> "$<username_env>"
51
- playwright-cli fill <password_ref> "$<password_env>"
60
+ playwright-cli snapshot # find form fields
61
+ playwright-cli fill <email_ref> "<username value>"
62
+ playwright-cli fill <password_ref> "<password value>"
52
63
  playwright-cli click <submit_ref>
53
64
  ```
54
- 3. Verify login succeeded (snapshot shows dashboard/profile)
55
- 4. Save state: `playwright-cli state-save .auth/<target-name>.json`
56
- 5. Set `auth.strategy: storage_state` with `state_file: ".auth/<target-name>.json"`
65
+ If the identity provider needs MFA, reopen headed (`playwright-cli open <login_url> --headed`) and let the user complete it by hand; never automate an MFA challenge.
66
+ 3. Verify login succeeded (snapshot shows dashboard/profile).
67
+ 4. Save state: `playwright-cli state-save .auth/<name>.json`
68
+ 5. Set `auth.strategy: storage_state` with `state_file: ".auth/<name>.json"`, plus `auth.credentials.username` / `auth.credentials.password` naming the env vars so a session can re-login when the state expires.
57
69
 
58
70
  **Option B: Token**
59
- 1. Ask for token env var name and injection method (cookie/localStorage)
60
- 2. Set `auth.strategy: token`
71
+ 1. Ask for the token env var name and the injection method (cookie or localStorage).
72
+ 2. Set `auth.strategy: token`, `auth.token: <ENV_VAR_NAME>`.
61
73
 
62
74
  ### 4. Detect Scope
63
75
 
@@ -67,52 +79,62 @@ playwright-cli snapshot
67
79
  ```
68
80
 
69
81
  From the snapshot, identify:
70
- - Top-level navigation sections → suggest as `start_pages`
71
- - Suggest `include_patterns` based on URL structure
72
- - Suggest `exclude_patterns` for common exclusions (/admin, /api, *.pdf)
82
+ - Top-level navigation sections → suggest as `scope.start_pages`
83
+ - Suggest `scope.include_patterns` based on the URL structure
84
+ - Suggest `scope.exclude_patterns` for common exclusions (`/admin`, `/api`, `*.pdf`)
85
+ - `scope.max_depth` (default 3)
73
86
 
74
- Ask user to confirm/modify scope.
87
+ Ask the user to confirm/modify scope.
75
88
 
76
89
  ### 5. Configure Browser Settings
77
90
 
78
91
  Suggest defaults:
79
- - `headless: false` (so user can watch)
92
+ - `headless: false` (so the user can watch)
80
93
  - `viewport: { width: 1280, height: 720 }`
81
- - Ask if mobile testing is needed (add device emulation)
94
+ - Ask whether mobile emulation is needed (`browser.device`, e.g. `iPhone 15`, used with `playwright-cli open --device="iPhone 15"`, or `--mobile` for a generic mobile device — the two flags are mutually exclusive, and `--mobile` is unsupported on Firefox)
82
95
 
83
96
  ### 6. Safety Settings
84
97
 
85
- Ask if this is a production environment:
86
- - If yes: set `safety.read_only: true`, `safety.no_form_submit: true`
87
- - If no: no safety restrictions
98
+ - If the production rule fired, or the user says this is production: `safety.read_only: true`, `safety.no_form_submit: true`.
99
+ - Otherwise: `safety.read_only: false`, `safety.no_form_submit: false`.
88
100
 
89
101
  ### 7. Save Config
90
102
 
91
- Write `data/targets/<target-name>.yml` with all gathered configuration.
103
+ Write `qa/target.yml` (default) or `data/targets/<name>.yml` (`--shared`) with all gathered
104
+ configuration; `id` equals the target name. Then close the session:
92
105
 
93
- Close browser:
94
106
  ```bash
95
107
  playwright-cli close
108
+ playwright-cli delete-data
96
109
  ```
97
110
 
98
- Update `.env.example` with any new env var names.
111
+ Update `qa/.env.example` (create it if missing) with every env var name the config references,
112
+ each as a commented `# NAME=` line. Never write values.
99
113
 
100
114
  ### 8. Confirm
101
115
 
102
- Present the saved config to user:
116
+ Present the saved config to the user:
103
117
  - Target name and URL
104
118
  - Auth strategy configured
105
119
  - Scope defined
106
- - How to use: `/qa-explore --target <target-name>`
120
+ - How to use: `/qa-explore` (picks up `qa/target.yml`) or `/qa-explore --target <name>` for a shared target
107
121
 
108
122
  ### 9. Optional — Offer Playwright Test Agents Bootstrap
109
123
 
110
- If the user plans to author full Playwright tests against this target later (not just exploratory sessions), offer to scaffold the Test Agents workspace now while you already have credentials and scope loaded:
124
+ If the user plans to author full Playwright tests against this target later (not just
125
+ exploratory sessions), offer to scaffold the Test Agents workspace now while credentials and
126
+ scope are already loaded:
111
127
 
112
128
  ```bash
113
129
  npx playwright init-agents --loop=claude
114
130
  ```
115
131
 
116
- This creates the planner / generator / healer agent definitions in the target project (Playwright 1.56+). It is **strictly optional** — declining does not affect the target config just written. If the user accepts, note in the target config's `notes` field that Test Agents are scaffolded; if they decline, continue normally.
132
+ With `--loop=claude` this writes the planner / generator / healer definitions into
133
+ `.claude/agents/`, a `.mcp.json` registering the `playwright-test` server, and a `specs/`
134
+ scaffold with a seed spec if the project has none (Playwright 1.56+; other loops: `codex`,
135
+ `copilot`, `opencode`, `vscode`, `vscode-legacy`). It is **strictly optional**; declining does not affect
136
+ the target config just written. If the user accepts, note in the config's `notes` field that
137
+ Test Agents are scaffolded.
117
138
 
118
- This step requires `@playwright/test` to be installed in the target project. If it is not, skip this step silently.
139
+ This step requires `@playwright/test` and a `playwright.config.*` in the target project. If
140
+ either is missing, skip it silently.
@@ -10,7 +10,8 @@ description: >
10
10
  "review the infra change", "does this meet the acceptance criteria", "test the backend",
11
11
  "test the API", "check the endpoint", or gives a backend/API/infra ticket whose ACs
12
12
  cannot be seen in a browser.
13
- allowed-tools: Read, Write, Glob, Grep, Bash(git:*), Bash(aws:*), Bash(jq:*), Bash(curl:*), Bash(playwright-cli:*), Bash(terraform validate:*), Bash(terraform fmt:*)
13
+ argument-hint: "[--target <id>] [--context <file>] [--static-only] [--api-only] [--no-e2e] [--parity <target-id>]"
14
+ allowed-tools: Read, Write, Glob, Grep, Bash(git:*), Bash(aws:*), Bash(jq:*), Bash(grep:*), Bash(curl:*), Bash(node:*), Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(terraform validate:*), Bash(terraform fmt:*)
14
15
  ---
15
16
 
16
17
  # Backend & Infrastructure AC Verification
@@ -70,26 +71,29 @@ Execute in order. Read and follow the linked file.
70
71
 
71
72
  | Phase | File | Summary |
72
73
  |-------|------|---------|
73
- | **Setup** | `phases/00-setup.md` | Resolve target, ticket context, repo branch, AWS access. Decide which lanes are runnable. |
74
- | **AC Decomposition** | `phases/01-ac-decomposition.md` | Turn each AC into a falsifiable check with a named evidence source. Flag untestable ACs. |
75
- | **Static Review** | `phases/02-static-review.md` | Read the branch diff against every AC. Producer/consumer contract. Spec drift. Negative space. |
76
- | **Live Verification** | `phases/03-live-verification.md` | Read-only aws-cli probes of the cloud resources. One probe per AC. Capture raw evidence. |
77
- | **API Verification** | `phases/03b-api-verification.md` | Fingerprint the environment, then call the endpoints directly from the authenticated session. Case matrix, cross-environment parity, UI-vs-API differential. |
78
- | **End-to-End Trigger** | `phases/04-e2e-trigger.md` | Drive the real write path (UI or API), then re-probe the data layer. Covers "verified in env" ACs. |
79
- | **Reporting** | `phases/05-reporting.md` | AC traceability matrix, bug reports with business impact, DoD gaps. |
74
+ | **Setup** | `${CLAUDE_SKILL_DIR}/phases/00-setup.md` | Resolve target, ticket context, repo branch, AWS access. Decide which lanes are runnable. |
75
+ | **AC Decomposition** | `${CLAUDE_SKILL_DIR}/phases/01-ac-decomposition.md` | Turn each AC into a falsifiable check with a named evidence source. Flag untestable ACs. |
76
+ | **Static Review** | `${CLAUDE_SKILL_DIR}/phases/02-static-review.md` | Read the branch diff against every AC. Producer/consumer contract. Spec drift. Negative space. |
77
+ | **Live Verification** | `${CLAUDE_SKILL_DIR}/phases/03-live-verification.md` | Read-only aws-cli probes of the cloud resources. One probe per AC. Capture raw evidence. |
78
+ | **API Verification** | `${CLAUDE_SKILL_DIR}/phases/03b-api-verification.md` | Fingerprint the environment, then call the endpoints directly from the authenticated session. Case matrix, cross-environment parity, UI-vs-API differential. |
79
+ | **End-to-End Trigger** | `${CLAUDE_SKILL_DIR}/phases/04-e2e-trigger.md` | Drive the real write path (UI or API), then re-probe the data layer. Covers "verified in env" ACs. |
80
+ | **Reporting** | `${CLAUDE_SKILL_DIR}/phases/05-reporting.md` | AC traceability matrix, bug reports with business impact, DoD gaps. |
80
81
 
81
82
  ## References
82
83
 
83
- - **`references/aws-readonly-probes.md`** — canonical probe commands per resource type, and how to read their output
84
- - **`references/api-probes.md`** — getting an authenticated request context without handling a token, the case families worth probing on every endpoint, and how to read the results
85
- - **`references/environment-fingerprinting.md`** — proving which build and which implementation an environment actually runs, before any verdict is written
86
- - **`references/payload-verification.md`** — recomputing derived values, saying what a single-source check does not prove, and comparing the rendered value against the payload it came from
87
- - **`references/release-readiness.md`** — verifying many tickets against one build: the deployment table, the result vocabulary, carry-overs, disposition
88
- - **`references/safety-rules.md`** — read-only discipline, production guardrails, redaction, confidentiality
84
+ - **`${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`** — target resolution (`--target` → `data/targets/<id>.yml`; none → `qa/target.yml`), data directory, credentials, session-directory scheme, index rows
85
+ - **`${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md`** — bug-report, session-report and `stats.json` format, confidentiality header
86
+ - **`${CLAUDE_SKILL_DIR}/../qa-explore/references/security-rules.md`** — the production rule with its exclude list, the redaction list, output classification; `references/safety-rules.md` below adds the backend-specific rules on top
87
+ - **`${CLAUDE_SKILL_DIR}/references/aws-readonly-probes.md`** — canonical probe commands per resource type, and how to read their output
88
+ - **`${CLAUDE_SKILL_DIR}/references/api-probes.md`** — getting an authenticated request context without handling a token, the case families worth probing on every endpoint, and how to read the results
89
+ - **`${CLAUDE_SKILL_DIR}/references/environment-fingerprinting.md`** — proving which build and which implementation an environment actually runs, before any verdict is written
90
+ - **`${CLAUDE_SKILL_DIR}/references/payload-verification.md`** — recomputing derived values, saying what a single-source check does not prove, and comparing the rendered value against the payload it came from
91
+ - **`${CLAUDE_SKILL_DIR}/references/release-readiness.md`** — verifying many tickets against one build: the deployment table, the result vocabulary, carry-overs, disposition
92
+ - **`${CLAUDE_SKILL_DIR}/references/safety-rules.md`** — read-only discipline, account confirmation, destructive runbooks, session credentials, repo hygiene, untrusted content
89
93
 
90
94
  ## Knowledge Base
91
95
 
92
- Load these from `data/knowledge/` before the static and live lanes. They are the
96
+ Load these from `<data>/knowledge/` (resolved per `paths.md`) before the static and live lanes. They are the
93
97
  BE/API verification layer — techniques derived from real sessions with this skill,
94
98
  not from published literature.
95
99
 
@@ -109,6 +113,8 @@ not from published literature.
109
113
  | `technique-expected-behaviour-specification` | The session found behaviour no AC covers — which is most of what an API probe turns up. |
110
114
  | `technique-config-surface-verification` | The change under test *is* the configuration mechanism — a flag endpoint, a settings surface, a shared config library. |
111
115
  | `technique-release-readiness-verification` | The ask is "is this release good to go", not "does this ticket meet its ACs". |
116
+ | `technique-exactly-once-verification` | Any money movement or ledger write: assert the balance delta, not the status field; attack identity collision and simultaneous redelivery. |
117
+ | `technique-async-callback-contracts` | The change consumes callbacks or webhooks: what the endpoint does with an event it cannot match, a duplicate, or a late one. |
112
118
 
113
119
  The mode selection entry governs the others: it decides *how* an AC gets observed,
114
120
  and the rest supply *what to look for* once the channel is chosen.
@@ -129,7 +135,7 @@ and the rest supply *what to look for* once the channel is chosen.
129
135
  size limits. Audit and event systems fail silently by default.
130
136
  6. **Absence is a finding.** An AC that nothing implements, a cleanup that was written
131
137
  but never run, a doc that still describes the deleted architecture.
132
- 7. **One bug = one report**, each with Business Impact, per `data/templates/bug-report.md`.
138
+ 7. **One bug = one report**, each with Business Impact, in the format of `${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md` (`bugs/BUG-NNN.md`).
133
139
  8. **Name the observation mode in every verdict.** `PASS (direct request, raw
134
140
  response attached)` and `PASS (read the diff)` are different claims. A verdict
135
141
  whose only evidence is a code reading is `UNVERIFIABLE`, not `PASS` — see
@@ -160,4 +166,7 @@ and the rest supply *what to look for* once the channel is chosen.
160
166
  shared table, no destructive cleanup — not even when a runbook in the repo says to.
161
167
  Propose it; let a human run it.
162
168
  17. **Redact.** Account IDs beyond what the target config already holds, tokens, emails
163
- of real people, and consumer PII get `[REDACTED]` before anything is written to disk.
169
+ of real people, and consumer PII get `[REDACTED]` before anything is written to disk
170
+ (the full list is in `security-rules.md`).
171
+ 18. **Session directory and header.** `output/sessions/<YYYY-MM-DD-HHmm>-backend-<ticket>/`,
172
+ and the confidentiality header is the first two lines of every artefact.
@@ -3,15 +3,18 @@
3
3
  ## Step 1: Resolve Inputs
4
4
 
5
5
  Extract from the user message:
6
- - **target**: id under `data/targets/` (with a `backend:` and/or `api:` block) — or inline
7
- resource names
6
+ - **target**: `--target <id>` → `data/targets/<id>.yml` (with a `backend:` and/or `api:`
7
+ block); no `--target` → the project-local `qa/target.yml` (resolution order in
8
+ `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`) — or inline resource names
8
9
  - **context**: path to a gathered ticket context file, or inline ticket text
9
10
  - **flags**: `--static-only` (skip live probes), `--api-only` (skip the cloud lane),
10
11
  `--no-e2e` (skip phase 4), `--parity <target-id>` (run the API matrix in a second
11
12
  environment too)
12
13
 
13
- Read the target config. Read `data/domains/<domain>.md` for the domain's data-integrity
14
- checks — they become extra checks beyond the ticket's own ACs.
14
+ Read the target config. Read `<data>/domains/<domain>.yml` for the domain's
15
+ `data_integrity_checks` — they become extra checks beyond the ticket's own ACs. Apply the
16
+ production rule from `security-rules.md` to the account id, profile name, resource names and
17
+ `api.base_url` now; `environment.kind: production` also triggers it.
15
18
 
16
19
  ## Step 2: Resolve the Source Branch
17
20
 
@@ -39,28 +42,64 @@ Compare the returned account to `backend.account_id`.
39
42
  | Matches | Live lane is GO |
40
43
  | Different account | **STOP the live lane.** Report it. Do not probe an account the ticket does not concern. |
41
44
  | No credentials | Live lane is **BLOCKED**, not failed. Continue with static + write the probe commands into the report so a human can run them. |
42
- | Account looks like production | Read `references/safety-rules.md` and apply the production hard stop. |
45
+ | Account looks like production | Read `${CLAUDE_SKILL_DIR}/references/safety-rules.md` and apply the production hard stop. |
43
46
 
44
47
  ## Step 3b: Establish the API Lane (skip if the ticket has no HTTP surface)
45
48
 
46
- From `target.api`, confirm the endpoints under test and the probe allowlist. Open an
47
- authenticated session once — headed, against a persistent profile, because an
49
+ From `target.api`, confirm the endpoints under test, the `probe_allowlist` (read-only,
50
+ callable now) and the `write_allowlist` (state-changing, phase 4 only). How you get a
51
+ request context depends on `api.auth`:
52
+
53
+ ### `session-cookie` — the service sits behind a UI
54
+
55
+ Open an authenticated session once — headed, against a persistent profile, because an
48
56
  interactive identity provider cannot complete headless:
49
57
 
50
58
  ```bash
51
- playwright-cli -s=<target-id> open <base_url> --headed --profile .auth/<profile>
59
+ playwright-cli -s=backend-<HHmm>-<ticket> open <api.base_url or base_url> --headed --profile <api.browser_profile>
52
60
  ```
53
61
 
62
+ `api.browser_profile` (e.g. `.auth/my-service-dev-profile`) is the persistent profile
63
+ directory the target declares; it holds a live session for a real account and is gitignored.
64
+
65
+ The session id follows the one scheme in
66
+ `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md` — `-s=<kind>-<HHmm>-<slug>`, here
67
+ `-s=backend-<HHmm>-<ticket>`. Every later `playwright-cli` call in this session carries the
68
+ same `-s=` id, and the session ends in phase 5 with `close` then `delete-data` (the profile
69
+ directory itself is kept).
70
+
54
71
  Confirm the session is live before building any matrix — a probe run against a logged-out
55
72
  page returns a login page with a `200`, which reads exactly like a passing case.
56
73
 
57
- If the target declares `api.parity_targets` or `--parity` was passed, prime that session
74
+ ### `api-key-env` / `bearer-env` — the service has no UI
75
+
76
+ **Open no browser.** There is no session to inherit and no profile to persist. The request
77
+ context is an out-of-browser runner under `probes/` that reads the credential from the env
78
+ var named by `api.token_env` and sends it in `api.header_name` (default `X-Api-Key`), or as
79
+ `Authorization: Bearer` for `bearer-env`. The value is read from the environment at run
80
+ time and never written into a script, a report or this repo.
81
+
82
+ Prove the credential before the first probe, and record both calls in
83
+ `evidence/fingerprint.md`:
84
+
85
+ ```bash
86
+ curl -s -o /dev/null -w '%{http_code}\n' "$BASE/<a read endpoint>" # expect 401/403
87
+ curl -s -o /dev/null -w '%{http_code}\n' -H "X-Api-Key: $KEY" "$BASE/<same endpoint>" # expect 200
88
+ ```
89
+
90
+ A service that answers the same either way is not authenticating, which is a finding
91
+ before it is a setup step. If the key is missing the lane is **BLOCKED**, not failed —
92
+ write the commands into the report and continue.
93
+
94
+ Charter line for this shape: `api: GO (out-of-browser, api-key)`.
95
+
96
+ If the target declares `api.parity_targets` or `--parity` was passed, prime that context
58
97
  too. Same matrix, both environments, or the comparison is not one.
59
98
 
60
99
  ## Step 3c: Fingerprint the Environment
61
100
 
62
101
  **Before the first probe of any lane.** Read
63
- `references/environment-fingerprinting.md` and answer its three questions: which build is
102
+ `${CLAUDE_SKILL_DIR}/references/environment-fingerprinting.md` and answer its three questions: which build is
64
103
  deployed in every component of the path, whether the changed code path is *selected*
65
104
  here, and whether the commit under test is genuinely an ancestor of what is running.
66
105
 
@@ -73,8 +112,11 @@ needed a probe becomes `BLOCKED` with the exact command attached.
73
112
 
74
113
  ## Step 4: Create the Session Directory
75
114
 
76
- `output/sessions/<YYYY-MM-DD-HHmm>-<ticket>-backend/` containing:
77
- `charter.md`, `ac-matrix.md`, `evidence/`, `bugs/`, `probes/`, `session-log.md`
115
+ `output/sessions/<YYYY-MM-DD-HHmm>-backend-<ticket>/` (scheme in `paths.md`; `<ticket>` is the
116
+ ticket id, else the target id) containing:
117
+ `charter.md`, `ac-matrix.md`, `evidence/`, `bugs/`, `probes/`, `session-log.md`, `progress.json`
118
+ (`"kind": "backend"`). Every markdown file starts with the confidentiality header from
119
+ `output-contract.md`.
78
120
 
79
121
  `probes/` holds the probe scripts and case files, so the run is repeatable by someone
80
122
  else — and by you, against the next environment.
@@ -82,6 +124,9 @@ else — and by you, against the next environment.
82
124
  ## Step 5: Write the Charter
83
125
 
84
126
  ```markdown
127
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
128
+ > and application details. Do not share outside your organization without review.
129
+
85
130
  # Charter — <TICKET>
86
131
  **Mission:** Verify <n> acceptance criteria for <ticket summary>
87
132
  **Target:** <target id> (<environment kind>)
@@ -43,7 +43,7 @@ An AC is untestable when it has no observable falsifier. Common shapes:
43
43
  Every claim is a claim about an environment. Write it in: *"in staging, an empty query
44
44
  returns 400"*. An AC verified in one environment and asserted for another is the single
45
45
  most common false `PASS` this skill exists to prevent — see
46
- `references/environment-fingerprinting.md`.
46
+ `${CLAUDE_SKILL_DIR}/references/environment-fingerprinting.md`.
47
47
 
48
48
  Report these as **DoR gaps**, not as failures of the developer. They are ticket-writing
49
49
  defects and they belong in the report — an untestable AC will be marked done by whoever
@@ -51,7 +51,7 @@ is under the least time pressure.
51
51
 
52
52
  ## Add Domain Checks
53
53
 
54
- From `data/domains/<domain>.md`, pull the data-integrity checks and add any that this
54
+ From `<data>/domains/<domain>.yml`, pull the `data_integrity_checks` and add any that this
55
55
  change touches but the ticket never mentions. These become findings in the
56
56
  **negative space** — the things the ACs forgot to require.
57
57
 
@@ -3,13 +3,13 @@
3
3
  Skip if the live lane is BLOCKED — but still write every probe command into
4
4
  `evidence/live-probes.md` so a human with credentials can run them unchanged.
5
5
 
6
- Read `references/aws-readonly-probes.md` for the command catalogue. Read
7
- `references/safety-rules.md` before the first probe, and confirm the fingerprint from
6
+ Read `${CLAUDE_SKILL_DIR}/references/aws-readonly-probes.md` for the command catalogue. Read
7
+ `${CLAUDE_SKILL_DIR}/references/safety-rules.md` before the first probe, and confirm the fingerprint from
8
8
  phase 0 step 3c is on file — a probe of an environment that does not run the change
9
9
  measures something else.
10
10
 
11
11
  This phase covers the **cloud resources**. The service's own HTTP endpoints are phase 3b
12
- (`phases/03b-api-verification.md`); most tickets need both.
12
+ (`${CLAUDE_SKILL_DIR}/phases/03b-api-verification.md`); most tickets need both.
13
13
 
14
14
  ## Rules
15
15
 
@@ -8,9 +8,9 @@ Run it whenever the ticket touches a request/response contract: a search or list
8
8
  endpoint, validation rules, pagination, filter values, an error contract, a payload
9
9
  shape a consumer reads. Skip it when the change has no HTTP surface at all.
10
10
 
11
- Read `references/api-probes.md` for the mechanics and the case catalogue,
12
- `references/environment-fingerprinting.md` before you believe any result, and
13
- `references/payload-verification.md` once you have one — a well-shaped `200` says nothing
11
+ Read `${CLAUDE_SKILL_DIR}/references/api-probes.md` for the mechanics and the case catalogue,
12
+ `${CLAUDE_SKILL_DIR}/references/environment-fingerprinting.md` before you believe any result, and
13
+ `${CLAUDE_SKILL_DIR}/references/payload-verification.md` once you have one — a well-shaped `200` says nothing
14
14
  about whether the numbers in it are right.
15
15
 
16
16
  ## Step 1: Prove Which Implementation You Are Probing
@@ -45,11 +45,17 @@ playwright-cli -s=<session> eval "$(cat probes/<probe>.js)" --raw
45
45
  cannot complete headless. `--profile` is what makes it stick — a persistent profile
46
46
  directory outlives a `storage_state` JSON by a wide margin.
47
47
 
48
- Fall back to an out-of-browser runner (a script driving `curl`) only when you need
49
- volume or a machine-readable matrix. Take the credential from the authenticated profile
48
+ Fall back to an out-of-browser runner (a script driving `curl`) when you need volume or a
49
+ machine-readable matrix. Take the credential from the authenticated profile
50
50
  (`playwright-cli cookie-list`), keep it in a file outside the repo, and reference it by
51
51
  path — never inline it into a script, a report, or this repo.
52
52
 
53
+ **When the target has no UI** (`api.auth: api-key-env` or `bearer-env`), the out-of-browser
54
+ runner is not a fallback, it is the only mode: there is no page to run inside. Phase 0 step
55
+ 3b has already proved the credential works and that the endpoint rejects a request without
56
+ it. Read the key from the env var at run time; the script contains the variable name, never
57
+ the value.
58
+
53
59
  Record which mode produced each result. An in-page probe and a bare `curl` do not prove
54
60
  the same thing: the first proves the endpoint behaves this way *for a real logged-in
55
61
  session*, the second proves it behaves this way *for whoever holds that credential*.
@@ -58,7 +64,7 @@ session*, the second proves it behaves this way *for whoever holds that credenti
58
64
 
59
65
  Write the cases down first, as data, so the run is repeatable and the evidence is a
60
66
  table rather than a scroll of output. Derive them from the ACs, then add the standard
61
- families from `references/api-probes.md` — the ACs are a floor:
67
+ families from `${CLAUDE_SKILL_DIR}/references/api-probes.md` — the ACs are a floor:
62
68
 
63
69
  boundary lengths · tokenisation and whitespace · metacharacters of whatever query
64
70
  language sits underneath · empty, null, absent and whitespace-only · enum values that
@@ -112,10 +118,16 @@ and add the structural invariants the values must satisfy regardless of magnitud
112
118
 
113
119
  Then write down what the check does not prove: when both sides come from one payload you
114
120
  have verified the derivation, not the inputs. Name the independent oracle that would close
115
- it, and say whether you ran it. Full guidance in `references/payload-verification.md`.
121
+ it, and say whether you ran it. Full guidance in `${CLAUDE_SKILL_DIR}/references/payload-verification.md`.
116
122
 
117
123
  ## Step 6: The Differential Pass — the Same Matrix at Both Surfaces
118
124
 
125
+ **Skip this step when the service has no user interface, and say so in the report** — with
126
+ no second surface there is no differential, and every finding is by definition "API only".
127
+ That is not a reason to downgrade one: an endpoint with no browser guard in front of it is
128
+ reachable exactly as probed, by every client there will ever be. Note instead which findings
129
+ a future client could mask, so the eventual UI is not credited with fixing them.
130
+
119
131
  If the ticket also has a UI, run the overlapping cases through the screen too, and sort
120
132
  every finding into four buckets:
121
133
 
@@ -146,7 +158,8 @@ report:
146
158
  The API lane calls **read** endpoints only. Reads that the ticket's own ACs describe,
147
159
  plus the standard families above, against endpoints the target's `api.probe_allowlist`
148
160
  declares. Anything that creates, mutates or deletes belongs to phase 4, goes through the
149
- real write path, and never runs against production. See `references/safety-rules.md`.
161
+ real write path, is limited to `api.write_allowlist`, and never runs against production.
162
+ See `${CLAUDE_SKILL_DIR}/references/safety-rules.md`.
150
163
 
151
164
  Response bodies are real records. Redact before they touch the disk.
152
165
 
@@ -154,4 +167,4 @@ Response bodies are real records. Redact before they touch the disk.
154
167
 
155
168
  Update `ac-matrix.md` as you go, one row per AC, each verdict naming the environment it
156
169
  holds in. Write the case table to `evidence/api-probe-matrix.md` using
157
- `data/templates/api-probe-matrix.md`.
170
+ `<data>/templates/api-probe-matrix.md`.
@@ -3,7 +3,9 @@
3
3
  Covers the "verified in environment" style of AC: a real change through the real write
4
4
  path, then re-probe the data layer.
5
5
 
6
- Skip if `--no-e2e`, if the environment is production, or if the live lane is BLOCKED.
6
+ Skip if `--no-e2e`, if the target declares `environment.kind: production` (or the production
7
+ rule in `security-rules.md` fired in setup), or if the live lane is BLOCKED. Record the skip
8
+ and its reason in the charter and the report.
7
9
 
8
10
  ## Step 1: Capture the Before State
9
11
 
@@ -25,6 +27,13 @@ If the UI is unavailable, call the write API directly with the same auth — but
25
27
  in the report, because an API-only trigger does not prove the UI's identity
26
28
  propagation works.
27
29
 
30
+ **For a service that has no UI at all, the write API *is* the real path** and no caveat is
31
+ owed. Restrict every state-changing call to the target's `api.write_allowlist`; an endpoint
32
+ absent from that list stays uncalled even when its verb looks harmless. Where the service
33
+ offers a workspace reset, treat it as destructive: it is the one call that erases the
34
+ evidence for everything already observed, so make it deliberate and never routine cleanup
35
+ between cases.
36
+
28
37
  Conversely, if the UI trigger succeeds, that is not evidence the endpoint accepts what
29
38
  other clients will send. The client's own guards are still in the path. Phase 3b covers
30
39
  what happens without them.
@@ -41,7 +50,7 @@ state. Check, in this order:
41
50
  3. **Fidelity** — the stored snapshot matches what the source store now holds, field for
42
51
  field. Compare the actual documents; do not eyeball two summaries. Where the record
43
52
  carries derived values, recompute them from the raw figures beside them
44
- (`references/payload-verification.md`) rather than trusting that they were computed
53
+ (`${CLAUDE_SKILL_DIR}/references/payload-verification.md`) rather than trusting that they were computed
45
54
  once and correctly.
46
55
  4. **Key shape** — the sort key format matches what the consumer will query and sort by.
47
56
  5. **Completeness** — every attribute the ACs require is present, with the required type.
@@ -61,6 +70,13 @@ and event pipelines actually break:
61
70
  another.
62
71
  - **DLQ after.** Re-check depth. Anything new means the pipeline dropped something
63
72
  while you watched.
73
+ - **The replayed event.** Send the same event twice — sequentially, then concurrently, then
74
+ with one field altered. Which field the system deduplicates on is rarely the one the
75
+ documentation implies, and "deduplicated" is not the same claim as "applied exactly once":
76
+ check the effect, not just the status.
77
+ - **The late event.** Deliver an event *after* a timeout, sweeper or retry has already
78
+ resolved the record. A system that closed the record and then still applies the arriving
79
+ event has two truths at once — and in a money system that is the expensive kind.
64
80
 
65
81
  Record each with its own evidence file. Any of these that fails is a bug report,
66
82
  whether or not an AC covers it — the ACs are a floor, not a ceiling.
@@ -32,8 +32,26 @@ know what was running cannot use the rest of the table.
32
32
 
33
33
  ## Step 2: Bug Reports
34
34
 
35
- One bug per report in `bugs/`, using `data/templates/bug-report.md`. Backend findings
36
- need translation — nobody funds a fix for "StreamViewType is NEW_AND_OLD_IMAGES".
35
+ One bug per file, `bugs/BUG-NNN.md`, in the format of
36
+ `${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md` (the template
37
+ `<data>/templates/bug-report.md` is the same format with guidance). The first two lines are
38
+ the confidentiality header, verbatim:
39
+
40
+ ```
41
+ > CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
42
+ > and application details. Do not share outside your organization without review.
43
+ ```
44
+
45
+ then `# BUG-NNN: [Component] fails [Condition] causing [Impact]`, then `**Severity:**`,
46
+ `**Priority:**`, `**Component:**`, `**URL:** <endpoint, table, queue or resource ARN>`,
47
+ `**Environment:** <env kind + build/commit per component from the fingerprint>`,
48
+ `**Reproduction rate:**`, then `## Summary`, `## Expected Behavior`, `## Actual Behavior`,
49
+ `## Steps to Reproduce` (the probe command or trigger sequence), `## Business Impact`,
50
+ `## Evidence` (`- Screenshot:`, `- Video:`, `- Log: evidence/<file>` — the raw probe
51
+ output —, `- Console errors:`, `- Network failures:`) and `## Recommended Fix Priority`.
52
+
53
+ Backend findings need translation — nobody funds a fix for
54
+ "StreamViewType is NEW_AND_OLD_IMAGES".
37
55
 
38
56
  Write the **Business Impact** in terms of consequence:
39
57
  - What can go wrong for a customer, an operator, or an auditor?
@@ -104,7 +122,7 @@ of green rows that is the line the reader acts on.
104
122
 
105
123
  Most of what an API probe finds has no acceptance criterion behind it, so there is nothing
106
124
  to file the finding against and it turns into an argument instead of a fix. Produce an
107
- expected-behaviour specification using `data/templates/expected-behaviour.md`: observed
125
+ expected-behaviour specification using `<data>/templates/expected-behaviour.md`: observed
108
126
  against expected, grouped by cause rather than by case, with the decisions the fix forces
109
127
  made explicit (reject or clamp; an error or an empty result; where validation lives), and
110
128
  ranked by what real users can reach **today** rather than by how bad each one reads.
@@ -114,9 +132,16 @@ section is what gets the work scheduled.
114
132
 
115
133
  ## Step 4: Session Report
116
134
 
117
- Use `data/templates/session-report.md`. Lead with the matrix summary
118
- (`n PASS / n PARTIAL / n FAIL / n BLOCKED`), then the findings ranked by severity, then
119
- what you could not check and exactly what access would unblock it.
135
+ Write `session-report.md` in the format of
136
+ `${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md` with `Kind: backend`, and
137
+ insert `## AC Matrix Summary` right after the executive summary:
138
+ `n PASS / n PARTIAL / n FAIL / n BLOCKED / n NOT-REACHABLE / n UNVERIFIABLE` — all six
139
+ verdicts, every time, so that what could not be observed stays visible. Then the findings
140
+ ranked by severity, then what you could not check and exactly what access would unblock it.
141
+ The coverage map uses the contract's `| Area | Risk | Status | Bugs | Notes |` header with
142
+ `code-verified-only` for anything whose only evidence is a code reading. Write `stats.json`
143
+ with `"kind": "backend"` and the verdict counts under `coverage.verdicts`, then append the
144
+ rows to `output/sessions/INDEX.md` and `output/bugs/all-bugs.md` (columns in `paths.md`).
120
145
 
121
146
  Recommend a disposition in one line: ship, ship-with-follow-ups, or hold — and say what
122
147
  would change your mind. Where the scope of what you checked is narrower than the question
@@ -124,12 +149,15 @@ being asked, say so in the same breath; a recommendation with an unstated scope
124
149
  someone will over-read.
125
150
 
126
151
  For a release-level session covering many tickets against one build, follow
127
- `references/release-readiness.md` instead of this phase's per-AC shape: the deployment
152
+ `${CLAUDE_SKILL_DIR}/references/release-readiness.md` instead of this phase's per-AC shape: the deployment
128
153
  table first, then a fixed result vocabulary per ticket that keeps *not testable here* and
129
154
  *not tested* visible rather than letting them vanish between passed and failed.
130
155
 
131
156
  ## Step 5: Confidentiality
132
157
 
133
- Every output file gets the confidentiality header. Scan the whole session directory for
134
- tokens, keys, real user emails and consumer PII before finishing; replace with
135
- `[REDACTED]`. Session output stays local — never transmit it anywhere.
158
+ Every output file starts with the confidentiality header from `output-contract.md`. Scan the
159
+ whole session directory against the redaction list in `security-rules.md` (tokens, keys,
160
+ cookies, real user emails, consumer PII, account ids not already in the target) before
161
+ finishing; replace with `[REDACTED]`. Session output stays local; never transmit it anywhere.
162
+ If a browser session was opened for the API lane, end it now: `playwright-cli -s=<sid> close`
163
+ then `playwright-cli -s=<sid> delete-data`.