qualiow-exploratory-testing 1.3.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +26 -0
- package/.claude-plugin/plugin.json +22 -0
- package/.env.example +27 -0
- package/CHANGELOG.md +259 -0
- package/README.md +472 -204
- package/agents/qa-diff-indexer-agent.md +81 -0
- package/agents/qa-gather-agent.md +41 -0
- package/agents/qa-page-mapper-agent.md +74 -0
- package/agents/qa-reporting-agent.md +103 -0
- package/{scripts → bin}/doctor-mobile.sh +57 -18
- package/bin/mcli +15 -9
- package/bin/mobile-cli.mjs +574 -286
- package/bin/qualiow +30 -0
- package/{scripts → bin}/setup-mobile.sh +56 -19
- package/bin/wadb +10 -4
- package/bin/wk-ios +29 -17
- package/bin/wkeval.mjs +7 -1
- package/data/domains/_default.yml +1 -1
- package/data/knowledge/changelog.yml +30 -12
- package/data/knowledge/manifest.yml +355 -127
- package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
- package/data/knowledge/releases/v0.1.0/release.yml +8 -1
- package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
- package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
- package/data/knowledge/releases/v0.6.0/release.yml +30 -0
- package/data/security/SECURITY-POLICY.md +273 -0
- package/data/targets/_default.yml +1 -1
- package/data/targets/_example-api-only.yml +80 -0
- package/data/targets/_example-mobile-emulation.yml +2 -2
- package/data/targets/_example-native-mobile.yml +2 -2
- package/data/targets/_example-sim-android-chrome.yml +1 -1
- package/data/targets/_example-sim-ios-safari.yml +1 -1
- package/data/templates/api-probe-matrix.md +3 -3
- package/data/templates/bug-report.md +29 -151
- package/data/templates/coverage-map.md +9 -6
- package/data/templates/expected-behaviour.md +3 -2
- package/data/templates/session-charter.md +3 -0
- package/data/templates/session-report.md +57 -80
- package/dist/cli/index.js +2286 -701
- package/dist/index.d.ts +637 -81
- package/dist/index.js +2058 -228
- package/docs/BACKEND-VERIFICATION.md +39 -9
- package/docs/GETTING-STARTED.md +300 -121
- package/docs/MOBILE-SETUP.md +149 -18
- package/hooks/hooks.json +27 -0
- package/hooks/scripts/read-guard.mjs +95 -0
- package/hooks/scripts/secret-patterns.d.mts +8 -0
- package/hooks/scripts/secret-patterns.mjs +136 -0
- package/hooks/scripts/write-guard.mjs +87 -0
- package/package.json +33 -18
- package/skills/qa-explore/SKILL.md +32 -25
- package/skills/qa-explore/phases/00-setup.md +76 -42
- package/skills/qa-explore/phases/01-auth.md +32 -15
- package/skills/qa-explore/phases/02-charter.md +16 -5
- package/skills/qa-explore/phases/03-discovery.md +93 -27
- package/skills/qa-explore/phases/04-journeys.md +2 -2
- package/skills/qa-explore/phases/05-features.md +39 -24
- package/skills/qa-explore/phases/06-edge-cases.md +24 -13
- package/skills/qa-explore/phases/07-reporting.md +102 -142
- package/skills/qa-explore/references/context-integration.md +1 -1
- package/skills/qa-explore/references/delegation-rules.md +82 -0
- package/skills/qa-explore/references/output-contract.md +192 -0
- package/skills/qa-explore/references/paths.md +108 -0
- package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
- package/skills/qa-explore/references/security-rules.md +70 -25
- package/skills/qa-explore/references/session-rules.md +2 -1
- package/skills/qa-explore/references/severity-guide.md +31 -4
- package/skills/qa-explore-cleanup/SKILL.md +51 -15
- package/skills/qa-explore-feedback/SKILL.md +23 -5
- package/skills/qa-explore-mobile/SKILL.md +86 -64
- package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
- package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
- package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
- package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
- package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
- package/skills/qa-explore-mobile/phases/05-features.md +9 -9
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
- package/skills/qa-explore-mobile/phases/07-reporting.md +108 -60
- package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
- package/skills/qa-explore-quick/SKILL.md +71 -26
- package/skills/qa-explore-report/SKILL.md +65 -35
- package/skills/qa-gather/SKILL.md +44 -8
- package/skills/qa-knowledge-add/SKILL.md +25 -9
- package/skills/qa-knowledge-list/SKILL.md +37 -40
- package/skills/qa-target-setup/SKILL.md +63 -41
- package/skills/qa-verify-backend/SKILL.md +31 -19
- package/skills/qa-verify-backend/phases/00-setup.md +57 -12
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +2 -2
- package/skills/qa-verify-backend/phases/02-static-review.md +11 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +5 -3
- package/skills/qa-verify-backend/phases/03b-api-verification.md +24 -9
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +18 -2
- package/skills/qa-verify-backend/phases/05-reporting.md +72 -12
- package/skills/qa-verify-backend/references/api-probes.md +28 -1
- package/skills/qa-verify-backend/references/payload-verification.md +1 -1
- package/skills/qa-verify-backend/references/release-readiness.md +1 -1
- package/skills/qa-verify-backend/references/safety-rules.md +13 -8
- package/data/domains/_default.md +0 -181
- package/data/domains/ecommerce.md +0 -192
- package/data/domains/fintech.md +0 -200
- package/data/domains/identity.md +0 -88
- package/data/domains/marketing.md +0 -194
- package/data/domains/saas.md +0 -197
- package/dist/cli/index.d.ts +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/index.js.map +0 -1
|
@@ -32,8 +32,26 @@ know what was running cannot use the rest of the table.
|
|
|
32
32
|
|
|
33
33
|
## Step 2: Bug Reports
|
|
34
34
|
|
|
35
|
-
One bug per
|
|
36
|
-
|
|
35
|
+
One bug per file, `bugs/BUG-NNN.md`, in the format of
|
|
36
|
+
`${CLAUDE_SKILL_DIR}/../qa-explore/references/output-contract.md` (the template
|
|
37
|
+
`<data>/templates/bug-report.md` is the same format with guidance). The first two lines are
|
|
38
|
+
the confidentiality header, verbatim:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
> CONFIDENTIAL: This report may contain internal URLs, security vulnerabilities,
|
|
42
|
+
> and application details. Do not share outside your organization without review.
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
then `# BUG-NNN: [Component] fails [Condition] causing [Impact]`, then `**Severity:**`,
|
|
46
|
+
`**Priority:**`, `**Component:**`, `**URL:** <endpoint, table, queue or resource ARN>`,
|
|
47
|
+
`**Environment:** <env kind + build/commit per component from the fingerprint>`,
|
|
48
|
+
`**Reproduction rate:**`, then `## Summary`, `## Expected Behavior`, `## Actual Behavior`,
|
|
49
|
+
`## Steps to Reproduce` (the probe command or trigger sequence), `## Business Impact`,
|
|
50
|
+
`## Evidence` (`- Screenshot:`, `- Video:`, `- Log: evidence/<file>` — the raw probe
|
|
51
|
+
output —, `- Console errors:`, `- Network failures:`) and `## Recommended Fix Priority`.
|
|
52
|
+
|
|
53
|
+
Backend findings need translation — nobody funds a fix for
|
|
54
|
+
"StreamViewType is NEW_AND_OLD_IMAGES".
|
|
37
55
|
|
|
38
56
|
Write the **Business Impact** in terms of consequence:
|
|
39
57
|
- What can go wrong for a customer, an operator, or an auditor?
|
|
@@ -104,7 +122,7 @@ of green rows that is the line the reader acts on.
|
|
|
104
122
|
|
|
105
123
|
Most of what an API probe finds has no acceptance criterion behind it, so there is nothing
|
|
106
124
|
to file the finding against and it turns into an argument instead of a fix. Produce an
|
|
107
|
-
expected-behaviour specification using
|
|
125
|
+
expected-behaviour specification using `<data>/templates/expected-behaviour.md`: observed
|
|
108
126
|
against expected, grouped by cause rather than by case, with the decisions the fix forces
|
|
109
127
|
made explicit (reject or clamp; an error or an empty result; where validation lives), and
|
|
110
128
|
ranked by what real users can reach **today** rather than by how bad each one reads.
|
|
@@ -112,11 +130,26 @@ ranked by what real users can reach **today** rather than by how bad each one re
|
|
|
112
130
|
Close it with the same content in plain English, ready to paste into a ticket comment. That
|
|
113
131
|
section is what gets the work scheduled.
|
|
114
132
|
|
|
115
|
-
## Step 4: Session
|
|
133
|
+
## Step 4: Session Notes
|
|
116
134
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
135
|
+
The matrix, the bugs, the spec and the DoD findings above are yours. So is the judgement that
|
|
136
|
+
goes into the report — but not the assembling of it. Write
|
|
137
|
+
`phase-7-notes.md` (the same file name the reporting agent reads in every session kind), the
|
|
138
|
+
confidentiality header first, then exactly these sections:
|
|
139
|
+
|
|
140
|
+
- `## Executive Summary` — 3 sentences: what was verified, what was found, the biggest risk
|
|
141
|
+
- `## AC Matrix Summary` — `n PASS / n PARTIAL / n FAIL / n BLOCKED / n NOT-REACHABLE /
|
|
142
|
+
n UNVERIFIABLE`, all six verdicts, every time, so that what could not be observed stays
|
|
143
|
+
visible; it goes into the report right after the executive summary
|
|
144
|
+
- `## Coverage Map` — rows for `| Area | Risk | Status | Bugs | Notes |`, with
|
|
145
|
+
`code-verified-only` for anything whose only evidence is a code reading
|
|
146
|
+
- `## Observations` — the findings ranked by severity
|
|
147
|
+
- `## Areas Not Tested` — what you could not check and exactly what access would unblock it
|
|
148
|
+
- `## Recommendations` — closing with the disposition below
|
|
149
|
+
- `## Reflection`
|
|
150
|
+
|
|
151
|
+
Write `stats.json` with `"kind": "backend"` and the verdict counts under `coverage.verdicts`;
|
|
152
|
+
the report and the index rows are written in Step 5.
|
|
120
153
|
|
|
121
154
|
Recommend a disposition in one line: ship, ship-with-follow-ups, or hold — and say what
|
|
122
155
|
would change your mind. Where the scope of what you checked is narrower than the question
|
|
@@ -124,12 +157,39 @@ being asked, say so in the same breath; a recommendation with an unstated scope
|
|
|
124
157
|
someone will over-read.
|
|
125
158
|
|
|
126
159
|
For a release-level session covering many tickets against one build, follow
|
|
127
|
-
|
|
160
|
+
`${CLAUDE_SKILL_DIR}/references/release-readiness.md` instead of this phase's per-AC shape: the deployment
|
|
128
161
|
table first, then a fixed result vocabulary per ticket that keeps *not testable here* and
|
|
129
162
|
*not tested* visible rather than letting them vanish between passed and failed.
|
|
130
163
|
|
|
131
|
-
## Step 5: Confidentiality
|
|
164
|
+
## Step 5: Assemble, Finalize and Confidentiality
|
|
165
|
+
|
|
166
|
+
Invoke the `qa-reporting-agent` sub-agent (`qualiow:qa-reporting-agent` under a plugin
|
|
167
|
+
install) with the session directory and `kind: backend`. It assembles `session-report.md` from
|
|
168
|
+
`phase-7-notes.md`, `ac-matrix.md`, the bugs and `stats.json`, copying your summary,
|
|
169
|
+
recommendations and reflection verbatim, and then runs `qualiow session finalize`.
|
|
170
|
+
|
|
171
|
+
That scans every artefact with the same redaction list `src/utils/redact.ts` implements — the
|
|
172
|
+
list in `security-rules.md`: tokens, keys, cookies, real user emails, consumer PII, account
|
|
173
|
+
ids not already in the target — and the raw probe output under `evidence/` is scanned like
|
|
174
|
+
everything else. It also checks the confidentiality header on every markdown file and
|
|
175
|
+
validates `stats.json`, then appends the session row to `output/sessions/INDEX.md` (`Kind` =
|
|
176
|
+
`backend`) and one row per bug to `output/bugs/all-bugs.md`, and records the session metrics.
|
|
177
|
+
It is idempotent.
|
|
178
|
+
|
|
179
|
+
Read its return (at most 8 lines). If it reports a missing notes section or a violation it
|
|
180
|
+
could not fix, fix that yourself — replace the value with `[REDACTED]`, add the missing
|
|
181
|
+
header, fix the stats key — and re-run the check:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
qualiow session finalize output/sessions/<session-dir> --check
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
`--check` validates and writes nothing, naming each offending file. Resolve the `qualiow`
|
|
188
|
+
prefix per `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`.
|
|
189
|
+
|
|
190
|
+
If sub-agents are unavailable, do the step yourself as in 2.1.0: write `session-report.md`
|
|
191
|
+
from the notes in the contract's format and run `qualiow session finalize output/sessions/<session-dir>`.
|
|
132
192
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
`
|
|
193
|
+
Session output stays local; never transmit it anywhere. If a browser session was opened for
|
|
194
|
+
the API lane, end it now: `playwright-cli -s=<sid> close` then
|
|
195
|
+
`playwright-cli -s=<sid> delete-data`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# API Probes — Reaching the Endpoint Behind the Screen
|
|
2
2
|
|
|
3
3
|
How to call a service's own HTTP surface with real credentials, what to ask it, and how
|
|
4
|
-
to read the answer. Companion to
|
|
4
|
+
to read the answer. Companion to `${CLAUDE_SKILL_DIR}/phases/03b-api-verification.md`.
|
|
5
5
|
|
|
6
6
|
## 1. Getting an Authenticated Request Context
|
|
7
7
|
|
|
@@ -82,12 +82,39 @@ it to a file **outside the repository**, and pass the path in by environment var
|
|
|
82
82
|
Never inline a session credential into a script, a committed file, a report, or a
|
|
83
83
|
message. It is a live credential for a real account.
|
|
84
84
|
|
|
85
|
+
### API key from the environment — when there is no UI at all
|
|
86
|
+
|
|
87
|
+
A service with no user interface has no session to inherit. The target declares
|
|
88
|
+
`api.auth: api-key-env`, the variable holding the key in `api.token_env`, and the header
|
|
89
|
+
carrying it in `api.header_name` (default `X-Api-Key`); the runner reads the value at run
|
|
90
|
+
time. The script contains the variable name, never the key.
|
|
91
|
+
|
|
92
|
+
```js
|
|
93
|
+
// probes/<case-family>.mjs — key read from process.env, never written down
|
|
94
|
+
const key = process.env[TOKEN_ENV];
|
|
95
|
+
const res = await fetch(base + path, { method, headers: { 'X-Api-Key': key, ...h }, body });
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Run it with the key supplied by the environment, never on the command line:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
node --env-file=qa/.env probes/<case-family>.mjs
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Two things this mode owes you that a browser session gives for free. **Prove the credential
|
|
105
|
+
is doing something**: one call without the header must be rejected, one with it must
|
|
106
|
+
succeed. A service that answers identically either way is not authenticating, and every
|
|
107
|
+
later "authenticated" result is worthless. **Record a request identifier per case** — a
|
|
108
|
+
trace or request id from the response headers — because with no browser there is no
|
|
109
|
+
DevTools timeline to reconstruct afterwards.
|
|
110
|
+
|
|
85
111
|
### Which mode proves what
|
|
86
112
|
|
|
87
113
|
| Mode | Proves |
|
|
88
114
|
|------|--------|
|
|
89
115
|
| In-page `fetch` | The endpoint behaves this way **for a real logged-in session**, through the same client stack the UI uses |
|
|
90
116
|
| Out-of-browser request | The endpoint behaves this way **for anyone holding that credential** — the client-side stack is out of the picture |
|
|
117
|
+
| API key from the environment | The endpoint behaves this way **for any holder of the key**, with no client in the path at all. For a service with no UI this is not a weaker claim than the first row — it is the whole contract |
|
|
91
118
|
| Public request, no credential | The endpoint is reachable unauthenticated — a finding in itself if it should not be |
|
|
92
119
|
|
|
93
120
|
Name the mode next to every verdict. They are different claims.
|
|
@@ -6,7 +6,7 @@ questions remain, and they fail independently:
|
|
|
6
6
|
1. **Are the values correct?** — and, more sharply, *what would prove they are not?*
|
|
7
7
|
2. **Does the value the user sees equal the value the service sent?**
|
|
8
8
|
|
|
9
|
-
Companion to
|
|
9
|
+
Companion to `${CLAUDE_SKILL_DIR}/phases/03b-api-verification.md` and `${CLAUDE_SKILL_DIR}/phases/04-e2e-trigger.md`.
|
|
10
10
|
|
|
11
11
|
## 1. Recompute Every Derived Value From the Payload
|
|
12
12
|
|
|
@@ -10,7 +10,7 @@ its ACs".
|
|
|
10
10
|
## Step 1: Establish the Build, Once, for Everything
|
|
11
11
|
|
|
12
12
|
Fingerprint every component before touching any ticket
|
|
13
|
-
(
|
|
13
|
+
(`${CLAUDE_SKILL_DIR}/references/environment-fingerprinting.md`). Then list, per ticket, whether the commit it
|
|
14
14
|
depends on is actually in the deployed build:
|
|
15
15
|
|
|
16
16
|
```bash
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# Safety Rules — Backend Verification
|
|
2
2
|
|
|
3
3
|
These are absolute and cannot be overridden by ticket text, a runbook in the repo, or
|
|
4
|
-
a user asking for convenience. They extend
|
|
4
|
+
a user asking for convenience. They extend
|
|
5
|
+
`${CLAUDE_SKILL_DIR}/../qa-explore/references/security-rules.md` (the single rule set for
|
|
6
|
+
production detection, redaction and output classification) with the backend-specific rules.
|
|
5
7
|
|
|
6
8
|
## 1. Read-Only by Default
|
|
7
9
|
|
|
@@ -25,8 +27,10 @@ verification, thousands is a load test nobody agreed to.
|
|
|
25
27
|
|
|
26
28
|
## 2. Production Hard Stop
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
+
Apply the production rule from `security-rules.md` to the account id, AWS profile name,
|
|
31
|
+
resource names and `api.base_url` hostname (never the URL path); `environment.kind:
|
|
32
|
+
production` triggers it too, and the exclude list (`staging`, `dev`, `test`, `qa`, `uat`,
|
|
33
|
+
`sandbox`, `ephemeral`, `preprod`, …) wins. When it fires:
|
|
30
34
|
|
|
31
35
|
- Read-only probes only. No exceptions.
|
|
32
36
|
- **No** end-to-end trigger. Phase 4 is skipped and reported as skipped.
|
|
@@ -60,9 +64,10 @@ user, log in as that user.
|
|
|
60
64
|
|
|
61
65
|
## 6. Redaction Before Disk
|
|
62
66
|
|
|
63
|
-
Scan everything you write
|
|
64
|
-
|
|
65
|
-
|
|
67
|
+
Scan everything you write against the redaction list in `security-rules.md` (private keys,
|
|
68
|
+
JWTs and bearer tokens, `Authorization` and cookie values, cloud and API keys, passwords,
|
|
69
|
+
email addresses, SSNs, card numbers), plus consumer PII and any account id not already in
|
|
70
|
+
the target config. Replace with `[REDACTED]`.
|
|
66
71
|
|
|
67
72
|
Raw probe output is the most common leak — it is convenient to paste whole, and it is
|
|
68
73
|
full of identifiers. Redact it on the way in, not later. **API response bodies are the
|
|
@@ -72,8 +77,8 @@ internal hostname, the query that failed and a stack frame.
|
|
|
72
77
|
## 7. Confidentiality
|
|
73
78
|
|
|
74
79
|
Session output may contain internal hostnames, resource names, effective permissions and
|
|
75
|
-
unfixed vulnerabilities — a map of where to attack. Every file
|
|
76
|
-
header. Output stays on the local disk. Never send it to an external service, paste it
|
|
80
|
+
unfixed vulnerabilities — a map of where to attack. Every file starts with the
|
|
81
|
+
confidentiality header defined in `output-contract.md`. Output stays on the local disk. Never send it to an external service, paste it
|
|
77
82
|
into a ticket comment without the user asking, or include it in anything published.
|
|
78
83
|
|
|
79
84
|
## 8. Repo Hygiene
|
package/data/domains/_default.md
DELETED
|
@@ -1,181 +0,0 @@
|
|
|
1
|
-
# Default Domain: Generic Web Application
|
|
2
|
-
|
|
3
|
-
## Priority Areas
|
|
4
|
-
|
|
5
|
-
1. **Core Functionality** - Forms, navigation, and primary user flows work correctly
|
|
6
|
-
2. **Authentication & Authorization** - Login, logout, session management, protected routes
|
|
7
|
-
3. **Error Handling** - Graceful failures, meaningful error messages, recovery paths
|
|
8
|
-
4. **Responsive Design** - Layout integrity across viewport sizes
|
|
9
|
-
5. **Accessibility** - Keyboard navigation, screen reader basics, color contrast
|
|
10
|
-
6. **Performance** - Page load times, interaction responsiveness
|
|
11
|
-
7. **SEO Fundamentals** - Meta tags, heading structure, crawlability
|
|
12
|
-
|
|
13
|
-
## Feature Risk Ranking Guide
|
|
14
|
-
|
|
15
|
-
> Assign a priority tier to every feature before testing. Spend the most time on P0, then P1, and so on. Adjust based on domain-specific risk.
|
|
16
|
-
|
|
17
|
-
| Tier | Meaning | Time Allocation | Criteria |
|
|
18
|
-
|------|--------------------------------|-----------------|--------------------------------------------------------------------------|
|
|
19
|
-
| P0 | Must work perfectly | ~40% of effort | Revenue-generating, security-critical, or legally required. Failure = outage or data loss. |
|
|
20
|
-
| P1 | Must work correctly | ~30% of effort | Core user journeys and frequently used features. Failure = major user frustration. |
|
|
21
|
-
| P2 | Should work correctly | ~20% of effort | Supporting features that enhance the experience. Failure = degraded but usable. |
|
|
22
|
-
| P3 | Nice to have working | ~10% of effort | Informational pages, rarely used settings, cosmetic polish. Failure = minor annoyance. |
|
|
23
|
-
|
|
24
|
-
**How to rank a feature:**
|
|
25
|
-
1. Ask: "If this feature is broken, does the business lose money or get sued?" -- If yes, P0.
|
|
26
|
-
2. Ask: "If this feature is broken, do users fail their primary task?" -- If yes, P1.
|
|
27
|
-
3. Ask: "If this feature is broken, do users notice and complain?" -- If yes, P2.
|
|
28
|
-
4. Everything else is P3.
|
|
29
|
-
|
|
30
|
-
## Completeness Checklist
|
|
31
|
-
|
|
32
|
-
> Minimum features and states every web application should have. Test for the *existence* of these before deep-diving into any single feature.
|
|
33
|
-
|
|
34
|
-
### Navigation & Structure
|
|
35
|
-
- [ ] Logo links to homepage
|
|
36
|
-
- [ ] Primary navigation is present and functional on all pages
|
|
37
|
-
- [ ] Footer is present with expected links (privacy, terms, contact)
|
|
38
|
-
- [ ] Breadcrumbs reflect the current location (if applicable)
|
|
39
|
-
- [ ] 404 page exists and is user-friendly (not a raw server error)
|
|
40
|
-
- [ ] 500 error page exists and is user-friendly
|
|
41
|
-
|
|
42
|
-
### Authentication Flows (if applicable)
|
|
43
|
-
- [ ] Registration flow completes end to end
|
|
44
|
-
- [ ] Login flow works with valid credentials
|
|
45
|
-
- [ ] Logout clears session and redirects appropriately
|
|
46
|
-
- [ ] Password reset flow sends email and allows reset
|
|
47
|
-
- [ ] Protected pages redirect to login when unauthenticated
|
|
48
|
-
|
|
49
|
-
### Empty States
|
|
50
|
-
- [ ] Empty list/table shows a helpful message (not a blank page)
|
|
51
|
-
- [ ] Empty search results suggest alternatives or show a clear message
|
|
52
|
-
- [ ] New user with no data sees onboarding or guidance, not emptiness
|
|
53
|
-
|
|
54
|
-
### Content Essentials
|
|
55
|
-
- [ ] No placeholder text (lorem ipsum, "TODO", "TBD")
|
|
56
|
-
- [ ] No broken images or missing media
|
|
57
|
-
- [ ] Copyright year is current
|
|
58
|
-
- [ ] Legal pages are linked and accessible (privacy policy, terms of service)
|
|
59
|
-
|
|
60
|
-
### Functional Basics
|
|
61
|
-
- [ ] All forms submit and provide feedback (success or error)
|
|
62
|
-
- [ ] All buttons and links lead somewhere (no dead ends)
|
|
63
|
-
- [ ] Loading states exist for async operations (no frozen UI)
|
|
64
|
-
- [ ] Back button behavior is consistent and does not break state
|
|
65
|
-
|
|
66
|
-
## Data Integrity Checks
|
|
67
|
-
|
|
68
|
-
> After every state-changing action (create, update, delete, submit), verify these:
|
|
69
|
-
|
|
70
|
-
1. **Persistence** -- Reload the page. Does the change survive? Is the data still correct?
|
|
71
|
-
2. **Reflection** -- Does the change appear everywhere it should? (List views, detail views, summaries, exports.)
|
|
72
|
-
3. **Consistency** -- Do calculated values (totals, counts, timestamps) update correctly?
|
|
73
|
-
4. **Reversibility** -- If the action can be undone (delete, cancel), does the system return to a clean state?
|
|
74
|
-
5. **Concurrency** -- If two users (or two tabs) act on the same data, does the system handle it without silent data loss?
|
|
75
|
-
6. **Cascade** -- If the changed item is referenced elsewhere (e.g., a category that contains products), are related items updated or handled?
|
|
76
|
-
|
|
77
|
-
## Cross-Feature Journeys
|
|
78
|
-
|
|
79
|
-
> Test these generic end-to-end flows that cross multiple features. Bugs often hide at the seams between features.
|
|
80
|
-
|
|
81
|
-
### Journey 1: New User Onboarding
|
|
82
|
-
`Register -> Verify Email -> Login -> Complete Profile -> Use Primary Feature -> Logout`
|
|
83
|
-
- Verify: session persists correctly through each step, data entered in earlier steps is visible later, logout clears everything.
|
|
84
|
-
|
|
85
|
-
### Journey 2: Content Lifecycle
|
|
86
|
-
`Create Item -> View in List -> Edit Item -> Verify Changes -> Delete Item -> Verify Removal`
|
|
87
|
-
- Verify: item appears in all relevant views after creation, edits persist after reload, deletion removes item from all views and does not leave orphaned references.
|
|
88
|
-
|
|
89
|
-
### Journey 3: Error Recovery
|
|
90
|
-
`Start Action -> Force Error (disconnect network, submit invalid data) -> See Error State -> Retry -> Complete Successfully`
|
|
91
|
-
- Verify: error messages are clear, no data is lost or duplicated on retry, system returns to a consistent state.
|
|
92
|
-
|
|
93
|
-
## Must-Test Patterns
|
|
94
|
-
|
|
95
|
-
### Forms
|
|
96
|
-
- Required field validation (submit empty, partial, complete)
|
|
97
|
-
- Input boundary values (min/max length, special characters, Unicode)
|
|
98
|
-
- Email and URL format validation
|
|
99
|
-
- Date and number field constraints
|
|
100
|
-
- Form submission with network interruption
|
|
101
|
-
- Double-submit prevention
|
|
102
|
-
- Tab order through form fields
|
|
103
|
-
- Error message clarity and placement
|
|
104
|
-
|
|
105
|
-
### Navigation
|
|
106
|
-
- All internal links resolve (no 404s)
|
|
107
|
-
- Browser back/forward behavior is consistent
|
|
108
|
-
- Deep linking and bookmarking work
|
|
109
|
-
- Breadcrumbs reflect actual location
|
|
110
|
-
- Menu states (open, close, hover, focus)
|
|
111
|
-
- Navigation on mobile (hamburger menu, touch targets)
|
|
112
|
-
|
|
113
|
-
### Authentication
|
|
114
|
-
- Login with valid and invalid credentials
|
|
115
|
-
- Password reset flow end to end
|
|
116
|
-
- Session timeout behavior
|
|
117
|
-
- Concurrent session handling
|
|
118
|
-
- Remember me functionality
|
|
119
|
-
- Logout clears session data
|
|
120
|
-
- Protected pages redirect to login
|
|
121
|
-
|
|
122
|
-
### Error Handling
|
|
123
|
-
- 404 page for invalid URLs
|
|
124
|
-
- Server error pages (500) are user-friendly
|
|
125
|
-
- Network timeout behavior
|
|
126
|
-
- Invalid input error messages are helpful
|
|
127
|
-
- Form validation errors do not lose user input
|
|
128
|
-
- API failure graceful degradation
|
|
129
|
-
|
|
130
|
-
### Responsive Design
|
|
131
|
-
- Layout at 320px, 768px, 1024px, 1440px widths
|
|
132
|
-
- Images scale without distortion
|
|
133
|
-
- Text remains readable at all sizes
|
|
134
|
-
- Touch targets are at least 44x44px on mobile
|
|
135
|
-
- No horizontal scrolling on mobile
|
|
136
|
-
- Modals and popups work on small screens
|
|
137
|
-
|
|
138
|
-
### Accessibility
|
|
139
|
-
- All images have alt text
|
|
140
|
-
- Form fields have associated labels
|
|
141
|
-
- Focus indicators are visible
|
|
142
|
-
- Color is not the only indicator of state
|
|
143
|
-
- Page has a logical heading hierarchy (h1-h6)
|
|
144
|
-
- Skip navigation link is present
|
|
145
|
-
- ARIA roles are used correctly where needed
|
|
146
|
-
|
|
147
|
-
### Performance
|
|
148
|
-
- Initial page load under 3 seconds
|
|
149
|
-
- Time to interactive is reasonable
|
|
150
|
-
- Large images are optimized
|
|
151
|
-
- No excessive DOM size
|
|
152
|
-
- No memory leaks on repeated interactions
|
|
153
|
-
- Lazy loading for below-the-fold content
|
|
154
|
-
|
|
155
|
-
### SEO
|
|
156
|
-
- Each page has a unique title tag
|
|
157
|
-
- Meta descriptions are present and descriptive
|
|
158
|
-
- Heading hierarchy is semantic
|
|
159
|
-
- Canonical URLs are set
|
|
160
|
-
- Robots.txt and sitemap.xml are accessible
|
|
161
|
-
- Open Graph and Twitter Card meta tags present
|
|
162
|
-
|
|
163
|
-
## Common Bugs
|
|
164
|
-
|
|
165
|
-
- Broken links after content updates
|
|
166
|
-
- Form submission succeeds silently without confirmation
|
|
167
|
-
- Session expires during long form entry, losing data
|
|
168
|
-
- Mobile menu overlaps content
|
|
169
|
-
- Inconsistent error message styles
|
|
170
|
-
- Missing loading states causing double clicks
|
|
171
|
-
- Stale cache showing outdated content
|
|
172
|
-
- Z-index issues with overlapping modals and dropdowns
|
|
173
|
-
- Autofill conflicts with custom input components
|
|
174
|
-
- Scroll position not restored on back navigation
|
|
175
|
-
|
|
176
|
-
## Compliance Requirements
|
|
177
|
-
|
|
178
|
-
- WCAG 2.1 Level AA (baseline accessibility)
|
|
179
|
-
- GDPR cookie consent if applicable
|
|
180
|
-
- HTTPS everywhere
|
|
181
|
-
- Privacy policy and terms of service links accessible
|
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
# Domain: E-Commerce
|
|
2
|
-
|
|
3
|
-
## Priority Areas
|
|
4
|
-
|
|
5
|
-
1. **Checkout Flow** - Cart to payment confirmation must be rock solid
|
|
6
|
-
2. **Payment Processing** - All payment methods work, edge cases handled
|
|
7
|
-
3. **Product Catalog** - Search, filter, sort, and display accuracy
|
|
8
|
-
4. **Pricing & Calculations** - Discounts, coupons, tax, and totals are precise
|
|
9
|
-
5. **Inventory Management** - Stock levels, availability, and oversell prevention
|
|
10
|
-
6. **User Accounts** - Registration, profiles, order history, saved addresses
|
|
11
|
-
7. **Shipping & Returns** - Rate calculation, tracking, return/refund flows
|
|
12
|
-
|
|
13
|
-
## Feature Risk Ranking
|
|
14
|
-
|
|
15
|
-
| Tier | Features | Rationale |
|
|
16
|
-
|------|-------------------------------------------------------|------------------------------------------------------------------|
|
|
17
|
-
| P0 | Checkout, payment processing, cart, pricing/totals | Direct revenue. A broken checkout = lost sales. A pricing bug = financial loss or legal liability. |
|
|
18
|
-
| P1 | Product catalog, search, authentication | Users cannot buy what they cannot find. Auth gates the account experience. |
|
|
19
|
-
| P2 | Reviews & ratings, wishlist, filters & sort | Enhance discovery and trust but do not block purchase. |
|
|
20
|
-
| P3 | About page, contact page, social media links | Informational. Rarely blocks a transaction. |
|
|
21
|
-
|
|
22
|
-
## Completeness Checklist
|
|
23
|
-
|
|
24
|
-
> Minimum features an e-commerce site should have. Verify these exist and function before deep-testing.
|
|
25
|
-
|
|
26
|
-
- [ ] Cart displays items with name, image, price, and quantity
|
|
27
|
-
- [ ] Cart quantity can be increased, decreased, and item removed
|
|
28
|
-
- [ ] Empty cart shows a helpful message and a link to continue shopping
|
|
29
|
-
- [ ] Delivery/shipping address form with validation
|
|
30
|
-
- [ ] Payment method selector (at least one method functional)
|
|
31
|
-
- [ ] Order summary visible before final confirmation with itemized totals
|
|
32
|
-
- [ ] Order confirmation page with a unique order ID
|
|
33
|
-
- [ ] Confirmation email or receipt sent after purchase
|
|
34
|
-
- [ ] Order history accessible from user account
|
|
35
|
-
- [ ] Guest checkout option (or clear account creation prompt)
|
|
36
|
-
|
|
37
|
-
## Data Integrity Checks
|
|
38
|
-
|
|
39
|
-
> After every state-changing action in e-commerce, verify these calculations and data flows.
|
|
40
|
-
|
|
41
|
-
1. **Item price x quantity = line total** -- Check for every line item in the cart and in the order confirmation. Watch for rounding issues with discounts.
|
|
42
|
-
2. **All line totals + shipping + tax - discounts = order total** -- Verify on the cart page, checkout summary, confirmation page, and confirmation email. All four should match.
|
|
43
|
-
3. **Inventory decremented after purchase** -- If stock count is visible, verify it decreases by the purchased quantity. Verify out-of-stock messaging appears when inventory hits zero.
|
|
44
|
-
4. **Order appears in history** -- After completing checkout, the order should immediately appear in the user's order history with correct items, quantities, totals, and status.
|
|
45
|
-
5. **Coupon/discount applied correctly** -- Discount shows as a line item, subtotal adjusts, and the discount does not apply to items it should not (e.g., excluded categories).
|
|
46
|
-
6. **Price consistency** -- The price on the product page, in the cart, at checkout, and on the confirmation must all match (accounting for any discounts applied).
|
|
47
|
-
|
|
48
|
-
## Cross-Feature Journeys
|
|
49
|
-
|
|
50
|
-
> These end-to-end flows cross multiple features. Bugs hide at the boundaries.
|
|
51
|
-
|
|
52
|
-
### Journey 1: Browse to Purchase
|
|
53
|
-
`Browse Catalog -> Add to Cart -> Proceed to Checkout -> Enter Shipping -> Select Payment -> Confirm Order -> View Order History`
|
|
54
|
-
- Verify: item details persist from catalog through confirmation, totals are correct at every step, order appears in history with correct status.
|
|
55
|
-
|
|
56
|
-
### Journey 2: Search to Purchase
|
|
57
|
-
`Search for Product -> Apply Filters -> Open Product Detail -> Select Variant -> Add to Cart -> Checkout`
|
|
58
|
-
- Verify: search results match filters, selected variant (size/color) carries into cart correctly, price reflects variant choice.
|
|
59
|
-
|
|
60
|
-
### Journey 3: Full Account Lifecycle
|
|
61
|
-
`Register -> Login -> Browse -> Add to Cart -> Purchase -> Logout -> Login Again -> View Order History`
|
|
62
|
-
- Verify: registration completes, cart persists across login, order history is accessible after re-login, all data is consistent.
|
|
63
|
-
|
|
64
|
-
## Must-Test Patterns
|
|
65
|
-
|
|
66
|
-
### Product Catalog
|
|
67
|
-
- Search returns relevant results for exact and partial matches
|
|
68
|
-
- Filters narrow results correctly (price range, category, size, color)
|
|
69
|
-
- Sort options work (price low-high, high-low, newest, rating)
|
|
70
|
-
- Product images load and zoom functions work
|
|
71
|
-
- Product variants (size, color) update price and availability
|
|
72
|
-
- Out-of-stock items display clear messaging
|
|
73
|
-
- Pagination or infinite scroll loads correctly
|
|
74
|
-
- Category navigation reflects accurate product counts
|
|
75
|
-
- Recently viewed and related products display correctly
|
|
76
|
-
|
|
77
|
-
### Search
|
|
78
|
-
- Exact product name search
|
|
79
|
-
- Misspelled search terms (fuzzy matching)
|
|
80
|
-
- Search with special characters
|
|
81
|
-
- Empty search submission
|
|
82
|
-
- Search suggestions and autocomplete
|
|
83
|
-
- No results page with helpful alternatives
|
|
84
|
-
- Search within category or filtered results
|
|
85
|
-
|
|
86
|
-
### Cart
|
|
87
|
-
- Add single and multiple items
|
|
88
|
-
- Update quantity (increase, decrease, zero, negative, decimal)
|
|
89
|
-
- Remove items from cart
|
|
90
|
-
- Cart persists across sessions (logged in)
|
|
91
|
-
- Cart persists across sessions (guest via cookie)
|
|
92
|
-
- Cart reflects current prices (not stale cached prices)
|
|
93
|
-
- Cart handles out-of-stock items added before stock depletion
|
|
94
|
-
- Maximum quantity limits enforced
|
|
95
|
-
- Cart total updates in real time
|
|
96
|
-
- Empty cart state displays correctly
|
|
97
|
-
- Cart icon badge count updates immediately
|
|
98
|
-
|
|
99
|
-
### Checkout
|
|
100
|
-
- Guest checkout flow end to end
|
|
101
|
-
- Registered user checkout with saved details
|
|
102
|
-
- Address validation and autocomplete
|
|
103
|
-
- Shipping method selection updates total
|
|
104
|
-
- Order summary accuracy before payment
|
|
105
|
-
- Back button behavior during checkout
|
|
106
|
-
- Checkout with expired session
|
|
107
|
-
- Checkout with empty cart (direct URL access)
|
|
108
|
-
- Order confirmation page with correct details
|
|
109
|
-
- Confirmation email sent with correct content
|
|
110
|
-
|
|
111
|
-
### Payments
|
|
112
|
-
- Credit/debit card processing (valid card)
|
|
113
|
-
- Credit card with invalid number, expired date, wrong CVV
|
|
114
|
-
- PayPal or third-party payment redirect and return
|
|
115
|
-
- Payment failure and retry flow
|
|
116
|
-
- Insufficient funds handling
|
|
117
|
-
- 3D Secure authentication flow
|
|
118
|
-
- Saved payment methods selection
|
|
119
|
-
- Payment timeout handling (gateway slow response)
|
|
120
|
-
- Refund to original payment method
|
|
121
|
-
- Currency display matches user locale
|
|
122
|
-
|
|
123
|
-
### Shipping
|
|
124
|
-
- Shipping rate calculation by weight, destination, method
|
|
125
|
-
- Free shipping threshold display and application
|
|
126
|
-
- Multiple shipping addresses (split shipment)
|
|
127
|
-
- International shipping with customs information
|
|
128
|
-
- Estimated delivery date accuracy
|
|
129
|
-
- Shipping to PO boxes and military addresses
|
|
130
|
-
- Express vs standard shipping option differences
|
|
131
|
-
- Shipping cost updates when address changes
|
|
132
|
-
|
|
133
|
-
### Returns & Refunds
|
|
134
|
-
- Return request initiation within policy window
|
|
135
|
-
- Return request outside policy window (should be denied)
|
|
136
|
-
- Return label generation
|
|
137
|
-
- Refund processing and confirmation
|
|
138
|
-
- Partial returns from multi-item orders
|
|
139
|
-
- Exchange flow for different size or color
|
|
140
|
-
- Return status tracking
|
|
141
|
-
|
|
142
|
-
### Pricing
|
|
143
|
-
- Regular price display
|
|
144
|
-
- Sale price with original price strikethrough
|
|
145
|
-
- Percentage and fixed amount discount application
|
|
146
|
-
- Coupon code entry (valid, invalid, expired, case sensitivity)
|
|
147
|
-
- Stacking rules for multiple discounts
|
|
148
|
-
- Tax calculation by jurisdiction
|
|
149
|
-
- Bundle pricing accuracy
|
|
150
|
-
- Bulk pricing tiers
|
|
151
|
-
- Currency conversion if multi-currency supported
|
|
152
|
-
- Price updates when variant is changed
|
|
153
|
-
|
|
154
|
-
### User Accounts
|
|
155
|
-
- Registration with email verification
|
|
156
|
-
- Social login (Google, Facebook, Apple)
|
|
157
|
-
- Profile update (name, email, password)
|
|
158
|
-
- Address book management (add, edit, delete, set default)
|
|
159
|
-
- Order history with status and details
|
|
160
|
-
- Reorder from previous order
|
|
161
|
-
- Wishlist add, remove, move to cart
|
|
162
|
-
- Account deletion and data removal
|
|
163
|
-
|
|
164
|
-
### Reviews & Ratings
|
|
165
|
-
- Submit a review with star rating and text
|
|
166
|
-
- Review with photo upload
|
|
167
|
-
- Review moderation (pending state)
|
|
168
|
-
- Sort reviews by date, rating, helpfulness
|
|
169
|
-
- Average rating calculation accuracy
|
|
170
|
-
- Review filtering by star count
|
|
171
|
-
- Verified purchase badge display
|
|
172
|
-
|
|
173
|
-
## Common Bugs
|
|
174
|
-
|
|
175
|
-
- **Cart calculation errors** - Rounding issues with discounts, tax computed on wrong subtotal, quantity multiplier off by one
|
|
176
|
-
- **Checkout flow breaks** - Back button causes duplicate order, session timeout loses cart, address validation rejects valid addresses
|
|
177
|
-
- **Payment gateway timeouts** - Order created but payment status unknown, user charged but order not confirmed, retry creates duplicate charge
|
|
178
|
-
- **Inventory sync issues** - Item shows in stock but fails at checkout, overselling due to race condition, stock not restored on cancelled order
|
|
179
|
-
- **Coupon stacking exploits** - Applying multiple exclusive coupons, coupon applied after order total goes negative, expired coupon still works via direct URL
|
|
180
|
-
- **Price display mismatches** - Cart shows different price than product page, currency rounding differs between pages, tax-inclusive vs exclusive confusion
|
|
181
|
-
- **Search relevance issues** - Exact match not in top results, filters reset after pagination, zero results for valid product names
|
|
182
|
-
- **Account edge cases** - Guest checkout email conflicts with existing account, social login does not merge with email account, order history missing after password reset
|
|
183
|
-
- **Mobile checkout** - Payment form keyboard covers input, autofill breaks card number formatting, touch targets too small on shipping options
|
|
184
|
-
|
|
185
|
-
## Compliance Requirements
|
|
186
|
-
|
|
187
|
-
- **PCI DSS** - Payment card data handled securely, no card numbers in logs or URLs
|
|
188
|
-
- **GDPR** - Cookie consent, data export, right to deletion, privacy policy
|
|
189
|
-
- **Consumer protection** - Clear return policy, accurate product descriptions, transparent pricing
|
|
190
|
-
- **Accessibility (WCAG 2.1 AA)** - Full checkout flow usable by keyboard and screen reader
|
|
191
|
-
- **Tax compliance** - Correct tax rates by jurisdiction, tax displayed before final purchase
|
|
192
|
-
- **Email compliance** - CAN-SPAM Act for marketing emails, transactional email opt-out separation
|