qualiow-exploratory-testing 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +177 -0
  2. package/bin/mcli +16 -0
  3. package/bin/mobile-cli.mjs +573 -0
  4. package/bin/wadb +6 -0
  5. package/bin/wk-ios +58 -0
  6. package/bin/wkeval.mjs +44 -0
  7. package/data/domains/_default.md +181 -0
  8. package/data/domains/ecommerce.md +192 -0
  9. package/data/domains/fintech.md +200 -0
  10. package/data/domains/identity.md +88 -0
  11. package/data/domains/identity.yml +138 -0
  12. package/data/domains/marketing.md +194 -0
  13. package/data/domains/saas.md +197 -0
  14. package/data/knowledge/changelog.yml +86 -0
  15. package/data/knowledge/learned-patterns.md +172 -1
  16. package/data/knowledge/manifest.yml +210 -4
  17. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  18. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  19. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  20. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  21. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  22. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  23. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  24. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  25. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  26. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  28. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  29. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  30. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  31. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  34. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  35. package/data/targets/_example-backend.yml +123 -0
  36. package/data/targets/_example-mobile-emulation.yml +52 -0
  37. package/data/targets/_example-native-mobile.yml +61 -0
  38. package/data/targets/_example-sim-android-chrome.yml +81 -0
  39. package/data/targets/_example-sim-ios-safari.yml +80 -0
  40. package/data/templates/api-probe-matrix.md +99 -0
  41. package/data/templates/coverage-map.md +14 -0
  42. package/data/templates/expected-behaviour.md +93 -0
  43. package/dist/cli/index.js +137 -5
  44. package/dist/cli/index.js.map +1 -1
  45. package/dist/index.d.ts +350 -527
  46. package/dist/index.js +144 -5
  47. package/dist/index.js.map +1 -1
  48. package/docs/BACKEND-VERIFICATION.md +277 -0
  49. package/docs/MOBILE-SETUP.md +92 -0
  50. package/package.json +26 -12
  51. package/scripts/doctor-mobile.sh +143 -0
  52. package/scripts/setup-mobile.sh +213 -0
  53. package/skills/qa-explore/phases/00-setup.md +8 -1
  54. package/skills/qa-explore/phases/01-auth.md +4 -0
  55. package/skills/qa-explore-mobile/SKILL.md +146 -0
  56. package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
  57. package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
  58. package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
  59. package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
  60. package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
  61. package/skills/qa-explore-mobile/phases/05-features.md +72 -0
  62. package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
  63. package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
  64. package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
  65. package/skills/qa-explore-quick/SKILL.md +6 -0
  66. package/skills/qa-verify-backend/SKILL.md +163 -0
  67. package/skills/qa-verify-backend/phases/00-setup.md +97 -0
  68. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  69. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  70. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  71. package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
  72. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
  73. package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
  74. package/skills/qa-verify-backend/references/api-probes.md +141 -0
  75. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  76. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  77. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  78. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  79. package/skills/qa-verify-backend/references/safety-rules.md +90 -0
@@ -0,0 +1,88 @@
1
+ # Domain: Identity & Access Platforms
2
+
3
+ Covers identity and account platforms **and** the internal admin consoles that
4
+ configure them (tenant/scope configuration, consent definitions, audit history).
5
+
6
+ ## Priority Areas
7
+
8
+ 1. **Audit Trail Integrity** - every change attributable, complete, ordered, immutable
9
+ 2. **Identity Propagation** - the real human reaches the persisted record
10
+ 3. **Consent & Privacy** - capture, versioning, data minimisation
11
+ 4. **Configuration Correctness** - tenant isolation, validation, promotion
12
+ 5. **Admin RBAC** - write gating enforced at the API, not just the UI
13
+ 6. **Log & Data Hygiene** - no personal data leaking into logs or downstream platforms
14
+
15
+ ## Feature Risk Ranking
16
+
17
+ | Tier | Features | Rationale |
18
+ |------|----------|-----------|
19
+ | P0 | Authentication, consent & privacy, audit trail integrity, tenant configuration, personal data handling | A gap here is a regulatory and incident-response failure, not a UX annoyance. |
20
+ | P1 | Registration journeys, account recovery, admin RBAC, config promotion | Core operator and end-user workflows; failure blocks the business. |
21
+ | P2 | Config search & filtering, notifications, reporting views | Supporting features; failure is friction. |
22
+ | P3 | Static help content, about/version pages | Informational. |
23
+
24
+ ## Completeness Checklist
25
+
26
+ - [ ] Every state-changing action on identity or configuration data produces an audit record (who, what, when)
27
+ - [ ] Audit records are immutable — no update or delete path exists for the actor who wrote them
28
+ - [ ] Actor identity on an audit record is the authenticated user, never a service account or `unknown`
29
+ - [ ] Consent capture records purpose, version, timestamp and locale for every tenant
30
+ - [ ] Personal data is never written to application logs, traces or error payloads
31
+ - [ ] Admin writes are gated by role — a read-only operator cannot save configuration
32
+ - [ ] Configuration changes are scoped per tenant
33
+ - [ ] Config promotion between environments preserves the audit chain and does not rewrite authorship
34
+ - [ ] Deletion of a configuration entity is auditable and reversible via the audit trail
35
+ - [ ] Every environment has the same audit guarantees — ephemeral envs are not a special case
36
+
37
+ ## Data Integrity Checks
38
+
39
+ > After every state-changing action in an identity system, verify these.
40
+
41
+ 1. **Audit entry exists per change** — exactly one record per modified item. Not zero, not duplicated.
42
+ 2. **Actor is correct** — equals the logged-in user's identity; not `unknown`, not the previous editor, not a service role.
43
+ 3. **Timestamp ordering holds** — two rapid changes produce two distinct, ordered records; neither overwrites the other.
44
+ 4. **Snapshot fidelity** — the stored post-change snapshot matches the config store field for field.
45
+ 5. **Delete attribution** — a deletion names the deleter, not the last editor.
46
+ 6. **Tenant isolation** — changing tenant A produces records only for tenant A.
47
+ 7. **No silent loss** — a downstream failure surfaces (error, DLQ, alarm); it never disappears behind a success response.
48
+ 8. **Personal data containment** — audit record, log stream and error path carry no personal data beyond what the design authorises.
49
+
50
+ ## Cross-Feature Journeys
51
+
52
+ ### Journey 1: Config Change to Audit Trail
53
+ `Login → Open tenant config → Change one field → Save → Verify config store → Verify audit record → Verify actor/timestamp/snapshot/version`
54
+ - Verify: attributable, chronologically sortable, matches the config store exactly.
55
+
56
+ ### Journey 2: Rapid Successive Edits
57
+ `Open config → Save change 1 → Immediately save change 2 → Query audit store`
58
+ - Verify: **two** distinct records with distinct sort keys. Same-second collisions are a classic audit-loss bug.
59
+
60
+ ### Journey 3: Delete and Attribute
61
+ `Login as A → Edit and save → Logout → Login as B → Delete → Query audit store`
62
+ - Verify: the deletion record names **B**.
63
+
64
+ ### Journey 4: Degraded Identity
65
+ `Trigger a write where the token carries no identity claim`
66
+ - Verify: rejected, or recorded with an explicit flag and an alert. `unknown` actors must never be silently accepted.
67
+
68
+ ## Testing Guidance
69
+
70
+ The strongest bugs in this domain are **absences**: the audit record that was never
71
+ written, the permission that was never checked, the cleanup that was never run.
72
+
73
+ Never accept "the happy path wrote a record" as evidence the audit trail works.
74
+ Test the failure path, the same-second path, the delete path, and the
75
+ missing-identity path — that is where audit trails actually break.
76
+
77
+ **Heuristic focus:**
78
+ - **SFDIPOT** — Data (stored vs should-be), Operations (who may write), Time (ordering, collisions, skew)
79
+ - **FEW HICCUPPS** — Claims (does it do what the ticket claims?), Standards (data protection law, least privilege), Product (does the producer match the consumer's contract?)
80
+
81
+ ## Compliance
82
+
83
+ - **Data protection law** — lawful basis, consent records, right to erasure, data minimisation on audit payloads
84
+ - **Audit Trail Integrity** — attributable, tamper-evident, retained for the stated period
85
+ - **Least Privilege** — producers hold write-only; readers are separate principals
86
+ - **Encryption** — at rest with managed keys, in transit end to end
87
+ - **Log Hygiene** — no personal data or credentials in logs; production log levels constrained
88
+ - **Separation of Environments** — ephemeral/dev data never mixed with production
@@ -0,0 +1,138 @@
1
+ id: identity
2
+ name: "Identity & Access Platforms"
3
+
4
+ risk_ranking:
5
+ p0: ["authentication", "consent & privacy", "audit trail integrity", "tenant configuration", "personal data handling"]
6
+ p1: ["registration journeys", "account recovery", "admin console RBAC", "config promotion between environments"]
7
+ p2: ["config search & filtering", "notifications", "reporting views"]
8
+ p3: ["static help content", "about/version pages"]
9
+
10
+ completeness_checklist:
11
+ - "Every state-changing action on identity or configuration data produces an audit record (who, what, when)"
12
+ - "Audit records are immutable -- no update or delete path exists for the actor who wrote them"
13
+ - "Actor identity on an audit record is the authenticated user, never a service account or 'unknown'"
14
+ - "Consent capture records purpose, version, timestamp and locale for every tenant"
15
+ - "Personal data is never written to application logs, traces or error payloads"
16
+ - "Admin writes are gated by role -- a read-only operator cannot save configuration"
17
+ - "Configuration changes are scoped per tenant -- a change to one tenant cannot alter another"
18
+ - "Config promotion between environments preserves the audit chain and does not rewrite authorship"
19
+ - "Deletion of a configuration entity is auditable and reversible via the audit trail"
20
+ - "Every environment has the same audit guarantees -- ephemeral envs are not a special case"
21
+
22
+ data_integrity_checks:
23
+ - "Audit entry exists per change -- After every save in the admin console, exactly one audit record per modified item appears in the audit store. Not zero, not duplicated."
24
+ - "Actor is correct -- The audit record's actor equals the logged-in user's identity, not 'unknown', not the previous editor, not a service role."
25
+ - "Timestamp ordering holds -- Two changes made in quick succession produce two distinct, chronologically ordered audit records. Neither overwrites the other."
26
+ - "Snapshot fidelity -- The stored post-change snapshot matches, field for field, what the config store holds after the save."
27
+ - "Delete attribution -- Deleting an entity attributes the audit record to the user who deleted it, not the user who last edited it."
28
+ - "Tenant isolation -- Changing tenant A produces audit records only for tenant A's entity, with no records touching tenant B."
29
+ - "No silent loss -- A change that fails downstream surfaces as an error or a DLQ message; it never disappears with a success response to the user."
30
+ - "Personal data containment -- The audit record, the log stream and the error path contain no personal data beyond what the audit design authorises."
31
+
32
+ journeys:
33
+ - name: "Config Change to Audit Trail"
34
+ steps:
35
+ - "Login to the admin console as a known user"
36
+ - "Open a tenant configuration"
37
+ - "Change one identifiable field"
38
+ - "Save"
39
+ - "Verify the config store holds the new value"
40
+ - "Verify exactly one audit record was written for that entity"
41
+ - "Verify actor, timestamp, snapshot and version on the audit record"
42
+ - "Verify: the audit record is attributable, chronologically sortable, and matches the config store exactly"
43
+ - name: "Rapid Successive Edits"
44
+ steps:
45
+ - "Open a configuration"
46
+ - "Save a change"
47
+ - "Immediately save a second, different change"
48
+ - "Query the audit store for that entity"
49
+ - "Verify: TWO distinct audit records exist with distinct sort keys; neither overwrote the other"
50
+ - name: "Delete and Attribute"
51
+ steps:
52
+ - "Login as user A"
53
+ - "Edit and save an entity"
54
+ - "Logout, login as user B"
55
+ - "Delete that entity"
56
+ - "Verify: the deletion audit record names user B, not user A"
57
+ - name: "Unauthenticated / Degraded Identity"
58
+ steps:
59
+ - "Trigger a config write where the token carries no identity claim"
60
+ - "Verify: the write is rejected OR the audit record clearly flags the missing identity and an alert fires"
61
+ - "Verify: 'unknown' actors are not silently accepted into the audit trail"
62
+
63
+ must_test_patterns:
64
+ audit_trail:
65
+ - "One change produces one audit record -- not zero, not N duplicates"
66
+ - "Bulk save of N items produces N attributable records"
67
+ - "Sort key uniqueness under same-second writes"
68
+ - "Actor resolution for create, update and delete separately"
69
+ - "Audit write failure surfaces (DLQ, alarm) rather than silently dropping"
70
+ - "Audit store is write-only for the producer -- no update/delete permission"
71
+ - "Reprocessing or replay does not duplicate or clobber existing records"
72
+ - "Records survive redeploys and stream position resets"
73
+ identity_propagation:
74
+ - "User identity flows from token -> API handler -> persisted item -> audit record"
75
+ - "Missing or malformed identity claim handled explicitly, not defaulted"
76
+ - "Service-initiated writes are distinguishable from human writes"
77
+ - "Impersonation or support-mode writes are labelled as such"
78
+ configuration_management:
79
+ - "Change to one tenant or scope does not affect another"
80
+ - "Concurrent edits by two operators -- last-write-wins is detected, not silent"
81
+ - "Validation rejects malformed config before it reaches the store"
82
+ - "Promotion between environments preserves or correctly re-authors the audit chain"
83
+ data_protection:
84
+ - "No personal data or secrets in the log platform at the configured log level"
85
+ - "Log level in production is not debug/trace"
86
+ - "Encryption at rest with a customer-managed key where required"
87
+ - "Retention on audit data matches the stated compliance period"
88
+ - "API responses do not leak internal audit metadata to unauthorised roles"
89
+ admin_rbac:
90
+ - "Read-only role cannot save, delete or promote"
91
+ - "UI hiding a control does not substitute for API enforcement"
92
+ - "Role change takes effect without needing a redeploy"
93
+
94
+ common_bugs:
95
+ - "Audit gaps -- The trigger is enabled but a failure path drops entries silently; the trail looks complete but has holes nobody notices until an incident review"
96
+ - "Wrong actor -- Delete events attributed to the last editor, or every entry recorded as 'unknown' because the identity claim never reached the write path"
97
+ - "Sort key collisions -- Timestamps at second granularity cause two changes in the same second to share a key, so one overwrites the other"
98
+ - "Spec drift between producer and consumer -- Attribute names or types differ from what the reading UI expects, so the feature works in isolation but breaks on integration"
99
+ - "Over-collection -- Storing more than the design authorises (old images, full payloads) creating privacy exposure and item-size limits"
100
+ - "Trace logging in production -- Full event payloads written to logs, exporting configuration and personal data to the logging platform"
101
+ - "Stale documentation -- READMEs still describe the removed architecture, misleading the next engineer and failing the DoD"
102
+ - "Manual cleanup never executed -- A runbook is written and merged, but the data it was meant to clean is still there"
103
+
104
+ compliance:
105
+ - "Data protection law -- Lawful basis, consent records, right to erasure, data minimisation on audit payloads"
106
+ - "Audit Trail Integrity -- Records attributable, tamper-evident, retained for the stated period"
107
+ - "Least Privilege -- Producers hold write-only permission; readers are separate principals"
108
+ - "Encryption -- At rest with managed keys, in transit end to end"
109
+ - "Log Hygiene -- No personal data or credentials in logs; production log levels constrained"
110
+ - "Separation of Environments -- Ephemeral and dev data never mixed with production"
111
+
112
+ guidance: |
113
+ Identity-platform testing priority: attribution and completeness of the identity
114
+ and audit record. A platform whose audit trail has silent gaps cannot answer "who
115
+ changed this, and when" during an incident or a regulatory request -- which is the
116
+ entire point of the system.
117
+
118
+ Priority areas in order:
119
+ 1. Audit trail integrity -- every change attributable, complete, ordered, immutable
120
+ 2. Identity propagation -- the real human reaches the persisted record
121
+ 3. Consent & privacy -- capture, versioning, data minimisation
122
+ 4. Configuration correctness -- tenant isolation, validation, promotion
123
+ 5. Admin RBAC -- write gating enforced at the API, not just the UI
124
+ 6. Log & data hygiene -- no personal data leaking into logs or downstream platforms
125
+
126
+ Key heuristic focus for identity platforms:
127
+ - SFDIPOT: Data (what is stored vs what should be), Operations (who may write),
128
+ Time (ordering, same-second collisions, clock skew)
129
+ - FEW HICCUPPS: Claims (does it do what the ticket claims?), Standards (data
130
+ protection law, least privilege), Product (does the producer match the
131
+ consumer's contract?)
132
+ - Negative space: the strongest bugs here are absences -- the audit record that
133
+ was never written, the permission that was never checked, the cleanup that was
134
+ never run.
135
+
136
+ Critical rule: never accept "the happy path wrote a record" as evidence the audit
137
+ trail works. Test the failure path, the same-second path, the delete path and the
138
+ missing-identity path -- that is where audit trails actually break.
@@ -0,0 +1,194 @@
1
+ # Domain: Marketing Website
2
+
3
+ ## Priority Areas
4
+
5
+ 1. **Landing Pages & CTAs** - Conversion-critical elements must work flawlessly
6
+ 2. **Forms** - Contact, signup, and newsletter forms capture and submit correctly
7
+ 3. **SEO** - Meta tags, structured data, canonical URLs, crawlability
8
+ 4. **Analytics & Tracking** - Events fire correctly, attribution works
9
+ 5. **Mobile Responsiveness** - Layout and interactions work on all devices
10
+ 6. **Page Load Speed** - Fast load times directly impact conversion rates
11
+ 7. **Content Quality** - No broken links, typos, or placeholder content
12
+
13
+ ## Feature Risk Ranking
14
+
15
+ | Tier | Features | Rationale |
16
+ |------|---------------------------------------------------|------------------------------------------------------------------|
17
+ | P0 | Forms/lead capture, CTAs, page load speed | Direct revenue pipeline. Broken forms = lost leads. Broken CTAs = lost conversions. Slow pages = high bounce rate. |
18
+ | P1 | SEO, analytics tracking, responsive design | Discoverability and measurement. Without SEO, no traffic. Without analytics, no insight. Without responsive, lost mobile users. |
19
+ | P2 | Content pages, images/media, social sharing | Supports brand and engagement but does not directly block conversion. |
20
+ | P3 | Blog, careers page, press/media section | Informational. Rarely impacts lead generation or conversion. |
21
+
22
+ ## Completeness Checklist
23
+
24
+ > Minimum features a marketing website should have. Verify these exist and function before deep-testing.
25
+
26
+ - [ ] All CTAs lead to a valid destination (no dead links, no 404s, no "#" hrefs)
27
+ - [ ] All forms submit successfully and display a confirmation message or redirect to a thank-you page
28
+ - [ ] Meta tags present on every page (title, description, og:title, og:description, og:image)
29
+ - [ ] Structured data valid (JSON-LD for Organization, BreadcrumbList, FAQ, etc.)
30
+ - [ ] Cookie consent banner present and functional (blocks non-essential cookies until accepted)
31
+ - [ ] Privacy policy page linked from footer and accessible
32
+ - [ ] Mobile responsive -- layout does not break at 320px, 375px, or 768px
33
+ - [ ] No placeholder content (lorem ipsum, "TODO", sample images)
34
+ - [ ] Favicon present
35
+ - [ ] 404 page is custom and helpful (not a raw server error)
36
+
37
+ ## Data Integrity Checks
38
+
39
+ > Marketing sites are lighter on data, but lead capture and tracking integrity are critical.
40
+
41
+ 1. **Form submission arrives in CRM/inbox** -- Submit every form with test data and verify the submission appears in the destination system (CRM, email inbox, marketing automation platform). Check all fields map correctly.
42
+ 2. **Analytics events fire correctly** -- Open browser dev tools or a tag debugger. Verify page view events fire on load, CTA click events fire on click (with correct labels), and form submission events fire on successful submit. No duplicate events.
43
+ 3. **UTM parameters preserved through redirects** -- Navigate to the site with UTM parameters (e.g., `?utm_source=test&utm_medium=cpc&utm_campaign=spring`). Click through CTAs and form submissions. Verify UTM values are captured in analytics and/or CRM fields. Check that redirects (301, 302) do not strip the parameters.
44
+ 4. **Cookie consent respected** -- Before accepting cookies, non-essential scripts (analytics, ad pixels) should not load. After accepting, they should load. After rejecting, they should remain blocked.
45
+
46
+ ## Cross-Feature Journeys
47
+
48
+ > End-to-end flows that cross feature boundaries. Conversion path bugs are costly.
49
+
50
+ ### Journey 1: Landing Page to Lead Capture
51
+ `Land on Page (with UTM params) -> Scroll to CTA -> Click CTA -> Fill Form -> Submit -> See Thank-You Page -> Verify Lead Captured in CRM`
52
+ - Verify: UTM parameters preserved, CTA links to correct form, form submits without error, thank-you page loads, lead appears in CRM with correct source/campaign data.
53
+
54
+ ### Journey 2: Content Discovery to Social Sharing
55
+ `Homepage -> Navigate to Blog -> Read Article -> Click Share Button -> Verify OG Tags in Share Preview`
56
+ - Verify: blog navigation works, article loads fully, share buttons function, Open Graph tags produce correct title, description, and image in the share preview. Check og:image dimensions meet platform minimums (1200x630).
57
+
58
+ ### Journey 3: Organic Search Entry
59
+ `Enter via Search (simulate with direct URL + UTM) -> Browse Multiple Pages -> Click CTA -> Convert`
60
+ - Verify: page has correct canonical URL, meta tags match page content, internal links work, CTA conversion tracked as organic in analytics.
61
+
62
+ ## Must-Test Patterns
63
+
64
+ ### Landing Pages
65
+ - Hero section renders correctly with image, headline, subheading
66
+ - Primary CTA button is visible above the fold
67
+ - CTA links to correct destination (signup, demo, pricing)
68
+ - Multiple CTAs on the page all function correctly
69
+ - Social proof elements display (testimonials, logos, stats)
70
+ - Video embeds play without error
71
+ - Responsive layout does not break the visual hierarchy
72
+ - A/B test variants load correctly when configured
73
+ - UTM parameters preserved through CTA clicks
74
+ - Page loads correctly when accessed from paid ad link
75
+
76
+ ### Forms
77
+ - Contact form submits successfully with valid data
78
+ - Required field validation on all mandatory fields
79
+ - Email format validation
80
+ - Phone number format handling (international formats)
81
+ - Form submission sends confirmation email
82
+ - Form data reaches CRM or email marketing system
83
+ - Honeypot or reCAPTCHA spam prevention works
84
+ - Form does not submit with empty honeypot field triggered
85
+ - Newsletter signup with double opt-in flow
86
+ - Unsubscribe link works in marketing emails
87
+ - Form error messages are clear and positioned near the field
88
+ - Form submit button shows loading state and prevents double click
89
+
90
+ ### SEO
91
+ - Unique title tag on every page (under 60 characters)
92
+ - Meta description present and relevant (under 160 characters)
93
+ - Only one h1 per page
94
+ - Heading hierarchy is logical (h1 > h2 > h3, no skips)
95
+ - Canonical URL set on all pages
96
+ - Canonical URL is self-referencing or points to correct primary page
97
+ - Hreflang tags for multi-language sites
98
+ - Structured data (JSON-LD) for organization, breadcrumbs, FAQ, articles
99
+ - Structured data validates in Google Rich Results Test
100
+ - XML sitemap includes all indexable pages
101
+ - XML sitemap excludes noindex pages
102
+ - Robots.txt does not block critical resources
103
+ - Image alt text present and descriptive
104
+ - Clean URL slugs (no query parameters for main content)
105
+ - 301 redirects in place for moved or renamed pages
106
+ - No broken internal links (404 errors)
107
+ - Open Graph tags (og:title, og:description, og:image) present
108
+ - Twitter Card tags present
109
+ - og:image dimensions meet minimum (1200x630)
110
+
111
+ ### Analytics & Tracking
112
+ - Google Analytics (GA4) or equivalent loads on all pages
113
+ - Page view events fire on every page load
114
+ - CTA click events tracked with correct labels
115
+ - Form submission events fire on successful submit
116
+ - Scroll depth tracking active
117
+ - UTM parameters captured in analytics
118
+ - Conversion goals or events configured for key actions
119
+ - Cross-domain tracking works if multi-domain setup
120
+ - Cookie consent does not block analytics before acceptance
121
+ - Analytics data matches actual behavior (no duplicate events)
122
+ - Facebook Pixel, LinkedIn Insight, or other ad pixels fire correctly
123
+ - Event parameters contain correct values (page name, CTA label)
124
+
125
+ ### A/B Testing
126
+ - Control and variant pages render correctly
127
+ - Traffic split is approximately correct
128
+ - Variant does not flash original before changing (no flicker)
129
+ - A/B test tool does not significantly impact page load time
130
+ - Conversion tracking attributes to correct variant
131
+ - Test preview mode works for QA
132
+
133
+ ### Social Sharing
134
+ - Share buttons for major platforms (Facebook, Twitter, LinkedIn, email)
135
+ - Share URL includes correct page link
136
+ - Shared content preview shows correct title, description, image
137
+ - Share count displays (if applicable)
138
+ - Copy link button works and shows confirmation
139
+
140
+ ### Mobile Responsiveness
141
+ - Layout at 320px (small phone), 375px (standard phone), 768px (tablet)
142
+ - Navigation hamburger menu opens and closes
143
+ - Touch targets at least 44x44px
144
+ - No horizontal scrolling
145
+ - Images scale without overflow or distortion
146
+ - Text is readable without zooming (minimum 16px body text)
147
+ - Fixed headers do not cover content
148
+ - Phone numbers are clickable (tel: links)
149
+ - Maps and embeds resize correctly
150
+ - Popup and modal overlays work on mobile
151
+
152
+ ### Page Load Speed
153
+ - Largest Contentful Paint (LCP) under 2.5 seconds
154
+ - First Input Delay (FID) under 100ms
155
+ - Cumulative Layout Shift (CLS) under 0.1
156
+ - Images use modern formats (WebP, AVIF) with fallbacks
157
+ - CSS and JS are minified
158
+ - No render-blocking resources above the fold
159
+ - Fonts load without causing layout shift (font-display: swap)
160
+ - Third-party scripts do not block main thread significantly
161
+ - Lazy loading for images below the fold
162
+ - CDN caching configured for static assets
163
+
164
+ ### Content Quality
165
+ - No lorem ipsum or placeholder text
166
+ - No broken images (missing files or wrong paths)
167
+ - No typos or grammatical errors in headlines and CTAs
168
+ - Legal pages accessible (Privacy Policy, Terms of Service, Cookie Policy)
169
+ - Copyright year is current
170
+ - External links open in new tab and have rel="noopener"
171
+ - Downloads work (PDF brochures, whitepapers)
172
+ - Pricing page is accurate and up to date
173
+ - Team photos and bios are current
174
+
175
+ ## Common Bugs
176
+
177
+ - **CTA link errors** - Primary CTA links to wrong page or staging URL, UTM parameters stripped on redirect, anchor link targets missing element
178
+ - **Form delivery failures** - Form submits with success message but data never reaches inbox or CRM, email integration token expired silently, spam filter catches legitimate submissions
179
+ - **SEO regressions** - New deploy removes meta tags, sitemap not updated after adding pages, canonical URL points to staging, noindex left on from staging environment
180
+ - **Analytics gaps** - Cookie consent blocks all tracking including essential analytics, duplicate event firing on single-page app navigation, ad blocker breaks conversion tracking
181
+ - **Mobile layout breaks** - Hero image pushes CTA below fold on mobile, navigation menu overlaps content, horizontal scroll caused by uncontained element, video embed breaks layout
182
+ - **Performance regressions** - New hero video slows LCP, unoptimized image added to homepage, third-party chat widget blocks interactivity, font loading causes visible text shift
183
+ - **Social sharing preview** - og:image URL broken, description shows default text instead of page-specific, Twitter Card validator fails
184
+ - **A/B test interference** - Test JavaScript causes page flicker, wrong variant shown to returning users, test not stopped after reaching significance
185
+
186
+ ## Compliance Requirements
187
+
188
+ - **GDPR** - Cookie consent banner before non-essential cookies, privacy policy link, data processing disclosure
189
+ - **CCPA** - "Do Not Sell My Personal Information" link for California visitors
190
+ - **CAN-SPAM** - Unsubscribe link in all marketing emails, physical address in email footer
191
+ - **ADA / WCAG 2.1 AA** - Accessible forms, sufficient color contrast, keyboard navigable
192
+ - **ePrivacy Directive** - Cookie consent specific to EU visitors
193
+ - **FTC Guidelines** - Truthful advertising, clear disclosure of endorsements and sponsorships
194
+ - **COPPA** - Age verification if content could attract children under 13
@@ -0,0 +1,197 @@
1
+ # Domain: SaaS (Software as a Service)
2
+
3
+ ## Priority Areas
4
+
5
+ 1. **Authentication & Authorization** - Login flows, MFA, session security
6
+ 2. **Role-Based Access Control** - Permissions enforced at every level
7
+ 3. **Multi-Tenancy** - Strict data isolation between organizations
8
+ 4. **Billing & Subscriptions** - Plan management, upgrades, downgrades, cancellations
9
+ 5. **Data Integrity** - Export, import, backup, and consistency across views
10
+ 6. **Onboarding** - First-time user experience, setup wizards, guided tours
11
+ 7. **API Integrations** - Third-party connections, webhooks, OAuth flows
12
+
13
+ ## Feature Risk Ranking
14
+
15
+ | Tier | Features | Rationale |
16
+ |------|-----------------------------------------------------------|------------------------------------------------------------------|
17
+ | P0 | Authentication, permissions/RBAC, data CRUD, billing | Security breaches, unauthorized access, data loss, or billing errors are existential risks. |
18
+ | P1 | Dashboard, search, data export | Core daily workflows. Broken dashboards or export = users cannot do their job. |
19
+ | P2 | Settings, notifications, integrations | Supporting features. Failure is annoying but does not block primary tasks. |
20
+ | P3 | Help center, about page, changelog | Informational. Rarely impacts user workflows or business outcomes. |
21
+
22
+ ## Completeness Checklist
23
+
24
+ > Minimum features a SaaS application should have. Verify these exist and function before deep-testing.
25
+
26
+ - [ ] Role-based access control enforced (admin vs member vs viewer at minimum)
27
+ - [ ] Multi-tenancy isolation -- user cannot see another organization's data
28
+ - [ ] Data export available (CSV, JSON, or PDF for key data)
29
+ - [ ] Audit log present for sensitive actions (who did what, when)
30
+ - [ ] Session management -- timeout, concurrent session handling, logout
31
+ - [ ] Account deletion or deactivation flow exists
32
+ - [ ] Empty states for all list views (new account with no data shows guidance, not a blank page)
33
+ - [ ] Onboarding flow or first-use guidance for new users
34
+ - [ ] Billing/subscription page shows current plan, usage, and upgrade path
35
+ - [ ] Settings page allows profile, notification, and security configuration
36
+
37
+ ## Data Integrity Checks
38
+
39
+ > After every state-changing action in SaaS, verify these.
40
+
41
+ 1. **Created item appears in list** -- After creating any entity (project, record, user, etc.), it should immediately appear in the relevant list/table view with correct data.
42
+ 2. **Edited fields persist after reload** -- Edit an item, save, reload the page. Every changed field should retain its new value. Check all field types (text, dates, dropdowns, toggles).
43
+ 3. **Deleted items do not appear** -- After deletion, the item should vanish from all list views, search results, dashboards, and reports. Check that direct URL access to the deleted item returns a proper 404 or "not found" message.
44
+ 4. **Data isolated between tenants/users** -- Switch to a different user account or organization. The created/edited/deleted item should NOT be visible. Verify in list views, search, API responses, and exports.
45
+ 5. **Aggregations update** -- Dashboard counts, totals, and charts should reflect the change immediately (or after a documented refresh interval).
46
+ 6. **Audit trail records the action** -- If an audit log exists, verify the create/edit/delete action is logged with correct timestamp, user, and details.
47
+
48
+ ## Cross-Feature Journeys
49
+
50
+ > End-to-end flows that cross feature boundaries. Bugs often hide at the seams.
51
+
52
+ ### Journey 1: Full Data Lifecycle
53
+ `Register -> Complete Onboarding -> Create Item -> Edit Item -> Delete Item -> Verify Empty State`
54
+ - Verify: onboarding completes and does not reappear, created item is visible everywhere, edits persist, deletion is clean with no orphaned references, empty state is shown.
55
+
56
+ ### Journey 2: Role-Based Access Verification
57
+ `Login as Admin -> Change User Role to Viewer -> Logout -> Login as Viewer -> Verify Read-Only Access -> Logout -> Login as Admin -> Restore Role`
58
+ - Verify: role change takes effect immediately, viewer cannot create/edit/delete, UI hides write controls AND API rejects write attempts, role restoration works cleanly.
59
+
60
+ ### Journey 3: Billing and Feature Gating
61
+ `Login -> View Current Plan -> Upgrade Plan -> Verify New Features Unlocked -> Downgrade Plan -> Verify Features Locked`
62
+ - Verify: plan change reflects in billing page, new features are immediately accessible after upgrade, downgrade restricts access without data loss.
63
+
64
+ ## Must-Test Patterns
65
+
66
+ ### Authentication
67
+ - Email and password login
68
+ - Social login (Google, GitHub, SSO providers)
69
+ - Multi-factor authentication setup and verification
70
+ - Magic link login flow
71
+ - Password reset with email verification
72
+ - Session timeout and re-authentication prompts
73
+ - Concurrent session limits (same user, multiple devices)
74
+ - Account lockout after failed attempts
75
+ - SSO/SAML integration for enterprise accounts
76
+ - Token refresh and expiration handling
77
+
78
+ ### Authorization & Roles
79
+ - Admin can access all features
80
+ - Member restricted from admin-only pages
81
+ - Viewer cannot modify data (read-only enforcement)
82
+ - Custom role creation and assignment
83
+ - Permission changes take effect immediately (no stale cache)
84
+ - API endpoints enforce same permissions as UI
85
+ - Shared links respect permission boundaries
86
+ - Invitation flow assigns correct role
87
+ - Role downgrade removes access to restricted features
88
+ - Owner transfer flow
89
+
90
+ ### Multi-Tenancy
91
+ - User A cannot see User B's data (different organizations)
92
+ - Switching between organizations loads correct data
93
+ - Search results scoped to current organization only
94
+ - API responses never leak cross-tenant data
95
+ - Shared resources (templates, integrations) scoped correctly
96
+ - Organization deletion removes all associated data
97
+ - Subdomain or workspace slug routing to correct tenant
98
+ - Invitation to wrong organization prevented
99
+ - Audit logs scoped to correct organization
100
+
101
+ ### Dashboards & Data Views
102
+ - Dashboard loads with correct data for the current user
103
+ - Widgets display accurate numbers (totals, averages, counts)
104
+ - Date range filters update all widgets consistently
105
+ - Real-time data updates (if applicable) without full reload
106
+ - Empty states for new accounts with no data
107
+ - Large dataset performance (1000+ records in tables)
108
+ - Sort, filter, and pagination on data tables
109
+ - Export matches what is displayed on screen
110
+ - Chart tooltips show correct values
111
+ - Dashboard customization persists across sessions
112
+
113
+ ### Data Export & Import
114
+ - CSV export contains all visible columns
115
+ - Export with filters applied exports only filtered data
116
+ - Large export does not time out (background job with download link)
117
+ - CSV import with valid data
118
+ - Import with missing required fields (error handling)
119
+ - Import with duplicate records (merge or skip strategy)
120
+ - Import encoding issues (UTF-8, special characters)
121
+ - PDF report generation accuracy
122
+ - API data export matches UI data
123
+
124
+ ### Settings & Configuration
125
+ - Organization settings saved and applied
126
+ - User profile settings (name, avatar, timezone, language)
127
+ - Notification preferences (email, in-app, push)
128
+ - Integration settings (connect, disconnect, reconfigure)
129
+ - Billing settings accessible only to billing admins
130
+ - Danger zone actions (delete account, reset data) require confirmation
131
+ - Settings changes propagate to all active sessions
132
+
133
+ ### Billing & Subscriptions
134
+ - Plan selection and upgrade flow
135
+ - Downgrade with feature restriction warnings
136
+ - Trial expiration behavior (grace period, feature lockout)
137
+ - Payment method add, update, remove
138
+ - Invoice history and download
139
+ - Proration calculation on mid-cycle plan change
140
+ - Cancellation flow with retention offer
141
+ - Reactivation after cancellation
142
+ - Failed payment retry and dunning notifications
143
+ - Seat-based pricing (add/remove seats updates billing)
144
+ - Annual vs monthly billing toggle
145
+
146
+ ### API & Integrations
147
+ - OAuth connection flow (connect, authorize, callback)
148
+ - API key generation and revocation
149
+ - Webhook delivery and retry on failure
150
+ - Rate limiting behavior (correct error codes and headers)
151
+ - API pagination consistency
152
+ - API error responses follow documented format
153
+ - Third-party integration data sync accuracy
154
+ - Disconnecting integration cleans up related data
155
+ - API versioning (deprecated endpoints still work or return clear error)
156
+
157
+ ### Onboarding
158
+ - First login guided tour or setup wizard
159
+ - Skip onboarding option
160
+ - Onboarding checklist progress tracking
161
+ - Sample data population for new accounts
162
+ - Invite team members during onboarding
163
+ - Onboarding completion state persists
164
+ - Re-trigger onboarding from settings
165
+
166
+ ### Notification System
167
+ - Email notifications delivered for configured events
168
+ - In-app notification bell shows unread count
169
+ - Mark as read and mark all as read
170
+ - Notification links navigate to correct resource
171
+ - Notification preferences respected (no unwanted emails)
172
+ - Real-time notifications (WebSocket or polling)
173
+ - Notification for mentions and assignments
174
+ - Digest email frequency settings
175
+
176
+ ## Common Bugs
177
+
178
+ - **Permission escalation** - User discovers admin endpoint via URL manipulation, API does not check permissions on update/delete, role change cached and stale
179
+ - **Data leakage between tenants** - Search index not scoped, autocomplete suggests other tenant's data, export includes cross-tenant records, error messages reveal other tenant info
180
+ - **Broken role-based access** - New feature added without permission check, UI hides button but API allows action, permission matrix incomplete for custom roles
181
+ - **Billing edge cases** - Double charge on retry, downgrade does not reduce invoice, cancelled account still charged, proration math incorrect
182
+ - **Session management** - Token not invalidated on password change, session persists after role removal, refresh token reuse after revocation
183
+ - **Onboarding state bugs** - Wizard reappears after completion, skip does not properly set all defaults, onboarding blocks access to needed features
184
+ - **Data consistency** - Dashboard totals do not match detail view, export numbers differ from UI, cached aggregations stale after data change
185
+ - **Integration failures** - OAuth token expired silently, webhook payload format changed, rate limit not communicated to user, disconnected integration still sends data
186
+ - **Multi-device conflicts** - Same record edited on two devices, last write wins without warning, real-time sync delays cause confusion
187
+
188
+ ## Compliance Requirements
189
+
190
+ - **SOC 2 Type II** - Access controls, audit logging, data encryption at rest and in transit
191
+ - **GDPR** - Data portability (export), right to erasure, consent management, DPA availability
192
+ - **CCPA** - California consumer data rights, opt-out of data sale
193
+ - **HIPAA** (if health data) - BAA availability, PHI encryption, access logging
194
+ - **ISO 27001** - Information security management practices
195
+ - **Accessibility (WCAG 2.1 AA)** - All core workflows usable by keyboard and screen reader
196
+ - **Audit Trail** - All data modifications logged with timestamp, user, and action
197
+ - **Data Residency** - Data stored in region matching customer requirements