qualiow-exploratory-testing 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +177 -0
  2. package/bin/mcli +16 -0
  3. package/bin/mobile-cli.mjs +573 -0
  4. package/bin/wadb +6 -0
  5. package/bin/wk-ios +58 -0
  6. package/bin/wkeval.mjs +44 -0
  7. package/data/domains/_default.md +181 -0
  8. package/data/domains/ecommerce.md +192 -0
  9. package/data/domains/fintech.md +200 -0
  10. package/data/domains/identity.md +88 -0
  11. package/data/domains/identity.yml +138 -0
  12. package/data/domains/marketing.md +194 -0
  13. package/data/domains/saas.md +197 -0
  14. package/data/knowledge/changelog.yml +86 -0
  15. package/data/knowledge/learned-patterns.md +172 -1
  16. package/data/knowledge/manifest.yml +210 -4
  17. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  18. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  19. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  20. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  21. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  22. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  23. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  24. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  25. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  26. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  28. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  29. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  30. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  31. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  34. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  35. package/data/targets/_example-backend.yml +123 -0
  36. package/data/targets/_example-mobile-emulation.yml +52 -0
  37. package/data/targets/_example-native-mobile.yml +61 -0
  38. package/data/targets/_example-sim-android-chrome.yml +81 -0
  39. package/data/targets/_example-sim-ios-safari.yml +80 -0
  40. package/data/templates/api-probe-matrix.md +99 -0
  41. package/data/templates/coverage-map.md +14 -0
  42. package/data/templates/expected-behaviour.md +93 -0
  43. package/dist/cli/index.js +137 -5
  44. package/dist/cli/index.js.map +1 -1
  45. package/dist/index.d.ts +350 -527
  46. package/dist/index.js +144 -5
  47. package/dist/index.js.map +1 -1
  48. package/docs/BACKEND-VERIFICATION.md +277 -0
  49. package/docs/MOBILE-SETUP.md +92 -0
  50. package/package.json +26 -12
  51. package/scripts/doctor-mobile.sh +143 -0
  52. package/scripts/setup-mobile.sh +213 -0
  53. package/skills/qa-explore/phases/00-setup.md +8 -1
  54. package/skills/qa-explore/phases/01-auth.md +4 -0
  55. package/skills/qa-explore-mobile/SKILL.md +146 -0
  56. package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
  57. package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
  58. package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
  59. package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
  60. package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
  61. package/skills/qa-explore-mobile/phases/05-features.md +72 -0
  62. package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
  63. package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
  64. package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
  65. package/skills/qa-explore-quick/SKILL.md +6 -0
  66. package/skills/qa-verify-backend/SKILL.md +163 -0
  67. package/skills/qa-verify-backend/phases/00-setup.md +97 -0
  68. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  69. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  70. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  71. package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
  72. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
  73. package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
  74. package/skills/qa-verify-backend/references/api-probes.md +141 -0
  75. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  76. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  77. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  78. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  79. package/skills/qa-verify-backend/references/safety-rules.md +90 -0
@@ -0,0 +1,77 @@
1
+ id: technique-config-surface-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Config & Feature-Flag Surface Verification"
5
+ description: >
6
+ When the change under test is the configuration mechanism itself — a feature-flag
7
+ endpoint, a settings surface, a shared config library — the thing to verify is not only
8
+ that it answers, but that it answers with the deployed values, exposes nothing it should
9
+ not, is genuinely the single source of truth for the behaviour it advertises, and is
10
+ reachable by the same route in every environment it will be promoted to.
11
+ author: "Qualiow — BE/API verification layer"
12
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
13
+ tags: [backend, api, feature-flags, environment, security, configuration, deployment, verification]
14
+ domains: [all]
15
+ priority: medium
16
+ added: "2026-09-03"
17
+ updated: "2026-09-03"
18
+
19
+ content:
20
+ summary: >
21
+ Call the config surface authenticated and unauthenticated. Compare every value against
22
+ the environment's own deployment configuration. Check the response carries nothing but
23
+ what it advertises. Then check that the code actually reads the flag through that
24
+ surface rather than reaching around it — and that the route exists in the environments
25
+ the change will be promoted to.
26
+
27
+ core_principle: >
28
+ A config surface that reports a value the runtime does not use is worse than no config
29
+ surface: it is a source of confident, wrong answers, and every later verification built
30
+ on it inherits the error.
31
+
32
+ the_checks:
33
+ - check: "Values match the deployed configuration"
34
+ how: >
35
+ Compare each returned value against the environment's own deployment config file or
36
+ equivalent — not the default in the source. A mismatch means the surface and the
37
+ runtime disagree, which invalidates every conclusion drawn from either.
38
+ - check: "Nothing else leaks"
39
+ how: >
40
+ Read the whole response body. A config or flag endpoint that also returns
41
+ connection strings, secrets, internal hostnames or unrelated settings is a
42
+ disclosure finding regardless of the ticket's ACs.
43
+ - check: "Authentication and method"
44
+ how: >
45
+ Call it unauthenticated — it should not answer if it is not meant to. Call it with
46
+ the wrong verbs and expect 405. Confirm the response content type is what callers
47
+ will parse, and that an unauthenticated call does not silently redirect to a login
48
+ page that returns 200.
49
+ - check: "Single source of truth"
50
+ how: >
51
+ Grep the codebase for direct reads of the same key. A service that reads the
52
+ underlying configuration directly bypasses the interface: the flag appears on the
53
+ surface, but the surface is not authoritative for the behaviour it advertises. This
54
+ is a finding even when every AC passes, and it is exactly the shape that makes a
55
+ later verification session draw a confident wrong conclusion.
56
+ - check: "Route reachability, per environment"
57
+ how: >
58
+ Build a small matrix of path against result and meaning. Then confirm the same
59
+ routing or ingress rule exists in every environment the change will be promoted to
60
+ — a route added in one environment and not the others turns into a production
61
+ incident at promotion time, and it is invisible until then.
62
+ - check: "The flag actually switches something"
63
+ how: >
64
+ With the flag on, confirm the behaviour it selects is the behaviour observed.
65
+ Reporting a flag as true is not evidence that the path it selects is running — see
66
+ technique-environment-fingerprinting.
67
+
68
+ reporting:
69
+ - "Distinguish ACs satisfied by measurement from ACs satisfied by inference. 'The endpoint auto-exposes implemented flags, so the interface must be implemented' is inference — mark it, and say what would confirm it."
70
+ - "Carry the promotion risk forward explicitly: which environments still need the route, the value, or both."
71
+ - "Keep pre-existing findings on the same surface in a separate carry-over section."
72
+
73
+ anti_patterns:
74
+ - "Reading the flag's value from the source default instead of the environment's deployed config."
75
+ - "Accepting a 200 without reading the body — a login redirect answers 200 too."
76
+ - "Verifying the surface and never checking that the runtime reads through it."
77
+ - "Confirming a route in one environment and assuming it exists in the next."
@@ -0,0 +1,81 @@
1
+ id: technique-derived-value-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Derived Value Verification — Recomputing the Number and Naming What It Does Not Prove"
5
+ description: >
6
+ Percentages, totals, ratios, deltas and aggregates are derived, and they are verified by
7
+ recomputing them from the raw figures in the same response against the formula from the
8
+ specification — never from the code that implements it. The discipline that makes this
9
+ worth anything is the second half: when both sides of the check come from one payload,
10
+ you have verified the derivation and not the inputs. Say so, name the independent oracle
11
+ that would close the gap, and say whether you ran it.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
14
+ tags: [data-integrity, api, backend, calculations, verification, evidence, oracles, acceptance-criteria]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Take the formula from the spec, recompute every derived value from the raw figures in
23
+ the same response, and choose cases that stress sign, zero, scale and cardinality. Add
24
+ the structural invariants the values must satisfy regardless of magnitude. Then write
25
+ down, explicitly, that internal consistency is not correctness, and name the second
26
+ source that would settle it.
27
+
28
+ core_principle: >
29
+ A check whose two sides come from the same place can only find an inconsistency, never
30
+ an error. It is still worth running — it is cheap and it catches a real class of bug —
31
+ but reporting it as "the numbers are correct" is a claim the evidence does not support.
32
+
33
+ where_the_formula_comes_from: >
34
+ The specification, the ticket, or the person who owns the definition. A formula read out
35
+ of the implementation cannot falsify that implementation — recomputing the code's own
36
+ arithmetic proves only that arithmetic is deterministic.
37
+
38
+ choosing_the_cases:
39
+ - case: "A negative value"
40
+ why: "Sign handling is where derivation breaks, and a screen full of positive numbers never exercises it."
41
+ - case: "A zero, and a zero denominator"
42
+ why: "Division by zero, and the difference between 0, null and not-applicable."
43
+ - case: "A very large and a very small value"
44
+ why: "Rounding, precision loss and scale errors live at both ends."
45
+ - case: "One item and many items"
46
+ why: >
47
+ An aggregate over a single item should equal that item. It is the cleanest
48
+ apples-to-apples comparison available and removes aggregation ambiguity entirely —
49
+ reach for it first when comparing against another system.
50
+
51
+ structural_invariants: >
52
+ Relationships that must hold regardless of magnitude — a total equal to the sum of its
53
+ parts, one metric bounded by another, a derived cost positive, a percentage inside its
54
+ own range. They cost nothing, they generalise past the cases you happened to pick, and
55
+ a violation is unambiguous.
56
+
57
+ the_limitation_to_write_down: >
58
+ "This validates that the percentage equals the amount divided by the base. It does not
59
+ validate that the amount is itself correct, because both figures come from the same
60
+ payload." Then: "The outstanding oracle is <the other system / report / source of
61
+ truth> and it has not been run." That is a complete, honest, actionable verdict.
62
+
63
+ finding_an_independent_oracle:
64
+ - "A second system that holds the same figure for the same entity."
65
+ - "A report, dashboard or extract the business already trusts."
66
+ - "The source store the value is computed from, queried directly."
67
+ - "A worked example agreed with whoever owns the definition."
68
+ - "Where none exists, say that: 'no independent oracle is available' is itself a finding about the feature."
69
+
70
+ reporting:
71
+ - "Record every computed-vs-returned pair, not the conclusion drawn from them."
72
+ - "Record the formula and where it came from."
73
+ - "Record the invariants checked."
74
+ - "Record what the check does not prove, and the oracle that would."
75
+ - "Scope the verdict: 'internal consistency verified' is not 'values verified'."
76
+
77
+ anti_patterns:
78
+ - "Taking the formula from the code under test."
79
+ - "Checking only positive, mid-range values."
80
+ - "Reporting 'the numbers are correct' from a single-source check."
81
+ - "Comparing an aggregate against a per-item figure without a single-item case to anchor it."
@@ -0,0 +1,97 @@
1
+ id: technique-expected-behaviour-specification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Expected Behaviour Specification — Writing the Spec That Should Have Existed"
5
+ description: >
6
+ Most of what an API probe turns up has no acceptance criterion behind it, so there is
7
+ nothing to file the finding against. A bug report says "this is wrong"; this artifact
8
+ says "here is what right looks like" — observed behaviour and expected behaviour side by
9
+ side, per area, with the decisions the fix forces made explicit, ranked by what real
10
+ users can reach today, and closed with a plain-English reply the person funding the fix
11
+ can act on.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
14
+ tags: [acceptance-criteria, api, backend, requirements, reporting, communication, contract, verification]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Group the findings by cause, not by case. For each area write what the system does
23
+ today as evidence, what it should do as numbered falsifiable statements, and the
24
+ decisions the fix forces. Rank by production reachability. Finish with the same content
25
+ in plain English, ready to paste into a ticket or a chat.
26
+
27
+ core_principle: >
28
+ A finding with no acceptance criterion behind it will be argued about instead of fixed.
29
+ Converting a pile of observations into one reviewable specification turns a debate into
30
+ a decision — and the specification is frequently the most valuable thing a session
31
+ produces.
32
+
33
+ structure:
34
+ - part: "Observed"
35
+ content: >
36
+ Exact request, exact status, exact response, and which environments it holds in.
37
+ Behaviour often differs between environments, and that difference is part of the
38
+ finding rather than a distraction from it.
39
+ - part: "Expected"
40
+ content: >
41
+ Numbered, falsifiable statements — each a testable assertion, including the
42
+ boundary values themselves. Not sentiment, not "should handle this gracefully".
43
+ - part: "Decisions this forces"
44
+ content: >
45
+ Made explicit, because leaving them implicit is how one spec becomes three
46
+ implementations.
47
+ - part: "What must not change"
48
+ content: >
49
+ The valid inputs and their current answers, so the fix is verifiable as
50
+ non-breaking.
51
+
52
+ the_recurring_decisions:
53
+ - decision: "Reject, or clamp?"
54
+ answer: >
55
+ Reject. Silently serving a different page size, a truncated list or a corrected
56
+ value than was asked for breaks every caller that trusts the value back — and does
57
+ it invisibly.
58
+ - decision: "An error, or an empty result?"
59
+ answer: >
60
+ An error. A silent zero is indistinguishable from a genuine no-match and will be
61
+ read as data.
62
+ - decision: "Where does validation live?"
63
+ answer: >
64
+ At the layer that covers every implementation of the feature. Not in one of two
65
+ code paths, and never only in the client — the client is not in the path for any
66
+ other caller.
67
+ - decision: "Is the fallback reachable?"
68
+ answer: >
69
+ Say so explicitly. A match-all or default-everything branch reached by accident
70
+ from ordinary input is a different defect from the one that was reported.
71
+
72
+ ranking: >
73
+ Rank by what real users can reach **today**, not by how bad each finding reads. Name
74
+ which one is live in production now and which only appears in an environment nobody
75
+ ships from. The two need very different urgency and the difference is invisible unless
76
+ somebody states it.
77
+
78
+ the_plain_english_reply: >
79
+ The same content with no jargon, written for whoever decides whether to fund the fix:
80
+ what happens in a sentence they can picture; whether it is live and where; what the fix
81
+ is in outline, enough to size and not enough to design; where you agree with their
82
+ reading and precisely where you do not, and why. Close with the one item you would put
83
+ ahead of the rest and the reason — usually that it is the only one users are hitting
84
+ today. Writing this section is what gets the work scheduled.
85
+
86
+ disagreeing_well: >
87
+ When the reporter proposed a fix that would not work — most commonly guarding in the
88
+ client — agree with the part that is right, then say why the guard has to live on the
89
+ server: the cases in question cannot come from the app at all, they come from calling
90
+ the interface directly, and no client-side change can affect them.
91
+
92
+ anti_patterns:
93
+ - "One section per failing input instead of one per cause."
94
+ - "Expected behaviour written as a sentiment rather than an assertion."
95
+ - "Leaving reject-vs-clamp and error-vs-empty to the implementer."
96
+ - "Ranking by severity language instead of by who can reach it today."
97
+ - "Stopping before the plain-English reply, so the work never gets prioritised."
@@ -0,0 +1,78 @@
1
+ id: technique-presentation-integrity
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Presentation Integrity — Does the Screen Show What the Service Sent?"
5
+ description: >
6
+ A correct response can still reach the user as a wrong number. Formatting helpers that
7
+ inspect a value to guess what it is, units applied twice, rounding that crosses a
8
+ threshold, truncation presented as a total — each corrupts a payload that was right when
9
+ it left the service. The defect is invisible from any API tool and invisible from the
10
+ screen alone; it appears only when the raw response is held next to what is rendered. It
11
+ also gets attributed to the wrong change more often than any other kind.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from paired UI and API sessions on the same features, 2026-07 to 2026-09"
14
+ tags: [ui, api, data-integrity, verification, differential, calculations, regression, error-handling]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ For every value on screen, open the raw response beside it and compare. Probe any
23
+ formatter that branches on the value itself by feeding it inputs on both sides of the
24
+ threshold it tests. When they differ, work out which change owns the corruption before
25
+ filing it — it is frequently not the change under test.
26
+
27
+ core_principle: >
28
+ "The API is correct" and "the user sees the correct value" are two verdicts. Proving
29
+ the first says nothing about the second, and a session that only ever looks at one
30
+ surface cannot tell which half is broken.
31
+
32
+ the_formatter_that_guesses:
33
+ shape: >
34
+ A helper that inspects the value to decide what kind of value it is — the classic
35
+ being a ratio-or-percentage test such as `value > 1 ? value : value * 100`.
36
+ why_it_survives_review: >
37
+ It is right across most of the range, so it looks correct in every casual check and
38
+ in every test written from the same assumption.
39
+ why_it_is_wrong: >
40
+ The threshold is a guess about data, not a fact about it. Every legitimate value on
41
+ the wrong side of it is rendered wrong — here, every percentage at or below 1 renders
42
+ one hundred times too large.
43
+ how_to_probe_it: >
44
+ Find the threshold in the helper, then obtain a payload with a value just either side
45
+ of it. If no reachable record has one, that is worth reporting on its own: the case
46
+ cannot be verified from the product.
47
+ the_tell: >
48
+ A tile that contradicts itself — a percentage printed above the fraction it was
49
+ derived from, and the two disagree.
50
+
51
+ other_shapes:
52
+ - "A unit or currency applied twice, or not at all."
53
+ - "Rounding that changes the answer rather than the display — a value rounded before a comparison or a threshold."
54
+ - "Truncation presented as a total, so a partial figure reads as the whole."
55
+ - "A field the client silently drops because it does not render it — invisible until a consumer needs it."
56
+ - "Locale and format settings whose output merely happens to match: consistent is not causal. Change the setting and watch the output follow, or record the AC as partial."
57
+ - "An error rendered as a legitimate value — see technique-silent-failure-audit."
58
+
59
+ attributing_it_correctly: >
60
+ A correct payload corrupted on its way to the screen is not the backend ticket's
61
+ defect. It usually belongs to a different commit — frequently one deployed on one
62
+ environment and not another, which is why the same feature can look broken in one place
63
+ and fine in another. Identify the owning change, state plainly that the change under
64
+ test is not at fault, and say which environments are exposed. Reviewers who see a
65
+ contradictory screen reject correct work; that is the business impact, and it is worth
66
+ writing in those terms.
67
+
68
+ reporting:
69
+ - "Show the payload value and the rendered value side by side. That pair is the finding."
70
+ - "Give the range over which the corruption occurs, not just the one example that was noticed."
71
+ - "Name the owning change and the environments it is deployed on."
72
+ - "Say explicitly whether the change under test is implicated, especially when it is not."
73
+
74
+ anti_patterns:
75
+ - "Signing off a value from the screen alone."
76
+ - "Signing off a value from the API alone and assuming the screen follows."
77
+ - "Filing a rendering defect against the backend ticket that happened to surface it."
78
+ - "Accepting a setting-driven format as verified without ever changing the setting."
@@ -0,0 +1,110 @@
1
+ id: technique-release-readiness-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Release Readiness Verification — Many Tickets, One Build"
5
+ description: >
6
+ Verifying a release is not a longer ticket verification. The question is what is actually
7
+ in this build and what that makes each ticket's result mean — which is answered once, for
8
+ everything, before any testing starts. Adds a fixed result vocabulary that keeps
9
+ not-testable and not-tested visible, a coverage map with four states including
10
+ code-verified-only, a carry-over section that keeps pre-existing defects out of the
11
+ release's risk picture, and a disposition line with the condition that would change it.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from multi-ticket release verification sessions, 2026-08 to 2026-09"
14
+ tags: [regression, release, deployment, reporting, coverage, risk, acceptance-criteria, verification, environment]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Fingerprint the build once, list per ticket whether its commit is genuinely deployed,
23
+ and reclassify the session before touching anything. Then give every ticket a result
24
+ from a fixed vocabulary, keep a coverage map that admits what it did not do, separate
25
+ carry-overs, and end with a disposition and the condition that would reverse it.
26
+
27
+ core_principle: >
28
+ A result is only meaningful once you know what was running when you took it. Establish
29
+ that first, for the whole set, or half the session's conclusions will be about a build
30
+ that does not contain the change.
31
+
32
+ step_one_the_build_table: >
33
+ For every ticket, run the ancestry check against the deployed commit
34
+ (`git merge-base --is-ancestor`). A ticket whose backend is not deployed is NOT
35
+ TESTABLE here — not a failure. Testing its UI anyway produces a convincing and
36
+ meaningless result: the frontend ships the feature, the API does not answer, and it
37
+ renders as dashes, zeros or blanks that look exactly like a data defect and get filed
38
+ as one.
39
+
40
+ result_vocabulary:
41
+ - result: "Clean pass"
42
+ means: "Every AC met, with evidence."
43
+ - result: "Pass with caveats"
44
+ means: "The feature works; named ACs remain unproven, or a defect is open against it."
45
+ - result: "Fail"
46
+ means: "An AC is not met in this build."
47
+ - result: "Not testable here"
48
+ means: "The dependency is not deployed or not selected in this environment."
49
+ - result: "Not tested"
50
+ means: "In scope, not reached. Say so — never let it disappear into a summary."
51
+
52
+ coverage_map_states:
53
+ - state: "tested and passing"
54
+ means: "Observed, with evidence."
55
+ - state: "partially tested"
56
+ means: "Exercised, but a named part of the AC was not."
57
+ - state: "not tested"
58
+ means: "In scope, never reached."
59
+ - state: "code-verified only"
60
+ means: >
61
+ Read in the source and believed. This is UNVERIFIABLE, not a pass — mark it
62
+ distinctly so it cannot be skimmed as green.
63
+
64
+ two_pedantic_distinctions:
65
+ - distinction: "Consistent is not causal"
66
+ detail: >
67
+ A setting whose current value happens to match the output proves nothing. Change
68
+ the setting and watch the output follow, or record the AC as partial.
69
+ - distinction: "The data has to reach the case"
70
+ detail: >
71
+ "Negative values render in the negative colour" cannot be verified against records
72
+ that are all zero. If no reachable record exercises the AC, it is untested — find
73
+ one, create one, or say plainly it is unproven and name what data would settle it.
74
+
75
+ name_the_biggest_untested_item: >
76
+ One sentence identifying which gap carries the most risk. In a report full of green
77
+ rows it is the line the reader actually acts on, and it is the one most often left out.
78
+
79
+ carry_overs: >
80
+ Defects that pre-date the changes under test get their own section — pre-existing, not
81
+ caused by these tickets, tracked separately. Mixing them into the release's results
82
+ inflates the apparent risk of shipping and buries the findings that belong to it. The
83
+ reverse also holds: a defect that appears only because two changes met in this build
84
+ belongs to the release even though neither ticket owns it alone.
85
+
86
+ a_deploy_landing_mid_session:
87
+ - "Re-fingerprint immediately and record both build ids."
88
+ - "Keep pre-deploy findings in their own file — they are the evidence for what the previous build did."
89
+ - "Re-run the affected checks and state which build each result refers to."
90
+ - "Never merge pre- and post-deploy results into one table."
91
+
92
+ baselines: >
93
+ When a ticket's effect is a change in a number, take the before figure from the same
94
+ place as the after figure and state the tolerance — a few per cent on a large live
95
+ collection is churn, not a regression. Where a system-wide figure and a per-user figure
96
+ disagree, name which one describes the behaviour the feature actually changed.
97
+
98
+ the_disposition: >
99
+ Close with ship / ship-with-follow-ups / hold, plus the condition that would change your
100
+ mind, plus the scope of what you checked when it is narrower than the question being
101
+ asked. "Feature functionality verified; absolute value accuracy is out of scope and
102
+ validated by the product team" is a recommendation someone can act on. One with an
103
+ unstated scope is one someone will over-read.
104
+
105
+ anti_patterns:
106
+ - "Testing tickets before establishing what is deployed."
107
+ - "Letting 'not tested' vanish between 'passed' and 'failed'."
108
+ - "Reporting a code reading in the same colour as an observation."
109
+ - "Mixing carry-over defects into the release's risk picture."
110
+ - "A recommendation with no stated scope and no stated reversal condition."
@@ -0,0 +1,33 @@
1
+ # Release metadata for v0.5.0
2
+ version: "0.5.0"
3
+ date: "2026-09-03"
4
+ author: "qualiow"
5
+ status: active
6
+
7
+ notes: >
8
+ Payload correctness and release-level verification. v0.4.0 established how to reach an
9
+ endpoint and how to know which build is answering; this release covers what to do with
10
+ the answer once you have it, and how to report a set of tickets rather than one. Derived
11
+ Value Verification recomputes every percentage, total and aggregate from the raw figures
12
+ in the same response — and then says, in the report, that internal consistency is not
13
+ correctness and names the independent oracle that would close the gap. Presentation
14
+ Integrity holds the payload next to the screen, because a correct response can still
15
+ reach the user as a wrong number, and because that defect is routinely attributed to the
16
+ wrong change. Expected Behaviour Specification is the artifact for everything an API
17
+ probe turns up that has no acceptance criterion behind it: observed against expected, the
18
+ reject-or-clamp and error-or-empty decisions made explicit, ranked by what users can
19
+ reach today, and closed with a plain-English reply the person funding the fix can act on.
20
+ Config & Feature-Flag Surface Verification covers the case where the configuration
21
+ mechanism is itself the change. Release Readiness Verification handles many tickets in
22
+ one build: establish what is deployed before testing anything, keep not-testable and
23
+ not-tested visible, mark code-verified-only as UNVERIFIABLE rather than green, separate
24
+ carry-overs, and end with a disposition and the condition that would reverse it.
25
+
26
+ entry_count: 5
27
+
28
+ entries:
29
+ - technique-derived-value-verification
30
+ - technique-presentation-integrity
31
+ - technique-expected-behaviour-specification
32
+ - technique-config-surface-verification
33
+ - technique-release-readiness-verification
@@ -0,0 +1,123 @@
1
+ # Example target for /qa-verify-backend — copy, rename, and fill in.
2
+ #
3
+ # A backend-verification target is an ordinary web target plus four optional
4
+ # blocks: `backend:` (the cloud surface to probe read-only), `api:` (the service's
5
+ # own HTTP surface), `source:` (the repo and branch to review statically) and
6
+ # `environment:` (what the skill is allowed to do here). All four are optional — a
7
+ # target with none of them still works for /qa-explore.
8
+ #
9
+ # NEVER put credentials in this file. Reference environment variables by NAME
10
+ # and keep the values in .env (gitignored).
11
+
12
+ id: _example-backend
13
+ name: "Example Service — dev environment"
14
+ base_url: "https://example-service-dev.example.com"
15
+ domain: saas # or: identity, ecommerce, fintech, marketing, _default
16
+
17
+ auth:
18
+ strategy: credentials # switch to storage_state once .auth/ is primed
19
+ login_url: "https://example-service-dev.example.com/login" # absolute URL
20
+ state_file: ".auth/_example-backend.json"
21
+ notes: >
22
+ If login needs SSO/MFA that cannot be automated, sign in once by hand, save
23
+ the storage state, then set strategy to storage_state.
24
+
25
+ browser:
26
+ headless: false
27
+ viewport: {width: 1440, height: 900}
28
+
29
+ scope:
30
+ start_pages: ["/"]
31
+ max_depth: 3
32
+
33
+ safety:
34
+ read_only: false # writes allowed — this is a dev environment
35
+ no_form_submit: false
36
+
37
+ # --- What the skill may do here (consumed by /qa-verify-backend) -------------
38
+ environment:
39
+ kind: dev # dev | ephemeral | staging | production
40
+ destroyed_automatically: false # true for short-lived ephemeral envs — capture
41
+ # evidence as you go, it will not be there later
42
+ # kind: production forces read-only probes and skips the end-to-end phase.
43
+ # See .claude/skills/qa-verify-backend/references/safety-rules.md
44
+
45
+ # --- Cloud surface to probe (read-only) --------------------------------------
46
+ backend:
47
+ provider: aws
48
+ aws_profile_env: QA_AWS_PROFILE # NAME of the env var holding the profile
49
+ region_env: QA_AWS_REGION # NAME of the env var holding the region
50
+ account_id: "000000000000" # expected account — probes stop if it differs
51
+ env_suffix: dev
52
+ resources:
53
+ config_table: "example-service-dev-config"
54
+ audit_table: "example-service-dev-audit"
55
+ stream_lambda: "example-service-dev-change-processor"
56
+ stream_lambda_role: "example-service-dev-change-processor-role"
57
+ dlq: "example-service-dev-change-processor-dlq"
58
+ notes: >
59
+ Resource names here are ASSERTIONS, not proof. The skill confirms them with
60
+ `aws dynamodb list-tables` / `aws lambda list-functions` before trusting them —
61
+ a probe against a mistyped name returns an error that reads like a failure.
62
+
63
+ # --- The service's own HTTP surface (probed read-only) -----------------------
64
+ api:
65
+ # base_url: defaults to the target's base_url above when omitted.
66
+ auth: session-cookie # session-cookie | bearer-env | none
67
+ #
68
+ # session-cookie is the default and the one to prefer: the skill runs the
69
+ # request from inside the already-authenticated page, so it inherits the
70
+ # session cookie, the CSRF token and the client's interceptors. Nothing to
71
+ # store, and it works with SSO/MFA that no scripted login can pass.
72
+ #
73
+ # token_env: QA_API_TOKEN # NAME of the env var, for `auth: bearer-env`
74
+
75
+ # Persistent profile holding the logged-in session. Prime it once, headed:
76
+ # playwright-cli -s=<id> open <base_url> --headed --profile .auth/<dir>
77
+ # A profile directory outlives a saved storage_state by a wide margin.
78
+ browser_profile: ".auth/_example-backend-profile"
79
+
80
+ # Used to fingerprint the environment before any verdict is written.
81
+ version_endpoint: "/api/v1/version"
82
+
83
+ endpoints:
84
+ search: "POST /api/v1/search" # note the verb — a GET here returns 405
85
+ item_detail: "GET /api/v1/item/{id}"
86
+
87
+ # The ONLY endpoints the read-only API lane may call without asking. Leave an
88
+ # endpoint off this list if its verb is safe but its effect is not — billed per
89
+ # call, queues a job, or rate-limited into an outage.
90
+ probe_allowlist:
91
+ - "POST /api/v1/search"
92
+ - "GET /api/v1/item/{id}"
93
+ - "GET /api/v1/version"
94
+
95
+ # Run the same case matrix here too, and compare SHAPES — never counts.
96
+ parity_targets: ["_example-backend-staging"]
97
+
98
+ # Flag name -> where its DEPLOYED value is declared for this environment.
99
+ # A flag can select between two implementations of one feature inside a single
100
+ # identical build, which is why "same commit, different behaviour" is a
101
+ # configuration finding rather than deploy lag.
102
+ feature_flags:
103
+ USE_NEW_SEARCH: "deploy/config/example-service-dev.yml"
104
+
105
+ notes: >
106
+ The API lane is read-only in every environment. Writes go through the real
107
+ write path in phase 4, never against production. Response bodies are real
108
+ records — they are redacted before anything is written to disk.
109
+
110
+ # --- Implementation branch to review statically ------------------------------
111
+ source:
112
+ repo_path: "/absolute/path/to/the/service/repo"
113
+ branch: "origin/feature/TICKET-123-short-description"
114
+ base_branch: "origin/main"
115
+ components:
116
+ iac: "infra/modules/audit, infra/modules/service"
117
+ stream_lambda: "services/audit/lambdas"
118
+ write_api: "services/api"
119
+ ui: "apps/console"
120
+
121
+ notes: >
122
+ Ticket under verification and the primary journey — e.g. "change a setting in
123
+ the console, save, and confirm exactly one correct audit record is written."
@@ -0,0 +1,52 @@
1
+ # Template target — MOBILE-WEB EMULATION via Playwright (desktop engine, mobile device
2
+ # descriptor). This is the /qa-explore path, NOT /qa-explore-mobile — no simulator needed.
3
+ #
4
+ # WHEN TO USE WHICH:
5
+ # - This template (Playwright emulation): fast, supports storage_state auth, full
6
+ # console/network/eval — but the engine is desktop Chromium/WebKit pretending to be
7
+ # a phone. Good for layout, flows, and most functional bugs.
8
+ # - _example-sim-ios-safari.yml / _example-sim-android-chrome.yml (real sim browsers via
9
+ # /qa-explore-mobile): the genuine mobile engines — required for engine-identity
10
+ # behavior (UA/feature detection, real keyboard/viewport behavior, Safari quirks).
11
+ # Playwright's "webkit" is desktop macOS WebKit, NOT iOS Safari.
12
+ id: example-mobile-emulation
13
+ name: "Example Mobile Web App — Playwright Mobile Emulation (WebKit)"
14
+ base_url: "https://staging.m.example.com"
15
+ domain: _default
16
+ auth:
17
+ strategy: storage_state
18
+ state_file: ".auth/example-mobile-emulation.json"
19
+ login_url: "https://staging.m.example.com/login"
20
+ browser:
21
+ headless: false
22
+ engine: webkit # webkit (Safari-like) | chromium (Chrome-like)
23
+ channel: webkit
24
+ device: "iPhone 15" # Playwright device descriptor — sets mobile UA + viewport + touch
25
+ viewport: { width: 390, height: 844 }
26
+ safety:
27
+ read_only: false
28
+ no_form_submit: false
29
+ scope:
30
+ start_pages:
31
+ - "/"
32
+ max_depth: 3
33
+ exclude_patterns:
34
+ - "/auth/*"
35
+ - "/api/*"
36
+ - "**/*.pdf"
37
+ notes: |
38
+ Mobile-web emulation target: Playwright with a mobile device descriptor. Closest proxy
39
+ to real mobile browsers that Playwright offers (caveat: WebKit != Safari — no Apple Pay,
40
+ ITP nuances differ; Chromium != Android Chrome).
41
+
42
+ The device descriptor matters: many mobile hosts redirect desktop UAs to the desktop
43
+ site — viewport resize alone is NOT enough; the descriptor sets the mobile UA. Launch:
44
+ PLAYWRIGHT_MCP_DEVICE="iPhone 15" PLAYWRIGHT_MCP_BROWSER="webkit" playwright-cli ...
45
+ or pass --browser=webkit --device="iPhone 15" explicitly.
46
+
47
+ Create a chromium twin of this config to cross-check engine-specific behavior. WebKit
48
+ and Chromium maintain independent cookie jars — state files are NOT interchangeable
49
+ between engine variants; each needs its own one-time login via /qa-target-setup.
50
+
51
+ For bugs that depend on the real engine identity (browser detection, install prompts,
52
+ engine-specific rendering), escalate to the real-sim targets and /qa-explore-mobile.