qualiow-exploratory-testing 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/.claude-plugin/plugin.json +22 -0
  2. package/.env.example +27 -0
  3. package/CHANGELOG.md +194 -0
  4. package/README.md +428 -179
  5. package/agents/qa-gather-agent.md +24 -0
  6. package/{scripts → bin}/doctor-mobile.sh +57 -18
  7. package/bin/mcli +15 -9
  8. package/bin/mobile-cli.mjs +574 -286
  9. package/{scripts → bin}/setup-mobile.sh +56 -19
  10. package/bin/wadb +10 -4
  11. package/bin/wk-ios +29 -17
  12. package/bin/wkeval.mjs +7 -1
  13. package/data/domains/_default.yml +1 -1
  14. package/data/domains/identity.yml +138 -0
  15. package/data/knowledge/changelog.yml +104 -0
  16. package/data/knowledge/learned-patterns.md +172 -86
  17. package/data/knowledge/manifest.yml +481 -70
  18. package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
  19. package/data/knowledge/releases/v0.1.0/release.yml +8 -1
  20. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  21. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  22. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  23. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  24. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  25. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  26. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  28. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  29. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  30. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  31. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  34. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  35. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  36. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  37. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  38. package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
  39. package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
  40. package/data/knowledge/releases/v0.6.0/release.yml +30 -0
  41. package/data/security/SECURITY-POLICY.md +273 -0
  42. package/data/targets/_default.yml +1 -1
  43. package/data/targets/_example-api-only.yml +80 -0
  44. package/data/targets/_example-backend.yml +123 -0
  45. package/data/targets/_example-mobile-emulation.yml +2 -2
  46. package/data/targets/_example-native-mobile.yml +2 -2
  47. package/data/targets/_example-sim-android-chrome.yml +1 -1
  48. package/data/targets/_example-sim-ios-safari.yml +1 -1
  49. package/data/templates/api-probe-matrix.md +99 -0
  50. package/data/templates/bug-report.md +29 -151
  51. package/data/templates/coverage-map.md +23 -6
  52. package/data/templates/expected-behaviour.md +94 -0
  53. package/data/templates/session-charter.md +3 -0
  54. package/data/templates/session-report.md +57 -80
  55. package/dist/cli/index.js +1500 -739
  56. package/dist/index.d.ts +673 -75
  57. package/dist/index.js +1664 -270
  58. package/docs/BACKEND-VERIFICATION.md +307 -0
  59. package/docs/GETTING-STARTED.md +188 -128
  60. package/docs/MOBILE-SETUP.md +149 -18
  61. package/package.json +32 -16
  62. package/skills/qa-explore/SKILL.md +31 -25
  63. package/skills/qa-explore/phases/00-setup.md +67 -41
  64. package/skills/qa-explore/phases/01-auth.md +32 -15
  65. package/skills/qa-explore/phases/02-charter.md +16 -5
  66. package/skills/qa-explore/phases/03-discovery.md +50 -26
  67. package/skills/qa-explore/phases/04-journeys.md +2 -2
  68. package/skills/qa-explore/phases/05-features.md +39 -24
  69. package/skills/qa-explore/phases/06-edge-cases.md +24 -13
  70. package/skills/qa-explore/phases/07-reporting.md +79 -140
  71. package/skills/qa-explore/references/context-integration.md +1 -1
  72. package/skills/qa-explore/references/output-contract.md +187 -0
  73. package/skills/qa-explore/references/paths.md +84 -0
  74. package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
  75. package/skills/qa-explore/references/security-rules.md +66 -25
  76. package/skills/qa-explore/references/severity-guide.md +31 -4
  77. package/skills/qa-explore-cleanup/SKILL.md +17 -6
  78. package/skills/qa-explore-feedback/SKILL.md +18 -5
  79. package/skills/qa-explore-mobile/SKILL.md +86 -64
  80. package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
  81. package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
  82. package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
  83. package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
  84. package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
  85. package/skills/qa-explore-mobile/phases/05-features.md +9 -9
  86. package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
  87. package/skills/qa-explore-mobile/phases/07-reporting.md +82 -56
  88. package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
  89. package/skills/qa-explore-quick/SKILL.md +60 -25
  90. package/skills/qa-explore-report/SKILL.md +40 -34
  91. package/skills/qa-gather/SKILL.md +17 -2
  92. package/skills/qa-knowledge-add/SKILL.md +20 -9
  93. package/skills/qa-knowledge-list/SKILL.md +12 -3
  94. package/skills/qa-target-setup/SKILL.md +63 -41
  95. package/skills/qa-verify-backend/SKILL.md +172 -0
  96. package/skills/qa-verify-backend/phases/00-setup.md +142 -0
  97. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  98. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  99. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  100. package/skills/qa-verify-backend/phases/03b-api-verification.md +170 -0
  101. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +82 -0
  102. package/skills/qa-verify-backend/phases/05-reporting.md +163 -0
  103. package/skills/qa-verify-backend/references/api-probes.md +168 -0
  104. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  105. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  106. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  107. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  108. package/skills/qa-verify-backend/references/safety-rules.md +95 -0
  109. package/dist/cli/index.d.ts +0 -1
  110. package/dist/cli/index.js.map +0 -1
  111. package/dist/index.js.map +0 -1
@@ -0,0 +1,77 @@
1
+ id: technique-config-surface-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Config & Feature-Flag Surface Verification"
5
+ description: >
6
+ When the change under test is the configuration mechanism itself — a feature-flag
7
+ endpoint, a settings surface, a shared config library — the thing to verify is not only
8
+ that it answers, but that it answers with the deployed values, exposes nothing it should
9
+ not, is genuinely the single source of truth for the behaviour it advertises, and is
10
+ reachable by the same route in every environment it will be promoted to.
11
+ author: "Qualiow — BE/API verification layer"
12
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
13
+ tags: [backend, api, feature-flags, environment, security, configuration, deployment, verification]
14
+ domains: [all]
15
+ priority: medium
16
+ added: "2026-09-03"
17
+ updated: "2026-09-03"
18
+
19
+ content:
20
+ summary: >
21
+ Call the config surface authenticated and unauthenticated. Compare every value against
22
+ the environment's own deployment configuration. Check the response carries nothing but
23
+ what it advertises. Then check that the code actually reads the flag through that
24
+ surface rather than reaching around it — and that the route exists in the environments
25
+ the change will be promoted to.
26
+
27
+ core_principle: >
28
+ A config surface that reports a value the runtime does not use is worse than no config
29
+ surface: it is a source of confident, wrong answers, and every later verification built
30
+ on it inherits the error.
31
+
32
+ the_checks:
33
+ - check: "Values match the deployed configuration"
34
+ how: >
35
+ Compare each returned value against the environment's own deployment config file or
36
+ equivalent — not the default in the source. A mismatch means the surface and the
37
+ runtime disagree, which invalidates every conclusion drawn from either.
38
+ - check: "Nothing else leaks"
39
+ how: >
40
+ Read the whole response body. A config or flag endpoint that also returns
41
+ connection strings, secrets, internal hostnames or unrelated settings is a
42
+ disclosure finding regardless of the ticket's ACs.
43
+ - check: "Authentication and method"
44
+ how: >
45
+ Call it unauthenticated — it should not answer if it is not meant to. Call it with
46
+ the wrong verbs and expect 405. Confirm the response content type is what callers
47
+ will parse, and that an unauthenticated call does not silently redirect to a login
48
+ page that returns 200.
49
+ - check: "Single source of truth"
50
+ how: >
51
+ Grep the codebase for direct reads of the same key. A service that reads the
52
+ underlying configuration directly bypasses the interface: the flag appears on the
53
+ surface, but the surface is not authoritative for the behaviour it advertises. This
54
+ is a finding even when every AC passes, and it is exactly the shape that makes a
55
+ later verification session draw a confident wrong conclusion.
56
+ - check: "Route reachability, per environment"
57
+ how: >
58
+ Build a small matrix of path against result and meaning. Then confirm the same
59
+ routing or ingress rule exists in every environment the change will be promoted to
60
+ — a route added in one environment and not the others turns into a production
61
+ incident at promotion time, and it is invisible until then.
62
+ - check: "The flag actually switches something"
63
+ how: >
64
+ With the flag on, confirm the behaviour it selects is the behaviour observed.
65
+ Reporting a flag as true is not evidence that the path it selects is running — see
66
+ technique-environment-fingerprinting.
67
+
68
+ reporting:
69
+ - "Distinguish ACs satisfied by measurement from ACs satisfied by inference. 'The endpoint auto-exposes implemented flags, so the interface must be implemented' is inference — mark it, and say what would confirm it."
70
+ - "Carry the promotion risk forward explicitly: which environments still need the route, the value, or both."
71
+ - "Keep pre-existing findings on the same surface in a separate carry-over section."
72
+
73
+ anti_patterns:
74
+ - "Reading the flag's value from the source default instead of the environment's deployed config."
75
+ - "Accepting a 200 without reading the body — a login redirect answers 200 too."
76
+ - "Verifying the surface and never checking that the runtime reads through it."
77
+ - "Confirming a route in one environment and assuming it exists in the next."
@@ -0,0 +1,81 @@
1
+ id: technique-derived-value-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Derived Value Verification — Recomputing the Number and Naming What It Does Not Prove"
5
+ description: >
6
+ Percentages, totals, ratios, deltas and aggregates are derived, and they are verified by
7
+ recomputing them from the raw figures in the same response against the formula from the
8
+ specification — never from the code that implements it. The discipline that makes this
9
+ worth anything is the second half: when both sides of the check come from one payload,
10
+ you have verified the derivation and not the inputs. Say so, name the independent oracle
11
+ that would close the gap, and say whether you ran it.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
14
+ tags: [data-integrity, api, backend, calculations, verification, evidence, oracles, acceptance-criteria]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Take the formula from the spec, recompute every derived value from the raw figures in
23
+ the same response, and choose cases that stress sign, zero, scale and cardinality. Add
24
+ the structural invariants the values must satisfy regardless of magnitude. Then write
25
+ down, explicitly, that internal consistency is not correctness, and name the second
26
+ source that would settle it.
27
+
28
+ core_principle: >
29
+ A check whose two sides come from the same place can only find an inconsistency, never
30
+ an error. It is still worth running — it is cheap and it catches a real class of bug —
31
+ but reporting it as "the numbers are correct" is a claim the evidence does not support.
32
+
33
+ where_the_formula_comes_from: >
34
+ The specification, the ticket, or the person who owns the definition. A formula read out
35
+ of the implementation cannot falsify that implementation — recomputing the code's own
36
+ arithmetic proves only that arithmetic is deterministic.
37
+
38
+ choosing_the_cases:
39
+ - case: "A negative value"
40
+ why: "Sign handling is where derivation breaks, and a screen full of positive numbers never exercises it."
41
+ - case: "A zero, and a zero denominator"
42
+ why: "Division by zero, and the difference between 0, null and not-applicable."
43
+ - case: "A very large and a very small value"
44
+ why: "Rounding, precision loss and scale errors live at both ends."
45
+ - case: "One item and many items"
46
+ why: >
47
+ An aggregate over a single item should equal that item. It is the cleanest
48
+ apples-to-apples comparison available and removes aggregation ambiguity entirely —
49
+ reach for it first when comparing against another system.
50
+
51
+ structural_invariants: >
52
+ Relationships that must hold regardless of magnitude — a total equal to the sum of its
53
+ parts, one metric bounded by another, a derived cost positive, a percentage inside its
54
+ own range. They cost nothing, they generalise past the cases you happened to pick, and
55
+ a violation is unambiguous.
56
+
57
+ the_limitation_to_write_down: >
58
+ "This validates that the percentage equals the amount divided by the base. It does not
59
+ validate that the amount is itself correct, because both figures come from the same
60
+ payload." Then: "The outstanding oracle is <the other system / report / source of
61
+ truth> and it has not been run." That is a complete, honest, actionable verdict.
62
+
63
+ finding_an_independent_oracle:
64
+ - "A second system that holds the same figure for the same entity."
65
+ - "A report, dashboard or extract the business already trusts."
66
+ - "The source store the value is computed from, queried directly."
67
+ - "A worked example agreed with whoever owns the definition."
68
+ - "Where none exists, say that: 'no independent oracle is available' is itself a finding about the feature."
69
+
70
+ reporting:
71
+ - "Record every computed-vs-returned pair, not the conclusion drawn from them."
72
+ - "Record the formula and where it came from."
73
+ - "Record the invariants checked."
74
+ - "Record what the check does not prove, and the oracle that would."
75
+ - "Scope the verdict: 'internal consistency verified' is not 'values verified'."
76
+
77
+ anti_patterns:
78
+ - "Taking the formula from the code under test."
79
+ - "Checking only positive, mid-range values."
80
+ - "Reporting 'the numbers are correct' from a single-source check."
81
+ - "Comparing an aggregate against a per-item figure without a single-item case to anchor it."
@@ -0,0 +1,97 @@
1
+ id: technique-expected-behaviour-specification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Expected Behaviour Specification — Writing the Spec That Should Have Existed"
5
+ description: >
6
+ Most of what an API probe turns up has no acceptance criterion behind it, so there is
7
+ nothing to file the finding against. A bug report says "this is wrong"; this artifact
8
+ says "here is what right looks like" — observed behaviour and expected behaviour side by
9
+ side, per area, with the decisions the fix forces made explicit, ranked by what real
10
+ users can reach today, and closed with a plain-English reply the person funding the fix
11
+ can act on.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
14
+ tags: [acceptance-criteria, api, backend, requirements, reporting, communication, contract, verification]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Group the findings by cause, not by case. For each area write what the system does
23
+ today as evidence, what it should do as numbered falsifiable statements, and the
24
+ decisions the fix forces. Rank by production reachability. Finish with the same content
25
+ in plain English, ready to paste into a ticket or a chat.
26
+
27
+ core_principle: >
28
+ A finding with no acceptance criterion behind it will be argued about instead of fixed.
29
+ Converting a pile of observations into one reviewable specification turns a debate into
30
+ a decision — and the specification is frequently the most valuable thing a session
31
+ produces.
32
+
33
+ structure:
34
+ - part: "Observed"
35
+ content: >
36
+ Exact request, exact status, exact response, and which environments it holds in.
37
+ Behaviour often differs between environments, and that difference is part of the
38
+ finding rather than a distraction from it.
39
+ - part: "Expected"
40
+ content: >
41
+ Numbered, falsifiable statements — each a testable assertion, including the
42
+ boundary values themselves. Not sentiment, not "should handle this gracefully".
43
+ - part: "Decisions this forces"
44
+ content: >
45
+ Made explicit, because leaving them implicit is how one spec becomes three
46
+ implementations.
47
+ - part: "What must not change"
48
+ content: >
49
+ The valid inputs and their current answers, so the fix is verifiable as
50
+ non-breaking.
51
+
52
+ the_recurring_decisions:
53
+ - decision: "Reject, or clamp?"
54
+ answer: >
55
+ Reject. Silently serving a different page size, a truncated list or a corrected
56
+ value than was asked for breaks every caller that trusts the value back — and does
57
+ it invisibly.
58
+ - decision: "An error, or an empty result?"
59
+ answer: >
60
+ An error. A silent zero is indistinguishable from a genuine no-match and will be
61
+ read as data.
62
+ - decision: "Where does validation live?"
63
+ answer: >
64
+ At the layer that covers every implementation of the feature. Not in one of two
65
+ code paths, and never only in the client — the client is not in the path for any
66
+ other caller.
67
+ - decision: "Is the fallback reachable?"
68
+ answer: >
69
+ Say so explicitly. A match-all or default-everything branch reached by accident
70
+ from ordinary input is a different defect from the one that was reported.
71
+
72
+ ranking: >
73
+ Rank by what real users can reach **today**, not by how bad each finding reads. Name
74
+ which one is live in production now and which only appears in an environment nobody
75
+ ships from. The two need very different urgency and the difference is invisible unless
76
+ somebody states it.
77
+
78
+ the_plain_english_reply: >
79
+ The same content with no jargon, written for whoever decides whether to fund the fix:
80
+ what happens in a sentence they can picture; whether it is live and where; what the fix
81
+ is in outline, enough to size and not enough to design; where you agree with their
82
+ reading and precisely where you do not, and why. Close with the one item you would put
83
+ ahead of the rest and the reason — usually that it is the only one users are hitting
84
+ today. Writing this section is what gets the work scheduled.
85
+
86
+ disagreeing_well: >
87
+ When the reporter proposed a fix that would not work — most commonly guarding in the
88
+ client — agree with the part that is right, then say why the guard has to live on the
89
+ server: the cases in question cannot come from the app at all, they come from calling
90
+ the interface directly, and no client-side change can affect them.
91
+
92
+ anti_patterns:
93
+ - "One section per failing input instead of one per cause."
94
+ - "Expected behaviour written as a sentiment rather than an assertion."
95
+ - "Leaving reject-vs-clamp and error-vs-empty to the implementer."
96
+ - "Ranking by severity language instead of by who can reach it today."
97
+ - "Stopping before the plain-English reply, so the work never gets prioritised."
@@ -0,0 +1,78 @@
1
+ id: technique-presentation-integrity
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Presentation Integrity — Does the Screen Show What the Service Sent?"
5
+ description: >
6
+ A correct response can still reach the user as a wrong number. Formatting helpers that
7
+ inspect a value to guess what it is, units applied twice, rounding that crosses a
8
+ threshold, truncation presented as a total — each corrupts a payload that was right when
9
+ it left the service. The defect is invisible from any API tool and invisible from the
10
+ screen alone; it appears only when the raw response is held next to what is rendered. It
11
+ also gets attributed to the wrong change more often than any other kind.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from paired UI and API sessions on the same features, 2026-07 to 2026-09"
14
+ tags: [ui, api, data-integrity, verification, differential, calculations, regression, error-handling]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ For every value on screen, open the raw response beside it and compare. Probe any
23
+ formatter that branches on the value itself by feeding it inputs on both sides of the
24
+ threshold it tests. When they differ, work out which change owns the corruption before
25
+ filing it — it is frequently not the change under test.
26
+
27
+ core_principle: >
28
+ "The API is correct" and "the user sees the correct value" are two verdicts. Proving
29
+ the first says nothing about the second, and a session that only ever looks at one
30
+ surface cannot tell which half is broken.
31
+
32
+ the_formatter_that_guesses:
33
+ shape: >
34
+ A helper that inspects the value to decide what kind of value it is — the classic
35
+ being a ratio-or-percentage test such as `value > 1 ? value : value * 100`.
36
+ why_it_survives_review: >
37
+ It is right across most of the range, so it looks correct in every casual check and
38
+ in every test written from the same assumption.
39
+ why_it_is_wrong: >
40
+ The threshold is a guess about data, not a fact about it. Every legitimate value on
41
+ the wrong side of it is rendered wrong — here, every percentage at or below 1 renders
42
+ one hundred times too large.
43
+ how_to_probe_it: >
44
+ Find the threshold in the helper, then obtain a payload with a value just either side
45
+ of it. If no reachable record has one, that is worth reporting on its own: the case
46
+ cannot be verified from the product.
47
+ the_tell: >
48
+ A tile that contradicts itself — a percentage printed above the fraction it was
49
+ derived from, and the two disagree.
50
+
51
+ other_shapes:
52
+ - "A unit or currency applied twice, or not at all."
53
+ - "Rounding that changes the answer rather than the display — a value rounded before a comparison or a threshold."
54
+ - "Truncation presented as a total, so a partial figure reads as the whole."
55
+ - "A field the client silently drops because it does not render it — invisible until a consumer needs it."
56
+ - "Locale and format settings whose output merely happens to match: consistent is not causal. Change the setting and watch the output follow, or record the AC as partial."
57
+ - "An error rendered as a legitimate value — see technique-silent-failure-audit."
58
+
59
+ attributing_it_correctly: >
60
+ A correct payload corrupted on its way to the screen is not the backend ticket's
61
+ defect. It usually belongs to a different commit — frequently one deployed on one
62
+ environment and not another, which is why the same feature can look broken in one place
63
+ and fine in another. Identify the owning change, state plainly that the change under
64
+ test is not at fault, and say which environments are exposed. Reviewers who see a
65
+ contradictory screen reject correct work; that is the business impact, and it is worth
66
+ writing in those terms.
67
+
68
+ reporting:
69
+ - "Show the payload value and the rendered value side by side. That pair is the finding."
70
+ - "Give the range over which the corruption occurs, not just the one example that was noticed."
71
+ - "Name the owning change and the environments it is deployed on."
72
+ - "Say explicitly whether the change under test is implicated, especially when it is not."
73
+
74
+ anti_patterns:
75
+ - "Signing off a value from the screen alone."
76
+ - "Signing off a value from the API alone and assuming the screen follows."
77
+ - "Filing a rendering defect against the backend ticket that happened to surface it."
78
+ - "Accepting a setting-driven format as verified without ever changing the setting."
@@ -0,0 +1,110 @@
1
+ id: technique-release-readiness-verification
2
+ version: "0.5.0"
3
+ type: technique
4
+ name: "Release Readiness Verification — Many Tickets, One Build"
5
+ description: >
6
+ Verifying a release is not a longer ticket verification. The question is what is actually
7
+ in this build and what that makes each ticket's result mean — which is answered once, for
8
+ everything, before any testing starts. Adds a fixed result vocabulary that keeps
9
+ not-testable and not-tested visible, a coverage map with four states including
10
+ code-verified-only, a carry-over section that keeps pre-existing defects out of the
11
+ release's risk picture, and a disposition line with the condition that would change it.
12
+ author: "Qualiow — BE/API verification layer"
13
+ source: "Derived from multi-ticket release verification sessions, 2026-08 to 2026-09"
14
+ tags: [regression, release, deployment, reporting, coverage, risk, acceptance-criteria, verification, environment]
15
+ domains: [all]
16
+ priority: high
17
+ added: "2026-09-03"
18
+ updated: "2026-09-03"
19
+
20
+ content:
21
+ summary: >
22
+ Fingerprint the build once, list per ticket whether its commit is genuinely deployed,
23
+ and reclassify the session before touching anything. Then give every ticket a result
24
+ from a fixed vocabulary, keep a coverage map that admits what it did not do, separate
25
+ carry-overs, and end with a disposition and the condition that would reverse it.
26
+
27
+ core_principle: >
28
+ A result is only meaningful once you know what was running when you took it. Establish
29
+ that first, for the whole set, or half the session's conclusions will be about a build
30
+ that does not contain the change.
31
+
32
+ step_one_the_build_table: >
33
+ For every ticket, run the ancestry check against the deployed commit
34
+ (`git merge-base --is-ancestor`). A ticket whose backend is not deployed is NOT
35
+ TESTABLE here — not a failure. Testing its UI anyway produces a convincing and
36
+ meaningless result: the frontend ships the feature, the API does not answer, and it
37
+ renders as dashes, zeros or blanks that look exactly like a data defect and get filed
38
+ as one.
39
+
40
+ result_vocabulary:
41
+ - result: "Clean pass"
42
+ means: "Every AC met, with evidence."
43
+ - result: "Pass with caveats"
44
+ means: "The feature works; named ACs remain unproven, or a defect is open against it."
45
+ - result: "Fail"
46
+ means: "An AC is not met in this build."
47
+ - result: "Not testable here"
48
+ means: "The dependency is not deployed or not selected in this environment."
49
+ - result: "Not tested"
50
+ means: "In scope, not reached. Say so — never let it disappear into a summary."
51
+
52
+ coverage_map_states:
53
+ - state: "tested and passing"
54
+ means: "Observed, with evidence."
55
+ - state: "partially tested"
56
+ means: "Exercised, but a named part of the AC was not."
57
+ - state: "not tested"
58
+ means: "In scope, never reached."
59
+ - state: "code-verified only"
60
+ means: >
61
+ Read in the source and believed. This is UNVERIFIABLE, not a pass — mark it
62
+ distinctly so it cannot be skimmed as green.
63
+
64
+ two_pedantic_distinctions:
65
+ - distinction: "Consistent is not causal"
66
+ detail: >
67
+ A setting whose current value happens to match the output proves nothing. Change
68
+ the setting and watch the output follow, or record the AC as partial.
69
+ - distinction: "The data has to reach the case"
70
+ detail: >
71
+ "Negative values render in the negative colour" cannot be verified against records
72
+ that are all zero. If no reachable record exercises the AC, it is untested — find
73
+ one, create one, or say plainly it is unproven and name what data would settle it.
74
+
75
+ name_the_biggest_untested_item: >
76
+ One sentence identifying which gap carries the most risk. In a report full of green
77
+ rows it is the line the reader actually acts on, and it is the one most often left out.
78
+
79
+ carry_overs: >
80
+ Defects that pre-date the changes under test get their own section — pre-existing, not
81
+ caused by these tickets, tracked separately. Mixing them into the release's results
82
+ inflates the apparent risk of shipping and buries the findings that belong to it. The
83
+ reverse also holds: a defect that appears only because two changes met in this build
84
+ belongs to the release even though neither ticket owns it alone.
85
+
86
+ a_deploy_landing_mid_session:
87
+ - "Re-fingerprint immediately and record both build ids."
88
+ - "Keep pre-deploy findings in their own file — they are the evidence for what the previous build did."
89
+ - "Re-run the affected checks and state which build each result refers to."
90
+ - "Never merge pre- and post-deploy results into one table."
91
+
92
+ baselines: >
93
+ When a ticket's effect is a change in a number, take the before figure from the same
94
+ place as the after figure and state the tolerance — a few per cent on a large live
95
+ collection is churn, not a regression. Where a system-wide figure and a per-user figure
96
+ disagree, name which one describes the behaviour the feature actually changed.
97
+
98
+ the_disposition: >
99
+ Close with ship / ship-with-follow-ups / hold, plus the condition that would change your
100
+ mind, plus the scope of what you checked when it is narrower than the question being
101
+ asked. "Feature functionality verified; absolute value accuracy is out of scope and
102
+ validated by the product team" is a recommendation someone can act on. One with an
103
+ unstated scope is one someone will over-read.
104
+
105
+ anti_patterns:
106
+ - "Testing tickets before establishing what is deployed."
107
+ - "Letting 'not tested' vanish between 'passed' and 'failed'."
108
+ - "Reporting a code reading in the same colour as an observation."
109
+ - "Mixing carry-over defects into the release's risk picture."
110
+ - "A recommendation with no stated scope and no stated reversal condition."
@@ -0,0 +1,33 @@
1
+ # Release metadata for v0.5.0
2
+ version: "0.5.0"
3
+ date: "2026-09-03"
4
+ author: "qualiow"
5
+ status: active
6
+
7
+ notes: >
8
+ Payload correctness and release-level verification. v0.4.0 established how to reach an
9
+ endpoint and how to know which build is answering; this release covers what to do with
10
+ the answer once you have it, and how to report a set of tickets rather than one. Derived
11
+ Value Verification recomputes every percentage, total and aggregate from the raw figures
12
+ in the same response — and then says, in the report, that internal consistency is not
13
+ correctness and names the independent oracle that would close the gap. Presentation
14
+ Integrity holds the payload next to the screen, because a correct response can still
15
+ reach the user as a wrong number, and because that defect is routinely attributed to the
16
+ wrong change. Expected Behaviour Specification is the artifact for everything an API
17
+ probe turns up that has no acceptance criterion behind it: observed against expected, the
18
+ reject-or-clamp and error-or-empty decisions made explicit, ranked by what users can
19
+ reach today, and closed with a plain-English reply the person funding the fix can act on.
20
+ Config & Feature-Flag Surface Verification covers the case where the configuration
21
+ mechanism is itself the change. Release Readiness Verification handles many tickets in
22
+ one build: establish what is deployed before testing anything, keep not-testable and
23
+ not-tested visible, mark code-verified-only as UNVERIFIABLE rather than green, separate
24
+ carry-overs, and end with a disposition and the condition that would reverse it.
25
+
26
+ entry_count: 5
27
+
28
+ entries:
29
+ - technique-derived-value-verification
30
+ - technique-presentation-integrity
31
+ - technique-expected-behaviour-specification
32
+ - technique-config-surface-verification
33
+ - technique-release-readiness-verification
@@ -0,0 +1,98 @@
1
+ id: technique-async-callback-contracts
2
+ version: "0.6.0"
3
+ type: technique
4
+ name: "Asynchronous Callback Contracts — What a Webhook Endpoint Owes Its Caller"
5
+ description: >
6
+ A callback endpoint is an API whose caller you do not control, cannot ask to retry, and
7
+ cannot expect to behave. This covers what to probe on the receiving end: the identity it
8
+ keys on, what it does with an event it cannot match, whether it distinguishes acknowledging
9
+ from applying, whether its answer is honest enough for the sender to act on, and whether an
10
+ event that arrives out of order is applied, rejected or silently swallowed.
11
+ author: "Qualiow — BE/API verification layer"
12
+ source: "Derived from a black-box assessment of an asynchronous mobile-money top-up service, 2026-09"
13
+ tags: [webhooks, callbacks, api, backend, contracts, silent-failure, integration, payments]
14
+ domains: [fintech, all]
15
+ priority: high
16
+ added: "2026-09-03"
17
+ updated: "2026-09-03"
18
+
19
+ content:
20
+ summary: >
21
+ Establish four things before anything else: is the endpoint authenticated, what field does
22
+ it key on, what does it do with an event that matches nothing, and can the caller tell the
23
+ difference between accepted, duplicate and discarded. Everything else about the integration
24
+ is downstream of those answers.
25
+
26
+ core_principle: >
27
+ The endpoint is the entire contract. There is no client in front of it, no form validation,
28
+ no retry the receiver controls. Judge every case on the consequence of the call succeeding,
29
+ never on how hard it was to make.
30
+
31
+ first_question_authentication: >
32
+ Probe the callback endpoint with no credential and with a wrong one, before anything else.
33
+ A callback endpoint that accepts unauthenticated requests is a "credit any account" endpoint
34
+ on a payments platform, and every other finding is secondary to it. It is also the single
35
+ most commonly unprotected route in an integration, because it is written to be called by
36
+ someone else's system and is easy to think of as inbound plumbing rather than as an API.
37
+ When it IS protected, say so explicitly and prominently — it is worth as much to the reader
38
+ as a defect.
39
+
40
+ second_question_the_dedup_key: >
41
+ The documentation says "duplicates are deduplicated" and never says duplicate of what. Find
42
+ out empirically by varying one field at a time against a settled record: same everything;
43
+ same reference with a new receipt; same receipt against a different reference; same reference
44
+ with a different amount. The field it actually keys on is frequently not the one that is
45
+ unique in the sender's system. A receipt that is globally unique upstream but keyed on
46
+ locally, or a reference that is unique locally but reused upstream, both produce silent
47
+ misapplication.
48
+
49
+ third_question_the_unmatched_event: >
50
+ Send an event whose identifier matches nothing — well-formed, then malformed. Three answers
51
+ are possible and they are not equally good:
52
+ a 4xx that makes a retrying sender retry forever against a record that will never exist;
53
+ a 200 that claims it was applied, which is a broken error contract; and a 200 that is
54
+ explicitly distinguishable, such as an "ignored" flag, which is usually right for a sender
55
+ that does not retry. Whichever it does, the question that follows is the one you cannot see
56
+ from outside — does anything count or alert on it? An unmatched callback for a real payment
57
+ means money collected and never credited, and if nothing counts them the platform learns
58
+ from the customer. Report that as a suspected finding about an absence and say plainly that
59
+ it is not observable through the API.
60
+
61
+ fourth_question_the_honest_acknowledgement: >
62
+ The response body has to let the sender distinguish applied, already-applied and discarded.
63
+ Collapsing them into one 200 is a silent-failure shape: it means no sender, no monitor and
64
+ no engineer can tell a working integration from a broken one. Check what a response says
65
+ when the event applied to MORE records than intended, too — a body listing the records it
66
+ touched is the cheapest possible detection for an identity collision, and a body that just
67
+ says "ok" hides it completely.
68
+
69
+ ordering_and_lateness:
70
+ - "Deliver two events for one record in the wrong order and see which one wins."
71
+ - "Deliver an event after the record has been closed by a timeout or reconciliation job."
72
+ - "Deliver two contradicting events simultaneously and see whether the result is stable."
73
+ - >
74
+ Ask, for each: what was the customer or the downstream system told at the moment of the
75
+ first event? Every out-of-order defect gets its severity from what had already been
76
+ promised, not from the record's final state.
77
+
78
+ simulating_the_provider: >
79
+ Where the system under test offers provider behaviour profiles — healthy, silent, duplicate,
80
+ late, contradictory — those are the specification of what the integration is supposed to
81
+ survive, and they should be exercised one by one and named in the coverage map. Where it does
82
+ not offer them, the equivalent is a mode where the platform schedules nothing and the tester
83
+ sends every event by hand: that mode is worth more than all the others combined, because it
84
+ is the only one that gives deterministic control of timing, ordering and concurrency. Reach
85
+ for it by default, and use a real profile only when the provider's own behaviour is the
86
+ subject of the test.
87
+
88
+ common_mistakes:
89
+ - "Testing the callback endpoint only through the happy path the platform itself triggers."
90
+ - "Assuming the documented dedup key is the implemented one."
91
+ - "Reading a 200 as success without checking what the body says was actually done."
92
+ - "Treating an unmatchable event as a low-value case; it is how collected money goes uncredited."
93
+ - "Reporting 'the endpoint is authenticated' nowhere, so the reader cannot tell it was checked."
94
+
95
+ relationship_to_other_entries: >
96
+ The receiving half of technique-exactly-once-verification, which covers the ledger effects.
97
+ Shares its result-reading vocabulary with technique-silent-failure-audit, and its
98
+ identity-and-scope discipline with technique-authenticated-api-probing.