qualiow-exploratory-testing 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -0
- package/bin/mcli +16 -0
- package/bin/mobile-cli.mjs +573 -0
- package/bin/wadb +6 -0
- package/bin/wk-ios +58 -0
- package/bin/wkeval.mjs +44 -0
- package/data/domains/_default.md +181 -0
- package/data/domains/ecommerce.md +192 -0
- package/data/domains/fintech.md +200 -0
- package/data/domains/identity.md +88 -0
- package/data/domains/identity.yml +138 -0
- package/data/domains/marketing.md +194 -0
- package/data/domains/saas.md +197 -0
- package/data/knowledge/changelog.yml +86 -0
- package/data/knowledge/learned-patterns.md +172 -1
- package/data/knowledge/manifest.yml +210 -4
- package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
- package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
- package/data/knowledge/releases/v0.2.0/release.yml +20 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
- package/data/knowledge/releases/v0.3.0/release.yml +25 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
- package/data/knowledge/releases/v0.4.0/release.yml +28 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
- package/data/knowledge/releases/v0.5.0/release.yml +33 -0
- package/data/targets/_example-backend.yml +123 -0
- package/data/targets/_example-mobile-emulation.yml +52 -0
- package/data/targets/_example-native-mobile.yml +61 -0
- package/data/targets/_example-sim-android-chrome.yml +81 -0
- package/data/targets/_example-sim-ios-safari.yml +80 -0
- package/data/templates/api-probe-matrix.md +99 -0
- package/data/templates/coverage-map.md +14 -0
- package/data/templates/expected-behaviour.md +93 -0
- package/dist/cli/index.js +137 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +350 -527
- package/dist/index.js +144 -5
- package/dist/index.js.map +1 -1
- package/docs/BACKEND-VERIFICATION.md +277 -0
- package/docs/MOBILE-SETUP.md +92 -0
- package/package.json +26 -12
- package/scripts/doctor-mobile.sh +143 -0
- package/scripts/setup-mobile.sh +213 -0
- package/skills/qa-explore/phases/00-setup.md +8 -1
- package/skills/qa-explore/phases/01-auth.md +4 -0
- package/skills/qa-explore-mobile/SKILL.md +146 -0
- package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
- package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
- package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
- package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
- package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
- package/skills/qa-explore-mobile/phases/05-features.md +72 -0
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
- package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
- package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
- package/skills/qa-explore-quick/SKILL.md +6 -0
- package/skills/qa-verify-backend/SKILL.md +163 -0
- package/skills/qa-verify-backend/phases/00-setup.md +97 -0
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
- package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
- package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
- package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
- package/skills/qa-verify-backend/references/api-probes.md +141 -0
- package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
- package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
- package/skills/qa-verify-backend/references/payload-verification.md +112 -0
- package/skills/qa-verify-backend/references/release-readiness.md +97 -0
- package/skills/qa-verify-backend/references/safety-rules.md +90 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
id: technique-config-surface-verification
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
type: technique
|
|
4
|
+
name: "Config & Feature-Flag Surface Verification"
|
|
5
|
+
description: >
|
|
6
|
+
When the change under test is the configuration mechanism itself — a feature-flag
|
|
7
|
+
endpoint, a settings surface, a shared config library — the thing to verify is not only
|
|
8
|
+
that it answers, but that it answers with the deployed values, exposes nothing it should
|
|
9
|
+
not, is genuinely the single source of truth for the behaviour it advertises, and is
|
|
10
|
+
reachable by the same route in every environment it will be promoted to.
|
|
11
|
+
author: "Qualiow — BE/API verification layer"
|
|
12
|
+
source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
|
|
13
|
+
tags: [backend, api, feature-flags, environment, security, configuration, deployment, verification]
|
|
14
|
+
domains: [all]
|
|
15
|
+
priority: medium
|
|
16
|
+
added: "2026-09-03"
|
|
17
|
+
updated: "2026-09-03"
|
|
18
|
+
|
|
19
|
+
content:
|
|
20
|
+
summary: >
|
|
21
|
+
Call the config surface authenticated and unauthenticated. Compare every value against
|
|
22
|
+
the environment's own deployment configuration. Check the response carries nothing but
|
|
23
|
+
what it advertises. Then check that the code actually reads the flag through that
|
|
24
|
+
surface rather than reaching around it — and that the route exists in the environments
|
|
25
|
+
the change will be promoted to.
|
|
26
|
+
|
|
27
|
+
core_principle: >
|
|
28
|
+
A config surface that reports a value the runtime does not use is worse than no config
|
|
29
|
+
surface: it is a source of confident, wrong answers, and every later verification built
|
|
30
|
+
on it inherits the error.
|
|
31
|
+
|
|
32
|
+
the_checks:
|
|
33
|
+
- check: "Values match the deployed configuration"
|
|
34
|
+
how: >
|
|
35
|
+
Compare each returned value against the environment's own deployment config file or
|
|
36
|
+
equivalent — not the default in the source. A mismatch means the surface and the
|
|
37
|
+
runtime disagree, which invalidates every conclusion drawn from either.
|
|
38
|
+
- check: "Nothing else leaks"
|
|
39
|
+
how: >
|
|
40
|
+
Read the whole response body. A config or flag endpoint that also returns
|
|
41
|
+
connection strings, secrets, internal hostnames or unrelated settings is a
|
|
42
|
+
disclosure finding regardless of the ticket's ACs.
|
|
43
|
+
- check: "Authentication and method"
|
|
44
|
+
how: >
|
|
45
|
+
Call it unauthenticated — it should not answer if it is not meant to. Call it with
|
|
46
|
+
the wrong verbs and expect 405. Confirm the response content type is what callers
|
|
47
|
+
will parse, and that an unauthenticated call does not silently redirect to a login
|
|
48
|
+
page that returns 200.
|
|
49
|
+
- check: "Single source of truth"
|
|
50
|
+
how: >
|
|
51
|
+
Grep the codebase for direct reads of the same key. A service that reads the
|
|
52
|
+
underlying configuration directly bypasses the interface: the flag appears on the
|
|
53
|
+
surface, but the surface is not authoritative for the behaviour it advertises. This
|
|
54
|
+
is a finding even when every AC passes, and it is exactly the shape that makes a
|
|
55
|
+
later verification session draw a confident wrong conclusion.
|
|
56
|
+
- check: "Route reachability, per environment"
|
|
57
|
+
how: >
|
|
58
|
+
Build a small matrix of path against result and meaning. Then confirm the same
|
|
59
|
+
routing or ingress rule exists in every environment the change will be promoted to
|
|
60
|
+
— a route added in one environment and not the others turns into a production
|
|
61
|
+
incident at promotion time, and it is invisible until then.
|
|
62
|
+
- check: "The flag actually switches something"
|
|
63
|
+
how: >
|
|
64
|
+
With the flag on, confirm the behaviour it selects is the behaviour observed.
|
|
65
|
+
Reporting a flag as true is not evidence that the path it selects is running — see
|
|
66
|
+
technique-environment-fingerprinting.
|
|
67
|
+
|
|
68
|
+
reporting:
|
|
69
|
+
- "Distinguish ACs satisfied by measurement from ACs satisfied by inference. 'The endpoint auto-exposes implemented flags, so the interface must be implemented' is inference — mark it, and say what would confirm it."
|
|
70
|
+
- "Carry the promotion risk forward explicitly: which environments still need the route, the value, or both."
|
|
71
|
+
- "Keep pre-existing findings on the same surface in a separate carry-over section."
|
|
72
|
+
|
|
73
|
+
anti_patterns:
|
|
74
|
+
- "Reading the flag's value from the source default instead of the environment's deployed config."
|
|
75
|
+
- "Accepting a 200 without reading the body — a login redirect answers 200 too."
|
|
76
|
+
- "Verifying the surface and never checking that the runtime reads through it."
|
|
77
|
+
- "Confirming a route in one environment and assuming it exists in the next."
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
id: technique-derived-value-verification
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
type: technique
|
|
4
|
+
name: "Derived Value Verification — Recomputing the Number and Naming What It Does Not Prove"
|
|
5
|
+
description: >
|
|
6
|
+
Percentages, totals, ratios, deltas and aggregates are derived, and they are verified by
|
|
7
|
+
recomputing them from the raw figures in the same response against the formula from the
|
|
8
|
+
specification — never from the code that implements it. The discipline that makes this
|
|
9
|
+
worth anything is the second half: when both sides of the check come from one payload,
|
|
10
|
+
you have verified the derivation and not the inputs. Say so, name the independent oracle
|
|
11
|
+
that would close the gap, and say whether you ran it.
|
|
12
|
+
author: "Qualiow — BE/API verification layer"
|
|
13
|
+
source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
|
|
14
|
+
tags: [data-integrity, api, backend, calculations, verification, evidence, oracles, acceptance-criteria]
|
|
15
|
+
domains: [all]
|
|
16
|
+
priority: high
|
|
17
|
+
added: "2026-09-03"
|
|
18
|
+
updated: "2026-09-03"
|
|
19
|
+
|
|
20
|
+
content:
|
|
21
|
+
summary: >
|
|
22
|
+
Take the formula from the spec, recompute every derived value from the raw figures in
|
|
23
|
+
the same response, and choose cases that stress sign, zero, scale and cardinality. Add
|
|
24
|
+
the structural invariants the values must satisfy regardless of magnitude. Then write
|
|
25
|
+
down, explicitly, that internal consistency is not correctness, and name the second
|
|
26
|
+
source that would settle it.
|
|
27
|
+
|
|
28
|
+
core_principle: >
|
|
29
|
+
A check whose two sides come from the same place can only find an inconsistency, never
|
|
30
|
+
an error. It is still worth running — it is cheap and it catches a real class of bug —
|
|
31
|
+
but reporting it as "the numbers are correct" is a claim the evidence does not support.
|
|
32
|
+
|
|
33
|
+
where_the_formula_comes_from: >
|
|
34
|
+
The specification, the ticket, or the person who owns the definition. A formula read out
|
|
35
|
+
of the implementation cannot falsify that implementation — recomputing the code's own
|
|
36
|
+
arithmetic proves only that arithmetic is deterministic.
|
|
37
|
+
|
|
38
|
+
choosing_the_cases:
|
|
39
|
+
- case: "A negative value"
|
|
40
|
+
why: "Sign handling is where derivation breaks, and a screen full of positive numbers never exercises it."
|
|
41
|
+
- case: "A zero, and a zero denominator"
|
|
42
|
+
why: "Division by zero, and the difference between 0, null and not-applicable."
|
|
43
|
+
- case: "A very large and a very small value"
|
|
44
|
+
why: "Rounding, precision loss and scale errors live at both ends."
|
|
45
|
+
- case: "One item and many items"
|
|
46
|
+
why: >
|
|
47
|
+
An aggregate over a single item should equal that item. It is the cleanest
|
|
48
|
+
apples-to-apples comparison available and removes aggregation ambiguity entirely —
|
|
49
|
+
reach for it first when comparing against another system.
|
|
50
|
+
|
|
51
|
+
structural_invariants: >
|
|
52
|
+
Relationships that must hold regardless of magnitude — a total equal to the sum of its
|
|
53
|
+
parts, one metric bounded by another, a derived cost positive, a percentage inside its
|
|
54
|
+
own range. They cost nothing, they generalise past the cases you happened to pick, and
|
|
55
|
+
a violation is unambiguous.
|
|
56
|
+
|
|
57
|
+
the_limitation_to_write_down: >
|
|
58
|
+
"This validates that the percentage equals the amount divided by the base. It does not
|
|
59
|
+
validate that the amount is itself correct, because both figures come from the same
|
|
60
|
+
payload." Then: "The outstanding oracle is <the other system / report / source of
|
|
61
|
+
truth> and it has not been run." That is a complete, honest, actionable verdict.
|
|
62
|
+
|
|
63
|
+
finding_an_independent_oracle:
|
|
64
|
+
- "A second system that holds the same figure for the same entity."
|
|
65
|
+
- "A report, dashboard or extract the business already trusts."
|
|
66
|
+
- "The source store the value is computed from, queried directly."
|
|
67
|
+
- "A worked example agreed with whoever owns the definition."
|
|
68
|
+
- "Where none exists, say that: 'no independent oracle is available' is itself a finding about the feature."
|
|
69
|
+
|
|
70
|
+
reporting:
|
|
71
|
+
- "Record every computed-vs-returned pair, not the conclusion drawn from them."
|
|
72
|
+
- "Record the formula and where it came from."
|
|
73
|
+
- "Record the invariants checked."
|
|
74
|
+
- "Record what the check does not prove, and the oracle that would."
|
|
75
|
+
- "Scope the verdict: 'internal consistency verified' is not 'values verified'."
|
|
76
|
+
|
|
77
|
+
anti_patterns:
|
|
78
|
+
- "Taking the formula from the code under test."
|
|
79
|
+
- "Checking only positive, mid-range values."
|
|
80
|
+
- "Reporting 'the numbers are correct' from a single-source check."
|
|
81
|
+
- "Comparing an aggregate against a per-item figure without a single-item case to anchor it."
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
id: technique-expected-behaviour-specification
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
type: technique
|
|
4
|
+
name: "Expected Behaviour Specification — Writing the Spec That Should Have Existed"
|
|
5
|
+
description: >
|
|
6
|
+
Most of what an API probe turns up has no acceptance criterion behind it, so there is
|
|
7
|
+
nothing to file the finding against. A bug report says "this is wrong"; this artifact
|
|
8
|
+
says "here is what right looks like" — observed behaviour and expected behaviour side by
|
|
9
|
+
side, per area, with the decisions the fix forces made explicit, ranked by what real
|
|
10
|
+
users can reach today, and closed with a plain-English reply the person funding the fix
|
|
11
|
+
can act on.
|
|
12
|
+
author: "Qualiow — BE/API verification layer"
|
|
13
|
+
source: "Derived from backend and API verification sessions, 2026-07 to 2026-09"
|
|
14
|
+
tags: [acceptance-criteria, api, backend, requirements, reporting, communication, contract, verification]
|
|
15
|
+
domains: [all]
|
|
16
|
+
priority: high
|
|
17
|
+
added: "2026-09-03"
|
|
18
|
+
updated: "2026-09-03"
|
|
19
|
+
|
|
20
|
+
content:
|
|
21
|
+
summary: >
|
|
22
|
+
Group the findings by cause, not by case. For each area write what the system does
|
|
23
|
+
today as evidence, what it should do as numbered falsifiable statements, and the
|
|
24
|
+
decisions the fix forces. Rank by production reachability. Finish with the same content
|
|
25
|
+
in plain English, ready to paste into a ticket or a chat.
|
|
26
|
+
|
|
27
|
+
core_principle: >
|
|
28
|
+
A finding with no acceptance criterion behind it will be argued about instead of fixed.
|
|
29
|
+
Converting a pile of observations into one reviewable specification turns a debate into
|
|
30
|
+
a decision — and the specification is frequently the most valuable thing a session
|
|
31
|
+
produces.
|
|
32
|
+
|
|
33
|
+
structure:
|
|
34
|
+
- part: "Observed"
|
|
35
|
+
content: >
|
|
36
|
+
Exact request, exact status, exact response, and which environments it holds in.
|
|
37
|
+
Behaviour often differs between environments, and that difference is part of the
|
|
38
|
+
finding rather than a distraction from it.
|
|
39
|
+
- part: "Expected"
|
|
40
|
+
content: >
|
|
41
|
+
Numbered, falsifiable statements — each a testable assertion, including the
|
|
42
|
+
boundary values themselves. Not sentiment, not "should handle this gracefully".
|
|
43
|
+
- part: "Decisions this forces"
|
|
44
|
+
content: >
|
|
45
|
+
Made explicit, because leaving them implicit is how one spec becomes three
|
|
46
|
+
implementations.
|
|
47
|
+
- part: "What must not change"
|
|
48
|
+
content: >
|
|
49
|
+
The valid inputs and their current answers, so the fix is verifiable as
|
|
50
|
+
non-breaking.
|
|
51
|
+
|
|
52
|
+
the_recurring_decisions:
|
|
53
|
+
- decision: "Reject, or clamp?"
|
|
54
|
+
answer: >
|
|
55
|
+
Reject. Silently serving a different page size, a truncated list or a corrected
|
|
56
|
+
value than was asked for breaks every caller that trusts the value back — and does
|
|
57
|
+
it invisibly.
|
|
58
|
+
- decision: "An error, or an empty result?"
|
|
59
|
+
answer: >
|
|
60
|
+
An error. A silent zero is indistinguishable from a genuine no-match and will be
|
|
61
|
+
read as data.
|
|
62
|
+
- decision: "Where does validation live?"
|
|
63
|
+
answer: >
|
|
64
|
+
At the layer that covers every implementation of the feature. Not in one of two
|
|
65
|
+
code paths, and never only in the client — the client is not in the path for any
|
|
66
|
+
other caller.
|
|
67
|
+
- decision: "Is the fallback reachable?"
|
|
68
|
+
answer: >
|
|
69
|
+
Say so explicitly. A match-all or default-everything branch reached by accident
|
|
70
|
+
from ordinary input is a different defect from the one that was reported.
|
|
71
|
+
|
|
72
|
+
ranking: >
|
|
73
|
+
Rank by what real users can reach **today**, not by how bad each finding reads. Name
|
|
74
|
+
which one is live in production now and which only appears in an environment nobody
|
|
75
|
+
ships from. The two need very different urgency and the difference is invisible unless
|
|
76
|
+
somebody states it.
|
|
77
|
+
|
|
78
|
+
the_plain_english_reply: >
|
|
79
|
+
The same content with no jargon, written for whoever decides whether to fund the fix:
|
|
80
|
+
what happens in a sentence they can picture; whether it is live and where; what the fix
|
|
81
|
+
is in outline, enough to size and not enough to design; where you agree with their
|
|
82
|
+
reading and precisely where you do not, and why. Close with the one item you would put
|
|
83
|
+
ahead of the rest and the reason — usually that it is the only one users are hitting
|
|
84
|
+
today. Writing this section is what gets the work scheduled.
|
|
85
|
+
|
|
86
|
+
disagreeing_well: >
|
|
87
|
+
When the reporter proposed a fix that would not work — most commonly guarding in the
|
|
88
|
+
client — agree with the part that is right, then say why the guard has to live on the
|
|
89
|
+
server: the cases in question cannot come from the app at all, they come from calling
|
|
90
|
+
the interface directly, and no client-side change can affect them.
|
|
91
|
+
|
|
92
|
+
anti_patterns:
|
|
93
|
+
- "One section per failing input instead of one per cause."
|
|
94
|
+
- "Expected behaviour written as a sentiment rather than an assertion."
|
|
95
|
+
- "Leaving reject-vs-clamp and error-vs-empty to the implementer."
|
|
96
|
+
- "Ranking by severity language instead of by who can reach it today."
|
|
97
|
+
- "Stopping before the plain-English reply, so the work never gets prioritised."
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
id: technique-presentation-integrity
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
type: technique
|
|
4
|
+
name: "Presentation Integrity — Does the Screen Show What the Service Sent?"
|
|
5
|
+
description: >
|
|
6
|
+
A correct response can still reach the user as a wrong number. Formatting helpers that
|
|
7
|
+
inspect a value to guess what it is, units applied twice, rounding that crosses a
|
|
8
|
+
threshold, truncation presented as a total — each corrupts a payload that was right when
|
|
9
|
+
it left the service. The defect is invisible from any API tool and invisible from the
|
|
10
|
+
screen alone; it appears only when the raw response is held next to what is rendered. It
|
|
11
|
+
also gets attributed to the wrong change more often than any other kind.
|
|
12
|
+
author: "Qualiow — BE/API verification layer"
|
|
13
|
+
source: "Derived from paired UI and API sessions on the same features, 2026-07 to 2026-09"
|
|
14
|
+
tags: [ui, api, data-integrity, verification, differential, calculations, regression, error-handling]
|
|
15
|
+
domains: [all]
|
|
16
|
+
priority: high
|
|
17
|
+
added: "2026-09-03"
|
|
18
|
+
updated: "2026-09-03"
|
|
19
|
+
|
|
20
|
+
content:
|
|
21
|
+
summary: >
|
|
22
|
+
For every value on screen, open the raw response beside it and compare. Probe any
|
|
23
|
+
formatter that branches on the value itself by feeding it inputs on both sides of the
|
|
24
|
+
threshold it tests. When they differ, work out which change owns the corruption before
|
|
25
|
+
filing it — it is frequently not the change under test.
|
|
26
|
+
|
|
27
|
+
core_principle: >
|
|
28
|
+
"The API is correct" and "the user sees the correct value" are two verdicts. Proving
|
|
29
|
+
the first says nothing about the second, and a session that only ever looks at one
|
|
30
|
+
surface cannot tell which half is broken.
|
|
31
|
+
|
|
32
|
+
the_formatter_that_guesses:
|
|
33
|
+
shape: >
|
|
34
|
+
A helper that inspects the value to decide what kind of value it is — the classic
|
|
35
|
+
being a ratio-or-percentage test such as `value > 1 ? value : value * 100`.
|
|
36
|
+
why_it_survives_review: >
|
|
37
|
+
It is right across most of the range, so it looks correct in every casual check and
|
|
38
|
+
in every test written from the same assumption.
|
|
39
|
+
why_it_is_wrong: >
|
|
40
|
+
The threshold is a guess about data, not a fact about it. Every legitimate value on
|
|
41
|
+
the wrong side of it is rendered wrong — here, every percentage at or below 1 renders
|
|
42
|
+
one hundred times too large.
|
|
43
|
+
how_to_probe_it: >
|
|
44
|
+
Find the threshold in the helper, then obtain a payload with a value just either side
|
|
45
|
+
of it. If no reachable record has one, that is worth reporting on its own: the case
|
|
46
|
+
cannot be verified from the product.
|
|
47
|
+
the_tell: >
|
|
48
|
+
A tile that contradicts itself — a percentage printed above the fraction it was
|
|
49
|
+
derived from, and the two disagree.
|
|
50
|
+
|
|
51
|
+
other_shapes:
|
|
52
|
+
- "A unit or currency applied twice, or not at all."
|
|
53
|
+
- "Rounding that changes the answer rather than the display — a value rounded before a comparison or a threshold."
|
|
54
|
+
- "Truncation presented as a total, so a partial figure reads as the whole."
|
|
55
|
+
- "A field the client silently drops because it does not render it — invisible until a consumer needs it."
|
|
56
|
+
- "Locale and format settings whose output merely happens to match: consistent is not causal. Change the setting and watch the output follow, or record the AC as partial."
|
|
57
|
+
- "An error rendered as a legitimate value — see technique-silent-failure-audit."
|
|
58
|
+
|
|
59
|
+
attributing_it_correctly: >
|
|
60
|
+
A correct payload corrupted on its way to the screen is not the backend ticket's
|
|
61
|
+
defect. It usually belongs to a different commit — frequently one deployed on one
|
|
62
|
+
environment and not another, which is why the same feature can look broken in one place
|
|
63
|
+
and fine in another. Identify the owning change, state plainly that the change under
|
|
64
|
+
test is not at fault, and say which environments are exposed. Reviewers who see a
|
|
65
|
+
contradictory screen reject correct work; that is the business impact, and it is worth
|
|
66
|
+
writing in those terms.
|
|
67
|
+
|
|
68
|
+
reporting:
|
|
69
|
+
- "Show the payload value and the rendered value side by side. That pair is the finding."
|
|
70
|
+
- "Give the range over which the corruption occurs, not just the one example that was noticed."
|
|
71
|
+
- "Name the owning change and the environments it is deployed on."
|
|
72
|
+
- "Say explicitly whether the change under test is implicated, especially when it is not."
|
|
73
|
+
|
|
74
|
+
anti_patterns:
|
|
75
|
+
- "Signing off a value from the screen alone."
|
|
76
|
+
- "Signing off a value from the API alone and assuming the screen follows."
|
|
77
|
+
- "Filing a rendering defect against the backend ticket that happened to surface it."
|
|
78
|
+
- "Accepting a setting-driven format as verified without ever changing the setting."
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
id: technique-release-readiness-verification
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
type: technique
|
|
4
|
+
name: "Release Readiness Verification — Many Tickets, One Build"
|
|
5
|
+
description: >
|
|
6
|
+
Verifying a release is not a longer ticket verification. The question is what is actually
|
|
7
|
+
in this build and what that makes each ticket's result mean — which is answered once, for
|
|
8
|
+
everything, before any testing starts. Adds a fixed result vocabulary that keeps
|
|
9
|
+
not-testable and not-tested visible, a coverage map with four states including
|
|
10
|
+
code-verified-only, a carry-over section that keeps pre-existing defects out of the
|
|
11
|
+
release's risk picture, and a disposition line with the condition that would change it.
|
|
12
|
+
author: "Qualiow — BE/API verification layer"
|
|
13
|
+
source: "Derived from multi-ticket release verification sessions, 2026-08 to 2026-09"
|
|
14
|
+
tags: [regression, release, deployment, reporting, coverage, risk, acceptance-criteria, verification, environment]
|
|
15
|
+
domains: [all]
|
|
16
|
+
priority: high
|
|
17
|
+
added: "2026-09-03"
|
|
18
|
+
updated: "2026-09-03"
|
|
19
|
+
|
|
20
|
+
content:
|
|
21
|
+
summary: >
|
|
22
|
+
Fingerprint the build once, list per ticket whether its commit is genuinely deployed,
|
|
23
|
+
and reclassify the session before touching anything. Then give every ticket a result
|
|
24
|
+
from a fixed vocabulary, keep a coverage map that admits what it did not do, separate
|
|
25
|
+
carry-overs, and end with a disposition and the condition that would reverse it.
|
|
26
|
+
|
|
27
|
+
core_principle: >
|
|
28
|
+
A result is only meaningful once you know what was running when you took it. Establish
|
|
29
|
+
that first, for the whole set, or half the session's conclusions will be about a build
|
|
30
|
+
that does not contain the change.
|
|
31
|
+
|
|
32
|
+
step_one_the_build_table: >
|
|
33
|
+
For every ticket, run the ancestry check against the deployed commit
|
|
34
|
+
(`git merge-base --is-ancestor`). A ticket whose backend is not deployed is NOT
|
|
35
|
+
TESTABLE here — not a failure. Testing its UI anyway produces a convincing and
|
|
36
|
+
meaningless result: the frontend ships the feature, the API does not answer, and it
|
|
37
|
+
renders as dashes, zeros or blanks that look exactly like a data defect and get filed
|
|
38
|
+
as one.
|
|
39
|
+
|
|
40
|
+
result_vocabulary:
|
|
41
|
+
- result: "Clean pass"
|
|
42
|
+
means: "Every AC met, with evidence."
|
|
43
|
+
- result: "Pass with caveats"
|
|
44
|
+
means: "The feature works; named ACs remain unproven, or a defect is open against it."
|
|
45
|
+
- result: "Fail"
|
|
46
|
+
means: "An AC is not met in this build."
|
|
47
|
+
- result: "Not testable here"
|
|
48
|
+
means: "The dependency is not deployed or not selected in this environment."
|
|
49
|
+
- result: "Not tested"
|
|
50
|
+
means: "In scope, not reached. Say so — never let it disappear into a summary."
|
|
51
|
+
|
|
52
|
+
coverage_map_states:
|
|
53
|
+
- state: "tested and passing"
|
|
54
|
+
means: "Observed, with evidence."
|
|
55
|
+
- state: "partially tested"
|
|
56
|
+
means: "Exercised, but a named part of the AC was not."
|
|
57
|
+
- state: "not tested"
|
|
58
|
+
means: "In scope, never reached."
|
|
59
|
+
- state: "code-verified only"
|
|
60
|
+
means: >
|
|
61
|
+
Read in the source and believed. This is UNVERIFIABLE, not a pass — mark it
|
|
62
|
+
distinctly so it cannot be skimmed as green.
|
|
63
|
+
|
|
64
|
+
two_pedantic_distinctions:
|
|
65
|
+
- distinction: "Consistent is not causal"
|
|
66
|
+
detail: >
|
|
67
|
+
A setting whose current value happens to match the output proves nothing. Change
|
|
68
|
+
the setting and watch the output follow, or record the AC as partial.
|
|
69
|
+
- distinction: "The data has to reach the case"
|
|
70
|
+
detail: >
|
|
71
|
+
"Negative values render in the negative colour" cannot be verified against records
|
|
72
|
+
that are all zero. If no reachable record exercises the AC, it is untested — find
|
|
73
|
+
one, create one, or say plainly it is unproven and name what data would settle it.
|
|
74
|
+
|
|
75
|
+
name_the_biggest_untested_item: >
|
|
76
|
+
One sentence identifying which gap carries the most risk. In a report full of green
|
|
77
|
+
rows it is the line the reader actually acts on, and it is the one most often left out.
|
|
78
|
+
|
|
79
|
+
carry_overs: >
|
|
80
|
+
Defects that pre-date the changes under test get their own section — pre-existing, not
|
|
81
|
+
caused by these tickets, tracked separately. Mixing them into the release's results
|
|
82
|
+
inflates the apparent risk of shipping and buries the findings that belong to it. The
|
|
83
|
+
reverse also holds: a defect that appears only because two changes met in this build
|
|
84
|
+
belongs to the release even though neither ticket owns it alone.
|
|
85
|
+
|
|
86
|
+
a_deploy_landing_mid_session:
|
|
87
|
+
- "Re-fingerprint immediately and record both build ids."
|
|
88
|
+
- "Keep pre-deploy findings in their own file — they are the evidence for what the previous build did."
|
|
89
|
+
- "Re-run the affected checks and state which build each result refers to."
|
|
90
|
+
- "Never merge pre- and post-deploy results into one table."
|
|
91
|
+
|
|
92
|
+
baselines: >
|
|
93
|
+
When a ticket's effect is a change in a number, take the before figure from the same
|
|
94
|
+
place as the after figure and state the tolerance — a few per cent on a large live
|
|
95
|
+
collection is churn, not a regression. Where a system-wide figure and a per-user figure
|
|
96
|
+
disagree, name which one describes the behaviour the feature actually changed.
|
|
97
|
+
|
|
98
|
+
the_disposition: >
|
|
99
|
+
Close with ship / ship-with-follow-ups / hold, plus the condition that would change your
|
|
100
|
+
mind, plus the scope of what you checked when it is narrower than the question being
|
|
101
|
+
asked. "Feature functionality verified; absolute value accuracy is out of scope and
|
|
102
|
+
validated by the product team" is a recommendation someone can act on. One with an
|
|
103
|
+
unstated scope is one someone will over-read.
|
|
104
|
+
|
|
105
|
+
anti_patterns:
|
|
106
|
+
- "Testing tickets before establishing what is deployed."
|
|
107
|
+
- "Letting 'not tested' vanish between 'passed' and 'failed'."
|
|
108
|
+
- "Reporting a code reading in the same colour as an observation."
|
|
109
|
+
- "Mixing carry-over defects into the release's risk picture."
|
|
110
|
+
- "A recommendation with no stated scope and no stated reversal condition."
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Release metadata for v0.5.0
|
|
2
|
+
version: "0.5.0"
|
|
3
|
+
date: "2026-09-03"
|
|
4
|
+
author: "qualiow"
|
|
5
|
+
status: active
|
|
6
|
+
|
|
7
|
+
notes: >
|
|
8
|
+
Payload correctness and release-level verification. v0.4.0 established how to reach an
|
|
9
|
+
endpoint and how to know which build is answering; this release covers what to do with
|
|
10
|
+
the answer once you have it, and how to report a set of tickets rather than one. Derived
|
|
11
|
+
Value Verification recomputes every percentage, total and aggregate from the raw figures
|
|
12
|
+
in the same response — and then says, in the report, that internal consistency is not
|
|
13
|
+
correctness and names the independent oracle that would close the gap. Presentation
|
|
14
|
+
Integrity holds the payload next to the screen, because a correct response can still
|
|
15
|
+
reach the user as a wrong number, and because that defect is routinely attributed to the
|
|
16
|
+
wrong change. Expected Behaviour Specification is the artifact for everything an API
|
|
17
|
+
probe turns up that has no acceptance criterion behind it: observed against expected, the
|
|
18
|
+
reject-or-clamp and error-or-empty decisions made explicit, ranked by what users can
|
|
19
|
+
reach today, and closed with a plain-English reply the person funding the fix can act on.
|
|
20
|
+
Config & Feature-Flag Surface Verification covers the case where the configuration
|
|
21
|
+
mechanism is itself the change. Release Readiness Verification handles many tickets in
|
|
22
|
+
one build: establish what is deployed before testing anything, keep not-testable and
|
|
23
|
+
not-tested visible, mark code-verified-only as UNVERIFIABLE rather than green, separate
|
|
24
|
+
carry-overs, and end with a disposition and the condition that would reverse it.
|
|
25
|
+
|
|
26
|
+
entry_count: 5
|
|
27
|
+
|
|
28
|
+
entries:
|
|
29
|
+
- technique-derived-value-verification
|
|
30
|
+
- technique-presentation-integrity
|
|
31
|
+
- technique-expected-behaviour-specification
|
|
32
|
+
- technique-config-surface-verification
|
|
33
|
+
- technique-release-readiness-verification
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Example target for /qa-verify-backend — copy, rename, and fill in.
|
|
2
|
+
#
|
|
3
|
+
# A backend-verification target is an ordinary web target plus four optional
|
|
4
|
+
# blocks: `backend:` (the cloud surface to probe read-only), `api:` (the service's
|
|
5
|
+
# own HTTP surface), `source:` (the repo and branch to review statically) and
|
|
6
|
+
# `environment:` (what the skill is allowed to do here). All four are optional — a
|
|
7
|
+
# target with none of them still works for /qa-explore.
|
|
8
|
+
#
|
|
9
|
+
# NEVER put credentials in this file. Reference environment variables by NAME
|
|
10
|
+
# and keep the values in .env (gitignored).
|
|
11
|
+
|
|
12
|
+
id: _example-backend
|
|
13
|
+
name: "Example Service — dev environment"
|
|
14
|
+
base_url: "https://example-service-dev.example.com"
|
|
15
|
+
domain: saas # or: identity, ecommerce, fintech, marketing, _default
|
|
16
|
+
|
|
17
|
+
auth:
|
|
18
|
+
strategy: credentials # switch to storage_state once .auth/ is primed
|
|
19
|
+
login_url: "https://example-service-dev.example.com/login" # absolute URL
|
|
20
|
+
state_file: ".auth/_example-backend.json"
|
|
21
|
+
notes: >
|
|
22
|
+
If login needs SSO/MFA that cannot be automated, sign in once by hand, save
|
|
23
|
+
the storage state, then set strategy to storage_state.
|
|
24
|
+
|
|
25
|
+
browser:
|
|
26
|
+
headless: false
|
|
27
|
+
viewport: {width: 1440, height: 900}
|
|
28
|
+
|
|
29
|
+
scope:
|
|
30
|
+
start_pages: ["/"]
|
|
31
|
+
max_depth: 3
|
|
32
|
+
|
|
33
|
+
safety:
|
|
34
|
+
read_only: false # writes allowed — this is a dev environment
|
|
35
|
+
no_form_submit: false
|
|
36
|
+
|
|
37
|
+
# --- What the skill may do here (consumed by /qa-verify-backend) -------------
|
|
38
|
+
environment:
|
|
39
|
+
kind: dev # dev | ephemeral | staging | production
|
|
40
|
+
destroyed_automatically: false # true for short-lived ephemeral envs — capture
|
|
41
|
+
# evidence as you go, it will not be there later
|
|
42
|
+
# kind: production forces read-only probes and skips the end-to-end phase.
|
|
43
|
+
# See .claude/skills/qa-verify-backend/references/safety-rules.md
|
|
44
|
+
|
|
45
|
+
# --- Cloud surface to probe (read-only) --------------------------------------
|
|
46
|
+
backend:
|
|
47
|
+
provider: aws
|
|
48
|
+
aws_profile_env: QA_AWS_PROFILE # NAME of the env var holding the profile
|
|
49
|
+
region_env: QA_AWS_REGION # NAME of the env var holding the region
|
|
50
|
+
account_id: "000000000000" # expected account — probes stop if it differs
|
|
51
|
+
env_suffix: dev
|
|
52
|
+
resources:
|
|
53
|
+
config_table: "example-service-dev-config"
|
|
54
|
+
audit_table: "example-service-dev-audit"
|
|
55
|
+
stream_lambda: "example-service-dev-change-processor"
|
|
56
|
+
stream_lambda_role: "example-service-dev-change-processor-role"
|
|
57
|
+
dlq: "example-service-dev-change-processor-dlq"
|
|
58
|
+
notes: >
|
|
59
|
+
Resource names here are ASSERTIONS, not proof. The skill confirms them with
|
|
60
|
+
`aws dynamodb list-tables` / `aws lambda list-functions` before trusting them —
|
|
61
|
+
a probe against a mistyped name returns an error that reads like a failure.
|
|
62
|
+
|
|
63
|
+
# --- The service's own HTTP surface (probed read-only) -----------------------
|
|
64
|
+
api:
|
|
65
|
+
# base_url: defaults to the target's base_url above when omitted.
|
|
66
|
+
auth: session-cookie # session-cookie | bearer-env | none
|
|
67
|
+
#
|
|
68
|
+
# session-cookie is the default and the one to prefer: the skill runs the
|
|
69
|
+
# request from inside the already-authenticated page, so it inherits the
|
|
70
|
+
# session cookie, the CSRF token and the client's interceptors. Nothing to
|
|
71
|
+
# store, and it works with SSO/MFA that no scripted login can pass.
|
|
72
|
+
#
|
|
73
|
+
# token_env: QA_API_TOKEN # NAME of the env var, for `auth: bearer-env`
|
|
74
|
+
|
|
75
|
+
# Persistent profile holding the logged-in session. Prime it once, headed:
|
|
76
|
+
# playwright-cli -s=<id> open <base_url> --headed --profile .auth/<dir>
|
|
77
|
+
# A profile directory outlives a saved storage_state by a wide margin.
|
|
78
|
+
browser_profile: ".auth/_example-backend-profile"
|
|
79
|
+
|
|
80
|
+
# Used to fingerprint the environment before any verdict is written.
|
|
81
|
+
version_endpoint: "/api/v1/version"
|
|
82
|
+
|
|
83
|
+
endpoints:
|
|
84
|
+
search: "POST /api/v1/search" # note the verb — a GET here returns 405
|
|
85
|
+
item_detail: "GET /api/v1/item/{id}"
|
|
86
|
+
|
|
87
|
+
# The ONLY endpoints the read-only API lane may call without asking. Leave an
|
|
88
|
+
# endpoint off this list if its verb is safe but its effect is not — billed per
|
|
89
|
+
# call, queues a job, or rate-limited into an outage.
|
|
90
|
+
probe_allowlist:
|
|
91
|
+
- "POST /api/v1/search"
|
|
92
|
+
- "GET /api/v1/item/{id}"
|
|
93
|
+
- "GET /api/v1/version"
|
|
94
|
+
|
|
95
|
+
# Run the same case matrix here too, and compare SHAPES — never counts.
|
|
96
|
+
parity_targets: ["_example-backend-staging"]
|
|
97
|
+
|
|
98
|
+
# Flag name -> where its DEPLOYED value is declared for this environment.
|
|
99
|
+
# A flag can select between two implementations of one feature inside a single
|
|
100
|
+
# identical build, which is why "same commit, different behaviour" is a
|
|
101
|
+
# configuration finding rather than deploy lag.
|
|
102
|
+
feature_flags:
|
|
103
|
+
USE_NEW_SEARCH: "deploy/config/example-service-dev.yml"
|
|
104
|
+
|
|
105
|
+
notes: >
|
|
106
|
+
The API lane is read-only in every environment. Writes go through the real
|
|
107
|
+
write path in phase 4, never against production. Response bodies are real
|
|
108
|
+
records — they are redacted before anything is written to disk.
|
|
109
|
+
|
|
110
|
+
# --- Implementation branch to review statically ------------------------------
|
|
111
|
+
source:
|
|
112
|
+
repo_path: "/absolute/path/to/the/service/repo"
|
|
113
|
+
branch: "origin/feature/TICKET-123-short-description"
|
|
114
|
+
base_branch: "origin/main"
|
|
115
|
+
components:
|
|
116
|
+
iac: "infra/modules/audit, infra/modules/service"
|
|
117
|
+
stream_lambda: "services/audit/lambdas"
|
|
118
|
+
write_api: "services/api"
|
|
119
|
+
ui: "apps/console"
|
|
120
|
+
|
|
121
|
+
notes: >
|
|
122
|
+
Ticket under verification and the primary journey — e.g. "change a setting in
|
|
123
|
+
the console, save, and confirm exactly one correct audit record is written."
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Template target — MOBILE-WEB EMULATION via Playwright (desktop engine, mobile device
|
|
2
|
+
# descriptor). This is the /qa-explore path, NOT /qa-explore-mobile — no simulator needed.
|
|
3
|
+
#
|
|
4
|
+
# WHEN TO USE WHICH:
|
|
5
|
+
# - This template (Playwright emulation): fast, supports storage_state auth, full
|
|
6
|
+
# console/network/eval — but the engine is desktop Chromium/WebKit pretending to be
|
|
7
|
+
# a phone. Good for layout, flows, and most functional bugs.
|
|
8
|
+
# - _example-sim-ios-safari.yml / _example-sim-android-chrome.yml (real sim browsers via
|
|
9
|
+
# /qa-explore-mobile): the genuine mobile engines — required for engine-identity
|
|
10
|
+
# behavior (UA/feature detection, real keyboard/viewport behavior, Safari quirks).
|
|
11
|
+
# Playwright's "webkit" is desktop macOS WebKit, NOT iOS Safari.
|
|
12
|
+
id: example-mobile-emulation
|
|
13
|
+
name: "Example Mobile Web App — Playwright Mobile Emulation (WebKit)"
|
|
14
|
+
base_url: "https://staging.m.example.com"
|
|
15
|
+
domain: _default
|
|
16
|
+
auth:
|
|
17
|
+
strategy: storage_state
|
|
18
|
+
state_file: ".auth/example-mobile-emulation.json"
|
|
19
|
+
login_url: "https://staging.m.example.com/login"
|
|
20
|
+
browser:
|
|
21
|
+
headless: false
|
|
22
|
+
engine: webkit # webkit (Safari-like) | chromium (Chrome-like)
|
|
23
|
+
channel: webkit
|
|
24
|
+
device: "iPhone 15" # Playwright device descriptor — sets mobile UA + viewport + touch
|
|
25
|
+
viewport: { width: 390, height: 844 }
|
|
26
|
+
safety:
|
|
27
|
+
read_only: false
|
|
28
|
+
no_form_submit: false
|
|
29
|
+
scope:
|
|
30
|
+
start_pages:
|
|
31
|
+
- "/"
|
|
32
|
+
max_depth: 3
|
|
33
|
+
exclude_patterns:
|
|
34
|
+
- "/auth/*"
|
|
35
|
+
- "/api/*"
|
|
36
|
+
- "**/*.pdf"
|
|
37
|
+
notes: |
|
|
38
|
+
Mobile-web emulation target: Playwright with a mobile device descriptor. Closest proxy
|
|
39
|
+
to real mobile browsers that Playwright offers (caveat: WebKit != Safari — no Apple Pay,
|
|
40
|
+
ITP nuances differ; Chromium != Android Chrome).
|
|
41
|
+
|
|
42
|
+
The device descriptor matters: many mobile hosts redirect desktop UAs to the desktop
|
|
43
|
+
site — viewport resize alone is NOT enough; the descriptor sets the mobile UA. Launch:
|
|
44
|
+
PLAYWRIGHT_MCP_DEVICE="iPhone 15" PLAYWRIGHT_MCP_BROWSER="webkit" playwright-cli ...
|
|
45
|
+
or pass --browser=webkit --device="iPhone 15" explicitly.
|
|
46
|
+
|
|
47
|
+
Create a chromium twin of this config to cross-check engine-specific behavior. WebKit
|
|
48
|
+
and Chromium maintain independent cookie jars — state files are NOT interchangeable
|
|
49
|
+
between engine variants; each needs its own one-time login via /qa-target-setup.
|
|
50
|
+
|
|
51
|
+
For bugs that depend on the real engine identity (browser detection, install prompts,
|
|
52
|
+
engine-specific rendering), escalate to the real-sim targets and /qa-explore-mobile.
|