qualiow-exploratory-testing 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +91 -0
- package/agents/qa-bug-judge.md +54 -6
- package/agents/qa-reporting-agent.md +5 -3
- package/data/knowledge/learned-patterns.md +34 -0
- package/dist/cli/index.js +766 -234
- package/dist/index.d.ts +105 -2
- package/dist/index.js +945 -449
- package/docs/GETTING-STARTED.md +14 -2
- package/hooks/scripts/secret-patterns.d.mts +9 -0
- package/hooks/scripts/secret-patterns.mjs +53 -0
- package/hooks/scripts/write-guard.mjs +69 -3
- package/package.json +1 -1
- package/skills/qa-explore/phases/01-auth.md +14 -4
- package/skills/qa-explore/phases/07-reporting.md +43 -5
- package/skills/qa-explore/references/output-contract.md +26 -4
- package/skills/qa-explore/references/security-rules.md +11 -2
- package/skills/qa-explore-quick/SKILL.md +17 -2
- package/skills/qa-target-setup/SKILL.md +10 -3
package/docs/GETTING-STARTED.md
CHANGED
|
@@ -270,6 +270,7 @@ output/sessions/2026-09-08-1813-explore-parabank/
|
|
|
270
270
|
session-report.md # the deliverable
|
|
271
271
|
evidence-level.md # coverage level, written by `qualiow session level --write`
|
|
272
272
|
backlog.md # areas not tested, rendered by the same command
|
|
273
|
+
evidence-manifest.json # every evidence file with size and SHA-256, written by finalize
|
|
273
274
|
stats.json # machine-readable session metrics
|
|
274
275
|
```
|
|
275
276
|
|
|
@@ -281,8 +282,13 @@ Plus, at the top level:
|
|
|
281
282
|
|
|
282
283
|
Both are written by `qualiow session finalize`, which the session runs as its last step. It
|
|
283
284
|
checks the session against the output contract first — a confidentiality header on every
|
|
284
|
-
markdown file, `stats.json` in the metrics shape, no unredacted secret anywhere
|
|
285
|
-
|
|
285
|
+
markdown file, `stats.json` in the metrics shape, no unredacted secret anywhere (including the
|
|
286
|
+
literal value of a credential variable such as `QA_PASS`, named by variable, never printed),
|
|
287
|
+
and, when the bug judge ran, every bug's `**Verification:**` line agreeing with its verdict
|
|
288
|
+
file — and refuses to append rows until the violations it lists are fixed. A passing run also
|
|
289
|
+
writes `evidence-manifest.json`: every file under `screenshots/`, `videos/`, `traces/`,
|
|
290
|
+
`logs/` and `evidence/` with its size, SHA-256 and whether the secret scan read it
|
|
291
|
+
(`text-clean`) or could not (`binary-not-scanned`). Run it by hand at any time:
|
|
286
292
|
|
|
287
293
|
```bash
|
|
288
294
|
qualiow session finalize latest --check # report violations, write nothing
|
|
@@ -420,6 +426,12 @@ QA_API_TOKEN=... # only for a target declaring api.auth: bearer-env
|
|
|
420
426
|
EXAMPLE_API_KEY=... # a no-UI service names its own variable; see _example-api-only.yml
|
|
421
427
|
```
|
|
422
428
|
|
|
429
|
+
A web session types the password with `qualiow auth fill --session <sid> --ref <ref> --env QA_PASS --target <id>`,
|
|
430
|
+
which reads the value itself and fills the field through `playwright-cli`, so the value never
|
|
431
|
+
passes through the model or into the transcript. `finalize` refuses a session that has the
|
|
432
|
+
literal value of a credential variable anywhere in its files. Mobile sessions still type the
|
|
433
|
+
password through the model.
|
|
434
|
+
|
|
423
435
|
## 5. Add knowledge
|
|
424
436
|
|
|
425
437
|
The knowledge base drives Claude's testing strategy: 29 entries across releases v0.1.0–v0.6.0,
|
|
@@ -9,3 +9,12 @@ export const SECRET_PATTERNS: SecretPattern[];
|
|
|
9
9
|
export function findSecretCategories(text: string): string[];
|
|
10
10
|
/** Number of secret-shaped matches per category in `text`; never the matched text. */
|
|
11
11
|
export function countSecrets(text: string): Map<string, number>;
|
|
12
|
+
/** Values shorter than this are not matched. Same as `src/utils/secret-values.ts`. */
|
|
13
|
+
export const MIN_SECRET_VALUE_LENGTH: number;
|
|
14
|
+
/** Literal, URL-encoded, JSON-escaped and base64 forms of `value`, longest first. */
|
|
15
|
+
export function secretValueVariants(value: string): string[];
|
|
16
|
+
/** Occurrences of each known value (any variant) in `text`, keyed by env var NAME; never the value. */
|
|
17
|
+
export function countSecretValues(
|
|
18
|
+
text: string,
|
|
19
|
+
values: ReadonlyArray<{ name: string; value: string }>,
|
|
20
|
+
): Map<string, number>;
|
|
@@ -226,3 +226,56 @@ export function findSecretCategories(text) {
|
|
|
226
226
|
}
|
|
227
227
|
return [...found];
|
|
228
228
|
}
|
|
229
|
+
|
|
230
|
+
// ─── Known values ────────────────────────────────────────────────────
|
|
231
|
+
//
|
|
232
|
+
// The patterns above catch secret-shaped text. A plain password typed into a form
|
|
233
|
+
// has no shape, so the write guard also looks for the literal values of the
|
|
234
|
+
// credential env vars. Same variants and the same minimum length as
|
|
235
|
+
// `src/utils/secret-values.ts` (`tests/unit/write-guard.test.ts` compares them).
|
|
236
|
+
|
|
237
|
+
/** Shorter values are skipped: matching every `1234` in a report would make it unwritable. */
|
|
238
|
+
export const MIN_SECRET_VALUE_LENGTH = 6;
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* The forms a value takes on disk: literal, URL-encoded (encodeURIComponent and both form
|
|
242
|
+
* encodings), JSON-escaped, and base64 (standard and URL-safe, padded or not). Longest first.
|
|
243
|
+
*/
|
|
244
|
+
export function secretValueVariants(value) {
|
|
245
|
+
const b64 = Buffer.from(value, 'utf-8').toString('base64');
|
|
246
|
+
const b64url = Buffer.from(value, 'utf-8').toString('base64url');
|
|
247
|
+
const variants = new Set([
|
|
248
|
+
value,
|
|
249
|
+
encodeURIComponent(value),
|
|
250
|
+
encodeURIComponent(value).replace(/%20/g, '+'),
|
|
251
|
+
new URLSearchParams({ v: value }).toString().slice(2),
|
|
252
|
+
JSON.stringify(value).slice(1, -1),
|
|
253
|
+
b64,
|
|
254
|
+
b64.replace(/=+$/, ''),
|
|
255
|
+
b64url,
|
|
256
|
+
]);
|
|
257
|
+
return [...variants]
|
|
258
|
+
.filter((v) => v.length >= MIN_SECRET_VALUE_LENGTH)
|
|
259
|
+
.sort((a, b) => b.length - a.length);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Occurrences of each known value (any variant) in `text`, keyed by the env var NAME.
|
|
264
|
+
* `values` is `[{ name, value }]`; the result never carries a value.
|
|
265
|
+
*/
|
|
266
|
+
export function countSecretValues(text, values) {
|
|
267
|
+
const counts = new Map();
|
|
268
|
+
if (!text) return counts;
|
|
269
|
+
for (const { name, value } of values) {
|
|
270
|
+
let n = 0;
|
|
271
|
+
for (const variant of secretValueVariants(value)) {
|
|
272
|
+
let at = text.indexOf(variant);
|
|
273
|
+
while (at !== -1) {
|
|
274
|
+
n += 1;
|
|
275
|
+
at = text.indexOf(variant, at + variant.length);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (n > 0) counts.set(name, (counts.get(name) || 0) + n);
|
|
279
|
+
}
|
|
280
|
+
return counts;
|
|
281
|
+
}
|
|
@@ -15,19 +15,64 @@
|
|
|
15
15
|
* boundary is caught; edits that only remove or keep secrets already there
|
|
16
16
|
* pass, so a file can be cleaned up one value at a time.
|
|
17
17
|
*
|
|
18
|
+
* It also denies the literal value of a credential env var — a plain password
|
|
19
|
+
* has no shape the patterns could catch. The values come from `<cwd>/qa/.env`
|
|
20
|
+
* and `<cwd>/.env` (and the hook's own environment, for the same names): every
|
|
21
|
+
* variable whose NAME ends in PASS, PASSWORD, TOKEN, SECRET, API_KEY or _KEY,
|
|
22
|
+
* plus QA_PASS, QA_TOKEN and QA_API_TOKEN, six characters or longer, matched
|
|
23
|
+
* literally and URL-encoded, JSON-escaped or base64-encoded.
|
|
24
|
+
*
|
|
18
25
|
* Node built-ins only, plus the sibling `secret-patterns.mjs` — a
|
|
19
26
|
* git-sourced plugin install has neither `dist/` nor `node_modules`. Never
|
|
20
27
|
* throws and never echoes the matched text back: the denial reason names
|
|
21
|
-
* categories only.
|
|
28
|
+
* categories and variable names only.
|
|
22
29
|
*/
|
|
23
30
|
import { readFileSync, statSync } from 'node:fs';
|
|
24
31
|
import { resolve } from 'node:path';
|
|
25
|
-
import {
|
|
32
|
+
import { parseEnv } from 'node:util';
|
|
33
|
+
import {
|
|
34
|
+
MIN_SECRET_VALUE_LENGTH,
|
|
35
|
+
countSecretValues,
|
|
36
|
+
countSecrets,
|
|
37
|
+
findSecretCategories,
|
|
38
|
+
} from './secret-patterns.mjs';
|
|
26
39
|
|
|
27
40
|
const WRITE_TOOLS = new Set(['Write', 'Edit', 'MultiEdit', 'NotebookEdit']);
|
|
28
41
|
// Larger files are judged on the inserted text alone.
|
|
29
42
|
const MAX_POST_EDIT_BYTES = 5 * 1024 * 1024;
|
|
30
43
|
|
|
44
|
+
const SECRET_ENV_NAME = /(PASS|PASSWORD|TOKEN|SECRET|API_KEY|_KEY)$/;
|
|
45
|
+
const DEFAULT_SECRET_ENV_NAMES = ['QA_PASS', 'QA_TOKEN', 'QA_API_TOKEN'];
|
|
46
|
+
const MAX_ENV_FILE_BYTES = 256 * 1024;
|
|
47
|
+
|
|
48
|
+
/** `[{ name, value }]` for the credential env vars this project defines. Values stay in memory. */
|
|
49
|
+
function knownSecretValues(cwd) {
|
|
50
|
+
const out = [];
|
|
51
|
+
const seen = new Set();
|
|
52
|
+
const add = (name, value) => {
|
|
53
|
+
if (typeof value !== 'string' || value.length < MIN_SECRET_VALUE_LENGTH || seen.has(value)) return;
|
|
54
|
+
seen.add(value);
|
|
55
|
+
out.push({ name, value });
|
|
56
|
+
};
|
|
57
|
+
const names = new Set(DEFAULT_SECRET_ENV_NAMES);
|
|
58
|
+
for (const file of [resolve(cwd, 'qa', '.env'), resolve(cwd, '.env')]) {
|
|
59
|
+
let parsed;
|
|
60
|
+
try {
|
|
61
|
+
if (statSync(file).size > MAX_ENV_FILE_BYTES) continue;
|
|
62
|
+
parsed = parseEnv(readFileSync(file, 'utf-8'));
|
|
63
|
+
} catch {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
for (const [name, value] of Object.entries(parsed)) {
|
|
67
|
+
if (!SECRET_ENV_NAME.test(name) && !DEFAULT_SECRET_ENV_NAMES.includes(name)) continue;
|
|
68
|
+
names.add(name);
|
|
69
|
+
add(name, value);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
for (const name of names) add(name, process.env[name]);
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
|
|
31
76
|
function deny(reason) {
|
|
32
77
|
process.stdout.write(
|
|
33
78
|
JSON.stringify({
|
|
@@ -128,7 +173,28 @@ try {
|
|
|
128
173
|
const edited = postEditContent(input.tool_name, toolInput, absPath);
|
|
129
174
|
if (edited) introducedCategories(edited[0], edited[1]).forEach((c) => found.add(c));
|
|
130
175
|
|
|
131
|
-
|
|
176
|
+
const values = knownSecretValues(cwd);
|
|
177
|
+
const leaked = new Set();
|
|
178
|
+
if (values.length > 0) {
|
|
179
|
+
for (const name of countSecretValues(insertedText(input.tool_name, toolInput), values).keys()) leaked.add(name);
|
|
180
|
+
if (edited) {
|
|
181
|
+
const was = countSecretValues(edited[0], values);
|
|
182
|
+
for (const [name, n] of countSecretValues(edited[1], values)) {
|
|
183
|
+
if (n > (was.get(name) || 0)) leaked.add(name);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (leaked.size > 0) {
|
|
189
|
+
const names = [...leaked].join(', ');
|
|
190
|
+
deny(
|
|
191
|
+
`Refusing to write ${filePath}: it contains the value of ${names}` +
|
|
192
|
+
(found.size > 0 ? ` and ${[...found].join(', ')}` : '') +
|
|
193
|
+
'. Replace it with [REDACTED] before writing; a credential is filled with ' +
|
|
194
|
+
'`qualiow auth fill --session <sid> --ref <ref> --env <NAME>`, never typed. ' +
|
|
195
|
+
'Set QUALIOW_HOOKS=off to disable.',
|
|
196
|
+
);
|
|
197
|
+
} else if (found.size > 0) {
|
|
132
198
|
deny(
|
|
133
199
|
`Refusing to write ${filePath}: it contains ${[...found].join(', ')}. ` +
|
|
134
200
|
'Redact per security-rules.md ([REDACTED]) before writing; ' +
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "qualiow-exploratory-testing",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "qualiow — AI-powered exploratory testing skills for Claude Code: web via playwright-cli, iOS/Android simulators via Maestro/mobile-cli, and backend/API acceptance-criteria verification",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -14,19 +14,29 @@ credentials when the login page appears.
|
|
|
14
14
|
|
|
15
15
|
## Credentials (adaptive: uses snapshot to find form fields)
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
order from `${CLAUDE_SKILL_DIR}/references/paths.md` (`qa/.env`, then `.env`).
|
|
19
|
-
|
|
17
|
+
`auth.credentials.username` and `auth.credentials.password` name env vars, resolved in the
|
|
18
|
+
order from `${CLAUDE_SKILL_DIR}/references/paths.md` (environment, `qa/.env`, then `.env`).
|
|
19
|
+
The username is not a secret and may be filled as a literal; it is often an email address,
|
|
20
|
+
which the redaction list still replaces in anything written to `output/`. The password is
|
|
21
|
+
**never read and never typed by you**: `qualiow auth fill` reads the variable named by
|
|
22
|
+
`auth.credentials.password`, fills the field through playwright-cli itself and prints the
|
|
23
|
+
result with the value redacted. Pass it the session id, not the `-s=` flag, and the target id:
|
|
24
|
+
it types only a variable the target declares as a login credential (`auth.credentials`,
|
|
25
|
+
`auth.token`, or `QA_USER`/`QA_PASS`/`QA_TOKEN`), so a page that asks for anything else gets
|
|
26
|
+
nothing.
|
|
20
27
|
|
|
21
28
|
```bash
|
|
22
29
|
playwright-cli open <login_url>
|
|
23
30
|
playwright-cli snapshot
|
|
24
31
|
playwright-cli fill <email_ref> "<username value>"
|
|
25
|
-
|
|
32
|
+
qualiow auth fill --session <sid> --ref <pass_ref> --env <the NAME from auth.credentials.password> --target <target id>
|
|
26
33
|
playwright-cli click <submit_ref>
|
|
27
34
|
playwright-cli state-save .auth/<target>.json
|
|
28
35
|
```
|
|
29
36
|
|
|
37
|
+
Exit 2 means the variable is not set: say which one and stop the credentials path — never
|
|
38
|
+
ask for the value in the conversation.
|
|
39
|
+
|
|
30
40
|
## Token (`auth.strategy: token`)
|
|
31
41
|
|
|
32
42
|
Inject the token named by `auth.token` where the app expects it, then reload:
|
|
@@ -69,6 +69,22 @@ A screenshot remains sufficient for Medium / Low.
|
|
|
69
69
|
|
|
70
70
|
## Write Bug Drafts
|
|
71
71
|
|
|
72
|
+
**Before you draft a claim, rule these out.** Each one reads as a defect in a snapshot or a
|
|
73
|
+
single screenshot and is not one; the full entries, with their disqualifiers, are under
|
|
74
|
+
`## UI False Positive Patterns` in `<data>/knowledge/learned-patterns.md`. While the browser
|
|
75
|
+
is still open:
|
|
76
|
+
|
|
77
|
+
- a link that "does nothing" — `tab-list`: it may have opened a new tab;
|
|
78
|
+
- a label "missing" or "cut short" — an inline link splits it into separate nodes;
|
|
79
|
+
- list items "missing" — scroll the infinite-scroll sentinel into view and wait;
|
|
80
|
+
- images "broken" — lazy images below the fold load only once scrolled into view;
|
|
81
|
+
- a "dead" page — a development server compiling the route on its first request; reload;
|
|
82
|
+
- data that "changed on its own" — another session on the same account;
|
|
83
|
+
- an "error page" — a firewall, bot-challenge or rate-limit interstitial in front of the app.
|
|
84
|
+
|
|
85
|
+
A candidate that falls to one of these is not drafted; note it in `session-log.md` as
|
|
86
|
+
`[<timestamp>] [RULED-OUT] <one line> — <pattern>`. The judge is a second check, not the first.
|
|
87
|
+
|
|
72
88
|
One file per candidate bug, `verification/drafts/BUG-NNN.md`, in exactly the format of
|
|
73
89
|
`${CLAUDE_SKILL_DIR}/references/output-contract.md` (the template
|
|
74
90
|
`<data>/templates/bug-report.md` is the same format with guidance) — everything a shipped bug
|
|
@@ -208,7 +224,10 @@ under a plugin install) with exactly this prompt:
|
|
|
208
224
|
### 4. Record the verdicts
|
|
209
225
|
|
|
210
226
|
Copy the returned fenced block VERBATIM — confidentiality header first, then the block — to
|
|
211
|
-
`verification/VERDICT-NNN.md`. Do not edit or summarise it
|
|
227
|
+
`verification/VERDICT-NNN.md`. Do not edit or summarise it, and do not reorder its fields: the
|
|
228
|
+
judge writes its reasoning before its verdict, and a live re-run ends with a
|
|
229
|
+
`REPRO_COMMANDS:` list (one command per line, indented two spaces) that stays in the file
|
|
230
|
+
exactly as returned. If the judge fails, times out or
|
|
212
231
|
returns no parseable block, write the file yourself with
|
|
213
232
|
`VERDICT: UNVERIFIED (judge unavailable: <reason>)`. **Fail open, flagged, never silent** —
|
|
214
233
|
the bug still ships, marked Unverified.
|
|
@@ -232,8 +251,11 @@ severity, the business impact and the priority are yours:
|
|
|
232
251
|
`**Verification:** Verified` (or `Verified (severity adjusted from <X>)`, or
|
|
233
252
|
`Unverified (<reason>)`) directly after `**Reproduction rate:**`, and a `## Verification`
|
|
234
253
|
section after `## Recommended Fix Priority`: verdict with method and confidence, the judge's
|
|
235
|
-
repro result in one line, the severity change if any,
|
|
236
|
-
`Full verdict: ../verification/VERDICT-NNN.md
|
|
254
|
+
repro result in one line, the severity change if any,
|
|
255
|
+
`Full verdict: ../verification/VERDICT-NNN.md`, and — only when the verdict carries a
|
|
256
|
+
`REPRO_COMMANDS:` list — `Replay: ../verification/VERDICT-NNN.md (REPRO_COMMANDS)`, so a
|
|
257
|
+
developer can find the judge's own commands. The commands stay in the verdict file; never
|
|
258
|
+
copy them into the bug. Accept an adjusted severity unless you can
|
|
237
259
|
say why not; when you keep yours, write `Verified (severity kept at X; judge proposed Y)`
|
|
238
260
|
and the reason.
|
|
239
261
|
- **Under `triage-shadow`**, after the verdict is in: in every shipped bug whose triage file
|
|
@@ -246,6 +268,16 @@ severity, the business impact and the priority are yours:
|
|
|
246
268
|
you can disprove — ship it as `Verified (judge overruled: <reason>)` and say so in
|
|
247
269
|
`## Verification`.
|
|
248
270
|
|
|
271
|
+
`qualiow session finalize` cross-checks every line above against its verdict file and refuses
|
|
272
|
+
the session when they disagree: a `Verified` bug whose `VERDICT-NNN.md` is missing or is not
|
|
273
|
+
CONFIRMED or CONFIRMED-ADJUSTED (unless the line says `judge overruled`), a bug under
|
|
274
|
+
`bugs/refuted/` whose verdict is not the REFUTED or UNREPRODUCIBLE its line says, a `Verified`
|
|
275
|
+
line in a session with no `verification/` directory, and a verdict file that does not parse
|
|
276
|
+
to a known verdict, `METHOD` or `CONFIDENCE`. `Unverified (…)` bugs are not cross-checked. The
|
|
277
|
+
fix is the bug's line or its directory, never a verdict edited to match; a verdict file that
|
|
278
|
+
does not parse is the judge returning no parseable block — rewrite it as step 4 says
|
|
279
|
+
(`VERDICT: UNVERIFIED (judge unavailable: …)`) and ship that bug `Unverified`.
|
|
280
|
+
|
|
249
281
|
Leave `verification/drafts/` in place once every draft is sorted: it is a working directory,
|
|
250
282
|
header-checked and secret-scanned like the rest, and nothing reads it for the report. Every
|
|
251
283
|
file under `bugs/`, `bugs/refuted/` and `verification/` starts with the confidentiality header.
|
|
@@ -351,10 +383,16 @@ Recommendations, Reflection and Refuted Findings verbatim, adding the `Verificat
|
|
|
351
383
|
column to `## Bugs Found` from each bug file, and copying the body of `evidence-level.md`
|
|
352
384
|
verbatim into `## Coverage Level` — and then runs `qualiow session finalize`, which
|
|
353
385
|
validates `stats.json` against the strict schema, checks the confidentiality header on every
|
|
354
|
-
artefact, scans the directory against the redaction list
|
|
386
|
+
artefact, scans the directory against the redaction list and for the literal values of the
|
|
387
|
+
target's credential variables (named by variable, never by value), cross-checks each bug's
|
|
388
|
+
`**Verification:**` line against its verdict file, re-checks a contract-2 session and
|
|
355
389
|
refuses it while anything is missing or the level is stale, appends the session row to
|
|
356
390
|
`output/sessions/INDEX.md` and one row per bug to `output/bugs/all-bugs.md`, records the
|
|
357
|
-
session metrics and sets `progress.json` to `complete`. It is
|
|
391
|
+
session metrics, writes `evidence-manifest.json` and sets `progress.json` to `complete`. It is
|
|
392
|
+
idempotent. `evidence-manifest.json` lists every file under `screenshots/`, `videos/`,
|
|
393
|
+
`traces/`, `logs/` and `evidence/` with its size, SHA-256 and scan result (`text-clean` or
|
|
394
|
+
`binary-not-scanned` — nobody has vouched for the pixels of a screenshot or the contents of a
|
|
395
|
+
trace). The CLI writes it on every passing finalize; never write or edit it by hand.
|
|
358
396
|
|
|
359
397
|
Read its return (at most 8 lines). If it reports a missing notes section or a violation it
|
|
360
398
|
could not fix, fix that yourself and re-run the check:
|
|
@@ -29,13 +29,14 @@ session report, coverage map, AC matrix, expected-behaviour spec):
|
|
|
29
29
|
| `evidence/` | explore — any other file an area or bug cites; backend — raw probe output |
|
|
30
30
|
| `snapshots/*.yml` | raw accessibility trees from `playwright-cli --raw snapshot`; working files, never shipped; the session's top-level `snapshots/` is the only directory excluded from the secrets scan |
|
|
31
31
|
| `phase-7-notes.md` | phase 7 (backend: phase 5) — the session's own executive summary, coverage rows, observations, areas not tested, recommendations and reflection; the input the report is assembled from |
|
|
32
|
-
| `verification/claims/CLAIM-NNN.md`, `verification/VERDICT-NNN.md`, `verification/proposed-patterns.md` | phase 7 verification (explore) — the claim card the `qa-bug-judge` sub-agent sees, its verdict copied verbatim, and the false-positive patterns it proposed; `verification/drafts/BUG-NNN.md` holds each draft until its verdict sorts it and is then left as a working file. All header-first, all secret-scanned. Under `verification.mode: triage-shadow`, also `verification/JEV-NNN.md` / `LAYA-NNN.md` (the advisory triage block) and `.json` (the exact request sent) |
|
|
32
|
+
| `verification/claims/CLAIM-NNN.md`, `verification/VERDICT-NNN.md`, `verification/proposed-patterns.md` | phase 7 verification (explore) — the claim card the `qa-bug-judge` sub-agent sees, its verdict copied verbatim (reasoning before the verdict; a live re-run ends with `REPRO_COMMANDS:`, one command per line indented two spaces, the session id written `-s=<sid>`, no credential), and the false-positive patterns it proposed; `verification/drafts/BUG-NNN.md` holds each draft until its verdict sorts it and is then left as a working file. All header-first, all secret-scanned. Under `verification.mode: triage-shadow`, also `verification/JEV-NNN.md` / `LAYA-NNN.md` (the advisory triage block) and `.json` (the exact request sent) |
|
|
33
33
|
| `bugs/refuted/BUG-NNN.md` | phase 7 — a candidate the judge REFUTED or found UNREPRODUCIBLE, with the verdict under `## Refutation (Judge)`. Excluded from `bugs_found`, the index rows and `## Bugs Found`; listed only in `## Refuted Findings` |
|
|
34
34
|
| `bugs/BUG-NNN.md` | phase 7 |
|
|
35
35
|
| `session-report.md` | the `qa-reporting-agent` sub-agent, from `phase-7-notes.md`, the bugs, `stats.json` and the phase files (quick sessions write it directly) |
|
|
36
36
|
| `stats.json` | phase 7 |
|
|
37
37
|
| `evidence-level.md` | contract 2 only — written by `qualiow session level <dir> --write`, never by hand: the coverage level overall and per risk tier, the findings line, the gaps and the "To raise this level" list. Its body is copied verbatim into the report's `## Coverage Level` |
|
|
38
38
|
| `backlog.md` | contract 2 only — rendered by the same command from the `coverage.areas` entries that are not `tested`; never hand-written. The only part of a session `--continue` carries forward, through `qualiow session continue-check` |
|
|
39
|
+
| `evidence-manifest.json` | every session kind — written by `qualiow session finalize` on each passing run, never by hand: `{ "generated_by": "qualiow session finalize", "files": [{ "path", "size", "sha256", "scan" }] }`, one entry per file under `screenshots/`, `videos/`, `traces/`, `logs/` and `evidence/`, sorted by path. `scan` is `text-clean` (a text file that passed the secret scan) or `binary-not-scanned` (images, video, archives such as a trace — nobody has vouched for their contents). A record of what was there and what was checked, not a signature |
|
|
39
40
|
| mobile: `logs/BUG-NNN.log` · backend: `ac-matrix.md`, `evidence/`, `probes/`, `expected-behaviour.md` | as produced |
|
|
40
41
|
|
|
41
42
|
The coverage map lives inside `session-report.md`. There is no separate session-level
|
|
@@ -71,7 +72,7 @@ or `evidence/` and nowhere else. `snapshots/` and `.auth/` are never evidence.
|
|
|
71
72
|
<be specific; quote error messages verbatim>
|
|
72
73
|
|
|
73
74
|
## Steps to Reproduce
|
|
74
|
-
1. <exact steps a developer can follow>
|
|
75
|
+
1. <exact steps a developer can follow — a test card by provider and type, never its number>
|
|
75
76
|
|
|
76
77
|
## Business Impact
|
|
77
78
|
- **Revenue impact:** <or "none identified">
|
|
@@ -95,6 +96,7 @@ or `evidence/` and nowhere else. `snapshots/` and `.auth/` are never evidence.
|
|
|
95
96
|
- Judge repro result: <one line>
|
|
96
97
|
- Severity: <original X → final Y — only when adjusted>
|
|
97
98
|
- Full verdict: `../verification/VERDICT-NNN.md`
|
|
99
|
+
- Replay: `../verification/VERDICT-NNN.md` (REPRO_COMMANDS) (only when the verdict carries a `REPRO_COMMANDS:` list)
|
|
98
100
|
- Triage (advisory): <predicted verdict> · <ROUTE> · P(refuted)=<x> — `../verification/JEV-NNN.md` (only under `triage-shadow`)
|
|
99
101
|
```
|
|
100
102
|
|
|
@@ -102,6 +104,24 @@ The `**Verification:**` line and the `## Verification` section are present only
|
|
|
102
104
|
session ran the adversarial judge (`/qa-explore` does unless verification is `off`). A bug
|
|
103
105
|
under `bugs/refuted/` carries `**Verification:** Refuted` or `Unreproducible` and, instead of
|
|
104
106
|
`## Verification`, a `## Refutation (Judge)` section holding the verdict block verbatim.
|
|
107
|
+
The judge's commands stay in the verdict file; the `Replay:` line points at them and nothing
|
|
108
|
+
copies them into the bug.
|
|
109
|
+
|
|
110
|
+
`qualiow session finalize` cross-checks these lines against `verification/VERDICT-NNN.md`
|
|
111
|
+
(found through the `Full verdict:` link, else by the bug's number) and refuses the session
|
|
112
|
+
when:
|
|
113
|
+
|
|
114
|
+
- a bug in `bugs/` says `Verified…` and its verdict file is missing, or is not CONFIRMED or
|
|
115
|
+
CONFIRMED-ADJUSTED — unless the line says `judge overruled`, which needs the file to exist
|
|
116
|
+
and nothing more;
|
|
117
|
+
- a bug in `bugs/` says `Verified…` and the session has no `verification/` directory;
|
|
118
|
+
- a bug in `bugs/refuted/` says `Refuted` without a REFUTED verdict, or `Unreproducible`
|
|
119
|
+
without an UNREPRODUCIBLE one;
|
|
120
|
+
- any `verification/VERDICT-*.md` does not parse to a known verdict, or carries a `METHOD`
|
|
121
|
+
other than `live-repro` / `evidence-only` or a `CONFIDENCE` other than `high` / `medium` /
|
|
122
|
+
`low`.
|
|
123
|
+
|
|
124
|
+
`Unverified (…)` bugs are not cross-checked: they make the conservative claim.
|
|
105
125
|
|
|
106
126
|
The `**Area:**` line is present only in a contract-2 session (`/qa-explore`,
|
|
107
127
|
`/qa-explore-quick`): the ID of the charter risk row the bug was found in, or `none`. It never
|
|
@@ -273,8 +293,10 @@ removed.
|
|
|
273
293
|
Every session ends by appending one row to `output/sessions/INDEX.md` and one row per bug
|
|
274
294
|
to `output/bugs/all-bugs.md`. Both rows are written by `qualiow session finalize <session-dir>`,
|
|
275
295
|
which refuses the session — exit 1, naming the file — when an artefact is missing the
|
|
276
|
-
confidentiality header, an unredacted secret survives the scan
|
|
277
|
-
|
|
296
|
+
confidentiality header, an unredacted secret survives the scan — including the literal value,
|
|
297
|
+
or an encoded form of it, of a credential variable the target names (the violation names the
|
|
298
|
+
variable, never the value) — a bug's `**Verification:**` line disagrees with its verdict file
|
|
299
|
+
(above), or `stats.json` does not conform; for a contract-2 session also when an area, its evidence or a bug's `**Area:**` fails
|
|
278
300
|
the checks above, or the level is missing or stale ("re-run
|
|
279
301
|
`qualiow session level <dir> --write`"). The column order (defined once in `paths.md`) is:
|
|
280
302
|
|
|
@@ -70,14 +70,23 @@ Before writing ANY file to `output/`, scan it and replace every match with `[RED
|
|
|
70
70
|
- US social-security numbers (SSN)
|
|
71
71
|
- card numbers (13–19 digits that pass Luhn)
|
|
72
72
|
|
|
73
|
+
Payment-provider test cards pass Luhn too, and the scan cannot tell them from real ones: name
|
|
74
|
+
a test card by provider and type ("the provider's test Mastercard"), never by number, in
|
|
75
|
+
steps, notes and evidence.
|
|
76
|
+
|
|
73
77
|
Never write actual `.env` values, storage-state contents, or session cookies to any output
|
|
74
78
|
file. Raw probe output and API response bodies are the most common leak; redact them on the
|
|
75
|
-
way in.
|
|
79
|
+
way in. Credential values are filled by `qualiow auth fill --session <sid> --ref <ref> --env <NAME>`,
|
|
80
|
+
never typed by the model: the CLI reads the variable, runs the fill itself and redacts the value
|
|
81
|
+
from what it prints, so it never enters a command line, the transcript or a tool result. It
|
|
82
|
+
refuses any variable that is not a login credential the target declares, so a page that asks for
|
|
83
|
+
an API or cloud key gets nothing.
|
|
76
84
|
|
|
77
85
|
Two layers check the same list behind you: `qualiow session finalize` scans every text file in
|
|
78
86
|
the session directory whatever its extension (binary files and the top-level `snapshots/` are
|
|
79
87
|
skipped; a symlink leading outside the session, or a text file over 32MB, refuses finalize), and the plugin's write guard applies it before a file under `output/`
|
|
80
|
-
is written
|
|
88
|
+
is written, together with the literal and encoded values of the credential variables in
|
|
89
|
+
`qa/.env` and `.env` (`QUALIOW_HOOKS=off` disables the guard). Neither replaces redacting on the way in.
|
|
81
90
|
|
|
82
91
|
## Session Isolation
|
|
83
92
|
|
|
@@ -87,6 +87,18 @@ Based on what's on the page, apply relevant heuristics:
|
|
|
87
87
|
- Responsive mobile: `playwright-cli resize 375 667`
|
|
88
88
|
- Ambiguous element in a repro step: `playwright-cli generate-locator <ref>` for a stable locator
|
|
89
89
|
|
|
90
|
+
**Before you write a bug, rule these out** — a quick session has no judge, so this check is the
|
|
91
|
+
only one (full entries: `## UI False Positive Patterns` in the knowledge base's
|
|
92
|
+
`learned-patterns.md`):
|
|
93
|
+
|
|
94
|
+
- a link that "does nothing" — `playwright-cli tab-list`: it may have opened a new tab;
|
|
95
|
+
- a label "missing" or "cut short" — an inline link splits it into separate nodes;
|
|
96
|
+
- list items "missing" — scroll the infinite-scroll sentinel into view and wait;
|
|
97
|
+
- images "broken" — lazy images below the fold load only once scrolled into view;
|
|
98
|
+
- a "dead" page — a development server compiling the route on its first request; reload;
|
|
99
|
+
- data that "changed on its own" — another session on the same account;
|
|
100
|
+
- an "error page" — a firewall, bot-challenge or rate-limit interstitial in front of the app.
|
|
101
|
+
|
|
90
102
|
For each bug: `playwright-cli screenshot --filename=output/sessions/<session-dir>/screenshots/BUG-NNN.png` and document immediately.
|
|
91
103
|
|
|
92
104
|
When you are done with an area — fully tested or only partly — capture it under its charter ID
|
|
@@ -149,9 +161,12 @@ qualiow session finalize output/sessions/<session-dir>
|
|
|
149
161
|
```
|
|
150
162
|
|
|
151
163
|
It validates `stats.json`, checks the confidentiality header on every artefact and scans them
|
|
152
|
-
against the redaction list
|
|
164
|
+
against the redaction list and for the literal values of the target's credential variables,
|
|
165
|
+
re-checks contract 2 (a level that is missing or stale after a later
|
|
153
166
|
edit: re-run `qualiow session level <dir> --write`), then appends the session row to `output/sessions/INDEX.md` and one
|
|
154
|
-
row per bug to `output/bugs/all-bugs.md
|
|
167
|
+
row per bug to `output/bugs/all-bugs.md`, records the metrics and writes
|
|
168
|
+
`evidence-manifest.json` (every evidence file with its size and SHA-256; never write or edit
|
|
169
|
+
it by hand). On exit 1 it names each
|
|
155
170
|
offending file: fix it and run again (`--check` validates without writing). Resolve the
|
|
156
171
|
`qualiow` prefix per `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`.
|
|
157
172
|
|
|
@@ -5,7 +5,7 @@ description: >
|
|
|
5
5
|
walks through login flow creation, auto-detects scope, and saves target config.
|
|
6
6
|
Use when user says: "setup target", "add target", "configure app", "new target".
|
|
7
7
|
argument-hint: "<url> [--name <id>] [--shared]"
|
|
8
|
-
allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(npx playwright:*), Read, Write, Glob, Grep
|
|
8
|
+
allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(npx playwright:*), Bash(qualiow:*), Bash(npx -y -p qualiow-exploratory-testing qualiow:*), Read, Write, Glob, Grep
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
# Target Application Setup
|
|
@@ -55,13 +55,20 @@ From the snapshot, detect:
|
|
|
55
55
|
|
|
56
56
|
**Option A: Storage State (recommended)**
|
|
57
57
|
1. Ask the user for the env var **names** for username and password (values go in `qa/.env`).
|
|
58
|
-
2. Log in via playwright-cli
|
|
58
|
+
2. Log in via playwright-cli. The username may be filled as a literal; the password is filled
|
|
59
|
+
by `qualiow auth fill`, which reads the variable itself so the value never passes through
|
|
60
|
+
you. `--session` takes the `setup-<HHmm>-<name>` id without `-s=`; write `qualiow` as the
|
|
61
|
+
prefix resolved from `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md` (CLI):
|
|
59
62
|
```bash
|
|
60
63
|
playwright-cli snapshot # find form fields
|
|
61
64
|
playwright-cli fill <email_ref> "<username value>"
|
|
62
|
-
|
|
65
|
+
qualiow auth fill --session <sid> --ref <password_ref> --env <the password env var NAME> --target <name>
|
|
63
66
|
playwright-cli click <submit_ref>
|
|
64
67
|
```
|
|
68
|
+
`auth fill` types only a login credential the target declares. With names other than
|
|
69
|
+
`QA_USER`/`QA_PASS`, write the target file's `auth.credentials` block (step 5) before logging
|
|
70
|
+
in. Exit 2 means the variable is not set in `qa/.env` yet: ask the user to add it there, never
|
|
71
|
+
to paste the value into the conversation.
|
|
65
72
|
If the identity provider needs MFA, reopen headed (`playwright-cli open <login_url> --headed`) and let the user complete it by hand; never automate an MFA challenge.
|
|
66
73
|
3. Verify login succeeded (snapshot shows dashboard/profile).
|
|
67
74
|
4. Save state: `playwright-cli state-save .auth/<name>.json`
|