qualiow-exploratory-testing 2.4.1 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -270,6 +270,7 @@ output/sessions/2026-09-08-1813-explore-parabank/
270
270
  session-report.md # the deliverable
271
271
  evidence-level.md # coverage level, written by `qualiow session level --write`
272
272
  backlog.md # areas not tested, rendered by the same command
273
+ evidence-manifest.json # every evidence file with size and SHA-256, written by finalize
273
274
  stats.json # machine-readable session metrics
274
275
  ```
275
276
 
@@ -281,8 +282,13 @@ Plus, at the top level:
281
282
 
282
283
  Both are written by `qualiow session finalize`, which the session runs as its last step. It
283
284
  checks the session against the output contract first — a confidentiality header on every
284
- markdown file, `stats.json` in the metrics shape, no unredacted secret anywhere — and refuses
285
- to append rows until the violations it lists are fixed. Run it by hand at any time:
285
+ markdown file, `stats.json` in the metrics shape, no unredacted secret anywhere (including the
286
+ literal value of a credential variable such as `QA_PASS`, named by variable, never printed),
287
+ and, when the bug judge ran, every bug's `**Verification:**` line agreeing with its verdict
288
+ file — and refuses to append rows until the violations it lists are fixed. A passing run also
289
+ writes `evidence-manifest.json`: every file under `screenshots/`, `videos/`, `traces/`,
290
+ `logs/` and `evidence/` with its size, SHA-256 and whether the secret scan read it
291
+ (`text-clean`) or could not (`binary-not-scanned`). Run it by hand at any time:
286
292
 
287
293
  ```bash
288
294
  qualiow session finalize latest --check # report violations, write nothing
@@ -420,6 +426,12 @@ QA_API_TOKEN=... # only for a target declaring api.auth: bearer-env
420
426
  EXAMPLE_API_KEY=... # a no-UI service names its own variable; see _example-api-only.yml
421
427
  ```
422
428
 
429
+ A web session types the password with `qualiow auth fill --session <sid> --ref <ref> --env QA_PASS --target <id>`,
430
+ which reads the value itself and fills the field through `playwright-cli`, so the value never
431
+ passes through the model or into the transcript. `finalize` refuses a session that has the
432
+ literal value of a credential variable anywhere in its files. Mobile sessions still type the
433
+ password through the model.
434
+
423
435
  ## 5. Add knowledge
424
436
 
425
437
  The knowledge base drives Claude's testing strategy: 29 entries across releases v0.1.0–v0.6.0,
@@ -9,3 +9,12 @@ export const SECRET_PATTERNS: SecretPattern[];
9
9
  export function findSecretCategories(text: string): string[];
10
10
  /** Number of secret-shaped matches per category in `text`; never the matched text. */
11
11
  export function countSecrets(text: string): Map<string, number>;
12
+ /** Values shorter than this are not matched. Same as `src/utils/secret-values.ts`. */
13
+ export const MIN_SECRET_VALUE_LENGTH: number;
14
+ /** Literal, URL-encoded, JSON-escaped and base64 forms of `value`, longest first. */
15
+ export function secretValueVariants(value: string): string[];
16
+ /** Occurrences of each known value (any variant) in `text`, keyed by env var NAME; never the value. */
17
+ export function countSecretValues(
18
+ text: string,
19
+ values: ReadonlyArray<{ name: string; value: string }>,
20
+ ): Map<string, number>;
@@ -226,3 +226,56 @@ export function findSecretCategories(text) {
226
226
  }
227
227
  return [...found];
228
228
  }
229
+
230
+ // ─── Known values ────────────────────────────────────────────────────
231
+ //
232
+ // The patterns above catch secret-shaped text. A plain password typed into a form
233
+ // has no shape, so the write guard also looks for the literal values of the
234
+ // credential env vars. Same variants and the same minimum length as
235
+ // `src/utils/secret-values.ts` (`tests/unit/write-guard.test.ts` compares them).
236
+
237
+ /** Shorter values are skipped: matching every `1234` in a report would make it unwritable. */
238
+ export const MIN_SECRET_VALUE_LENGTH = 6;
239
+
240
+ /**
241
+ * The forms a value takes on disk: literal, URL-encoded (encodeURIComponent and both form
242
+ * encodings), JSON-escaped, and base64 (standard and URL-safe, padded or not). Longest first.
243
+ */
244
+ export function secretValueVariants(value) {
245
+ const b64 = Buffer.from(value, 'utf-8').toString('base64');
246
+ const b64url = Buffer.from(value, 'utf-8').toString('base64url');
247
+ const variants = new Set([
248
+ value,
249
+ encodeURIComponent(value),
250
+ encodeURIComponent(value).replace(/%20/g, '+'),
251
+ new URLSearchParams({ v: value }).toString().slice(2),
252
+ JSON.stringify(value).slice(1, -1),
253
+ b64,
254
+ b64.replace(/=+$/, ''),
255
+ b64url,
256
+ ]);
257
+ return [...variants]
258
+ .filter((v) => v.length >= MIN_SECRET_VALUE_LENGTH)
259
+ .sort((a, b) => b.length - a.length);
260
+ }
261
+
262
+ /**
263
+ * Occurrences of each known value (any variant) in `text`, keyed by the env var NAME.
264
+ * `values` is `[{ name, value }]`; the result never carries a value.
265
+ */
266
+ export function countSecretValues(text, values) {
267
+ const counts = new Map();
268
+ if (!text) return counts;
269
+ for (const { name, value } of values) {
270
+ let n = 0;
271
+ for (const variant of secretValueVariants(value)) {
272
+ let at = text.indexOf(variant);
273
+ while (at !== -1) {
274
+ n += 1;
275
+ at = text.indexOf(variant, at + variant.length);
276
+ }
277
+ }
278
+ if (n > 0) counts.set(name, (counts.get(name) || 0) + n);
279
+ }
280
+ return counts;
281
+ }
@@ -15,19 +15,64 @@
15
15
  * boundary is caught; edits that only remove or keep secrets already there
16
16
  * pass, so a file can be cleaned up one value at a time.
17
17
  *
18
+ * It also denies the literal value of a credential env var — a plain password
19
+ * has no shape the patterns could catch. The values come from `<cwd>/qa/.env`
20
+ * and `<cwd>/.env` (and the hook's own environment, for the same names): every
21
+ * variable whose NAME ends in PASS, PASSWORD, TOKEN, SECRET, API_KEY or _KEY,
22
+ * plus QA_PASS, QA_TOKEN and QA_API_TOKEN, six characters or longer, matched
23
+ * literally and URL-encoded, JSON-escaped or base64-encoded.
24
+ *
18
25
  * Node built-ins only, plus the sibling `secret-patterns.mjs` — a
19
26
  * git-sourced plugin install has neither `dist/` nor `node_modules`. Never
20
27
  * throws and never echoes the matched text back: the denial reason names
21
- * categories only.
28
+ * categories and variable names only.
22
29
  */
23
30
  import { readFileSync, statSync } from 'node:fs';
24
31
  import { resolve } from 'node:path';
25
- import { countSecrets, findSecretCategories } from './secret-patterns.mjs';
32
+ import { parseEnv } from 'node:util';
33
+ import {
34
+ MIN_SECRET_VALUE_LENGTH,
35
+ countSecretValues,
36
+ countSecrets,
37
+ findSecretCategories,
38
+ } from './secret-patterns.mjs';
26
39
 
27
40
  const WRITE_TOOLS = new Set(['Write', 'Edit', 'MultiEdit', 'NotebookEdit']);
28
41
  // Larger files are judged on the inserted text alone.
29
42
  const MAX_POST_EDIT_BYTES = 5 * 1024 * 1024;
30
43
 
44
+ const SECRET_ENV_NAME = /(PASS|PASSWORD|TOKEN|SECRET|API_KEY|_KEY)$/;
45
+ const DEFAULT_SECRET_ENV_NAMES = ['QA_PASS', 'QA_TOKEN', 'QA_API_TOKEN'];
46
+ const MAX_ENV_FILE_BYTES = 256 * 1024;
47
+
48
+ /** `[{ name, value }]` for the credential env vars this project defines. Values stay in memory. */
49
+ function knownSecretValues(cwd) {
50
+ const out = [];
51
+ const seen = new Set();
52
+ const add = (name, value) => {
53
+ if (typeof value !== 'string' || value.length < MIN_SECRET_VALUE_LENGTH || seen.has(value)) return;
54
+ seen.add(value);
55
+ out.push({ name, value });
56
+ };
57
+ const names = new Set(DEFAULT_SECRET_ENV_NAMES);
58
+ for (const file of [resolve(cwd, 'qa', '.env'), resolve(cwd, '.env')]) {
59
+ let parsed;
60
+ try {
61
+ if (statSync(file).size > MAX_ENV_FILE_BYTES) continue;
62
+ parsed = parseEnv(readFileSync(file, 'utf-8'));
63
+ } catch {
64
+ continue;
65
+ }
66
+ for (const [name, value] of Object.entries(parsed)) {
67
+ if (!SECRET_ENV_NAME.test(name) && !DEFAULT_SECRET_ENV_NAMES.includes(name)) continue;
68
+ names.add(name);
69
+ add(name, value);
70
+ }
71
+ }
72
+ for (const name of names) add(name, process.env[name]);
73
+ return out;
74
+ }
75
+
31
76
  function deny(reason) {
32
77
  process.stdout.write(
33
78
  JSON.stringify({
@@ -128,7 +173,28 @@ try {
128
173
  const edited = postEditContent(input.tool_name, toolInput, absPath);
129
174
  if (edited) introducedCategories(edited[0], edited[1]).forEach((c) => found.add(c));
130
175
 
131
- if (found.size > 0) {
176
+ const values = knownSecretValues(cwd);
177
+ const leaked = new Set();
178
+ if (values.length > 0) {
179
+ for (const name of countSecretValues(insertedText(input.tool_name, toolInput), values).keys()) leaked.add(name);
180
+ if (edited) {
181
+ const was = countSecretValues(edited[0], values);
182
+ for (const [name, n] of countSecretValues(edited[1], values)) {
183
+ if (n > (was.get(name) || 0)) leaked.add(name);
184
+ }
185
+ }
186
+ }
187
+
188
+ if (leaked.size > 0) {
189
+ const names = [...leaked].join(', ');
190
+ deny(
191
+ `Refusing to write ${filePath}: it contains the value of ${names}` +
192
+ (found.size > 0 ? ` and ${[...found].join(', ')}` : '') +
193
+ '. Replace it with [REDACTED] before writing; a credential is filled with ' +
194
+ '`qualiow auth fill --session <sid> --ref <ref> --env <NAME>`, never typed. ' +
195
+ 'Set QUALIOW_HOOKS=off to disable.',
196
+ );
197
+ } else if (found.size > 0) {
132
198
  deny(
133
199
  `Refusing to write ${filePath}: it contains ${[...found].join(', ')}. ` +
134
200
  'Redact per security-rules.md ([REDACTED]) before writing; ' +
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "qualiow-exploratory-testing",
3
- "version": "2.4.1",
3
+ "version": "2.5.0",
4
4
  "description": "qualiow — AI-powered exploratory testing skills for Claude Code: web via playwright-cli, iOS/Android simulators via Maestro/mobile-cli, and backend/API acceptance-criteria verification",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -14,19 +14,29 @@ credentials when the login page appears.
14
14
 
15
15
  ## Credentials (adaptive: uses snapshot to find form fields)
16
16
 
17
- Resolve the values named by `auth.credentials.username` / `auth.credentials.password` in the
18
- order from `${CLAUDE_SKILL_DIR}/references/paths.md` (`qa/.env`, then `.env`). Read the value,
19
- pass it to the command, and never echo it into the log or the report.
17
+ `auth.credentials.username` and `auth.credentials.password` name env vars, resolved in the
18
+ order from `${CLAUDE_SKILL_DIR}/references/paths.md` (environment, `qa/.env`, then `.env`).
19
+ The username is not a secret and may be filled as a literal; it is often an email address,
20
+ which the redaction list still replaces in anything written to `output/`. The password is
21
+ **never read and never typed by you**: `qualiow auth fill` reads the variable named by
22
+ `auth.credentials.password`, fills the field through playwright-cli itself and prints the
23
+ result with the value redacted. Pass it the session id, not the `-s=` flag, and the target id:
24
+ it types only a variable the target declares as a login credential (`auth.credentials`,
25
+ `auth.token`, or `QA_USER`/`QA_PASS`/`QA_TOKEN`), so a page that asks for anything else gets
26
+ nothing.
20
27
 
21
28
  ```bash
22
29
  playwright-cli open <login_url>
23
30
  playwright-cli snapshot
24
31
  playwright-cli fill <email_ref> "<username value>"
25
- playwright-cli fill <pass_ref> "<password value>"
32
+ qualiow auth fill --session <sid> --ref <pass_ref> --env <the NAME from auth.credentials.password> --target <target id>
26
33
  playwright-cli click <submit_ref>
27
34
  playwright-cli state-save .auth/<target>.json
28
35
  ```
29
36
 
37
+ Exit 2 means the variable is not set: say which one and stop the credentials path — never
38
+ ask for the value in the conversation.
39
+
30
40
  ## Token (`auth.strategy: token`)
31
41
 
32
42
  Inject the token named by `auth.token` where the app expects it, then reload:
@@ -69,6 +69,22 @@ A screenshot remains sufficient for Medium / Low.
69
69
 
70
70
  ## Write Bug Drafts
71
71
 
72
+ **Before you draft a claim, rule these out.** Each one reads as a defect in a snapshot or a
73
+ single screenshot and is not one; the full entries, with their disqualifiers, are under
74
+ `## UI False Positive Patterns` in `<data>/knowledge/learned-patterns.md`. While the browser
75
+ is still open:
76
+
77
+ - a link that "does nothing" — `tab-list`: it may have opened a new tab;
78
+ - a label "missing" or "cut short" — an inline link splits it into separate nodes;
79
+ - list items "missing" — scroll the infinite-scroll sentinel into view and wait;
80
+ - images "broken" — lazy images below the fold load only once scrolled into view;
81
+ - a "dead" page — a development server compiling the route on its first request; reload;
82
+ - data that "changed on its own" — another session on the same account;
83
+ - an "error page" — a firewall, bot-challenge or rate-limit interstitial in front of the app.
84
+
85
+ A candidate that falls to one of these is not drafted; note it in `session-log.md` as
86
+ `[<timestamp>] [RULED-OUT] <one line> — <pattern>`. The judge is a second check, not the first.
87
+
72
88
  One file per candidate bug, `verification/drafts/BUG-NNN.md`, in exactly the format of
73
89
  `${CLAUDE_SKILL_DIR}/references/output-contract.md` (the template
74
90
  `<data>/templates/bug-report.md` is the same format with guidance) — everything a shipped bug
@@ -208,7 +224,10 @@ under a plugin install) with exactly this prompt:
208
224
  ### 4. Record the verdicts
209
225
 
210
226
  Copy the returned fenced block VERBATIM — confidentiality header first, then the block — to
211
- `verification/VERDICT-NNN.md`. Do not edit or summarise it. If the judge fails, times out or
227
+ `verification/VERDICT-NNN.md`. Do not edit or summarise it, and do not reorder its fields: the
228
+ judge writes its reasoning before its verdict, and a live re-run ends with a
229
+ `REPRO_COMMANDS:` list (one command per line, indented two spaces) that stays in the file
230
+ exactly as returned. If the judge fails, times out or
212
231
  returns no parseable block, write the file yourself with
213
232
  `VERDICT: UNVERIFIED (judge unavailable: <reason>)`. **Fail open, flagged, never silent** —
214
233
  the bug still ships, marked Unverified.
@@ -232,8 +251,11 @@ severity, the business impact and the priority are yours:
232
251
  `**Verification:** Verified` (or `Verified (severity adjusted from <X>)`, or
233
252
  `Unverified (<reason>)`) directly after `**Reproduction rate:**`, and a `## Verification`
234
253
  section after `## Recommended Fix Priority`: verdict with method and confidence, the judge's
235
- repro result in one line, the severity change if any, and
236
- `Full verdict: ../verification/VERDICT-NNN.md`. Accept an adjusted severity unless you can
254
+ repro result in one line, the severity change if any,
255
+ `Full verdict: ../verification/VERDICT-NNN.md`, and — only when the verdict carries a
256
+ `REPRO_COMMANDS:` list — `Replay: ../verification/VERDICT-NNN.md (REPRO_COMMANDS)`, so a
257
+ developer can find the judge's own commands. The commands stay in the verdict file; never
258
+ copy them into the bug. Accept an adjusted severity unless you can
237
259
  say why not; when you keep yours, write `Verified (severity kept at X; judge proposed Y)`
238
260
  and the reason.
239
261
  - **Under `triage-shadow`**, after the verdict is in: in every shipped bug whose triage file
@@ -246,6 +268,16 @@ severity, the business impact and the priority are yours:
246
268
  you can disprove — ship it as `Verified (judge overruled: <reason>)` and say so in
247
269
  `## Verification`.
248
270
 
271
+ `qualiow session finalize` cross-checks every line above against its verdict file and refuses
272
+ the session when they disagree: a `Verified` bug whose `VERDICT-NNN.md` is missing or is not
273
+ CONFIRMED or CONFIRMED-ADJUSTED (unless the line says `judge overruled`), a bug under
274
+ `bugs/refuted/` whose verdict is not the REFUTED or UNREPRODUCIBLE its line says, a `Verified`
275
+ line in a session with no `verification/` directory, and a verdict file that does not parse
276
+ to a known verdict, `METHOD` or `CONFIDENCE`. `Unverified (…)` bugs are not cross-checked. The
277
+ fix is the bug's line or its directory, never a verdict edited to match; a verdict file that
278
+ does not parse is the judge returning no parseable block — rewrite it as step 4 says
279
+ (`VERDICT: UNVERIFIED (judge unavailable: …)`) and ship that bug `Unverified`.
280
+
249
281
  Leave `verification/drafts/` in place once every draft is sorted: it is a working directory,
250
282
  header-checked and secret-scanned like the rest, and nothing reads it for the report. Every
251
283
  file under `bugs/`, `bugs/refuted/` and `verification/` starts with the confidentiality header.
@@ -351,10 +383,16 @@ Recommendations, Reflection and Refuted Findings verbatim, adding the `Verificat
351
383
  column to `## Bugs Found` from each bug file, and copying the body of `evidence-level.md`
352
384
  verbatim into `## Coverage Level` — and then runs `qualiow session finalize`, which
353
385
  validates `stats.json` against the strict schema, checks the confidentiality header on every
354
- artefact, scans the directory against the redaction list, re-checks a contract-2 session and
386
+ artefact, scans the directory against the redaction list and for the literal values of the
387
+ target's credential variables (named by variable, never by value), cross-checks each bug's
388
+ `**Verification:**` line against its verdict file, re-checks a contract-2 session and
355
389
  refuses it while anything is missing or the level is stale, appends the session row to
356
390
  `output/sessions/INDEX.md` and one row per bug to `output/bugs/all-bugs.md`, records the
357
- session metrics and sets `progress.json` to `complete`. It is idempotent.
391
+ session metrics, writes `evidence-manifest.json` and sets `progress.json` to `complete`. It is
392
+ idempotent. `evidence-manifest.json` lists every file under `screenshots/`, `videos/`,
393
+ `traces/`, `logs/` and `evidence/` with its size, SHA-256 and scan result (`text-clean` or
394
+ `binary-not-scanned` — nobody has vouched for the pixels of a screenshot or the contents of a
395
+ trace). The CLI writes it on every passing finalize; never write or edit it by hand.
358
396
 
359
397
  Read its return (at most 8 lines). If it reports a missing notes section or a violation it
360
398
  could not fix, fix that yourself and re-run the check:
@@ -29,13 +29,14 @@ session report, coverage map, AC matrix, expected-behaviour spec):
29
29
  | `evidence/` | explore — any other file an area or bug cites; backend — raw probe output |
30
30
  | `snapshots/*.yml` | raw accessibility trees from `playwright-cli --raw snapshot`; working files, never shipped; the session's top-level `snapshots/` is the only directory excluded from the secrets scan |
31
31
  | `phase-7-notes.md` | phase 7 (backend: phase 5) — the session's own executive summary, coverage rows, observations, areas not tested, recommendations and reflection; the input the report is assembled from |
32
- | `verification/claims/CLAIM-NNN.md`, `verification/VERDICT-NNN.md`, `verification/proposed-patterns.md` | phase 7 verification (explore) — the claim card the `qa-bug-judge` sub-agent sees, its verdict copied verbatim, and the false-positive patterns it proposed; `verification/drafts/BUG-NNN.md` holds each draft until its verdict sorts it and is then left as a working file. All header-first, all secret-scanned. Under `verification.mode: triage-shadow`, also `verification/JEV-NNN.md` / `LAYA-NNN.md` (the advisory triage block) and `.json` (the exact request sent) |
32
+ | `verification/claims/CLAIM-NNN.md`, `verification/VERDICT-NNN.md`, `verification/proposed-patterns.md` | phase 7 verification (explore) — the claim card the `qa-bug-judge` sub-agent sees, its verdict copied verbatim (reasoning before the verdict; a live re-run ends with `REPRO_COMMANDS:`, one command per line indented two spaces, the session id written `-s=<sid>`, no credential), and the false-positive patterns it proposed; `verification/drafts/BUG-NNN.md` holds each draft until its verdict sorts it and is then left as a working file. All header-first, all secret-scanned. Under `verification.mode: triage-shadow`, also `verification/JEV-NNN.md` / `LAYA-NNN.md` (the advisory triage block) and `.json` (the exact request sent) |
33
33
  | `bugs/refuted/BUG-NNN.md` | phase 7 — a candidate the judge REFUTED or found UNREPRODUCIBLE, with the verdict under `## Refutation (Judge)`. Excluded from `bugs_found`, the index rows and `## Bugs Found`; listed only in `## Refuted Findings` |
34
34
  | `bugs/BUG-NNN.md` | phase 7 |
35
35
  | `session-report.md` | the `qa-reporting-agent` sub-agent, from `phase-7-notes.md`, the bugs, `stats.json` and the phase files (quick sessions write it directly) |
36
36
  | `stats.json` | phase 7 |
37
37
  | `evidence-level.md` | contract 2 only — written by `qualiow session level <dir> --write`, never by hand: the coverage level overall and per risk tier, the findings line, the gaps and the "To raise this level" list. Its body is copied verbatim into the report's `## Coverage Level` |
38
38
  | `backlog.md` | contract 2 only — rendered by the same command from the `coverage.areas` entries that are not `tested`; never hand-written. The only part of a session `--continue` carries forward, through `qualiow session continue-check` |
39
+ | `evidence-manifest.json` | every session kind — written by `qualiow session finalize` on each passing run, never by hand: `{ "generated_by": "qualiow session finalize", "files": [{ "path", "size", "sha256", "scan" }] }`, one entry per file under `screenshots/`, `videos/`, `traces/`, `logs/` and `evidence/`, sorted by path. `scan` is `text-clean` (a text file that passed the secret scan) or `binary-not-scanned` (images, video, archives such as a trace — nobody has vouched for their contents). A record of what was there and what was checked, not a signature |
39
40
  | mobile: `logs/BUG-NNN.log` · backend: `ac-matrix.md`, `evidence/`, `probes/`, `expected-behaviour.md` | as produced |
40
41
 
41
42
  The coverage map lives inside `session-report.md`. There is no separate session-level
@@ -95,6 +96,7 @@ or `evidence/` and nowhere else. `snapshots/` and `.auth/` are never evidence.
95
96
  - Judge repro result: <one line>
96
97
  - Severity: <original X → final Y — only when adjusted>
97
98
  - Full verdict: `../verification/VERDICT-NNN.md`
99
+ - Replay: `../verification/VERDICT-NNN.md` (REPRO_COMMANDS) (only when the verdict carries a `REPRO_COMMANDS:` list)
98
100
  - Triage (advisory): <predicted verdict> · <ROUTE> · P(refuted)=<x> — `../verification/JEV-NNN.md` (only under `triage-shadow`)
99
101
  ```
100
102
 
@@ -102,6 +104,24 @@ The `**Verification:**` line and the `## Verification` section are present only
102
104
  session ran the adversarial judge (`/qa-explore` does unless verification is `off`). A bug
103
105
  under `bugs/refuted/` carries `**Verification:** Refuted` or `Unreproducible` and, instead of
104
106
  `## Verification`, a `## Refutation (Judge)` section holding the verdict block verbatim.
107
+ The judge's commands stay in the verdict file; the `Replay:` line points at them and nothing
108
+ copies them into the bug.
109
+
110
+ `qualiow session finalize` cross-checks these lines against `verification/VERDICT-NNN.md`
111
+ (found through the `Full verdict:` link, else by the bug's number) and refuses the session
112
+ when:
113
+
114
+ - a bug in `bugs/` says `Verified…` and its verdict file is missing, or is not CONFIRMED or
115
+ CONFIRMED-ADJUSTED — unless the line says `judge overruled`, which needs the file to exist
116
+ and nothing more;
117
+ - a bug in `bugs/` says `Verified…` and the session has no `verification/` directory;
118
+ - a bug in `bugs/refuted/` says `Refuted` without a REFUTED verdict, or `Unreproducible`
119
+ without an UNREPRODUCIBLE one;
120
+ - any `verification/VERDICT-*.md` does not parse to a known verdict, or carries a `METHOD`
121
+ other than `live-repro` / `evidence-only` or a `CONFIDENCE` other than `high` / `medium` /
122
+ `low`.
123
+
124
+ `Unverified (…)` bugs are not cross-checked: they make the conservative claim.
105
125
 
106
126
  The `**Area:**` line is present only in a contract-2 session (`/qa-explore`,
107
127
  `/qa-explore-quick`): the ID of the charter risk row the bug was found in, or `none`. It never
@@ -273,8 +293,10 @@ removed.
273
293
  Every session ends by appending one row to `output/sessions/INDEX.md` and one row per bug
274
294
  to `output/bugs/all-bugs.md`. Both rows are written by `qualiow session finalize <session-dir>`,
275
295
  which refuses the session — exit 1, naming the file — when an artefact is missing the
276
- confidentiality header, an unredacted secret survives the scan, or `stats.json` does not
277
- conform; for a contract-2 session also when an area, its evidence or a bug's `**Area:**` fails
296
+ confidentiality header, an unredacted secret survives the scan — including the literal value,
297
+ or an encoded form of it, of a credential variable the target names (the violation names the
298
+ variable, never the value) — a bug's `**Verification:**` line disagrees with its verdict file
299
+ (above), or `stats.json` does not conform; for a contract-2 session also when an area, its evidence or a bug's `**Area:**` fails
278
300
  the checks above, or the level is missing or stale ("re-run
279
301
  `qualiow session level <dir> --write`"). The column order (defined once in `paths.md`) is:
280
302
 
@@ -76,12 +76,17 @@ steps, notes and evidence.
76
76
 
77
77
  Never write actual `.env` values, storage-state contents, or session cookies to any output
78
78
  file. Raw probe output and API response bodies are the most common leak; redact them on the
79
- way in.
79
+ way in. Credential values are filled by `qualiow auth fill --session <sid> --ref <ref> --env <NAME>`,
80
+ never typed by the model: the CLI reads the variable, runs the fill itself and redacts the value
81
+ from what it prints, so it never enters a command line, the transcript or a tool result. It
82
+ refuses any variable that is not a login credential the target declares, so a page that asks for
83
+ an API or cloud key gets nothing.
80
84
 
81
85
  Two layers check the same list behind you: `qualiow session finalize` scans every text file in
82
86
  the session directory whatever its extension (binary files and the top-level `snapshots/` are
83
87
  skipped; a symlink leading outside the session, or a text file over 32MB, refuses finalize), and the plugin's write guard applies it before a file under `output/`
84
- is written (`QUALIOW_HOOKS=off` disables the guard). Neither replaces redacting on the way in.
88
+ is written, together with the literal and encoded values of the credential variables in
89
+ `qa/.env` and `.env` (`QUALIOW_HOOKS=off` disables the guard). Neither replaces redacting on the way in.
85
90
 
86
91
  ## Session Isolation
87
92
 
@@ -87,6 +87,18 @@ Based on what's on the page, apply relevant heuristics:
87
87
  - Responsive mobile: `playwright-cli resize 375 667`
88
88
  - Ambiguous element in a repro step: `playwright-cli generate-locator <ref>` for a stable locator
89
89
 
90
+ **Before you write a bug, rule these out** — a quick session has no judge, so this check is the
91
+ only one (full entries: `## UI False Positive Patterns` in the knowledge base's
92
+ `learned-patterns.md`):
93
+
94
+ - a link that "does nothing" — `playwright-cli tab-list`: it may have opened a new tab;
95
+ - a label "missing" or "cut short" — an inline link splits it into separate nodes;
96
+ - list items "missing" — scroll the infinite-scroll sentinel into view and wait;
97
+ - images "broken" — lazy images below the fold load only once scrolled into view;
98
+ - a "dead" page — a development server compiling the route on its first request; reload;
99
+ - data that "changed on its own" — another session on the same account;
100
+ - an "error page" — a firewall, bot-challenge or rate-limit interstitial in front of the app.
101
+
90
102
  For each bug: `playwright-cli screenshot --filename=output/sessions/<session-dir>/screenshots/BUG-NNN.png` and document immediately.
91
103
 
92
104
  When you are done with an area — fully tested or only partly — capture it under its charter ID
@@ -149,9 +161,12 @@ qualiow session finalize output/sessions/<session-dir>
149
161
  ```
150
162
 
151
163
  It validates `stats.json`, checks the confidentiality header on every artefact and scans them
152
- against the redaction list, re-checks contract 2 (a level that is missing or stale after a later
164
+ against the redaction list and for the literal values of the target's credential variables,
165
+ re-checks contract 2 (a level that is missing or stale after a later
153
166
  edit: re-run `qualiow session level <dir> --write`), then appends the session row to `output/sessions/INDEX.md` and one
154
- row per bug to `output/bugs/all-bugs.md` and records the metrics. On exit 1 it names each
167
+ row per bug to `output/bugs/all-bugs.md`, records the metrics and writes
168
+ `evidence-manifest.json` (every evidence file with its size and SHA-256; never write or edit
169
+ it by hand). On exit 1 it names each
155
170
  offending file: fix it and run again (`--check` validates without writing). Resolve the
156
171
  `qualiow` prefix per `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md`.
157
172
 
@@ -5,7 +5,7 @@ description: >
5
5
  walks through login flow creation, auto-detects scope, and saves target config.
6
6
  Use when user says: "setup target", "add target", "configure app", "new target".
7
7
  argument-hint: "<url> [--name <id>] [--shared]"
8
- allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(npx playwright:*), Read, Write, Glob, Grep
8
+ allowed-tools: Bash(playwright-cli:*), Bash(npx playwright-cli:*), Bash(npx playwright:*), Bash(qualiow:*), Bash(npx -y -p qualiow-exploratory-testing qualiow:*), Read, Write, Glob, Grep
9
9
  ---
10
10
 
11
11
  # Target Application Setup
@@ -55,13 +55,20 @@ From the snapshot, detect:
55
55
 
56
56
  **Option A: Storage State (recommended)**
57
57
  1. Ask the user for the env var **names** for username and password (values go in `qa/.env`).
58
- 2. Log in via playwright-cli, reading the values from `qa/.env` (never echo them):
58
+ 2. Log in via playwright-cli. The username may be filled as a literal; the password is filled
59
+ by `qualiow auth fill`, which reads the variable itself so the value never passes through
60
+ you. `--session` takes the `setup-<HHmm>-<name>` id without `-s=`; write `qualiow` as the
61
+ prefix resolved from `${CLAUDE_SKILL_DIR}/../qa-explore/references/paths.md` (CLI):
59
62
  ```bash
60
63
  playwright-cli snapshot # find form fields
61
64
  playwright-cli fill <email_ref> "<username value>"
62
- playwright-cli fill <password_ref> "<password value>"
65
+ qualiow auth fill --session <sid> --ref <password_ref> --env <the password env var NAME> --target <name>
63
66
  playwright-cli click <submit_ref>
64
67
  ```
68
+ `auth fill` types only a login credential the target declares. With names other than
69
+ `QA_USER`/`QA_PASS`, write the target file's `auth.credentials` block (step 5) before logging
70
+ in. Exit 2 means the variable is not set in `qa/.env` yet: ask the user to add it there, never
71
+ to paste the value into the conversation.
65
72
  If the identity provider needs MFA, reopen headed (`playwright-cli open <login_url> --headed`) and let the user complete it by hand; never automate an MFA challenge.
66
73
  3. Verify login succeeded (snapshot shows dashboard/profile).
67
74
  4. Save state: `playwright-cli state-save .auth/<name>.json`