qualiow-exploratory-testing 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -0
- package/data/domains/_default.md +181 -0
- package/data/domains/ecommerce.md +192 -0
- package/data/domains/fintech.md +200 -0
- package/data/domains/identity.md +88 -0
- package/data/domains/identity.yml +138 -0
- package/data/domains/marketing.md +194 -0
- package/data/domains/saas.md +197 -0
- package/data/knowledge/changelog.yml +86 -0
- package/data/knowledge/learned-patterns.md +172 -86
- package/data/knowledge/manifest.yml +211 -6
- package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
- package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
- package/data/knowledge/releases/v0.2.0/release.yml +20 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
- package/data/knowledge/releases/v0.3.0/release.yml +25 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
- package/data/knowledge/releases/v0.4.0/release.yml +28 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
- package/data/knowledge/releases/v0.5.0/release.yml +33 -0
- package/data/targets/_example-backend.yml +123 -0
- package/data/templates/api-probe-matrix.md +99 -0
- package/data/templates/coverage-map.md +14 -0
- package/data/templates/expected-behaviour.md +93 -0
- package/dist/cli/index.js +79 -70
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +90 -0
- package/dist/index.js +58 -49
- package/dist/index.js.map +1 -1
- package/docs/BACKEND-VERIFICATION.md +277 -0
- package/package.json +5 -3
- package/skills/qa-verify-backend/SKILL.md +163 -0
- package/skills/qa-verify-backend/phases/00-setup.md +97 -0
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
- package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
- package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
- package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
- package/skills/qa-verify-backend/references/api-probes.md +141 -0
- package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
- package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
- package/skills/qa-verify-backend/references/payload-verification.md +112 -0
- package/skills/qa-verify-backend/references/release-readiness.md +97 -0
- package/skills/qa-verify-backend/references/safety-rules.md +90 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Example target for /qa-verify-backend — copy, rename, and fill in.
|
|
2
|
+
#
|
|
3
|
+
# A backend-verification target is an ordinary web target plus four optional
|
|
4
|
+
# blocks: `backend:` (the cloud surface to probe read-only), `api:` (the service's
|
|
5
|
+
# own HTTP surface), `source:` (the repo and branch to review statically) and
|
|
6
|
+
# `environment:` (what the skill is allowed to do here). All four are optional — a
|
|
7
|
+
# target with none of them still works for /qa-explore.
|
|
8
|
+
#
|
|
9
|
+
# NEVER put credentials in this file. Reference environment variables by NAME
|
|
10
|
+
# and keep the values in .env (gitignored).
|
|
11
|
+
|
|
12
|
+
id: _example-backend
|
|
13
|
+
name: "Example Service — dev environment"
|
|
14
|
+
base_url: "https://example-service-dev.example.com"
|
|
15
|
+
domain: saas # or: identity, ecommerce, fintech, marketing, _default
|
|
16
|
+
|
|
17
|
+
auth:
|
|
18
|
+
strategy: credentials # switch to storage_state once .auth/ is primed
|
|
19
|
+
login_url: "https://example-service-dev.example.com/login" # absolute URL
|
|
20
|
+
state_file: ".auth/_example-backend.json"
|
|
21
|
+
notes: >
|
|
22
|
+
If login needs SSO/MFA that cannot be automated, sign in once by hand, save
|
|
23
|
+
the storage state, then set strategy to storage_state.
|
|
24
|
+
|
|
25
|
+
browser:
|
|
26
|
+
headless: false
|
|
27
|
+
viewport: {width: 1440, height: 900}
|
|
28
|
+
|
|
29
|
+
scope:
|
|
30
|
+
start_pages: ["/"]
|
|
31
|
+
max_depth: 3
|
|
32
|
+
|
|
33
|
+
safety:
|
|
34
|
+
read_only: false # writes allowed — this is a dev environment
|
|
35
|
+
no_form_submit: false
|
|
36
|
+
|
|
37
|
+
# --- What the skill may do here (consumed by /qa-verify-backend) -------------
|
|
38
|
+
environment:
|
|
39
|
+
kind: dev # dev | ephemeral | staging | production
|
|
40
|
+
destroyed_automatically: false # true for short-lived ephemeral envs — capture
|
|
41
|
+
# evidence as you go, it will not be there later
|
|
42
|
+
# kind: production forces read-only probes and skips the end-to-end phase.
|
|
43
|
+
# See .claude/skills/qa-verify-backend/references/safety-rules.md
|
|
44
|
+
|
|
45
|
+
# --- Cloud surface to probe (read-only) --------------------------------------
|
|
46
|
+
backend:
|
|
47
|
+
provider: aws
|
|
48
|
+
aws_profile_env: QA_AWS_PROFILE # NAME of the env var holding the profile
|
|
49
|
+
region_env: QA_AWS_REGION # NAME of the env var holding the region
|
|
50
|
+
account_id: "000000000000" # expected account — probes stop if it differs
|
|
51
|
+
env_suffix: dev
|
|
52
|
+
resources:
|
|
53
|
+
config_table: "example-service-dev-config"
|
|
54
|
+
audit_table: "example-service-dev-audit"
|
|
55
|
+
stream_lambda: "example-service-dev-change-processor"
|
|
56
|
+
stream_lambda_role: "example-service-dev-change-processor-role"
|
|
57
|
+
dlq: "example-service-dev-change-processor-dlq"
|
|
58
|
+
notes: >
|
|
59
|
+
Resource names here are ASSERTIONS, not proof. The skill confirms them with
|
|
60
|
+
`aws dynamodb list-tables` / `aws lambda list-functions` before trusting them —
|
|
61
|
+
a probe against a mistyped name returns an error that reads like a failure.
|
|
62
|
+
|
|
63
|
+
# --- The service's own HTTP surface (probed read-only) -----------------------
|
|
64
|
+
api:
|
|
65
|
+
# base_url: defaults to the target's base_url above when omitted.
|
|
66
|
+
auth: session-cookie # session-cookie | bearer-env | none
|
|
67
|
+
#
|
|
68
|
+
# session-cookie is the default and the one to prefer: the skill runs the
|
|
69
|
+
# request from inside the already-authenticated page, so it inherits the
|
|
70
|
+
# session cookie, the CSRF token and the client's interceptors. Nothing to
|
|
71
|
+
# store, and it works with SSO/MFA that no scripted login can pass.
|
|
72
|
+
#
|
|
73
|
+
# token_env: QA_API_TOKEN # NAME of the env var, for `auth: bearer-env`
|
|
74
|
+
|
|
75
|
+
# Persistent profile holding the logged-in session. Prime it once, headed:
|
|
76
|
+
# playwright-cli -s=<id> open <base_url> --headed --profile .auth/<dir>
|
|
77
|
+
# A profile directory outlives a saved storage_state by a wide margin.
|
|
78
|
+
browser_profile: ".auth/_example-backend-profile"
|
|
79
|
+
|
|
80
|
+
# Used to fingerprint the environment before any verdict is written.
|
|
81
|
+
version_endpoint: "/api/v1/version"
|
|
82
|
+
|
|
83
|
+
endpoints:
|
|
84
|
+
search: "POST /api/v1/search" # note the verb — a GET here returns 405
|
|
85
|
+
item_detail: "GET /api/v1/item/{id}"
|
|
86
|
+
|
|
87
|
+
# The ONLY endpoints the read-only API lane may call without asking. Leave an
|
|
88
|
+
# endpoint off this list if its verb is safe but its effect is not — billed per
|
|
89
|
+
# call, queues a job, or rate-limited into an outage.
|
|
90
|
+
probe_allowlist:
|
|
91
|
+
- "POST /api/v1/search"
|
|
92
|
+
- "GET /api/v1/item/{id}"
|
|
93
|
+
- "GET /api/v1/version"
|
|
94
|
+
|
|
95
|
+
# Run the same case matrix here too, and compare SHAPES — never counts.
|
|
96
|
+
parity_targets: ["_example-backend-staging"]
|
|
97
|
+
|
|
98
|
+
# Flag name -> where its DEPLOYED value is declared for this environment.
|
|
99
|
+
# A flag can select between two implementations of one feature inside a single
|
|
100
|
+
# identical build, which is why "same commit, different behaviour" is a
|
|
101
|
+
# configuration finding rather than deploy lag.
|
|
102
|
+
feature_flags:
|
|
103
|
+
USE_NEW_SEARCH: "deploy/config/example-service-dev.yml"
|
|
104
|
+
|
|
105
|
+
notes: >
|
|
106
|
+
The API lane is read-only in every environment. Writes go through the real
|
|
107
|
+
write path in phase 4, never against production. Response bodies are real
|
|
108
|
+
records — they are redacted before anything is written to disk.
|
|
109
|
+
|
|
110
|
+
# --- Implementation branch to review statically ------------------------------
|
|
111
|
+
source:
|
|
112
|
+
repo_path: "/absolute/path/to/the/service/repo"
|
|
113
|
+
branch: "origin/feature/TICKET-123-short-description"
|
|
114
|
+
base_branch: "origin/main"
|
|
115
|
+
components:
|
|
116
|
+
iac: "infra/modules/audit, infra/modules/service"
|
|
117
|
+
stream_lambda: "services/audit/lambdas"
|
|
118
|
+
write_api: "services/api"
|
|
119
|
+
ui: "apps/console"
|
|
120
|
+
|
|
121
|
+
notes: >
|
|
122
|
+
Ticket under verification and the primary journey — e.g. "change a setting in
|
|
123
|
+
the console, save, and confirm exactly one correct audit record is written."
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# API Probe Matrix
|
|
2
|
+
|
|
3
|
+
> CONFIDENTIAL — internal QA artifact. Contains endpoint behaviour and environment
|
|
4
|
+
> configuration. Keep local. Redact response bodies before saving.
|
|
5
|
+
|
|
6
|
+
## Session Reference
|
|
7
|
+
|
|
8
|
+
| Property | Value |
|
|
9
|
+
|-----------------|-----------------------------------------|
|
|
10
|
+
| Session ID | [e.g., SESSION-2026-03-28-001] |
|
|
11
|
+
| Ticket | [ticket id, or "none"] |
|
|
12
|
+
| Endpoint | [`POST /api/v1/…` — note the verb] |
|
|
13
|
+
| Date | [YYYY-MM-DD] |
|
|
14
|
+
| Observation mode| [in-page fetch / out-of-browser request / unauthenticated] |
|
|
15
|
+
| Identity | [role or persona used — never a credential] |
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Environment Fingerprint
|
|
20
|
+
|
|
21
|
+
Fill this in **before** the matrix. Without it the results below are uninterpretable.
|
|
22
|
+
|
|
23
|
+
| Component | Environment A `[name]` | Environment B `[name]` |
|
|
24
|
+
|------------------|------------------------|------------------------|
|
|
25
|
+
| Frontend build | [build id / date] | [build id / date] |
|
|
26
|
+
| API build | [build id / commit] | [build id / commit] |
|
|
27
|
+
| Service under test | [build id / commit] | [build id / commit] |
|
|
28
|
+
| Changed path selected? | [yes / no — name the flag or config that decides it] | [yes / no] |
|
|
29
|
+
| Commit under test deployed? | [ancestor / not an ancestor] | [ancestor / not an ancestor] |
|
|
30
|
+
|
|
31
|
+
> **Same build, different behaviour ⇒ configuration, not deploy lag.**
|
|
32
|
+
> If the changed path is not selected in an environment, every AC that depends on it is
|
|
33
|
+
> `NOT-REACHABLE` there — not `PASS`, not `FAIL`.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Cases
|
|
38
|
+
|
|
39
|
+
One row per case. The **label** is the stable identifier the report, the bug and the
|
|
40
|
+
re-run all refer to. Record passes as well as failures — a matrix of only failures
|
|
41
|
+
cannot show a regression later.
|
|
42
|
+
|
|
43
|
+
| # | Label | Request | Expected | Env A: status / count / rows | Env B: status / count / rows | Verdict |
|
|
44
|
+
|---|-------|---------|----------|------------------------------|------------------------------|---------|
|
|
45
|
+
| 1 | identity | `{"query":"<known value>"}` | 200, stable non-zero | 200 / 77 / 20 | 200 / 91 / 20 | ✅ |
|
|
46
|
+
| 2 | empty string | `{"query":""}` | 400 + error body | | | |
|
|
47
|
+
| 3 | body absent | `{}` | 400 + error body | | | |
|
|
48
|
+
| 4 | metachar — lone `(` | `{"query":"("}` | 400 or escaped 200 | | | |
|
|
49
|
+
| 5 | limit above cap | `?limit=<cap+1>` | 400, or capped | | | |
|
|
50
|
+
| 6 | invalid enum | `{"filter":["<nonsense>"]}` | 400 + error body | | | |
|
|
51
|
+
| 7 | second identity | same request, other user | scoped result | | | |
|
|
52
|
+
|
|
53
|
+
> **Compare status codes and shapes. Never compare absolute counts** — different
|
|
54
|
+
> environments hold different data and it drifts between runs.
|
|
55
|
+
|
|
56
|
+
### Case families to cover
|
|
57
|
+
|
|
58
|
+
Boundary lengths · tokenisation and whitespace · metacharacters of the underlying query
|
|
59
|
+
language · empty / null / absent / whitespace-only · enum values valid, invalid, empty,
|
|
60
|
+
wildcard, injected · pagination bounds and types · type confusion · casing and
|
|
61
|
+
normalisation · second identity and second scope.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Findings
|
|
66
|
+
|
|
67
|
+
One row per finding, not one per failing case — a family of inputs failing on the same
|
|
68
|
+
character is **one** bug with several examples.
|
|
69
|
+
|
|
70
|
+
| ID | Cases | Finding | Surface | Severity | Bug report |
|
|
71
|
+
|----|-------|---------|---------|----------|------------|
|
|
72
|
+
| F1 | 4, 9–14 | [what is wrong, in one line] | API / UI / both | High | `bugs/BUG-0xx-….md` |
|
|
73
|
+
|
|
74
|
+
**Surface** classifies where the defect lives, from the differential pass:
|
|
75
|
+
|
|
76
|
+
| Surface | Meaning | Owner |
|
|
77
|
+
|---------|---------|-------|
|
|
78
|
+
| **Both** | The UI faithfully reflects the endpoint | Service |
|
|
79
|
+
| **API only** | Real defect the client's guard is hiding — reachable by every other client | Service |
|
|
80
|
+
| **UI only** | The client invents or masks behaviour the service does not have | Client |
|
|
81
|
+
| **Neither** | Environment or access noise, not a product defect | — |
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Evidence
|
|
86
|
+
|
|
87
|
+
| File | Contents |
|
|
88
|
+
|------|----------|
|
|
89
|
+
| `evidence/fingerprint.md` | Build ids, flag values, commit ancestry |
|
|
90
|
+
| `evidence/<case-set>-raw.json` | Raw per-case output, redacted |
|
|
91
|
+
| `probes/<probe>.js` | The probe script, so this is repeatable |
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Notes and Open Questions
|
|
96
|
+
|
|
97
|
+
- [What could not be checked, and exactly what access would unblock it]
|
|
98
|
+
- [Which environment would answer a `NOT-REACHABLE` verdict]
|
|
99
|
+
- [Counts recorded here are valid for the date above only]
|
|
@@ -23,6 +23,20 @@
|
|
|
23
23
|
- **tested** -- Area was fully explored with the planned heuristics.
|
|
24
24
|
- **partial** -- Area was visited but not all heuristics or paths were exercised.
|
|
25
25
|
- **not-tested** -- Area was in scope but was not reached during the session.
|
|
26
|
+
- **code-verified-only** -- Believed correct from reading the source, never observed
|
|
27
|
+
running. This is `UNVERIFIABLE`, **not** a pass -- keep it visually distinct so it
|
|
28
|
+
cannot be skimmed as green.
|
|
29
|
+
|
|
30
|
+
Two distinctions that routinely produce a false **tested**:
|
|
31
|
+
|
|
32
|
+
- **Consistent is not causal.** A setting whose current value happens to match the output
|
|
33
|
+
proves nothing until the setting is changed and the output is watched to follow.
|
|
34
|
+
- **The data has to reach the case.** An area whose behaviour depends on values no
|
|
35
|
+
reachable record has -- a negative, a zero, an empty collection -- is **not-tested** for
|
|
36
|
+
that behaviour. Find such a record, create one, or say plainly what data would settle it.
|
|
37
|
+
|
|
38
|
+
Name the single **not-tested** item that carries the most risk. In a map full of green rows
|
|
39
|
+
that is the line the reader acts on.
|
|
26
40
|
|
|
27
41
|
---
|
|
28
42
|
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Expected Behaviour Specification
|
|
2
|
+
|
|
3
|
+
> CONFIDENTIAL — internal QA artifact.
|
|
4
|
+
|
|
5
|
+
Use this when a verification session finds behaviour nobody wrote an acceptance criterion
|
|
6
|
+
for — which is most of what an API probe turns up. A bug report says *this is wrong*. This
|
|
7
|
+
says *here is what right looks like*, in enough detail that someone can implement it and
|
|
8
|
+
someone else can verify it. It is frequently the most valuable artifact a session produces,
|
|
9
|
+
because it converts a pile of findings into one reviewable decision.
|
|
10
|
+
|
|
11
|
+
Write it **per area**, not per case. Ten failing inputs that share a cause are one section.
|
|
12
|
+
|
|
13
|
+
## Header
|
|
14
|
+
|
|
15
|
+
| Property | Value |
|
|
16
|
+
|----------|-------|
|
|
17
|
+
| Date | [YYYY-MM-DD] |
|
|
18
|
+
| Endpoint / surface | [`POST /api/v1/…` — note the verb] |
|
|
19
|
+
| Environment(s) observed | [name, and the build each one was running] |
|
|
20
|
+
| Source findings | [link to the session that produced the observations] |
|
|
21
|
+
| Status | [proposed / agreed / implemented] |
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## [N]. [Area — e.g. Minimum input length]
|
|
26
|
+
|
|
27
|
+
**Observed**
|
|
28
|
+
|
|
29
|
+
> What the system does today, as evidence. Exact request, exact status, exact response.
|
|
30
|
+
> Note which environments it holds in — behaviour frequently differs, and that difference
|
|
31
|
+
> is part of the finding.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
{"query":"L"} → 200 / <the entire collection> [alpha]
|
|
35
|
+
{"query":"L"} → 200 / 0 [beta]
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
**Expected**
|
|
39
|
+
|
|
40
|
+
> What it should do. Numbered, falsifiable statements — each one a testable assertion, not
|
|
41
|
+
> a sentiment. Include the boundary values themselves.
|
|
42
|
+
|
|
43
|
+
- An input whose tokens are all below the minimum → **`400`**, with the minimum named in
|
|
44
|
+
the message.
|
|
45
|
+
- "No usable tokens after filtering" and "no query supplied" are **different branches**.
|
|
46
|
+
Neither falls through to a match-all.
|
|
47
|
+
- The match-all path is not reachable from this parameter at any input. If "browse
|
|
48
|
+
everything" is wanted, it is an explicit endpoint with a hard server-side cap.
|
|
49
|
+
- The minimum is **N**, matching the existing rule, so that [known-good short inputs] keep
|
|
50
|
+
working.
|
|
51
|
+
- Client: no request fires below N; the user sees a hint rather than an empty result.
|
|
52
|
+
|
|
53
|
+
### Decisions this section forces
|
|
54
|
+
|
|
55
|
+
State them explicitly — leaving them implicit is how a spec gets implemented three
|
|
56
|
+
different ways:
|
|
57
|
+
|
|
58
|
+
- **Reject, or clamp?** *Reject.* Silently serving a different page size than was asked for
|
|
59
|
+
breaks every caller that trusts the value back.
|
|
60
|
+
- **`400`, or a silent empty result?** *`400`.* A silent zero is indistinguishable from a
|
|
61
|
+
genuine no-match, and it will be read as data.
|
|
62
|
+
- **Where does validation live?** At the layer that covers **every** implementation of the
|
|
63
|
+
feature — not in one of two code paths, and never only in the client.
|
|
64
|
+
- **What must not change?** List the valid inputs and their current answers, so the fix is
|
|
65
|
+
verifiable as non-breaking.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Priority
|
|
70
|
+
|
|
71
|
+
Rank the areas by what is reachable by real users **today**, not by how bad each one reads.
|
|
72
|
+
Name explicitly which finding is live in production right now, and which only appears in an
|
|
73
|
+
environment nobody ships from — the two need very different urgency, and the difference is
|
|
74
|
+
invisible unless someone says it.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Reply — plain English
|
|
79
|
+
|
|
80
|
+
> Ready to paste into a ticket comment, a pull request or a chat reply. Same content, no
|
|
81
|
+
> jargon, no status codes unless they carry meaning to the reader. Written for the person
|
|
82
|
+
> who has to decide whether to fund the fix.
|
|
83
|
+
|
|
84
|
+
For each area, in a short paragraph:
|
|
85
|
+
|
|
86
|
+
- **What happens**, in a sentence a non-engineer can picture.
|
|
87
|
+
- **Whether it is live for users**, and where.
|
|
88
|
+
- **What the fix is**, in outline — enough to size it, not enough to design it.
|
|
89
|
+
- **Where you agree with their reading**, if they proposed one, and precisely where you do
|
|
90
|
+
not, and why.
|
|
91
|
+
|
|
92
|
+
Close with the one thing you would put ahead of everything else in the list, and the reason
|
|
93
|
+
— usually that it is the only one users are hitting today.
|
package/dist/cli/index.js
CHANGED
|
@@ -276,6 +276,51 @@ var SafetyConfigSchema = z.object({
|
|
|
276
276
|
no_file_upload: z.boolean().optional(),
|
|
277
277
|
no_delete_actions: z.boolean().optional()
|
|
278
278
|
});
|
|
279
|
+
var EnvironmentConfigSchema = z.object({
|
|
280
|
+
kind: z.enum(["dev", "ephemeral", "staging", "production"]),
|
|
281
|
+
destroyed_automatically: z.boolean().optional()
|
|
282
|
+
});
|
|
283
|
+
var BackendConfigSchema = z.object({
|
|
284
|
+
provider: z.enum(["aws"]).optional(),
|
|
285
|
+
aws_profile_env: z.string().optional(),
|
|
286
|
+
region_env: z.string().optional(),
|
|
287
|
+
region: z.string().optional(),
|
|
288
|
+
// Expected account. The skill compares it to `sts get-caller-identity` and
|
|
289
|
+
// stops the live lane on a mismatch rather than probing someone else's account.
|
|
290
|
+
account_id: z.string().optional(),
|
|
291
|
+
env_suffix: z.string().optional(),
|
|
292
|
+
// Free-form logical name -> deployed resource name (config_table, dlq, ...).
|
|
293
|
+
resources: z.record(z.string(), z.string()).optional(),
|
|
294
|
+
notes: z.string().optional()
|
|
295
|
+
});
|
|
296
|
+
var ApiSurfaceConfigSchema = z.object({
|
|
297
|
+
// Defaults to the target's base_url when omitted.
|
|
298
|
+
base_url: z.string().optional(),
|
|
299
|
+
auth: z.enum(["session-cookie", "bearer-env", "none"]).optional(),
|
|
300
|
+
// NAME of the env var holding a bearer token, for `auth: bearer-env`.
|
|
301
|
+
token_env: z.string().optional(),
|
|
302
|
+
// Persistent browser profile directory holding the authenticated session,
|
|
303
|
+
// e.g. `.auth/my-service-dev-profile`. Gitignored.
|
|
304
|
+
browser_profile: z.string().optional(),
|
|
305
|
+
// Endpoint returning the deployed build, used to fingerprint the environment.
|
|
306
|
+
version_endpoint: z.string().optional(),
|
|
307
|
+
// Logical name -> "VERB /path", e.g. search: "POST /api/v1/search".
|
|
308
|
+
endpoints: z.record(z.string(), z.string()).optional(),
|
|
309
|
+
// The ONLY endpoints the read-only API lane may call without asking.
|
|
310
|
+
probe_allowlist: z.array(z.string()).optional(),
|
|
311
|
+
// Other target ids to run the same matrix against, for parity comparison.
|
|
312
|
+
parity_targets: z.array(z.string()).optional(),
|
|
313
|
+
// Flag name -> where its deployed value is declared for this environment.
|
|
314
|
+
// A flag can select between two implementations inside one identical build.
|
|
315
|
+
feature_flags: z.record(z.string(), z.string()).optional(),
|
|
316
|
+
notes: z.string().optional()
|
|
317
|
+
});
|
|
318
|
+
var SourceBranchConfigSchema = z.object({
|
|
319
|
+
repo_path: z.string(),
|
|
320
|
+
branch: z.string(),
|
|
321
|
+
base_branch: z.string().optional(),
|
|
322
|
+
components: z.record(z.string(), z.string()).optional()
|
|
323
|
+
});
|
|
279
324
|
var WebTargetConfigSchema = z.object({
|
|
280
325
|
id: z.string().min(1),
|
|
281
326
|
name: z.string().min(1),
|
|
@@ -287,6 +332,12 @@ var WebTargetConfigSchema = z.object({
|
|
|
287
332
|
browser: BrowserConfigSchema,
|
|
288
333
|
scope: ScopeConfigSchema,
|
|
289
334
|
safety: SafetyConfigSchema.optional(),
|
|
335
|
+
// Optional blocks consumed by /qa-verify-backend. A target without them is
|
|
336
|
+
// still a valid /qa-explore target.
|
|
337
|
+
environment: EnvironmentConfigSchema.optional(),
|
|
338
|
+
backend: BackendConfigSchema.optional(),
|
|
339
|
+
api: ApiSurfaceConfigSchema.optional(),
|
|
340
|
+
source: SourceBranchConfigSchema.optional(),
|
|
290
341
|
notes: z.string().optional()
|
|
291
342
|
});
|
|
292
343
|
var MobileDeviceConfigSchema = z.object({
|
|
@@ -697,13 +748,11 @@ function pad(str, len) {
|
|
|
697
748
|
|
|
698
749
|
// src/cli/commands/report.ts
|
|
699
750
|
import { Command as Command4 } from "commander";
|
|
700
|
-
import { resolve as
|
|
701
|
-
import { existsSync as
|
|
751
|
+
import { resolve as resolve5, join as join5, basename as basename2 } from "path";
|
|
752
|
+
import { existsSync as existsSync5, readdirSync as readdirSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
702
753
|
import chalk4 from "chalk";
|
|
703
754
|
|
|
704
755
|
// src/formatters/html-report.ts
|
|
705
|
-
import { readFileSync as readFileSync5, existsSync as existsSync5 } from "fs";
|
|
706
|
-
import { resolve as resolve5, extname } from "path";
|
|
707
756
|
import Handlebars from "handlebars";
|
|
708
757
|
|
|
709
758
|
// src/utils/parse-session.ts
|
|
@@ -954,17 +1003,6 @@ var HTML_TEMPLATE = `<!DOCTYPE html>
|
|
|
954
1003
|
a { color: #3498db; text-decoration: none; }
|
|
955
1004
|
a:hover { text-decoration: underline; }
|
|
956
1005
|
|
|
957
|
-
.screenshot-container { margin: 0.75rem 0; }
|
|
958
|
-
.screenshot-img {
|
|
959
|
-
max-width: 100%;
|
|
960
|
-
border-radius: 6px;
|
|
961
|
-
border: 1px solid #1e2a3a;
|
|
962
|
-
cursor: pointer;
|
|
963
|
-
transition: transform 0.2s;
|
|
964
|
-
}
|
|
965
|
-
.screenshot-img:hover { transform: scale(1.02); }
|
|
966
|
-
.screenshot-label { font-size: 0.8rem; color: #8899aa; margin-top: 0.25rem; }
|
|
967
|
-
|
|
968
1006
|
.container { max-width: 960px; margin: 0 auto; }
|
|
969
1007
|
|
|
970
1008
|
/* Header */
|
|
@@ -1252,11 +1290,8 @@ var HTML_TEMPLATE = `<!DOCTYPE html>
|
|
|
1252
1290
|
|
|
1253
1291
|
{{#if this.hasEvidence}}
|
|
1254
1292
|
<div class="bug-section-label">Evidence</div>
|
|
1255
|
-
{{#each this.
|
|
1256
|
-
<
|
|
1257
|
-
<img src="{{this.dataUri}}" alt="{{this.filename}}" class="screenshot-img" />
|
|
1258
|
-
<p class="screenshot-label">{{this.filename}}</p>
|
|
1259
|
-
</div>
|
|
1293
|
+
{{#each this.evidence.screenshots}}
|
|
1294
|
+
<p class="evidence-item">Screenshot: {{this}}</p>
|
|
1260
1295
|
{{/each}}
|
|
1261
1296
|
{{#each this.evidence.console_errors}}
|
|
1262
1297
|
<p class="evidence-item">Console: {{this}}</p>
|
|
@@ -1330,37 +1365,11 @@ async function generateHtmlReport(sessionDir) {
|
|
|
1330
1365
|
for (const bug of session.bugs) {
|
|
1331
1366
|
counts[bug.severity]++;
|
|
1332
1367
|
}
|
|
1333
|
-
const bugsForTemplate = session.bugs.map((bug) => {
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
resolve5(sessionDir, "screenshots", cleaned),
|
|
1339
|
-
resolve5(sessionDir, cleaned.replace(/^screenshots\//, "")),
|
|
1340
|
-
resolve5(sessionDir, "screenshots", cleaned.replace(/^screenshots\//, ""))
|
|
1341
|
-
];
|
|
1342
|
-
for (const imgPath of possiblePaths) {
|
|
1343
|
-
if (existsSync5(imgPath)) {
|
|
1344
|
-
try {
|
|
1345
|
-
const ext = extname(imgPath).toLowerCase().replace(".", "");
|
|
1346
|
-
const mime = ext === "jpg" ? "image/jpeg" : `image/${ext || "png"}`;
|
|
1347
|
-
const data = readFileSync5(imgPath).toString("base64");
|
|
1348
|
-
const filename = imgPath.split("/").pop() || screenshotRef;
|
|
1349
|
-
return { dataUri: `data:${mime};base64,${data}`, filename };
|
|
1350
|
-
} catch {
|
|
1351
|
-
return null;
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
1354
|
-
}
|
|
1355
|
-
return null;
|
|
1356
|
-
}).filter(Boolean);
|
|
1357
|
-
return {
|
|
1358
|
-
...bug,
|
|
1359
|
-
business_impact_summary: summarizeBusinessImpact(bug.business_impact),
|
|
1360
|
-
hasEvidence: embeddedScreenshots.length > 0 || bug.evidence.console_errors.length > 0,
|
|
1361
|
-
embeddedScreenshots
|
|
1362
|
-
};
|
|
1363
|
-
});
|
|
1368
|
+
const bugsForTemplate = session.bugs.map((bug) => ({
|
|
1369
|
+
...bug,
|
|
1370
|
+
business_impact_summary: summarizeBusinessImpact(bug.business_impact),
|
|
1371
|
+
hasEvidence: bug.evidence.screenshots.length > 0 || bug.evidence.console_errors.length > 0
|
|
1372
|
+
}));
|
|
1364
1373
|
const html = compiledTemplate({
|
|
1365
1374
|
meta,
|
|
1366
1375
|
totalBugs: session.bugs.length,
|
|
@@ -1573,8 +1582,8 @@ function reportCommand() {
|
|
|
1573
1582
|
}
|
|
1574
1583
|
async function runReport(options) {
|
|
1575
1584
|
const cwd = process.cwd();
|
|
1576
|
-
const sessionsDir =
|
|
1577
|
-
if (!
|
|
1585
|
+
const sessionsDir = resolve5(cwd, "output", "sessions");
|
|
1586
|
+
if (!existsSync5(sessionsDir)) {
|
|
1578
1587
|
console.error(chalk4.red("No sessions directory found. Run `npx qualiow init` first."));
|
|
1579
1588
|
process.exit(1);
|
|
1580
1589
|
}
|
|
@@ -1590,7 +1599,7 @@ Session: ${sessionName}`));
|
|
|
1590
1599
|
switch (format) {
|
|
1591
1600
|
case "md": {
|
|
1592
1601
|
const reportPath = join5(sessionDir, "session-report.md");
|
|
1593
|
-
if (
|
|
1602
|
+
if (existsSync5(reportPath)) {
|
|
1594
1603
|
console.log(chalk4.green(`
|
|
1595
1604
|
Report: ${reportPath}`));
|
|
1596
1605
|
} else {
|
|
@@ -1635,7 +1644,7 @@ function resolveSessionDir(sessionsDir, sessionId) {
|
|
|
1635
1644
|
return join5(sessionsDir, dirs2[dirs2.length - 1]);
|
|
1636
1645
|
}
|
|
1637
1646
|
const exactPath = join5(sessionsDir, sessionId);
|
|
1638
|
-
if (
|
|
1647
|
+
if (existsSync5(exactPath)) {
|
|
1639
1648
|
return exactPath;
|
|
1640
1649
|
}
|
|
1641
1650
|
const dirs = readdirSync4(sessionsDir, { withFileTypes: true }).filter((d) => d.isDirectory() && d.name.includes(sessionId)).map((d) => d.name).sort();
|
|
@@ -1647,8 +1656,8 @@ function resolveSessionDir(sessionsDir, sessionId) {
|
|
|
1647
1656
|
|
|
1648
1657
|
// src/cli/commands/explore.ts
|
|
1649
1658
|
import { Command as Command5 } from "commander";
|
|
1650
|
-
import { resolve as
|
|
1651
|
-
import { existsSync as
|
|
1659
|
+
import { resolve as resolve6, join as join6 } from "path";
|
|
1660
|
+
import { existsSync as existsSync6, mkdirSync as mkdirSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
1652
1661
|
import { execSync } from "child_process";
|
|
1653
1662
|
import chalk5 from "chalk";
|
|
1654
1663
|
function exploreCommand() {
|
|
@@ -1674,8 +1683,8 @@ async function runExplore(url, options) {
|
|
|
1674
1683
|
}
|
|
1675
1684
|
let targetName = "adhoc";
|
|
1676
1685
|
if (options.target) {
|
|
1677
|
-
const targetPath =
|
|
1678
|
-
if (!
|
|
1686
|
+
const targetPath = resolve6(cwd, "data", "targets", `${options.target}.yml`);
|
|
1687
|
+
if (!existsSync6(targetPath)) {
|
|
1679
1688
|
console.error(
|
|
1680
1689
|
chalk5.red(`Target config not found: ${targetPath}`)
|
|
1681
1690
|
);
|
|
@@ -1694,8 +1703,8 @@ async function runExplore(url, options) {
|
|
|
1694
1703
|
console.log(chalk5.green(` \u2713 Target config "${options.target}" is valid`));
|
|
1695
1704
|
}
|
|
1696
1705
|
if (options.context) {
|
|
1697
|
-
const contextPath =
|
|
1698
|
-
if (!
|
|
1706
|
+
const contextPath = resolve6(cwd, options.context);
|
|
1707
|
+
if (!existsSync6(contextPath)) {
|
|
1699
1708
|
console.error(chalk5.red(`Context file not found: ${contextPath}`));
|
|
1700
1709
|
process.exit(1);
|
|
1701
1710
|
}
|
|
@@ -1711,7 +1720,7 @@ async function runExplore(url, options) {
|
|
|
1711
1720
|
String(now.getMinutes()).padStart(2, "0")
|
|
1712
1721
|
].join("");
|
|
1713
1722
|
const sessionName = `${timestamp}-${targetName}`;
|
|
1714
|
-
const sessionDir =
|
|
1723
|
+
const sessionDir = resolve6(cwd, "output", "sessions", sessionName);
|
|
1715
1724
|
if (options.dryRun) {
|
|
1716
1725
|
console.log("");
|
|
1717
1726
|
console.log(chalk5.cyan.bold("Dry run \u2014 no files created"));
|
|
@@ -1789,8 +1798,8 @@ async function runExplore(url, options) {
|
|
|
1789
1798
|
|
|
1790
1799
|
// src/cli/commands/gather.ts
|
|
1791
1800
|
import { Command as Command6 } from "commander";
|
|
1792
|
-
import { resolve as
|
|
1793
|
-
import { existsSync as
|
|
1801
|
+
import { resolve as resolve7 } from "path";
|
|
1802
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync3 } from "fs";
|
|
1794
1803
|
import { execSync as execSync2 } from "child_process";
|
|
1795
1804
|
import chalk6 from "chalk";
|
|
1796
1805
|
function gatherCommand() {
|
|
@@ -1807,7 +1816,7 @@ function gatherCommand() {
|
|
|
1807
1816
|
async function runGather(source, options) {
|
|
1808
1817
|
const cwd = process.cwd();
|
|
1809
1818
|
const isUrl = isValidUrl(source);
|
|
1810
|
-
const isFile = !isUrl &&
|
|
1819
|
+
const isFile = !isUrl && existsSync7(resolve7(cwd, source));
|
|
1811
1820
|
if (!isUrl && !isFile) {
|
|
1812
1821
|
console.error(chalk6.red(`Source not found: "${source}"`));
|
|
1813
1822
|
console.error(chalk6.white(" Provide a valid file path or URL."));
|
|
@@ -1816,21 +1825,21 @@ async function runGather(source, options) {
|
|
|
1816
1825
|
if (isUrl) {
|
|
1817
1826
|
console.log(chalk6.green(` \u2713 Source is a URL: ${source}`));
|
|
1818
1827
|
} else {
|
|
1819
|
-
console.log(chalk6.green(` \u2713 Source file found: ${
|
|
1828
|
+
console.log(chalk6.green(` \u2713 Source file found: ${resolve7(cwd, source)}`));
|
|
1820
1829
|
}
|
|
1821
1830
|
if (options.domain) {
|
|
1822
|
-
const domainPath =
|
|
1823
|
-
if (!
|
|
1831
|
+
const domainPath = resolve7(cwd, "data", "domains", `${options.domain}.yml`);
|
|
1832
|
+
if (!existsSync7(domainPath)) {
|
|
1824
1833
|
console.error(chalk6.red(`Domain config not found: ${domainPath}`));
|
|
1825
1834
|
console.error(chalk6.white(" Available domains: npx qualiow list domains"));
|
|
1826
1835
|
process.exit(1);
|
|
1827
1836
|
}
|
|
1828
1837
|
console.log(chalk6.green(` \u2713 Domain config "${options.domain}" found`));
|
|
1829
1838
|
}
|
|
1830
|
-
const outputDir =
|
|
1839
|
+
const outputDir = resolve7(cwd, "output", "context");
|
|
1831
1840
|
mkdirSync3(outputDir, { recursive: true });
|
|
1832
1841
|
if (options.output) {
|
|
1833
|
-
console.log(chalk6.green(` \u2713 Output will be saved to: ${
|
|
1842
|
+
console.log(chalk6.green(` \u2713 Output will be saved to: ${resolve7(cwd, options.output)}`));
|
|
1834
1843
|
} else {
|
|
1835
1844
|
console.log(chalk6.green(` \u2713 Output directory ready: ${outputDir}`));
|
|
1836
1845
|
}
|