qualiow-exploratory-testing 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +177 -0
  2. package/bin/mcli +16 -0
  3. package/bin/mobile-cli.mjs +573 -0
  4. package/bin/wadb +6 -0
  5. package/bin/wk-ios +58 -0
  6. package/bin/wkeval.mjs +44 -0
  7. package/data/domains/_default.md +181 -0
  8. package/data/domains/ecommerce.md +192 -0
  9. package/data/domains/fintech.md +200 -0
  10. package/data/domains/identity.md +88 -0
  11. package/data/domains/identity.yml +138 -0
  12. package/data/domains/marketing.md +194 -0
  13. package/data/domains/saas.md +197 -0
  14. package/data/knowledge/changelog.yml +86 -0
  15. package/data/knowledge/learned-patterns.md +172 -1
  16. package/data/knowledge/manifest.yml +210 -4
  17. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  18. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  19. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  20. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  21. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  22. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  23. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  24. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  25. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  26. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  28. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  29. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  30. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  31. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  34. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  35. package/data/targets/_example-backend.yml +123 -0
  36. package/data/targets/_example-mobile-emulation.yml +52 -0
  37. package/data/targets/_example-native-mobile.yml +61 -0
  38. package/data/targets/_example-sim-android-chrome.yml +81 -0
  39. package/data/targets/_example-sim-ios-safari.yml +80 -0
  40. package/data/templates/api-probe-matrix.md +99 -0
  41. package/data/templates/coverage-map.md +14 -0
  42. package/data/templates/expected-behaviour.md +93 -0
  43. package/dist/cli/index.js +137 -5
  44. package/dist/cli/index.js.map +1 -1
  45. package/dist/index.d.ts +350 -527
  46. package/dist/index.js +144 -5
  47. package/dist/index.js.map +1 -1
  48. package/docs/BACKEND-VERIFICATION.md +277 -0
  49. package/docs/MOBILE-SETUP.md +92 -0
  50. package/package.json +26 -12
  51. package/scripts/doctor-mobile.sh +143 -0
  52. package/scripts/setup-mobile.sh +213 -0
  53. package/skills/qa-explore/phases/00-setup.md +8 -1
  54. package/skills/qa-explore/phases/01-auth.md +4 -0
  55. package/skills/qa-explore-mobile/SKILL.md +146 -0
  56. package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
  57. package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
  58. package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
  59. package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
  60. package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
  61. package/skills/qa-explore-mobile/phases/05-features.md +72 -0
  62. package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
  63. package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
  64. package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
  65. package/skills/qa-explore-quick/SKILL.md +6 -0
  66. package/skills/qa-verify-backend/SKILL.md +163 -0
  67. package/skills/qa-verify-backend/phases/00-setup.md +97 -0
  68. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  69. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  70. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  71. package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
  72. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
  73. package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
  74. package/skills/qa-verify-backend/references/api-probes.md +141 -0
  75. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  76. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  77. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  78. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  79. package/skills/qa-verify-backend/references/safety-rules.md +90 -0
@@ -0,0 +1,277 @@
1
+ # Backend & Infrastructure AC Verification
2
+
3
+ A guide to `/qa-verify-backend` — the lane for tickets whose acceptance criteria cannot be
4
+ seen in a browser.
5
+
6
+ ## Why this exists
7
+
8
+ `/qa-explore` drives a browser. A backend ticket — a stream, a table, an IAM policy, a queue
9
+ consumer, a webhook — has most of its acceptance criteria below the UI, where a browser cannot
10
+ reach. Those ACs get waved through on "it works in the dev env", and the gaps surface months
11
+ later during an incident.
12
+
13
+ Three failure modes this skill is built against, all subtler than a missed bug:
14
+
15
+ - **A confident `PASS` backed by a code reading** looks exactly like a `PASS` backed by a
16
+ measurement, and is worth far less. Refusing to fake that is what makes the other verdicts
17
+ credible.
18
+ - **A client-side guard is not the endpoint's behaviour.** "The button is disabled until you
19
+ type" describes the browser. Every other client — mobile, integration, script — sends the
20
+ request the guard was preventing, and that request is frequently the one that fails.
21
+ - **A verdict is a claim about code; a probe is an observation of an environment.** They
22
+ coincide only when that environment runs the change. One identical build can hold two
23
+ implementations of one feature with a flag choosing between them.
24
+
25
+ ## The workflow
26
+
27
+ ### 1. Analyse the ticket → `output/context/<TICKET>-context.md`
28
+
29
+ Run `/qa-gather` on the ticket, or write the context file by hand. The job is to turn prose
30
+ ACs into falsifiable claims **before** reading the implementation — doing it in that order is
31
+ what stops you rationalising whatever the code happens to do.
32
+
33
+ Flag **DoR gaps** while you are here: ACs with no observable falsifier, missing criteria for
34
+ behaviour the change clearly has, internal contradictions. Take each AC literally and ask what
35
+ observation would prove it false. An AC whose spec is wrong is worth finding before anyone
36
+ implements it.
37
+
38
+ ### 2. Create a target config
39
+
40
+ ```bash
41
+ cp data/targets/_example-backend.yml data/targets/local-my-service.yml
42
+ ```
43
+
44
+ Fill in `base_url`, `environment.kind`, the `backend.resources` names, and `source.branch`.
45
+
46
+ Two rules:
47
+
48
+ - **Credentials are referenced by env var NAME, never by value.** `aws_profile_env:
49
+ QA_AWS_PROFILE` — the value lives in `.env`, which is gitignored.
50
+ - **Name a private target `local-*.yml`.** That prefix is gitignored, so internal hostnames,
51
+ account ids and resource names stay on your machine. Add your own prefix to `.gitignore` if
52
+ you prefer a different convention.
53
+
54
+ ### 3. Verify
55
+
56
+ ```bash
57
+ /qa-verify-backend --target local-my-service --context output/context/TICKET-123-context.md
58
+ ```
59
+
60
+ Flags: `--static-only` (skip live probes — use when you have no cloud credentials),
61
+ `--api-only` (skip the cloud lane and verify the HTTP surface), `--no-e2e` (skip the
62
+ write-path phase), `--parity <target-id>` (run the same API matrix in a second environment
63
+ and compare).
64
+
65
+ ### 4. Explore the UI, if the ticket has one
66
+
67
+ ```bash
68
+ /qa-explore --target local-my-service --context output/context/TICKET-123-context.md
69
+ ```
70
+
71
+ ### 5. Hand off
72
+
73
+ `ac-matrix.md` is the headline deliverable. Bug reports land in `bugs/`, one per finding, each
74
+ with a Business Impact written in terms of consequence rather than mechanism — nobody funds a
75
+ fix for "StreamViewType is NEW_AND_OLD_IMAGES".
76
+
77
+ ## The four lanes
78
+
79
+ | Lane | Reads | Never |
80
+ |------|-------|-------|
81
+ | **Static** | The implementation branch via `git show <ref>:<path>` | Checks out, fetches without asking, or creates branches — you may have uncommitted work |
82
+ | **Live** | `describe-*`, `get-*`, `list-*`, `query`, `scan`, `simulate-principal-policy` | Mutates anything, or probes an account other than the one the target declares |
83
+ | **API** | The service's own endpoints, from the authenticated session — the cases a browser cannot send | Calls anything outside `api.probe_allowlist`, or anything that creates, mutates or deletes |
84
+ | **End-to-end** | The real write path in a non-production environment, then the data layer again | Runs against production, or writes directly to a data store (that bypasses the path under test and proves nothing) |
85
+
86
+ ### Before any lane: fingerprint the environment
87
+
88
+ A green result against an environment that does not run the changed code proves nothing, and a
89
+ red result there is not the ticket's bug. Phase 0 establishes, with evidence, which build is
90
+ deployed in **every** component of the request path, whether the changed path is *selected*
91
+ here (a flag, a config value, a routing rule), and whether the commit under test is genuinely
92
+ an ancestor of what is running.
93
+
94
+ > **Same build, different behaviour ⇒ configuration, not deploy lag.**
95
+
96
+ The consequence is a verdict: an environment that does not run the change gets
97
+ `NOT-REACHABLE`. And a fix that only lands behind a flag which is off in the environments
98
+ users are in has reached nobody — that is a finding in its own right, because flipping the
99
+ flag is a separate decision carrying its own risk.
100
+
101
+ ### The API lane, specifically
102
+
103
+ The request runs **inside the already-authenticated page**, so it carries the same session
104
+ cookie, CSRF token and interceptors the UI has. There is no token to obtain, nothing to store,
105
+ and it works with SSO and MFA that no scripted login can pass:
106
+
107
+ ```bash
108
+ playwright-cli -s=my-env open https://service.example.com/ --headed --profile .auth/my-env
109
+ playwright-cli -s=my-env eval "$(cat probes/search-cases.js)" --raw
110
+ ```
111
+
112
+ Cases come from the ACs plus the standard families — length boundaries, tokenisation,
113
+ metacharacters of whatever query language sits underneath, the four different kinds of
114
+ nothing (`""`, `null`, whitespace-only, field absent), enum values valid and invalid,
115
+ pagination bounds, type confusion, a second identity and a second scope.
116
+
117
+ If the ticket also has a screen, run the overlapping cases through it too and sort every
118
+ finding into **both** (the UI is faithful — fix it in the service), **API only** (a real
119
+ defect the client's guard is hiding, reachable by every other client), **UI only** (the client
120
+ invents or masks behaviour the service does not have) and **neither**.
121
+
122
+ ### A well-shaped `200` is not a correct answer
123
+
124
+ Once the contract holds, the values are still an open question. Every derived number — a
125
+ percentage, total, ratio, delta or aggregate — is recomputed from the raw figures in the same
126
+ response, using the formula from the **specification** rather than from the code under test,
127
+ with cases chosen to stress sign, zero, scale and cardinality, plus the structural invariants
128
+ that must hold regardless of magnitude.
129
+
130
+ Then the report says what that does not prove. When both sides of the check come from one
131
+ payload, the *derivation* is verified and the *inputs* are not — so the limitation is written
132
+ down and the **independent oracle** that would close it is named, along with whether it was
133
+ run. "Internal consistency verified; the outstanding oracle is X and has not been run" is an
134
+ honest, useful verdict. "The numbers are correct" is not.
135
+
136
+ And the payload is held next to the screen, because a correct response can still reach the
137
+ user as a wrong number — a formatter that guesses what a value is, a unit applied twice,
138
+ rounding that crosses a threshold, a truncated figure shown as a total. That defect is
139
+ invisible from either surface alone, and it usually belongs to a different change than the one
140
+ under test. See `.claude/skills/qa-verify-backend/references/payload-verification.md`.
141
+
142
+ ## Verdicts
143
+
144
+ | Verdict | Means |
145
+ |---------|-------|
146
+ | `PASS` | Verified, with cited evidence |
147
+ | `PARTIAL` | The intent is met but something material deviates from the stated spec |
148
+ | `FAIL` | The AC as written is not true |
149
+ | `BLOCKED` | Not verifiable with available access — the probe command is attached, ready to run |
150
+ | `NOT-REACHABLE` | This environment does not run the changed code path — a flag, a config value, or an undeployed commit. Not a pass and not a failure |
151
+ | `UNVERIFIABLE` | The AC has no falsifier as written — a DoR defect |
152
+
153
+ Every verdict names the observation mode that produced it **and the environment it holds in**.
154
+ `PASS (dev)` and `PASS` are different claims, and only one of them is honest when the change is
155
+ behind a flag that is off everywhere else. A row is never left at `PENDING`: if it was not
156
+ checked, it is `BLOCKED` and says why.
157
+
158
+ ## Choosing how to observe
159
+
160
+ Each AC is routed to the channel that can actually falsify it
161
+ (`technique-verification-mode-selection`):
162
+
163
+ - **API-behind-the-screen** — a user flow reaches the changed code, but you assert on the
164
+ network calls underneath, not the rendered screen. The UI is a lossy renderer: it hides
165
+ fields it does not display, coerces types, and swallows partial failures.
166
+ - **Direct request** — webhooks, internal endpoints, queue consumers, scheduled jobs, and the
167
+ inputs a screen's own guards prevent. Nothing in any product flow reaches these, so you fire
168
+ the request yourself and follow through to what it produced
169
+ (`technique-authenticated-api-probing`).
170
+ - **LLM / agent output** — a prompt, tool definition, model version or retrieval step changed.
171
+ Run a fixed input set against a written rubric; assert on properties and tool trajectory, not
172
+ on generated text (`technique-llm-output-verification`).
173
+ - **Needs a human** — a pure refactor, or something you lack the access to observe. Say so and
174
+ write out what someone with access should run.
175
+
176
+ ## Safety
177
+
178
+ The full policy is in
179
+ `.claude/skills/qa-verify-backend/references/safety-rules.md`. The parts worth knowing before
180
+ your first session:
181
+
182
+ - **Production hard stop.** If the account, profile, resource name or URL contains `prod`,
183
+ `prd` or `live`, probes are read-only and the end-to-end phase is skipped and reported as
184
+ skipped. If you cannot tell whether an environment is production, it is treated as production.
185
+ - **Account confirmation first.** `sts get-caller-identity` runs before the first probe and is
186
+ compared to the target's declared account. A probe fired at the wrong account is an access
187
+ event in someone else's environment.
188
+ - **Destructive runbooks are findings, not instructions.** A cleanup script with `delete-item`
189
+ commands found in a repo gets reported ("cleanup documented, no evidence of execution"), never
190
+ executed.
191
+ - **The API lane is read-only in every environment.** It calls only what
192
+ `api.probe_allowlist` declares. An endpoint whose verb is safe but whose effect is not —
193
+ billed per call, queues a job, rate-limited into an outage — needs explicit confirmation.
194
+ Matrix volume stays proportionate: a few dozen requests is verification, thousands is a load
195
+ test nobody agreed to.
196
+ - **Session credentials never leave the machine.** An authenticated browser profile and any
197
+ cookie taken from it are live credentials for a real account. They stay in `.auth/` and are
198
+ never inlined into a probe script, a committed file, a report or a message.
199
+ - **Redaction before disk.** Tokens, keys, real user emails, PII and any account id not already
200
+ in the target config become `[REDACTED]`. Raw probe output is the most common leak — it is
201
+ convenient to paste whole, and it is full of identifiers. API response bodies are the worst
202
+ offender: they are whole records, and an error body routinely carries the internal hostname,
203
+ the query that failed and a stack frame.
204
+ - **Session output is confidential and local.** It maps internal resources, effective
205
+ permissions and unfixed defects. It never goes to an external service.
206
+
207
+ ## The techniques behind it
208
+
209
+ Knowledge base entries loaded by the skill (`data/knowledge/`):
210
+
211
+ | Entry | Use it when |
212
+ |-------|-------------|
213
+ | `technique-verification-mode-selection` | Every session, at AC decomposition |
214
+ | `technique-functional-diff-analysis` | The static lane — eight passes that read a diff for behaviour, not style |
215
+ | `technique-contract-narrowing` | Any ticket that swaps a data source on a read path |
216
+ | `technique-test-suite-audit` | The change rewrites the tests meant to prove it works |
217
+ | `technique-llm-output-verification` | The change touches a prompt, tool, model or retrieval step |
218
+ | `technique-environment-fingerprinting` | Every session, before the first probe — which build and which implementation is actually running |
219
+ | `technique-authenticated-api-probing` | Any AC about an endpoint's own behaviour: validation, filters, pagination, error contract, payload shape |
220
+ | `technique-ui-api-differential` | The ticket has both a screen and an endpoint |
221
+ | `technique-silent-failure-audit` | Any read path with an error branch — failures rendered as ordinary-looking empty or zero results |
222
+ | `technique-derived-value-verification` | The AC concerns a calculated value — percentage, total, ratio, delta, aggregate |
223
+ | `technique-presentation-integrity` | The value is also shown on a screen |
224
+ | `technique-expected-behaviour-specification` | The session found behaviour no AC covers |
225
+ | `technique-config-surface-verification` | The change under test *is* the configuration mechanism |
226
+ | `technique-release-readiness-verification` | The ask is "is this release good to go" |
227
+
228
+ Browse them with `/qa-knowledge-list`.
229
+
230
+ ## Verifying a release rather than a ticket
231
+
232
+ When the ask is "is this release good to go" instead of "does this ticket meet its ACs", follow
233
+ `.claude/skills/qa-verify-backend/references/release-readiness.md`. It changes the shape of
234
+ the session:
235
+
236
+ - **The build table comes first, for everything.** Per ticket, is the commit it depends on
237
+ genuinely an ancestor of what is deployed? This reclassifies half the session before any
238
+ testing happens. A ticket whose backend is not deployed is **not testable here** — and testing
239
+ its UI anyway produces a convincing, meaningless result, because the frontend ships the
240
+ feature, the API does not answer, and it renders as dashes or zeros that look exactly like a
241
+ data bug.
242
+ - **A fixed result vocabulary** — clean pass · pass with caveats · fail · not testable here ·
243
+ not tested — so the last two stay visible instead of vanishing between passed and failed.
244
+ - **A coverage map with four states**, where 🔍 *code-verified only* is marked distinctly
245
+ because it is `UNVERIFIABLE`, not a pass. Plus the one sentence naming which untested item
246
+ carries the most risk.
247
+ - **Carry-overs in their own section** — pre-existing defects mixed into a release's results
248
+ inflate the apparent risk of shipping it and bury what belongs to it.
249
+ - **A disposition with its reversal condition**, and the scope of what was checked stated
250
+ explicitly when it is narrower than the question being asked.
251
+
252
+ ## Writing the spec where none exists
253
+
254
+ Most of what an API probe turns up has no acceptance criterion behind it, so there is nothing to
255
+ file it against and it becomes an argument rather than a fix.
256
+ `data/templates/expected-behaviour.md` is the artifact for that: observed against expected,
257
+ grouped by cause rather than by case, with the decisions the fix forces made explicit —
258
+
259
+ - **Reject, or clamp?** Reject. Silently serving something other than what was asked for breaks
260
+ every caller that trusts the value back.
261
+ - **An error, or an empty result?** An error. A silent zero is indistinguishable from a genuine
262
+ no-match and will be read as data.
263
+ - **Where does validation live?** At the layer covering every implementation of the feature —
264
+ never only in the client, which is not in the path for any other caller.
265
+
266
+ — ranked by what real users can reach *today* rather than by how bad each one reads, and closed
267
+ with the same content in plain English, ready to paste into a ticket comment. That last section
268
+ is what gets the work scheduled.
269
+
270
+ ## Templates
271
+
272
+ | Template | Use |
273
+ |----------|-----|
274
+ | `data/templates/api-probe-matrix.md` | The environment fingerprint, the case table with a column per environment, the four-bucket findings table, the evidence index. Copy it in at the start of the API lane |
275
+ | `data/templates/expected-behaviour.md` | The specification that should have existed, plus the plain-English reply |
276
+ | `data/templates/coverage-map.md` | Coverage with four states, including code-verified-only |
277
+ | `data/templates/bug-report.md` | One bug per report, with Business Impact |
@@ -0,0 +1,92 @@
1
+ # Mobile Setup — running `/qa-explore-mobile` on a fresh Mac
2
+
3
+ The mobile skill drives **real** simulator/emulator browsers and native apps via
4
+ `mobile-cli` (a Maestro / `xcrun simctl` / `adb` shim). Unlike the desktop web skill
5
+ (`/qa-explore`, which only needs `playwright-cli`), the mobile path depends on the iOS
6
+ and/or Android toolchains being installed on the machine. This guide gets a clean Mac to
7
+ the point where a session can start.
8
+
9
+ ## TL;DR
10
+
11
+ ```bash
12
+ # 1. Install everything scriptable + create standard devices (idempotent, re-runnable)
13
+ scripts/setup-mobile.sh # or --android / --ios, add --yes for non-interactive
14
+
15
+ # 2. Confirm the machine is ready (read-only; installs nothing)
16
+ scripts/doctor-mobile.sh
17
+
18
+ # 3. Run a mobile session (once doctor says READY)
19
+ # e.g. iOS Safari against your mobile web app:
20
+ # drive via bin/mcli — see the target config notes in data/targets/_example-sim-*.yml
21
+ ```
22
+
23
+ `scripts/setup-mobile.sh` automates everything it can and **prints clear MANUAL steps**
24
+ for the few things it can't (Xcode + the iOS runtime). It finishes by running the doctor.
25
+
26
+ ## What gets installed / created
27
+
28
+ | Component | Needed for | Auto by `setup-mobile.sh`? |
29
+ |-----------|-----------|----------------------------|
30
+ | Homebrew | base | ✅ (offers to install) |
31
+ | Node ≥ 18 | runs `mobile-cli.mjs` | ✅ |
32
+ | **Maestro** | the driver (hierarchy/tap/input) | ✅ (`get.maestro.mobile.dev`) |
33
+ | **JDK 17** | Android SDK tooling | ✅ `brew install openjdk@17` |
34
+ | Android cmdline-tools | `sdkmanager`/`avdmanager` | ✅ `brew install --cask android-commandline-tools` |
35
+ | platform-tools (`adb`), `emulator` | drive Android | ✅ `sdkmanager` |
36
+ | Google **Play** system image (API 35) | **Chrome must be present** | ✅ `sdkmanager` |
37
+ | SDK licenses | unblock installs | ✅ `sdkmanager --licenses` |
38
+ | Android AVD `qa_pixel_api35` | a device to boot | ✅ `avdmanager create avd` |
39
+ | **Xcode** + CLI tools | iOS simulators at all | ⚠️ **manual** (App Store, multi-GB) |
40
+ | iOS runtime | a simulator OS to run | ⚠️ **manual** if none installed (Xcode ▸ Settings ▸ Components) |
41
+ | iOS simulator device `qa-iphone` | a device to boot | ✅ `simctl create` (once Xcode + a runtime exist) |
42
+ | Safari / Chrome (the browsers) | web-mode SUT | ✅ preinstalled (Safari always; Chrome via the Play image) |
43
+
44
+ ### The two genuinely manual iOS steps
45
+ Xcode can't be reliably scripted:
46
+ ```bash
47
+ # After installing Xcode from the App Store:
48
+ sudo xcode-select -s /Applications/Xcode.app/Contents/Developer
49
+ sudo xcodebuild -license accept
50
+ # If no iOS runtime is installed:
51
+ xcodebuild -downloadPlatform iOS # or Xcode ▸ Settings ▸ Components
52
+ ```
53
+ Then re-run `scripts/setup-mobile.sh` — it will create the `qa-iphone` device.
54
+
55
+ ## Standard device names (referenced by the target configs)
56
+ - Android AVD: **`qa_pixel_api35`** (Google-Play API 35) — set `ANDROID_AVD_NAME` to override.
57
+ - iOS sim: **`qa-iphone`** — set `IOS_DEVICE_NAME` to override.
58
+
59
+ The `data/targets/_example-sim-*.yml` configs resolve the device **by name** (not a
60
+ hardcoded UDID) so they're portable across machines. `00-setup` boots the named device,
61
+ creating it if missing.
62
+
63
+ ## Why a Google **Play** (or `google_apis`) Android image
64
+ A stock **AOSP** system image ships **no `com.android.chrome`** — only a basic AOSP
65
+ browser or none. Web-mode Android targets need Chrome, so `setup-mobile.sh` installs a
66
+ `google_apis_playstore` image. Verify on any AVD with:
67
+ ```bash
68
+ adb shell pm list packages | grep chrome
69
+ ```
70
+
71
+ ## Auth reality (web targets)
72
+ There is **no transferable `storage_state`** for a real device browser (unlike Playwright).
73
+ SSO + MFA logins are done **once, interactively, by a human** in the sim/emulator browser;
74
+ the device persists the browser profile across launches. Never script MFA. Never
75
+ `relaunch-clean` a web target (it `pm clear`s the logged-in Chrome profile). See the
76
+ `notes:` blocks in `data/targets/_example-sim-*.yml`.
77
+
78
+ ## Troubleshooting
79
+ - **`doctor-mobile.sh` says NOT READY** → fix each `✗` with the printed command, or run
80
+ `setup-mobile.sh`, then re-run the doctor.
81
+ - **`JAVA_HOME` / `adb` not found mid-session** → `bin/mcli` resolves JDK 17 via
82
+ `/usr/libexec/java_home -v 17` and defaults `ANDROID_HOME` to `~/Library/Android/sdk`.
83
+ Export them explicitly if your SDK lives elsewhere.
84
+ - **Intel Mac** → `setup-mobile.sh` selects `x86_64` system images automatically (`arm64`
85
+ Macs get `arm64-v8a`).
86
+ - **Emulator slow/flaky to boot or Chrome absent** → fall back to the iOS Safari target;
87
+ both engines are independent.
88
+
89
+ ## Scope
90
+ This covers iOS Simulator + Android Emulator (browser **and** native modes). It does **not**
91
+ provision physical devices, CI runners, or the desktop web path (`/qa-explore`, which only
92
+ needs `playwright-cli`).
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "qualiow-exploratory-testing",
3
- "version": "1.1.0",
4
- "description": "qualiow — AI-powered exploratory testing skills using Claude Code + Playwright CLI",
3
+ "version": "1.3.0",
4
+ "description": "qualiow — AI-powered exploratory testing skills using Claude Code + Playwright CLI (web) and Maestro/mobile-cli (iOS/Android simulators)",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
7
7
  "types": "dist/index.d.ts",
@@ -11,12 +11,22 @@
11
11
  "files": [
12
12
  "dist/",
13
13
  "skills/",
14
- "data/domains/*.yml",
14
+ "bin/",
15
+ "scripts/setup-mobile.sh",
16
+ "scripts/doctor-mobile.sh",
17
+ "data/domains/",
15
18
  "data/knowledge/",
16
19
  "data/templates/",
17
20
  "data/targets/_default.yml",
21
+ "data/targets/_example-native-mobile.yml",
22
+ "data/targets/_example-sim-ios-safari.yml",
23
+ "data/targets/_example-sim-android-chrome.yml",
24
+ "data/targets/_example-mobile-emulation.yml",
25
+ "data/targets/_example-backend.yml",
18
26
  "data/targets/testers-ai.yml",
19
- "docs/GETTING-STARTED.md"
27
+ "docs/GETTING-STARTED.md",
28
+ "docs/MOBILE-SETUP.md",
29
+ "docs/BACKEND-VERIFICATION.md"
20
30
  ],
21
31
  "scripts": {
22
32
  "sync:skills": "rm -rf skills && cp -R .claude/skills skills",
@@ -35,6 +45,10 @@
35
45
  "qa",
36
46
  "testing",
37
47
  "playwright",
48
+ "maestro",
49
+ "mobile-testing",
50
+ "ios-simulator",
51
+ "android-emulator",
38
52
  "ai",
39
53
  "claude",
40
54
  "skills",
@@ -55,13 +69,13 @@
55
69
  "url": "https://github.com/willcoliveira/qualiow-exploratory-testing-skills/issues"
56
70
  },
57
71
  "dependencies": {
58
- "@playwright/cli": "^0.1.6",
72
+ "@playwright/cli": "^0.1.15",
59
73
  "chalk": "^5.3.0",
60
- "commander": "^13.1.0",
61
- "glob": "^11.0.0",
74
+ "commander": "^15.0.0",
75
+ "glob": "^13.0.6",
62
76
  "handlebars": "^4.7.8",
63
- "yaml": "^2.7.0",
64
- "zod": "^3.24.0"
77
+ "yaml": "^2.9.0",
78
+ "zod": "^4.4.3"
65
79
  },
66
80
  "peerDependencies": {
67
81
  "@playwright/test": "^1.59.1"
@@ -72,9 +86,9 @@
72
86
  }
73
87
  },
74
88
  "devDependencies": {
75
- "@types/node": "^22.0.0",
89
+ "@types/node": "^26.1.1",
76
90
  "tsup": "^8.4.0",
77
- "typescript": "^5.7.0",
78
- "vitest": "^3.0.0"
91
+ "typescript": "^5.9.3",
92
+ "vitest": "^4.1.10"
79
93
  }
80
94
  }
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env bash
2
+ # doctor-mobile.sh — read-only preflight for the qa-explore-mobile skill.
3
+ #
4
+ # Verifies that everything mobile-cli needs to drive iOS Simulator Safari/native and
5
+ # Android Emulator Chrome/native is present on THIS machine. Installs NOTHING — it only
6
+ # reports ✓/✗ with the exact fix command for each gap. The mobile session's 00-setup
7
+ # phase runs this first so a fresh machine fails fast with actionable guidance instead of
8
+ # a cryptic error deep in a session.
9
+ #
10
+ # Usage:
11
+ # scripts/doctor-mobile.sh # check both platforms
12
+ # scripts/doctor-mobile.sh --ios # iOS only
13
+ # scripts/doctor-mobile.sh --android # Android only
14
+ # scripts/doctor-mobile.sh --quiet # only print failures + final verdict
15
+ #
16
+ # Exit code: 0 if all REQUIRED checks for the requested platform(s) pass, 1 otherwise.
17
+
18
+ set -uo pipefail
19
+
20
+ PLATFORM="both"
21
+ QUIET=0
22
+ for arg in "$@"; do
23
+ case "$arg" in
24
+ --ios) PLATFORM="ios" ;;
25
+ --android) PLATFORM="android" ;;
26
+ --quiet) QUIET=1 ;;
27
+ -h|--help) sed -n '2,18p' "$0"; exit 0 ;;
28
+ *) echo "unknown arg: $arg" >&2; exit 2 ;;
29
+ esac
30
+ done
31
+
32
+ # ---- resolution helpers (mirror what bin/mcli does, with fallbacks) ----
33
+ ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
34
+ JAVA_HOME_RESOLVED="${JAVA_HOME:-}"
35
+ if [ -z "$JAVA_HOME_RESOLVED" ] && [ -x /usr/libexec/java_home ]; then
36
+ JAVA_HOME_RESOLVED="$(/usr/libexec/java_home -v 17 2>/dev/null || true)"
37
+ fi
38
+ [ -z "$JAVA_HOME_RESOLVED" ] && [ -d /opt/homebrew/opt/openjdk@17 ] && JAVA_HOME_RESOLVED=/opt/homebrew/opt/openjdk@17
39
+ [ -z "$JAVA_HOME_RESOLVED" ] && [ -d /usr/local/opt/openjdk@17 ] && JAVA_HOME_RESOLVED=/usr/local/opt/openjdk@17
40
+ MAESTRO_BIN="$HOME/.maestro/bin/maestro"
41
+
42
+ PASS=0; FAIL=0; WARN=0
43
+ ok() { PASS=$((PASS+1)); [ "$QUIET" = 1 ] || printf ' \033[32m✓\033[0m %s\n' "$1"; }
44
+ bad() { FAIL=$((FAIL+1)); printf ' \033[31m✗\033[0m %s\n ↳ fix: %s\n' "$1" "$2"; }
45
+ warn() { WARN=$((WARN+1)); [ "$QUIET" = 1 ] || printf ' \033[33m!\033[0m %s\n ↳ %s\n' "$1" "$2"; }
46
+ hdr() { [ "$QUIET" = 1 ] || printf '\n\033[1m%s\033[0m\n' "$1"; }
47
+
48
+ # ============================ COMMON ============================
49
+ hdr "Common"
50
+ if [ "$(uname -s)" = "Darwin" ]; then ok "macOS ($(sw_vers -productVersion 2>/dev/null))"; else
51
+ bad "not macOS — iOS sims need macOS; Android may work on Linux but is unverified here" "use a Mac"; fi
52
+
53
+ if command -v brew >/dev/null 2>&1; then ok "Homebrew ($(brew --version 2>/dev/null | head -1))"; else
54
+ bad "Homebrew not found" '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'; fi
55
+
56
+ if command -v node >/dev/null 2>&1; then
57
+ NV="$(node -v | sed 's/^v//')"; NMAJ="${NV%%.*}"
58
+ if [ "${NMAJ:-0}" -ge 18 ]; then ok "Node $NV (>=18)"; else bad "Node $NV is < 18" "brew install node (or nvm install --lts)"; fi
59
+ else bad "Node not found" "brew install node"; fi
60
+
61
+ if [ -x "$MAESTRO_BIN" ] || command -v maestro >/dev/null 2>&1; then
62
+ ok "Maestro present ($MAESTRO_BIN)"
63
+ else bad "Maestro not found" 'curl -Ls "https://get.maestro.mobile.dev" | bash'; fi
64
+
65
+ # ============================ ANDROID ============================
66
+ if [ "$PLATFORM" = "both" ] || [ "$PLATFORM" = "android" ]; then
67
+ hdr "Android"
68
+ # JDK 17
69
+ if [ -n "$JAVA_HOME_RESOLVED" ] && [ -x "$JAVA_HOME_RESOLVED/bin/java" ]; then
70
+ JV="$("$JAVA_HOME_RESOLVED/bin/java" -version 2>&1 | head -1)"
71
+ case "$JV" in *\"17.*) ok "JDK 17 ($JAVA_HOME_RESOLVED)";; *) warn "Java found but not 17: $JV" "Android SDK tools expect JDK 17 — brew install openjdk@17";; esac
72
+ else bad "JDK 17 not found" "brew install openjdk@17"; fi
73
+
74
+ # SDK root
75
+ if [ -d "$ANDROID_HOME" ]; then ok "Android SDK root ($ANDROID_HOME)"; else
76
+ bad "Android SDK not found at $ANDROID_HOME" "scripts/setup-mobile.sh (installs cmdline-tools + SDK)"; fi
77
+
78
+ # adb / emulator / sdkmanager / avdmanager
79
+ for tuple in \
80
+ "platform-tools/adb:adb:sdkmanager 'platform-tools'" \
81
+ "emulator/emulator:emulator:sdkmanager 'emulator'" ; do
82
+ rel="${tuple%%:*}"; rest="${tuple#*:}"; label="${rest%%:*}"; fix="${rest#*:}"
83
+ if [ -x "$ANDROID_HOME/$rel" ]; then ok "$label ($ANDROID_HOME/$rel)"; else bad "$label missing" "$fix"; fi
84
+ done
85
+ SDKMGR="$(ls "$ANDROID_HOME"/cmdline-tools/*/bin/sdkmanager 2>/dev/null | head -1)"
86
+ AVDMGR="$(ls "$ANDROID_HOME"/cmdline-tools/*/bin/avdmanager 2>/dev/null | head -1)"
87
+ if [ -n "$SDKMGR" ]; then ok "sdkmanager present"; else bad "sdkmanager (cmdline-tools) missing" "brew install --cask android-commandlinetools (or Android Studio)"; fi
88
+ [ -n "$AVDMGR" ] && ok "avdmanager present"
89
+
90
+ # AVD existing? (Chrome needs a Google Play system image)
91
+ if [ -x "$ANDROID_HOME/emulator/emulator" ]; then
92
+ AVDS="$("$ANDROID_HOME/emulator/emulator" -list-avds 2>/dev/null)"
93
+ if [ -n "$AVDS" ]; then ok "AVD(s) available: $(echo "$AVDS" | tr '\n' ' ')"; else
94
+ bad "no Android AVD created" "scripts/setup-mobile.sh (creates a Google-Play API-35 AVD)"; fi
95
+ fi
96
+ warn "Chrome-on-Android needs a Google **Play** (or google_apis) system image" "AOSP images have no com.android.chrome — verify with: adb shell pm list packages | grep chrome"
97
+ fi
98
+
99
+ # ============================ iOS ============================
100
+ if [ "$PLATFORM" = "both" ] || [ "$PLATFORM" = "ios" ]; then
101
+ hdr "iOS"
102
+ # Command Line Tools are the floor (xcode-select/xcrun exist at all); full Xcode is
103
+ # still required on top of them for simulators.
104
+ if xcode-select -p >/dev/null 2>&1; then
105
+ ok "Xcode Command Line Tools path set ($(xcode-select -p))"
106
+ else
107
+ bad "Xcode Command Line Tools not installed" "xcode-select --install"
108
+ fi
109
+ if ! command -v xcrun >/dev/null 2>&1; then
110
+ bad "xcrun not found (no Xcode command line tools)" "xcode-select --install"
111
+ else
112
+ XPATH="$(xcode-select -p 2>/dev/null)"
113
+ case "$XPATH" in
114
+ *Xcode*.app*|*/Xcode/*) ok "Full Xcode selected ($XPATH)";;
115
+ *) bad "only Command Line Tools selected ($XPATH) — full Xcode is required for simulators" "install Xcode from the App Store, then: sudo xcode-select -s /Applications/Xcode.app && sudo xcodebuild -license accept";;
116
+ esac
117
+ if xcodebuild -version >/dev/null 2>&1; then ok "xcodebuild ($(xcodebuild -version 2>/dev/null | head -1))"; else
118
+ warn "xcodebuild not runnable (license not accepted?)" "sudo xcodebuild -license accept"; fi
119
+ RT="$(xcrun simctl list runtimes 2>/dev/null | grep -i 'iOS' | head -1)"
120
+ if [ -n "$RT" ]; then ok "iOS runtime: $(echo "$RT" | sed 's/ (.*//')"; else
121
+ bad "no iOS simulator runtime installed" "Xcode ▸ Settings ▸ Components ▸ install an iOS runtime (or: xcodebuild -downloadPlatform iOS)"; fi
122
+ # A usable simulator device
123
+ DEV="$(xcrun simctl list devices available 2>/dev/null | grep -iE 'iPhone' | head -1 | sed 's/^[[:space:]]*//')"
124
+ if [ -n "$DEV" ]; then ok "iPhone simulator available: ${DEV%% (*}"; else
125
+ bad "no available iPhone simulator device" "scripts/setup-mobile.sh (creates one), or: xcrun simctl create qa-iphone com.apple.CoreSimulator.SimDeviceType.iPhone-16 <runtime>"; fi
126
+ # Safari is always preinstalled on iOS sims — no check needed.
127
+ # WebKit DOM bridge (bin/wk-ios): restores JS eval / DOM assertions in sim Safari.
128
+ if command -v ios_webkit_debug_proxy >/dev/null 2>&1; then
129
+ ok "ios-webkit-debug-proxy present (bin/wk-ios DOM bridge)"
130
+ else
131
+ warn "ios-webkit-debug-proxy missing — bin/wk-ios (Safari JS/DOM eval) unavailable" "brew install ios-webkit-debug-proxy"
132
+ fi
133
+ fi
134
+ fi
135
+
136
+ # ============================ VERDICT ============================
137
+ printf '\n\033[1mSummary:\033[0m %d ok, %d warning(s), %d failure(s)\n' "$PASS" "$WARN" "$FAIL"
138
+ if [ "$FAIL" -gt 0 ]; then
139
+ printf '\033[31mNOT READY\033[0m — resolve the ✗ items above (or run: scripts/setup-mobile.sh) then re-run this doctor.\n'
140
+ exit 1
141
+ fi
142
+ printf '\033[32mREADY\033[0m — mobile-cli can drive the requested platform(s).\n'
143
+ exit 0