qualiow-exploratory-testing 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +177 -0
  2. package/bin/mcli +16 -0
  3. package/bin/mobile-cli.mjs +573 -0
  4. package/bin/wadb +6 -0
  5. package/bin/wk-ios +58 -0
  6. package/bin/wkeval.mjs +44 -0
  7. package/data/domains/_default.md +181 -0
  8. package/data/domains/ecommerce.md +192 -0
  9. package/data/domains/fintech.md +200 -0
  10. package/data/domains/identity.md +88 -0
  11. package/data/domains/identity.yml +138 -0
  12. package/data/domains/marketing.md +194 -0
  13. package/data/domains/saas.md +197 -0
  14. package/data/knowledge/changelog.yml +86 -0
  15. package/data/knowledge/learned-patterns.md +172 -1
  16. package/data/knowledge/manifest.yml +210 -4
  17. package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
  18. package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
  19. package/data/knowledge/releases/v0.2.0/release.yml +20 -0
  20. package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
  21. package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
  22. package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
  23. package/data/knowledge/releases/v0.3.0/release.yml +25 -0
  24. package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
  25. package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
  26. package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
  27. package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
  28. package/data/knowledge/releases/v0.4.0/release.yml +28 -0
  29. package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
  30. package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
  31. package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
  32. package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
  33. package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
  34. package/data/knowledge/releases/v0.5.0/release.yml +33 -0
  35. package/data/targets/_example-backend.yml +123 -0
  36. package/data/targets/_example-mobile-emulation.yml +52 -0
  37. package/data/targets/_example-native-mobile.yml +61 -0
  38. package/data/targets/_example-sim-android-chrome.yml +81 -0
  39. package/data/targets/_example-sim-ios-safari.yml +80 -0
  40. package/data/templates/api-probe-matrix.md +99 -0
  41. package/data/templates/coverage-map.md +14 -0
  42. package/data/templates/expected-behaviour.md +93 -0
  43. package/dist/cli/index.js +137 -5
  44. package/dist/cli/index.js.map +1 -1
  45. package/dist/index.d.ts +350 -527
  46. package/dist/index.js +144 -5
  47. package/dist/index.js.map +1 -1
  48. package/docs/BACKEND-VERIFICATION.md +277 -0
  49. package/docs/MOBILE-SETUP.md +92 -0
  50. package/package.json +26 -12
  51. package/scripts/doctor-mobile.sh +143 -0
  52. package/scripts/setup-mobile.sh +213 -0
  53. package/skills/qa-explore/phases/00-setup.md +8 -1
  54. package/skills/qa-explore/phases/01-auth.md +4 -0
  55. package/skills/qa-explore-mobile/SKILL.md +146 -0
  56. package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
  57. package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
  58. package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
  59. package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
  60. package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
  61. package/skills/qa-explore-mobile/phases/05-features.md +72 -0
  62. package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
  63. package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
  64. package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
  65. package/skills/qa-explore-quick/SKILL.md +6 -0
  66. package/skills/qa-verify-backend/SKILL.md +163 -0
  67. package/skills/qa-verify-backend/phases/00-setup.md +97 -0
  68. package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
  69. package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
  70. package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
  71. package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
  72. package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
  73. package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
  74. package/skills/qa-verify-backend/references/api-probes.md +141 -0
  75. package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
  76. package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
  77. package/skills/qa-verify-backend/references/payload-verification.md +112 -0
  78. package/skills/qa-verify-backend/references/release-readiness.md +97 -0
  79. package/skills/qa-verify-backend/references/safety-rules.md +90 -0
@@ -0,0 +1,61 @@
1
+ # Example target — NATIVE-MODE mobile app on a simulator/emulator.
2
+ # Copy this file, rename it, and fill in the values for your app. This documents the
3
+ # fields /qa-explore-mobile reads to run in NATIVE mode (the app itself is the system
4
+ # under test, driven via Maestro + adb / xcrun simctl through bin/mobile-cli.mjs).
5
+ #
6
+ # For WEB mode (a real device browser pointed at a URL), see the committed examples
7
+ # _example-sim-ios-safari.yml / _example-sim-android-chrome.yml instead.
8
+ #
9
+ # MODE DETECTION (done in phases/00-setup.md):
10
+ # - WEB mode -> app is a browser bundle (com.apple.mobilesafari / com.android.chrome)
11
+ # AND a web.base_url is present.
12
+ # - NATIVE mode -> app is a non-browser bundle/package with installable artifact paths
13
+ # (app_paths / apk_paths), no web.base_url.
14
+
15
+ id: example-native-mobile
16
+ name: "Example Native App (simulator/emulator)"
17
+ domain: _default
18
+ platform: ios # ios | android (required)
19
+
20
+ device:
21
+ # iOS: a simulator name or UDID. Android: an AVD name and/or a booted serial.
22
+ name: "iPhone 17" # iOS sim label
23
+ udid: "REPLACE-WITH-SIM-UDID" # iOS sim UDID (from `xcrun simctl list devices`)
24
+ # avd: my_test_avd # Android AVD name (created via Android Studio / avdmanager)
25
+ # serial: emulator-5554 # Android serial once the AVD is booted
26
+
27
+ app:
28
+ # The app under test (NOT a browser). Use whichever id matches the platform.
29
+ bundle_id: com.example.myapp # iOS bundle id
30
+ # package: com.example.myapp # Android application id
31
+ #
32
+ # Installable artifact(s) used to verify/install the build on the device. If the app is
33
+ # already installed and you don't need to (re)install, these can be omitted.
34
+ app_paths: # iOS .app bundle(s) (simulator build)
35
+ - "/absolute/path/to/MyApp.app"
36
+ # apk_paths: # Android APK(s)
37
+ # - "/absolute/path/to/myapp-release.apk"
38
+
39
+ # OPTIONAL — only needed if you want /qa-explore-mobile to be able to rebuild from source.
40
+ # Skipped unless the user passes --rebuild or the session goal mentions "latest"/"fresh".
41
+ # build_commands are referenced generically by 00-setup.md — put the EXACT commands your
42
+ # project uses here (no framework is assumed by the skill).
43
+ # source_repo:
44
+ # path: "/absolute/path/to/app-source"
45
+ # build_commands:
46
+ # # Each command should build AND install onto the configured device.
47
+ # android_release: "..." # e.g. your project's release build+install command
48
+ # ios_debug: "..." # e.g. your project's sim build+install command
49
+ # refresh: "git fetch && git status -sb" # used to detect upstream drift before rebuild
50
+
51
+ auth:
52
+ # in-app -> account creation / login happens inside the app (see phases/01-auth.md).
53
+ # none -> no auth.
54
+ strategy: in-app
55
+ # For staging/test builds that accept a fixed one-time code, record it here so the
56
+ # auth phase can use it instead of a real OTP. Leave unset for real OTP delivery.
57
+ # static_otp: "000000"
58
+ # test_email_pattern: "qa+{unique}@example.com" # if the app needs a unique email per signup
59
+
60
+ scope:
61
+ start_screen: launch # the app's first screen after launch
@@ -0,0 +1,81 @@
1
+ # Template target — WEB-MODE mobile app in the REAL Android Emulator Chrome.
2
+ # Copy this file, rename it, and point web.base_url at your mobile web app.
3
+ # For NATIVE mode (an installed app under test), see _example-native-mobile.yml.
4
+ id: example-sim-android-chrome
5
+ name: "Example Mobile Web App — Android Emulator Chrome"
6
+ platform: android
7
+ domain: _default
8
+ # This target drives a WEB app through a real mobile browser on a real emulator.
9
+ # The "app" is just Chrome; the system under test is the web app served at web.base_url.
10
+ # mobile-cli treats Chrome like any other app: set-app -> launch -> open-url -> snapshot.
11
+ device:
12
+ avd: qa_pixel_api35 # standard AVD created by scripts/setup-mobile.sh (Google-Play API 35). Override to any existing AVD (run: emulator -list-avds).
13
+ serial: emulator-5554 # populated once the AVD is booted
14
+ app:
15
+ # The browser IS the app. No native build/install needed — Chrome is preinstalled on
16
+ # Google-Play emulator images, so there is no apk_paths / build step here.
17
+ # NOTE: stock AOSP / non-Play emulator images may NOT ship Chrome — they often have only
18
+ # a basic AOSP browser or none. If `adb shell pm list packages | grep chrome` shows no
19
+ # com.android.chrome, use a Google-APIs/Play image AVD rather than assuming it's present.
20
+ package: com.android.chrome
21
+ web:
22
+ base_url: "https://staging.m.example.com"
23
+ start_url: "https://staging.m.example.com/login"
24
+ # Many mobile hosts enforce a mobile User-Agent and redirect desktop UAs to the desktop
25
+ # host. A real emulator Chrome sends a genuine mobile UA, so this is a non-issue.
26
+ auth:
27
+ strategy: interactive-sso # interactive SSO + MFA — cannot be automated
28
+ identity_provider: "your IdP (e.g. Microsoft Entra, Okta, Auth0)"
29
+ login_url: "https://staging.m.example.com/login"
30
+ # One-time interactive login, then the browser profile persists the session.
31
+ # Do NOT attempt to script the MFA challenge. Use strategy: none for public apps.
32
+ scope:
33
+ start_screen: login
34
+ start_url: "https://staging.m.example.com/login"
35
+ notes: |
36
+ WHAT THIS IS
37
+ A browser-as-app target: Android Emulator Chrome pointed at a mobile web app. The
38
+ Maestro/adb route used by mobile-cli drives the real Android Chrome engine.
39
+
40
+ NO NATIVE BUILD NEEDED (but verify Chrome exists — see app.package note above)
41
+ Boot the AVD, then:
42
+ bin/wadb devices # boot `emulator -avd <your-avd> ...` first if empty
43
+ bin/wadb shell pm list packages | grep chrome # confirm com.android.chrome is present
44
+ export MOBILE_CLI_STATE=/tmp/<target>-state.json
45
+ bin/mcli set-device emulator-5554
46
+ bin/mcli set-app com.android.chrome
47
+ bin/mcli launch
48
+ bin/mcli open-url <web.base_url>
49
+ bin/mcli snapshot
50
+ Use a dedicated MOBILE_CLI_STATE so you don't clobber another target's default state.
51
+ On a fresh Chrome profile you may see a one-time first-run / "Open tabs" promo bubble —
52
+ mobile-cli filters the promo (com.android.chrome:id/message) out of snapshots; if a
53
+ full-screen first-run/sign-in-to-Chrome interstitial appears, dismiss it once by hand.
54
+
55
+ A11Y-vs-DOM LIMITATION (document, do not try to solve)
56
+ snapshot reads the accessibility tree, not the DOM. Web content IS exposed to a11y —
57
+ headings, links, buttons, inputs and visible text become tappable refs. But versus
58
+ playwright-cli you LOSE: CSS/DOM selectors, console & network introspection, route-based
59
+ mocking, and reliable exact-DOM assertions. You drive purely by visible/accessible labels
60
+ and tap coordinates.
61
+ mobile-cli auto-filters Chrome's own chrome (omnibox/URL bar, tab switcher, menu, toolbar,
62
+ infobars/promos) from snapshots when the app id is a known browser, so the ref tree is the
63
+ page, not the browser furniture. The generic android:id/content root and the system status
64
+ bar may still appear (no stable app-specific id to filter on).
65
+
66
+ AUTH REALITY — SSO + MFA (do NOT automate)
67
+ If the app logs in via an interactive IdP with MFA: unlike Playwright, there is NO
68
+ transferable storage_state you can inject into a real emulator browser. The practical
69
+ approach is a ONE-TIME INTERACTIVE LOGIN performed by a human in the emulator's Chrome:
70
+ open the start_url, complete the username/password + MFA prompt by hand once. The
71
+ emulator persists Chrome's profile (cookies/session) across app launches and across
72
+ mobile-cli `launch`/`open-url` calls (as long as the AVD is not wiped / `pm clear`ed), so
73
+ subsequent exploration reuses that session without re-authenticating until it expires.
74
+ Never script the MFA step. Avoid `relaunch-clean` on this target — `pm clear
75
+ com.android.chrome` wipes the logged-in profile and forces a fresh SSO+MFA login.
76
+
77
+ EMULATOR / EXPLORATION NOTES
78
+ Android Chrome DOES expose the live page to uiautomator/Maestro's hierarchy, so snapshot
79
+ refs map to real web elements. Hardware BACK works here (`press BACK`) unlike iOS. If the
80
+ emulator is slow/flaky to boot or Chrome is absent on the chosen image, document that and
81
+ fall back to the iOS Safari target rather than blocking — the two engines are independent.
@@ -0,0 +1,80 @@
1
+ # Template target — WEB-MODE mobile app in the REAL iOS Simulator Safari.
2
+ # Copy this file, rename it, and point web.base_url at your mobile web app.
3
+ # For NATIVE mode (an installed app under test), see _example-native-mobile.yml.
4
+ id: example-sim-ios-safari
5
+ name: "Example Mobile Web App — iOS Simulator Safari"
6
+ platform: ios
7
+ domain: _default
8
+ # This target drives a WEB app through a real mobile browser on a real simulator.
9
+ # The "app" is just Safari; the system under test is the web app served at web.base_url.
10
+ # mobile-cli treats Safari like any other app: set-app -> launch -> open-url -> snapshot.
11
+ device:
12
+ # Resolve by NAME, not a hardcoded UDID (UDIDs are per-machine). scripts/setup-mobile.sh
13
+ # creates a standard sim named "qa-iphone"; on a machine that already has sims you can set
14
+ # this to any available iPhone (see: xcrun simctl list devices available). 00-setup boots
15
+ # the named device (creating it if missing) and resolves its UDID at runtime.
16
+ name: "qa-iphone"
17
+ # udid: optional — if set, takes precedence over name. Leave unset for portability.
18
+ app:
19
+ # The browser IS the app. No native build/install needed — Safari is preinstalled
20
+ # on every iOS simulator runtime, so there is no app_paths / build step here.
21
+ bundle_id: com.apple.mobilesafari
22
+ web:
23
+ base_url: "https://staging.m.example.com"
24
+ start_url: "https://staging.m.example.com/login"
25
+ # Many mobile hosts enforce a mobile User-Agent and redirect desktop UAs to the desktop
26
+ # host. A real simulator Safari already sends a genuine mobile UA, so this is a non-issue
27
+ # here (it only bites Playwright/desktop browsers).
28
+ auth:
29
+ strategy: interactive-sso # interactive SSO + MFA — cannot be automated
30
+ identity_provider: "your IdP (e.g. Microsoft Entra, Okta, Auth0)"
31
+ login_url: "https://staging.m.example.com/login"
32
+ # One-time interactive login, then the browser profile persists the session.
33
+ # Do NOT attempt to script the MFA challenge. Use strategy: none for public apps.
34
+ scope:
35
+ start_screen: login
36
+ start_url: "https://staging.m.example.com/login"
37
+ notes: |
38
+ WHAT THIS IS
39
+ A browser-as-app target: iOS Simulator Safari pointed at a mobile web app. The genuine
40
+ engine matters here — Playwright CANNOT drive iOS Simulator Safari, and Playwright's
41
+ "webkit" is desktop macOS WebKit, NOT the simulator's WebKit. The Maestro/simctl route
42
+ used by mobile-cli is the only way to exercise the real iOS engine.
43
+
44
+ NO NATIVE BUILD NEEDED
45
+ Safari ships with every simulator runtime — nothing to build or install. Just boot the
46
+ sim (`xcrun simctl boot <udid>; open -a Simulator`), then:
47
+ export MOBILE_CLI_STATE=/tmp/<target>-state.json
48
+ bin/mcli set-device <udid>
49
+ bin/mcli set-app com.apple.mobilesafari
50
+ bin/mcli launch
51
+ bin/mcli open-url <web.base_url>
52
+ bin/mcli snapshot
53
+ Use a dedicated MOBILE_CLI_STATE so you don't clobber another target's default state.
54
+
55
+ A11Y-vs-DOM LIMITATION (document, do not try to solve)
56
+ snapshot reads the accessibility tree, not the DOM. Web content IS exposed to a11y —
57
+ headings, links, buttons, inputs and visible text become tappable refs. But versus
58
+ playwright-cli you LOSE: CSS/DOM selectors, console & network introspection, route-based
59
+ mocking, and reliable exact-DOM assertions. You drive purely by visible/accessible labels
60
+ and tap coordinates.
61
+ mobile-cli auto-filters Safari's own chrome (URL/address bar, tabs, toolbar, on-screen
62
+ keyboard) from snapshots when the app id is a known browser, so the ref tree is the page,
63
+ not the browser furniture. Residual non-app noise that has no stable a11y id (the iOS
64
+ status bar clock/wifi/battery and transient "NN%" page-load progress) may still appear.
65
+
66
+ AUTH REALITY — SSO + MFA (do NOT automate)
67
+ If the app logs in via an interactive IdP with MFA: unlike Playwright, there is NO
68
+ transferable storage_state you can inject into a real simulator browser. The practical
69
+ approach is a ONE-TIME INTERACTIVE LOGIN performed by a human in the sim's Safari:
70
+ open the start_url, complete the username/password + MFA prompt by hand once. The
71
+ simulator persists Safari's profile (cookies/session) across app launches and across
72
+ mobile-cli `launch`/`open-url` calls, so subsequent exploration reuses that session
73
+ without re-authenticating (until the session expires or the sim is erased). Never script
74
+ the MFA step. If you land on the IdP domain mid-session, that just means the session
75
+ lapsed — hand back to a human for re-login.
76
+
77
+ iOS SIM CONSTRAINTS (inherited from the native iOS target)
78
+ No hardware BACK key (use an on-screen/web back control via snapshot+click), no biometric
79
+ flow, no easy CLI rotation. iOS a11y can also aggregate sibling nodes into one comma-joined
80
+ label on some screens — tap by coordinates from a `snapshot --full` when that happens.
@@ -0,0 +1,99 @@
1
+ # API Probe Matrix
2
+
3
+ > CONFIDENTIAL — internal QA artifact. Contains endpoint behaviour and environment
4
+ > configuration. Keep local. Redact response bodies before saving.
5
+
6
+ ## Session Reference
7
+
8
+ | Property | Value |
9
+ |-----------------|-----------------------------------------|
10
+ | Session ID | [e.g., SESSION-2026-03-28-001] |
11
+ | Ticket | [ticket id, or "none"] |
12
+ | Endpoint | [`POST /api/v1/…` — note the verb] |
13
+ | Date | [YYYY-MM-DD] |
14
+ | Observation mode| [in-page fetch / out-of-browser request / unauthenticated] |
15
+ | Identity | [role or persona used — never a credential] |
16
+
17
+ ---
18
+
19
+ ## Environment Fingerprint
20
+
21
+ Fill this in **before** the matrix. Without it the results below are uninterpretable.
22
+
23
+ | Component | Environment A `[name]` | Environment B `[name]` |
24
+ |------------------|------------------------|------------------------|
25
+ | Frontend build | [build id / date] | [build id / date] |
26
+ | API build | [build id / commit] | [build id / commit] |
27
+ | Service under test | [build id / commit] | [build id / commit] |
28
+ | Changed path selected? | [yes / no — name the flag or config that decides it] | [yes / no] |
29
+ | Commit under test deployed? | [ancestor / not an ancestor] | [ancestor / not an ancestor] |
30
+
31
+ > **Same build, different behaviour ⇒ configuration, not deploy lag.**
32
+ > If the changed path is not selected in an environment, every AC that depends on it is
33
+ > `NOT-REACHABLE` there — not `PASS`, not `FAIL`.
34
+
35
+ ---
36
+
37
+ ## Cases
38
+
39
+ One row per case. The **label** is the stable identifier the report, the bug and the
40
+ re-run all refer to. Record passes as well as failures — a matrix of only failures
41
+ cannot show a regression later.
42
+
43
+ | # | Label | Request | Expected | Env A: status / count / rows | Env B: status / count / rows | Verdict |
44
+ |---|-------|---------|----------|------------------------------|------------------------------|---------|
45
+ | 1 | identity | `{"query":"<known value>"}` | 200, stable non-zero | 200 / 77 / 20 | 200 / 91 / 20 | ✅ |
46
+ | 2 | empty string | `{"query":""}` | 400 + error body | | | |
47
+ | 3 | body absent | `{}` | 400 + error body | | | |
48
+ | 4 | metachar — lone `(` | `{"query":"("}` | 400 or escaped 200 | | | |
49
+ | 5 | limit above cap | `?limit=<cap+1>` | 400, or capped | | | |
50
+ | 6 | invalid enum | `{"filter":["<nonsense>"]}` | 400 + error body | | | |
51
+ | 7 | second identity | same request, other user | scoped result | | | |
52
+
53
+ > **Compare status codes and shapes. Never compare absolute counts** — different
54
+ > environments hold different data and it drifts between runs.
55
+
56
+ ### Case families to cover
57
+
58
+ Boundary lengths · tokenisation and whitespace · metacharacters of the underlying query
59
+ language · empty / null / absent / whitespace-only · enum values valid, invalid, empty,
60
+ wildcard, injected · pagination bounds and types · type confusion · casing and
61
+ normalisation · second identity and second scope.
62
+
63
+ ---
64
+
65
+ ## Findings
66
+
67
+ One row per finding, not one per failing case — a family of inputs failing on the same
68
+ character is **one** bug with several examples.
69
+
70
+ | ID | Cases | Finding | Surface | Severity | Bug report |
71
+ |----|-------|---------|---------|----------|------------|
72
+ | F1 | 4, 9–14 | [what is wrong, in one line] | API / UI / both | High | `bugs/BUG-0xx-….md` |
73
+
74
+ **Surface** classifies where the defect lives, from the differential pass:
75
+
76
+ | Surface | Meaning | Owner |
77
+ |---------|---------|-------|
78
+ | **Both** | The UI faithfully reflects the endpoint | Service |
79
+ | **API only** | Real defect the client's guard is hiding — reachable by every other client | Service |
80
+ | **UI only** | The client invents or masks behaviour the service does not have | Client |
81
+ | **Neither** | Environment or access noise, not a product defect | — |
82
+
83
+ ---
84
+
85
+ ## Evidence
86
+
87
+ | File | Contents |
88
+ |------|----------|
89
+ | `evidence/fingerprint.md` | Build ids, flag values, commit ancestry |
90
+ | `evidence/<case-set>-raw.json` | Raw per-case output, redacted |
91
+ | `probes/<probe>.js` | The probe script, so this is repeatable |
92
+
93
+ ---
94
+
95
+ ## Notes and Open Questions
96
+
97
+ - [What could not be checked, and exactly what access would unblock it]
98
+ - [Which environment would answer a `NOT-REACHABLE` verdict]
99
+ - [Counts recorded here are valid for the date above only]
@@ -23,6 +23,20 @@
23
23
  - **tested** -- Area was fully explored with the planned heuristics.
24
24
  - **partial** -- Area was visited but not all heuristics or paths were exercised.
25
25
  - **not-tested** -- Area was in scope but was not reached during the session.
26
+ - **code-verified-only** -- Believed correct from reading the source, never observed
27
+ running. This is `UNVERIFIABLE`, **not** a pass -- keep it visually distinct so it
28
+ cannot be skimmed as green.
29
+
30
+ Two distinctions that routinely produce a false **tested**:
31
+
32
+ - **Consistent is not causal.** A setting whose current value happens to match the output
33
+ proves nothing until the setting is changed and the output is watched to follow.
34
+ - **The data has to reach the case.** An area whose behaviour depends on values no
35
+ reachable record has -- a negative, a zero, an empty collection -- is **not-tested** for
36
+ that behaviour. Find such a record, create one, or say plainly what data would settle it.
37
+
38
+ Name the single **not-tested** item that carries the most risk. In a map full of green rows
39
+ that is the line the reader acts on.
26
40
 
27
41
  ---
28
42
 
@@ -0,0 +1,93 @@
1
+ # Expected Behaviour Specification
2
+
3
+ > CONFIDENTIAL — internal QA artifact.
4
+
5
+ Use this when a verification session finds behaviour nobody wrote an acceptance criterion
6
+ for — which is most of what an API probe turns up. A bug report says *this is wrong*. This
7
+ says *here is what right looks like*, in enough detail that someone can implement it and
8
+ someone else can verify it. It is frequently the most valuable artifact a session produces,
9
+ because it converts a pile of findings into one reviewable decision.
10
+
11
+ Write it **per area**, not per case. Ten failing inputs that share a cause are one section.
12
+
13
+ ## Header
14
+
15
+ | Property | Value |
16
+ |----------|-------|
17
+ | Date | [YYYY-MM-DD] |
18
+ | Endpoint / surface | [`POST /api/v1/…` — note the verb] |
19
+ | Environment(s) observed | [name, and the build each one was running] |
20
+ | Source findings | [link to the session that produced the observations] |
21
+ | Status | [proposed / agreed / implemented] |
22
+
23
+ ---
24
+
25
+ ## [N]. [Area — e.g. Minimum input length]
26
+
27
+ **Observed**
28
+
29
+ > What the system does today, as evidence. Exact request, exact status, exact response.
30
+ > Note which environments it holds in — behaviour frequently differs, and that difference
31
+ > is part of the finding.
32
+
33
+ ```
34
+ {"query":"L"} → 200 / <the entire collection> [alpha]
35
+ {"query":"L"} → 200 / 0 [beta]
36
+ ```
37
+
38
+ **Expected**
39
+
40
+ > What it should do. Numbered, falsifiable statements — each one a testable assertion, not
41
+ > a sentiment. Include the boundary values themselves.
42
+
43
+ - An input whose tokens are all below the minimum → **`400`**, with the minimum named in
44
+ the message.
45
+ - "No usable tokens after filtering" and "no query supplied" are **different branches**.
46
+ Neither falls through to a match-all.
47
+ - The match-all path is not reachable from this parameter at any input. If "browse
48
+ everything" is wanted, it is an explicit endpoint with a hard server-side cap.
49
+ - The minimum is **N**, matching the existing rule, so that [known-good short inputs] keep
50
+ working.
51
+ - Client: no request fires below N; the user sees a hint rather than an empty result.
52
+
53
+ ### Decisions this section forces
54
+
55
+ State them explicitly — leaving them implicit is how a spec gets implemented three
56
+ different ways:
57
+
58
+ - **Reject, or clamp?** *Reject.* Silently serving a different page size than was asked for
59
+ breaks every caller that trusts the value back.
60
+ - **`400`, or a silent empty result?** *`400`.* A silent zero is indistinguishable from a
61
+ genuine no-match, and it will be read as data.
62
+ - **Where does validation live?** At the layer that covers **every** implementation of the
63
+ feature — not in one of two code paths, and never only in the client.
64
+ - **What must not change?** List the valid inputs and their current answers, so the fix is
65
+ verifiable as non-breaking.
66
+
67
+ ---
68
+
69
+ ## Priority
70
+
71
+ Rank the areas by what is reachable by real users **today**, not by how bad each one reads.
72
+ Name explicitly which finding is live in production right now, and which only appears in an
73
+ environment nobody ships from — the two need very different urgency, and the difference is
74
+ invisible unless someone says it.
75
+
76
+ ---
77
+
78
+ ## Reply — plain English
79
+
80
+ > Ready to paste into a ticket comment, a pull request or a chat reply. Same content, no
81
+ > jargon, no status codes unless they carry meaning to the reader. Written for the person
82
+ > who has to decide whether to fund the fix.
83
+
84
+ For each area, in a short paragraph:
85
+
86
+ - **What happens**, in a sentence a non-engineer can picture.
87
+ - **Whether it is live for users**, and where.
88
+ - **What the fix is**, in outline — enough to size it, not enough to design it.
89
+ - **Where you agree with their reading**, if they proposed one, and precisely where you do
90
+ not, and why.
91
+
92
+ Close with the one thing you would put ahead of everything else in the list, and the reason
93
+ — usually that it is the only one users are hitting today.
package/dist/cli/index.js CHANGED
@@ -228,21 +228,41 @@ import { ZodError } from "zod";
228
228
  // src/schemas/target.schema.ts
229
229
  import { z } from "zod";
230
230
  var AuthConfigSchema = z.object({
231
- strategy: z.enum(["none", "storage_state", "credentials", "token"]),
231
+ // 'in-app' (native mobile: login/signup inside the app) and 'interactive-sso'
232
+ // (mobile web: one-time human SSO/MFA login the browser profile persists) are
233
+ // mobile-only strategies used by /qa-explore-mobile.
234
+ strategy: z.enum([
235
+ "none",
236
+ "storage_state",
237
+ "credentials",
238
+ "token",
239
+ "in-app",
240
+ "interactive-sso"
241
+ ]),
232
242
  state_file: z.string().optional(),
233
243
  login_url: z.string().url().optional(),
234
244
  credentials: z.object({
235
245
  username: z.string(),
236
246
  password: z.string()
237
247
  }).optional(),
238
- token: z.string().optional()
248
+ token: z.string().optional(),
249
+ identity_provider: z.string().optional(),
250
+ // Staging builds sometimes accept a fixed OTP — recorded so the auth phase can
251
+ // use it instead of waiting for a real code. Never a real credential.
252
+ static_otp: z.string().optional(),
253
+ test_email_pattern: z.string().optional()
239
254
  });
240
255
  var BrowserConfigSchema = z.object({
241
256
  headless: z.boolean(),
242
257
  viewport: z.object({
243
258
  width: z.number().int().positive(),
244
259
  height: z.number().int().positive()
245
- })
260
+ }),
261
+ // Optional engine selection + Playwright device descriptor for mobile-web
262
+ // emulation targets (e.g. engine: webkit, device: "iPhone 15").
263
+ engine: z.enum(["chromium", "webkit", "firefox"]).optional(),
264
+ channel: z.string().optional(),
265
+ device: z.string().optional()
246
266
  });
247
267
  var ScopeConfigSchema = z.object({
248
268
  start_pages: z.array(z.string()).optional(),
@@ -256,17 +276,129 @@ var SafetyConfigSchema = z.object({
256
276
  no_file_upload: z.boolean().optional(),
257
277
  no_delete_actions: z.boolean().optional()
258
278
  });
259
- var TargetConfigSchema = z.object({
279
+ var EnvironmentConfigSchema = z.object({
280
+ kind: z.enum(["dev", "ephemeral", "staging", "production"]),
281
+ destroyed_automatically: z.boolean().optional()
282
+ });
283
+ var BackendConfigSchema = z.object({
284
+ provider: z.enum(["aws"]).optional(),
285
+ aws_profile_env: z.string().optional(),
286
+ region_env: z.string().optional(),
287
+ region: z.string().optional(),
288
+ // Expected account. The skill compares it to `sts get-caller-identity` and
289
+ // stops the live lane on a mismatch rather than probing someone else's account.
290
+ account_id: z.string().optional(),
291
+ env_suffix: z.string().optional(),
292
+ // Free-form logical name -> deployed resource name (config_table, dlq, ...).
293
+ resources: z.record(z.string(), z.string()).optional(),
294
+ notes: z.string().optional()
295
+ });
296
+ var ApiSurfaceConfigSchema = z.object({
297
+ // Defaults to the target's base_url when omitted.
298
+ base_url: z.string().optional(),
299
+ auth: z.enum(["session-cookie", "bearer-env", "none"]).optional(),
300
+ // NAME of the env var holding a bearer token, for `auth: bearer-env`.
301
+ token_env: z.string().optional(),
302
+ // Persistent browser profile directory holding the authenticated session,
303
+ // e.g. `.auth/my-service-dev-profile`. Gitignored.
304
+ browser_profile: z.string().optional(),
305
+ // Endpoint returning the deployed build, used to fingerprint the environment.
306
+ version_endpoint: z.string().optional(),
307
+ // Logical name -> "VERB /path", e.g. search: "POST /api/v1/search".
308
+ endpoints: z.record(z.string(), z.string()).optional(),
309
+ // The ONLY endpoints the read-only API lane may call without asking.
310
+ probe_allowlist: z.array(z.string()).optional(),
311
+ // Other target ids to run the same matrix against, for parity comparison.
312
+ parity_targets: z.array(z.string()).optional(),
313
+ // Flag name -> where its deployed value is declared for this environment.
314
+ // A flag can select between two implementations inside one identical build.
315
+ feature_flags: z.record(z.string(), z.string()).optional(),
316
+ notes: z.string().optional()
317
+ });
318
+ var SourceBranchConfigSchema = z.object({
319
+ repo_path: z.string(),
320
+ branch: z.string(),
321
+ base_branch: z.string().optional(),
322
+ components: z.record(z.string(), z.string()).optional()
323
+ });
324
+ var WebTargetConfigSchema = z.object({
260
325
  id: z.string().min(1),
261
326
  name: z.string().min(1),
262
- base_url: z.string().url(),
327
+ // '' is allowed for the ad-hoc template (_default.yml) where the URL is
328
+ // provided at session start rather than in the config.
329
+ base_url: z.union([z.literal(""), z.string().url()]),
263
330
  domain: z.string().min(1),
264
331
  auth: AuthConfigSchema,
265
332
  browser: BrowserConfigSchema,
266
333
  scope: ScopeConfigSchema,
267
334
  safety: SafetyConfigSchema.optional(),
335
+ // Optional blocks consumed by /qa-verify-backend. A target without them is
336
+ // still a valid /qa-explore target.
337
+ environment: EnvironmentConfigSchema.optional(),
338
+ backend: BackendConfigSchema.optional(),
339
+ api: ApiSurfaceConfigSchema.optional(),
340
+ source: SourceBranchConfigSchema.optional(),
341
+ notes: z.string().optional()
342
+ });
343
+ var MobileDeviceConfigSchema = z.object({
344
+ name: z.string().optional(),
345
+ // iOS simulator name (portable across machines)
346
+ udid: z.string().optional(),
347
+ // iOS simulator UDID (takes precedence over name)
348
+ avd: z.string().optional(),
349
+ // Android AVD name
350
+ serial: z.string().optional()
351
+ // Android serial once booted (e.g. emulator-5554)
352
+ });
353
+ var MobileAppConfigSchema = z.object({
354
+ bundle_id: z.string().optional(),
355
+ // iOS bundle id (or browser bundle in web mode)
356
+ package: z.string().optional(),
357
+ // Android application id (or browser package)
358
+ app_paths: z.array(z.string()).optional(),
359
+ // iOS .app artifact(s) — native mode
360
+ apk_paths: z.array(z.string()).optional()
361
+ // Android APK(s) — native mode
362
+ });
363
+ var MobileWebConfigSchema = z.object({
364
+ base_url: z.string().url(),
365
+ start_url: z.string().url().optional()
366
+ });
367
+ var SourceRepoConfigSchema = z.object({
368
+ path: z.string(),
369
+ build_commands: z.record(z.string(), z.string()).optional()
370
+ });
371
+ var MobileScopeConfigSchema = z.object({
372
+ start_screen: z.string().optional(),
373
+ start_url: z.string().optional(),
374
+ include_patterns: z.array(z.string()).optional(),
375
+ exclude_patterns: z.array(z.string()).optional()
376
+ });
377
+ var MobileTargetConfigSchema = z.object({
378
+ id: z.string().min(1),
379
+ name: z.string().min(1),
380
+ platform: z.enum(["ios", "android"]),
381
+ // discriminates mobile from web targets
382
+ domain: z.string().min(1),
383
+ device: MobileDeviceConfigSchema,
384
+ app: MobileAppConfigSchema,
385
+ web: MobileWebConfigSchema.optional(),
386
+ // present → WEB mode; absent → NATIVE mode
387
+ auth: AuthConfigSchema,
388
+ scope: MobileScopeConfigSchema.optional(),
389
+ safety: SafetyConfigSchema.optional(),
390
+ source_repo: SourceRepoConfigSchema.optional(),
268
391
  notes: z.string().optional()
269
392
  });
393
+ var TargetConfigSchema = z.custom().superRefine((data, ctx) => {
394
+ const isMobile = typeof data === "object" && data !== null && "platform" in data;
395
+ const branch = isMobile ? MobileTargetConfigSchema : WebTargetConfigSchema;
396
+ const result = branch.safeParse(data);
397
+ if (!result.success) {
398
+ for (const issue of result.error.issues)
399
+ ctx.addIssue(issue);
400
+ }
401
+ });
270
402
 
271
403
  // src/schemas/knowledge-entry.schema.ts
272
404
  import { z as z2 } from "zod";