qualiow-exploratory-testing 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -0
- package/bin/mcli +16 -0
- package/bin/mobile-cli.mjs +573 -0
- package/bin/wadb +6 -0
- package/bin/wk-ios +58 -0
- package/bin/wkeval.mjs +44 -0
- package/data/domains/_default.md +181 -0
- package/data/domains/ecommerce.md +192 -0
- package/data/domains/fintech.md +200 -0
- package/data/domains/identity.md +88 -0
- package/data/domains/identity.yml +138 -0
- package/data/domains/marketing.md +194 -0
- package/data/domains/saas.md +197 -0
- package/data/knowledge/changelog.yml +86 -0
- package/data/knowledge/learned-patterns.md +172 -1
- package/data/knowledge/manifest.yml +210 -4
- package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
- package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
- package/data/knowledge/releases/v0.2.0/release.yml +20 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
- package/data/knowledge/releases/v0.3.0/release.yml +25 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
- package/data/knowledge/releases/v0.4.0/release.yml +28 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
- package/data/knowledge/releases/v0.5.0/release.yml +33 -0
- package/data/targets/_example-backend.yml +123 -0
- package/data/targets/_example-mobile-emulation.yml +52 -0
- package/data/targets/_example-native-mobile.yml +61 -0
- package/data/targets/_example-sim-android-chrome.yml +81 -0
- package/data/targets/_example-sim-ios-safari.yml +80 -0
- package/data/templates/api-probe-matrix.md +99 -0
- package/data/templates/coverage-map.md +14 -0
- package/data/templates/expected-behaviour.md +93 -0
- package/dist/cli/index.js +137 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +350 -527
- package/dist/index.js +144 -5
- package/dist/index.js.map +1 -1
- package/docs/BACKEND-VERIFICATION.md +277 -0
- package/docs/MOBILE-SETUP.md +92 -0
- package/package.json +26 -12
- package/scripts/doctor-mobile.sh +143 -0
- package/scripts/setup-mobile.sh +213 -0
- package/skills/qa-explore/phases/00-setup.md +8 -1
- package/skills/qa-explore/phases/01-auth.md +4 -0
- package/skills/qa-explore-mobile/SKILL.md +146 -0
- package/skills/qa-explore-mobile/phases/00-setup.md +203 -0
- package/skills/qa-explore-mobile/phases/01-auth.md +127 -0
- package/skills/qa-explore-mobile/phases/02-charter.md +70 -0
- package/skills/qa-explore-mobile/phases/03-discovery.md +79 -0
- package/skills/qa-explore-mobile/phases/04-journeys.md +49 -0
- package/skills/qa-explore-mobile/phases/05-features.md +72 -0
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +157 -0
- package/skills/qa-explore-mobile/phases/07-reporting.md +123 -0
- package/skills/qa-explore-mobile/references/mobile-edges.md +117 -0
- package/skills/qa-explore-quick/SKILL.md +6 -0
- package/skills/qa-verify-backend/SKILL.md +163 -0
- package/skills/qa-verify-backend/phases/00-setup.md +97 -0
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
- package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
- package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
- package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
- package/skills/qa-verify-backend/references/api-probes.md +141 -0
- package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
- package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
- package/skills/qa-verify-backend/references/payload-verification.md +112 -0
- package/skills/qa-verify-backend/references/release-readiness.md +97 -0
- package/skills/qa-verify-backend/references/safety-rules.md +90 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Example target — NATIVE-MODE mobile app on a simulator/emulator.
|
|
2
|
+
# Copy this file, rename it, and fill in the values for your app. This documents the
|
|
3
|
+
# fields /qa-explore-mobile reads to run in NATIVE mode (the app itself is the system
|
|
4
|
+
# under test, driven via Maestro + adb / xcrun simctl through bin/mobile-cli.mjs).
|
|
5
|
+
#
|
|
6
|
+
# For WEB mode (a real device browser pointed at a URL), see the committed examples
|
|
7
|
+
# _example-sim-ios-safari.yml / _example-sim-android-chrome.yml instead.
|
|
8
|
+
#
|
|
9
|
+
# MODE DETECTION (done in phases/00-setup.md):
|
|
10
|
+
# - WEB mode -> app is a browser bundle (com.apple.mobilesafari / com.android.chrome)
|
|
11
|
+
# AND a web.base_url is present.
|
|
12
|
+
# - NATIVE mode -> app is a non-browser bundle/package with installable artifact paths
|
|
13
|
+
# (app_paths / apk_paths), no web.base_url.
|
|
14
|
+
|
|
15
|
+
id: example-native-mobile
|
|
16
|
+
name: "Example Native App (simulator/emulator)"
|
|
17
|
+
domain: _default
|
|
18
|
+
platform: ios # ios | android (required)
|
|
19
|
+
|
|
20
|
+
device:
|
|
21
|
+
# iOS: a simulator name or UDID. Android: an AVD name and/or a booted serial.
|
|
22
|
+
name: "iPhone 17" # iOS sim label
|
|
23
|
+
udid: "REPLACE-WITH-SIM-UDID" # iOS sim UDID (from `xcrun simctl list devices`)
|
|
24
|
+
# avd: my_test_avd # Android AVD name (created via Android Studio / avdmanager)
|
|
25
|
+
# serial: emulator-5554 # Android serial once the AVD is booted
|
|
26
|
+
|
|
27
|
+
app:
|
|
28
|
+
# The app under test (NOT a browser). Use whichever id matches the platform.
|
|
29
|
+
bundle_id: com.example.myapp # iOS bundle id
|
|
30
|
+
# package: com.example.myapp # Android application id
|
|
31
|
+
#
|
|
32
|
+
# Installable artifact(s) used to verify/install the build on the device. If the app is
|
|
33
|
+
# already installed and you don't need to (re)install, these can be omitted.
|
|
34
|
+
app_paths: # iOS .app bundle(s) (simulator build)
|
|
35
|
+
- "/absolute/path/to/MyApp.app"
|
|
36
|
+
# apk_paths: # Android APK(s)
|
|
37
|
+
# - "/absolute/path/to/myapp-release.apk"
|
|
38
|
+
|
|
39
|
+
# OPTIONAL — only needed if you want /qa-explore-mobile to be able to rebuild from source.
|
|
40
|
+
# Skipped unless the user passes --rebuild or the session goal mentions "latest"/"fresh".
|
|
41
|
+
# build_commands are referenced generically by 00-setup.md — put the EXACT commands your
|
|
42
|
+
# project uses here (no framework is assumed by the skill).
|
|
43
|
+
# source_repo:
|
|
44
|
+
# path: "/absolute/path/to/app-source"
|
|
45
|
+
# build_commands:
|
|
46
|
+
# # Each command should build AND install onto the configured device.
|
|
47
|
+
# android_release: "..." # e.g. your project's release build+install command
|
|
48
|
+
# ios_debug: "..." # e.g. your project's sim build+install command
|
|
49
|
+
# refresh: "git fetch && git status -sb" # used to detect upstream drift before rebuild
|
|
50
|
+
|
|
51
|
+
auth:
|
|
52
|
+
# in-app -> account creation / login happens inside the app (see phases/01-auth.md).
|
|
53
|
+
# none -> no auth.
|
|
54
|
+
strategy: in-app
|
|
55
|
+
# For staging/test builds that accept a fixed one-time code, record it here so the
|
|
56
|
+
# auth phase can use it instead of a real OTP. Leave unset for real OTP delivery.
|
|
57
|
+
# static_otp: "000000"
|
|
58
|
+
# test_email_pattern: "qa+{unique}@example.com" # if the app needs a unique email per signup
|
|
59
|
+
|
|
60
|
+
scope:
|
|
61
|
+
start_screen: launch # the app's first screen after launch
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Template target — WEB-MODE mobile app in the REAL Android Emulator Chrome.
|
|
2
|
+
# Copy this file, rename it, and point web.base_url at your mobile web app.
|
|
3
|
+
# For NATIVE mode (an installed app under test), see _example-native-mobile.yml.
|
|
4
|
+
id: example-sim-android-chrome
|
|
5
|
+
name: "Example Mobile Web App — Android Emulator Chrome"
|
|
6
|
+
platform: android
|
|
7
|
+
domain: _default
|
|
8
|
+
# This target drives a WEB app through a real mobile browser on a real emulator.
|
|
9
|
+
# The "app" is just Chrome; the system under test is the web app served at web.base_url.
|
|
10
|
+
# mobile-cli treats Chrome like any other app: set-app -> launch -> open-url -> snapshot.
|
|
11
|
+
device:
|
|
12
|
+
avd: qa_pixel_api35 # standard AVD created by scripts/setup-mobile.sh (Google-Play API 35). Override to any existing AVD (run: emulator -list-avds).
|
|
13
|
+
serial: emulator-5554 # populated once the AVD is booted
|
|
14
|
+
app:
|
|
15
|
+
# The browser IS the app. No native build/install needed — Chrome is preinstalled on
|
|
16
|
+
# Google-Play emulator images, so there is no apk_paths / build step here.
|
|
17
|
+
# NOTE: stock AOSP / non-Play emulator images may NOT ship Chrome — they often have only
|
|
18
|
+
# a basic AOSP browser or none. If `adb shell pm list packages | grep chrome` shows no
|
|
19
|
+
# com.android.chrome, use a Google-APIs/Play image AVD rather than assuming it's present.
|
|
20
|
+
package: com.android.chrome
|
|
21
|
+
web:
|
|
22
|
+
base_url: "https://staging.m.example.com"
|
|
23
|
+
start_url: "https://staging.m.example.com/login"
|
|
24
|
+
# Many mobile hosts enforce a mobile User-Agent and redirect desktop UAs to the desktop
|
|
25
|
+
# host. A real emulator Chrome sends a genuine mobile UA, so this is a non-issue.
|
|
26
|
+
auth:
|
|
27
|
+
strategy: interactive-sso # interactive SSO + MFA — cannot be automated
|
|
28
|
+
identity_provider: "your IdP (e.g. Microsoft Entra, Okta, Auth0)"
|
|
29
|
+
login_url: "https://staging.m.example.com/login"
|
|
30
|
+
# One-time interactive login, then the browser profile persists the session.
|
|
31
|
+
# Do NOT attempt to script the MFA challenge. Use strategy: none for public apps.
|
|
32
|
+
scope:
|
|
33
|
+
start_screen: login
|
|
34
|
+
start_url: "https://staging.m.example.com/login"
|
|
35
|
+
notes: |
|
|
36
|
+
WHAT THIS IS
|
|
37
|
+
A browser-as-app target: Android Emulator Chrome pointed at a mobile web app. The
|
|
38
|
+
Maestro/adb route used by mobile-cli drives the real Android Chrome engine.
|
|
39
|
+
|
|
40
|
+
NO NATIVE BUILD NEEDED (but verify Chrome exists — see app.package note above)
|
|
41
|
+
Boot the AVD, then:
|
|
42
|
+
bin/wadb devices # boot `emulator -avd <your-avd> ...` first if empty
|
|
43
|
+
bin/wadb shell pm list packages | grep chrome # confirm com.android.chrome is present
|
|
44
|
+
export MOBILE_CLI_STATE=/tmp/<target>-state.json
|
|
45
|
+
bin/mcli set-device emulator-5554
|
|
46
|
+
bin/mcli set-app com.android.chrome
|
|
47
|
+
bin/mcli launch
|
|
48
|
+
bin/mcli open-url <web.base_url>
|
|
49
|
+
bin/mcli snapshot
|
|
50
|
+
Use a dedicated MOBILE_CLI_STATE so you don't clobber another target's default state.
|
|
51
|
+
On a fresh Chrome profile you may see a one-time first-run / "Open tabs" promo bubble —
|
|
52
|
+
mobile-cli filters the promo (com.android.chrome:id/message) out of snapshots; if a
|
|
53
|
+
full-screen first-run/sign-in-to-Chrome interstitial appears, dismiss it once by hand.
|
|
54
|
+
|
|
55
|
+
A11Y-vs-DOM LIMITATION (document, do not try to solve)
|
|
56
|
+
snapshot reads the accessibility tree, not the DOM. Web content IS exposed to a11y —
|
|
57
|
+
headings, links, buttons, inputs and visible text become tappable refs. But versus
|
|
58
|
+
playwright-cli you LOSE: CSS/DOM selectors, console & network introspection, route-based
|
|
59
|
+
mocking, and reliable exact-DOM assertions. You drive purely by visible/accessible labels
|
|
60
|
+
and tap coordinates.
|
|
61
|
+
mobile-cli auto-filters Chrome's own chrome (omnibox/URL bar, tab switcher, menu, toolbar,
|
|
62
|
+
infobars/promos) from snapshots when the app id is a known browser, so the ref tree is the
|
|
63
|
+
page, not the browser furniture. The generic android:id/content root and the system status
|
|
64
|
+
bar may still appear (no stable app-specific id to filter on).
|
|
65
|
+
|
|
66
|
+
AUTH REALITY — SSO + MFA (do NOT automate)
|
|
67
|
+
If the app logs in via an interactive IdP with MFA: unlike Playwright, there is NO
|
|
68
|
+
transferable storage_state you can inject into a real emulator browser. The practical
|
|
69
|
+
approach is a ONE-TIME INTERACTIVE LOGIN performed by a human in the emulator's Chrome:
|
|
70
|
+
open the start_url, complete the username/password + MFA prompt by hand once. The
|
|
71
|
+
emulator persists Chrome's profile (cookies/session) across app launches and across
|
|
72
|
+
mobile-cli `launch`/`open-url` calls (as long as the AVD is not wiped / `pm clear`ed), so
|
|
73
|
+
subsequent exploration reuses that session without re-authenticating until it expires.
|
|
74
|
+
Never script the MFA step. Avoid `relaunch-clean` on this target — `pm clear
|
|
75
|
+
com.android.chrome` wipes the logged-in profile and forces a fresh SSO+MFA login.
|
|
76
|
+
|
|
77
|
+
EMULATOR / EXPLORATION NOTES
|
|
78
|
+
Android Chrome DOES expose the live page to uiautomator/Maestro's hierarchy, so snapshot
|
|
79
|
+
refs map to real web elements. Hardware BACK works here (`press BACK`) unlike iOS. If the
|
|
80
|
+
emulator is slow/flaky to boot or Chrome is absent on the chosen image, document that and
|
|
81
|
+
fall back to the iOS Safari target rather than blocking — the two engines are independent.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Template target — WEB-MODE mobile app in the REAL iOS Simulator Safari.
|
|
2
|
+
# Copy this file, rename it, and point web.base_url at your mobile web app.
|
|
3
|
+
# For NATIVE mode (an installed app under test), see _example-native-mobile.yml.
|
|
4
|
+
id: example-sim-ios-safari
|
|
5
|
+
name: "Example Mobile Web App — iOS Simulator Safari"
|
|
6
|
+
platform: ios
|
|
7
|
+
domain: _default
|
|
8
|
+
# This target drives a WEB app through a real mobile browser on a real simulator.
|
|
9
|
+
# The "app" is just Safari; the system under test is the web app served at web.base_url.
|
|
10
|
+
# mobile-cli treats Safari like any other app: set-app -> launch -> open-url -> snapshot.
|
|
11
|
+
device:
|
|
12
|
+
# Resolve by NAME, not a hardcoded UDID (UDIDs are per-machine). scripts/setup-mobile.sh
|
|
13
|
+
# creates a standard sim named "qa-iphone"; on a machine that already has sims you can set
|
|
14
|
+
# this to any available iPhone (see: xcrun simctl list devices available). 00-setup boots
|
|
15
|
+
# the named device (creating it if missing) and resolves its UDID at runtime.
|
|
16
|
+
name: "qa-iphone"
|
|
17
|
+
# udid: optional — if set, takes precedence over name. Leave unset for portability.
|
|
18
|
+
app:
|
|
19
|
+
# The browser IS the app. No native build/install needed — Safari is preinstalled
|
|
20
|
+
# on every iOS simulator runtime, so there is no app_paths / build step here.
|
|
21
|
+
bundle_id: com.apple.mobilesafari
|
|
22
|
+
web:
|
|
23
|
+
base_url: "https://staging.m.example.com"
|
|
24
|
+
start_url: "https://staging.m.example.com/login"
|
|
25
|
+
# Many mobile hosts enforce a mobile User-Agent and redirect desktop UAs to the desktop
|
|
26
|
+
# host. A real simulator Safari already sends a genuine mobile UA, so this is a non-issue
|
|
27
|
+
# here (it only bites Playwright/desktop browsers).
|
|
28
|
+
auth:
|
|
29
|
+
strategy: interactive-sso # interactive SSO + MFA — cannot be automated
|
|
30
|
+
identity_provider: "your IdP (e.g. Microsoft Entra, Okta, Auth0)"
|
|
31
|
+
login_url: "https://staging.m.example.com/login"
|
|
32
|
+
# One-time interactive login, then the browser profile persists the session.
|
|
33
|
+
# Do NOT attempt to script the MFA challenge. Use strategy: none for public apps.
|
|
34
|
+
scope:
|
|
35
|
+
start_screen: login
|
|
36
|
+
start_url: "https://staging.m.example.com/login"
|
|
37
|
+
notes: |
|
|
38
|
+
WHAT THIS IS
|
|
39
|
+
A browser-as-app target: iOS Simulator Safari pointed at a mobile web app. The genuine
|
|
40
|
+
engine matters here — Playwright CANNOT drive iOS Simulator Safari, and Playwright's
|
|
41
|
+
"webkit" is desktop macOS WebKit, NOT the simulator's WebKit. The Maestro/simctl route
|
|
42
|
+
used by mobile-cli is the only way to exercise the real iOS engine.
|
|
43
|
+
|
|
44
|
+
NO NATIVE BUILD NEEDED
|
|
45
|
+
Safari ships with every simulator runtime — nothing to build or install. Just boot the
|
|
46
|
+
sim (`xcrun simctl boot <udid>; open -a Simulator`), then:
|
|
47
|
+
export MOBILE_CLI_STATE=/tmp/<target>-state.json
|
|
48
|
+
bin/mcli set-device <udid>
|
|
49
|
+
bin/mcli set-app com.apple.mobilesafari
|
|
50
|
+
bin/mcli launch
|
|
51
|
+
bin/mcli open-url <web.base_url>
|
|
52
|
+
bin/mcli snapshot
|
|
53
|
+
Use a dedicated MOBILE_CLI_STATE so you don't clobber another target's default state.
|
|
54
|
+
|
|
55
|
+
A11Y-vs-DOM LIMITATION (document, do not try to solve)
|
|
56
|
+
snapshot reads the accessibility tree, not the DOM. Web content IS exposed to a11y —
|
|
57
|
+
headings, links, buttons, inputs and visible text become tappable refs. But versus
|
|
58
|
+
playwright-cli you LOSE: CSS/DOM selectors, console & network introspection, route-based
|
|
59
|
+
mocking, and reliable exact-DOM assertions. You drive purely by visible/accessible labels
|
|
60
|
+
and tap coordinates.
|
|
61
|
+
mobile-cli auto-filters Safari's own chrome (URL/address bar, tabs, toolbar, on-screen
|
|
62
|
+
keyboard) from snapshots when the app id is a known browser, so the ref tree is the page,
|
|
63
|
+
not the browser furniture. Residual non-app noise that has no stable a11y id (the iOS
|
|
64
|
+
status bar clock/wifi/battery and transient "NN%" page-load progress) may still appear.
|
|
65
|
+
|
|
66
|
+
AUTH REALITY — SSO + MFA (do NOT automate)
|
|
67
|
+
If the app logs in via an interactive IdP with MFA: unlike Playwright, there is NO
|
|
68
|
+
transferable storage_state you can inject into a real simulator browser. The practical
|
|
69
|
+
approach is a ONE-TIME INTERACTIVE LOGIN performed by a human in the sim's Safari:
|
|
70
|
+
open the start_url, complete the username/password + MFA prompt by hand once. The
|
|
71
|
+
simulator persists Safari's profile (cookies/session) across app launches and across
|
|
72
|
+
mobile-cli `launch`/`open-url` calls, so subsequent exploration reuses that session
|
|
73
|
+
without re-authenticating (until the session expires or the sim is erased). Never script
|
|
74
|
+
the MFA step. If you land on the IdP domain mid-session, that just means the session
|
|
75
|
+
lapsed — hand back to a human for re-login.
|
|
76
|
+
|
|
77
|
+
iOS SIM CONSTRAINTS (inherited from the native iOS target)
|
|
78
|
+
No hardware BACK key (use an on-screen/web back control via snapshot+click), no biometric
|
|
79
|
+
flow, no easy CLI rotation. iOS a11y can also aggregate sibling nodes into one comma-joined
|
|
80
|
+
label on some screens — tap by coordinates from a `snapshot --full` when that happens.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# API Probe Matrix
|
|
2
|
+
|
|
3
|
+
> CONFIDENTIAL — internal QA artifact. Contains endpoint behaviour and environment
|
|
4
|
+
> configuration. Keep local. Redact response bodies before saving.
|
|
5
|
+
|
|
6
|
+
## Session Reference
|
|
7
|
+
|
|
8
|
+
| Property | Value |
|
|
9
|
+
|-----------------|-----------------------------------------|
|
|
10
|
+
| Session ID | [e.g., SESSION-2026-03-28-001] |
|
|
11
|
+
| Ticket | [ticket id, or "none"] |
|
|
12
|
+
| Endpoint | [`POST /api/v1/…` — note the verb] |
|
|
13
|
+
| Date | [YYYY-MM-DD] |
|
|
14
|
+
| Observation mode| [in-page fetch / out-of-browser request / unauthenticated] |
|
|
15
|
+
| Identity | [role or persona used — never a credential] |
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Environment Fingerprint
|
|
20
|
+
|
|
21
|
+
Fill this in **before** the matrix. Without it the results below are uninterpretable.
|
|
22
|
+
|
|
23
|
+
| Component | Environment A `[name]` | Environment B `[name]` |
|
|
24
|
+
|------------------|------------------------|------------------------|
|
|
25
|
+
| Frontend build | [build id / date] | [build id / date] |
|
|
26
|
+
| API build | [build id / commit] | [build id / commit] |
|
|
27
|
+
| Service under test | [build id / commit] | [build id / commit] |
|
|
28
|
+
| Changed path selected? | [yes / no — name the flag or config that decides it] | [yes / no] |
|
|
29
|
+
| Commit under test deployed? | [ancestor / not an ancestor] | [ancestor / not an ancestor] |
|
|
30
|
+
|
|
31
|
+
> **Same build, different behaviour ⇒ configuration, not deploy lag.**
|
|
32
|
+
> If the changed path is not selected in an environment, every AC that depends on it is
|
|
33
|
+
> `NOT-REACHABLE` there — not `PASS`, not `FAIL`.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Cases
|
|
38
|
+
|
|
39
|
+
One row per case. The **label** is the stable identifier the report, the bug and the
|
|
40
|
+
re-run all refer to. Record passes as well as failures — a matrix of only failures
|
|
41
|
+
cannot show a regression later.
|
|
42
|
+
|
|
43
|
+
| # | Label | Request | Expected | Env A: status / count / rows | Env B: status / count / rows | Verdict |
|
|
44
|
+
|---|-------|---------|----------|------------------------------|------------------------------|---------|
|
|
45
|
+
| 1 | identity | `{"query":"<known value>"}` | 200, stable non-zero | 200 / 77 / 20 | 200 / 91 / 20 | ✅ |
|
|
46
|
+
| 2 | empty string | `{"query":""}` | 400 + error body | | | |
|
|
47
|
+
| 3 | body absent | `{}` | 400 + error body | | | |
|
|
48
|
+
| 4 | metachar — lone `(` | `{"query":"("}` | 400 or escaped 200 | | | |
|
|
49
|
+
| 5 | limit above cap | `?limit=<cap+1>` | 400, or capped | | | |
|
|
50
|
+
| 6 | invalid enum | `{"filter":["<nonsense>"]}` | 400 + error body | | | |
|
|
51
|
+
| 7 | second identity | same request, other user | scoped result | | | |
|
|
52
|
+
|
|
53
|
+
> **Compare status codes and shapes. Never compare absolute counts** — different
|
|
54
|
+
> environments hold different data and it drifts between runs.
|
|
55
|
+
|
|
56
|
+
### Case families to cover
|
|
57
|
+
|
|
58
|
+
Boundary lengths · tokenisation and whitespace · metacharacters of the underlying query
|
|
59
|
+
language · empty / null / absent / whitespace-only · enum values valid, invalid, empty,
|
|
60
|
+
wildcard, injected · pagination bounds and types · type confusion · casing and
|
|
61
|
+
normalisation · second identity and second scope.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Findings
|
|
66
|
+
|
|
67
|
+
One row per finding, not one per failing case — a family of inputs failing on the same
|
|
68
|
+
character is **one** bug with several examples.
|
|
69
|
+
|
|
70
|
+
| ID | Cases | Finding | Surface | Severity | Bug report |
|
|
71
|
+
|----|-------|---------|---------|----------|------------|
|
|
72
|
+
| F1 | 4, 9–14 | [what is wrong, in one line] | API / UI / both | High | `bugs/BUG-0xx-….md` |
|
|
73
|
+
|
|
74
|
+
**Surface** classifies where the defect lives, from the differential pass:
|
|
75
|
+
|
|
76
|
+
| Surface | Meaning | Owner |
|
|
77
|
+
|---------|---------|-------|
|
|
78
|
+
| **Both** | The UI faithfully reflects the endpoint | Service |
|
|
79
|
+
| **API only** | Real defect the client's guard is hiding — reachable by every other client | Service |
|
|
80
|
+
| **UI only** | The client invents or masks behaviour the service does not have | Client |
|
|
81
|
+
| **Neither** | Environment or access noise, not a product defect | — |
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Evidence
|
|
86
|
+
|
|
87
|
+
| File | Contents |
|
|
88
|
+
|------|----------|
|
|
89
|
+
| `evidence/fingerprint.md` | Build ids, flag values, commit ancestry |
|
|
90
|
+
| `evidence/<case-set>-raw.json` | Raw per-case output, redacted |
|
|
91
|
+
| `probes/<probe>.js` | The probe script, so this is repeatable |
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Notes and Open Questions
|
|
96
|
+
|
|
97
|
+
- [What could not be checked, and exactly what access would unblock it]
|
|
98
|
+
- [Which environment would answer a `NOT-REACHABLE` verdict]
|
|
99
|
+
- [Counts recorded here are valid for the date above only]
|
|
@@ -23,6 +23,20 @@
|
|
|
23
23
|
- **tested** -- Area was fully explored with the planned heuristics.
|
|
24
24
|
- **partial** -- Area was visited but not all heuristics or paths were exercised.
|
|
25
25
|
- **not-tested** -- Area was in scope but was not reached during the session.
|
|
26
|
+
- **code-verified-only** -- Believed correct from reading the source, never observed
|
|
27
|
+
running. This is `UNVERIFIABLE`, **not** a pass -- keep it visually distinct so it
|
|
28
|
+
cannot be skimmed as green.
|
|
29
|
+
|
|
30
|
+
Two distinctions that routinely produce a false **tested**:
|
|
31
|
+
|
|
32
|
+
- **Consistent is not causal.** A setting whose current value happens to match the output
|
|
33
|
+
proves nothing until the setting is changed and the output is watched to follow.
|
|
34
|
+
- **The data has to reach the case.** An area whose behaviour depends on values no
|
|
35
|
+
reachable record has -- a negative, a zero, an empty collection -- is **not-tested** for
|
|
36
|
+
that behaviour. Find such a record, create one, or say plainly what data would settle it.
|
|
37
|
+
|
|
38
|
+
Name the single **not-tested** item that carries the most risk. In a map full of green rows
|
|
39
|
+
that is the line the reader acts on.
|
|
26
40
|
|
|
27
41
|
---
|
|
28
42
|
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Expected Behaviour Specification
|
|
2
|
+
|
|
3
|
+
> CONFIDENTIAL — internal QA artifact.
|
|
4
|
+
|
|
5
|
+
Use this when a verification session finds behaviour nobody wrote an acceptance criterion
|
|
6
|
+
for — which is most of what an API probe turns up. A bug report says *this is wrong*. This
|
|
7
|
+
says *here is what right looks like*, in enough detail that someone can implement it and
|
|
8
|
+
someone else can verify it. It is frequently the most valuable artifact a session produces,
|
|
9
|
+
because it converts a pile of findings into one reviewable decision.
|
|
10
|
+
|
|
11
|
+
Write it **per area**, not per case. Ten failing inputs that share a cause are one section.
|
|
12
|
+
|
|
13
|
+
## Header
|
|
14
|
+
|
|
15
|
+
| Property | Value |
|
|
16
|
+
|----------|-------|
|
|
17
|
+
| Date | [YYYY-MM-DD] |
|
|
18
|
+
| Endpoint / surface | [`POST /api/v1/…` — note the verb] |
|
|
19
|
+
| Environment(s) observed | [name, and the build each one was running] |
|
|
20
|
+
| Source findings | [link to the session that produced the observations] |
|
|
21
|
+
| Status | [proposed / agreed / implemented] |
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## [N]. [Area — e.g. Minimum input length]
|
|
26
|
+
|
|
27
|
+
**Observed**
|
|
28
|
+
|
|
29
|
+
> What the system does today, as evidence. Exact request, exact status, exact response.
|
|
30
|
+
> Note which environments it holds in — behaviour frequently differs, and that difference
|
|
31
|
+
> is part of the finding.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
{"query":"L"} → 200 / <the entire collection> [alpha]
|
|
35
|
+
{"query":"L"} → 200 / 0 [beta]
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
**Expected**
|
|
39
|
+
|
|
40
|
+
> What it should do. Numbered, falsifiable statements — each one a testable assertion, not
|
|
41
|
+
> a sentiment. Include the boundary values themselves.
|
|
42
|
+
|
|
43
|
+
- An input whose tokens are all below the minimum → **`400`**, with the minimum named in
|
|
44
|
+
the message.
|
|
45
|
+
- "No usable tokens after filtering" and "no query supplied" are **different branches**.
|
|
46
|
+
Neither falls through to a match-all.
|
|
47
|
+
- The match-all path is not reachable from this parameter at any input. If "browse
|
|
48
|
+
everything" is wanted, it is an explicit endpoint with a hard server-side cap.
|
|
49
|
+
- The minimum is **N**, matching the existing rule, so that [known-good short inputs] keep
|
|
50
|
+
working.
|
|
51
|
+
- Client: no request fires below N; the user sees a hint rather than an empty result.
|
|
52
|
+
|
|
53
|
+
### Decisions this section forces
|
|
54
|
+
|
|
55
|
+
State them explicitly — leaving them implicit is how a spec gets implemented three
|
|
56
|
+
different ways:
|
|
57
|
+
|
|
58
|
+
- **Reject, or clamp?** *Reject.* Silently serving a different page size than was asked for
|
|
59
|
+
breaks every caller that trusts the value back.
|
|
60
|
+
- **`400`, or a silent empty result?** *`400`.* A silent zero is indistinguishable from a
|
|
61
|
+
genuine no-match, and it will be read as data.
|
|
62
|
+
- **Where does validation live?** At the layer that covers **every** implementation of the
|
|
63
|
+
feature — not in one of two code paths, and never only in the client.
|
|
64
|
+
- **What must not change?** List the valid inputs and their current answers, so the fix is
|
|
65
|
+
verifiable as non-breaking.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Priority
|
|
70
|
+
|
|
71
|
+
Rank the areas by what is reachable by real users **today**, not by how bad each one reads.
|
|
72
|
+
Name explicitly which finding is live in production right now, and which only appears in an
|
|
73
|
+
environment nobody ships from — the two need very different urgency, and the difference is
|
|
74
|
+
invisible unless someone says it.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Reply — plain English
|
|
79
|
+
|
|
80
|
+
> Ready to paste into a ticket comment, a pull request or a chat reply. Same content, no
|
|
81
|
+
> jargon, no status codes unless they carry meaning to the reader. Written for the person
|
|
82
|
+
> who has to decide whether to fund the fix.
|
|
83
|
+
|
|
84
|
+
For each area, in a short paragraph:
|
|
85
|
+
|
|
86
|
+
- **What happens**, in a sentence a non-engineer can picture.
|
|
87
|
+
- **Whether it is live for users**, and where.
|
|
88
|
+
- **What the fix is**, in outline — enough to size it, not enough to design it.
|
|
89
|
+
- **Where you agree with their reading**, if they proposed one, and precisely where you do
|
|
90
|
+
not, and why.
|
|
91
|
+
|
|
92
|
+
Close with the one thing you would put ahead of everything else in the list, and the reason
|
|
93
|
+
— usually that it is the only one users are hitting today.
|
package/dist/cli/index.js
CHANGED
|
@@ -228,21 +228,41 @@ import { ZodError } from "zod";
|
|
|
228
228
|
// src/schemas/target.schema.ts
|
|
229
229
|
import { z } from "zod";
|
|
230
230
|
var AuthConfigSchema = z.object({
|
|
231
|
-
|
|
231
|
+
// 'in-app' (native mobile: login/signup inside the app) and 'interactive-sso'
|
|
232
|
+
// (mobile web: one-time human SSO/MFA login the browser profile persists) are
|
|
233
|
+
// mobile-only strategies used by /qa-explore-mobile.
|
|
234
|
+
strategy: z.enum([
|
|
235
|
+
"none",
|
|
236
|
+
"storage_state",
|
|
237
|
+
"credentials",
|
|
238
|
+
"token",
|
|
239
|
+
"in-app",
|
|
240
|
+
"interactive-sso"
|
|
241
|
+
]),
|
|
232
242
|
state_file: z.string().optional(),
|
|
233
243
|
login_url: z.string().url().optional(),
|
|
234
244
|
credentials: z.object({
|
|
235
245
|
username: z.string(),
|
|
236
246
|
password: z.string()
|
|
237
247
|
}).optional(),
|
|
238
|
-
token: z.string().optional()
|
|
248
|
+
token: z.string().optional(),
|
|
249
|
+
identity_provider: z.string().optional(),
|
|
250
|
+
// Staging builds sometimes accept a fixed OTP — recorded so the auth phase can
|
|
251
|
+
// use it instead of waiting for a real code. Never a real credential.
|
|
252
|
+
static_otp: z.string().optional(),
|
|
253
|
+
test_email_pattern: z.string().optional()
|
|
239
254
|
});
|
|
240
255
|
var BrowserConfigSchema = z.object({
|
|
241
256
|
headless: z.boolean(),
|
|
242
257
|
viewport: z.object({
|
|
243
258
|
width: z.number().int().positive(),
|
|
244
259
|
height: z.number().int().positive()
|
|
245
|
-
})
|
|
260
|
+
}),
|
|
261
|
+
// Optional engine selection + Playwright device descriptor for mobile-web
|
|
262
|
+
// emulation targets (e.g. engine: webkit, device: "iPhone 15").
|
|
263
|
+
engine: z.enum(["chromium", "webkit", "firefox"]).optional(),
|
|
264
|
+
channel: z.string().optional(),
|
|
265
|
+
device: z.string().optional()
|
|
246
266
|
});
|
|
247
267
|
var ScopeConfigSchema = z.object({
|
|
248
268
|
start_pages: z.array(z.string()).optional(),
|
|
@@ -256,17 +276,129 @@ var SafetyConfigSchema = z.object({
|
|
|
256
276
|
no_file_upload: z.boolean().optional(),
|
|
257
277
|
no_delete_actions: z.boolean().optional()
|
|
258
278
|
});
|
|
259
|
-
var
|
|
279
|
+
var EnvironmentConfigSchema = z.object({
|
|
280
|
+
kind: z.enum(["dev", "ephemeral", "staging", "production"]),
|
|
281
|
+
destroyed_automatically: z.boolean().optional()
|
|
282
|
+
});
|
|
283
|
+
var BackendConfigSchema = z.object({
|
|
284
|
+
provider: z.enum(["aws"]).optional(),
|
|
285
|
+
aws_profile_env: z.string().optional(),
|
|
286
|
+
region_env: z.string().optional(),
|
|
287
|
+
region: z.string().optional(),
|
|
288
|
+
// Expected account. The skill compares it to `sts get-caller-identity` and
|
|
289
|
+
// stops the live lane on a mismatch rather than probing someone else's account.
|
|
290
|
+
account_id: z.string().optional(),
|
|
291
|
+
env_suffix: z.string().optional(),
|
|
292
|
+
// Free-form logical name -> deployed resource name (config_table, dlq, ...).
|
|
293
|
+
resources: z.record(z.string(), z.string()).optional(),
|
|
294
|
+
notes: z.string().optional()
|
|
295
|
+
});
|
|
296
|
+
var ApiSurfaceConfigSchema = z.object({
|
|
297
|
+
// Defaults to the target's base_url when omitted.
|
|
298
|
+
base_url: z.string().optional(),
|
|
299
|
+
auth: z.enum(["session-cookie", "bearer-env", "none"]).optional(),
|
|
300
|
+
// NAME of the env var holding a bearer token, for `auth: bearer-env`.
|
|
301
|
+
token_env: z.string().optional(),
|
|
302
|
+
// Persistent browser profile directory holding the authenticated session,
|
|
303
|
+
// e.g. `.auth/my-service-dev-profile`. Gitignored.
|
|
304
|
+
browser_profile: z.string().optional(),
|
|
305
|
+
// Endpoint returning the deployed build, used to fingerprint the environment.
|
|
306
|
+
version_endpoint: z.string().optional(),
|
|
307
|
+
// Logical name -> "VERB /path", e.g. search: "POST /api/v1/search".
|
|
308
|
+
endpoints: z.record(z.string(), z.string()).optional(),
|
|
309
|
+
// The ONLY endpoints the read-only API lane may call without asking.
|
|
310
|
+
probe_allowlist: z.array(z.string()).optional(),
|
|
311
|
+
// Other target ids to run the same matrix against, for parity comparison.
|
|
312
|
+
parity_targets: z.array(z.string()).optional(),
|
|
313
|
+
// Flag name -> where its deployed value is declared for this environment.
|
|
314
|
+
// A flag can select between two implementations inside one identical build.
|
|
315
|
+
feature_flags: z.record(z.string(), z.string()).optional(),
|
|
316
|
+
notes: z.string().optional()
|
|
317
|
+
});
|
|
318
|
+
var SourceBranchConfigSchema = z.object({
|
|
319
|
+
repo_path: z.string(),
|
|
320
|
+
branch: z.string(),
|
|
321
|
+
base_branch: z.string().optional(),
|
|
322
|
+
components: z.record(z.string(), z.string()).optional()
|
|
323
|
+
});
|
|
324
|
+
var WebTargetConfigSchema = z.object({
|
|
260
325
|
id: z.string().min(1),
|
|
261
326
|
name: z.string().min(1),
|
|
262
|
-
|
|
327
|
+
// '' is allowed for the ad-hoc template (_default.yml) where the URL is
|
|
328
|
+
// provided at session start rather than in the config.
|
|
329
|
+
base_url: z.union([z.literal(""), z.string().url()]),
|
|
263
330
|
domain: z.string().min(1),
|
|
264
331
|
auth: AuthConfigSchema,
|
|
265
332
|
browser: BrowserConfigSchema,
|
|
266
333
|
scope: ScopeConfigSchema,
|
|
267
334
|
safety: SafetyConfigSchema.optional(),
|
|
335
|
+
// Optional blocks consumed by /qa-verify-backend. A target without them is
|
|
336
|
+
// still a valid /qa-explore target.
|
|
337
|
+
environment: EnvironmentConfigSchema.optional(),
|
|
338
|
+
backend: BackendConfigSchema.optional(),
|
|
339
|
+
api: ApiSurfaceConfigSchema.optional(),
|
|
340
|
+
source: SourceBranchConfigSchema.optional(),
|
|
341
|
+
notes: z.string().optional()
|
|
342
|
+
});
|
|
343
|
+
var MobileDeviceConfigSchema = z.object({
|
|
344
|
+
name: z.string().optional(),
|
|
345
|
+
// iOS simulator name (portable across machines)
|
|
346
|
+
udid: z.string().optional(),
|
|
347
|
+
// iOS simulator UDID (takes precedence over name)
|
|
348
|
+
avd: z.string().optional(),
|
|
349
|
+
// Android AVD name
|
|
350
|
+
serial: z.string().optional()
|
|
351
|
+
// Android serial once booted (e.g. emulator-5554)
|
|
352
|
+
});
|
|
353
|
+
var MobileAppConfigSchema = z.object({
|
|
354
|
+
bundle_id: z.string().optional(),
|
|
355
|
+
// iOS bundle id (or browser bundle in web mode)
|
|
356
|
+
package: z.string().optional(),
|
|
357
|
+
// Android application id (or browser package)
|
|
358
|
+
app_paths: z.array(z.string()).optional(),
|
|
359
|
+
// iOS .app artifact(s) — native mode
|
|
360
|
+
apk_paths: z.array(z.string()).optional()
|
|
361
|
+
// Android APK(s) — native mode
|
|
362
|
+
});
|
|
363
|
+
var MobileWebConfigSchema = z.object({
|
|
364
|
+
base_url: z.string().url(),
|
|
365
|
+
start_url: z.string().url().optional()
|
|
366
|
+
});
|
|
367
|
+
var SourceRepoConfigSchema = z.object({
|
|
368
|
+
path: z.string(),
|
|
369
|
+
build_commands: z.record(z.string(), z.string()).optional()
|
|
370
|
+
});
|
|
371
|
+
var MobileScopeConfigSchema = z.object({
|
|
372
|
+
start_screen: z.string().optional(),
|
|
373
|
+
start_url: z.string().optional(),
|
|
374
|
+
include_patterns: z.array(z.string()).optional(),
|
|
375
|
+
exclude_patterns: z.array(z.string()).optional()
|
|
376
|
+
});
|
|
377
|
+
var MobileTargetConfigSchema = z.object({
|
|
378
|
+
id: z.string().min(1),
|
|
379
|
+
name: z.string().min(1),
|
|
380
|
+
platform: z.enum(["ios", "android"]),
|
|
381
|
+
// discriminates mobile from web targets
|
|
382
|
+
domain: z.string().min(1),
|
|
383
|
+
device: MobileDeviceConfigSchema,
|
|
384
|
+
app: MobileAppConfigSchema,
|
|
385
|
+
web: MobileWebConfigSchema.optional(),
|
|
386
|
+
// present → WEB mode; absent → NATIVE mode
|
|
387
|
+
auth: AuthConfigSchema,
|
|
388
|
+
scope: MobileScopeConfigSchema.optional(),
|
|
389
|
+
safety: SafetyConfigSchema.optional(),
|
|
390
|
+
source_repo: SourceRepoConfigSchema.optional(),
|
|
268
391
|
notes: z.string().optional()
|
|
269
392
|
});
|
|
393
|
+
var TargetConfigSchema = z.custom().superRefine((data, ctx) => {
|
|
394
|
+
const isMobile = typeof data === "object" && data !== null && "platform" in data;
|
|
395
|
+
const branch = isMobile ? MobileTargetConfigSchema : WebTargetConfigSchema;
|
|
396
|
+
const result = branch.safeParse(data);
|
|
397
|
+
if (!result.success) {
|
|
398
|
+
for (const issue of result.error.issues)
|
|
399
|
+
ctx.addIssue(issue);
|
|
400
|
+
}
|
|
401
|
+
});
|
|
270
402
|
|
|
271
403
|
// src/schemas/knowledge-entry.schema.ts
|
|
272
404
|
import { z as z2 } from "zod";
|