qualiow-exploratory-testing 1.3.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +26 -0
- package/.claude-plugin/plugin.json +22 -0
- package/.env.example +27 -0
- package/CHANGELOG.md +259 -0
- package/README.md +472 -204
- package/agents/qa-diff-indexer-agent.md +81 -0
- package/agents/qa-gather-agent.md +41 -0
- package/agents/qa-page-mapper-agent.md +74 -0
- package/agents/qa-reporting-agent.md +103 -0
- package/{scripts → bin}/doctor-mobile.sh +57 -18
- package/bin/mcli +15 -9
- package/bin/mobile-cli.mjs +574 -286
- package/bin/qualiow +30 -0
- package/{scripts → bin}/setup-mobile.sh +56 -19
- package/bin/wadb +10 -4
- package/bin/wk-ios +29 -17
- package/bin/wkeval.mjs +7 -1
- package/data/domains/_default.yml +1 -1
- package/data/knowledge/changelog.yml +30 -12
- package/data/knowledge/manifest.yml +355 -127
- package/data/knowledge/releases/v0.1.0/entries/technique-business-logic-race-conditions.yml +1 -1
- package/data/knowledge/releases/v0.1.0/release.yml +8 -1
- package/data/knowledge/releases/v0.6.0/entries/technique-async-callback-contracts.yml +98 -0
- package/data/knowledge/releases/v0.6.0/entries/technique-exactly-once-verification.yml +116 -0
- package/data/knowledge/releases/v0.6.0/release.yml +30 -0
- package/data/security/SECURITY-POLICY.md +273 -0
- package/data/targets/_default.yml +1 -1
- package/data/targets/_example-api-only.yml +80 -0
- package/data/targets/_example-mobile-emulation.yml +2 -2
- package/data/targets/_example-native-mobile.yml +2 -2
- package/data/targets/_example-sim-android-chrome.yml +1 -1
- package/data/targets/_example-sim-ios-safari.yml +1 -1
- package/data/templates/api-probe-matrix.md +3 -3
- package/data/templates/bug-report.md +29 -151
- package/data/templates/coverage-map.md +9 -6
- package/data/templates/expected-behaviour.md +3 -2
- package/data/templates/session-charter.md +3 -0
- package/data/templates/session-report.md +57 -80
- package/dist/cli/index.js +2286 -701
- package/dist/index.d.ts +637 -81
- package/dist/index.js +2058 -228
- package/docs/BACKEND-VERIFICATION.md +39 -9
- package/docs/GETTING-STARTED.md +300 -121
- package/docs/MOBILE-SETUP.md +149 -18
- package/hooks/hooks.json +27 -0
- package/hooks/scripts/read-guard.mjs +95 -0
- package/hooks/scripts/secret-patterns.d.mts +8 -0
- package/hooks/scripts/secret-patterns.mjs +136 -0
- package/hooks/scripts/write-guard.mjs +87 -0
- package/package.json +33 -18
- package/skills/qa-explore/SKILL.md +32 -25
- package/skills/qa-explore/phases/00-setup.md +76 -42
- package/skills/qa-explore/phases/01-auth.md +32 -15
- package/skills/qa-explore/phases/02-charter.md +16 -5
- package/skills/qa-explore/phases/03-discovery.md +93 -27
- package/skills/qa-explore/phases/04-journeys.md +2 -2
- package/skills/qa-explore/phases/05-features.md +39 -24
- package/skills/qa-explore/phases/06-edge-cases.md +24 -13
- package/skills/qa-explore/phases/07-reporting.md +102 -142
- package/skills/qa-explore/references/context-integration.md +1 -1
- package/skills/qa-explore/references/delegation-rules.md +82 -0
- package/skills/qa-explore/references/output-contract.md +192 -0
- package/skills/qa-explore/references/paths.md +108 -0
- package/skills/qa-explore/references/playwright-agents-integration.md +61 -84
- package/skills/qa-explore/references/security-rules.md +70 -25
- package/skills/qa-explore/references/session-rules.md +2 -1
- package/skills/qa-explore/references/severity-guide.md +31 -4
- package/skills/qa-explore-cleanup/SKILL.md +51 -15
- package/skills/qa-explore-feedback/SKILL.md +23 -5
- package/skills/qa-explore-mobile/SKILL.md +86 -64
- package/skills/qa-explore-mobile/phases/00-setup.md +71 -62
- package/skills/qa-explore-mobile/phases/01-auth.md +28 -31
- package/skills/qa-explore-mobile/phases/02-charter.md +8 -4
- package/skills/qa-explore-mobile/phases/03-discovery.md +17 -11
- package/skills/qa-explore-mobile/phases/04-journeys.md +6 -6
- package/skills/qa-explore-mobile/phases/05-features.md +9 -9
- package/skills/qa-explore-mobile/phases/06-edge-cases.md +35 -40
- package/skills/qa-explore-mobile/phases/07-reporting.md +108 -60
- package/skills/qa-explore-mobile/references/mobile-edges.md +26 -26
- package/skills/qa-explore-quick/SKILL.md +71 -26
- package/skills/qa-explore-report/SKILL.md +65 -35
- package/skills/qa-gather/SKILL.md +44 -8
- package/skills/qa-knowledge-add/SKILL.md +25 -9
- package/skills/qa-knowledge-list/SKILL.md +37 -40
- package/skills/qa-target-setup/SKILL.md +63 -41
- package/skills/qa-verify-backend/SKILL.md +31 -19
- package/skills/qa-verify-backend/phases/00-setup.md +57 -12
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +2 -2
- package/skills/qa-verify-backend/phases/02-static-review.md +11 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +5 -3
- package/skills/qa-verify-backend/phases/03b-api-verification.md +24 -9
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +18 -2
- package/skills/qa-verify-backend/phases/05-reporting.md +72 -12
- package/skills/qa-verify-backend/references/api-probes.md +28 -1
- package/skills/qa-verify-backend/references/payload-verification.md +1 -1
- package/skills/qa-verify-backend/references/release-readiness.md +1 -1
- package/skills/qa-verify-backend/references/safety-rules.md +13 -8
- package/data/domains/_default.md +0 -181
- package/data/domains/ecommerce.md +0 -192
- package/data/domains/fintech.md +0 -200
- package/data/domains/identity.md +0 -88
- package/data/domains/marketing.md +0 -194
- package/data/domains/saas.md +0 -197
- package/dist/cli/index.d.ts +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/index.js.map +0 -1
package/bin/qualiow
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# qualiow — launcher shim so `${CLAUDE_PLUGIN_ROOT}/bin/qualiow` (or a checkout's
|
|
3
|
+
# `bin/qualiow`) works even when this repo was installed as a Claude Code plugin
|
|
4
|
+
# straight from git — a marketplace/plugin install has no `dist/` (tsup output)
|
|
5
|
+
# and no `node_modules` (tsup does not bundle deps), so the real CLI entry point
|
|
6
|
+
# `dist/cli/index.js` cannot run in place. When a built local copy exists, run it
|
|
7
|
+
# directly; otherwise fall through to the published npm package via `npx`, pinned
|
|
8
|
+
# to this checkout's own `.claude-plugin/plugin.json` version so the shim and the
|
|
9
|
+
# skills it ships alongside stay in lockstep. Release order matters for that pin:
|
|
10
|
+
# `npm publish` must land before this version becomes the current git tag, or the
|
|
11
|
+
# pinned `npx` fetch 404s — see docs/ARCHITECTURE-DECISIONS.md ADR-012.
|
|
12
|
+
set -euo pipefail
|
|
13
|
+
|
|
14
|
+
SELF="${BASH_SOURCE[0]}"
|
|
15
|
+
while [ -L "$SELF" ]; do
|
|
16
|
+
LINK="$(readlink "$SELF")"
|
|
17
|
+
case "$LINK" in /*) SELF="$LINK" ;; *) SELF="$(dirname "$SELF")/$LINK" ;; esac
|
|
18
|
+
done
|
|
19
|
+
ROOT="$(cd "$(dirname "$SELF")/.." && pwd)"
|
|
20
|
+
|
|
21
|
+
if [ -f "$ROOT/dist/cli/index.js" ] && [ -d "$ROOT/node_modules" ]; then
|
|
22
|
+
exec node "$ROOT/dist/cli/index.js" "$@"
|
|
23
|
+
fi
|
|
24
|
+
|
|
25
|
+
export CLAUDE_PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$ROOT}"
|
|
26
|
+
|
|
27
|
+
VER="$(node -p "try{require('$ROOT/.claude-plugin/plugin.json').version}catch{''}" 2>/dev/null || true)"
|
|
28
|
+
[ -z "$VER" ] && VER="latest"
|
|
29
|
+
|
|
30
|
+
exec npx -y -p "qualiow-exploratory-testing@$VER" qualiow "$@"
|
|
@@ -5,33 +5,41 @@
|
|
|
5
5
|
# devices the target configs expect. Re-runnable: every step checks before acting.
|
|
6
6
|
#
|
|
7
7
|
# What it can do automatically:
|
|
8
|
-
# - Homebrew (offers to install if missing), Node, JDK 17, Maestro
|
|
8
|
+
# - Homebrew (offers to install if missing), Node >= 22.4, JDK 17, Maestro (pinned)
|
|
9
9
|
# - Android: cmdline-tools, platform-tools, emulator, a Google-Play system image,
|
|
10
10
|
# license acceptance, and a standard AVD ($ANDROID_AVD_NAME, default qa_pixel_api35)
|
|
11
11
|
# - iOS: a standard simulator device ($IOS_DEVICE_NAME, default qa-iphone) from an
|
|
12
|
-
# installed iOS runtime
|
|
12
|
+
# installed iOS runtime, plus ios-webkit-debug-proxy for the bin/wk-ios DOM bridge
|
|
13
13
|
#
|
|
14
14
|
# What stays MANUAL (printed as guidance, never silently skipped):
|
|
15
15
|
# - Xcode itself (multi-GB App Store install) + `xcodebuild -license accept`
|
|
16
16
|
# - iOS runtime download (Xcode ▸ Settings ▸ Components) if none is installed
|
|
17
17
|
#
|
|
18
18
|
# Usage:
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
19
|
+
# bin/setup-mobile.sh # both platforms
|
|
20
|
+
# bin/setup-mobile.sh --android # Android only
|
|
21
|
+
# bin/setup-mobile.sh --ios # iOS only
|
|
22
|
+
# bin/setup-mobile.sh --yes # non-interactive (assume yes to prompts)
|
|
23
|
+
# bin/setup-mobile.sh --yes --allow-curl-bash # also let `curl | bash` installers run unattended
|
|
24
|
+
#
|
|
25
|
+
# Env: MAESTRO_VERSION (default 2.10.0) pins the Maestro release the official installer
|
|
26
|
+
# fetches; ANDROID_API (35), ANDROID_AVD_NAME, IOS_DEVICE_NAME, ANDROID_HOME as above.
|
|
27
|
+
#
|
|
28
|
+
# `curl | bash` installers (Homebrew, Maestro) always ask first; under --yes they are
|
|
29
|
+
# SKIPPED with the exact command printed unless --allow-curl-bash is also given.
|
|
23
30
|
#
|
|
24
31
|
# After it finishes it runs doctor-mobile.sh to confirm the result.
|
|
25
32
|
|
|
26
33
|
set -uo pipefail
|
|
27
34
|
|
|
28
|
-
PLATFORM="both"; ASSUME_YES=0
|
|
35
|
+
PLATFORM="both"; ASSUME_YES=0; ALLOW_CURL_BASH=0
|
|
29
36
|
for arg in "$@"; do
|
|
30
37
|
case "$arg" in
|
|
31
38
|
--android) PLATFORM="android" ;;
|
|
32
39
|
--ios) PLATFORM="ios" ;;
|
|
33
40
|
--yes|-y) ASSUME_YES=1 ;;
|
|
34
|
-
-
|
|
41
|
+
--allow-curl-bash) ALLOW_CURL_BASH=1 ;;
|
|
42
|
+
-h|--help) sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
35
43
|
*) echo "unknown arg: $arg" >&2; exit 2 ;;
|
|
36
44
|
esac
|
|
37
45
|
done
|
|
@@ -42,6 +50,8 @@ ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
|
|
|
42
50
|
ANDROID_AVD_NAME="${ANDROID_AVD_NAME:-qa_pixel_api35}"
|
|
43
51
|
ANDROID_API="${ANDROID_API:-35}"
|
|
44
52
|
IOS_DEVICE_NAME="${IOS_DEVICE_NAME:-qa-iphone}"
|
|
53
|
+
MAESTRO_VERSION="${MAESTRO_VERSION:-2.10.0}"
|
|
54
|
+
NODE_MIN_MAJOR=22; NODE_MIN_MINOR=4
|
|
45
55
|
if [ "$ARCH" = "arm64" ]; then ABI="arm64-v8a"; else ABI="x86_64"; fi
|
|
46
56
|
SYS_IMAGE="system-images;android-${ANDROID_API};google_apis_playstore;${ABI}"
|
|
47
57
|
|
|
@@ -54,11 +64,28 @@ confirm() {
|
|
|
54
64
|
printf ' %s [y/N] ' "$1"; read -r ans </dev/tty 2>/dev/null || ans=n
|
|
55
65
|
case "$ans" in y|Y|yes) return 0;; *) return 1;; esac
|
|
56
66
|
}
|
|
67
|
+
# A `curl | bash` installer runs only after an interactive yes, or under --yes when
|
|
68
|
+
# --allow-curl-bash was given too. Otherwise the exact command is printed as a manual step.
|
|
69
|
+
confirm_curl_bash() {
|
|
70
|
+
if [ "$ASSUME_YES" = 1 ]; then
|
|
71
|
+
[ "$ALLOW_CURL_BASH" = 1 ] && return 0
|
|
72
|
+
manual "skipped under --yes (pass --allow-curl-bash to run it unattended): $2"
|
|
73
|
+
return 1
|
|
74
|
+
fi
|
|
75
|
+
confirm "$1"
|
|
76
|
+
}
|
|
77
|
+
node_ok() {
|
|
78
|
+
command -v node >/dev/null 2>&1 || return 1
|
|
79
|
+
local v maj min
|
|
80
|
+
v="$(node -v | sed 's/^v//')"; maj="${v%%.*}"; min="${v#*.}"; min="${min%%.*}"
|
|
81
|
+
[ "${maj:-0}" -gt "$NODE_MIN_MAJOR" ] || { [ "${maj:-0}" -eq "$NODE_MIN_MAJOR" ] && [ "${min:-0}" -ge "$NODE_MIN_MINOR" ]; }
|
|
82
|
+
}
|
|
57
83
|
|
|
58
84
|
# ----------------------------------------------------------------- Homebrew
|
|
59
85
|
say "Homebrew"
|
|
86
|
+
HOMEBREW_CMD='/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
|
|
60
87
|
if command -v brew >/dev/null 2>&1; then skip "brew $(brew --version | head -1)"; else
|
|
61
|
-
if
|
|
88
|
+
if confirm_curl_bash "Homebrew not found. Install it now? (runs the official curl | bash installer)" "$HOMEBREW_CMD"; then
|
|
62
89
|
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
|
63
90
|
else manual "Install Homebrew, then re-run: https://brew.sh"; fi
|
|
64
91
|
fi
|
|
@@ -67,17 +94,23 @@ BREW="$(command -v brew || true)"
|
|
|
67
94
|
[ -z "$BREW" ] && [ -x /usr/local/bin/brew ] && BREW=/usr/local/bin/brew
|
|
68
95
|
|
|
69
96
|
# ----------------------------------------------------------------- Node
|
|
70
|
-
say "Node.js (>=
|
|
71
|
-
if
|
|
97
|
+
say "Node.js (>= $NODE_MIN_MAJOR.$NODE_MIN_MINOR)"
|
|
98
|
+
if node_ok; then
|
|
72
99
|
skip "node $(node -v)"
|
|
73
|
-
elif [ -n "$BREW" ]; then info "installing node..."; "$BREW" install node || manual "brew install node failed — install Node >=
|
|
100
|
+
elif [ -n "$BREW" ]; then info "installing node..."; "$BREW" install node || manual "brew install node failed — install Node >= $NODE_MIN_MAJOR.$NODE_MIN_MINOR manually (nvm install 22)"
|
|
101
|
+
else manual "Install Node >= $NODE_MIN_MAJOR.$NODE_MIN_MINOR (nvm install 22)"; fi
|
|
74
102
|
|
|
75
103
|
# ----------------------------------------------------------------- Maestro
|
|
76
|
-
say "Maestro"
|
|
104
|
+
say "Maestro (pinned $MAESTRO_VERSION)"
|
|
105
|
+
MAESTRO_CMD="curl -Ls https://get.maestro.mobile.dev | MAESTRO_VERSION=$MAESTRO_VERSION bash"
|
|
77
106
|
if [ -x "$HOME/.maestro/bin/maestro" ] || command -v maestro >/dev/null 2>&1; then
|
|
78
|
-
skip "maestro present"
|
|
107
|
+
skip "maestro present ($( (command -v maestro || echo "$HOME/.maestro/bin/maestro") ) — doctor checks the version)"
|
|
79
108
|
else
|
|
80
|
-
if
|
|
109
|
+
if confirm_curl_bash "Install Maestro $MAESTRO_VERSION now? (runs the official curl | bash installer)" "$MAESTRO_CMD"; then
|
|
110
|
+
# The official installer honours MAESTRO_VERSION and downloads
|
|
111
|
+
# github.com/mobile-dev-inc/maestro/releases/download/cli-<version>/maestro.zip
|
|
112
|
+
curl -Ls "https://get.maestro.mobile.dev" | MAESTRO_VERSION="$MAESTRO_VERSION" bash || manual "$MAESTRO_CMD"
|
|
113
|
+
fi
|
|
81
114
|
fi
|
|
82
115
|
|
|
83
116
|
# ============================================================ ANDROID
|
|
@@ -157,7 +190,7 @@ if [ "$PLATFORM" = "both" ] || [ "$PLATFORM" = "ios" ]; then
|
|
|
157
190
|
fi
|
|
158
191
|
fi
|
|
159
192
|
|
|
160
|
-
say "iOS — WebKit DOM bridge (ios-webkit-debug-proxy for bin/wk-ios)"
|
|
193
|
+
say "iOS — WebKit DOM bridge (ios-webkit-debug-proxy + python3 for bin/wk-ios)"
|
|
161
194
|
if command -v ios_webkit_debug_proxy >/dev/null 2>&1; then
|
|
162
195
|
skip "ios-webkit-debug-proxy present"
|
|
163
196
|
elif [ -n "$BREW" ]; then
|
|
@@ -166,6 +199,8 @@ if [ "$PLATFORM" = "both" ] || [ "$PLATFORM" = "ios" ]; then
|
|
|
166
199
|
else
|
|
167
200
|
manual "brew install ios-webkit-debug-proxy (enables bin/wk-ios JS/DOM eval in sim Safari)"
|
|
168
201
|
fi
|
|
202
|
+
if command -v python3 >/dev/null 2>&1; then skip "python3 present"; else
|
|
203
|
+
manual "python3 missing — brew install python3 (or xcode-select --install); bin/wk-ios needs it"; fi
|
|
169
204
|
|
|
170
205
|
say "iOS — Xcode"
|
|
171
206
|
XPATH="$(xcode-select -p 2>/dev/null || true)"
|
|
@@ -196,14 +231,16 @@ if [ "$PLATFORM" = "both" ] || [ "$PLATFORM" = "ios" ]; then
|
|
|
196
231
|
fi
|
|
197
232
|
fi
|
|
198
233
|
else
|
|
199
|
-
manual "Xcode not runnable yet — finish the Xcode steps above, then re-run
|
|
234
|
+
manual "Xcode not runnable yet — finish the Xcode steps above, then re-run bin/setup-mobile.sh"
|
|
200
235
|
fi
|
|
201
236
|
fi
|
|
202
237
|
|
|
203
238
|
# ----------------------------------------------------------------- verify
|
|
204
239
|
say "Verifying with doctor-mobile.sh"
|
|
205
|
-
DOCTOR_ARGS=
|
|
206
|
-
|
|
240
|
+
DOCTOR_ARGS=()
|
|
241
|
+
[ "$PLATFORM" = "ios" ] && DOCTOR_ARGS+=(--ios)
|
|
242
|
+
[ "$PLATFORM" = "android" ] && DOCTOR_ARGS+=(--android)
|
|
243
|
+
bash "$HERE/doctor-mobile.sh" ${DOCTOR_ARGS[@]+"${DOCTOR_ARGS[@]}"} || {
|
|
207
244
|
echo
|
|
208
245
|
echo "Some checks still fail — resolve the MANUAL steps above (Xcode / iOS runtime are the usual ones) and re-run."
|
|
209
246
|
exit 1
|
package/bin/wadb
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# wadb — wrapped adb for
|
|
3
|
-
#
|
|
4
|
-
#
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# wadb — wrapped adb for agents. Uses the adb already on PATH when present, else the
|
|
3
|
+
# SDK copy under ANDROID_HOME, so direct adb commands (logcat, pm clear, am start, ...)
|
|
4
|
+
# work regardless of the caller's environment.
|
|
5
|
+
set -euo pipefail
|
|
5
6
|
export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
|
|
7
|
+
if command -v adb >/dev/null 2>&1; then exec adb "$@"; fi
|
|
8
|
+
if [ ! -x "$ANDROID_HOME/platform-tools/adb" ]; then
|
|
9
|
+
echo "wadb: adb not found on PATH or at $ANDROID_HOME/platform-tools/adb — run bin/setup-mobile.sh --android" >&2
|
|
10
|
+
exit 127
|
|
11
|
+
fi
|
|
6
12
|
exec "$ANDROID_HOME/platform-tools/adb" "$@"
|
package/bin/wk-ios
CHANGED
|
@@ -10,46 +10,58 @@
|
|
|
10
10
|
# USAGE:
|
|
11
11
|
# bin/wk-ios "<js-expression>" # evaluate JS in the sim's Safari, print result
|
|
12
12
|
# bin/wk-ios --url # just print the page WebSocket URL
|
|
13
|
-
# bin/wk-ios --stop # stop the background iwdp proxy
|
|
13
|
+
# bin/wk-ios --stop # stop the background iwdp proxy this script started
|
|
14
14
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
15
|
+
# ENV: WK_IOS_PORT (default 9221) — iwdp device-list port; pages are served on WK_IOS_PORT+1.
|
|
16
|
+
#
|
|
17
|
+
# REQUIRES: brew install ios-webkit-debug-proxy ; python3 ; Node >= 22.4 ; a booted iOS
|
|
18
|
+
# Simulator with Safari on a page (bin/mcli set-app com.apple.mobilesafari; bin/mcli open-url <url>).
|
|
17
19
|
set -euo pipefail
|
|
18
20
|
|
|
19
21
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
PORT="${WK_IOS_PORT:-9221}"
|
|
23
|
+
PAGE_PORT=$((PORT + 1))
|
|
24
|
+
IWDP_LOG="${TMPDIR:-/tmp}/wk-ios-iwdp-$PORT.log"
|
|
25
|
+
IWDP_PIDFILE="${TMPDIR:-/tmp}/wk-ios-iwdp-$PORT.pid"
|
|
22
26
|
|
|
27
|
+
# Stop only the proxy we started (recorded in our pidfile) — never other users' iwdp.
|
|
23
28
|
stop_proxy() {
|
|
24
|
-
if [[ -f "$IWDP_PIDFILE" ]]; then
|
|
25
|
-
|
|
29
|
+
if [[ -f "$IWDP_PIDFILE" ]]; then
|
|
30
|
+
PID="$(cat "$IWDP_PIDFILE" 2>/dev/null || true)"
|
|
31
|
+
if [[ -n "$PID" ]]; then kill "$PID" 2>/dev/null || true; fi
|
|
32
|
+
rm -f "$IWDP_PIDFILE"
|
|
33
|
+
fi
|
|
26
34
|
}
|
|
27
35
|
|
|
28
36
|
if [[ "${1:-}" == "--stop" ]]; then stop_proxy; echo "iwdp stopped"; exit 0; fi
|
|
29
37
|
|
|
30
38
|
command -v ios_webkit_debug_proxy >/dev/null || { echo "missing: brew install ios-webkit-debug-proxy" >&2; exit 1; }
|
|
39
|
+
command -v python3 >/dev/null || { echo "missing: python3 (parses the iwdp page list)" >&2; exit 1; }
|
|
31
40
|
|
|
32
41
|
# 1) Find the simulator's web inspector unix socket.
|
|
33
42
|
SOCK="$(lsof -U 2>/dev/null | grep -oE '/private/var/tmp/[^ ]+webinspectord_sim.socket' | head -1 || true)"
|
|
34
43
|
[[ -n "$SOCK" ]] || { echo "no webinspectord_sim socket — is a Simulator booted with Safari open?" >&2; exit 1; }
|
|
35
44
|
|
|
36
|
-
# 2) Ensure iwdp is running against that socket.
|
|
37
|
-
if ! curl -s http://localhost
|
|
38
|
-
ios_webkit_debug_proxy -F -s "unix:$SOCK" > "$IWDP_LOG" 2>&1 &
|
|
45
|
+
# 2) Ensure iwdp is running against that socket on our ports.
|
|
46
|
+
if ! curl -s "http://localhost:$PORT/json" >/dev/null 2>&1; then
|
|
47
|
+
ios_webkit_debug_proxy -c "null:$PORT,:$PAGE_PORT-$((PAGE_PORT + 100))" -F -s "unix:$SOCK" > "$IWDP_LOG" 2>&1 &
|
|
39
48
|
echo $! > "$IWDP_PIDFILE"
|
|
40
|
-
for
|
|
49
|
+
for _ in $(seq 1 20); do curl -s "http://localhost:$PORT/json" >/dev/null 2>&1 && break; sleep 0.25; done
|
|
41
50
|
fi
|
|
42
51
|
|
|
43
|
-
# 3) Resolve the frontmost page's WebSocket URL
|
|
44
|
-
#
|
|
45
|
-
# page list
|
|
52
|
+
# 3) Resolve the frontmost page's WebSocket URL: the LAST page that is not about:blank
|
|
53
|
+
# (iwdp lists pages in creation order; the newest non-blank one is what the user sees).
|
|
54
|
+
# Retry briefly: right after iwdp starts, :$PORT answers before the page list is populated.
|
|
46
55
|
WS=""
|
|
47
|
-
for
|
|
48
|
-
WS="$(curl -s http://localhost
|
|
56
|
+
for _ in $(seq 1 20); do
|
|
57
|
+
WS="$(curl -s "http://localhost:$PAGE_PORT/json" 2>/dev/null | python3 -c 'import sys,json
|
|
58
|
+
d=json.load(sys.stdin)
|
|
59
|
+
p=[x for x in d if x.get("url") and x["url"]!="about:blank"] or d
|
|
60
|
+
print(p[-1]["webSocketDebuggerUrl"] if p else "")' 2>/dev/null || true)"
|
|
49
61
|
[[ -n "$WS" ]] && break
|
|
50
62
|
sleep 0.25
|
|
51
63
|
done
|
|
52
|
-
[[ -n "$WS" ]] || { echo "no inspectable Safari page found on
|
|
64
|
+
[[ -n "$WS" ]] || { echo "no inspectable Safari page found on :$PAGE_PORT (open a URL in the sim Safari first)" >&2; exit 1; }
|
|
53
65
|
|
|
54
66
|
if [[ "${1:-}" == "--url" ]]; then echo "$WS"; exit 0; fi
|
|
55
67
|
[[ $# -ge 1 ]] || { echo "usage: wk-ios \"<js-expression>\" | --url | --stop" >&2; exit 2; }
|
package/bin/wkeval.mjs
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
// wkeval.mjs — evaluate JS in iOS Simulator Safari via WebKit Remote Inspector (through iwdp)
|
|
2
2
|
// Modern WebKit uses a Target-wrapped protocol: commands go via Target.sendMessageToTarget,
|
|
3
3
|
// responses come back via Target.dispatchMessageFromTarget.
|
|
4
|
-
// Usage: node wkeval.mjs '<js-expression>'
|
|
4
|
+
// Usage: node wkeval.mjs '<js-expression>' (WK_WS = page WebSocket URL, set by bin/wk-ios)
|
|
5
|
+
// Exit codes: 0 ok · 2 usage · 3 timeout · 4 websocket error · 5 eval failed · 6 Node too old
|
|
6
|
+
if (typeof WebSocket === 'undefined') {
|
|
7
|
+
console.error('Node >= 22.4 required (global WebSocket is unavailable in this Node version)');
|
|
8
|
+
process.exit(6);
|
|
9
|
+
}
|
|
5
10
|
const WS_URL = process.env.WK_WS || 'ws://localhost:9222/devtools/page/1';
|
|
6
11
|
const expr = process.argv[2];
|
|
7
12
|
if (!expr) { console.error('need a JS expression arg'); process.exit(2); }
|
|
@@ -33,6 +38,7 @@ ws.addEventListener('message', async (ev) => {
|
|
|
33
38
|
const r = res && res.result && res.result.result ? res.result.result : (res && res.result);
|
|
34
39
|
if (r && 'value' in r) console.log(typeof r.value === 'object' ? JSON.stringify(r.value) : String(r.value));
|
|
35
40
|
else console.log(JSON.stringify(r));
|
|
41
|
+
try { ws.close(); } catch { /* ignore */ }
|
|
36
42
|
process.exit(0);
|
|
37
43
|
} catch (e) { console.error('EVAL_FAIL', e.message || e); process.exit(5); }
|
|
38
44
|
}
|
|
@@ -2,21 +2,37 @@
|
|
|
2
2
|
# Tracks what changed in each release so /qa-knowledge-list can show history.
|
|
3
3
|
|
|
4
4
|
releases:
|
|
5
|
+
- version: "0.6.0"
|
|
6
|
+
date: "2026-09-03"
|
|
7
|
+
summary: "Asynchronous money movement — verifying that a payment credited a ledger exactly once, and what a callback endpoint owes the caller it cannot control."
|
|
8
|
+
entries_added:
|
|
9
|
+
- id: technique-exactly-once-verification
|
|
10
|
+
type: technique
|
|
11
|
+
name: "Exactly-Once Verification — Proving a Payment Moved Money Once"
|
|
12
|
+
description: "Assert the balance delta, never the status field, then attack the flow with the five cases that break it: an identity collision between records created in the same instant, the same event redelivered simultaneously rather than sequentially, a contradicting event, an event arriving after a timeout closed the record, and a settlement amount that does not match the request. Includes what to do when the defect corrupts your own parallel test run."
|
|
13
|
+
- id: technique-async-callback-contracts
|
|
14
|
+
type: technique
|
|
15
|
+
name: "Asynchronous Callback Contracts — What a Webhook Endpoint Owes Its Caller"
|
|
16
|
+
description: "Is the endpoint authenticated, which field does it actually deduplicate on rather than which one is documented, what does it do with an event matching no record, and can the caller distinguish applied from duplicate from discarded. The unmatched-event case is where collected money goes uncredited with nothing to show for it."
|
|
17
|
+
entries_modified: []
|
|
18
|
+
entries_removed: []
|
|
19
|
+
origin: "A black-box assessment of an asynchronous mobile-money top-up service. Every sequential path through it was correct and every concurrent one was not, so the existing request-and-response techniques reported it as healthy while two independent defects were crediting wallets with money no provider had collected."
|
|
20
|
+
|
|
5
21
|
- version: "0.5.0"
|
|
6
22
|
date: "2026-09-03"
|
|
7
23
|
summary: "Payload correctness and release-level verification — what to do with the answer once you have it, and how to report a set of tickets rather than one."
|
|
8
24
|
entries_added:
|
|
9
25
|
- id: technique-derived-value-verification
|
|
10
26
|
type: technique
|
|
11
|
-
name: "Derived Value Verification
|
|
27
|
+
name: "Derived Value Verification — Recomputing the Number and Naming What It Does Not Prove"
|
|
12
28
|
description: "Recompute every percentage, total, ratio and aggregate from the raw figures in the same response, using the formula from the spec rather than the code. Choose cases that stress sign, zero, scale and cardinality, add the structural invariants, then state explicitly that internal consistency is not correctness and name the independent oracle that would close the gap."
|
|
13
29
|
- id: technique-presentation-integrity
|
|
14
30
|
type: technique
|
|
15
|
-
name: "Presentation Integrity
|
|
31
|
+
name: "Presentation Integrity — Does the Screen Show What the Service Sent?"
|
|
16
32
|
description: "A correct payload can still reach the user as a wrong number. Formatting helpers that guess what a value is, units applied twice, rounding across a threshold, truncation shown as a total. Invisible from either surface alone, and routinely attributed to the wrong change."
|
|
17
33
|
- id: technique-expected-behaviour-specification
|
|
18
34
|
type: technique
|
|
19
|
-
name: "Expected Behaviour Specification
|
|
35
|
+
name: "Expected Behaviour Specification — Writing the Spec That Should Have Existed"
|
|
20
36
|
description: "Observed against expected, per area rather than per case, with the reject-or-clamp and error-or-empty decisions made explicit, ranked by what users can reach today, and closed with a plain-English reply for whoever decides to fund the fix."
|
|
21
37
|
- id: technique-config-surface-verification
|
|
22
38
|
type: technique
|
|
@@ -24,7 +40,7 @@ releases:
|
|
|
24
40
|
description: "When the configuration mechanism is itself the change: values matching the deployed config rather than the source default, nothing leaking alongside them, the runtime genuinely reading through the surface rather than around it, and the route existing in every environment the change will be promoted to."
|
|
25
41
|
- id: technique-release-readiness-verification
|
|
26
42
|
type: technique
|
|
27
|
-
name: "Release Readiness Verification
|
|
43
|
+
name: "Release Readiness Verification — Many Tickets, One Build"
|
|
28
44
|
description: "Establish what is deployed before testing anything, give every ticket a result from a fixed vocabulary that keeps not-testable and not-tested visible, mark code-verified-only as UNVERIFIABLE rather than green, separate carry-overs, and end with a disposition and the condition that would reverse it."
|
|
29
45
|
entries_modified: []
|
|
30
46
|
entries_removed: []
|
|
@@ -36,19 +52,19 @@ releases:
|
|
|
36
52
|
entries_added:
|
|
37
53
|
- id: technique-environment-fingerprinting
|
|
38
54
|
type: technique
|
|
39
|
-
name: "Environment Fingerprinting
|
|
55
|
+
name: "Environment Fingerprinting — Proving What Is Actually Running"
|
|
40
56
|
description: "Before the first probe: which build is deployed in every component of the path, whether the changed code path is SELECTED here, and whether the commit under test is an ancestor of what is running. Same build plus different behaviour means configuration, not deploy lag. Adds NOT-REACHABLE as the honest verdict when the change is not live in the environment being measured."
|
|
41
57
|
- id: technique-authenticated-api-probing
|
|
42
58
|
type: technique
|
|
43
|
-
name: "Authenticated API Probing
|
|
59
|
+
name: "Authenticated API Probing — Calling the Endpoint Behind the Screen"
|
|
44
60
|
description: "Get a real authenticated request context without handling a token by running fetch inside the already-logged-in page, then drive a written case matrix through it. Ten case families, how to read each signal, and the evidence and safety rules that keep credentials and response bodies off disk."
|
|
45
61
|
- id: technique-ui-api-differential
|
|
46
62
|
type: technique
|
|
47
|
-
name: "UI-vs-API Differential
|
|
63
|
+
name: "UI-vs-API Differential — Running the Same Matrix at Both Surfaces"
|
|
48
64
|
description: "Sort every finding into reproducible-at-both, API-only, UI-only and neither. Corrects the most common false PASS in exploratory testing: a client-side guard recorded as the endpoint's behaviour."
|
|
49
65
|
- id: technique-silent-failure-audit
|
|
50
66
|
type: technique
|
|
51
|
-
name: "Silent Failure Audit
|
|
67
|
+
name: "Silent Failure Audit — Failures That Render as Ordinary Results"
|
|
52
68
|
description: "Seven shapes of failure that look like a legitimate empty or zero answer, how to force each one, and why they recur as a class rather than as individual bugs."
|
|
53
69
|
entries_modified: []
|
|
54
70
|
entries_removed: []
|
|
@@ -60,15 +76,15 @@ releases:
|
|
|
60
76
|
entries_added:
|
|
61
77
|
- id: technique-verification-mode-selection
|
|
62
78
|
type: technique
|
|
63
|
-
name: "Verification Mode Selection
|
|
79
|
+
name: "Verification Mode Selection — Choosing How to Observe a Backend Change"
|
|
64
80
|
description: "Route each AC to the observation channel that can falsify it: API-behind-the-screen, direct request to a no-UI endpoint, LLM output judgement, or needs-a-human. Makes UNVERIFIABLE a first-class verdict instead of a PASS inferred from source."
|
|
65
81
|
- id: technique-functional-diff-analysis
|
|
66
82
|
type: technique
|
|
67
|
-
name: "Functional Diff Analysis
|
|
83
|
+
name: "Functional Diff Analysis — Reading a Backend Change for Behaviour, Not Style"
|
|
68
84
|
description: "Eight passes over a BE/API/LLM diff that produce falsifiable hypotheses rather than code comments: spec drift, value tracing to the service boundary, failure paths, identity propagation, producer/consumer contract, scope, and the tests that shipped with it."
|
|
69
85
|
- id: technique-llm-output-verification
|
|
70
86
|
type: technique
|
|
71
|
-
name: "LLM & Agent Output Verification
|
|
87
|
+
name: "LLM & Agent Output Verification — Running It and Judging the Result"
|
|
72
88
|
description: "Fixed input set, written rubric, before-and-after on the same inputs, N runs to expose variance, assertions on properties and tool trajectory rather than on generated text. Includes what not to claim from a passing run."
|
|
73
89
|
entries_modified: []
|
|
74
90
|
entries_removed: []
|
|
@@ -80,12 +96,14 @@ releases:
|
|
|
80
96
|
entries_added:
|
|
81
97
|
- id: technique-contract-narrowing
|
|
82
98
|
type: technique
|
|
83
|
-
name: "Contract Narrowing
|
|
99
|
+
name: "Contract Narrowing — Verifying a Swapped Data Source"
|
|
84
100
|
description: "When a full-record read is replaced by a projection, the request's field-selection list becomes the payload specification. Trace every consumed field to a selection token; unrequested fields vanish with no error, no DLQ, and a green suite."
|
|
85
101
|
- id: technique-test-suite-audit
|
|
86
102
|
type: technique
|
|
87
103
|
name: "Auditing the Branch's Own Tests"
|
|
88
104
|
description: "When a change rewrites the tests that are meant to prove it works, those tests become part of the change. Check that each assertion still falsifies what its name claims, and that artefacts named by an AC actually exist."
|
|
105
|
+
entries_modified: []
|
|
106
|
+
entries_removed: []
|
|
89
107
|
origin: "A read-path swap on an event-processing service — a full-record read replaced by a projection read. The contract-narrowing audit found a dropped webhook field that 175 green unit tests could not see; the test audit found an e2e negative scenario that could only pass when the system was broken."
|
|
90
108
|
|
|
91
109
|
- version: "0.1.0"
|