visionclaw 0.1.187-beta.8 → 0.1.187-dev.refactor-computer-use-direct-coordinates.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/loop.js +1 -1
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/providers/client-factory.d.ts +1 -1
- package/dist/agent/providers/client-factory.js +1 -1
- package/dist/agent/runtime-surface.d.ts +1 -1
- package/dist/agent/runtime-surface.d.ts.map +1 -1
- package/dist/agent/runtime-surface.js +35 -18
- package/dist/agent/runtime-surface.js.map +1 -1
- package/dist/agent/system-prompt.d.ts.map +1 -1
- package/dist/agent/system-prompt.js +1 -3
- package/dist/agent/system-prompt.js.map +1 -1
- package/dist/builtin-skills/macos-automation/SKILL.md +13 -10
- package/dist/onboarding/generate-wallpaper.d.ts +3 -8
- package/dist/onboarding/generate-wallpaper.d.ts.map +1 -1
- package/dist/onboarding/generate-wallpaper.js +3 -123
- package/dist/onboarding/generate-wallpaper.js.map +1 -1
- package/dist/tools/computer-use.d.ts +56 -6
- package/dist/tools/computer-use.d.ts.map +1 -1
- package/dist/tools/computer-use.js +129 -286
- package/dist/tools/computer-use.js.map +1 -1
- package/dist-agent/bundle.cjs +208 -574
- package/package.json +1 -1
- package/dist/agent/applied-credential-signature.d.ts +0 -53
- package/dist/agent/applied-credential-signature.d.ts.map +0 -1
- package/dist/agent/applied-credential-signature.js +0 -137
- package/dist/agent/applied-credential-signature.js.map +0 -1
- package/dist/agent/tunnel-credential-handler.d.ts +0 -90
- package/dist/agent/tunnel-credential-handler.d.ts.map +0 -1
- package/dist/agent/tunnel-credential-handler.js +0 -162
- package/dist/agent/tunnel-credential-handler.js.map +0 -1
- package/dist/billing/payg-handler.d.ts +0 -29
- package/dist/billing/payg-handler.d.ts.map +0 -1
- package/dist/billing/payg-handler.js +0 -92
- package/dist/billing/payg-handler.js.map +0 -1
- package/dist/billing/payment-handler.d.ts +0 -24
- package/dist/billing/payment-handler.d.ts.map +0 -1
- package/dist/billing/payment-handler.js +0 -101
- package/dist/billing/payment-handler.js.map +0 -1
- package/dist/builtin-skills/catalog/phone-adb-automation/SKILL.md +0 -412
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_input.sh +0 -132
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_launch.sh +0 -166
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_screenshot.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_security_kbd.py +0 -174
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_setup.sh +0 -274
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_swipe.sh +0 -111
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_tap.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_ui_parse.py +0 -176
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_wake_unlock.sh +0 -67
- package/dist/builtin-skills/transcribe-audio/SKILL.md +0 -122
- package/dist/data-processing/convert-demo-cli.d.ts +0 -7
- package/dist/data-processing/convert-demo-cli.d.ts.map +0 -1
- package/dist/data-processing/convert-demo-cli.js +0 -30
- package/dist/data-processing/convert-demo-cli.js.map +0 -1
- package/dist/data-processing/convert-demo.d.ts +0 -26
- package/dist/data-processing/convert-demo.d.ts.map +0 -1
- package/dist/data-processing/convert-demo.js +0 -233
- package/dist/data-processing/convert-demo.js.map +0 -1
- package/dist/obs/rdp/icons/icons/app_windows.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_get.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_send.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/control.svg +0 -4
- package/dist/obs/rdp/icons/icons/desktop.svg +0 -4
- package/dist/obs/rdp/icons/icons/display.svg +0 -4
- package/dist/obs/rdp/icons/icons/launchpad.svg +0 -4
- package/dist/obs/rdp/icons/icons/mission_control.svg +0 -4
- package/dist/obs/rdp/icons/icons/screenshot.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_actual.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_fit.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_in.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_out.svg +0 -4
- package/dist/obs/tunnel-telemetry.d.ts +0 -46
- package/dist/obs/tunnel-telemetry.d.ts.map +0 -1
- package/dist/obs/tunnel-telemetry.js +0 -70
- package/dist/obs/tunnel-telemetry.js.map +0 -1
- package/dist/service/gbox-tun.d.ts +0 -14
- package/dist/service/gbox-tun.d.ts.map +0 -1
- package/dist/service/gbox-tun.js +0 -315
- package/dist/service/gbox-tun.js.map +0 -1
- package/dist/tools/coordinate-resolver.d.ts +0 -30
- package/dist/tools/coordinate-resolver.d.ts.map +0 -1
- package/dist/tools/coordinate-resolver.js +0 -104
- package/dist/tools/coordinate-resolver.js.map +0 -1
- package/dist/utils/wechat-monitor.d.ts +0 -21
- package/dist/utils/wechat-monitor.d.ts.map +0 -1
- package/dist/utils/wechat-monitor.js +0 -88
- package/dist/utils/wechat-monitor.js.map +0 -1
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# phone_launch.sh - Launch an Android app by name alias or package name
|
|
3
|
-
#
|
|
4
|
-
# Usage:
|
|
5
|
-
# bash phone_launch.sh <app_name_or_package> [options]
|
|
6
|
-
#
|
|
7
|
-
# Options:
|
|
8
|
-
# -d DEVICE_ID Target specific device
|
|
9
|
-
# -l List all known app aliases
|
|
10
|
-
# -q Quiet mode
|
|
11
|
-
#
|
|
12
|
-
# Supports common Chinese app aliases (case-insensitive):
|
|
13
|
-
# wechat, wecom, meituan, ctrip, xiaohongshu/xhs, bilibili,
|
|
14
|
-
# baidu, boss/bosszhipin, doubao, taobao, alipay, jd, didi,
|
|
15
|
-
# douyin/tiktok, kuaishou, zhihu, ele, pinduoduo/pdd, gaode/amap
|
|
16
|
-
#
|
|
17
|
-
# Examples:
|
|
18
|
-
# bash phone_launch.sh wechat
|
|
19
|
-
# bash phone_launch.sh com.tencent.mm
|
|
20
|
-
# bash phone_launch.sh -l
|
|
21
|
-
|
|
22
|
-
set -eo pipefail
|
|
23
|
-
|
|
24
|
-
DEVICE=""
|
|
25
|
-
QUIET=false
|
|
26
|
-
LIST=false
|
|
27
|
-
|
|
28
|
-
# Parse options manually to allow "phone_launch.sh <app> -d DEV" order
|
|
29
|
-
ARGV=()
|
|
30
|
-
i=1
|
|
31
|
-
while [ $i -le $# ]; do
|
|
32
|
-
arg="${!i}"
|
|
33
|
-
case "$arg" in
|
|
34
|
-
-d) i=$((i+1)); DEVICE="${!i}" ;;
|
|
35
|
-
-l) LIST=true ;;
|
|
36
|
-
-q) QUIET=true ;;
|
|
37
|
-
-h|--help) head -21 "$0" | tail -16; exit 0 ;;
|
|
38
|
-
-*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
|
39
|
-
*) ARGV+=("$arg") ;;
|
|
40
|
-
esac
|
|
41
|
-
i=$((i+1))
|
|
42
|
-
done
|
|
43
|
-
|
|
44
|
-
# App alias -> package name lookup (bash 3.2 compatible)
|
|
45
|
-
resolve_app() {
|
|
46
|
-
local app
|
|
47
|
-
app=$(echo "$1" | tr '[:upper:]' '[:lower:]')
|
|
48
|
-
case "$app" in
|
|
49
|
-
wechat|weixin) echo "com.tencent.mm" ;;
|
|
50
|
-
wecom|qiyeweixin) echo "com.tencent.wework" ;;
|
|
51
|
-
meituan) echo "com.sankuai.meituan" ;;
|
|
52
|
-
ctrip|xiecheng) echo "ctrip.android.view" ;;
|
|
53
|
-
xiaohongshu|xhs) echo "com.xingin.xhs" ;;
|
|
54
|
-
bilibili) echo "tv.danmaku.bili" ;;
|
|
55
|
-
baidu) echo "com.baidu.searchbox" ;;
|
|
56
|
-
boss|bosszhipin) echo "com.hpbr.bosszhipin" ;;
|
|
57
|
-
doubao) echo "com.larus.nova" ;;
|
|
58
|
-
taobao) echo "com.taobao.taobao" ;;
|
|
59
|
-
alipay|zhifubao) echo "com.eg.android.AlipayGphone" ;;
|
|
60
|
-
jd|jingdong) echo "com.jingdong.app.mall" ;;
|
|
61
|
-
didi) echo "com.sdu.didi.psnger" ;;
|
|
62
|
-
douyin|tiktok) echo "com.ss.android.ugc.aweme" ;;
|
|
63
|
-
kuaishou) echo "com.smile.gifmaker" ;;
|
|
64
|
-
zhihu) echo "com.zhihu.android" ;;
|
|
65
|
-
ele|eleme) echo "me.ele" ;;
|
|
66
|
-
pinduoduo|pdd) echo "com.xunmeng.pinduoduo" ;;
|
|
67
|
-
gaode|amap) echo "com.autonavi.minimap" ;;
|
|
68
|
-
qq) echo "com.tencent.mobileqq" ;;
|
|
69
|
-
wps) echo "cn.wps.moffice_eng" ;;
|
|
70
|
-
settings) echo "com.android.settings" ;;
|
|
71
|
-
camera) echo "com.android.camera" ;;
|
|
72
|
-
chrome) echo "com.android.chrome" ;;
|
|
73
|
-
calendar) echo "com.google.android.calendar" ;;
|
|
74
|
-
maps) echo "com.google.android.apps.maps" ;;
|
|
75
|
-
phone) echo "com.android.dialer" ;;
|
|
76
|
-
contacts) echo "com.android.contacts" ;;
|
|
77
|
-
messages) echo "com.google.android.apps.messaging" ;;
|
|
78
|
-
gallery) echo "com.miui.gallery" ;;
|
|
79
|
-
files) echo "com.android.fileexplorer" ;;
|
|
80
|
-
*) echo "" ;;
|
|
81
|
-
esac
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
list_aliases() {
|
|
85
|
-
cat <<'ALIASES'
|
|
86
|
-
Known app aliases:
|
|
87
|
-
alipay/zhifubao -> com.eg.android.AlipayGphone
|
|
88
|
-
amap/gaode -> com.autonavi.minimap
|
|
89
|
-
baidu -> com.baidu.searchbox
|
|
90
|
-
bilibili -> tv.danmaku.bili
|
|
91
|
-
boss/bosszhipin -> com.hpbr.bosszhipin
|
|
92
|
-
calendar -> com.google.android.calendar
|
|
93
|
-
camera -> com.android.camera
|
|
94
|
-
chrome -> com.android.chrome
|
|
95
|
-
contacts -> com.android.contacts
|
|
96
|
-
ctrip/xiecheng -> ctrip.android.view
|
|
97
|
-
didi -> com.sdu.didi.psnger
|
|
98
|
-
doubao -> com.larus.nova
|
|
99
|
-
douyin/tiktok -> com.ss.android.ugc.aweme
|
|
100
|
-
ele/eleme -> me.ele
|
|
101
|
-
files -> com.android.fileexplorer
|
|
102
|
-
gallery -> com.miui.gallery
|
|
103
|
-
jd/jingdong -> com.jingdong.app.mall
|
|
104
|
-
kuaishou -> com.smile.gifmaker
|
|
105
|
-
maps -> com.google.android.apps.maps
|
|
106
|
-
meituan -> com.sankuai.meituan
|
|
107
|
-
messages -> com.google.android.apps.messaging
|
|
108
|
-
phone -> com.android.dialer
|
|
109
|
-
pinduoduo/pdd -> com.xunmeng.pinduoduo
|
|
110
|
-
qq -> com.tencent.mobileqq
|
|
111
|
-
settings -> com.android.settings
|
|
112
|
-
taobao -> com.taobao.taobao
|
|
113
|
-
wechat/weixin -> com.tencent.mm
|
|
114
|
-
wecom/qiyeweixin -> com.tencent.wework
|
|
115
|
-
wps -> cn.wps.moffice_eng
|
|
116
|
-
xiaohongshu/xhs -> com.xingin.xhs
|
|
117
|
-
zhihu -> com.zhihu.android
|
|
118
|
-
ALIASES
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
if $LIST; then
|
|
122
|
-
list_aliases
|
|
123
|
-
exit 0
|
|
124
|
-
fi
|
|
125
|
-
|
|
126
|
-
if [ ${#ARGV[@]} -eq 0 ]; then
|
|
127
|
-
echo "Usage: $0 <app_name_or_package> [-d device] [-l] [-q]" >&2
|
|
128
|
-
exit 1
|
|
129
|
-
fi
|
|
130
|
-
|
|
131
|
-
APP_INPUT="${ARGV[0]}"
|
|
132
|
-
|
|
133
|
-
ADB_CMD="adb"
|
|
134
|
-
if [ -n "$DEVICE" ]; then
|
|
135
|
-
ADB_CMD="adb -s $DEVICE"
|
|
136
|
-
fi
|
|
137
|
-
|
|
138
|
-
info() { $QUIET || echo "[phone_launch] $*" >&2; }
|
|
139
|
-
|
|
140
|
-
# Resolve package name
|
|
141
|
-
PACKAGE=$(resolve_app "$APP_INPUT")
|
|
142
|
-
if [ -n "$PACKAGE" ]; then
|
|
143
|
-
info "Resolved '$APP_INPUT' -> $PACKAGE"
|
|
144
|
-
elif echo "$APP_INPUT" | grep -q '\.'; then
|
|
145
|
-
PACKAGE="$APP_INPUT"
|
|
146
|
-
info "Using package name directly: $PACKAGE"
|
|
147
|
-
else
|
|
148
|
-
# Try fuzzy search on device
|
|
149
|
-
info "Unknown alias '$APP_INPUT', searching on device..."
|
|
150
|
-
MATCHES=$($ADB_CMD shell pm list packages | grep -i "$APP_INPUT" | sed 's/package://' || true)
|
|
151
|
-
if [ -z "$MATCHES" ]; then
|
|
152
|
-
echo "ERROR: No package found matching '$APP_INPUT'" >&2
|
|
153
|
-
echo "Use -l to list known aliases, or provide full package name" >&2
|
|
154
|
-
exit 1
|
|
155
|
-
fi
|
|
156
|
-
PACKAGE=$(echo "$MATCHES" | head -1 | tr -d '\r')
|
|
157
|
-
info "Found package: $PACKAGE"
|
|
158
|
-
fi
|
|
159
|
-
|
|
160
|
-
# Launch the app
|
|
161
|
-
info "Launching $PACKAGE..."
|
|
162
|
-
$ADB_CMD shell monkey -p "$PACKAGE" -c android.intent.category.LAUNCHER 1 >/dev/null 2>&1
|
|
163
|
-
|
|
164
|
-
sleep 1
|
|
165
|
-
info "Done."
|
|
166
|
-
echo "$PACKAGE"
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# phone_screenshot.sh - Take a screenshot from connected Android phone via ADB
|
|
3
|
-
#
|
|
4
|
-
# Usage:
|
|
5
|
-
# bash phone_screenshot.sh [options]
|
|
6
|
-
#
|
|
7
|
-
# Options:
|
|
8
|
-
# -d DEVICE_ID Target specific device (default: first connected device)
|
|
9
|
-
# -o OUTPUT_PATH Output file path (default: /tmp/phone_screen.png)
|
|
10
|
-
# -s SCALE Scale percentage 1-100 (default: 50)
|
|
11
|
-
# -r Raw only - skip scaling, save native resolution
|
|
12
|
-
# -q Quiet mode - suppress info messages
|
|
13
|
-
#
|
|
14
|
-
# Output:
|
|
15
|
-
# Saves scaled screenshot to OUTPUT_PATH
|
|
16
|
-
# Prints the output file path on success
|
|
17
|
-
#
|
|
18
|
-
# Examples:
|
|
19
|
-
# bash phone_screenshot.sh # Default: 50% scale to /tmp/phone_screen.png
|
|
20
|
-
# bash phone_screenshot.sh -o /tmp/my.png -s 40 # 40% scale to custom path
|
|
21
|
-
# bash phone_screenshot.sh -d abc123 -r # Raw native resolution for specific device
|
|
22
|
-
|
|
23
|
-
set -euo pipefail
|
|
24
|
-
|
|
25
|
-
DEVICE=""
|
|
26
|
-
OUTPUT="/tmp/phone_screen.png"
|
|
27
|
-
SCALE=50
|
|
28
|
-
RAW=false
|
|
29
|
-
QUIET=false
|
|
30
|
-
|
|
31
|
-
while getopts "d:o:s:rq" opt; do
|
|
32
|
-
case $opt in
|
|
33
|
-
d) DEVICE="$OPTARG" ;;
|
|
34
|
-
o) OUTPUT="$OPTARG" ;;
|
|
35
|
-
s) SCALE="$OPTARG" ;;
|
|
36
|
-
r) RAW=true ;;
|
|
37
|
-
q) QUIET=true ;;
|
|
38
|
-
*) echo "Usage: $0 [-d device] [-o output] [-s scale%] [-r] [-q]" >&2; exit 1 ;;
|
|
39
|
-
esac
|
|
40
|
-
done
|
|
41
|
-
|
|
42
|
-
ADB_CMD="adb"
|
|
43
|
-
if [ -n "$DEVICE" ]; then
|
|
44
|
-
ADB_CMD="adb -s $DEVICE"
|
|
45
|
-
fi
|
|
46
|
-
|
|
47
|
-
info() { $QUIET || echo "[phone_screenshot] $*" >&2; }
|
|
48
|
-
|
|
49
|
-
# Verify device connection
|
|
50
|
-
if ! $ADB_CMD get-state >/dev/null 2>&1; then
|
|
51
|
-
echo "ERROR: No device connected or device unauthorized" >&2
|
|
52
|
-
echo "Run 'adb devices' to check connection status" >&2
|
|
53
|
-
exit 1
|
|
54
|
-
fi
|
|
55
|
-
|
|
56
|
-
REMOTE_PATH="/sdcard/screen_$(date +%s).png"
|
|
57
|
-
RAW_PATH="/tmp/phone_raw_$$.png"
|
|
58
|
-
|
|
59
|
-
info "Capturing screenshot..."
|
|
60
|
-
$ADB_CMD shell screencap -p "$REMOTE_PATH"
|
|
61
|
-
|
|
62
|
-
info "Pulling to local..."
|
|
63
|
-
$ADB_CMD pull "$REMOTE_PATH" "$RAW_PATH" >/dev/null 2>&1
|
|
64
|
-
|
|
65
|
-
info "Cleaning up device temp file..."
|
|
66
|
-
$ADB_CMD shell rm -f "$REMOTE_PATH"
|
|
67
|
-
|
|
68
|
-
if $RAW; then
|
|
69
|
-
mv "$RAW_PATH" "$OUTPUT"
|
|
70
|
-
info "Saved raw screenshot to $OUTPUT"
|
|
71
|
-
else
|
|
72
|
-
# Get native resolution from the image
|
|
73
|
-
NATIVE_W=$(sips -g pixelWidth "$RAW_PATH" | tail -1 | awk '{print $2}')
|
|
74
|
-
NATIVE_H=$(sips -g pixelHeight "$RAW_PATH" | tail -1 | awk '{print $2}')
|
|
75
|
-
|
|
76
|
-
SCALED_W=$((NATIVE_W * SCALE / 100))
|
|
77
|
-
SCALED_H=$((NATIVE_H * SCALE / 100))
|
|
78
|
-
|
|
79
|
-
info "Native: ${NATIVE_W}x${NATIVE_H} -> Scaled (${SCALE}%): ${SCALED_W}x${SCALED_H}"
|
|
80
|
-
sips --resampleHeightWidth "$SCALED_H" "$SCALED_W" "$RAW_PATH" --out "$OUTPUT" >/dev/null 2>&1
|
|
81
|
-
|
|
82
|
-
rm -f "$RAW_PATH"
|
|
83
|
-
info "Saved scaled screenshot to $OUTPUT"
|
|
84
|
-
fi
|
|
85
|
-
|
|
86
|
-
# Output the path for easy capture by calling scripts
|
|
87
|
-
echo "$OUTPUT"
|
|
@@ -1,174 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
phone_security_kbd.py - Handle randomized security keyboards on Android.
|
|
4
|
-
|
|
5
|
-
Financial apps (banking, stock trading) use security keyboards where digit
|
|
6
|
-
positions change every time. This script reads the UI dump to map digit
|
|
7
|
-
positions and types a sequence by tapping each digit.
|
|
8
|
-
|
|
9
|
-
Usage:
|
|
10
|
-
python3 phone_security_kbd.py <digits> [options]
|
|
11
|
-
|
|
12
|
-
Options:
|
|
13
|
-
-d DEVICE_ID Target specific device
|
|
14
|
-
-f XML_FILE Use existing UI dump XML file
|
|
15
|
-
--delay MS Delay between taps in ms (default: 200)
|
|
16
|
-
--dry-run Show mapped positions without tapping
|
|
17
|
-
-q Quiet mode
|
|
18
|
-
|
|
19
|
-
How it works:
|
|
20
|
-
1. Dumps the UI hierarchy via uiautomator
|
|
21
|
-
2. Finds all clickable elements with numeric content-desc (0-9)
|
|
22
|
-
3. Maps each digit to its center coordinates
|
|
23
|
-
4. Taps each digit in sequence with configurable delay
|
|
24
|
-
|
|
25
|
-
Examples:
|
|
26
|
-
python3 phone_security_kbd.py "123456" # Type PIN 123456
|
|
27
|
-
python3 phone_security_kbd.py "888888" --dry-run # Show positions only
|
|
28
|
-
python3 phone_security_kbd.py "1234" --delay 300 # Slower tapping
|
|
29
|
-
python3 phone_security_kbd.py "0000" -f /tmp/ui.xml # Use existing dump
|
|
30
|
-
"""
|
|
31
|
-
|
|
32
|
-
import argparse
|
|
33
|
-
import os
|
|
34
|
-
import re
|
|
35
|
-
import subprocess
|
|
36
|
-
import sys
|
|
37
|
-
import tempfile
|
|
38
|
-
import time
|
|
39
|
-
import xml.etree.ElementTree as ET
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def run_adb(device, *args):
|
|
43
|
-
cmd = ["adb"]
|
|
44
|
-
if device:
|
|
45
|
-
cmd += ["-s", device]
|
|
46
|
-
cmd += list(args)
|
|
47
|
-
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
48
|
-
return result.stdout, result.stderr, result.returncode
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def dump_ui(device=None):
|
|
52
|
-
remote_path = "/sdcard/ui_kbd.xml"
|
|
53
|
-
local_path = os.path.join(tempfile.gettempdir(), f"phone_kbd_{os.getpid()}.xml")
|
|
54
|
-
|
|
55
|
-
_, stderr, rc = run_adb(device, "shell", "uiautomator", "dump", remote_path)
|
|
56
|
-
if rc != 0:
|
|
57
|
-
print(f"ERROR: uiautomator dump failed: {stderr}", file=sys.stderr)
|
|
58
|
-
sys.exit(1)
|
|
59
|
-
|
|
60
|
-
_, stderr, rc = run_adb(device, "pull", remote_path, local_path)
|
|
61
|
-
if rc != 0:
|
|
62
|
-
print(f"ERROR: Failed to pull UI dump: {stderr}", file=sys.stderr)
|
|
63
|
-
sys.exit(1)
|
|
64
|
-
|
|
65
|
-
run_adb(device, "shell", "rm", "-f", remote_path)
|
|
66
|
-
return local_path
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def parse_keyboard_map(xml_path):
|
|
70
|
-
"""Parse UI dump and return dict of digit/char -> (center_x, center_y)."""
|
|
71
|
-
tree = ET.parse(xml_path)
|
|
72
|
-
root = tree.getroot()
|
|
73
|
-
kbd_map = {}
|
|
74
|
-
|
|
75
|
-
for node in root.iter():
|
|
76
|
-
desc = node.get("content-desc", "").strip()
|
|
77
|
-
text = node.get("text", "").strip()
|
|
78
|
-
bounds = node.get("bounds", "")
|
|
79
|
-
clickable = node.get("clickable", "false")
|
|
80
|
-
|
|
81
|
-
if not bounds or clickable != "true":
|
|
82
|
-
continue
|
|
83
|
-
|
|
84
|
-
# Use content-desc first, fallback to text
|
|
85
|
-
label = desc or text
|
|
86
|
-
if not label:
|
|
87
|
-
continue
|
|
88
|
-
|
|
89
|
-
m = re.findall(r"\d+", bounds)
|
|
90
|
-
if len(m) != 4:
|
|
91
|
-
continue
|
|
92
|
-
|
|
93
|
-
x1, y1, x2, y2 = int(m[0]), int(m[1]), int(m[2]), int(m[3])
|
|
94
|
-
cx = (x1 + x2) // 2
|
|
95
|
-
cy = (y1 + y2) // 2
|
|
96
|
-
|
|
97
|
-
# Map single digits and common labels
|
|
98
|
-
if label in "0123456789":
|
|
99
|
-
kbd_map[label] = (cx, cy)
|
|
100
|
-
elif label.lower() in ("delete", "backspace", "del", "删除"):
|
|
101
|
-
kbd_map["delete"] = (cx, cy)
|
|
102
|
-
elif label.lower() in ("confirm", "ok", "done", "确定", "确认", "完成"):
|
|
103
|
-
kbd_map["confirm"] = (cx, cy)
|
|
104
|
-
elif label == ".":
|
|
105
|
-
kbd_map["."] = (cx, cy)
|
|
106
|
-
|
|
107
|
-
return kbd_map
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def tap_digit(device, x, y):
|
|
111
|
-
run_adb(device, "shell", "input", "tap", str(x), str(y))
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
def main():
|
|
115
|
-
parser = argparse.ArgumentParser(description="Type on security keyboard")
|
|
116
|
-
parser.add_argument("digits", help="Digits to type (e.g., '123456')")
|
|
117
|
-
parser.add_argument("-d", "--device", help="Target device ID")
|
|
118
|
-
parser.add_argument("-f", "--file", help="Existing UI dump XML")
|
|
119
|
-
parser.add_argument("--delay", type=int, default=200, help="Delay between taps (ms)")
|
|
120
|
-
parser.add_argument("--dry-run", action="store_true", help="Show positions only")
|
|
121
|
-
parser.add_argument("-q", "--quiet", action="store_true", help="Quiet mode")
|
|
122
|
-
args = parser.parse_args()
|
|
123
|
-
|
|
124
|
-
info = lambda msg: None if args.quiet else print(f"[security_kbd] {msg}", file=sys.stderr)
|
|
125
|
-
|
|
126
|
-
# Get UI dump
|
|
127
|
-
if args.file:
|
|
128
|
-
xml_path = args.file
|
|
129
|
-
cleanup = False
|
|
130
|
-
else:
|
|
131
|
-
info("Dumping UI hierarchy...")
|
|
132
|
-
xml_path = dump_ui(args.device)
|
|
133
|
-
cleanup = True
|
|
134
|
-
|
|
135
|
-
# Parse keyboard layout
|
|
136
|
-
kbd_map = parse_keyboard_map(xml_path)
|
|
137
|
-
|
|
138
|
-
if cleanup:
|
|
139
|
-
os.unlink(xml_path)
|
|
140
|
-
|
|
141
|
-
if not kbd_map:
|
|
142
|
-
print("ERROR: No keyboard digits found in UI dump", file=sys.stderr)
|
|
143
|
-
print("Make sure a security keyboard is visible on screen", file=sys.stderr)
|
|
144
|
-
sys.exit(1)
|
|
145
|
-
|
|
146
|
-
info(f"Keyboard map: { {k: v for k, v in sorted(kbd_map.items()) if k in '0123456789'} }")
|
|
147
|
-
|
|
148
|
-
# Validate all digits are mappable
|
|
149
|
-
for ch in args.digits:
|
|
150
|
-
if ch not in kbd_map:
|
|
151
|
-
print(f"ERROR: Digit '{ch}' not found on keyboard. Available: {sorted(kbd_map.keys())}", file=sys.stderr)
|
|
152
|
-
sys.exit(1)
|
|
153
|
-
|
|
154
|
-
if args.dry_run:
|
|
155
|
-
print("Dry run - would tap:")
|
|
156
|
-
for ch in args.digits:
|
|
157
|
-
x, y = kbd_map[ch]
|
|
158
|
-
print(f" '{ch}' -> ({x}, {y})")
|
|
159
|
-
return
|
|
160
|
-
|
|
161
|
-
# Type each digit
|
|
162
|
-
for i, ch in enumerate(args.digits):
|
|
163
|
-
x, y = kbd_map[ch]
|
|
164
|
-
info(f"Tapping '{ch}' at ({x}, {y})")
|
|
165
|
-
tap_digit(args.device, x, y)
|
|
166
|
-
if i < len(args.digits) - 1:
|
|
167
|
-
time.sleep(args.delay / 1000.0)
|
|
168
|
-
|
|
169
|
-
info(f"Typed {len(args.digits)} digits successfully")
|
|
170
|
-
print("kbd_input_ok")
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
if __name__ == "__main__":
|
|
174
|
-
main()
|
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# phone_setup.sh - One-time setup for ADB phone automation
|
|
3
|
-
#
|
|
4
|
-
# What it does:
|
|
5
|
-
# 1. Checks ADB connectivity
|
|
6
|
-
# 2. Downloads & installs ADBKeyboard APK (for Chinese text input)
|
|
7
|
-
# 3. Enables ADBKeyboard as the active input method
|
|
8
|
-
# 4. Verifies everything works
|
|
9
|
-
#
|
|
10
|
-
# Usage:
|
|
11
|
-
# bash phone_setup.sh [options]
|
|
12
|
-
#
|
|
13
|
-
# Options:
|
|
14
|
-
# -d DEVICE_ID Target specific device
|
|
15
|
-
# -q Quiet mode
|
|
16
|
-
# --skip-kbd Skip ADBKeyboard installation
|
|
17
|
-
# --uninstall Remove ADBKeyboard and restore default IME
|
|
18
|
-
# --status Just check current setup status
|
|
19
|
-
#
|
|
20
|
-
# Examples:
|
|
21
|
-
# bash phone_setup.sh # Full setup
|
|
22
|
-
# bash phone_setup.sh -d DEVICE_ID # Setup specific device
|
|
23
|
-
# bash phone_setup.sh --status # Check status only
|
|
24
|
-
# bash phone_setup.sh --uninstall # Remove ADBKeyboard
|
|
25
|
-
|
|
26
|
-
set -euo pipefail
|
|
27
|
-
|
|
28
|
-
DEVICE=""
|
|
29
|
-
QUIET=false
|
|
30
|
-
SKIP_KBD=false
|
|
31
|
-
UNINSTALL=false
|
|
32
|
-
STATUS_ONLY=false
|
|
33
|
-
|
|
34
|
-
ADBKBD_PKG="com.android.adbkeyboard"
|
|
35
|
-
ADBKBD_IME="$ADBKBD_PKG/.AdbIME"
|
|
36
|
-
ADBKBD_APK_URL="https://github.com/senzhk/ADBKeyBoard/releases/download/v2.4-dev/keyboardservice-debug.apk"
|
|
37
|
-
ADBKBD_APK="/tmp/ADBKeyboard.apk"
|
|
38
|
-
|
|
39
|
-
# Parse args
|
|
40
|
-
ARGV=("$@")
|
|
41
|
-
i=0
|
|
42
|
-
while [ $i -lt ${#ARGV[@]} ]; do
|
|
43
|
-
case "${ARGV[$i]}" in
|
|
44
|
-
-d) DEVICE="${ARGV[$((i+1))]}"; i=$((i+2)) ;;
|
|
45
|
-
-q) QUIET=true; i=$((i+1)) ;;
|
|
46
|
-
--skip-kbd) SKIP_KBD=true; i=$((i+1)) ;;
|
|
47
|
-
--uninstall) UNINSTALL=true; i=$((i+1)) ;;
|
|
48
|
-
--status) STATUS_ONLY=true; i=$((i+1)) ;;
|
|
49
|
-
-h|--help) head -25 "$0" | tail -20; exit 0 ;;
|
|
50
|
-
*) echo "Unknown option: ${ARGV[$i]}" >&2; exit 1 ;;
|
|
51
|
-
esac
|
|
52
|
-
done
|
|
53
|
-
|
|
54
|
-
ADB_CMD="adb"
|
|
55
|
-
if [ -n "$DEVICE" ]; then
|
|
56
|
-
ADB_CMD="adb -s $DEVICE"
|
|
57
|
-
fi
|
|
58
|
-
|
|
59
|
-
info() { $QUIET || echo "✅ $*"; }
|
|
60
|
-
warn() { echo "⚠️ $*" >&2; }
|
|
61
|
-
error() { echo "❌ $*" >&2; }
|
|
62
|
-
step() { $QUIET || echo ""; $QUIET || echo "▶ $*"; }
|
|
63
|
-
|
|
64
|
-
# ─── Check ADB connectivity ─────────────────────────────────────────
|
|
65
|
-
|
|
66
|
-
check_device() {
|
|
67
|
-
step "Checking ADB device connection..."
|
|
68
|
-
if ! command -v adb &>/dev/null; then
|
|
69
|
-
error "adb not found. Install Android SDK platform-tools."
|
|
70
|
-
echo " brew install android-platform-tools" >&2
|
|
71
|
-
exit 1
|
|
72
|
-
fi
|
|
73
|
-
|
|
74
|
-
local devices
|
|
75
|
-
devices=$($ADB_CMD devices 2>/dev/null | grep -v "^List" | grep -v "^$" | wc -l | tr -d ' ')
|
|
76
|
-
if [ "$devices" -eq 0 ]; then
|
|
77
|
-
error "No devices connected. Connect via USB or WiFi:"
|
|
78
|
-
echo " USB: adb devices" >&2
|
|
79
|
-
echo " WiFi: adb connect <IP>:5555" >&2
|
|
80
|
-
exit 1
|
|
81
|
-
fi
|
|
82
|
-
|
|
83
|
-
local device_model
|
|
84
|
-
device_model=$($ADB_CMD shell getprop ro.product.model 2>/dev/null | tr -d '\r')
|
|
85
|
-
local android_ver
|
|
86
|
-
android_ver=$($ADB_CMD shell getprop ro.build.version.release 2>/dev/null | tr -d '\r')
|
|
87
|
-
local resolution
|
|
88
|
-
resolution=$($ADB_CMD shell wm size 2>/dev/null | grep "Physical" | awk '{print $3}' | tr -d '\r')
|
|
89
|
-
|
|
90
|
-
info "Device: $device_model (Android $android_ver, $resolution)"
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
# ─── Check ADBKeyboard status ───────────────────────────────────────
|
|
94
|
-
|
|
95
|
-
check_adbkbd_status() {
|
|
96
|
-
local installed=false
|
|
97
|
-
local enabled=false
|
|
98
|
-
local active=false
|
|
99
|
-
|
|
100
|
-
# Check if installed
|
|
101
|
-
if $ADB_CMD shell pm list packages 2>/dev/null | grep -q "$ADBKBD_PKG"; then
|
|
102
|
-
installed=true
|
|
103
|
-
fi
|
|
104
|
-
|
|
105
|
-
# Check if enabled as IME
|
|
106
|
-
if $ADB_CMD shell ime list -s 2>/dev/null | grep -q "adbkeyboard"; then
|
|
107
|
-
enabled=true
|
|
108
|
-
fi
|
|
109
|
-
|
|
110
|
-
# Check if currently active
|
|
111
|
-
local current_ime
|
|
112
|
-
current_ime=$($ADB_CMD shell settings get secure default_input_method 2>/dev/null | tr -d '\r')
|
|
113
|
-
if echo "$current_ime" | grep -q "adbkeyboard"; then
|
|
114
|
-
active=true
|
|
115
|
-
fi
|
|
116
|
-
|
|
117
|
-
if $STATUS_ONLY; then
|
|
118
|
-
echo ""
|
|
119
|
-
echo "=== ADB Phone Automation Status ==="
|
|
120
|
-
echo "ADBKeyboard installed: $installed"
|
|
121
|
-
echo "ADBKeyboard enabled: $enabled"
|
|
122
|
-
echo "ADBKeyboard active: $active"
|
|
123
|
-
echo "Current IME: $current_ime"
|
|
124
|
-
echo ""
|
|
125
|
-
# Test input if active
|
|
126
|
-
if $active; then
|
|
127
|
-
echo "Testing ADBKeyboard broadcast..."
|
|
128
|
-
$ADB_CMD shell am broadcast -a ADB_INPUT_TEXT --es msg "test" 2>/dev/null | grep -q "result=0" && \
|
|
129
|
-
echo "✅ ADBKeyboard working!" || echo "⚠️ ADBKeyboard broadcast failed"
|
|
130
|
-
fi
|
|
131
|
-
return
|
|
132
|
-
fi
|
|
133
|
-
|
|
134
|
-
echo "installed:$installed enabled:$enabled active:$active current:$current_ime"
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
# ─── Install ADBKeyboard ────────────────────────────────────────────
|
|
138
|
-
|
|
139
|
-
install_adbkbd() {
|
|
140
|
-
step "Installing ADBKeyboard for Chinese text input support..."
|
|
141
|
-
|
|
142
|
-
# Check if already installed
|
|
143
|
-
if $ADB_CMD shell pm list packages 2>/dev/null | grep -q "$ADBKBD_PKG"; then
|
|
144
|
-
info "ADBKeyboard already installed"
|
|
145
|
-
else
|
|
146
|
-
# Download APK
|
|
147
|
-
info "Downloading ADBKeyboard APK..."
|
|
148
|
-
if ! curl -sL -o "$ADBKBD_APK" "$ADBKBD_APK_URL"; then
|
|
149
|
-
error "Failed to download ADBKeyboard APK from GitHub"
|
|
150
|
-
warn "You can manually download from: $ADBKBD_APK_URL"
|
|
151
|
-
return 1
|
|
152
|
-
fi
|
|
153
|
-
|
|
154
|
-
# Verify download is a valid APK (zip archive), not an error page
|
|
155
|
-
if [ ! -s "$ADBKBD_APK" ]; then
|
|
156
|
-
error "Downloaded APK is empty"
|
|
157
|
-
return 1
|
|
158
|
-
fi
|
|
159
|
-
if ! file "$ADBKBD_APK" | grep -qi "zip"; then
|
|
160
|
-
error "Downloaded file is not a valid APK (got: $(file -b "$ADBKBD_APK"))"
|
|
161
|
-
rm -f "$ADBKBD_APK"
|
|
162
|
-
return 1
|
|
163
|
-
fi
|
|
164
|
-
|
|
165
|
-
# Install
|
|
166
|
-
info "Installing APK on device..."
|
|
167
|
-
if ! $ADB_CMD install "$ADBKBD_APK" 2>/dev/null; then
|
|
168
|
-
error "Failed to install ADBKeyboard"
|
|
169
|
-
warn "Try: adb install $ADBKBD_APK"
|
|
170
|
-
return 1
|
|
171
|
-
fi
|
|
172
|
-
info "ADBKeyboard installed successfully"
|
|
173
|
-
|
|
174
|
-
# Clean up
|
|
175
|
-
rm -f "$ADBKBD_APK"
|
|
176
|
-
fi
|
|
177
|
-
|
|
178
|
-
# Enable ADBKeyboard as an input method
|
|
179
|
-
step "Enabling ADBKeyboard as input method..."
|
|
180
|
-
$ADB_CMD shell ime enable "$ADBKBD_IME" 2>/dev/null || true
|
|
181
|
-
# Some devices (e.g. Xiaomi/MIUI) don't add to enabled list via 'ime enable'
|
|
182
|
-
# Manually append to enabled_input_methods if needed
|
|
183
|
-
local enabled_imes
|
|
184
|
-
enabled_imes=$($ADB_CMD shell settings get secure enabled_input_methods 2>/dev/null | tr -d '\r')
|
|
185
|
-
if ! echo "$enabled_imes" | grep -q "adbkeyboard"; then
|
|
186
|
-
$ADB_CMD shell settings put secure enabled_input_methods "$enabled_imes:$ADBKBD_IME" 2>/dev/null || true
|
|
187
|
-
fi
|
|
188
|
-
info "ADBKeyboard enabled"
|
|
189
|
-
|
|
190
|
-
# Set as active input method
|
|
191
|
-
$ADB_CMD shell ime set "$ADBKBD_IME" 2>/dev/null || true
|
|
192
|
-
info "ADBKeyboard set as active IME"
|
|
193
|
-
|
|
194
|
-
# Verify
|
|
195
|
-
step "Verifying ADBKeyboard..."
|
|
196
|
-
local current_ime
|
|
197
|
-
current_ime=$($ADB_CMD shell settings get secure default_input_method 2>/dev/null | tr -d '\r')
|
|
198
|
-
if echo "$current_ime" | grep -q "adbkeyboard"; then
|
|
199
|
-
info "ADBKeyboard is ready!"
|
|
200
|
-
# Quick test
|
|
201
|
-
$ADB_CMD shell am broadcast -a ADB_INPUT_TEXT --es msg "setup_ok" 2>/dev/null | grep -q "result=0" && \
|
|
202
|
-
info "Broadcast test passed" || warn "Broadcast test inconclusive (may still work)"
|
|
203
|
-
else
|
|
204
|
-
warn "ADBKeyboard might need manual enabling in Settings > Language & Input"
|
|
205
|
-
fi
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
# ─── Uninstall ADBKeyboard ──────────────────────────────────────────
|
|
209
|
-
|
|
210
|
-
uninstall_adbkbd() {
|
|
211
|
-
step "Uninstalling ADBKeyboard..."
|
|
212
|
-
|
|
213
|
-
# Restore default IME first
|
|
214
|
-
local default_ime
|
|
215
|
-
default_ime=$($ADB_CMD shell settings get secure default_input_method 2>/dev/null | tr -d '\r')
|
|
216
|
-
if echo "$default_ime" | grep -q "adbkeyboard"; then
|
|
217
|
-
# Try to switch to a standard IME
|
|
218
|
-
local other_ime
|
|
219
|
-
other_ime=$($ADB_CMD shell ime list -s 2>/dev/null | grep -v "adbkeyboard" | head -1 | tr -d '\r')
|
|
220
|
-
if [ -n "$other_ime" ]; then
|
|
221
|
-
$ADB_CMD shell ime set "$other_ime" 2>/dev/null
|
|
222
|
-
info "Switched IME to: $other_ime"
|
|
223
|
-
fi
|
|
224
|
-
fi
|
|
225
|
-
|
|
226
|
-
# Uninstall
|
|
227
|
-
if $ADB_CMD shell pm list packages 2>/dev/null | grep -q "$ADBKBD_PKG"; then
|
|
228
|
-
$ADB_CMD uninstall "$ADBKBD_PKG" 2>/dev/null && \
|
|
229
|
-
info "ADBKeyboard uninstalled" || error "Failed to uninstall"
|
|
230
|
-
else
|
|
231
|
-
info "ADBKeyboard was not installed"
|
|
232
|
-
fi
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
# ─── Main ────────────────────────────────────────────────────────────
|
|
236
|
-
|
|
237
|
-
check_device
|
|
238
|
-
|
|
239
|
-
if $STATUS_ONLY; then
|
|
240
|
-
check_adbkbd_status
|
|
241
|
-
exit 0
|
|
242
|
-
fi
|
|
243
|
-
|
|
244
|
-
if $UNINSTALL; then
|
|
245
|
-
uninstall_adbkbd
|
|
246
|
-
exit 0
|
|
247
|
-
fi
|
|
248
|
-
|
|
249
|
-
if ! $SKIP_KBD; then
|
|
250
|
-
install_adbkbd
|
|
251
|
-
fi
|
|
252
|
-
|
|
253
|
-
if ! $QUIET; then
|
|
254
|
-
echo ""
|
|
255
|
-
echo "══════════════════════════════════════════"
|
|
256
|
-
echo " Setup complete! Quick reference:"
|
|
257
|
-
echo "══════════════════════════════════════════"
|
|
258
|
-
echo ""
|
|
259
|
-
echo " # Take screenshot"
|
|
260
|
-
echo " bash phone_screenshot.sh"
|
|
261
|
-
echo ""
|
|
262
|
-
echo " # Tap at coordinates"
|
|
263
|
-
echo " bash phone_tap.sh 500 800"
|
|
264
|
-
echo ""
|
|
265
|
-
echo " # Type Chinese text"
|
|
266
|
-
echo " bash phone_input.sh \"你好世界\" -m adbkbd"
|
|
267
|
-
echo ""
|
|
268
|
-
echo " # Type English + press Enter"
|
|
269
|
-
echo " bash phone_input.sh \"hello\" -e"
|
|
270
|
-
echo ""
|
|
271
|
-
echo " # Check status anytime"
|
|
272
|
-
echo " bash phone_setup.sh --status"
|
|
273
|
-
echo ""
|
|
274
|
-
fi
|