visionclaw 0.1.187-beta.8 → 0.1.187-dev.refactor-computer-use-direct-coordinates.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/loop.js +1 -1
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/providers/client-factory.d.ts +1 -1
- package/dist/agent/providers/client-factory.js +1 -1
- package/dist/agent/runtime-surface.d.ts +1 -1
- package/dist/agent/runtime-surface.d.ts.map +1 -1
- package/dist/agent/runtime-surface.js +35 -18
- package/dist/agent/runtime-surface.js.map +1 -1
- package/dist/agent/system-prompt.d.ts.map +1 -1
- package/dist/agent/system-prompt.js +1 -3
- package/dist/agent/system-prompt.js.map +1 -1
- package/dist/builtin-skills/macos-automation/SKILL.md +13 -10
- package/dist/onboarding/generate-wallpaper.d.ts +3 -8
- package/dist/onboarding/generate-wallpaper.d.ts.map +1 -1
- package/dist/onboarding/generate-wallpaper.js +3 -123
- package/dist/onboarding/generate-wallpaper.js.map +1 -1
- package/dist/tools/computer-use.d.ts +56 -6
- package/dist/tools/computer-use.d.ts.map +1 -1
- package/dist/tools/computer-use.js +129 -286
- package/dist/tools/computer-use.js.map +1 -1
- package/dist-agent/bundle.cjs +208 -574
- package/package.json +1 -1
- package/dist/agent/applied-credential-signature.d.ts +0 -53
- package/dist/agent/applied-credential-signature.d.ts.map +0 -1
- package/dist/agent/applied-credential-signature.js +0 -137
- package/dist/agent/applied-credential-signature.js.map +0 -1
- package/dist/agent/tunnel-credential-handler.d.ts +0 -90
- package/dist/agent/tunnel-credential-handler.d.ts.map +0 -1
- package/dist/agent/tunnel-credential-handler.js +0 -162
- package/dist/agent/tunnel-credential-handler.js.map +0 -1
- package/dist/billing/payg-handler.d.ts +0 -29
- package/dist/billing/payg-handler.d.ts.map +0 -1
- package/dist/billing/payg-handler.js +0 -92
- package/dist/billing/payg-handler.js.map +0 -1
- package/dist/billing/payment-handler.d.ts +0 -24
- package/dist/billing/payment-handler.d.ts.map +0 -1
- package/dist/billing/payment-handler.js +0 -101
- package/dist/billing/payment-handler.js.map +0 -1
- package/dist/builtin-skills/catalog/phone-adb-automation/SKILL.md +0 -412
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_input.sh +0 -132
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_launch.sh +0 -166
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_screenshot.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_security_kbd.py +0 -174
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_setup.sh +0 -274
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_swipe.sh +0 -111
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_tap.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_ui_parse.py +0 -176
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_wake_unlock.sh +0 -67
- package/dist/builtin-skills/transcribe-audio/SKILL.md +0 -122
- package/dist/data-processing/convert-demo-cli.d.ts +0 -7
- package/dist/data-processing/convert-demo-cli.d.ts.map +0 -1
- package/dist/data-processing/convert-demo-cli.js +0 -30
- package/dist/data-processing/convert-demo-cli.js.map +0 -1
- package/dist/data-processing/convert-demo.d.ts +0 -26
- package/dist/data-processing/convert-demo.d.ts.map +0 -1
- package/dist/data-processing/convert-demo.js +0 -233
- package/dist/data-processing/convert-demo.js.map +0 -1
- package/dist/obs/rdp/icons/icons/app_windows.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_get.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_send.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/control.svg +0 -4
- package/dist/obs/rdp/icons/icons/desktop.svg +0 -4
- package/dist/obs/rdp/icons/icons/display.svg +0 -4
- package/dist/obs/rdp/icons/icons/launchpad.svg +0 -4
- package/dist/obs/rdp/icons/icons/mission_control.svg +0 -4
- package/dist/obs/rdp/icons/icons/screenshot.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_actual.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_fit.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_in.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_out.svg +0 -4
- package/dist/obs/tunnel-telemetry.d.ts +0 -46
- package/dist/obs/tunnel-telemetry.d.ts.map +0 -1
- package/dist/obs/tunnel-telemetry.js +0 -70
- package/dist/obs/tunnel-telemetry.js.map +0 -1
- package/dist/service/gbox-tun.d.ts +0 -14
- package/dist/service/gbox-tun.d.ts.map +0 -1
- package/dist/service/gbox-tun.js +0 -315
- package/dist/service/gbox-tun.js.map +0 -1
- package/dist/tools/coordinate-resolver.d.ts +0 -30
- package/dist/tools/coordinate-resolver.d.ts.map +0 -1
- package/dist/tools/coordinate-resolver.js +0 -104
- package/dist/tools/coordinate-resolver.js.map +0 -1
- package/dist/utils/wechat-monitor.d.ts +0 -21
- package/dist/utils/wechat-monitor.d.ts.map +0 -1
- package/dist/utils/wechat-monitor.js +0 -88
- package/dist/utils/wechat-monitor.js.map +0 -1
package/dist-agent/bundle.cjs
CHANGED
|
@@ -14607,63 +14607,6 @@ var init_types = __esm({
|
|
|
14607
14607
|
});
|
|
14608
14608
|
|
|
14609
14609
|
// dist/config/index.js
|
|
14610
|
-
var config_exports = {};
|
|
14611
|
-
__export(config_exports, {
|
|
14612
|
-
addCommandIdToPendingUpgrade: () => addCommandIdToPendingUpgrade,
|
|
14613
|
-
buildPricingUrl: () => buildPricingUrl,
|
|
14614
|
-
configExists: () => configExists,
|
|
14615
|
-
consumePendingRestart: () => consumePendingRestart,
|
|
14616
|
-
consumePendingUpgrade: () => consumePendingUpgrade,
|
|
14617
|
-
deleteOnboardingSession: () => deleteOnboardingSession,
|
|
14618
|
-
ensureConfigDir: () => ensureConfigDir,
|
|
14619
|
-
getBaseDir: () => getBaseDir,
|
|
14620
|
-
getClaudeAutoMemoryDir: () => getClaudeAutoMemoryDir,
|
|
14621
|
-
getClaudeLocalSettingsFile: () => getClaudeLocalSettingsFile,
|
|
14622
|
-
getConfigDir: () => getConfigDir,
|
|
14623
|
-
getConfigFile: () => getConfigFile,
|
|
14624
|
-
getDynamicMcpServersFile: () => getDynamicMcpServersFile,
|
|
14625
|
-
getFailedUploadsDir: () => getFailedUploadsDir,
|
|
14626
|
-
getMemoriesDir: () => getMemoriesDir,
|
|
14627
|
-
getOnboardingSessionFile: () => getOnboardingSessionFile,
|
|
14628
|
-
getOwnerFile: () => getOwnerFile,
|
|
14629
|
-
getPlaywrightProfileDir: () => getPlaywrightProfileDir,
|
|
14630
|
-
getProfile: () => getProfile,
|
|
14631
|
-
getProfilePhotosDir: () => getProfilePhotosDir,
|
|
14632
|
-
getProfileTunnelConfigPath: () => getProfileTunnelConfigPath,
|
|
14633
|
-
getProfileTunnelCredentialsPath: () => getProfileTunnelCredentialsPath,
|
|
14634
|
-
getProfileTunnelDir: () => getProfileTunnelDir,
|
|
14635
|
-
getProfilesDir: () => getProfilesDir,
|
|
14636
|
-
getScreenshotsDir: () => getScreenshotsDir,
|
|
14637
|
-
getSessionFile: () => getSessionFile,
|
|
14638
|
-
getSkillsDir: () => getSkillsDir,
|
|
14639
|
-
getTOTPSecretsFile: () => getTOTPSecretsFile,
|
|
14640
|
-
getTelegramAccessFile: () => getTelegramAccessFile,
|
|
14641
|
-
getTokensFile: () => getTokensFile,
|
|
14642
|
-
getUserCatalogDir: () => getUserCatalogDir,
|
|
14643
|
-
isOnboardingComplete: () => isOnboardingComplete,
|
|
14644
|
-
loadConfig: () => loadConfig,
|
|
14645
|
-
loadDynamicMcpServers: () => loadDynamicMcpServers,
|
|
14646
|
-
loadGoogleTokens: () => loadGoogleTokens,
|
|
14647
|
-
loadOnboardingSessionId: () => loadOnboardingSessionId,
|
|
14648
|
-
loadOwnerConfig: () => loadOwnerConfig,
|
|
14649
|
-
loadSessionId: () => loadSessionId,
|
|
14650
|
-
loadTOTPSecrets: () => loadTOTPSecrets,
|
|
14651
|
-
loadTelegramAccessConfig: () => loadTelegramAccessConfig,
|
|
14652
|
-
loadUsageSnapshot: () => loadUsageSnapshot,
|
|
14653
|
-
ownerConfigExists: () => ownerConfigExists,
|
|
14654
|
-
saveConfig: () => saveConfig,
|
|
14655
|
-
saveDynamicMcpServers: () => saveDynamicMcpServers,
|
|
14656
|
-
saveGoogleTokens: () => saveGoogleTokens,
|
|
14657
|
-
saveOnboardingSessionId: () => saveOnboardingSessionId,
|
|
14658
|
-
saveOwnerConfig: () => saveOwnerConfig,
|
|
14659
|
-
saveSessionId: () => saveSessionId,
|
|
14660
|
-
saveTOTPSecrets: () => saveTOTPSecrets,
|
|
14661
|
-
saveTelegramAccessConfig: () => saveTelegramAccessConfig,
|
|
14662
|
-
saveUsageSnapshot: () => saveUsageSnapshot,
|
|
14663
|
-
setProfile: () => setProfile,
|
|
14664
|
-
writePendingRestart: () => writePendingRestart,
|
|
14665
|
-
writePendingUpgrade: () => writePendingUpgrade
|
|
14666
|
-
});
|
|
14667
14610
|
function setProfile(name) {
|
|
14668
14611
|
if (!/^[a-z0-9][a-z0-9-]*$/.test(name)) {
|
|
14669
14612
|
throw new Error(`Invalid profile name "${name}". Use lowercase letters, numbers, and hyphens.`);
|
|
@@ -14676,9 +14619,6 @@ function getProfile() {
|
|
|
14676
14619
|
function getBaseDir() {
|
|
14677
14620
|
return BASE_DIR;
|
|
14678
14621
|
}
|
|
14679
|
-
function getProfilesDir() {
|
|
14680
|
-
return PROFILES_DIR;
|
|
14681
|
-
}
|
|
14682
14622
|
function getConfigDir() {
|
|
14683
14623
|
return import_node_path.default.join(PROFILES_DIR, currentProfile);
|
|
14684
14624
|
}
|
|
@@ -15007,22 +14947,6 @@ function saveSessionId(sessionId, mode) {
|
|
|
15007
14947
|
existing.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
15008
14948
|
import_node_fs.default.writeFileSync(sessionFile, JSON.stringify(existing, null, 2), "utf-8");
|
|
15009
14949
|
}
|
|
15010
|
-
function loadOnboardingSessionId() {
|
|
15011
|
-
const file2 = getOnboardingSessionFile();
|
|
15012
|
-
if (!import_node_fs.default.existsSync(file2))
|
|
15013
|
-
return null;
|
|
15014
|
-
try {
|
|
15015
|
-
const raw = JSON.parse(import_node_fs.default.readFileSync(file2, "utf-8"));
|
|
15016
|
-
const parsed = raw;
|
|
15017
|
-
return parsed.sessionId ?? null;
|
|
15018
|
-
} catch {
|
|
15019
|
-
return null;
|
|
15020
|
-
}
|
|
15021
|
-
}
|
|
15022
|
-
function saveOnboardingSessionId(sessionId) {
|
|
15023
|
-
ensureConfigDir();
|
|
15024
|
-
import_node_fs.default.writeFileSync(getOnboardingSessionFile(), JSON.stringify({ sessionId, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), "utf-8");
|
|
15025
|
-
}
|
|
15026
14950
|
function deleteOnboardingSession() {
|
|
15027
14951
|
const file2 = getOnboardingSessionFile();
|
|
15028
14952
|
if (import_node_fs.default.existsSync(file2)) {
|
|
@@ -20445,7 +20369,7 @@ var require_prompts3 = __commonJS({
|
|
|
20445
20369
|
|
|
20446
20370
|
// dist/utils/version-check.js
|
|
20447
20371
|
function isBundled() {
|
|
20448
|
-
const v16 = "0.1.187-
|
|
20372
|
+
const v16 = "0.1.187-dev.refactor-computer-use-direct-coordinates.1";
|
|
20449
20373
|
return typeof v16 === "string" && v16 !== "undefined";
|
|
20450
20374
|
}
|
|
20451
20375
|
function getPackageRoot() {
|
|
@@ -20462,7 +20386,7 @@ function getInstallationInfo() {
|
|
|
20462
20386
|
};
|
|
20463
20387
|
}
|
|
20464
20388
|
function getCurrentVersion() {
|
|
20465
|
-
const bundledVersion = "0.1.187-
|
|
20389
|
+
const bundledVersion = "0.1.187-dev.refactor-computer-use-direct-coordinates.1";
|
|
20466
20390
|
if (bundledVersion && bundledVersion !== "undefined") {
|
|
20467
20391
|
return bundledVersion;
|
|
20468
20392
|
}
|
|
@@ -24995,7 +24919,7 @@ var require_retry = __commonJS({
|
|
|
24995
24919
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
24996
24920
|
exports2.getRetryConfig = getRetryConfig;
|
|
24997
24921
|
async function getRetryConfig(err7) {
|
|
24998
|
-
let config2 =
|
|
24922
|
+
let config2 = getConfig3(err7);
|
|
24999
24923
|
if (!err7 || !err7.config || !config2 && !err7.config.retry) {
|
|
25000
24924
|
return { shouldRetry: false };
|
|
25001
24925
|
}
|
|
@@ -25047,7 +24971,7 @@ var require_retry = __commonJS({
|
|
|
25047
24971
|
}
|
|
25048
24972
|
function shouldRetryRequest(err7) {
|
|
25049
24973
|
var _a7;
|
|
25050
|
-
const config2 =
|
|
24974
|
+
const config2 = getConfig3(err7);
|
|
25051
24975
|
if (err7.name === "AbortError" || ((_a7 = err7.error) === null || _a7 === void 0 ? void 0 : _a7.name) === "AbortError") {
|
|
25052
24976
|
return false;
|
|
25053
24977
|
}
|
|
@@ -25079,7 +25003,7 @@ var require_retry = __commonJS({
|
|
|
25079
25003
|
}
|
|
25080
25004
|
return true;
|
|
25081
25005
|
}
|
|
25082
|
-
function
|
|
25006
|
+
function getConfig3(err7) {
|
|
25083
25007
|
if (err7 && err7.config && err7.config.retryConfig) {
|
|
25084
25008
|
return err7.config.retryConfig;
|
|
25085
25009
|
}
|
|
@@ -26328,7 +26252,7 @@ var require_dist2 = __commonJS({
|
|
|
26328
26252
|
};
|
|
26329
26253
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
26330
26254
|
exports2.Agent = void 0;
|
|
26331
|
-
var
|
|
26255
|
+
var net2 = __importStar3(require("net"));
|
|
26332
26256
|
var http7 = __importStar3(require("http"));
|
|
26333
26257
|
var https_1 = require("https");
|
|
26334
26258
|
__exportStar3(require_helpers(), exports2);
|
|
@@ -26368,7 +26292,7 @@ var require_dist2 = __commonJS({
|
|
|
26368
26292
|
if (!this.sockets[name]) {
|
|
26369
26293
|
this.sockets[name] = [];
|
|
26370
26294
|
}
|
|
26371
|
-
const fakeSocket = new
|
|
26295
|
+
const fakeSocket = new net2.Socket({ writable: false });
|
|
26372
26296
|
this.sockets[name].push(fakeSocket);
|
|
26373
26297
|
this.totalSocketCount++;
|
|
26374
26298
|
return fakeSocket;
|
|
@@ -26580,7 +26504,7 @@ var require_dist3 = __commonJS({
|
|
|
26580
26504
|
};
|
|
26581
26505
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
26582
26506
|
exports2.HttpsProxyAgent = void 0;
|
|
26583
|
-
var
|
|
26507
|
+
var net2 = __importStar3(require("net"));
|
|
26584
26508
|
var tls = __importStar3(require("tls"));
|
|
26585
26509
|
var assert_1 = __importDefault3(require("assert"));
|
|
26586
26510
|
var debug_1 = __importDefault3(require_src2());
|
|
@@ -26589,7 +26513,7 @@ var require_dist3 = __commonJS({
|
|
|
26589
26513
|
var parse_proxy_response_1 = require_parse_proxy_response();
|
|
26590
26514
|
var debug3 = (0, debug_1.default)("https-proxy-agent");
|
|
26591
26515
|
var setServernameFromNonIpHost = (options3) => {
|
|
26592
|
-
if (options3.servername === void 0 && options3.host && !
|
|
26516
|
+
if (options3.servername === void 0 && options3.host && !net2.isIP(options3.host)) {
|
|
26593
26517
|
return {
|
|
26594
26518
|
...options3,
|
|
26595
26519
|
servername: options3.host
|
|
@@ -26629,10 +26553,10 @@ var require_dist3 = __commonJS({
|
|
|
26629
26553
|
socket = tls.connect(setServernameFromNonIpHost(this.connectOpts));
|
|
26630
26554
|
} else {
|
|
26631
26555
|
debug3("Creating `net.Socket`: %o", this.connectOpts);
|
|
26632
|
-
socket =
|
|
26556
|
+
socket = net2.connect(this.connectOpts);
|
|
26633
26557
|
}
|
|
26634
26558
|
const headers = typeof this.proxyHeaders === "function" ? this.proxyHeaders() : { ...this.proxyHeaders };
|
|
26635
|
-
const host =
|
|
26559
|
+
const host = net2.isIPv6(opts.host) ? `[${opts.host}]` : opts.host;
|
|
26636
26560
|
let payload2 = `CONNECT ${host}:${opts.port} HTTP/1.1\r
|
|
26637
26561
|
`;
|
|
26638
26562
|
if (proxy.username || proxy.password) {
|
|
@@ -26665,7 +26589,7 @@ var require_dist3 = __commonJS({
|
|
|
26665
26589
|
return socket;
|
|
26666
26590
|
}
|
|
26667
26591
|
socket.destroy();
|
|
26668
|
-
const fakeSocket = new
|
|
26592
|
+
const fakeSocket = new net2.Socket({ writable: false });
|
|
26669
26593
|
fakeSocket.readable = true;
|
|
26670
26594
|
req.once("socket", (s11) => {
|
|
26671
26595
|
debug3("Replaying proxy buffer for failed request");
|
|
@@ -686554,108 +686478,14 @@ function generateAndSetWallpaper(agentName) {
|
|
|
686554
686478
|
cleanupOldWallpapers(dest);
|
|
686555
686479
|
return dest;
|
|
686556
686480
|
}
|
|
686557
|
-
function wallpaperHelperSwiftSource() {
|
|
686558
|
-
return `
|
|
686559
|
-
import AppKit
|
|
686560
|
-
|
|
686561
|
-
let args = CommandLine.arguments
|
|
686562
|
-
guard args.count >= 2 else {
|
|
686563
|
-
fputs("Usage: wallpaper-helper set <path> | get\\n", stderr)
|
|
686564
|
-
exit(1)
|
|
686565
|
-
}
|
|
686566
|
-
|
|
686567
|
-
let command = args[1]
|
|
686568
|
-
|
|
686569
|
-
switch command {
|
|
686570
|
-
case "set":
|
|
686571
|
-
guard args.count >= 3 else {
|
|
686572
|
-
fputs("Usage: wallpaper-helper set <imagePath>\\n", stderr)
|
|
686573
|
-
exit(1)
|
|
686574
|
-
}
|
|
686575
|
-
let imagePath = args[2]
|
|
686576
|
-
let url = URL(fileURLWithPath: imagePath)
|
|
686577
|
-
do {
|
|
686578
|
-
for screen in NSScreen.screens {
|
|
686579
|
-
// Preserve the screen's existing display options (scaling, fill colour,
|
|
686580
|
-
// clipping). Apple's docs note these come from System Settings; if we
|
|
686581
|
-
// pass [:] we silently reset them to defaults, which can change the
|
|
686582
|
-
// visual layout of the wallpaper on some setups.
|
|
686583
|
-
let existingOptions = NSWorkspace.shared.desktopImageOptions(for: screen) ?? [:]
|
|
686584
|
-
try NSWorkspace.shared.setDesktopImageURL(url, for: screen, options: existingOptions)
|
|
686585
|
-
}
|
|
686586
|
-
} catch {
|
|
686587
|
-
fputs("Failed to set wallpaper: \\(error)\\n", stderr)
|
|
686588
|
-
exit(1)
|
|
686589
|
-
}
|
|
686590
|
-
|
|
686591
|
-
case "get":
|
|
686592
|
-
if let screen = NSScreen.main,
|
|
686593
|
-
let url = NSWorkspace.shared.desktopImageURL(for: screen) {
|
|
686594
|
-
print(url.path)
|
|
686595
|
-
}
|
|
686596
|
-
|
|
686597
|
-
default:
|
|
686598
|
-
fputs("Unknown command: \\(command). Use 'set' or 'get'.\\n", stderr)
|
|
686599
|
-
exit(1)
|
|
686600
|
-
}
|
|
686601
|
-
`;
|
|
686602
|
-
}
|
|
686603
|
-
function getHelperHash() {
|
|
686604
|
-
_helperHash ??= (0, import_node_crypto3.createHash)("sha256").update(wallpaperHelperSwiftSource()).digest("hex").slice(0, 12);
|
|
686605
|
-
return _helperHash;
|
|
686606
|
-
}
|
|
686607
|
-
function getWallpaperHelperBin() {
|
|
686608
|
-
const hash3 = getHelperHash();
|
|
686609
|
-
const binFile = import_node_path11.default.join(import_node_os7.default.tmpdir(), `visionclaw-wallpaper-helper-${hash3}`);
|
|
686610
|
-
if (!import_node_fs14.default.existsSync(binFile)) {
|
|
686611
|
-
const srcFile = import_node_path11.default.join(import_node_os7.default.tmpdir(), `visionclaw-wallpaper-helper-${hash3}.swift`);
|
|
686612
|
-
import_node_fs14.default.writeFileSync(srcFile, wallpaperHelperSwiftSource(), "utf-8");
|
|
686613
|
-
const compile = (0, import_node_child_process9.spawnSync)("swiftc", ["-O", "-o", binFile, srcFile], {
|
|
686614
|
-
encoding: "utf-8",
|
|
686615
|
-
timeout: 6e4,
|
|
686616
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
686617
|
-
});
|
|
686618
|
-
try {
|
|
686619
|
-
import_node_fs14.default.unlinkSync(srcFile);
|
|
686620
|
-
} catch {
|
|
686621
|
-
}
|
|
686622
|
-
if (compile.status !== 0) {
|
|
686623
|
-
throw new Error(`Wallpaper helper compilation failed: ${compile.stderr}`);
|
|
686624
|
-
}
|
|
686625
|
-
}
|
|
686626
|
-
return binFile;
|
|
686627
|
-
}
|
|
686628
686481
|
function setDesktopWallpaper(imagePath) {
|
|
686629
686482
|
if (process.platform !== "darwin")
|
|
686630
686483
|
return;
|
|
686631
|
-
try {
|
|
686632
|
-
const bin = getWallpaperHelperBin();
|
|
686633
|
-
const result = (0, import_node_child_process9.spawnSync)(bin, ["set", imagePath], {
|
|
686634
|
-
encoding: "utf-8",
|
|
686635
|
-
timeout: 15e3,
|
|
686636
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
686637
|
-
});
|
|
686638
|
-
if (result.status === 0)
|
|
686639
|
-
return;
|
|
686640
|
-
} catch {
|
|
686641
|
-
}
|
|
686642
686484
|
(0, import_node_child_process9.execSync)(`osascript -e 'tell application "Finder" to set desktop picture to POSIX file "${imagePath}"'`, { stdio: "ignore", timeout: 15e3 });
|
|
686643
686485
|
}
|
|
686644
686486
|
function getCurrentDesktopWallpaper() {
|
|
686645
686487
|
if (process.platform !== "darwin")
|
|
686646
686488
|
return "";
|
|
686647
|
-
try {
|
|
686648
|
-
const bin = getWallpaperHelperBin();
|
|
686649
|
-
const result = (0, import_node_child_process9.spawnSync)(bin, ["get"], {
|
|
686650
|
-
encoding: "utf-8",
|
|
686651
|
-
timeout: 15e3,
|
|
686652
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
686653
|
-
});
|
|
686654
|
-
if (result.status === 0 && result.stdout.trim()) {
|
|
686655
|
-
return result.stdout.trim();
|
|
686656
|
-
}
|
|
686657
|
-
} catch {
|
|
686658
|
-
}
|
|
686659
686489
|
try {
|
|
686660
686490
|
return (0, import_node_child_process9.execSync)(`osascript -e 'tell application "Finder" to get POSIX path of (desktop picture as alias)'`, { encoding: "utf-8", timeout: 15e3, stdio: ["pipe", "pipe", "pipe"] }).trim();
|
|
686661
686491
|
} catch {
|
|
@@ -686679,7 +686509,7 @@ function ensureWallpaper(agentName) {
|
|
|
686679
686509
|
}
|
|
686680
686510
|
return false;
|
|
686681
686511
|
}
|
|
686682
|
-
var import_node_child_process9, import_node_crypto3, import_node_fs14, import_node_os7, import_node_path11, WALLPAPER_META_FILENAME, BG_SETUP, BG_RUNNING, _rendererHash, MACOS_CODENAMES
|
|
686512
|
+
var import_node_child_process9, import_node_crypto3, import_node_fs14, import_node_os7, import_node_path11, WALLPAPER_META_FILENAME, BG_SETUP, BG_RUNNING, _rendererHash, MACOS_CODENAMES;
|
|
686683
686513
|
var init_generate_wallpaper = __esm({
|
|
686684
686514
|
"dist/onboarding/generate-wallpaper.js"() {
|
|
686685
686515
|
"use strict";
|
|
@@ -776202,7 +776032,7 @@ var require_websocket = __commonJS({
|
|
|
776202
776032
|
var EventEmitter7 = require("events");
|
|
776203
776033
|
var https4 = require("https");
|
|
776204
776034
|
var http7 = require("http");
|
|
776205
|
-
var
|
|
776035
|
+
var net2 = require("net");
|
|
776206
776036
|
var tls = require("tls");
|
|
776207
776037
|
var { randomBytes: randomBytes5, createHash: createHash6 } = require("crypto");
|
|
776208
776038
|
var { Duplex, Readable: Readable3 } = require("stream");
|
|
@@ -776936,12 +776766,12 @@ var require_websocket = __commonJS({
|
|
|
776936
776766
|
}
|
|
776937
776767
|
function netConnect(options3) {
|
|
776938
776768
|
options3.path = options3.socketPath;
|
|
776939
|
-
return
|
|
776769
|
+
return net2.connect(options3);
|
|
776940
776770
|
}
|
|
776941
776771
|
function tlsConnect(options3) {
|
|
776942
776772
|
options3.path = void 0;
|
|
776943
776773
|
if (!options3.servername && options3.servername !== "") {
|
|
776944
|
-
options3.servername =
|
|
776774
|
+
options3.servername = net2.isIP(options3.host) ? "" : options3.host;
|
|
776945
776775
|
}
|
|
776946
776776
|
return tls.connect(options3);
|
|
776947
776777
|
}
|
|
@@ -810608,7 +810438,7 @@ var require_retry4 = __commonJS({
|
|
|
810608
810438
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
810609
810439
|
exports2.getRetryConfig = getRetryConfig;
|
|
810610
810440
|
async function getRetryConfig(err7) {
|
|
810611
|
-
let config2 =
|
|
810441
|
+
let config2 = getConfig3(err7);
|
|
810612
810442
|
if (!err7 || !err7.config || !config2 && !err7.config.retry) {
|
|
810613
810443
|
return { shouldRetry: false };
|
|
810614
810444
|
}
|
|
@@ -810659,7 +810489,7 @@ var require_retry4 = __commonJS({
|
|
|
810659
810489
|
return { shouldRetry: true, config: err7.config };
|
|
810660
810490
|
}
|
|
810661
810491
|
function shouldRetryRequest(err7) {
|
|
810662
|
-
const config2 =
|
|
810492
|
+
const config2 = getConfig3(err7);
|
|
810663
810493
|
if (err7.config.signal?.aborted && err7.code !== "TimeoutError" || err7.code === "AbortError") {
|
|
810664
810494
|
return false;
|
|
810665
810495
|
}
|
|
@@ -810691,7 +810521,7 @@ var require_retry4 = __commonJS({
|
|
|
810691
810521
|
}
|
|
810692
810522
|
return true;
|
|
810693
810523
|
}
|
|
810694
|
-
function
|
|
810524
|
+
function getConfig3(err7) {
|
|
810695
810525
|
if (err7 && err7.config && err7.config.retryConfig) {
|
|
810696
810526
|
return err7.config.retryConfig;
|
|
810697
810527
|
}
|
|
@@ -886173,7 +886003,7 @@ var require_forever_agent = __commonJS({
|
|
|
886173
886003
|
ForeverAgent.SSL = ForeverAgentSSL;
|
|
886174
886004
|
var util2 = require("util");
|
|
886175
886005
|
var Agent2 = require("http").Agent;
|
|
886176
|
-
var
|
|
886006
|
+
var net2 = require("net");
|
|
886177
886007
|
var tls = require("tls");
|
|
886178
886008
|
var AgentSSL = require("https").Agent;
|
|
886179
886009
|
function getConnectionName(host, port) {
|
|
@@ -886212,7 +886042,7 @@ var require_forever_agent = __commonJS({
|
|
|
886212
886042
|
}
|
|
886213
886043
|
util2.inherits(ForeverAgent, Agent2);
|
|
886214
886044
|
ForeverAgent.defaultMinSockets = 5;
|
|
886215
|
-
ForeverAgent.prototype.createConnection =
|
|
886045
|
+
ForeverAgent.prototype.createConnection = net2.createConnection;
|
|
886216
886046
|
ForeverAgent.prototype.addRequestNoreuse = Agent2.prototype.addRequest;
|
|
886217
886047
|
ForeverAgent.prototype.addRequest = function(req, host, port) {
|
|
886218
886048
|
var name = getConnectionName(host, port);
|
|
@@ -889086,7 +888916,7 @@ var require_redirect = __commonJS({
|
|
|
889086
888916
|
var require_tunnel_agent = __commonJS({
|
|
889087
888917
|
"node_modules/.pnpm/tunnel-agent@0.6.0/node_modules/tunnel-agent/index.js"(exports2) {
|
|
889088
888918
|
"use strict";
|
|
889089
|
-
var
|
|
888919
|
+
var net2 = require("net");
|
|
889090
888920
|
var tls = require("tls");
|
|
889091
888921
|
var http7 = require("http");
|
|
889092
888922
|
var https4 = require("https");
|
|
@@ -904361,7 +904191,7 @@ var require_util11 = __commonJS({
|
|
|
904361
904191
|
var { kDestroyed, kBodyUsed, kListeners, kBody } = require_symbols();
|
|
904362
904192
|
var { IncomingMessage } = require("node:http");
|
|
904363
904193
|
var stream = require("node:stream");
|
|
904364
|
-
var
|
|
904194
|
+
var net2 = require("node:net");
|
|
904365
904195
|
var { Blob: Blob4 } = require("node:buffer");
|
|
904366
904196
|
var nodeUtil = require("node:util");
|
|
904367
904197
|
var { stringify: stringify5 } = require("node:querystring");
|
|
@@ -904506,7 +904336,7 @@ var require_util11 = __commonJS({
|
|
|
904506
904336
|
}
|
|
904507
904337
|
assert4(typeof host === "string");
|
|
904508
904338
|
const servername = getHostname(host);
|
|
904509
|
-
if (
|
|
904339
|
+
if (net2.isIP(servername)) {
|
|
904510
904340
|
return "";
|
|
904511
904341
|
}
|
|
904512
904342
|
return servername;
|
|
@@ -905828,7 +905658,7 @@ var require_timers2 = __commonJS({
|
|
|
905828
905658
|
var require_connect2 = __commonJS({
|
|
905829
905659
|
"node_modules/.pnpm/undici@6.24.1/node_modules/undici/lib/core/connect.js"(exports2, module2) {
|
|
905830
905660
|
"use strict";
|
|
905831
|
-
var
|
|
905661
|
+
var net2 = require("node:net");
|
|
905832
905662
|
var assert4 = require("node:assert");
|
|
905833
905663
|
var util2 = require_util11();
|
|
905834
905664
|
var { InvalidArgumentError, ConnectTimeoutError } = require_errors11();
|
|
@@ -905924,7 +905754,7 @@ var require_connect2 = __commonJS({
|
|
|
905924
905754
|
} else {
|
|
905925
905755
|
assert4(!httpSocket, "httpSocket can only be sent on TLS update");
|
|
905926
905756
|
port = port || 80;
|
|
905927
|
-
socket =
|
|
905757
|
+
socket = net2.connect({
|
|
905928
905758
|
highWaterMark: 64 * 1024,
|
|
905929
905759
|
// Same as nodejs fs streams.
|
|
905930
905760
|
...options3,
|
|
@@ -910794,7 +910624,7 @@ var require_client = __commonJS({
|
|
|
910794
910624
|
"node_modules/.pnpm/undici@6.24.1/node_modules/undici/lib/dispatcher/client.js"(exports2, module2) {
|
|
910795
910625
|
"use strict";
|
|
910796
910626
|
var assert4 = require("node:assert");
|
|
910797
|
-
var
|
|
910627
|
+
var net2 = require("node:net");
|
|
910798
910628
|
var http7 = require("node:http");
|
|
910799
910629
|
var util2 = require_util11();
|
|
910800
910630
|
var { channels } = require_diagnostics();
|
|
@@ -910942,7 +910772,7 @@ var require_client = __commonJS({
|
|
|
910942
910772
|
if (maxRequestsPerClient != null && (!Number.isInteger(maxRequestsPerClient) || maxRequestsPerClient < 0)) {
|
|
910943
910773
|
throw new InvalidArgumentError("maxRequestsPerClient must be a positive number");
|
|
910944
910774
|
}
|
|
910945
|
-
if (localAddress != null && (typeof localAddress !== "string" ||
|
|
910775
|
+
if (localAddress != null && (typeof localAddress !== "string" || net2.isIP(localAddress) === 0)) {
|
|
910946
910776
|
throw new InvalidArgumentError("localAddress must be valid string IP address");
|
|
910947
910777
|
}
|
|
910948
910778
|
if (maxResponseSize != null && (!Number.isInteger(maxResponseSize) || maxResponseSize < -1)) {
|
|
@@ -911106,7 +910936,7 @@ var require_client = __commonJS({
|
|
|
911106
910936
|
const idx = hostname3.indexOf("]");
|
|
911107
910937
|
assert4(idx !== -1);
|
|
911108
910938
|
const ip = hostname3.substring(1, idx);
|
|
911109
|
-
assert4(
|
|
910939
|
+
assert4(net2.isIP(ip));
|
|
911110
910940
|
hostname3 = ip;
|
|
911111
910941
|
}
|
|
911112
910942
|
client[kConnecting] = true;
|
|
@@ -974873,7 +974703,7 @@ async function sendMessage(params) {
|
|
|
974873
974703
|
label: "sendMessage"
|
|
974874
974704
|
});
|
|
974875
974705
|
}
|
|
974876
|
-
async function
|
|
974706
|
+
async function getConfig2(params) {
|
|
974877
974707
|
const rawText = await apiPostFetch({
|
|
974878
974708
|
baseUrl: params.baseUrl,
|
|
974879
974709
|
endpoint: "ilink/bot/getconfig",
|
|
@@ -974942,7 +974772,7 @@ var init_config_cache = __esm({
|
|
|
974942
974772
|
if (shouldFetch) {
|
|
974943
974773
|
let fetchOk = false;
|
|
974944
974774
|
try {
|
|
974945
|
-
const resp = await
|
|
974775
|
+
const resp = await getConfig2({
|
|
974946
974776
|
baseUrl: this.apiOpts.baseUrl,
|
|
974947
974777
|
token: this.apiOpts.token,
|
|
974948
974778
|
ilinkUserId: userId,
|
|
@@ -976538,9 +976368,9 @@ var init_weixin = __esm({
|
|
|
976538
976368
|
setContextToken(deps.accountId, fromUserId, full.context_token);
|
|
976539
976369
|
this.contextTokens.set(fromUserId, full.context_token);
|
|
976540
976370
|
}
|
|
976541
|
-
const
|
|
976542
|
-
if (
|
|
976543
|
-
this.typingTickets.set(fromUserId,
|
|
976371
|
+
const cachedConfig = await deps.configManager.getForUser(fromUserId, full.context_token);
|
|
976372
|
+
if (cachedConfig.typingTicket) {
|
|
976373
|
+
this.typingTickets.set(fromUserId, cachedConfig.typingTicket);
|
|
976544
976374
|
}
|
|
976545
976375
|
const textBody = extractTextBody(full.item_list);
|
|
976546
976376
|
const attachments = [];
|
|
@@ -978209,6 +978039,15 @@ var init_login_qr = __esm({
|
|
|
978209
978039
|
}
|
|
978210
978040
|
});
|
|
978211
978041
|
|
|
978042
|
+
// node_modules/openclaw/plugin-sdk.js
|
|
978043
|
+
function normalizeAccountId2(id2) {
|
|
978044
|
+
return id2.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
|
|
978045
|
+
}
|
|
978046
|
+
var init_plugin_sdk2 = __esm({
|
|
978047
|
+
"node_modules/openclaw/plugin-sdk.js"() {
|
|
978048
|
+
}
|
|
978049
|
+
});
|
|
978050
|
+
|
|
978212
978051
|
// dist/agent/backup.js
|
|
978213
978052
|
var backup_exports = {};
|
|
978214
978053
|
__export(backup_exports, {
|
|
@@ -989725,7 +989564,7 @@ ${result.message}`);
|
|
|
989725
989564
|
if (!result.connected || !result.accountId) {
|
|
989726
989565
|
process.exit(1);
|
|
989727
989566
|
}
|
|
989728
|
-
const accountId =
|
|
989567
|
+
const accountId = normalizeAccountId2(result.accountId);
|
|
989729
989568
|
saveWeixinAccount(accountId, {
|
|
989730
989569
|
token: result.botToken,
|
|
989731
989570
|
baseUrl: result.baseUrl,
|
|
@@ -989758,7 +989597,7 @@ var init_weixin_login = __esm({
|
|
|
989758
989597
|
"use strict";
|
|
989759
989598
|
init_accounts();
|
|
989760
989599
|
init_login_qr();
|
|
989761
|
-
|
|
989600
|
+
init_plugin_sdk2();
|
|
989762
989601
|
init_config();
|
|
989763
989602
|
}
|
|
989764
989603
|
});
|
|
@@ -991671,9 +991510,7 @@ Even if you are using Playwright to operate the browser, you should bring the br
|
|
|
991671
991510
|
|
|
991672
991511
|
### Desktop Interactions (Computer Use)
|
|
991673
991512
|
|
|
991674
|
-
For desktop applications and UI outside the browser, use the available desktop/computer-use tools.
|
|
991675
|
-
|
|
991676
|
-
If you describe something but the computer use tool can not resolve the coordinates, try describe the target in more details. Also, the screenshot will always show the current cursor location.
|
|
991513
|
+
For desktop applications and UI outside the browser, use the available desktop/computer-use tools. The \`computer_use_*\` tools accept pixel coordinates from screenshots \u2014 always take a screenshot first with \`computer_use_screenshot\` to see the screen and determine coordinates, then use those coordinates for click/move/scroll/drag actions.
|
|
991677
991514
|
|
|
991678
991515
|
**Prefer ${config2.browserBackend === "playwriter" ? "Playwriter" : "Playwright"} browser tools** for all in-browser work. Use desktop/computer-use tools only for desktop apps, situations outside the browser, or as a fallback when the browser tools can't do the job.
|
|
991679
991516
|
|
|
@@ -996056,84 +995893,6 @@ init_transcribe_audio();
|
|
|
996056
995893
|
init_zod();
|
|
996057
995894
|
init_sdk2();
|
|
996058
995895
|
init_desktop_executor_factory();
|
|
996059
|
-
|
|
996060
|
-
// dist/tools/coordinate-resolver.js
|
|
996061
|
-
init_logger();
|
|
996062
|
-
init_screenshot();
|
|
996063
|
-
init_client_factory();
|
|
996064
|
-
function clamp(n6, min, max) {
|
|
996065
|
-
return Math.max(min, Math.min(max, n6));
|
|
996066
|
-
}
|
|
996067
|
-
async function resolveCoordinates(screenshotBase64, instruction, displayWidth, displayHeight, config2, screenshotFilePath) {
|
|
996068
|
-
const dims = screenshotFilePath ? await getImageDimensionsFromFile(screenshotFilePath) : await getImageDimensions(Buffer.from(screenshotBase64, "base64"));
|
|
996069
|
-
if (!dims) {
|
|
996070
|
-
logger.warn("Could not read screenshot dimensions, aborting coordinate resolution");
|
|
996071
|
-
return null;
|
|
996072
|
-
}
|
|
996073
|
-
const imgW = dims.width;
|
|
996074
|
-
const imgH = dims.height;
|
|
996075
|
-
const scaleX = displayWidth / imgW;
|
|
996076
|
-
const scaleY = displayHeight / imgH;
|
|
996077
|
-
logger.info(`Resolving coordinates: "${instruction}" (display ${displayWidth}x${displayHeight}, image ${imgW}x${imgH}, scaleX=${scaleX.toFixed(3)}, scaleY=${scaleY.toFixed(3)})`, { instruction, displayWidth, displayHeight, imgW, imgH, scaleX, scaleY });
|
|
996078
|
-
const client = createClient(config2);
|
|
996079
|
-
const response = await client.beta.messages.create({
|
|
996080
|
-
model: getModelId(config2),
|
|
996081
|
-
max_tokens: 1024,
|
|
996082
|
-
betas: ["computer-use-2025-11-24"],
|
|
996083
|
-
tools: [
|
|
996084
|
-
{
|
|
996085
|
-
type: "computer_20251124",
|
|
996086
|
-
name: "computer",
|
|
996087
|
-
display_width_px: imgW,
|
|
996088
|
-
display_height_px: imgH,
|
|
996089
|
-
display_number: 1
|
|
996090
|
-
}
|
|
996091
|
-
],
|
|
996092
|
-
system: "You are a helpful assistant that can resolve natural language targets to pixel coordinates using the computer tool. Call the tool directly, no questions or explanations.",
|
|
996093
|
-
tool_choice: { type: "tool", name: "computer" },
|
|
996094
|
-
messages: [
|
|
996095
|
-
{
|
|
996096
|
-
role: "user",
|
|
996097
|
-
content: [
|
|
996098
|
-
{
|
|
996099
|
-
type: "image",
|
|
996100
|
-
source: {
|
|
996101
|
-
type: "base64",
|
|
996102
|
-
media_type: "image/png",
|
|
996103
|
-
data: screenshotBase64
|
|
996104
|
-
}
|
|
996105
|
-
},
|
|
996106
|
-
{
|
|
996107
|
-
type: "text",
|
|
996108
|
-
text: instruction
|
|
996109
|
-
}
|
|
996110
|
-
]
|
|
996111
|
-
}
|
|
996112
|
-
]
|
|
996113
|
-
});
|
|
996114
|
-
for (const block of response.content) {
|
|
996115
|
-
if (block.type === "tool_use" && block.name === "computer") {
|
|
996116
|
-
const input = block.input;
|
|
996117
|
-
const coord = input.coordinate;
|
|
996118
|
-
const action = input.action ?? "left_click";
|
|
996119
|
-
logger.debug(`Coordinate resolver raw response: action=${action} coordinate=${JSON.stringify(coord)}`, { instruction, action, coordinate: coord, fullInput: input });
|
|
996120
|
-
if (Array.isArray(coord) && coord.length >= 2) {
|
|
996121
|
-
const rawX = coord[0];
|
|
996122
|
-
const rawY = coord[1];
|
|
996123
|
-
if (typeof rawX === "number" && typeof rawY === "number" && Number.isFinite(rawX) && Number.isFinite(rawY)) {
|
|
996124
|
-
const x14 = clamp(Math.round(rawX * scaleX), 0, displayWidth);
|
|
996125
|
-
const y12 = clamp(Math.round(rawY * scaleY), 0, displayHeight);
|
|
996126
|
-
logger.info(`Resolved coordinates: action=${action} raw=(${rawX},${rawY}) unscaled=(${x14},${y12}) scaleX=${scaleX.toFixed(3)} scaleY=${scaleY.toFixed(3)}`, { instruction, action, rawX, rawY, x: x14, y: y12, scaleX, scaleY });
|
|
996127
|
-
return { action, x: x14, y: y12 };
|
|
996128
|
-
}
|
|
996129
|
-
}
|
|
996130
|
-
}
|
|
996131
|
-
}
|
|
996132
|
-
logger.info(`Could not resolve coordinates: "${instruction}"`, { instruction });
|
|
996133
|
-
return null;
|
|
996134
|
-
}
|
|
996135
|
-
|
|
996136
|
-
// dist/tools/computer-use.js
|
|
996137
995896
|
init_screenshot();
|
|
996138
995897
|
init_logger();
|
|
996139
995898
|
function requireDesktop() {
|
|
@@ -996156,48 +995915,32 @@ function requireDesktop() {
|
|
|
996156
995915
|
}
|
|
996157
995916
|
return null;
|
|
996158
995917
|
}
|
|
996159
|
-
|
|
996160
|
-
|
|
996161
|
-
if (
|
|
996162
|
-
return
|
|
996163
|
-
|
|
996164
|
-
|
|
996165
|
-
|
|
996166
|
-
|
|
996167
|
-
} catch {
|
|
996168
|
-
if (process.env.ANTHROPIC_API_KEY) {
|
|
996169
|
-
cachedConfig = {
|
|
996170
|
-
model: "claude-sonnet-4-6",
|
|
996171
|
-
provider: "anthropic",
|
|
996172
|
-
anthropicApiKey: process.env.ANTHROPIC_API_KEY
|
|
996173
|
-
};
|
|
996174
|
-
return cachedConfig;
|
|
996175
|
-
}
|
|
996176
|
-
return null;
|
|
996177
|
-
}
|
|
995918
|
+
function getScreenshotSize(displaySize) {
|
|
995919
|
+
const scale = getVisionScaleFactor(displaySize.width, displaySize.height);
|
|
995920
|
+
if (scale >= 1)
|
|
995921
|
+
return { ...displaySize };
|
|
995922
|
+
return {
|
|
995923
|
+
width: Math.max(1, Math.round(displaySize.width * scale)),
|
|
995924
|
+
height: Math.max(1, Math.round(displaySize.height * scale))
|
|
995925
|
+
};
|
|
996178
995926
|
}
|
|
996179
|
-
|
|
996180
|
-
|
|
996181
|
-
|
|
996182
|
-
|
|
996183
|
-
|
|
996184
|
-
|
|
996185
|
-
|
|
996186
|
-
|
|
996187
|
-
|
|
996188
|
-
|
|
996189
|
-
|
|
996190
|
-
|
|
996191
|
-
isError: true
|
|
996192
|
-
};
|
|
996193
|
-
}
|
|
996194
|
-
await fn2(resolved.x, resolved.y);
|
|
996195
|
-
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
995927
|
+
function screenshotToDisplay(point, screenshotSize, displaySize) {
|
|
995928
|
+
return {
|
|
995929
|
+
x: Math.round(point.x / screenshotSize.width * displaySize.width),
|
|
995930
|
+
y: Math.round(point.y / screenshotSize.height * displaySize.height)
|
|
995931
|
+
};
|
|
995932
|
+
}
|
|
995933
|
+
var pointSchema = external_exports.object({
|
|
995934
|
+
x: external_exports.number().min(0).describe("X pixel coordinate in the screenshot image (0 = left edge)"),
|
|
995935
|
+
y: external_exports.number().min(0).describe("Y pixel coordinate in the screenshot image (0 = top edge)")
|
|
995936
|
+
});
|
|
995937
|
+
async function actionResult(text, delayMs = 1e3) {
|
|
995938
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
996196
995939
|
const { base64: resultScreenshot } = await captureScreen3();
|
|
996197
995940
|
logger.screenshot(resultScreenshot);
|
|
996198
995941
|
return {
|
|
996199
995942
|
content: [
|
|
996200
|
-
{ type: "text", text
|
|
995943
|
+
{ type: "text", text },
|
|
996201
995944
|
{
|
|
996202
995945
|
type: "image",
|
|
996203
995946
|
data: resultScreenshot,
|
|
@@ -996206,184 +995949,88 @@ async function resolveAndExecute(instruction, config2, fn2) {
|
|
|
996206
995949
|
]
|
|
996207
995950
|
};
|
|
996208
995951
|
}
|
|
996209
|
-
|
|
996210
|
-
|
|
995952
|
+
function withDesktopCheck(toolName, fn2) {
|
|
995953
|
+
return (async () => {
|
|
995954
|
+
const err7 = requireDesktop();
|
|
995955
|
+
if (err7)
|
|
995956
|
+
return err7;
|
|
995957
|
+
try {
|
|
995958
|
+
return await fn2();
|
|
995959
|
+
} catch (e11) {
|
|
995960
|
+
logger.err(`${toolName} failed`, { error: String(e11) });
|
|
995961
|
+
return {
|
|
995962
|
+
content: [
|
|
995963
|
+
{
|
|
995964
|
+
type: "text",
|
|
995965
|
+
text: `${toolName} failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
995966
|
+
}
|
|
995967
|
+
],
|
|
995968
|
+
isError: true
|
|
995969
|
+
};
|
|
995970
|
+
}
|
|
995971
|
+
})();
|
|
995972
|
+
}
|
|
995973
|
+
var computerUseClickTool = gl("computer_use_click", "Click at a point on the desktop screen. Coordinates are pixel positions in the screenshot image returned by computer_use_screenshot. Supports left-click, right-click, and double-click.", {
|
|
995974
|
+
point: pointSchema.describe("Pixel coordinates in the screenshot image to click"),
|
|
996211
995975
|
button: external_exports.enum(["left", "right", "double"]).optional().default("left")
|
|
996212
995976
|
}, async (args) => {
|
|
996213
|
-
|
|
996214
|
-
|
|
996215
|
-
|
|
996216
|
-
|
|
996217
|
-
|
|
996218
|
-
|
|
996219
|
-
|
|
996220
|
-
|
|
996221
|
-
}
|
|
996222
|
-
|
|
996223
|
-
|
|
996224
|
-
|
|
996225
|
-
|
|
996226
|
-
return await resolveAndExecute(instruction, cfg, fn2);
|
|
996227
|
-
} catch (e11) {
|
|
996228
|
-
logger.err("computer_use_click failed", { error: String(e11) });
|
|
996229
|
-
return {
|
|
996230
|
-
content: [
|
|
996231
|
-
{
|
|
996232
|
-
type: "text",
|
|
996233
|
-
text: `computer_use_click failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996234
|
-
}
|
|
996235
|
-
],
|
|
996236
|
-
isError: true
|
|
996237
|
-
};
|
|
996238
|
-
}
|
|
995977
|
+
return withDesktopCheck("computer_use_click", async () => {
|
|
995978
|
+
const displaySize = getDisplaySize3();
|
|
995979
|
+
const ssSize = getScreenshotSize(displaySize);
|
|
995980
|
+
const abs = screenshotToDisplay(args.point, ssSize, displaySize);
|
|
995981
|
+
if (args.button === "right") {
|
|
995982
|
+
await rightClick3(abs.x, abs.y);
|
|
995983
|
+
} else if (args.button === "double") {
|
|
995984
|
+
await doubleClick3(abs.x, abs.y);
|
|
995985
|
+
} else {
|
|
995986
|
+
await click3(abs.x, abs.y);
|
|
995987
|
+
}
|
|
995988
|
+
return actionResult(`${args.button === "double" ? "Double-clicked" : args.button === "right" ? "Right-clicked" : "Clicked"} at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) \u2192 display (${abs.x}, ${abs.y})`);
|
|
995989
|
+
});
|
|
996239
995990
|
});
|
|
996240
|
-
var computerUseMoveTool = gl("computer_use_move", "Move the mouse cursor to a
|
|
996241
|
-
|
|
995991
|
+
var computerUseMoveTool = gl("computer_use_move", "Move the mouse cursor to a point on the desktop screen. Coordinates are pixel positions in the screenshot image.", {
|
|
995992
|
+
point: pointSchema.describe("Pixel coordinates in the screenshot image to move the cursor to")
|
|
996242
995993
|
}, async (args) => {
|
|
996243
|
-
|
|
996244
|
-
|
|
996245
|
-
|
|
996246
|
-
|
|
996247
|
-
|
|
996248
|
-
return {
|
|
996249
|
-
|
|
996250
|
-
isError: true
|
|
996251
|
-
};
|
|
996252
|
-
}
|
|
996253
|
-
const instruction = `Move cursor to the center of: ${args.target}`;
|
|
996254
|
-
try {
|
|
996255
|
-
return await resolveAndExecute(instruction, cfg, async (x14, y12) => {
|
|
996256
|
-
await moveTo3(x14, y12);
|
|
996257
|
-
});
|
|
996258
|
-
} catch (e11) {
|
|
996259
|
-
logger.err("computer_use_move failed", { error: String(e11) });
|
|
996260
|
-
return {
|
|
996261
|
-
content: [
|
|
996262
|
-
{
|
|
996263
|
-
type: "text",
|
|
996264
|
-
text: `computer_use_move failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996265
|
-
}
|
|
996266
|
-
],
|
|
996267
|
-
isError: true
|
|
996268
|
-
};
|
|
996269
|
-
}
|
|
995994
|
+
return withDesktopCheck("computer_use_move", async () => {
|
|
995995
|
+
const displaySize = getDisplaySize3();
|
|
995996
|
+
const ssSize = getScreenshotSize(displaySize);
|
|
995997
|
+
const abs = screenshotToDisplay(args.point, ssSize, displaySize);
|
|
995998
|
+
await moveTo3(abs.x, abs.y);
|
|
995999
|
+
return actionResult(`Moved cursor to screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) \u2192 display (${abs.x}, ${abs.y})`);
|
|
996000
|
+
});
|
|
996270
996001
|
});
|
|
996271
|
-
var computerUseScrollTool = gl("computer_use_scroll", "Scroll at a
|
|
996272
|
-
|
|
996002
|
+
var computerUseScrollTool = gl("computer_use_scroll", "Scroll at a point on the desktop screen in a direction. Coordinates are pixel positions in the screenshot image.", {
|
|
996003
|
+
point: pointSchema.describe("Pixel coordinates in the screenshot image for scroll origin"),
|
|
996273
996004
|
direction: external_exports.enum(["up", "down", "left", "right"]),
|
|
996274
996005
|
amount: external_exports.number().optional().default(3).describe("Scroll amount (default 3)")
|
|
996275
996006
|
}, async (args) => {
|
|
996276
|
-
|
|
996277
|
-
|
|
996278
|
-
|
|
996279
|
-
|
|
996280
|
-
|
|
996281
|
-
return {
|
|
996282
|
-
|
|
996283
|
-
isError: true
|
|
996284
|
-
};
|
|
996285
|
-
}
|
|
996286
|
-
const instruction = `Move cursor to the center of: ${args.target} (for scrolling)`;
|
|
996287
|
-
const amount = args.amount;
|
|
996288
|
-
try {
|
|
996289
|
-
return await resolveAndExecute(instruction, cfg, async (x14, y12) => {
|
|
996290
|
-
await scroll3(x14, y12, args.direction, amount);
|
|
996291
|
-
});
|
|
996292
|
-
} catch (e11) {
|
|
996293
|
-
logger.err("computer_use_scroll failed", { error: String(e11) });
|
|
996294
|
-
return {
|
|
996295
|
-
content: [
|
|
996296
|
-
{
|
|
996297
|
-
type: "text",
|
|
996298
|
-
text: `computer_use_scroll failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996299
|
-
}
|
|
996300
|
-
],
|
|
996301
|
-
isError: true
|
|
996302
|
-
};
|
|
996303
|
-
}
|
|
996007
|
+
return withDesktopCheck("computer_use_scroll", async () => {
|
|
996008
|
+
const displaySize = getDisplaySize3();
|
|
996009
|
+
const ssSize = getScreenshotSize(displaySize);
|
|
996010
|
+
const abs = screenshotToDisplay(args.point, ssSize, displaySize);
|
|
996011
|
+
await scroll3(abs.x, abs.y, args.direction, args.amount);
|
|
996012
|
+
return actionResult(`Scrolled ${args.direction} at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) \u2192 display (${abs.x}, ${abs.y})`);
|
|
996013
|
+
});
|
|
996304
996014
|
});
|
|
996305
|
-
var computerUseDragTool = gl("computer_use_drag", "Drag from one
|
|
996306
|
-
|
|
996307
|
-
|
|
996015
|
+
var computerUseDragTool = gl("computer_use_drag", "Drag from one point to another on the desktop screen. Coordinates are pixel positions in the screenshot image.", {
|
|
996016
|
+
startPoint: pointSchema.describe("Pixel coordinates in the screenshot image (drag source)"),
|
|
996017
|
+
endPoint: pointSchema.describe("Pixel coordinates in the screenshot image (drop target)")
|
|
996308
996018
|
}, async (args) => {
|
|
996309
|
-
|
|
996310
|
-
|
|
996311
|
-
|
|
996312
|
-
|
|
996313
|
-
|
|
996314
|
-
|
|
996315
|
-
|
|
996316
|
-
|
|
996317
|
-
};
|
|
996318
|
-
}
|
|
996319
|
-
const { base64: screenshot, filePath: screenshotPath } = await captureScreen3();
|
|
996320
|
-
const size = getDisplaySize3();
|
|
996321
|
-
const [fromResolved, toResolved] = await Promise.all([
|
|
996322
|
-
resolveCoordinates(screenshot, `Move cursor to the center of: ${args.from} (for dragging)`, size.width, size.height, cfg, screenshotPath),
|
|
996323
|
-
resolveCoordinates(screenshot, `Move cursor to the center of: ${args.to} (for dragging)`, size.width, size.height, cfg, screenshotPath)
|
|
996324
|
-
]);
|
|
996325
|
-
if (!fromResolved) {
|
|
996326
|
-
return {
|
|
996327
|
-
content: [
|
|
996328
|
-
{
|
|
996329
|
-
type: "text",
|
|
996330
|
-
text: `Could not resolve 'from' coordinates: ${args.from}. Please try again with a clearer instruction.`
|
|
996331
|
-
}
|
|
996332
|
-
],
|
|
996333
|
-
isError: true
|
|
996334
|
-
};
|
|
996335
|
-
}
|
|
996336
|
-
if (!toResolved) {
|
|
996337
|
-
return {
|
|
996338
|
-
content: [
|
|
996339
|
-
{
|
|
996340
|
-
type: "text",
|
|
996341
|
-
text: `Could not resolve 'to' coordinates: ${args.to}. Please try again with a clearer instruction.`
|
|
996342
|
-
}
|
|
996343
|
-
],
|
|
996344
|
-
isError: true
|
|
996345
|
-
};
|
|
996346
|
-
}
|
|
996347
|
-
try {
|
|
996348
|
-
await drag3(fromResolved.x, fromResolved.y, toResolved.x, toResolved.y);
|
|
996349
|
-
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
996350
|
-
const { base64: resultScreenshot } = await captureScreen3();
|
|
996351
|
-
logger.screenshot(resultScreenshot);
|
|
996352
|
-
return {
|
|
996353
|
-
content: [
|
|
996354
|
-
{
|
|
996355
|
-
type: "text",
|
|
996356
|
-
text: `Dragged from (${fromResolved.x}, ${fromResolved.y}) to (${toResolved.x}, ${toResolved.y})`
|
|
996357
|
-
},
|
|
996358
|
-
{
|
|
996359
|
-
type: "image",
|
|
996360
|
-
data: resultScreenshot,
|
|
996361
|
-
mimeType: "image/png"
|
|
996362
|
-
}
|
|
996363
|
-
]
|
|
996364
|
-
};
|
|
996365
|
-
} catch (e11) {
|
|
996366
|
-
logger.err("computer_use_drag failed", { error: String(e11) });
|
|
996367
|
-
return {
|
|
996368
|
-
content: [
|
|
996369
|
-
{
|
|
996370
|
-
type: "text",
|
|
996371
|
-
text: `computer_use_drag failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996372
|
-
}
|
|
996373
|
-
],
|
|
996374
|
-
isError: true
|
|
996375
|
-
};
|
|
996376
|
-
}
|
|
996019
|
+
return withDesktopCheck("computer_use_drag", async () => {
|
|
996020
|
+
const displaySize = getDisplaySize3();
|
|
996021
|
+
const ssSize = getScreenshotSize(displaySize);
|
|
996022
|
+
const start = screenshotToDisplay(args.startPoint, ssSize, displaySize);
|
|
996023
|
+
const end = screenshotToDisplay(args.endPoint, ssSize, displaySize);
|
|
996024
|
+
await drag3(start.x, start.y, end.x, end.y);
|
|
996025
|
+
return actionResult(`Dragged from screenshot (${Math.round(args.startPoint.x)}, ${Math.round(args.startPoint.y)}) to (${Math.round(args.endPoint.x)}, ${Math.round(args.endPoint.y)}) \u2192 display (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
|
|
996026
|
+
});
|
|
996377
996027
|
});
|
|
996378
996028
|
var computerUseTypeTool = gl("computer_use_type", "Type text into the focused field. Optionally clear the field first (replace) and/or press Enter after.", {
|
|
996379
996029
|
text: external_exports.string().describe("Text to type"),
|
|
996380
996030
|
replace: external_exports.boolean().optional().default(false).describe("If true, select-all and replace existing text first"),
|
|
996381
996031
|
press_enter: external_exports.boolean().optional().default(false).describe("If true, press Enter after typing")
|
|
996382
996032
|
}, async (args) => {
|
|
996383
|
-
|
|
996384
|
-
if (err7)
|
|
996385
|
-
return err7;
|
|
996386
|
-
try {
|
|
996033
|
+
return withDesktopCheck("computer_use_type", async () => {
|
|
996387
996034
|
if (args.replace) {
|
|
996388
996035
|
await pressKey3(process.platform === "win32" ? "ctrl+a" : "cmd+a");
|
|
996389
996036
|
}
|
|
@@ -996391,78 +996038,33 @@ var computerUseTypeTool = gl("computer_use_type", "Type text into the focused fi
|
|
|
996391
996038
|
if (args.press_enter) {
|
|
996392
996039
|
await pressKey3("enter");
|
|
996393
996040
|
}
|
|
996394
|
-
|
|
996395
|
-
|
|
996396
|
-
logger.screenshot(resultScreenshot);
|
|
996397
|
-
return {
|
|
996398
|
-
content: [
|
|
996399
|
-
{ type: "text", text: `Typed text${args.press_enter ? " and pressed Enter" : ""}` },
|
|
996400
|
-
{
|
|
996401
|
-
type: "image",
|
|
996402
|
-
data: resultScreenshot,
|
|
996403
|
-
mimeType: "image/png"
|
|
996404
|
-
}
|
|
996405
|
-
]
|
|
996406
|
-
};
|
|
996407
|
-
} catch (e11) {
|
|
996408
|
-
logger.err("computer_use_type failed", { error: String(e11) });
|
|
996409
|
-
return {
|
|
996410
|
-
content: [
|
|
996411
|
-
{
|
|
996412
|
-
type: "text",
|
|
996413
|
-
text: `computer_use_type failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996414
|
-
}
|
|
996415
|
-
],
|
|
996416
|
-
isError: true
|
|
996417
|
-
};
|
|
996418
|
-
}
|
|
996041
|
+
return actionResult(`Typed text${args.press_enter ? " and pressed Enter" : ""}`);
|
|
996042
|
+
});
|
|
996419
996043
|
});
|
|
996420
996044
|
var computerUseKeyTool = gl("computer_use_key", "Press a key or key combination (e.g. 'enter', 'escape', 'cmd+s', 'ctrl+shift+t').", {
|
|
996421
996045
|
key: external_exports.string().describe("Key or combo: 'enter', 'escape', 'cmd+s', 'ctrl+shift+t', etc.")
|
|
996422
996046
|
}, async (args) => {
|
|
996423
|
-
|
|
996424
|
-
if (err7)
|
|
996425
|
-
return err7;
|
|
996426
|
-
try {
|
|
996047
|
+
return withDesktopCheck("computer_use_key", async () => {
|
|
996427
996048
|
await pressKey3(args.key);
|
|
996428
|
-
|
|
996429
|
-
|
|
996430
|
-
logger.screenshot(resultScreenshot);
|
|
996431
|
-
return {
|
|
996432
|
-
content: [
|
|
996433
|
-
{ type: "text", text: `Pressed: ${args.key}` },
|
|
996434
|
-
{
|
|
996435
|
-
type: "image",
|
|
996436
|
-
data: resultScreenshot,
|
|
996437
|
-
mimeType: "image/png"
|
|
996438
|
-
}
|
|
996439
|
-
]
|
|
996440
|
-
};
|
|
996441
|
-
} catch (e11) {
|
|
996442
|
-
logger.err("computer_use_key failed", { error: String(e11) });
|
|
996443
|
-
return {
|
|
996444
|
-
content: [
|
|
996445
|
-
{
|
|
996446
|
-
type: "text",
|
|
996447
|
-
text: `computer_use_key failed: ${e11 instanceof Error ? e11.message : String(e11)}`
|
|
996448
|
-
}
|
|
996449
|
-
],
|
|
996450
|
-
isError: true
|
|
996451
|
-
};
|
|
996452
|
-
}
|
|
996049
|
+
return actionResult(`Pressed: ${args.key}`);
|
|
996050
|
+
});
|
|
996453
996051
|
});
|
|
996454
|
-
var computerUseScreenshotTool = gl("computer_use_screenshot", "Capture a screenshot of the current desktop screen. Returns the image as base64 PNG
|
|
996052
|
+
var computerUseScreenshotTool = gl("computer_use_screenshot", "Capture a screenshot of the current desktop screen. Returns the image as base64 PNG. Use the pixel coordinates from this image when calling click, move, scroll, or drag tools.", {}, async (_args) => {
|
|
996455
996053
|
const err7 = requireDesktop();
|
|
996456
996054
|
if (err7)
|
|
996457
996055
|
return err7;
|
|
996458
996056
|
try {
|
|
996057
|
+
const displaySize = getDisplaySize3();
|
|
996058
|
+
const ssSize = getScreenshotSize(displaySize);
|
|
996459
996059
|
const { base64: base643, filePath } = await takeScreenshot();
|
|
996460
996060
|
logger.screenshot(base643);
|
|
996461
996061
|
return {
|
|
996462
996062
|
content: [
|
|
996463
996063
|
{
|
|
996464
996064
|
type: "text",
|
|
996465
|
-
text: `Screenshot saved to ${filePath}
|
|
996065
|
+
text: `Screenshot saved to ${filePath}
|
|
996066
|
+
Screenshot image size: ${ssSize.width}x${ssSize.height}px | Display resolution: ${displaySize.width}x${displaySize.height}px
|
|
996067
|
+
Use pixel coordinates from this image when calling click/move/scroll/drag tools.`
|
|
996466
996068
|
},
|
|
996467
996069
|
{
|
|
996468
996070
|
type: "image",
|
|
@@ -996901,7 +996503,7 @@ async function ensureDisplaySize() {
|
|
|
996901
996503
|
await initDisplaySize4();
|
|
996902
996504
|
}
|
|
996903
996505
|
}
|
|
996904
|
-
function
|
|
996506
|
+
function getScreenshotSize2(deviceSize) {
|
|
996905
996507
|
const scale = getVisionScaleFactor(deviceSize.width, deviceSize.height);
|
|
996906
996508
|
if (scale >= 1)
|
|
996907
996509
|
return { ...deviceSize };
|
|
@@ -996916,11 +996518,11 @@ function screenshotToDevice(point, screenshotSize, deviceSize) {
|
|
|
996916
996518
|
y: Math.round(point.y / screenshotSize.height * deviceSize.height)
|
|
996917
996519
|
};
|
|
996918
996520
|
}
|
|
996919
|
-
var
|
|
996521
|
+
var pointSchema2 = external_exports.object({
|
|
996920
996522
|
x: external_exports.number().min(0).describe("X pixel coordinate in the screenshot image (0 = left edge)"),
|
|
996921
996523
|
y: external_exports.number().min(0).describe("Y pixel coordinate in the screenshot image (0 = top edge)")
|
|
996922
996524
|
});
|
|
996923
|
-
async function
|
|
996525
|
+
async function actionResult2(text, delayMs = 1e3) {
|
|
996924
996526
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
996925
996527
|
const { base64: resultScreenshot } = await captureScreen4();
|
|
996926
996528
|
logger.screenshot(resultScreenshot);
|
|
@@ -996982,34 +996584,34 @@ function normalizeKey(raw) {
|
|
|
996982
996584
|
return KEY_ALIASES[normalized] ?? normalized;
|
|
996983
996585
|
}
|
|
996984
996586
|
var androidUseClickTool = gl("android_use_click", "Click (tap) at a point on the Android device screen. Coordinates are pixel positions in the screenshot image returned by android_use_screenshot. Supports single tap, double tap, and button variants.", {
|
|
996985
|
-
point:
|
|
996587
|
+
point: pointSchema2.describe("Pixel coordinates in the screenshot image to click"),
|
|
996986
996588
|
button: external_exports.enum(["left", "right", "middle"]).optional().default("left").describe("Click button (default: left). On Android, all buttons map to tap."),
|
|
996987
996589
|
double: external_exports.boolean().optional().default(false).describe("If true, perform a double-tap instead of a single tap")
|
|
996988
996590
|
}, async (args) => {
|
|
996989
996591
|
return withDeviceCheck("android_use_click", async () => {
|
|
996990
996592
|
await ensureDisplaySize();
|
|
996991
996593
|
const deviceSize = getDisplaySize4();
|
|
996992
|
-
const ssSize =
|
|
996594
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
996993
996595
|
const abs = screenshotToDevice(args.point, ssSize, deviceSize);
|
|
996994
996596
|
if (args.double) {
|
|
996995
996597
|
await doubleTap(abs.x, abs.y);
|
|
996996
996598
|
} else {
|
|
996997
996599
|
await tap(abs.x, abs.y);
|
|
996998
996600
|
}
|
|
996999
|
-
return
|
|
996601
|
+
return actionResult2(`${args.double ? "Double-clicked" : "Clicked"} at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) \u2192 device (${abs.x}, ${abs.y})`);
|
|
997000
996602
|
});
|
|
997001
996603
|
});
|
|
997002
996604
|
var androidUseLongPressTool = gl("android_use_long_press", "Long press at a point on the Android device screen. Coordinates are pixel positions in the screenshot image.", {
|
|
997003
|
-
point:
|
|
996605
|
+
point: pointSchema2.describe("Pixel coordinates in the screenshot image to long-press"),
|
|
997004
996606
|
durationMs: external_exports.number().int().positive().optional().default(1e3).describe("Long press duration in milliseconds (default: 1000)")
|
|
997005
996607
|
}, async (args) => {
|
|
997006
996608
|
return withDeviceCheck("android_use_long_press", async () => {
|
|
997007
996609
|
await ensureDisplaySize();
|
|
997008
996610
|
const deviceSize = getDisplaySize4();
|
|
997009
|
-
const ssSize =
|
|
996611
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
997010
996612
|
const abs = screenshotToDevice(args.point, ssSize, deviceSize);
|
|
997011
996613
|
await longPress(abs.x, abs.y, args.durationMs);
|
|
997012
|
-
return
|
|
996614
|
+
return actionResult2(`Long-pressed at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) \u2192 device (${abs.x}, ${abs.y}) for ${args.durationMs}ms`);
|
|
997013
996615
|
});
|
|
997014
996616
|
});
|
|
997015
996617
|
var androidUseTypeTool = gl("android_use_type", "Type text into the currently focused input field on the Android device. Optionally replace existing text first (select all + delete before typing).", {
|
|
@@ -997018,18 +996620,18 @@ var androidUseTypeTool = gl("android_use_type", "Type text into the currently fo
|
|
|
997018
996620
|
}, async (args) => {
|
|
997019
996621
|
return withDeviceCheck("android_use_type", async () => {
|
|
997020
996622
|
await typeText4(args.content, { replace: args.replace });
|
|
997021
|
-
return
|
|
996623
|
+
return actionResult2(`Typed "${args.content.length > 50 ? args.content.slice(0, 50) + "..." : args.content}"${args.replace ? " (replaced existing text)" : ""}`);
|
|
997022
996624
|
});
|
|
997023
996625
|
});
|
|
997024
996626
|
var androidUseScrollTool = gl("android_use_scroll", "Scroll the Android device screen in a direction from a point. Coordinates are pixel positions in the screenshot image. Amount is a float in [0, 1] representing scroll distance relative to screen size.", {
|
|
997025
996627
|
direction: external_exports.enum(["up", "down", "left", "right"]).describe("Scroll direction"),
|
|
997026
996628
|
amount: external_exports.number().min(0).max(1).describe("Scroll distance as fraction of screen size (0.0 to 1.0, e.g. 0.25 = quarter screen)"),
|
|
997027
|
-
point:
|
|
996629
|
+
point: pointSchema2.optional().describe("Pixel coordinates in the screenshot image for scroll origin (default: center of screen)")
|
|
997028
996630
|
}, async (args) => {
|
|
997029
996631
|
return withDeviceCheck("android_use_scroll", async () => {
|
|
997030
996632
|
await ensureDisplaySize();
|
|
997031
996633
|
const deviceSize = getDisplaySize4();
|
|
997032
|
-
const ssSize =
|
|
996634
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
997033
996635
|
const ssPoint = args.point ?? {
|
|
997034
996636
|
x: ssSize.width / 2,
|
|
997035
996637
|
y: ssSize.height / 2
|
|
@@ -997056,22 +996658,22 @@ var androidUseScrollTool = gl("android_use_scroll", "Scroll the Android device s
|
|
|
997056
996658
|
endX = Math.max(0, Math.min(deviceSize.width, endX));
|
|
997057
996659
|
endY = Math.max(0, Math.min(deviceSize.height, endY));
|
|
997058
996660
|
await swipe(origin.x, origin.y, endX, endY, 300);
|
|
997059
|
-
return
|
|
996661
|
+
return actionResult2(`Scrolled ${args.direction} by ${(args.amount * 100).toFixed(0)}% from screenshot (${Math.round(ssPoint.x)}, ${Math.round(ssPoint.y)})`);
|
|
997060
996662
|
});
|
|
997061
996663
|
});
|
|
997062
996664
|
var androidUseDragTool = gl("android_use_drag", "Drag from one point to another on the Android device. Coordinates are pixel positions in the screenshot image. Suitable for drag-and-drop operations (moving icons, reordering list items).", {
|
|
997063
|
-
startPoint:
|
|
997064
|
-
endPoint:
|
|
996665
|
+
startPoint: pointSchema2.describe("Pixel coordinates in the screenshot image (drag source)"),
|
|
996666
|
+
endPoint: pointSchema2.describe("Pixel coordinates in the screenshot image (drop target)"),
|
|
997065
996667
|
durationMs: external_exports.number().int().positive().optional().default(800).describe("Drag duration in milliseconds (default: 800, longer than swipe for drag-and-drop)")
|
|
997066
996668
|
}, async (args) => {
|
|
997067
996669
|
return withDeviceCheck("android_use_drag", async () => {
|
|
997068
996670
|
await ensureDisplaySize();
|
|
997069
996671
|
const deviceSize = getDisplaySize4();
|
|
997070
|
-
const ssSize =
|
|
996672
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
997071
996673
|
const start = screenshotToDevice(args.startPoint, ssSize, deviceSize);
|
|
997072
996674
|
const end = screenshotToDevice(args.endPoint, ssSize, deviceSize);
|
|
997073
996675
|
await swipe(start.x, start.y, end.x, end.y, args.durationMs);
|
|
997074
|
-
return
|
|
996676
|
+
return actionResult2(`Dragged from screenshot (${Math.round(args.startPoint.x)}, ${Math.round(args.startPoint.y)}) to (${Math.round(args.endPoint.x)}, ${Math.round(args.endPoint.y)}) over ${args.durationMs}ms`);
|
|
997075
996677
|
});
|
|
997076
996678
|
});
|
|
997077
996679
|
var androidUsePressTool = gl("android_use_press", "Press a key or button on the Android device. Supports standard keys: enter, tab, escape, backspace, delete, space, arrow keys (up/down/left/right), mobile keys (home, back, menu, app_switch, volume_up, volume_down, power), and function keys (f1-f12).", {
|
|
@@ -997080,7 +996682,7 @@ var androidUsePressTool = gl("android_use_press", "Press a key or button on the
|
|
|
997080
996682
|
return withDeviceCheck("android_use_press", async () => {
|
|
997081
996683
|
const normalized = normalizeKey(args.key);
|
|
997082
996684
|
await pressKey4(normalized);
|
|
997083
|
-
return
|
|
996685
|
+
return actionResult2(`Pressed: ${normalized}${normalized !== args.key ? ` (normalized from "${args.key}")` : ""}`);
|
|
997084
996686
|
});
|
|
997085
996687
|
});
|
|
997086
996688
|
var androidUseScreenshotTool = gl("android_use_screenshot", "Capture a screenshot of the connected Android device screen. Returns the image as base64 PNG. The image is resized for vision model processing. Use the pixel coordinates from this image when calling click, long_press, scroll, or drag tools.", {}, async (_args) => {
|
|
@@ -997090,7 +996692,7 @@ var androidUseScreenshotTool = gl("android_use_screenshot", "Capture a screensho
|
|
|
997090
996692
|
try {
|
|
997091
996693
|
await ensureDisplaySize();
|
|
997092
996694
|
const deviceSize = getDisplaySize4();
|
|
997093
|
-
const ssSize =
|
|
996695
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
997094
996696
|
const { base64: base643, filePath } = await captureScreen4();
|
|
997095
996697
|
logger.screenshot(base643);
|
|
997096
996698
|
return {
|
|
@@ -997128,7 +996730,7 @@ var androidUseOpenTool = gl("android_use_open", "Open (launch) an app on the And
|
|
|
997128
996730
|
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
997129
996731
|
await ensureDisplaySize();
|
|
997130
996732
|
const deviceSize = getDisplaySize4();
|
|
997131
|
-
const ssSize =
|
|
996733
|
+
const ssSize = getScreenshotSize2(deviceSize);
|
|
997132
996734
|
const { base64: resultScreenshot } = await captureScreen4();
|
|
997133
996735
|
logger.screenshot(resultScreenshot);
|
|
997134
996736
|
let activity = "";
|
|
@@ -1029358,7 +1028960,7 @@ var _config = void 0;
|
|
|
1029358
1028960
|
function resetArchiveConfigCache() {
|
|
1029359
1028961
|
_config = void 0;
|
|
1029360
1028962
|
}
|
|
1029361
|
-
function
|
|
1028963
|
+
function getConfig() {
|
|
1029362
1028964
|
if (_config === void 0) {
|
|
1029363
1028965
|
try {
|
|
1029364
1028966
|
_config = buildArchiveConfig(loadConfig());
|
|
@@ -1029369,7 +1028971,7 @@ function getConfig2() {
|
|
|
1029369
1028971
|
return _config;
|
|
1029370
1028972
|
}
|
|
1029371
1028973
|
function isArchiveEnabled() {
|
|
1029372
|
-
return
|
|
1028974
|
+
return getConfig() !== null;
|
|
1029373
1028975
|
}
|
|
1029374
1028976
|
var TEXT_CHUNK_KEYWORD = "vc-archive-id";
|
|
1029375
1028977
|
function pngCrc32(data2) {
|
|
@@ -1029517,7 +1029119,7 @@ async function drainFailedUploads(config2) {
|
|
|
1029517
1029119
|
}
|
|
1029518
1029120
|
}
|
|
1029519
1029121
|
function archiveImage(base64Data, mediaType, sessionId) {
|
|
1029520
|
-
const config2 =
|
|
1029122
|
+
const config2 = getConfig();
|
|
1029521
1029123
|
if (!config2)
|
|
1029522
1029124
|
return null;
|
|
1029523
1029125
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -1036084,7 +1035686,7 @@ async function buildStatusReport(options3) {
|
|
|
1036084
1035686
|
// dist/agent/command-handlers.js
|
|
1036085
1035687
|
init_accounts();
|
|
1036086
1035688
|
init_login_qr();
|
|
1036087
|
-
|
|
1035689
|
+
init_plugin_sdk2();
|
|
1036088
1035690
|
init_restart();
|
|
1036089
1035691
|
function registerCommandHandlers(deps) {
|
|
1036090
1035692
|
const { agentState: agentState2, channelManager, messageQueue, session, config: config2, ownerConfig, heartbeat } = deps;
|
|
@@ -1036280,7 +1035882,7 @@ function registerCommandHandlers(deps) {
|
|
|
1036280
1035882
|
await channelManager.sendMessage(req.channel, req.sender, t11("cmd.weixin.failed", getLocale(), result.message));
|
|
1036281
1035883
|
return;
|
|
1036282
1035884
|
}
|
|
1036283
|
-
const accountId =
|
|
1035885
|
+
const accountId = normalizeAccountId2(result.accountId);
|
|
1036284
1035886
|
saveWeixinAccount(accountId, {
|
|
1036285
1035887
|
token: result.botToken,
|
|
1036286
1035888
|
baseUrl: result.baseUrl,
|
|
@@ -1036622,21 +1036224,53 @@ function resolvePhonecallApiKey(config2) {
|
|
|
1036622
1036224
|
// dist/agent/runtime-surface.js
|
|
1036623
1036225
|
init_prepare_mac();
|
|
1036624
1036226
|
var import_url3 = require("url");
|
|
1036227
|
+
var import_net = __toESM(require("net"), 1);
|
|
1036625
1036228
|
var import_path61 = __toESM(require("path"), 1);
|
|
1036626
1036229
|
var import_fs11 = __toESM(require("fs"), 1);
|
|
1036627
1036230
|
init_logger();
|
|
1036628
|
-
function
|
|
1036231
|
+
function isPlaywriterRelayReachable(host, port, timeoutMs = 1e3) {
|
|
1036232
|
+
return new Promise((resolve) => {
|
|
1036233
|
+
const socket = import_net.default.createConnection({ host, port }, () => {
|
|
1036234
|
+
socket.destroy();
|
|
1036235
|
+
resolve(true);
|
|
1036236
|
+
});
|
|
1036237
|
+
socket.setTimeout(timeoutMs);
|
|
1036238
|
+
socket.on("timeout", () => {
|
|
1036239
|
+
socket.destroy();
|
|
1036240
|
+
resolve(false);
|
|
1036241
|
+
});
|
|
1036242
|
+
socket.on("error", () => {
|
|
1036243
|
+
resolve(false);
|
|
1036244
|
+
});
|
|
1036245
|
+
});
|
|
1036246
|
+
}
|
|
1036247
|
+
async function detectBrowserBackend(config2) {
|
|
1036629
1036248
|
if (process.platform !== "darwin")
|
|
1036630
1036249
|
return;
|
|
1036631
1036250
|
if (config2.browserBackend === "playwriter") {
|
|
1036632
|
-
|
|
1036251
|
+
const host = config2.playwriterRelayHost ?? "127.0.0.1";
|
|
1036252
|
+
const port = config2.playwriterRelayPort ?? 19988;
|
|
1036253
|
+
const reachable = await isPlaywriterRelayReachable(host, port);
|
|
1036254
|
+
if (reachable) {
|
|
1036255
|
+
logger.system("Browser backend: Playwriter (explicitly configured, relay reachable)");
|
|
1036256
|
+
} else {
|
|
1036257
|
+
config2.browserBackend = "playwright";
|
|
1036258
|
+
logger.system(`Browser backend: Playwright CDP (Playwriter relay unreachable at ${host}:${port}, falling back)`);
|
|
1036259
|
+
}
|
|
1036633
1036260
|
return;
|
|
1036634
1036261
|
}
|
|
1036635
1036262
|
try {
|
|
1036636
1036263
|
const { installed, profiles } = isPlaywriterExtensionInstalled();
|
|
1036637
1036264
|
if (installed) {
|
|
1036638
|
-
config2.
|
|
1036639
|
-
|
|
1036265
|
+
const host = config2.playwriterRelayHost ?? "127.0.0.1";
|
|
1036266
|
+
const port = config2.playwriterRelayPort ?? 19988;
|
|
1036267
|
+
const reachable = await isPlaywriterRelayReachable(host, port);
|
|
1036268
|
+
if (reachable) {
|
|
1036269
|
+
config2.browserBackend = "playwriter";
|
|
1036270
|
+
logger.system(`Browser backend: Playwriter (extension auto-detected in ${profiles.join(", ")}, relay reachable)`);
|
|
1036271
|
+
} else {
|
|
1036272
|
+
logger.system(`Browser backend: Playwright CDP (extension installed but relay unreachable at ${host}:${port})`);
|
|
1036273
|
+
}
|
|
1036640
1036274
|
} else {
|
|
1036641
1036275
|
logger.system("Browser backend: Playwright CDP (Playwriter extension not detected)");
|
|
1036642
1036276
|
}
|
|
@@ -1036644,8 +1036278,8 @@ function detectBrowserBackend(config2) {
|
|
|
1036644
1036278
|
logger.system("Browser backend: Playwright CDP (detection failed, using default)");
|
|
1036645
1036279
|
}
|
|
1036646
1036280
|
}
|
|
1036647
|
-
function buildRuntimeSurface(config2) {
|
|
1036648
|
-
detectBrowserBackend(config2);
|
|
1036281
|
+
async function buildRuntimeSurface(config2) {
|
|
1036282
|
+
await detectBrowserBackend(config2);
|
|
1036649
1036283
|
const dynamicExternalMcpServers = loadDynamicMcpServers();
|
|
1036650
1036284
|
const hasSerpApi = Boolean(config2.serpApiKey);
|
|
1036651
1036285
|
return {
|
|
@@ -1039376,7 +1039010,7 @@ async function startAgentLoop(config2, options3) {
|
|
|
1039376
1039010
|
cleanupOldTranscripts(transcriptDir);
|
|
1039377
1039011
|
} catch {
|
|
1039378
1039012
|
}
|
|
1039379
|
-
const { runtimeSurface, startupDynamicMcpServerCount } = buildRuntimeSurface(config2);
|
|
1039013
|
+
const { runtimeSurface, startupDynamicMcpServerCount } = await buildRuntimeSurface(config2);
|
|
1039380
1039014
|
if (startupDynamicMcpServerCount > 0) {
|
|
1039381
1039015
|
logger.system(`Loaded ${startupDynamicMcpServerCount} dynamic MCP server(s) from disk`);
|
|
1039382
1039016
|
}
|