visionclaw 0.1.91 → 0.1.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +4 -2
- package/dist/agent/browser-launcher.d.ts.map +1 -1
- package/dist/agent/browser-launcher.js +18 -1
- package/dist/agent/browser-launcher.js.map +1 -1
- package/dist/agent/command-handlers.d.ts +1 -1
- package/dist/agent/command-handlers.d.ts.map +1 -1
- package/dist/agent/command-handlers.js +1 -14
- package/dist/agent/command-handlers.js.map +1 -1
- package/dist/agent/context.js +1 -1
- package/dist/agent/context.js.map +1 -1
- package/dist/agent/interrupt-handler.d.ts +1 -0
- package/dist/agent/interrupt-handler.d.ts.map +1 -1
- package/dist/agent/interrupt-handler.js +22 -2
- package/dist/agent/interrupt-handler.js.map +1 -1
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/agent/loop.js +2 -1
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/openai-file-session.d.ts +19 -0
- package/dist/agent/openai-file-session.d.ts.map +1 -0
- package/dist/agent/openai-file-session.js +78 -0
- package/dist/agent/openai-file-session.js.map +1 -0
- package/dist/agent/openai-session.d.ts +50 -0
- package/dist/agent/openai-session.d.ts.map +1 -0
- package/dist/agent/openai-session.js +413 -0
- package/dist/agent/openai-session.js.map +1 -0
- package/dist/agent/openai-tools.d.ts +11 -0
- package/dist/agent/openai-tools.d.ts.map +1 -0
- package/dist/agent/openai-tools.js +167 -0
- package/dist/agent/openai-tools.js.map +1 -0
- package/dist/agent/providers/claude/session.d.ts +134 -0
- package/dist/agent/providers/claude/session.d.ts.map +1 -0
- package/dist/agent/providers/claude/session.js +525 -0
- package/dist/agent/providers/claude/session.js.map +1 -0
- package/dist/agent/providers/client-factory.d.ts +41 -0
- package/dist/agent/providers/client-factory.d.ts.map +1 -0
- package/dist/agent/providers/client-factory.js +190 -0
- package/dist/agent/providers/client-factory.js.map +1 -0
- package/dist/agent/providers/openai/file-session.d.ts +19 -0
- package/dist/agent/providers/openai/file-session.d.ts.map +1 -0
- package/dist/agent/providers/openai/file-session.js +78 -0
- package/dist/agent/providers/openai/file-session.js.map +1 -0
- package/dist/agent/providers/openai/session.d.ts +47 -0
- package/dist/agent/providers/openai/session.d.ts.map +1 -0
- package/dist/agent/providers/openai/session.js +414 -0
- package/dist/agent/providers/openai/session.js.map +1 -0
- package/dist/agent/providers/openai/tools.d.ts +13 -0
- package/dist/agent/providers/openai/tools.d.ts.map +1 -0
- package/dist/agent/providers/openai/tools.js +208 -0
- package/dist/agent/providers/openai/tools.js.map +1 -0
- package/dist/agent/providers/session-types.d.ts +123 -0
- package/dist/agent/providers/session-types.d.ts.map +1 -0
- package/dist/agent/providers/session-types.js +2 -0
- package/dist/agent/providers/session-types.js.map +1 -0
- package/dist/agent/runtime-surface.d.ts +23 -0
- package/dist/agent/runtime-surface.d.ts.map +1 -0
- package/dist/agent/runtime-surface.js +48 -0
- package/dist/agent/runtime-surface.js.map +1 -0
- package/dist/agent/session-types.d.ts +111 -0
- package/dist/agent/session-types.d.ts.map +1 -0
- package/dist/agent/session-types.js +2 -0
- package/dist/agent/session-types.js.map +1 -0
- package/dist/agent/status.d.ts.map +1 -1
- package/dist/agent/status.js +0 -7
- package/dist/agent/status.js.map +1 -1
- package/dist/agent/system-prompt.js +1 -1
- package/dist/agent/system-prompt.js.map +1 -1
- package/dist/builtin-skills/macos-automation/SKILL.md +43 -26
- package/dist/builtin-skills/visionclaw-manual/SKILL.md +3 -3
- package/dist/calendar/google-calendar.d.ts.map +1 -1
- package/dist/calendar/google-calendar.js +3 -1
- package/dist/calendar/google-calendar.js.map +1 -1
- package/dist/channels/interface.d.ts +0 -10
- package/dist/channels/interface.d.ts.map +1 -1
- package/dist/channels/manager.d.ts.map +1 -1
- package/dist/channels/manager.js +0 -3
- package/dist/channels/manager.js.map +1 -1
- package/dist/channels/telegram.d.ts.map +1 -1
- package/dist/channels/telegram.js +0 -13
- package/dist/channels/telegram.js.map +1 -1
- package/dist/config/types.d.ts +5 -5
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/types.js +15 -6
- package/dist/config/types.js.map +1 -1
- package/dist/drive/google-drive.d.ts.map +1 -1
- package/dist/drive/google-drive.js +3 -1
- package/dist/drive/google-drive.js.map +1 -1
- package/dist/email/gmail-utils.d.ts.map +1 -1
- package/dist/email/gmail-utils.js +3 -1
- package/dist/email/gmail-utils.js.map +1 -1
- package/dist/google/default-oauth-app.d.ts +6 -0
- package/dist/google/default-oauth-app.d.ts.map +1 -0
- package/dist/google/default-oauth-app.js +7 -0
- package/dist/google/default-oauth-app.js.map +1 -0
- package/dist/google/oauth-credentials.d.ts +10 -0
- package/dist/google/oauth-credentials.d.ts.map +1 -0
- package/dist/google/oauth-credentials.js +39 -0
- package/dist/google/oauth-credentials.js.map +1 -0
- package/dist/index.js +19 -3
- package/dist/index.js.map +1 -1
- package/dist/obs/server.js +1 -1
- package/dist/obs/server.js.map +1 -1
- package/dist/obs/tunnel.d.ts.map +1 -1
- package/dist/obs/tunnel.js +18 -2
- package/dist/obs/tunnel.js.map +1 -1
- package/dist/onboarding/google-auth.d.ts +2 -2
- package/dist/onboarding/google-auth.d.ts.map +1 -1
- package/dist/onboarding/google-auth.js +3 -1
- package/dist/onboarding/google-auth.js.map +1 -1
- package/dist/onboarding/google-cloud-setup.d.ts.map +1 -1
- package/dist/onboarding/google-cloud-setup.js +25 -6
- package/dist/onboarding/google-cloud-setup.js.map +1 -1
- package/dist/onboarding/index.d.ts.map +1 -1
- package/dist/onboarding/index.js +70 -21
- package/dist/onboarding/index.js.map +1 -1
- package/dist/onboarding/onboarding-session.d.ts +1 -0
- package/dist/onboarding/onboarding-session.d.ts.map +1 -1
- package/dist/onboarding/onboarding-session.js +2 -3
- package/dist/onboarding/onboarding-session.js.map +1 -1
- package/dist/onboarding/onboarding-tools.d.ts +0 -1
- package/dist/onboarding/onboarding-tools.d.ts.map +1 -1
- package/dist/onboarding/onboarding-tools.js +0 -6
- package/dist/onboarding/onboarding-tools.js.map +1 -1
- package/dist/onboarding/prepare-windows.d.ts +9 -0
- package/dist/onboarding/prepare-windows.d.ts.map +1 -0
- package/dist/onboarding/prepare-windows.js +250 -0
- package/dist/onboarding/prepare-windows.js.map +1 -0
- package/dist/onboarding/set-owner.js +1 -1
- package/dist/onboarding/set-owner.js.map +1 -1
- package/dist/onboarding/telegram-onboarding.d.ts.map +1 -1
- package/dist/onboarding/telegram-onboarding.js +0 -1
- package/dist/onboarding/telegram-onboarding.js.map +1 -1
- package/dist/onboarding/windows-permissions.d.ts +20 -0
- package/dist/onboarding/windows-permissions.d.ts.map +1 -0
- package/dist/onboarding/windows-permissions.js +195 -0
- package/dist/onboarding/windows-permissions.js.map +1 -0
- package/dist/reconfigure.d.ts.map +1 -1
- package/dist/reconfigure.js +40 -9
- package/dist/reconfigure.js.map +1 -1
- package/dist/tools/computer-use.d.ts +3 -0
- package/dist/tools/computer-use.d.ts.map +1 -1
- package/dist/tools/computer-use.js +43 -11
- package/dist/tools/computer-use.js.map +1 -1
- package/dist/tools/desktop-executor-factory.d.ts +20 -0
- package/dist/tools/desktop-executor-factory.d.ts.map +1 -0
- package/dist/tools/desktop-executor-factory.js +56 -0
- package/dist/tools/desktop-executor-factory.js.map +1 -0
- package/dist/tools/desktop-executor-windows.d.ts +42 -0
- package/dist/tools/desktop-executor-windows.d.ts.map +1 -0
- package/dist/tools/desktop-executor-windows.js +359 -0
- package/dist/tools/desktop-executor-windows.js.map +1 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +2 -1
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/screenshot.d.ts.map +1 -1
- package/dist/tools/screenshot.js +101 -17
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/web-fetch.d.ts +5 -0
- package/dist/tools/web-fetch.d.ts.map +1 -0
- package/dist/tools/web-fetch.js +175 -0
- package/dist/tools/web-fetch.js.map +1 -0
- package/dist/utils/restart.d.ts.map +1 -1
- package/dist/utils/restart.js +7 -2
- package/dist/utils/restart.js.map +1 -1
- package/dist/utils/transcribe.d.ts.map +1 -1
- package/dist/utils/transcribe.js +11 -5
- package/dist/utils/transcribe.js.map +1 -1
- package/dist/utils/version-check.d.ts.map +1 -1
- package/dist/utils/version-check.js +2 -1
- package/dist/utils/version-check.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Low-level Windows desktop executor using PowerShell + .NET P/Invoke.
|
|
3
|
+
* Used by computer_use MCP tools to perform mouse, keyboard, and screenshot actions.
|
|
4
|
+
*
|
|
5
|
+
* All input injection is done through a single reusable PowerShell helper script
|
|
6
|
+
* that accepts JSON commands on stdin. The script uses Add-Type to import user32.dll
|
|
7
|
+
* P/Invoke signatures for SetCursorPos, mouse_event, and keybd_event.
|
|
8
|
+
*/
|
|
9
|
+
/** Capture the screen. Returns base64 PNG and the saved file path. */
|
|
10
|
+
export declare function captureScreen(): Promise<{
|
|
11
|
+
base64: string;
|
|
12
|
+
filePath: string;
|
|
13
|
+
}>;
|
|
14
|
+
export declare function initDisplaySize(): Promise<void>;
|
|
15
|
+
export declare function getDisplaySize(): {
|
|
16
|
+
width: number;
|
|
17
|
+
height: number;
|
|
18
|
+
};
|
|
19
|
+
export declare function click(x: number, y: number): Promise<void>;
|
|
20
|
+
export declare function doubleClick(x: number, y: number): Promise<void>;
|
|
21
|
+
export declare function rightClick(x: number, y: number): Promise<void>;
|
|
22
|
+
export declare function moveTo(x: number, y: number): Promise<void>;
|
|
23
|
+
export declare function drag(x1: number, y1: number, x2: number, y2: number): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Type text into the currently focused element.
|
|
26
|
+
*
|
|
27
|
+
* Uses the clipboard (Set-Clipboard + Ctrl-V) for speed and full Unicode support.
|
|
28
|
+
* The previous clipboard content is saved and restored (text only — non-text
|
|
29
|
+
* clipboard data such as images cannot be preserved and will be logged as a warning).
|
|
30
|
+
*/
|
|
31
|
+
export declare function typeText(text: string): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Press a key or key combination (e.g. "enter", "escape", "cmd+s", "ctrl+shift+t").
|
|
34
|
+
* Modifiers cmd/meta are mapped to Ctrl on Windows.
|
|
35
|
+
*/
|
|
36
|
+
export declare function pressKey(key: string): Promise<void>;
|
|
37
|
+
/**
|
|
38
|
+
* Scroll at (x, y) in direction.
|
|
39
|
+
* Uses mouse_event with MOUSEEVENTF_WHEEL via the helper script.
|
|
40
|
+
*/
|
|
41
|
+
export declare function scroll(x: number, y: number, direction: "up" | "down" | "left" | "right", amount?: number): Promise<void>;
|
|
42
|
+
//# sourceMappingURL=desktop-executor-windows.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"desktop-executor-windows.d.ts","sourceRoot":"","sources":["../../src/tools/desktop-executor-windows.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAeH,sEAAsE;AACtE,wBAAsB,aAAa,IAAI,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAEnF;AAKD,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErD;AAED,wBAAgB,cAAc,IAAI;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAElE;AAyFD,wBAAsB,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/D;AAED,wBAAsB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAED,wBAAsB,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED,wBAAsB,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhE;AAED,wBAAsB,IAAI,CACxB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC,CAQf;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAwCD;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBzD;AAED;;;GAGG;AACH,wBAAsB,MAAM,CAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAC3C,MAAM,SAAI,GACT,OAAO,CAAC,IAAI,CAAC,CAef"}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Low-level Windows desktop executor using PowerShell + .NET P/Invoke.
|
|
3
|
+
* Used by computer_use MCP tools to perform mouse, keyboard, and screenshot actions.
|
|
4
|
+
*
|
|
5
|
+
* All input injection is done through a single reusable PowerShell helper script
|
|
6
|
+
* that accepts JSON commands on stdin. The script uses Add-Type to import user32.dll
|
|
7
|
+
* P/Invoke signatures for SetCursorPos, mouse_event, and keybd_event.
|
|
8
|
+
*/
|
|
9
|
+
import { exec } from "node:child_process";
|
|
10
|
+
import { promisify } from "node:util";
|
|
11
|
+
import fs from "node:fs";
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import os from "node:os";
|
|
14
|
+
import { takeScreenshot } from "./screenshot.js";
|
|
15
|
+
import { logger } from "../logger.js";
|
|
16
|
+
import { getConfigDir } from "../config/index.js";
|
|
17
|
+
const execAsync = promisify(exec);
|
|
18
|
+
const HELPER_VERSION = "3";
|
|
19
|
+
/** Capture the screen. Returns base64 PNG and the saved file path. */
|
|
20
|
+
export async function captureScreen() {
|
|
21
|
+
return takeScreenshot();
|
|
22
|
+
}
|
|
23
|
+
/** Cached display size, set once at agent startup. */
|
|
24
|
+
let cachedDisplaySize = null;
|
|
25
|
+
export async function initDisplaySize() {
|
|
26
|
+
cachedDisplaySize = await fetchDisplaySize();
|
|
27
|
+
}
|
|
28
|
+
export function getDisplaySize() {
|
|
29
|
+
return cachedDisplaySize ?? { width: 1920, height: 1080 };
|
|
30
|
+
}
|
|
31
|
+
async function fetchDisplaySize() {
|
|
32
|
+
const fallback = { width: 1920, height: 1080 };
|
|
33
|
+
try {
|
|
34
|
+
const ps1 = [
|
|
35
|
+
`$pinvoke = @'`,
|
|
36
|
+
`using System;`,
|
|
37
|
+
`using System.Runtime.InteropServices;`,
|
|
38
|
+
`public class DpiHelper {`,
|
|
39
|
+
` [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();`,
|
|
40
|
+
` [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);`,
|
|
41
|
+
`}`,
|
|
42
|
+
`'@`,
|
|
43
|
+
`Add-Type -TypeDefinition $pinvoke -Language CSharp`,
|
|
44
|
+
`[DpiHelper]::SetProcessDPIAware() | Out-Null`,
|
|
45
|
+
`$w = [DpiHelper]::GetSystemMetrics(0)`,
|
|
46
|
+
`$h = [DpiHelper]::GetSystemMetrics(1)`,
|
|
47
|
+
`Write-Output "$($w)x$($h)"`,
|
|
48
|
+
].join("\n");
|
|
49
|
+
const ps1File = path.join(os.tmpdir(), `visionclaw-displaysize-${Date.now()}.ps1`);
|
|
50
|
+
fs.writeFileSync(ps1File, ps1, "utf-8");
|
|
51
|
+
try {
|
|
52
|
+
const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${ps1File}"`, { timeout: 15_000 });
|
|
53
|
+
const m = /^(\d+)x(\d+)/.exec(stdout.trim());
|
|
54
|
+
if (m?.[1] && m[2]) {
|
|
55
|
+
const size = { width: parseInt(m[1], 10), height: parseInt(m[2], 10) };
|
|
56
|
+
logger.debug(`getDisplaySize: ${size.width}x${size.height}`, size);
|
|
57
|
+
return size;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
try {
|
|
62
|
+
fs.unlinkSync(ps1File);
|
|
63
|
+
}
|
|
64
|
+
catch { /* noop */ }
|
|
65
|
+
}
|
|
66
|
+
return fallback;
|
|
67
|
+
}
|
|
68
|
+
catch (e) {
|
|
69
|
+
logger.debug(`getDisplaySize: failed - ${e instanceof Error ? e.message : String(e)}`);
|
|
70
|
+
return fallback;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// ── PowerShell helper script management ─────────────────────────────────
|
|
74
|
+
function getHelperPath() {
|
|
75
|
+
return path.join(getConfigDir(), "visionclaw-input-helper.ps1");
|
|
76
|
+
}
|
|
77
|
+
function helperNeedsRegeneration(helperPath) {
|
|
78
|
+
if (!fs.existsSync(helperPath))
|
|
79
|
+
return true;
|
|
80
|
+
try {
|
|
81
|
+
const content = fs.readFileSync(helperPath, "utf-8");
|
|
82
|
+
return !content.includes(`# HELPER_VERSION=${HELPER_VERSION}`);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function ensureHelper() {
|
|
89
|
+
const helperPath = getHelperPath();
|
|
90
|
+
if (!helperNeedsRegeneration(helperPath))
|
|
91
|
+
return helperPath;
|
|
92
|
+
const dir = path.dirname(helperPath);
|
|
93
|
+
if (!fs.existsSync(dir)) {
|
|
94
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
95
|
+
}
|
|
96
|
+
fs.writeFileSync(helperPath, HELPER_SCRIPT, "utf-8");
|
|
97
|
+
logger.debug(`Windows input helper written to ${helperPath}`);
|
|
98
|
+
return helperPath;
|
|
99
|
+
}
|
|
100
|
+
async function runHelper(command) {
|
|
101
|
+
const helperPath = ensureHelper();
|
|
102
|
+
const json = JSON.stringify(command);
|
|
103
|
+
const commandBase64 = Buffer.from(json, "utf8").toString("base64");
|
|
104
|
+
const escapedBase64 = commandBase64.replace(/"/g, '\\"');
|
|
105
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "${helperPath}" -CommandBase64 "${escapedBase64}"`, { timeout: 15_000 });
|
|
106
|
+
return stdout.trim();
|
|
107
|
+
}
|
|
108
|
+
// ── Desktop automation functions ────────────────────────────────────────
|
|
109
|
+
export async function click(x, y) {
|
|
110
|
+
await runHelper({ action: "click", x: Math.round(x), y: Math.round(y) });
|
|
111
|
+
}
|
|
112
|
+
export async function doubleClick(x, y) {
|
|
113
|
+
await runHelper({ action: "doubleClick", x: Math.round(x), y: Math.round(y) });
|
|
114
|
+
}
|
|
115
|
+
export async function rightClick(x, y) {
|
|
116
|
+
await runHelper({ action: "rightClick", x: Math.round(x), y: Math.round(y) });
|
|
117
|
+
}
|
|
118
|
+
export async function moveTo(x, y) {
|
|
119
|
+
await runHelper({ action: "moveTo", x: Math.round(x), y: Math.round(y) });
|
|
120
|
+
}
|
|
121
|
+
export async function drag(x1, y1, x2, y2) {
|
|
122
|
+
await runHelper({
|
|
123
|
+
action: "drag",
|
|
124
|
+
x1: Math.round(x1),
|
|
125
|
+
y1: Math.round(y1),
|
|
126
|
+
x2: Math.round(x2),
|
|
127
|
+
y2: Math.round(y2),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Type text into the currently focused element.
|
|
132
|
+
*
|
|
133
|
+
* Uses the clipboard (Set-Clipboard + Ctrl-V) for speed and full Unicode support.
|
|
134
|
+
* The previous clipboard content is saved and restored (text only — non-text
|
|
135
|
+
* clipboard data such as images cannot be preserved and will be logged as a warning).
|
|
136
|
+
*/
|
|
137
|
+
export async function typeText(text) {
|
|
138
|
+
await runHelper({ action: "typeText", text });
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Windows virtual key codes for common special keys.
|
|
142
|
+
* Maps key names (lowercase) to VK_* hex values.
|
|
143
|
+
*/
|
|
144
|
+
const WIN_KEY_CODES = {
|
|
145
|
+
enter: 0x0D,
|
|
146
|
+
return: 0x0D,
|
|
147
|
+
tab: 0x09,
|
|
148
|
+
escape: 0x1B,
|
|
149
|
+
esc: 0x1B,
|
|
150
|
+
space: 0x20,
|
|
151
|
+
backspace: 0x08,
|
|
152
|
+
delete: 0x2E,
|
|
153
|
+
up: 0x26,
|
|
154
|
+
down: 0x28,
|
|
155
|
+
left: 0x25,
|
|
156
|
+
right: 0x27,
|
|
157
|
+
home: 0x24,
|
|
158
|
+
end: 0x23,
|
|
159
|
+
pageup: 0x21,
|
|
160
|
+
pagedown: 0x22,
|
|
161
|
+
f1: 0x70, f2: 0x71, f3: 0x72, f4: 0x73, f5: 0x74, f6: 0x75,
|
|
162
|
+
f7: 0x76, f8: 0x77, f9: 0x78, f10: 0x79, f11: 0x7A, f12: 0x7B,
|
|
163
|
+
};
|
|
164
|
+
/**
|
|
165
|
+
* Windows modifier key VK codes.
|
|
166
|
+
* cmd/meta map to Ctrl on Windows (macOS Cmd+key == Windows Ctrl+key).
|
|
167
|
+
*/
|
|
168
|
+
const WIN_MODIFIER_CODES = {
|
|
169
|
+
cmd: 0x11, // VK_CONTROL
|
|
170
|
+
meta: 0x11, // VK_CONTROL
|
|
171
|
+
ctrl: 0x11, // VK_CONTROL
|
|
172
|
+
alt: 0x12, // VK_MENU
|
|
173
|
+
option: 0x12, // VK_MENU
|
|
174
|
+
shift: 0x10, // VK_SHIFT
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* Press a key or key combination (e.g. "enter", "escape", "cmd+s", "ctrl+shift+t").
|
|
178
|
+
* Modifiers cmd/meta are mapped to Ctrl on Windows.
|
|
179
|
+
*/
|
|
180
|
+
export async function pressKey(key) {
|
|
181
|
+
const parts = key.toLowerCase().split("+");
|
|
182
|
+
const keyName = parts[parts.length - 1] ?? key;
|
|
183
|
+
const modParts = parts.slice(0, -1);
|
|
184
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
185
|
+
const modifiers = modParts
|
|
186
|
+
.map((m) => WIN_MODIFIER_CODES[m])
|
|
187
|
+
.filter((v) => v !== undefined);
|
|
188
|
+
const vk = WIN_KEY_CODES[keyName];
|
|
189
|
+
await runHelper({
|
|
190
|
+
action: "pressKey",
|
|
191
|
+
keyName,
|
|
192
|
+
vk: vk ?? null,
|
|
193
|
+
modifiers,
|
|
194
|
+
isSingleChar: vk === undefined && keyName.length === 1,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Scroll at (x, y) in direction.
|
|
199
|
+
* Uses mouse_event with MOUSEEVENTF_WHEEL via the helper script.
|
|
200
|
+
*/
|
|
201
|
+
export async function scroll(x, y, direction, amount = 3) {
|
|
202
|
+
await moveTo(x, y);
|
|
203
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
204
|
+
// WHEEL_DELTA = 120 per notch. Positive = up, negative = down.
|
|
205
|
+
const vertical = direction === "up" ? amount * 120 : direction === "down" ? -(amount * 120) : 0;
|
|
206
|
+
const horizontal = direction === "left" ? amount * 120 : direction === "right" ? -(amount * 120) : 0;
|
|
207
|
+
await runHelper({
|
|
208
|
+
action: "scroll",
|
|
209
|
+
vertical,
|
|
210
|
+
horizontal,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
// ── PowerShell helper script content ────────────────────────────────────
|
|
214
|
+
const HELPER_SCRIPT = `# HELPER_VERSION=${HELPER_VERSION}
|
|
215
|
+
# VisionClaw Windows Input Helper
|
|
216
|
+
# Accepts a UTF-8 JSON command encoded as base64 and performs the requested action.
|
|
217
|
+
|
|
218
|
+
param(
|
|
219
|
+
[string]$CommandBase64 = ""
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
223
|
+
|
|
224
|
+
$pinvoke = @"
|
|
225
|
+
using System;
|
|
226
|
+
using System.Runtime.InteropServices;
|
|
227
|
+
public class InputHelper {
|
|
228
|
+
[DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
|
|
229
|
+
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
|
230
|
+
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, int dwData, IntPtr dwExtraInfo);
|
|
231
|
+
[DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, IntPtr dwExtraInfo);
|
|
232
|
+
[DllImport("user32.dll")] public static extern short VkKeyScan(char ch);
|
|
233
|
+
|
|
234
|
+
public const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
|
|
235
|
+
public const uint MOUSEEVENTF_LEFTUP = 0x0004;
|
|
236
|
+
public const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
|
|
237
|
+
public const uint MOUSEEVENTF_RIGHTUP = 0x0010;
|
|
238
|
+
public const uint MOUSEEVENTF_WHEEL = 0x0800;
|
|
239
|
+
public const uint MOUSEEVENTF_HWHEEL = 0x1000;
|
|
240
|
+
public const uint KEYEVENTF_KEYUP = 0x0002;
|
|
241
|
+
}
|
|
242
|
+
"@
|
|
243
|
+
Add-Type -TypeDefinition $pinvoke -Language CSharp
|
|
244
|
+
[InputHelper]::SetProcessDPIAware() | Out-Null
|
|
245
|
+
|
|
246
|
+
$json = if ($CommandBase64) {
|
|
247
|
+
[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($CommandBase64))
|
|
248
|
+
} else {
|
|
249
|
+
# Backward-compat fallback for older callers.
|
|
250
|
+
[Console]::In.ReadToEnd()
|
|
251
|
+
}
|
|
252
|
+
$cmd = $json | ConvertFrom-Json
|
|
253
|
+
|
|
254
|
+
function Move-CursorTo([int]$x, [int]$y) {
|
|
255
|
+
[InputHelper]::SetCursorPos($x, $y) | Out-Null
|
|
256
|
+
Start-Sleep -Milliseconds 10
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function Send-Click([int]$x, [int]$y) {
|
|
260
|
+
Move-CursorTo $x $y
|
|
261
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
262
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
switch ($cmd.action) {
|
|
266
|
+
"click" {
|
|
267
|
+
Move-CursorTo $cmd.x $cmd.y
|
|
268
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
269
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
270
|
+
}
|
|
271
|
+
"doubleClick" {
|
|
272
|
+
Move-CursorTo $cmd.x $cmd.y
|
|
273
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
274
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
275
|
+
Start-Sleep -Milliseconds 50
|
|
276
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
277
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
278
|
+
}
|
|
279
|
+
"rightClick" {
|
|
280
|
+
Move-CursorTo $cmd.x $cmd.y
|
|
281
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_RIGHTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
282
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_RIGHTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
283
|
+
}
|
|
284
|
+
"moveTo" {
|
|
285
|
+
Move-CursorTo $cmd.x $cmd.y
|
|
286
|
+
}
|
|
287
|
+
"drag" {
|
|
288
|
+
Move-CursorTo $cmd.x1 $cmd.y1
|
|
289
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, [IntPtr]::Zero)
|
|
290
|
+
Start-Sleep -Milliseconds 100
|
|
291
|
+
Move-CursorTo $cmd.x2 $cmd.y2
|
|
292
|
+
Start-Sleep -Milliseconds 100
|
|
293
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_LEFTUP, 0, 0, 0, [IntPtr]::Zero)
|
|
294
|
+
}
|
|
295
|
+
"typeText" {
|
|
296
|
+
# Save clipboard, paste text, restore clipboard
|
|
297
|
+
$prevClip = $null
|
|
298
|
+
try { $prevClip = Get-Clipboard -Raw -ErrorAction SilentlyContinue } catch {}
|
|
299
|
+
|
|
300
|
+
Set-Clipboard -Value $cmd.text
|
|
301
|
+
Start-Sleep -Milliseconds 50
|
|
302
|
+
[InputHelper]::keybd_event(0x11, 0, 0, [IntPtr]::Zero) # Ctrl down
|
|
303
|
+
[InputHelper]::keybd_event(0x56, 0, 0, [IntPtr]::Zero) # V down
|
|
304
|
+
[InputHelper]::keybd_event(0x56, 0, [InputHelper]::KEYEVENTF_KEYUP, [IntPtr]::Zero) # V up
|
|
305
|
+
[InputHelper]::keybd_event(0x11, 0, [InputHelper]::KEYEVENTF_KEYUP, [IntPtr]::Zero) # Ctrl up
|
|
306
|
+
Start-Sleep -Milliseconds 50
|
|
307
|
+
|
|
308
|
+
# Restore previous clipboard
|
|
309
|
+
if ($null -ne $prevClip) {
|
|
310
|
+
try { Set-Clipboard -Value $prevClip } catch {}
|
|
311
|
+
} else {
|
|
312
|
+
try { Set-Clipboard -Value $null } catch {}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
"pressKey" {
|
|
316
|
+
# Press modifiers down
|
|
317
|
+
foreach ($mod in $cmd.modifiers) {
|
|
318
|
+
[InputHelper]::keybd_event([byte]$mod, 0, 0, [IntPtr]::Zero)
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if ($null -ne $cmd.vk) {
|
|
322
|
+
# Known special key
|
|
323
|
+
[InputHelper]::keybd_event([byte]$cmd.vk, 0, 0, [IntPtr]::Zero)
|
|
324
|
+
[InputHelper]::keybd_event([byte]$cmd.vk, 0, [InputHelper]::KEYEVENTF_KEYUP, [IntPtr]::Zero)
|
|
325
|
+
} elseif ($cmd.isSingleChar) {
|
|
326
|
+
# Single character — use VkKeyScan to get correct VK code
|
|
327
|
+
$vk = [InputHelper]::VkKeyScan([char]$cmd.keyName)
|
|
328
|
+
$lo = $vk -band 0xFF
|
|
329
|
+
if ($lo -ne 0xFF) {
|
|
330
|
+
[InputHelper]::keybd_event([byte]$lo, 0, 0, [IntPtr]::Zero)
|
|
331
|
+
[InputHelper]::keybd_event([byte]$lo, 0, [InputHelper]::KEYEVENTF_KEYUP, [IntPtr]::Zero)
|
|
332
|
+
} else {
|
|
333
|
+
[System.Windows.Forms.SendKeys]::SendWait($cmd.keyName)
|
|
334
|
+
}
|
|
335
|
+
} else {
|
|
336
|
+
# Unknown key — fallback to SendKeys
|
|
337
|
+
[System.Windows.Forms.SendKeys]::SendWait($cmd.keyName)
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
# Release modifiers
|
|
341
|
+
$reversed = @($cmd.modifiers)
|
|
342
|
+
[Array]::Reverse($reversed)
|
|
343
|
+
foreach ($mod in $reversed) {
|
|
344
|
+
[InputHelper]::keybd_event([byte]$mod, 0, [InputHelper]::KEYEVENTF_KEYUP, [IntPtr]::Zero)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
"scroll" {
|
|
348
|
+
if ($cmd.vertical -ne 0) {
|
|
349
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_WHEEL, 0, 0, $cmd.vertical, [IntPtr]::Zero)
|
|
350
|
+
}
|
|
351
|
+
if ($cmd.horizontal -ne 0) {
|
|
352
|
+
[InputHelper]::mouse_event([InputHelper]::MOUSEEVENTF_HWHEEL, 0, 0, $cmd.horizontal, [IntPtr]::Zero)
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
Write-Output "ok"
|
|
358
|
+
`;
|
|
359
|
+
//# sourceMappingURL=desktop-executor-windows.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"desktop-executor-windows.js","sourceRoot":"","sources":["../../src/tools/desktop-executor-windows.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;AAElC,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,cAAc,EAAE,CAAC;AAC1B,CAAC;AAED,sDAAsD;AACtD,IAAI,iBAAiB,GAA6C,IAAI,CAAC;AAEvE,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,iBAAiB,GAAG,MAAM,gBAAgB,EAAE,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,iBAAiB,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG;YACV,eAAe;YACf,eAAe;YACf,uCAAuC;YACvC,0BAA0B;YAC1B,6EAA6E;YAC7E,oFAAoF;YACpF,GAAG;YACH,IAAI;YACJ,oDAAoD;YACpD,8CAA8C;YAC9C,uCAAuC;YACvC,uCAAuC;YACvC,4BAA4B;SAC7B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,0BAA0B,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACnF,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAChC,wDAAwD,OAAO,GAAG,EAClE,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;YACF,MAAM,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;gBACnE,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,4BAA4B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACzE,CAAC;QACF,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,SAAS,aAAa;IACpB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,6BAA6B,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,uBAAuB,CAAC,UAAkB;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,cAAc,EAAE,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,CAAC,uBAAuB,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IAE5D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,CAAC,KAAK,CAAC,mCAAmC,UAAU,EAAE,CAAC,CAAC;IAC9D,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAgC;IACvD,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAChC,wEAAwE,UAAU,qBAAqB,aAAa,GAAG,EACvH,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;IACF,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED,2EAA2E;AAE3E,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,CAAS,EAAE,CAAS;IAC9C,MAAM,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,CAAS,EAAE,CAAS;IACpD,MAAM,SAAS,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,CAAS,EAAE,CAAS;IACnD,MAAM,SAAS,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,EAAU,EACV,EAAU,EACV,EAAU,EACV,EAAU;IAEV,MAAM,SAAS,CAAC;QACd,MAAM,EAAE,MAAM;QACd,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,MAAM,SAAS,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,aAAa,GAAoC;IACrD,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,IAAI;IACZ,GAAG,EAAE,IAAI;IACT,MAAM,EAAE,IAAI;IACZ,GAAG,EAAE,IAAI;IACT,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,MAAM,EAAE,IAAI;IACZ,EAAE,EAAE,IAAI;IACR,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,IAAI;IACV,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,IAAI;IACT,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,IAAI;IACd,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI;IAC1D,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI;CAC9D,CAAC;AAEF;;;GAGG;AACH,MAAM,kBAAkB,GAA2B;IACjD,GAAG,EAAE,IAAI,EAAK,aAAa;IAC3B,IAAI,EAAE,IAAI,EAAI,aAAa;IAC3B,IAAI,EAAE,IAAI,EAAI,aAAa;IAC3B,GAAG,EAAE,IAAI,EAAK,UAAU;IACxB,MAAM,EAAE,IAAI,EAAE,UAAU;IACxB,KAAK,EAAE,IAAI,EAAG,WAAW;CAC1B,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAW;IACxC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEpC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,QAAQ;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAuB,CAAC;SACvD,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAE/C,MAAM,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,SAAS,CAAC;QACd,MAAM,EAAE,UAAU;QAClB,OAAO;QACP,EAAE,EAAE,EAAE,IAAI,IAAI;QACd,SAAS;QACT,YAAY,EAAE,EAAE,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;KACvD,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,CAAS,EACT,CAAS,EACT,SAA2C,EAC3C,MAAM,GAAG,CAAC;IAEV,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAE7C,+DAA+D;IAC/D,MAAM,QAAQ,GACZ,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,UAAU,GACd,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpF,MAAM,SAAS,CAAC;QACd,MAAM,EAAE,QAAQ;QAChB,QAAQ;QACR,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,2EAA2E;AAE3E,MAAM,aAAa,GAAG,oBAAoB,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgJvD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAsBpE;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE;IACzC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GAAG,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAyBxC;AAED,MAAM,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,CAAC,EAAE;IAChD,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GAAG,iBAAiB,CAEpB"}
|
package/dist/tools/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import { memoryTool } from "./memory.js";
|
|
|
10
10
|
import { manageDriveTool } from "./drive.js";
|
|
11
11
|
import { switchSessionTool } from "./switch-session.js";
|
|
12
12
|
import { manageMcpServersTool } from "./mcp-manager.js";
|
|
13
|
-
import { computerUseClickTool, computerUseScrollTool, computerUseDragTool, computerUseTypeTool, computerUseKeyTool, computerUseScreenshotTool, } from "./computer-use.js";
|
|
13
|
+
import { computerUseClickTool, computerUseMoveTool, computerUseScrollTool, computerUseDragTool, computerUseTypeTool, computerUseKeyTool, computerUseScreenshotTool, } from "./computer-use.js";
|
|
14
14
|
/**
|
|
15
15
|
* Create the in-process MCP server that provides all
|
|
16
16
|
* VisionClaw system tools to the Claude Agent SDK session.
|
|
@@ -36,6 +36,7 @@ export function createToolServer(options) {
|
|
|
36
36
|
manageDriveTool,
|
|
37
37
|
memoryTool,
|
|
38
38
|
computerUseClickTool,
|
|
39
|
+
computerUseMoveTool,
|
|
39
40
|
computerUseScrollTool,
|
|
40
41
|
computerUseDragTool,
|
|
41
42
|
computerUseTypeTool,
|
package/dist/tools/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,mBAAmB,CAAC;AAE3B;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAEhC;IACC,OAAO,kBAAkB,CAAC;QACxB,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE;YACL,QAAQ;YACR,cAAc;YACd,UAAU;YACV,gBAAgB;YAChB,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,eAAe;YACf,UAAU;YACV,oBAAoB;YACpB,qBAAqB;YACrB,mBAAmB;YACnB,mBAAmB;YACnB,kBAAkB;YAClB,yBAAyB;YACzB,oBAAoB;YACpB,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D;KACF,CAAC,CAAC;AACL,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAEvC;IACC,OAAO,GAAG,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,mBAAmB,CAAC;AAE3B;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAEhC;IACC,OAAO,kBAAkB,CAAC;QACxB,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE;YACL,QAAQ;YACR,cAAc;YACd,UAAU;YACV,gBAAgB;YAChB,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,eAAe;YACf,UAAU;YACV,oBAAoB;YACpB,mBAAmB;YACnB,qBAAqB;YACrB,mBAAmB;YACnB,mBAAmB;YACnB,kBAAkB;YAClB,yBAAyB;YACzB,oBAAoB;YACpB,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D;KACF,CAAC,CAAC;AACL,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAEvC;IACC,OAAO,GAAG,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,iBAAiB,UAAY,CAAC;AAE3C;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAQ1E;
|
|
1
|
+
{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,iBAAiB,UAAY,CAAC;AAE3C;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAQ1E;AAuID;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAUnD;AAED;;GAEG;AACH,wBAAsB,0BAA0B,CAC9C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAgDnD;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBrE;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI3E;AAcD;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAmBtD;AAwCD;;;;GAIG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CA4BzD;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAmFhE"}
|
package/dist/tools/screenshot.js
CHANGED
|
@@ -25,15 +25,16 @@ export function getVisionScaleFactor(width, height) {
|
|
|
25
25
|
const totalPixelsScale = Math.sqrt(CLAUDE_MAX_PIXELS / totalPixels);
|
|
26
26
|
return Math.min(1.0, longEdgeScale, totalPixelsScale);
|
|
27
27
|
}
|
|
28
|
-
/** Resolve the absolute path to ffmpeg, checking common
|
|
28
|
+
/** Resolve the absolute path to ffmpeg, checking common system locations. */
|
|
29
29
|
let _ffmpegPath;
|
|
30
30
|
async function findFfmpeg() {
|
|
31
31
|
if (_ffmpegPath !== undefined)
|
|
32
32
|
return _ffmpegPath;
|
|
33
33
|
// Try PATH first
|
|
34
|
+
const whichCmd = process.platform === "win32" ? "where" : "which";
|
|
34
35
|
try {
|
|
35
|
-
const { stdout } = await execAsync(
|
|
36
|
-
const p = stdout.trim();
|
|
36
|
+
const { stdout } = await execAsync(`${whichCmd} ffmpeg`);
|
|
37
|
+
const p = stdout.trim().split(/\r?\n/)[0];
|
|
37
38
|
if (p) {
|
|
38
39
|
_ffmpegPath = p;
|
|
39
40
|
return p;
|
|
@@ -41,11 +42,17 @@ async function findFfmpeg() {
|
|
|
41
42
|
}
|
|
42
43
|
catch { /* not on PATH */ }
|
|
43
44
|
// Check well-known locations
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
const candidates = process.platform === "win32"
|
|
46
|
+
? [
|
|
47
|
+
"C:\\ProgramData\\chocolatey\\bin\\ffmpeg.exe",
|
|
48
|
+
"C:\\ffmpeg\\bin\\ffmpeg.exe",
|
|
49
|
+
]
|
|
50
|
+
: [
|
|
51
|
+
"/opt/homebrew/bin/ffmpeg",
|
|
52
|
+
"/usr/local/bin/ffmpeg",
|
|
53
|
+
"/usr/bin/ffmpeg",
|
|
54
|
+
];
|
|
55
|
+
for (const candidate of candidates) {
|
|
49
56
|
try {
|
|
50
57
|
fs.accessSync(candidate, fs.constants.X_OK);
|
|
51
58
|
_ffmpegPath = candidate;
|
|
@@ -95,8 +102,8 @@ async function compressWithFfmpeg(inputPath, outputPath, maxColors = 128) {
|
|
|
95
102
|
}
|
|
96
103
|
}
|
|
97
104
|
/**
|
|
98
|
-
* Resize a buffer using system tools (sips on macOS, ImageMagick on Linux
|
|
99
|
-
*
|
|
105
|
+
* Resize a buffer using system tools (sips on macOS, ImageMagick on Linux/Windows,
|
|
106
|
+
* PowerShell GDI+ on Windows as fallback).
|
|
100
107
|
* Returns the resized buffer, or null on failure.
|
|
101
108
|
*/
|
|
102
109
|
async function resizeWithSystemTools(buffer, targetW, targetH) {
|
|
@@ -107,6 +114,24 @@ async function resizeWithSystemTools(buffer, targetW, targetH) {
|
|
|
107
114
|
if (process.platform === "darwin") {
|
|
108
115
|
await execAsync(`sips -z ${targetH} ${targetW} "${tmpIn}" --out "${tmpOut}"`);
|
|
109
116
|
}
|
|
117
|
+
else if (process.platform === "win32") {
|
|
118
|
+
// Try ImageMagick first (uses 'magick' prefix on Windows)
|
|
119
|
+
try {
|
|
120
|
+
await execAsync(`magick convert "${tmpIn}" -resize ${targetW}x${targetH}! "${tmpOut}"`);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Fallback to PowerShell System.Drawing GDI+
|
|
124
|
+
await execAsync(`powershell -NoProfile -Command "` +
|
|
125
|
+
`Add-Type -AssemblyName System.Drawing; ` +
|
|
126
|
+
`$img = [System.Drawing.Image]::FromFile('${tmpIn}'); ` +
|
|
127
|
+
`$bmp = New-Object System.Drawing.Bitmap(${targetW}, ${targetH}); ` +
|
|
128
|
+
`$g = [System.Drawing.Graphics]::FromImage($bmp); ` +
|
|
129
|
+
`$g.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic; ` +
|
|
130
|
+
`$g.DrawImage($img, 0, 0, ${targetW}, ${targetH}); ` +
|
|
131
|
+
`$bmp.Save('${tmpOut}', [System.Drawing.Imaging.ImageFormat]::Png); ` +
|
|
132
|
+
`$g.Dispose(); $bmp.Dispose(); $img.Dispose()"`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
110
135
|
else {
|
|
111
136
|
await execAsync(`convert "${tmpIn}" -resize ${targetW}x${targetH}! "${tmpOut}"`);
|
|
112
137
|
}
|
|
@@ -165,6 +190,28 @@ export async function getImageDimensionsFromFile(filePath) {
|
|
|
165
190
|
return { width: parseInt(wMatch[1], 10), height: parseInt(hMatch[1], 10) };
|
|
166
191
|
}
|
|
167
192
|
}
|
|
193
|
+
else if (process.platform === "win32") {
|
|
194
|
+
// Try ImageMagick first
|
|
195
|
+
try {
|
|
196
|
+
const { stdout } = await execAsync(`magick identify -format "%w %h" "${filePath}"`);
|
|
197
|
+
const parts = stdout.trim().split(/\s+/);
|
|
198
|
+
if (parts.length >= 2) {
|
|
199
|
+
return { width: parseInt(parts[0], 10), height: parseInt(parts[1], 10) };
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
// Fallback to PowerShell System.Drawing
|
|
204
|
+
const { stdout } = await execAsync(`powershell -NoProfile -Command "` +
|
|
205
|
+
`Add-Type -AssemblyName System.Drawing; ` +
|
|
206
|
+
`$img = [System.Drawing.Image]::FromFile('${filePath}'); ` +
|
|
207
|
+
`Write-Output \\"$($img.Width) $($img.Height)\\"; ` +
|
|
208
|
+
`$img.Dispose()"`);
|
|
209
|
+
const parts = stdout.trim().split(/\s+/);
|
|
210
|
+
if (parts.length >= 2) {
|
|
211
|
+
return { width: parseInt(parts[0], 10), height: parseInt(parts[1], 10) };
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
168
215
|
else {
|
|
169
216
|
const { stdout } = await execAsync(`identify -format "%w %h" "${filePath}"`);
|
|
170
217
|
const parts = stdout.trim().split(/\s+/);
|
|
@@ -342,13 +389,50 @@ export async function takeScreenshot() {
|
|
|
342
389
|
await execAsync(`screencapture -x -C "${tmpFile}"`);
|
|
343
390
|
}
|
|
344
391
|
else if (process.platform === "win32") {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
`$
|
|
350
|
-
|
|
351
|
-
|
|
392
|
+
// Use DPI-aware capture + cursor overlay for complete screenshots on scaled displays
|
|
393
|
+
const ps1 = [
|
|
394
|
+
`Add-Type -AssemblyName System.Windows.Forms`,
|
|
395
|
+
`Add-Type -AssemblyName System.Drawing`,
|
|
396
|
+
`$pinvoke = @'`,
|
|
397
|
+
`using System;`,
|
|
398
|
+
`using System.Runtime.InteropServices;`,
|
|
399
|
+
`public class ScreenCapHelper {`,
|
|
400
|
+
` [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();`,
|
|
401
|
+
` [DllImport("user32.dll")] public static extern int GetSystemMetrics(int nIndex);`,
|
|
402
|
+
` [DllImport("user32.dll")] public static extern bool GetCursorInfo(ref CURSORINFO pci);`,
|
|
403
|
+
` [DllImport("user32.dll")] public static extern bool DrawIconEx(IntPtr hdc, int xLeft, int yTop, IntPtr hIcon, int cxWidth, int cyWidth, uint istepIfAniCur, IntPtr hbrFlickerFreeDraw, uint diFlags);`,
|
|
404
|
+
` [StructLayout(LayoutKind.Sequential)] public struct POINT { public int x; public int y; }`,
|
|
405
|
+
` [StructLayout(LayoutKind.Sequential)] public struct CURSORINFO { public int cbSize; public int flags; public IntPtr hCursor; public POINT ptScreenPos; }`,
|
|
406
|
+
`}`,
|
|
407
|
+
`'@`,
|
|
408
|
+
`Add-Type -TypeDefinition $pinvoke -Language CSharp`,
|
|
409
|
+
`[ScreenCapHelper]::SetProcessDPIAware() | Out-Null`,
|
|
410
|
+
`$w = [ScreenCapHelper]::GetSystemMetrics(0)`,
|
|
411
|
+
`$h = [ScreenCapHelper]::GetSystemMetrics(1)`,
|
|
412
|
+
`$bmp = New-Object System.Drawing.Bitmap($w, $h)`,
|
|
413
|
+
`$g = [System.Drawing.Graphics]::FromImage($bmp)`,
|
|
414
|
+
`$g.CopyFromScreen(0, 0, 0, 0, (New-Object System.Drawing.Size($w, $h)))`,
|
|
415
|
+
`$ci = New-Object ScreenCapHelper+CURSORINFO`,
|
|
416
|
+
`$ci.cbSize = [System.Runtime.InteropServices.Marshal]::SizeOf($ci)`,
|
|
417
|
+
`if ([ScreenCapHelper]::GetCursorInfo([ref]$ci) -and ($ci.flags -eq 1)) {`,
|
|
418
|
+
` $hdc = $g.GetHdc()`,
|
|
419
|
+
` [ScreenCapHelper]::DrawIconEx($hdc, $ci.ptScreenPos.x, $ci.ptScreenPos.y, $ci.hCursor, 0, 0, 0, [IntPtr]::Zero, 3) | Out-Null`,
|
|
420
|
+
` $g.ReleaseHdc($hdc)`,
|
|
421
|
+
`}`,
|
|
422
|
+
`$bmp.Save('${tmpFile}', [System.Drawing.Imaging.ImageFormat]::Png)`,
|
|
423
|
+
`$g.Dispose(); $bmp.Dispose()`,
|
|
424
|
+
].join("\n");
|
|
425
|
+
const ps1File = path.join(os.tmpdir(), `visionclaw-screenshot-${Date.now()}.ps1`);
|
|
426
|
+
await fsp.writeFile(ps1File, ps1, "utf-8");
|
|
427
|
+
try {
|
|
428
|
+
await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${ps1File}"`);
|
|
429
|
+
}
|
|
430
|
+
finally {
|
|
431
|
+
try {
|
|
432
|
+
await fsp.unlink(ps1File);
|
|
433
|
+
}
|
|
434
|
+
catch { /* noop */ }
|
|
435
|
+
}
|
|
352
436
|
}
|
|
353
437
|
else {
|
|
354
438
|
try {
|