screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env tsx
|
|
2
|
-
/**
|
|
3
|
-
* Helper script: Uses ScreenHand to navigate npm tokens page in Safari.
|
|
4
|
-
* Spawns the MCP server, starts a session, and automates Safari.
|
|
5
|
-
*/
|
|
6
|
-
import { spawn } from "node:child_process";
|
|
7
|
-
import { createInterface } from "node:readline";
|
|
8
|
-
import path from "node:path";
|
|
9
|
-
import { fileURLToPath } from "node:url";
|
|
10
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
|
-
const projectRoot = path.resolve(__dirname, "..");
|
|
12
|
-
const tsxBin = path.join(projectRoot, "node_modules", ".bin", "tsx");
|
|
13
|
-
const TIMEOUT_MS = 15_000;
|
|
14
|
-
const proc = spawn(tsxBin, [path.join(projectRoot, "src/mcp-entry.ts")], {
|
|
15
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
16
|
-
env: { ...process.env, SCREENHAND_ADAPTER: "accessibility" },
|
|
17
|
-
cwd: projectRoot,
|
|
18
|
-
});
|
|
19
|
-
let stderrBuf = "";
|
|
20
|
-
proc.stderr.on("data", (d) => {
|
|
21
|
-
stderrBuf += d.toString();
|
|
22
|
-
process.stderr.write(d);
|
|
23
|
-
});
|
|
24
|
-
let msgId = 0;
|
|
25
|
-
function send(msg) {
|
|
26
|
-
proc.stdin.write(JSON.stringify(msg) + "\n");
|
|
27
|
-
}
|
|
28
|
-
const rl = createInterface({ input: proc.stdout });
|
|
29
|
-
const lineQueue = [];
|
|
30
|
-
let lineWaiter = null;
|
|
31
|
-
rl.on("line", (line) => {
|
|
32
|
-
if (lineWaiter) {
|
|
33
|
-
const w = lineWaiter;
|
|
34
|
-
lineWaiter = null;
|
|
35
|
-
w(line);
|
|
36
|
-
}
|
|
37
|
-
else {
|
|
38
|
-
lineQueue.push(line);
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
function readResponse() {
|
|
42
|
-
return new Promise((resolve, reject) => {
|
|
43
|
-
const timer = setTimeout(() => {
|
|
44
|
-
lineWaiter = null;
|
|
45
|
-
reject(new Error(`Timeout. stderr: ${stderrBuf.slice(-500)}`));
|
|
46
|
-
}, TIMEOUT_MS);
|
|
47
|
-
const handle = (line) => {
|
|
48
|
-
clearTimeout(timer);
|
|
49
|
-
resolve(JSON.parse(line));
|
|
50
|
-
};
|
|
51
|
-
const queued = lineQueue.shift();
|
|
52
|
-
if (queued) {
|
|
53
|
-
clearTimeout(timer);
|
|
54
|
-
resolve(JSON.parse(queued));
|
|
55
|
-
}
|
|
56
|
-
else {
|
|
57
|
-
lineWaiter = handle;
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
async function callTool(name, args) {
|
|
62
|
-
const id = ++msgId;
|
|
63
|
-
send({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } });
|
|
64
|
-
const resp = await readResponse();
|
|
65
|
-
const result = resp.result;
|
|
66
|
-
const content = result?.content;
|
|
67
|
-
const data = content?.[0]?.text ? JSON.parse(content[0].text) : null;
|
|
68
|
-
if (result?.isError) {
|
|
69
|
-
console.error(` ERROR: ${content?.[0]?.text}`);
|
|
70
|
-
}
|
|
71
|
-
return data;
|
|
72
|
-
}
|
|
73
|
-
try {
|
|
74
|
-
// Wait for server to start
|
|
75
|
-
await new Promise((r) => setTimeout(r, 2000));
|
|
76
|
-
// Initialize MCP
|
|
77
|
-
send({
|
|
78
|
-
jsonrpc: "2.0", id: ++msgId, method: "initialize",
|
|
79
|
-
params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "npm-helper", version: "1.0" } },
|
|
80
|
-
});
|
|
81
|
-
await readResponse();
|
|
82
|
-
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
83
|
-
await new Promise((r) => setTimeout(r, 300));
|
|
84
|
-
// Start session
|
|
85
|
-
console.log("Starting session...");
|
|
86
|
-
const session = await callTool("session_start", {});
|
|
87
|
-
const sid = session.sessionId;
|
|
88
|
-
console.log(` Session: ${sid}`);
|
|
89
|
-
// Launch Safari
|
|
90
|
-
console.log("\nLaunching Safari...");
|
|
91
|
-
await callTool("app_launch", { sessionId: sid, bundleId: "com.apple.Safari" });
|
|
92
|
-
await new Promise((r) => setTimeout(r, 1500));
|
|
93
|
-
// Focus Safari
|
|
94
|
-
console.log("Focusing Safari...");
|
|
95
|
-
await callTool("app_focus", { sessionId: sid, bundleId: "com.apple.Safari" });
|
|
96
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
97
|
-
// Navigate to npm tokens page
|
|
98
|
-
console.log("Navigating to npm tokens page...");
|
|
99
|
-
await callTool("navigate", { sessionId: sid, url: "https://www.npmjs.com/settings/ai2hum/tokens" });
|
|
100
|
-
await new Promise((r) => setTimeout(r, 3000));
|
|
101
|
-
// Take screenshot to see the page
|
|
102
|
-
console.log("\nTaking screenshot...");
|
|
103
|
-
const screenshot = await callTool("screenshot", { sessionId: sid });
|
|
104
|
-
console.log(" Screenshot:", JSON.stringify(screenshot));
|
|
105
|
-
// Get element tree to see what's on the page
|
|
106
|
-
console.log("\nGetting element tree...");
|
|
107
|
-
const tree = await callTool("element_tree", { sessionId: sid, maxDepth: 4 });
|
|
108
|
-
console.log(" Tree:", JSON.stringify(tree, null, 2).slice(0, 3000));
|
|
109
|
-
console.log("\n--- Page loaded. Inspect the output above to determine next steps. ---");
|
|
110
|
-
proc.kill();
|
|
111
|
-
process.exit(0);
|
|
112
|
-
}
|
|
113
|
-
catch (e) {
|
|
114
|
-
console.error("Error:", e instanceof Error ? e.message : String(e));
|
|
115
|
-
proc.kill();
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
package/dist/npm-token-cdp.js
DELETED
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env tsx
|
|
2
|
-
/**
|
|
3
|
-
* Uses ScreenHand with CDP adapter to create npm token in Chrome.
|
|
4
|
-
*/
|
|
5
|
-
import { spawn } from "node:child_process";
|
|
6
|
-
import { createInterface } from "node:readline";
|
|
7
|
-
import path from "node:path";
|
|
8
|
-
import { fileURLToPath } from "node:url";
|
|
9
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
10
|
-
const projectRoot = path.resolve(__dirname, "..");
|
|
11
|
-
const tsxBin = path.join(projectRoot, "node_modules", ".bin", "tsx");
|
|
12
|
-
const proc = spawn(tsxBin, [path.join(projectRoot, "src/mcp-entry.ts")], {
|
|
13
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
14
|
-
env: { ...process.env, SCREENHAND_ADAPTER: "cdp" },
|
|
15
|
-
cwd: projectRoot,
|
|
16
|
-
});
|
|
17
|
-
proc.stderr.on("data", (d) => process.stderr.write(d));
|
|
18
|
-
let msgId = 0;
|
|
19
|
-
function send(msg) {
|
|
20
|
-
proc.stdin.write(JSON.stringify(msg) + "\n");
|
|
21
|
-
}
|
|
22
|
-
const rl = createInterface({ input: proc.stdout });
|
|
23
|
-
const lineQueue = [];
|
|
24
|
-
let lineWaiter = null;
|
|
25
|
-
rl.on("line", (line) => {
|
|
26
|
-
if (lineWaiter) {
|
|
27
|
-
const w = lineWaiter;
|
|
28
|
-
lineWaiter = null;
|
|
29
|
-
w(line);
|
|
30
|
-
}
|
|
31
|
-
else
|
|
32
|
-
lineQueue.push(line);
|
|
33
|
-
});
|
|
34
|
-
function readResponse() {
|
|
35
|
-
return new Promise((resolve, reject) => {
|
|
36
|
-
const timer = setTimeout(() => { lineWaiter = null; reject(new Error("Timeout")); }, 30000);
|
|
37
|
-
const handle = (line) => { clearTimeout(timer); resolve(JSON.parse(line)); };
|
|
38
|
-
const queued = lineQueue.shift();
|
|
39
|
-
if (queued) {
|
|
40
|
-
clearTimeout(timer);
|
|
41
|
-
resolve(JSON.parse(queued));
|
|
42
|
-
}
|
|
43
|
-
else
|
|
44
|
-
lineWaiter = handle;
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
async function callTool(name, args) {
|
|
48
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "tools/call", params: { name, arguments: args } });
|
|
49
|
-
const resp = await readResponse();
|
|
50
|
-
const result = resp.result;
|
|
51
|
-
const content = result?.content;
|
|
52
|
-
const text = content?.[0]?.text ?? "{}";
|
|
53
|
-
if (result?.isError) {
|
|
54
|
-
console.error(` ERROR: ${text}`);
|
|
55
|
-
return null;
|
|
56
|
-
}
|
|
57
|
-
try {
|
|
58
|
-
return JSON.parse(text);
|
|
59
|
-
}
|
|
60
|
-
catch {
|
|
61
|
-
return text;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
async function sleep(ms) { await new Promise(r => setTimeout(r, ms)); }
|
|
65
|
-
try {
|
|
66
|
-
await sleep(3000);
|
|
67
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "initialize",
|
|
68
|
-
params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "npm-cdp", version: "1.0" } } });
|
|
69
|
-
await readResponse();
|
|
70
|
-
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
71
|
-
await sleep(300);
|
|
72
|
-
const session = await callTool("session_start", {});
|
|
73
|
-
if (!session) {
|
|
74
|
-
console.error("Failed to start session");
|
|
75
|
-
process.exit(1);
|
|
76
|
-
}
|
|
77
|
-
const sid = session.sessionId;
|
|
78
|
-
console.log(`Session: ${sid}`);
|
|
79
|
-
// Navigate to npm tokens page
|
|
80
|
-
console.log("\nNavigating to npm tokens page...");
|
|
81
|
-
await callTool("navigate", { sessionId: sid, url: "https://www.npmjs.com/settings/ai2hum/tokens/new" });
|
|
82
|
-
await sleep(3000);
|
|
83
|
-
// Take screenshot
|
|
84
|
-
const ss1 = await callTool("screenshot", { sessionId: sid });
|
|
85
|
-
console.log("Screenshot:", JSON.stringify(ss1).slice(0, 200));
|
|
86
|
-
// Type token name
|
|
87
|
-
console.log("\nTyping token name...");
|
|
88
|
-
await callTool("type_into", { sessionId: sid, target: "css=#token-name", text: "screenhand-publish" });
|
|
89
|
-
await sleep(500);
|
|
90
|
-
// Check bypass 2FA
|
|
91
|
-
console.log("\nChecking bypass 2FA...");
|
|
92
|
-
await callTool("press", { sessionId: sid, target: "css=#bypass-2fa" });
|
|
93
|
-
await sleep(500);
|
|
94
|
-
// Set packages permission to "Read and write"
|
|
95
|
-
console.log("\nSetting packages permission...");
|
|
96
|
-
// Find the packages permissions select and change it
|
|
97
|
-
await callTool("press", { sessionId: sid, target: "css=select[name='packages-permission']" });
|
|
98
|
-
await sleep(300);
|
|
99
|
-
// Screenshot to see current state
|
|
100
|
-
const ss2 = await callTool("screenshot", { sessionId: sid });
|
|
101
|
-
console.log("Screenshot:", JSON.stringify(ss2).slice(0, 200));
|
|
102
|
-
// Try to get page HTML to understand the form structure
|
|
103
|
-
console.log("\nGetting page structure...");
|
|
104
|
-
const html = await callTool("extract", { sessionId: sid, target: "css=form", format: "text" });
|
|
105
|
-
console.log("Form text:", JSON.stringify(html).slice(0, 2000));
|
|
106
|
-
proc.kill();
|
|
107
|
-
process.exit(0);
|
|
108
|
-
}
|
|
109
|
-
catch (e) {
|
|
110
|
-
console.error("Error:", e instanceof Error ? e.message : String(e));
|
|
111
|
-
proc.kill();
|
|
112
|
-
process.exit(1);
|
|
113
|
-
}
|
package/dist/npm-token-create.js
DELETED
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env tsx
|
|
2
|
-
import { spawn } from "node:child_process";
|
|
3
|
-
import { createInterface } from "node:readline";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
import { fileURLToPath } from "node:url";
|
|
6
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
7
|
-
const projectRoot = path.resolve(__dirname, "..");
|
|
8
|
-
const tsxBin = path.join(projectRoot, "node_modules", ".bin", "tsx");
|
|
9
|
-
const proc = spawn(tsxBin, [path.join(projectRoot, "src/mcp-entry.ts")], {
|
|
10
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
11
|
-
env: { ...process.env, SCREENHAND_ADAPTER: "accessibility" },
|
|
12
|
-
cwd: projectRoot,
|
|
13
|
-
});
|
|
14
|
-
proc.stderr.on("data", (d) => process.stderr.write(d));
|
|
15
|
-
let msgId = 0;
|
|
16
|
-
function send(msg) {
|
|
17
|
-
proc.stdin.write(JSON.stringify(msg) + "\n");
|
|
18
|
-
}
|
|
19
|
-
const rl = createInterface({ input: proc.stdout });
|
|
20
|
-
const lineQueue = [];
|
|
21
|
-
let lineWaiter = null;
|
|
22
|
-
rl.on("line", (line) => {
|
|
23
|
-
if (lineWaiter) {
|
|
24
|
-
const w = lineWaiter;
|
|
25
|
-
lineWaiter = null;
|
|
26
|
-
w(line);
|
|
27
|
-
}
|
|
28
|
-
else
|
|
29
|
-
lineQueue.push(line);
|
|
30
|
-
});
|
|
31
|
-
function readResponse() {
|
|
32
|
-
return new Promise((resolve, reject) => {
|
|
33
|
-
const timer = setTimeout(() => { lineWaiter = null; reject(new Error("Timeout")); }, 15000);
|
|
34
|
-
const handle = (line) => { clearTimeout(timer); resolve(JSON.parse(line)); };
|
|
35
|
-
const queued = lineQueue.shift();
|
|
36
|
-
if (queued) {
|
|
37
|
-
clearTimeout(timer);
|
|
38
|
-
resolve(JSON.parse(queued));
|
|
39
|
-
}
|
|
40
|
-
else
|
|
41
|
-
lineWaiter = handle;
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
async function callTool(name, args) {
|
|
45
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "tools/call", params: { name, arguments: args } });
|
|
46
|
-
const resp = await readResponse();
|
|
47
|
-
const result = resp.result;
|
|
48
|
-
const content = result?.content;
|
|
49
|
-
const text = content?.[0]?.text ?? "{}";
|
|
50
|
-
if (result?.isError)
|
|
51
|
-
console.error(` ERROR: ${text}`);
|
|
52
|
-
try {
|
|
53
|
-
return JSON.parse(text);
|
|
54
|
-
}
|
|
55
|
-
catch {
|
|
56
|
-
return text;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
async function screenshot(sid) {
|
|
60
|
-
const result = await callTool("screenshot", { sessionId: sid });
|
|
61
|
-
const data = result.data;
|
|
62
|
-
return data?.path ?? "";
|
|
63
|
-
}
|
|
64
|
-
async function sleep(ms) { await new Promise(r => setTimeout(r, ms)); }
|
|
65
|
-
try {
|
|
66
|
-
await sleep(2000);
|
|
67
|
-
// Init MCP
|
|
68
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "initialize",
|
|
69
|
-
params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "npm-helper", version: "1.0" } } });
|
|
70
|
-
await readResponse();
|
|
71
|
-
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
72
|
-
await sleep(300);
|
|
73
|
-
// Start session
|
|
74
|
-
console.log("Starting session...");
|
|
75
|
-
const session = await callTool("session_start", {});
|
|
76
|
-
const sid = session.sessionId;
|
|
77
|
-
console.log(` Session: ${sid}`);
|
|
78
|
-
// Focus Safari (should already be on the tokens page)
|
|
79
|
-
console.log("\nFocusing Safari...");
|
|
80
|
-
await callTool("app_focus", { sessionId: sid, bundleId: "com.apple.Safari" });
|
|
81
|
-
await sleep(500);
|
|
82
|
-
// Step 1: Type token name
|
|
83
|
-
console.log("\nTyping token name...");
|
|
84
|
-
await callTool("press", { sessionId: sid, target: "Token name" });
|
|
85
|
-
await sleep(300);
|
|
86
|
-
await callTool("type_into", { sessionId: sid, target: "Token name", text: "screenhand-publish" });
|
|
87
|
-
await sleep(500);
|
|
88
|
-
// Step 2: Check "Bypass two-factor authentication (2FA)"
|
|
89
|
-
console.log("\nChecking bypass 2FA...");
|
|
90
|
-
await callTool("press", { sessionId: sid, target: "Bypass two-factor authentication (2FA)" });
|
|
91
|
-
await sleep(500);
|
|
92
|
-
// Step 3: Scroll down to see more options
|
|
93
|
-
console.log("\nScrolling down...");
|
|
94
|
-
await callTool("scroll", { sessionId: sid, direction: "down", amount: 5 });
|
|
95
|
-
await sleep(1000);
|
|
96
|
-
// Screenshot to see current state
|
|
97
|
-
console.log("\nScreenshot after scroll...");
|
|
98
|
-
const path1 = await screenshot(sid);
|
|
99
|
-
console.log(` Saved: ${path1}`);
|
|
100
|
-
// Step 4: Look for packages section - need to set permissions
|
|
101
|
-
console.log("\nScrolling down more...");
|
|
102
|
-
await callTool("scroll", { sessionId: sid, direction: "down", amount: 5 });
|
|
103
|
-
await sleep(1000);
|
|
104
|
-
const path2 = await screenshot(sid);
|
|
105
|
-
console.log(` Screenshot: ${path2}`);
|
|
106
|
-
// Try to find and click "Read and write" for packages permission
|
|
107
|
-
console.log("\nLooking for package permissions...");
|
|
108
|
-
await callTool("press", { sessionId: sid, target: "Read and write" });
|
|
109
|
-
await sleep(500);
|
|
110
|
-
// Scroll down to find Generate Token button
|
|
111
|
-
console.log("\nScrolling to Generate Token...");
|
|
112
|
-
await callTool("scroll", { sessionId: sid, direction: "down", amount: 5 });
|
|
113
|
-
await sleep(1000);
|
|
114
|
-
const path3 = await screenshot(sid);
|
|
115
|
-
console.log(` Screenshot: ${path3}`);
|
|
116
|
-
// Click Generate Token
|
|
117
|
-
console.log("\nClicking Generate Token...");
|
|
118
|
-
await callTool("press", { sessionId: sid, target: "Generate Token" });
|
|
119
|
-
await sleep(3000);
|
|
120
|
-
// Take final screenshot to see the token
|
|
121
|
-
console.log("\nFinal screenshot...");
|
|
122
|
-
const pathFinal = await screenshot(sid);
|
|
123
|
-
console.log(` Screenshot: ${pathFinal}`);
|
|
124
|
-
// Try to extract the token text from the page
|
|
125
|
-
console.log("\nExtracting page content...");
|
|
126
|
-
const tree = await callTool("element_tree", { sessionId: sid, maxDepth: 6 });
|
|
127
|
-
console.log(JSON.stringify(tree, null, 2).slice(0, 5000));
|
|
128
|
-
proc.kill();
|
|
129
|
-
process.exit(0);
|
|
130
|
-
}
|
|
131
|
-
catch (e) {
|
|
132
|
-
console.error("Error:", e instanceof Error ? e.message : String(e));
|
|
133
|
-
proc.kill();
|
|
134
|
-
process.exit(1);
|
|
135
|
-
}
|
package/dist/npm-token-finish.js
DELETED
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env tsx
|
|
2
|
-
import { spawn } from "node:child_process";
|
|
3
|
-
import { createInterface } from "node:readline";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
import { fileURLToPath } from "node:url";
|
|
6
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
7
|
-
const projectRoot = path.resolve(__dirname, "..");
|
|
8
|
-
const tsxBin = path.join(projectRoot, "node_modules", ".bin", "tsx");
|
|
9
|
-
const proc = spawn(tsxBin, [path.join(projectRoot, "src/mcp-entry.ts")], {
|
|
10
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
11
|
-
env: { ...process.env, SCREENHAND_ADAPTER: "accessibility" },
|
|
12
|
-
cwd: projectRoot,
|
|
13
|
-
});
|
|
14
|
-
proc.stderr.on("data", (d) => process.stderr.write(d));
|
|
15
|
-
let msgId = 0;
|
|
16
|
-
function send(msg) {
|
|
17
|
-
proc.stdin.write(JSON.stringify(msg) + "\n");
|
|
18
|
-
}
|
|
19
|
-
const rl = createInterface({ input: proc.stdout });
|
|
20
|
-
const lineQueue = [];
|
|
21
|
-
let lineWaiter = null;
|
|
22
|
-
rl.on("line", (line) => {
|
|
23
|
-
if (lineWaiter) {
|
|
24
|
-
const w = lineWaiter;
|
|
25
|
-
lineWaiter = null;
|
|
26
|
-
w(line);
|
|
27
|
-
}
|
|
28
|
-
else
|
|
29
|
-
lineQueue.push(line);
|
|
30
|
-
});
|
|
31
|
-
function readResponse() {
|
|
32
|
-
return new Promise((resolve, reject) => {
|
|
33
|
-
const timer = setTimeout(() => { lineWaiter = null; reject(new Error("Timeout")); }, 15000);
|
|
34
|
-
const handle = (line) => { clearTimeout(timer); resolve(JSON.parse(line)); };
|
|
35
|
-
const queued = lineQueue.shift();
|
|
36
|
-
if (queued) {
|
|
37
|
-
clearTimeout(timer);
|
|
38
|
-
resolve(JSON.parse(queued));
|
|
39
|
-
}
|
|
40
|
-
else
|
|
41
|
-
lineWaiter = handle;
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
async function callTool(name, args) {
|
|
45
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "tools/call", params: { name, arguments: args } });
|
|
46
|
-
const resp = await readResponse();
|
|
47
|
-
const result = resp.result;
|
|
48
|
-
const content = result?.content;
|
|
49
|
-
const text = content?.[0]?.text ?? "{}";
|
|
50
|
-
if (result?.isError)
|
|
51
|
-
console.error(` ERROR: ${text}`);
|
|
52
|
-
try {
|
|
53
|
-
return JSON.parse(text);
|
|
54
|
-
}
|
|
55
|
-
catch {
|
|
56
|
-
return text;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
async function screenshot(sid, label) {
|
|
60
|
-
const result = await callTool("screenshot", { sessionId: sid });
|
|
61
|
-
const data = result.data;
|
|
62
|
-
const p = data?.path ?? "";
|
|
63
|
-
console.log(` [${label}] ${p}`);
|
|
64
|
-
return p;
|
|
65
|
-
}
|
|
66
|
-
async function sleep(ms) { await new Promise(r => setTimeout(r, ms)); }
|
|
67
|
-
try {
|
|
68
|
-
await sleep(2000);
|
|
69
|
-
send({ jsonrpc: "2.0", id: ++msgId, method: "initialize",
|
|
70
|
-
params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "npm-helper", version: "1.0" } } });
|
|
71
|
-
await readResponse();
|
|
72
|
-
send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
73
|
-
await sleep(300);
|
|
74
|
-
const session = await callTool("session_start", {});
|
|
75
|
-
const sid = session.sessionId;
|
|
76
|
-
console.log(`Session: ${sid}`);
|
|
77
|
-
// Focus Safari
|
|
78
|
-
await callTool("app_focus", { sessionId: sid, bundleId: "com.apple.Safari" });
|
|
79
|
-
await sleep(500);
|
|
80
|
-
// Click the Packages "No access" dropdown to change permissions
|
|
81
|
-
console.log("\n1. Clicking Packages permissions dropdown...");
|
|
82
|
-
// The dropdown shows "No access" under "Packages and scopes"
|
|
83
|
-
await callTool("press", { sessionId: sid, target: { x: 220, y: 280 } });
|
|
84
|
-
await sleep(500);
|
|
85
|
-
await screenshot(sid, "after clicking packages dropdown");
|
|
86
|
-
// Try selecting "Read and write" from the dropdown
|
|
87
|
-
console.log("\n2. Selecting Read and write...");
|
|
88
|
-
await callTool("press", { sessionId: sid, target: "Read and write" });
|
|
89
|
-
await sleep(500);
|
|
90
|
-
await screenshot(sid, "after selecting read and write");
|
|
91
|
-
// Now we need to select which package - type "screenhand"
|
|
92
|
-
console.log("\n3. Looking for package selector...");
|
|
93
|
-
await callTool("scroll", { sessionId: sid, direction: "down", amount: 3 });
|
|
94
|
-
await sleep(500);
|
|
95
|
-
await screenshot(sid, "after scroll");
|
|
96
|
-
// Try to find a package input or "select packages" area
|
|
97
|
-
console.log("\n4. Trying to add screenhand package...");
|
|
98
|
-
await callTool("press", { sessionId: sid, target: "select" });
|
|
99
|
-
await sleep(300);
|
|
100
|
-
// Type screenhand in whatever input appeared
|
|
101
|
-
await callTool("key_combo", { sessionId: sid, keys: ["s", "c", "r", "e", "e", "n", "h", "a", "n", "d"] });
|
|
102
|
-
await sleep(1000);
|
|
103
|
-
await screenshot(sid, "after typing screenhand");
|
|
104
|
-
// Try to select it from autocomplete
|
|
105
|
-
await callTool("press", { sessionId: sid, target: "screenhand" });
|
|
106
|
-
await sleep(500);
|
|
107
|
-
// Scroll down to Generate Token
|
|
108
|
-
console.log("\n5. Scrolling to Generate Token...");
|
|
109
|
-
await callTool("scroll", { sessionId: sid, direction: "down", amount: 10 });
|
|
110
|
-
await sleep(500);
|
|
111
|
-
await screenshot(sid, "before generate");
|
|
112
|
-
// Click Generate Token
|
|
113
|
-
console.log("\n6. Clicking Generate Token...");
|
|
114
|
-
await callTool("press", { sessionId: sid, target: "Generate Token" });
|
|
115
|
-
await sleep(3000);
|
|
116
|
-
// Final screenshot - should show the token
|
|
117
|
-
console.log("\n7. Final result...");
|
|
118
|
-
const finalPath = await screenshot(sid, "FINAL");
|
|
119
|
-
proc.kill();
|
|
120
|
-
process.exit(0);
|
|
121
|
-
}
|
|
122
|
-
catch (e) {
|
|
123
|
-
console.error("Error:", e instanceof Error ? e.message : String(e));
|
|
124
|
-
proc.kill();
|
|
125
|
-
process.exit(1);
|
|
126
|
-
}
|
package/dist/playbook/engine.js
DELETED
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Playbook Engine — executes playbooks step-by-step
|
|
3
|
-
*
|
|
4
|
-
* Known path → playbook (fast, deterministic, no AI)
|
|
5
|
-
* Unknown state → AI fallback (slow, adaptive, learns)
|
|
6
|
-
*
|
|
7
|
-
* After AI recovers, the recovery steps get saved back into the playbook.
|
|
8
|
-
*/
|
|
9
|
-
const DEFAULT_VERIFY_TIMEOUT = 5000;
|
|
10
|
-
const STEP_DELAY_MS = 300;
|
|
11
|
-
export class PlaybookEngine {
|
|
12
|
-
runtime;
|
|
13
|
-
constructor(runtime) {
|
|
14
|
-
this.runtime = runtime;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Execute a playbook against a live session.
|
|
18
|
-
* Returns result with success/failure and which step broke.
|
|
19
|
-
*/
|
|
20
|
-
async run(sessionId, playbook, options = {}) {
|
|
21
|
-
const start = Date.now();
|
|
22
|
-
let stepsCompleted = 0;
|
|
23
|
-
for (let i = 0; i < playbook.steps.length; i++) {
|
|
24
|
-
const step = playbook.steps[i];
|
|
25
|
-
try {
|
|
26
|
-
const result = await this.executeStep(sessionId, step);
|
|
27
|
-
stepsCompleted++;
|
|
28
|
-
if (options.onStep) {
|
|
29
|
-
options.onStep(i, step, result);
|
|
30
|
-
}
|
|
31
|
-
// Verify step if needed
|
|
32
|
-
if (step.verify) {
|
|
33
|
-
const verified = await this.verifyStep(sessionId, step);
|
|
34
|
-
if (!verified && !step.optional) {
|
|
35
|
-
return {
|
|
36
|
-
playbook: playbook.id,
|
|
37
|
-
success: false,
|
|
38
|
-
stepsCompleted,
|
|
39
|
-
totalSteps: playbook.steps.length,
|
|
40
|
-
failedAtStep: i,
|
|
41
|
-
error: `Verification failed at step ${i}: ${step.description ?? step.action}`,
|
|
42
|
-
durationMs: Date.now() - start,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
// Small delay between steps for UI to settle
|
|
47
|
-
await sleep(STEP_DELAY_MS);
|
|
48
|
-
}
|
|
49
|
-
catch (err) {
|
|
50
|
-
if (step.optional) {
|
|
51
|
-
stepsCompleted++;
|
|
52
|
-
if (options.onStep) {
|
|
53
|
-
options.onStep(i, step, `Skipped (optional): ${err instanceof Error ? err.message : String(err)}`);
|
|
54
|
-
}
|
|
55
|
-
continue;
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
playbook: playbook.id,
|
|
59
|
-
success: false,
|
|
60
|
-
stepsCompleted,
|
|
61
|
-
totalSteps: playbook.steps.length,
|
|
62
|
-
failedAtStep: i,
|
|
63
|
-
error: err instanceof Error ? err.message : String(err),
|
|
64
|
-
durationMs: Date.now() - start,
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
return {
|
|
69
|
-
playbook: playbook.id,
|
|
70
|
-
success: true,
|
|
71
|
-
stepsCompleted,
|
|
72
|
-
totalSteps: playbook.steps.length,
|
|
73
|
-
failedAtStep: -1,
|
|
74
|
-
durationMs: Date.now() - start,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Execute a single playbook step.
|
|
79
|
-
*/
|
|
80
|
-
async executeStep(sessionId, step) {
|
|
81
|
-
const target = this.resolveTarget(step.target);
|
|
82
|
-
switch (step.action) {
|
|
83
|
-
case "navigate": {
|
|
84
|
-
if (!step.url)
|
|
85
|
-
throw new Error("navigate step missing url");
|
|
86
|
-
const r = await this.runtime.navigate({ sessionId, url: step.url });
|
|
87
|
-
if (!r.ok)
|
|
88
|
-
throw new Error(r.error.message);
|
|
89
|
-
return `Navigated to ${step.url}`;
|
|
90
|
-
}
|
|
91
|
-
case "press": {
|
|
92
|
-
if (!target)
|
|
93
|
-
throw new Error("press step missing target");
|
|
94
|
-
const r = await this.runtime.press({ sessionId, target });
|
|
95
|
-
if (!r.ok)
|
|
96
|
-
throw new Error(r.error.message);
|
|
97
|
-
return `Pressed ${JSON.stringify(step.target)}`;
|
|
98
|
-
}
|
|
99
|
-
case "type_into": {
|
|
100
|
-
if (!target)
|
|
101
|
-
throw new Error("type_into step missing target");
|
|
102
|
-
if (!step.text)
|
|
103
|
-
throw new Error("type_into step missing text");
|
|
104
|
-
const r = await this.runtime.typeInto({ sessionId, target, text: step.text });
|
|
105
|
-
if (!r.ok)
|
|
106
|
-
throw new Error(r.error.message);
|
|
107
|
-
return `Typed "${step.text}" into ${JSON.stringify(step.target)}`;
|
|
108
|
-
}
|
|
109
|
-
case "extract": {
|
|
110
|
-
if (!target)
|
|
111
|
-
throw new Error("extract step missing target");
|
|
112
|
-
const r = await this.runtime.extract({
|
|
113
|
-
sessionId,
|
|
114
|
-
target,
|
|
115
|
-
format: step.format ?? "text",
|
|
116
|
-
});
|
|
117
|
-
if (!r.ok)
|
|
118
|
-
throw new Error(r.error.message);
|
|
119
|
-
return `Extracted: ${JSON.stringify(r.data).slice(0, 200)}`;
|
|
120
|
-
}
|
|
121
|
-
case "key_combo": {
|
|
122
|
-
if (!step.keys || step.keys.length === 0)
|
|
123
|
-
throw new Error("key_combo step missing keys");
|
|
124
|
-
const r = await this.runtime.keyCombo({ sessionId, keys: step.keys });
|
|
125
|
-
if (!r.ok)
|
|
126
|
-
throw new Error(r.error.message);
|
|
127
|
-
return `Key combo: ${step.keys.join("+")}`;
|
|
128
|
-
}
|
|
129
|
-
case "scroll": {
|
|
130
|
-
const input = {
|
|
131
|
-
sessionId,
|
|
132
|
-
direction: step.direction ?? "down",
|
|
133
|
-
};
|
|
134
|
-
if (step.amount != null)
|
|
135
|
-
input.amount = step.amount;
|
|
136
|
-
const r = await this.runtime.scroll(input);
|
|
137
|
-
if (!r.ok)
|
|
138
|
-
throw new Error(r.error.message);
|
|
139
|
-
return `Scrolled ${step.direction ?? "down"}`;
|
|
140
|
-
}
|
|
141
|
-
case "wait": {
|
|
142
|
-
await sleep(step.ms ?? 1000);
|
|
143
|
-
return `Waited ${step.ms ?? 1000}ms`;
|
|
144
|
-
}
|
|
145
|
-
case "screenshot": {
|
|
146
|
-
const r = await this.runtime.screenshot({ sessionId });
|
|
147
|
-
if (!r.ok)
|
|
148
|
-
throw new Error(r.error.message);
|
|
149
|
-
return `Screenshot taken`;
|
|
150
|
-
}
|
|
151
|
-
default:
|
|
152
|
-
throw new Error(`Unknown action: ${step.action}`);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
/**
|
|
156
|
-
* Verify a step's postcondition via CSS selector check.
|
|
157
|
-
*/
|
|
158
|
-
async verifyStep(sessionId, step) {
|
|
159
|
-
if (!step.verify)
|
|
160
|
-
return true;
|
|
161
|
-
const timeout = step.verifyTimeoutMs ?? DEFAULT_VERIFY_TIMEOUT;
|
|
162
|
-
const r = await this.runtime.waitFor({
|
|
163
|
-
sessionId,
|
|
164
|
-
condition: { type: "selector_visible", selector: step.verify },
|
|
165
|
-
timeoutMs: timeout,
|
|
166
|
-
});
|
|
167
|
-
return r.ok && r.data.matched;
|
|
168
|
-
}
|
|
169
|
-
/**
|
|
170
|
-
* Convert playbook target format to runtime Target format.
|
|
171
|
-
*/
|
|
172
|
-
resolveTarget(target) {
|
|
173
|
-
if (!target)
|
|
174
|
-
return undefined;
|
|
175
|
-
if (typeof target === "string") {
|
|
176
|
-
// CSS selector if starts with common patterns, else treat as text
|
|
177
|
-
if (target.startsWith("[") || target.startsWith("#") || target.startsWith(".") || target.startsWith("css=")) {
|
|
178
|
-
return { type: "selector", value: target.replace(/^css=/, "") };
|
|
179
|
-
}
|
|
180
|
-
return { type: "text", value: target };
|
|
181
|
-
}
|
|
182
|
-
if ("selector" in target) {
|
|
183
|
-
return { type: "selector", value: target.selector };
|
|
184
|
-
}
|
|
185
|
-
if ("x" in target && "y" in target) {
|
|
186
|
-
return { type: "coordinates", x: target.x, y: target.y };
|
|
187
|
-
}
|
|
188
|
-
return undefined;
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
function sleep(ms) {
|
|
192
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
193
|
-
}
|
package/dist/playbook/index.js
DELETED