screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
#!/usr/bin/env npx tsx
|
|
2
|
+
/**
|
|
3
|
+
* Orchestrator Daemon — multi-agent task router and coordinator.
|
|
4
|
+
*
|
|
5
|
+
* Manages a pool of worker slots that process tasks in parallel:
|
|
6
|
+
* - Web slots (CDP-only): truly parallel, no mouse/keyboard conflict
|
|
7
|
+
* - Native slots: serialized per-app via lease locks
|
|
8
|
+
*
|
|
9
|
+
* Each worker slot runs a JobRunner independently. The orchestrator
|
|
10
|
+
* routes tasks to the right slot type and manages coordination.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* npx tsx scripts/orchestrator-daemon.ts
|
|
14
|
+
* npx tsx scripts/orchestrator-daemon.ts --web-slots 4 --native-slots 1 --poll 1000
|
|
15
|
+
*
|
|
16
|
+
* State files:
|
|
17
|
+
* ~/.screenhand/orchestrator/state.json — orchestrator state
|
|
18
|
+
* ~/.screenhand/orchestrator/orchestrator.pid — PID of this process
|
|
19
|
+
* ~/.screenhand/orchestrator/orchestrator.log — log output
|
|
20
|
+
*/
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import fs from "node:fs";
|
|
23
|
+
import os from "node:os";
|
|
24
|
+
import { BridgeClient } from "../src/native/bridge-client.js";
|
|
25
|
+
import { SessionSupervisor, LeaseManager } from "../src/supervisor/supervisor.js";
|
|
26
|
+
import { JobManager } from "../src/jobs/manager.js";
|
|
27
|
+
import { JobRunner } from "../src/jobs/runner.js";
|
|
28
|
+
import { PlaybookEngine } from "../src/playbook/engine.js";
|
|
29
|
+
import { PlaybookStore } from "../src/playbook/store.js";
|
|
30
|
+
import { AccessibilityAdapter } from "../src/runtime/accessibility-adapter.js";
|
|
31
|
+
import { AutomationRuntimeService } from "../src/runtime/service.js";
|
|
32
|
+
import { TimelineLogger } from "../src/logging/timeline-logger.js";
|
|
33
|
+
import { MemoryService } from "../src/memory/service.js";
|
|
34
|
+
import { writeOrchestratorState, getOrchestratorDaemonPid } from "../src/orchestrator/state.js";
|
|
35
|
+
import { ORCHESTRATOR_DIR, ORCHESTRATOR_PID_FILE, ORCHESTRATOR_LOG_FILE } from "../src/orchestrator/types.js";
|
|
36
|
+
// ── Config from CLI args ──
|
|
37
|
+
const args = process.argv.slice(2);
|
|
38
|
+
function getArg(name, fallback) {
|
|
39
|
+
const idx = args.indexOf("--" + name);
|
|
40
|
+
if (idx === -1)
|
|
41
|
+
return fallback;
|
|
42
|
+
return args[idx + 1] ?? fallback;
|
|
43
|
+
}
|
|
44
|
+
const WEB_SLOTS = Number(getArg("web-slots", "4"));
|
|
45
|
+
const NATIVE_SLOTS = Number(getArg("native-slots", "1"));
|
|
46
|
+
const POLL_MS = Number(getArg("poll", "1000"));
|
|
47
|
+
// ── Directories ──
|
|
48
|
+
const JOB_DIR = path.join(os.homedir(), ".screenhand", "jobs");
|
|
49
|
+
const LOCK_DIR = path.join(os.homedir(), ".screenhand", "locks");
|
|
50
|
+
const PLAYBOOKS_DIR = path.join(os.homedir(), ".screenhand", "playbooks");
|
|
51
|
+
const SUPERVISOR_STATE_DIR = path.join(os.homedir(), ".screenhand", "supervisor");
|
|
52
|
+
fs.mkdirSync(ORCHESTRATOR_DIR, { recursive: true });
|
|
53
|
+
fs.mkdirSync(JOB_DIR, { recursive: true });
|
|
54
|
+
// ── Logging ──
|
|
55
|
+
const logStream = fs.createWriteStream(ORCHESTRATOR_LOG_FILE, { flags: "a" });
|
|
56
|
+
let daemonized = false;
|
|
57
|
+
function log(msg) {
|
|
58
|
+
const line = `[${new Date().toISOString()}] ${msg}`;
|
|
59
|
+
logStream.write(line + "\n");
|
|
60
|
+
if (!daemonized)
|
|
61
|
+
process.stderr.write(line + "\n");
|
|
62
|
+
}
|
|
63
|
+
// ── Bridge setup ──
|
|
64
|
+
const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
|
|
65
|
+
const projectRoot = scriptDir.includes("/dist/")
|
|
66
|
+
? path.resolve(scriptDir, "../..")
|
|
67
|
+
: path.resolve(scriptDir, "..");
|
|
68
|
+
const bridgePath = process.platform === "win32"
|
|
69
|
+
? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
|
|
70
|
+
: path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
|
|
71
|
+
// Each worker slot gets its own bridge for true parallelism
|
|
72
|
+
function createBridge() {
|
|
73
|
+
return new BridgeClient(bridgePath);
|
|
74
|
+
}
|
|
75
|
+
// ── Shared services ──
|
|
76
|
+
const leaseManager = new LeaseManager(LOCK_DIR);
|
|
77
|
+
const supervisor = new SessionSupervisor({
|
|
78
|
+
stateDir: SUPERVISOR_STATE_DIR,
|
|
79
|
+
lockDir: LOCK_DIR,
|
|
80
|
+
});
|
|
81
|
+
const memory = new MemoryService(os.homedir());
|
|
82
|
+
const jobManager = new JobManager({ jobDir: JOB_DIR, memory, supervisor });
|
|
83
|
+
jobManager.init();
|
|
84
|
+
const playbookStore = new PlaybookStore(PLAYBOOKS_DIR);
|
|
85
|
+
// ── State ──
|
|
86
|
+
let stopped = false;
|
|
87
|
+
const startedAt = new Date().toISOString();
|
|
88
|
+
let totalSubmitted = 0;
|
|
89
|
+
let totalCompleted = 0;
|
|
90
|
+
let totalFailed = 0;
|
|
91
|
+
// Task queue — loaded from / persisted to state.json
|
|
92
|
+
let taskQueue = [];
|
|
93
|
+
// Worker slots
|
|
94
|
+
const workers = [];
|
|
95
|
+
const workerRunners = new Map();
|
|
96
|
+
// App locks for native tasks — bundleId → worker slot ID
|
|
97
|
+
const nativeLocks = new Map();
|
|
98
|
+
// ── Worker slot initialization ──
|
|
99
|
+
async function initWorkerSlots() {
|
|
100
|
+
let slotId = 0;
|
|
101
|
+
// Web slots
|
|
102
|
+
for (let i = 0; i < WEB_SLOTS; i++) {
|
|
103
|
+
const slot = {
|
|
104
|
+
id: slotId,
|
|
105
|
+
type: "web",
|
|
106
|
+
busy: false,
|
|
107
|
+
tasksCompleted: 0,
|
|
108
|
+
tasksFailed: 0,
|
|
109
|
+
};
|
|
110
|
+
workers.push(slot);
|
|
111
|
+
const bridge = createBridge();
|
|
112
|
+
await bridge.start();
|
|
113
|
+
const adapter = new AccessibilityAdapter(bridge);
|
|
114
|
+
const logger = new TimelineLogger();
|
|
115
|
+
const runtimeService = new AutomationRuntimeService(adapter, logger);
|
|
116
|
+
const playbookEngine = new PlaybookEngine(runtimeService);
|
|
117
|
+
const runner = new JobRunner(bridge, jobManager, leaseManager, supervisor, {
|
|
118
|
+
playbookEngine,
|
|
119
|
+
playbookStore,
|
|
120
|
+
runtimeService,
|
|
121
|
+
onLog: (msg) => log(`[W${slotId}] ${msg}`),
|
|
122
|
+
});
|
|
123
|
+
workerRunners.set(slotId, { runner, bridge, busy: false });
|
|
124
|
+
slotId++;
|
|
125
|
+
}
|
|
126
|
+
// Native slots
|
|
127
|
+
for (let i = 0; i < NATIVE_SLOTS; i++) {
|
|
128
|
+
const slot = {
|
|
129
|
+
id: slotId,
|
|
130
|
+
type: "native",
|
|
131
|
+
busy: false,
|
|
132
|
+
tasksCompleted: 0,
|
|
133
|
+
tasksFailed: 0,
|
|
134
|
+
};
|
|
135
|
+
workers.push(slot);
|
|
136
|
+
const bridge = createBridge();
|
|
137
|
+
await bridge.start();
|
|
138
|
+
const adapter = new AccessibilityAdapter(bridge);
|
|
139
|
+
const logger = new TimelineLogger();
|
|
140
|
+
const runtimeService = new AutomationRuntimeService(adapter, logger);
|
|
141
|
+
const playbookEngine = new PlaybookEngine(runtimeService);
|
|
142
|
+
const runner = new JobRunner(bridge, jobManager, leaseManager, supervisor, {
|
|
143
|
+
playbookEngine,
|
|
144
|
+
playbookStore,
|
|
145
|
+
runtimeService,
|
|
146
|
+
onLog: (msg) => log(`[W${slotId}] ${msg}`),
|
|
147
|
+
});
|
|
148
|
+
workerRunners.set(slotId, { runner, bridge, busy: false });
|
|
149
|
+
slotId++;
|
|
150
|
+
}
|
|
151
|
+
log(`Initialized ${WEB_SLOTS} web slots + ${NATIVE_SLOTS} native slots = ${workers.length} total`);
|
|
152
|
+
}
|
|
153
|
+
// ── Task routing ──
|
|
154
|
+
function findAvailableSlot(task) {
|
|
155
|
+
if (task.mode === "web") {
|
|
156
|
+
// Any free web slot
|
|
157
|
+
return workers.find(w => w.type === "web" && !w.busy) ?? null;
|
|
158
|
+
}
|
|
159
|
+
if (task.mode === "native" || task.mode === "mixed") {
|
|
160
|
+
// Native tasks need a free native slot AND the app must not be locked by another slot
|
|
161
|
+
const bundleId = task.bundleId ?? "unknown";
|
|
162
|
+
const lockHolder = nativeLocks.get(bundleId);
|
|
163
|
+
if (lockHolder !== undefined) {
|
|
164
|
+
// App is locked — only the lock holder can work on it
|
|
165
|
+
const slot = workers.find(w => w.id === lockHolder && !w.busy);
|
|
166
|
+
return slot ?? null;
|
|
167
|
+
}
|
|
168
|
+
// No lock — find any free native slot
|
|
169
|
+
return workers.find(w => w.type === "native" && !w.busy) ?? null;
|
|
170
|
+
}
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
// ── Task → Job conversion ──
|
|
174
|
+
function taskToJobParams(task) {
|
|
175
|
+
return {
|
|
176
|
+
task: task.task,
|
|
177
|
+
...(task.playbookId !== undefined ? { playbookId: task.playbookId } : {}),
|
|
178
|
+
...(task.bundleId !== undefined ? { bundleId: task.bundleId } : {}),
|
|
179
|
+
...(task.windowId !== undefined ? { windowId: task.windowId } : {}),
|
|
180
|
+
...(task.vars ? { vars: task.vars } : {}),
|
|
181
|
+
priority: task.priority,
|
|
182
|
+
tags: ["orchestrator", `task_${task.id}`],
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
// ── Task dispatch ──
|
|
186
|
+
async function dispatchTask(task, slot) {
|
|
187
|
+
const worker = workerRunners.get(slot.id);
|
|
188
|
+
if (!worker)
|
|
189
|
+
return;
|
|
190
|
+
// Mark slot as busy
|
|
191
|
+
slot.busy = true;
|
|
192
|
+
slot.currentTaskId = task.id;
|
|
193
|
+
worker.busy = true;
|
|
194
|
+
// Lock native app
|
|
195
|
+
if ((task.mode === "native" || task.mode === "mixed") && task.bundleId) {
|
|
196
|
+
nativeLocks.set(task.bundleId, slot.id);
|
|
197
|
+
}
|
|
198
|
+
// Update task status
|
|
199
|
+
task.status = "assigned";
|
|
200
|
+
task.assignedWorker = slot.id;
|
|
201
|
+
task.startedAt = new Date().toISOString();
|
|
202
|
+
log(`Dispatching task ${task.id} ("${task.task.slice(0, 50)}") to slot ${slot.id} (${slot.type})`);
|
|
203
|
+
// Create a job for this task
|
|
204
|
+
const jobParams = taskToJobParams(task);
|
|
205
|
+
const job = jobManager.create(jobParams);
|
|
206
|
+
task.jobId = job.id;
|
|
207
|
+
task.status = "running";
|
|
208
|
+
persistState();
|
|
209
|
+
// Run the job asynchronously
|
|
210
|
+
try {
|
|
211
|
+
const result = await worker.runner.run();
|
|
212
|
+
if (result) {
|
|
213
|
+
task.result = `${result.finalState}: ${result.stepsCompleted}/${result.totalSteps} steps in ${result.durationMs}ms`;
|
|
214
|
+
if (result.finalState === "done") {
|
|
215
|
+
task.status = "done";
|
|
216
|
+
slot.tasksCompleted++;
|
|
217
|
+
totalCompleted++;
|
|
218
|
+
log(`Task ${task.id} completed (${result.durationMs}ms)`);
|
|
219
|
+
}
|
|
220
|
+
else if (result.finalState === "blocked" || result.finalState === "waiting_human") {
|
|
221
|
+
task.status = "blocked";
|
|
222
|
+
task.error = result.error ?? "Blocked";
|
|
223
|
+
log(`Task ${task.id} blocked: ${result.error ?? "unknown"}`);
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
task.status = "failed";
|
|
227
|
+
task.error = result.error ?? "Failed";
|
|
228
|
+
slot.tasksFailed++;
|
|
229
|
+
totalFailed++;
|
|
230
|
+
log(`Task ${task.id} failed: ${result.error ?? "unknown"}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
// No job was dequeued — it may have been picked up already
|
|
235
|
+
task.status = "failed";
|
|
236
|
+
task.error = "No job to dequeue";
|
|
237
|
+
slot.tasksFailed++;
|
|
238
|
+
totalFailed++;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
catch (err) {
|
|
242
|
+
task.status = "failed";
|
|
243
|
+
task.error = err instanceof Error ? err.message : String(err);
|
|
244
|
+
slot.tasksFailed++;
|
|
245
|
+
totalFailed++;
|
|
246
|
+
log(`Task ${task.id} error: ${task.error}`);
|
|
247
|
+
}
|
|
248
|
+
// Release slot
|
|
249
|
+
task.completedAt = new Date().toISOString();
|
|
250
|
+
slot.busy = false;
|
|
251
|
+
delete slot.currentTaskId;
|
|
252
|
+
worker.busy = false;
|
|
253
|
+
// Release native app lock
|
|
254
|
+
if ((task.mode === "native" || task.mode === "mixed") && task.bundleId) {
|
|
255
|
+
nativeLocks.delete(task.bundleId);
|
|
256
|
+
}
|
|
257
|
+
persistState();
|
|
258
|
+
}
|
|
259
|
+
// ── State persistence ──
|
|
260
|
+
function buildState() {
|
|
261
|
+
const nativeLocksObj = {};
|
|
262
|
+
for (const [k, v] of nativeLocks)
|
|
263
|
+
nativeLocksObj[k] = v;
|
|
264
|
+
return {
|
|
265
|
+
pid: process.pid,
|
|
266
|
+
running: !stopped,
|
|
267
|
+
startedAt,
|
|
268
|
+
workers: [...workers],
|
|
269
|
+
webSlots: WEB_SLOTS,
|
|
270
|
+
nativeSlots: NATIVE_SLOTS,
|
|
271
|
+
tasks: taskQueue,
|
|
272
|
+
totalSubmitted,
|
|
273
|
+
totalCompleted,
|
|
274
|
+
totalFailed,
|
|
275
|
+
nativeLocks: nativeLocksObj,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
function persistState() {
|
|
279
|
+
try {
|
|
280
|
+
writeOrchestratorState(buildState());
|
|
281
|
+
}
|
|
282
|
+
catch {
|
|
283
|
+
// Non-fatal
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
// ── Load tasks from state (resume after restart) ──
|
|
287
|
+
function loadState() {
|
|
288
|
+
try {
|
|
289
|
+
const state = readExistingState();
|
|
290
|
+
if (state && state.tasks) {
|
|
291
|
+
// Resume queued/running tasks
|
|
292
|
+
for (const task of state.tasks) {
|
|
293
|
+
if (task.status === "queued" || task.status === "assigned" || task.status === "running") {
|
|
294
|
+
task.status = "queued"; // Re-queue interrupted tasks
|
|
295
|
+
delete task.assignedWorker;
|
|
296
|
+
taskQueue.push(task);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
totalSubmitted = state.totalSubmitted ?? 0;
|
|
300
|
+
totalCompleted = state.totalCompleted ?? 0;
|
|
301
|
+
totalFailed = state.totalFailed ?? 0;
|
|
302
|
+
if (taskQueue.length > 0) {
|
|
303
|
+
log(`Resumed ${taskQueue.length} tasks from previous state`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
// Fresh start
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
function readExistingState() {
|
|
312
|
+
try {
|
|
313
|
+
const data = fs.readFileSync(path.join(ORCHESTRATOR_DIR, "state.json"), "utf-8");
|
|
314
|
+
return JSON.parse(data);
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
// ── Main poll loop ──
|
|
321
|
+
async function poll() {
|
|
322
|
+
// Also reload playbooks periodically
|
|
323
|
+
playbookStore.load();
|
|
324
|
+
// Sort queue by priority (lower = higher priority)
|
|
325
|
+
const queued = taskQueue
|
|
326
|
+
.filter(t => t.status === "queued")
|
|
327
|
+
.sort((a, b) => a.priority - b.priority);
|
|
328
|
+
// Dispatch tasks to available slots
|
|
329
|
+
const dispatches = [];
|
|
330
|
+
for (const task of queued) {
|
|
331
|
+
const slot = findAvailableSlot(task);
|
|
332
|
+
if (slot) {
|
|
333
|
+
// Dispatch in parallel — don't await here
|
|
334
|
+
dispatches.push(dispatchTask(task, slot));
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
// Wait for all dispatched tasks in this cycle
|
|
338
|
+
if (dispatches.length > 0) {
|
|
339
|
+
await Promise.allSettled(dispatches);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
async function main() {
|
|
343
|
+
// Enforce single daemon
|
|
344
|
+
const existingPid = getOrchestratorDaemonPid();
|
|
345
|
+
if (existingPid !== null && existingPid !== process.pid) {
|
|
346
|
+
log(`Another orchestrator daemon already running (pid=${existingPid}). Aborting.`);
|
|
347
|
+
process.exit(1);
|
|
348
|
+
}
|
|
349
|
+
fs.writeFileSync(ORCHESTRATOR_PID_FILE, String(process.pid));
|
|
350
|
+
daemonized = true;
|
|
351
|
+
log(`Orchestrator daemon started (pid=${process.pid})`);
|
|
352
|
+
log(`Config: web-slots=${WEB_SLOTS} native-slots=${NATIVE_SLOTS} poll=${POLL_MS}ms`);
|
|
353
|
+
// Load previous state
|
|
354
|
+
loadState();
|
|
355
|
+
// Initialize worker slots (each with its own bridge)
|
|
356
|
+
await initWorkerSlots();
|
|
357
|
+
persistState();
|
|
358
|
+
// Poll loop
|
|
359
|
+
while (!stopped) {
|
|
360
|
+
try {
|
|
361
|
+
await poll();
|
|
362
|
+
}
|
|
363
|
+
catch (err) {
|
|
364
|
+
log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
|
|
365
|
+
}
|
|
366
|
+
await sleep(POLL_MS);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function sleep(ms) {
|
|
370
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
371
|
+
}
|
|
372
|
+
// ── Graceful shutdown ──
|
|
373
|
+
process.on("SIGINT", shutdown);
|
|
374
|
+
process.on("SIGTERM", shutdown);
|
|
375
|
+
async function shutdown() {
|
|
376
|
+
if (stopped)
|
|
377
|
+
return;
|
|
378
|
+
stopped = true;
|
|
379
|
+
log("Shutting down...");
|
|
380
|
+
// Stop all worker bridges
|
|
381
|
+
for (const [, w] of workerRunners) {
|
|
382
|
+
try {
|
|
383
|
+
await w.bridge.stop();
|
|
384
|
+
}
|
|
385
|
+
catch { /* ignore */ }
|
|
386
|
+
}
|
|
387
|
+
persistState();
|
|
388
|
+
try {
|
|
389
|
+
fs.unlinkSync(ORCHESTRATOR_PID_FILE);
|
|
390
|
+
}
|
|
391
|
+
catch { /* ignore */ }
|
|
392
|
+
logStream.end();
|
|
393
|
+
log(`Orchestrator exiting (${totalCompleted} done, ${totalFailed} failed)`);
|
|
394
|
+
process.exit(0);
|
|
395
|
+
}
|
|
396
|
+
main().catch((err) => {
|
|
397
|
+
log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
|
|
398
|
+
process.exit(1);
|
|
399
|
+
});
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ScreenHand Twitter/X 2-Hour Campaign Runner
|
|
3
|
+
*
|
|
4
|
+
* Executes the engagement cadence from threads-campaign.md
|
|
5
|
+
* Uses ScreenHand MCP tools via direct function calls
|
|
6
|
+
*
|
|
7
|
+
* Usage: tsx scripts/threads-campaign.ts
|
|
8
|
+
* With cron/loop: keeps alive for 2 hours
|
|
9
|
+
*/
|
|
10
|
+
import { join } from 'path';
|
|
11
|
+
// Campaign config
|
|
12
|
+
const CONFIG = {
|
|
13
|
+
duration: 2 * 60 * 60 * 1000, // 2 hours in ms
|
|
14
|
+
tabId: '', // filled at runtime
|
|
15
|
+
searchQueries: [
|
|
16
|
+
'"MCP server" -is:retweet',
|
|
17
|
+
'"AI agent" automation -is:retweet',
|
|
18
|
+
'"Claude Code" -is:retweet',
|
|
19
|
+
'"desktop automation" -is:retweet',
|
|
20
|
+
'"Selenium frustrating" OR "Playwright broken" -is:retweet',
|
|
21
|
+
'"browser automation" -is:retweet',
|
|
22
|
+
'"Figma to code" -is:retweet',
|
|
23
|
+
],
|
|
24
|
+
tweets: [
|
|
25
|
+
{
|
|
26
|
+
time: 0,
|
|
27
|
+
content: `AI agents can think, code, and reason.
|
|
28
|
+
|
|
29
|
+
But they can't click a button.
|
|
30
|
+
|
|
31
|
+
That's the gap ScreenHand fills — an MCP server giving any AI agent native desktop control on macOS + Windows.
|
|
32
|
+
|
|
33
|
+
No API keys. No cloud. No fixed scripts. Just real OS-level control. 82 tools. One protocol.`
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
time: 20,
|
|
37
|
+
content: `If your automation breaks when a button moves 10 pixels, you don't have automation. You have a time bomb.
|
|
38
|
+
|
|
39
|
+
ScreenHand doesn't rely on fixed selectors. It uses fallback chains:
|
|
40
|
+
|
|
41
|
+
Accessibility API → CDP → Vision → AppleScript
|
|
42
|
+
|
|
43
|
+
If one fails, it adapts. It also remembers what worked — gets faster every run.`
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
time: 40,
|
|
47
|
+
content: `The AI stack everyone's sleeping on:
|
|
48
|
+
|
|
49
|
+
Claude Code → thinks + codes
|
|
50
|
+
Figma MCP → design tokens
|
|
51
|
+
ScreenHand → opens the actual browser, verifies pixels, clicks deploy
|
|
52
|
+
|
|
53
|
+
What takes 45 min manually takes 3 min with this stack. The AI has a brain. ScreenHand gives it hands.`
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
time: 60,
|
|
57
|
+
content: `Every cloud automation tool has the same problem — your credentials pass through someone else's server.
|
|
58
|
+
|
|
59
|
+
ScreenHand runs locally. On your machine. No cloud relay. No API keys to leak. No third-party credential storage.
|
|
60
|
+
|
|
61
|
+
AGPL licensed. You own it completely.`
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
time: 80,
|
|
65
|
+
content: `Building AI agents? Here's what nobody tells you:
|
|
66
|
+
|
|
67
|
+
The hardest part isn't the reasoning. It's the last mile — actually executing actions in real software.
|
|
68
|
+
|
|
69
|
+
We built ScreenHand because we hit this wall ourselves. 82 MCP tools later, our agents can control any app on the desktop.
|
|
70
|
+
|
|
71
|
+
What are you building?`
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
time: 100,
|
|
75
|
+
content: `What ScreenHand can do in one MCP call:
|
|
76
|
+
|
|
77
|
+
- browser_human_click — clicks like a real human, anti-detection built in
|
|
78
|
+
- memory_recall — remembers patterns from previous runs
|
|
79
|
+
- execution_plan — plans multi-step workflows with fallback strategies
|
|
80
|
+
- export_playbook — saves learned flows for autonomous replay
|
|
81
|
+
|
|
82
|
+
Not scripts. Not recordings. Adaptive execution.`
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
replyTemplates: [
|
|
86
|
+
`This is exactly why we built ScreenHand — the execution layer is the missing piece. Agents can reason but need OS-level control to do real work. We use accessibility APIs + CDP, not screenshots.`,
|
|
87
|
+
`Interesting approach. One thing we've found — the fallback chain matters more than any single method. AX API fails? Try CDP. CDP fails? Try Vision. Resilience > precision.`,
|
|
88
|
+
`Have you tried giving your agent actual desktop control? We built an MCP server (ScreenHand) that lets Claude/GPT control any app natively. Game changer for automation.`,
|
|
89
|
+
`MCP is the right protocol for this. We built ScreenHand as an MCP server — 82 tools for desktop + browser automation. Any MCP client can drive it.`,
|
|
90
|
+
`The composability of MCP is what makes it powerful. ScreenHand (desktop control) stacks with tools like yours naturally. Brain + hands.`,
|
|
91
|
+
`Felt this. Selector-based automation is brittle. We switched to accessibility-tree-first with ScreenHand — the OS exposes UI elements semantically, not positionally.`,
|
|
92
|
+
`The maintenance tax on Selenium scripts is brutal. We built fallback chains (AX → CDP → Vision → AppleScript) because no single method is reliable enough alone.`,
|
|
93
|
+
`This is why we went API-free. No API keys = no rate limits, no platform approval needed. ScreenHand controls the actual UI like a human would.`,
|
|
94
|
+
`Claude Code + ScreenHand is a wild combo. Claude thinks and codes, ScreenHand handles everything outside the IDE — browser testing, deployment UIs, verification.`,
|
|
95
|
+
`Figma MCP gives you design tokens. But who verifies the implementation matches? ScreenHand opens the browser, screenshots the result, compares. Loop closed.`,
|
|
96
|
+
],
|
|
97
|
+
// Rate limits
|
|
98
|
+
maxLikesPerHour: 18,
|
|
99
|
+
maxRepliesPerHour: 6,
|
|
100
|
+
maxDmsPerHour: 2,
|
|
101
|
+
maxTweetsPerHour: 3,
|
|
102
|
+
};
|
|
103
|
+
// Randomized delay (human-like)
|
|
104
|
+
function randomDelay(minSec, maxSec) {
|
|
105
|
+
return (Math.floor(Math.random() * (maxSec - minSec + 1)) + minSec) * 1000;
|
|
106
|
+
}
|
|
107
|
+
function createInitialState() {
|
|
108
|
+
return {
|
|
109
|
+
startTime: Date.now(),
|
|
110
|
+
tweetsPosted: 0,
|
|
111
|
+
likesGiven: 0,
|
|
112
|
+
repliesSent: 0,
|
|
113
|
+
dmsSent: 0,
|
|
114
|
+
searchIndex: 0,
|
|
115
|
+
replyIndex: 0,
|
|
116
|
+
minutesElapsed: 0,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
const SCHEDULE = [
|
|
120
|
+
{ minute: 0, type: 'post', tweetIndex: 0 },
|
|
121
|
+
{ minute: 3, type: 'like', count: 3 },
|
|
122
|
+
{ minute: 8, type: 'reply', replyTemplate: 0 },
|
|
123
|
+
{ minute: 10, type: 'reply', replyTemplate: 3 },
|
|
124
|
+
{ minute: 15, type: 'like', count: 2 },
|
|
125
|
+
{ minute: 20, type: 'post', tweetIndex: 1 },
|
|
126
|
+
{ minute: 25, type: 'reply', replyTemplate: 5 },
|
|
127
|
+
{ minute: 27, type: 'reply', replyTemplate: 7 },
|
|
128
|
+
{ minute: 30, type: 'like', count: 3 },
|
|
129
|
+
{ minute: 35, type: 'reply', replyTemplate: 8 },
|
|
130
|
+
{ minute: 40, type: 'post', tweetIndex: 2 },
|
|
131
|
+
{ minute: 45, type: 'like', count: 3 },
|
|
132
|
+
{ minute: 50, type: 'reply', replyTemplate: 9 },
|
|
133
|
+
{ minute: 52, type: 'reply', replyTemplate: 1 },
|
|
134
|
+
{ minute: 55, type: 'search' },
|
|
135
|
+
{ minute: 60, type: 'post', tweetIndex: 3 },
|
|
136
|
+
{ minute: 65, type: 'like', count: 2 },
|
|
137
|
+
{ minute: 70, type: 'reply', replyTemplate: 2 },
|
|
138
|
+
{ minute: 72, type: 'reply', replyTemplate: 4 },
|
|
139
|
+
{ minute: 75, type: 'like', count: 3 },
|
|
140
|
+
{ minute: 80, type: 'post', tweetIndex: 4 },
|
|
141
|
+
{ minute: 85, type: 'reply', replyTemplate: 6 },
|
|
142
|
+
{ minute: 87, type: 'reply', replyTemplate: 1 },
|
|
143
|
+
{ minute: 90, type: 'search' },
|
|
144
|
+
{ minute: 95, type: 'like', count: 3 },
|
|
145
|
+
{ minute: 100, type: 'post', tweetIndex: 5 },
|
|
146
|
+
{ minute: 105, type: 'reply', replyTemplate: 3 },
|
|
147
|
+
{ minute: 107, type: 'reply', replyTemplate: 8 },
|
|
148
|
+
{ minute: 110, type: 'like', count: 2 },
|
|
149
|
+
{ minute: 115, type: 'like', count: 3 },
|
|
150
|
+
{ minute: 120, type: 'search' },
|
|
151
|
+
];
|
|
152
|
+
// Log action for tracking
|
|
153
|
+
function logAction(action, details) {
|
|
154
|
+
const ts = new Date().toISOString();
|
|
155
|
+
const line = `[${ts}] ${action}: ${details}`;
|
|
156
|
+
console.log(line);
|
|
157
|
+
// Append to campaign log
|
|
158
|
+
const logPath = join(__dirname, '..', 'threads_campaign_log.txt');
|
|
159
|
+
const { appendFileSync } = require('fs');
|
|
160
|
+
appendFileSync(logPath, line + '\n');
|
|
161
|
+
}
|
|
162
|
+
// Main — this file is designed to be driven by ScreenHand MCP tools
|
|
163
|
+
// When run standalone, it outputs the schedule for the current minute
|
|
164
|
+
if (require.main === module) {
|
|
165
|
+
const state = createInitialState();
|
|
166
|
+
const elapsed = Math.floor((Date.now() - state.startTime) / 60000);
|
|
167
|
+
// Find next actions
|
|
168
|
+
const upcoming = SCHEDULE.filter(a => a.minute >= elapsed).slice(0, 3);
|
|
169
|
+
console.log('=== ScreenHand Campaign Runner ===');
|
|
170
|
+
console.log(`Started: ${new Date(state.startTime).toLocaleTimeString()}`);
|
|
171
|
+
console.log(`Elapsed: ${elapsed} minutes`);
|
|
172
|
+
console.log(`\nNext actions:`);
|
|
173
|
+
for (const action of upcoming) {
|
|
174
|
+
const inMinutes = action.minute - elapsed;
|
|
175
|
+
let desc = '';
|
|
176
|
+
switch (action.type) {
|
|
177
|
+
case 'post':
|
|
178
|
+
desc = `Post tweet #${(action.tweetIndex ?? 0) + 1}`;
|
|
179
|
+
break;
|
|
180
|
+
case 'like':
|
|
181
|
+
desc = `Like ${action.count} relevant tweets`;
|
|
182
|
+
break;
|
|
183
|
+
case 'reply':
|
|
184
|
+
desc = `Reply using template #${(action.replyTemplate ?? 0) + 1}`;
|
|
185
|
+
break;
|
|
186
|
+
case 'search':
|
|
187
|
+
desc = `Search for new engagement targets`;
|
|
188
|
+
break;
|
|
189
|
+
case 'dm':
|
|
190
|
+
desc = `Send warm DM`;
|
|
191
|
+
break;
|
|
192
|
+
}
|
|
193
|
+
console.log(` ${inMinutes === 0 ? 'NOW' : `in ${inMinutes}m`}: ${desc}`);
|
|
194
|
+
}
|
|
195
|
+
console.log(`\nTweet content for next post:`);
|
|
196
|
+
const nextPost = upcoming.find(a => a.type === 'post');
|
|
197
|
+
if (nextPost?.tweetIndex !== undefined) {
|
|
198
|
+
console.log(CONFIG.tweets[nextPost.tweetIndex]?.content.substring(0, 200) + '...');
|
|
199
|
+
}
|
|
200
|
+
console.log(`\nReply template for next reply:`);
|
|
201
|
+
const nextReply = upcoming.find(a => a.type === 'reply');
|
|
202
|
+
if (nextReply?.replyTemplate !== undefined) {
|
|
203
|
+
console.log(CONFIG.replyTemplates[nextReply.replyTemplate]?.substring(0, 200) + '...');
|
|
204
|
+
}
|
|
205
|
+
console.log('\n--- Campaign will be executed via ScreenHand MCP tools ---');
|
|
206
|
+
console.log('Use /loop 5m to keep the campaign runner alive');
|
|
207
|
+
}
|
|
208
|
+
export { CONFIG, SCHEDULE, createInitialState, logAction, randomDelay };
|