screenhand 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/README.md +193 -109
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +5876 -0
  4. package/dist/scripts/codex-monitor-daemon.js +335 -0
  5. package/dist/scripts/export-help-center.js +112 -0
  6. package/dist/scripts/marketing-loop.js +117 -0
  7. package/dist/scripts/observer-daemon.js +288 -0
  8. package/dist/scripts/orchestrator-daemon.js +399 -0
  9. package/dist/scripts/supervisor-daemon.js +272 -0
  10. package/dist/scripts/threads-campaign.js +208 -0
  11. package/dist/scripts/worker-daemon.js +228 -0
  12. package/dist/src/agent/cli.js +82 -0
  13. package/dist/src/agent/loop.js +274 -0
  14. package/dist/src/community/fetcher.js +109 -0
  15. package/dist/src/community/index.js +6 -0
  16. package/dist/src/community/publisher.js +191 -0
  17. package/dist/src/community/remote-api.js +121 -0
  18. package/dist/src/community/types.js +3 -0
  19. package/dist/src/community/validator.js +95 -0
  20. package/{src/config.ts → dist/src/config.js} +5 -10
  21. package/dist/src/context-tracker.js +489 -0
  22. package/{src/index.ts → dist/src/index.js} +32 -52
  23. package/dist/src/ingestion/coverage-auditor.js +233 -0
  24. package/dist/src/ingestion/doc-parser.js +164 -0
  25. package/dist/src/ingestion/index.js +8 -0
  26. package/dist/src/ingestion/menu-scanner.js +152 -0
  27. package/dist/src/ingestion/reference-merger.js +186 -0
  28. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  29. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  30. package/dist/src/ingestion/types.js +3 -0
  31. package/dist/src/jobs/manager.js +305 -0
  32. package/dist/src/jobs/runner.js +806 -0
  33. package/dist/src/jobs/store.js +102 -0
  34. package/dist/src/jobs/types.js +30 -0
  35. package/dist/src/jobs/worker.js +97 -0
  36. package/dist/src/learning/engine.js +356 -0
  37. package/dist/src/learning/index.js +9 -0
  38. package/dist/src/learning/locator-policy.js +120 -0
  39. package/dist/src/learning/pattern-policy.js +89 -0
  40. package/dist/src/learning/recovery-policy.js +116 -0
  41. package/dist/src/learning/sensor-policy.js +115 -0
  42. package/dist/src/learning/timing-model.js +204 -0
  43. package/dist/src/learning/topology-policy.js +90 -0
  44. package/dist/src/learning/types.js +9 -0
  45. package/dist/src/logging/timeline-logger.js +48 -0
  46. package/dist/src/mcp/mcp-stdio-server.js +464 -0
  47. package/dist/src/mcp/server.js +363 -0
  48. package/dist/src/mcp-entry.js +60 -0
  49. package/dist/src/memory/playbook-seeds.js +200 -0
  50. package/dist/src/memory/recall.js +222 -0
  51. package/dist/src/memory/research.js +104 -0
  52. package/dist/src/memory/seeds.js +101 -0
  53. package/dist/src/memory/service.js +446 -0
  54. package/dist/src/memory/session.js +169 -0
  55. package/dist/src/memory/store.js +451 -0
  56. package/{src/runtime/locator-cache.ts → dist/src/memory/types.js} +1 -17
  57. package/dist/src/monitor/codex-monitor.js +382 -0
  58. package/dist/src/monitor/task-queue.js +97 -0
  59. package/dist/src/monitor/types.js +62 -0
  60. package/dist/src/native/bridge-client.js +412 -0
  61. package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
  62. package/dist/src/observer/state.js +199 -0
  63. package/dist/src/observer/types.js +43 -0
  64. package/dist/src/orchestrator/state.js +68 -0
  65. package/dist/src/orchestrator/types.js +22 -0
  66. package/dist/src/perception/ax-source.js +162 -0
  67. package/dist/src/perception/cdp-source.js +162 -0
  68. package/dist/src/perception/coordinator.js +771 -0
  69. package/dist/src/perception/frame-differ.js +287 -0
  70. package/dist/src/perception/index.js +22 -0
  71. package/dist/src/perception/manager.js +199 -0
  72. package/dist/src/perception/types.js +47 -0
  73. package/dist/src/perception/vision-source.js +399 -0
  74. package/dist/src/planner/deterministic.js +298 -0
  75. package/dist/src/planner/executor.js +870 -0
  76. package/dist/src/planner/goal-store.js +92 -0
  77. package/dist/src/planner/index.js +21 -0
  78. package/dist/src/planner/planner.js +520 -0
  79. package/dist/src/planner/tool-registry.js +71 -0
  80. package/dist/src/planner/types.js +22 -0
  81. package/dist/src/platform/explorer.js +213 -0
  82. package/dist/src/platform/help-center-markdown.js +527 -0
  83. package/dist/src/platform/learner.js +257 -0
  84. package/dist/src/playbook/engine.js +486 -0
  85. package/dist/src/playbook/index.js +20 -0
  86. package/dist/src/playbook/mcp-recorder.js +204 -0
  87. package/dist/src/playbook/recorder.js +536 -0
  88. package/dist/src/playbook/runner.js +408 -0
  89. package/dist/src/playbook/store.js +312 -0
  90. package/dist/src/playbook/types.js +17 -0
  91. package/dist/src/recovery/detectors.js +156 -0
  92. package/dist/src/recovery/engine.js +327 -0
  93. package/dist/src/recovery/index.js +20 -0
  94. package/dist/src/recovery/strategies.js +274 -0
  95. package/dist/src/recovery/types.js +20 -0
  96. package/dist/src/runtime/accessibility-adapter.js +430 -0
  97. package/dist/src/runtime/app-adapter.js +64 -0
  98. package/dist/src/runtime/applescript-adapter.js +305 -0
  99. package/dist/src/runtime/ax-role-map.js +96 -0
  100. package/dist/src/runtime/browser-adapter.js +52 -0
  101. package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
  102. package/dist/src/runtime/composite-adapter.js +221 -0
  103. package/dist/src/runtime/execution-contract.js +159 -0
  104. package/dist/src/runtime/executor.js +286 -0
  105. package/dist/src/runtime/locator-cache.js +50 -0
  106. package/dist/src/runtime/planning-loop.js +63 -0
  107. package/dist/src/runtime/service.js +432 -0
  108. package/dist/src/runtime/session-manager.js +63 -0
  109. package/dist/src/runtime/state-observer.js +121 -0
  110. package/dist/src/runtime/vision-adapter.js +225 -0
  111. package/dist/src/state/app-map-types.js +72 -0
  112. package/dist/src/state/app-map.js +1974 -0
  113. package/dist/src/state/entity-tracker.js +108 -0
  114. package/dist/src/state/fusion.js +96 -0
  115. package/dist/src/state/index.js +21 -0
  116. package/dist/src/state/ladder-generator.js +236 -0
  117. package/dist/src/state/persistence.js +156 -0
  118. package/dist/src/state/types.js +17 -0
  119. package/dist/src/state/world-model.js +1456 -0
  120. package/dist/src/supervisor/locks.js +186 -0
  121. package/dist/src/supervisor/supervisor.js +403 -0
  122. package/dist/src/supervisor/types.js +30 -0
  123. package/dist/src/test-mcp-protocol.js +154 -0
  124. package/dist/src/types.js +17 -0
  125. package/dist/src/util/atomic-write.js +133 -0
  126. package/dist/src/util/sanitize.js +146 -0
  127. package/dist-app-maps/com.figma.Desktop.json +959 -0
  128. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  129. package/dist-app-maps/notion.id.json +2831 -0
  130. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  131. package/dist-playbooks/codex-desktop.json +76 -0
  132. package/dist-playbooks/competitor-research-stack.json +122 -0
  133. package/dist-playbooks/davinci-color-grade.json +153 -0
  134. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  135. package/dist-playbooks/davinci-render.json +114 -0
  136. package/dist-playbooks/devto.json +52 -0
  137. package/dist-playbooks/discord.json +41 -0
  138. package/dist-playbooks/google-flow-create-project.json +59 -0
  139. package/dist-playbooks/google-flow-edit-image.json +90 -0
  140. package/dist-playbooks/google-flow-edit-video.json +90 -0
  141. package/dist-playbooks/google-flow-generate-image.json +68 -0
  142. package/dist-playbooks/google-flow-generate-video.json +191 -0
  143. package/dist-playbooks/google-flow-open-project.json +48 -0
  144. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  145. package/dist-playbooks/google-flow-search-assets.json +64 -0
  146. package/dist-playbooks/instagram.json +57 -0
  147. package/dist-playbooks/linkedin.json +52 -0
  148. package/dist-playbooks/n8n.json +43 -0
  149. package/dist-playbooks/reddit.json +52 -0
  150. package/dist-playbooks/threads.json +59 -0
  151. package/dist-playbooks/x-twitter.json +59 -0
  152. package/dist-playbooks/youtube.json +59 -0
  153. package/dist-references/canva.json +646 -0
  154. package/dist-references/codex-desktop.json +305 -0
  155. package/dist-references/davinci-resolve-keyboard.json +594 -0
  156. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  157. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  158. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  159. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  160. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  161. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  162. package/dist-references/devto.json +317 -0
  163. package/dist-references/discord.json +549 -0
  164. package/dist-references/figma.json +1186 -0
  165. package/dist-references/finder.json +146 -0
  166. package/dist-references/google-ads-transparency.json +95 -0
  167. package/dist-references/google-flow.json +649 -0
  168. package/dist-references/instagram.json +341 -0
  169. package/dist-references/linkedin.json +324 -0
  170. package/dist-references/meta-ad-library.json +86 -0
  171. package/dist-references/n8n.json +387 -0
  172. package/dist-references/notes.json +27 -0
  173. package/dist-references/notion.json +163 -0
  174. package/dist-references/reddit.json +341 -0
  175. package/dist-references/threads.json +337 -0
  176. package/dist-references/x-twitter.json +403 -0
  177. package/dist-references/youtube.json +373 -0
  178. package/native/macos-bridge/Package.swift +1 -0
  179. package/native/macos-bridge/Sources/AccessibilityBridge.swift +257 -36
  180. package/native/macos-bridge/Sources/AppManagement.swift +212 -2
  181. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +348 -53
  182. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  183. package/native/macos-bridge/Sources/VisionBridge.swift +165 -7
  184. package/native/macos-bridge/Sources/main.swift +169 -16
  185. package/native/windows-bridge/Program.cs +5 -0
  186. package/native/windows-bridge/ScreenCapture.cs +124 -0
  187. package/package.json +29 -4
  188. package/scripts/postinstall.cjs +127 -0
  189. package/.claude/commands/automate.md +0 -28
  190. package/.claude/commands/debug-ui.md +0 -19
  191. package/.claude/commands/screenshot.md +0 -15
  192. package/.github/FUNDING.yml +0 -1
  193. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
  194. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  195. package/.mcp.json +0 -8
  196. package/DESKTOP_MCP_GUIDE.md +0 -92
  197. package/SECURITY.md +0 -44
  198. package/docs/architecture.md +0 -47
  199. package/install-skills.sh +0 -19
  200. package/mcp-bridge.ts +0 -271
  201. package/mcp-desktop.ts +0 -1221
  202. package/playbooks/instagram.json +0 -41
  203. package/playbooks/instagram_v2.json +0 -201
  204. package/playbooks/x_v1.json +0 -211
  205. package/scripts/devpost-live-loop.mjs +0 -421
  206. package/src/logging/timeline-logger.ts +0 -55
  207. package/src/mcp/server.ts +0 -449
  208. package/src/memory/recall.ts +0 -191
  209. package/src/memory/research.ts +0 -146
  210. package/src/memory/seeds.ts +0 -123
  211. package/src/memory/session.ts +0 -201
  212. package/src/memory/store.ts +0 -434
  213. package/src/memory/types.ts +0 -69
  214. package/src/native/bridge-client.ts +0 -239
  215. package/src/runtime/accessibility-adapter.ts +0 -487
  216. package/src/runtime/app-adapter.ts +0 -169
  217. package/src/runtime/applescript-adapter.ts +0 -376
  218. package/src/runtime/ax-role-map.ts +0 -102
  219. package/src/runtime/browser-adapter.ts +0 -129
  220. package/src/runtime/cdp-chrome-adapter.ts +0 -676
  221. package/src/runtime/composite-adapter.ts +0 -274
  222. package/src/runtime/executor.ts +0 -396
  223. package/src/runtime/planning-loop.ts +0 -81
  224. package/src/runtime/service.ts +0 -448
  225. package/src/runtime/session-manager.ts +0 -50
  226. package/src/runtime/state-observer.ts +0 -136
  227. package/src/runtime/vision-adapter.ts +0 -297
  228. package/src/types.ts +0 -297
  229. package/tests/bridge-client.test.ts +0 -176
  230. package/tests/browser-stealth.test.ts +0 -210
  231. package/tests/composite-adapter.test.ts +0 -64
  232. package/tests/mcp-server.test.ts +0 -151
  233. package/tests/memory-recall.test.ts +0 -339
  234. package/tests/memory-research.test.ts +0 -159
  235. package/tests/memory-seeds.test.ts +0 -120
  236. package/tests/memory-store.test.ts +0 -392
  237. package/tests/types.test.ts +0 -92
  238. package/tsconfig.check.json +0 -17
  239. package/tsconfig.json +0 -19
  240. package/vitest.config.ts +0 -8
  241. /package/{playbooks → dist-references}/devpost.json +0 -0
@@ -1,239 +0,0 @@
1
- // Copyright (C) 2025 Clazro Technology Private Limited
2
- // SPDX-License-Identifier: AGPL-3.0-only
3
- //
4
- // This file is part of ScreenHand.
5
- //
6
- // ScreenHand is free software: you can redistribute it and/or modify
7
- // it under the terms of the GNU Affero General Public License as
8
- // published by the Free Software Foundation, version 3.
9
- //
10
- // ScreenHand is distributed in the hope that it will be useful,
11
- // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
- // GNU Affero General Public License for more details.
14
- //
15
- // You should have received a copy of the GNU Affero General Public License
16
- // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
-
18
- import { spawn, type ChildProcess } from "node:child_process";
19
- import { EventEmitter } from "node:events";
20
- import path from "node:path";
21
- import { createInterface } from "node:readline";
22
-
23
- interface JsonRpcRequest {
24
- id: number;
25
- method: string;
26
- params?: Record<string, unknown>;
27
- }
28
-
29
- interface JsonRpcResponse {
30
- id: number;
31
- result?: unknown;
32
- error?: { code: number; message: string };
33
- event?: Record<string, unknown>;
34
- }
35
-
36
- interface PendingRequest {
37
- resolve: (value: unknown) => void;
38
- reject: (error: Error) => void;
39
- timer: ReturnType<typeof setTimeout>;
40
- }
41
-
42
- /**
43
- * Per-method timeout overrides (ms).
44
- * Methods not listed here use the default 10s timeout.
45
- */
46
- const METHOD_TIMEOUTS: Record<string, number> = {
47
- "app.launch": 30_000,
48
- "cg.captureScreen": 15_000,
49
- "cg.captureWindow": 15_000,
50
- "vision.ocr": 20_000,
51
- "vision.findText": 20_000,
52
- };
53
-
54
- /**
55
- * Resolves the correct native bridge binary path for the current platform.
56
- */
57
- function defaultBinaryPath(): string {
58
- const base = import.meta.dirname ?? process.cwd();
59
-
60
- if (process.platform === "win32") {
61
- return path.resolve(
62
- base,
63
- "../../native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe",
64
- );
65
- }
66
-
67
- // macOS (default)
68
- return path.resolve(
69
- base,
70
- "../../native/macos-bridge/.build/release/macos-bridge",
71
- );
72
- }
73
-
74
- /**
75
- * Platform-aware native bridge client.
76
- * Spawns the correct bridge binary (macOS Swift or Windows C#) based on the OS,
77
- * communicating via the same JSON-RPC-over-stdio protocol.
78
- *
79
- * Drop-in replacement for the original MacOSBridgeClient.
80
- */
81
- export class BridgeClient extends EventEmitter {
82
- private process: ChildProcess | null = null;
83
- private nextId = 1;
84
- private pending = new Map<number, PendingRequest>();
85
- private readonly binaryPath: string;
86
- private restarting = false;
87
- private started = false;
88
-
89
- constructor(binaryPath?: string) {
90
- super();
91
- this.binaryPath = binaryPath ?? defaultBinaryPath();
92
- }
93
-
94
- async start(): Promise<void> {
95
- if (this.started) return;
96
- await this.spawn();
97
- this.started = true;
98
- }
99
-
100
- async stop(): Promise<void> {
101
- this.started = false;
102
- if (this.process) {
103
- this.process.kill();
104
- this.process = null;
105
- }
106
- // Reject all pending
107
- for (const [id, pending] of this.pending) {
108
- clearTimeout(pending.timer);
109
- pending.reject(new Error("Bridge stopped"));
110
- this.pending.delete(id);
111
- }
112
- }
113
-
114
- async call<T = unknown>(method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<T> {
115
- const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
116
- if (!this.process || this.process.exitCode !== null) {
117
- await this.restart();
118
- }
119
-
120
- const id = this.nextId++;
121
- const request: JsonRpcRequest = { id, method };
122
- if (params) {
123
- request.params = params;
124
- }
125
-
126
- return new Promise<T>((resolve, reject) => {
127
- const timer = setTimeout(() => {
128
- this.pending.delete(id);
129
- reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
130
- }, effectiveTimeout);
131
-
132
- this.pending.set(id, {
133
- resolve: resolve as (value: unknown) => void,
134
- reject,
135
- timer,
136
- });
137
-
138
- const line = JSON.stringify(request) + "\n";
139
- this.process!.stdin!.write(line);
140
- });
141
- }
142
-
143
- async ping(): Promise<{ pong: boolean; pid: number; accessible: boolean }> {
144
- return this.call("ping");
145
- }
146
-
147
- async checkPermissions(): Promise<{ trusted: boolean }> {
148
- return this.call("check_permissions");
149
- }
150
-
151
- private async spawn(): Promise<void> {
152
- const child = spawn(this.binaryPath, [], {
153
- stdio: ["pipe", "pipe", "pipe"],
154
- });
155
-
156
- child.on("error", (err) => {
157
- this.emit("error", err);
158
- if (this.started) {
159
- this.restart().catch(() => {});
160
- }
161
- });
162
-
163
- child.on("exit", (code) => {
164
- this.emit("exit", code);
165
- if (this.started && !this.restarting) {
166
- this.restart().catch(() => {});
167
- }
168
- });
169
-
170
- // Parse stdout line by line
171
- const rl = createInterface({ input: child.stdout! });
172
- rl.on("line", (line) => {
173
- this.handleLine(line);
174
- });
175
-
176
- // Log stderr
177
- child.stderr?.on("data", (data: Buffer) => {
178
- this.emit("stderr", data.toString());
179
- });
180
-
181
- this.process = child;
182
- }
183
-
184
- private handleLine(line: string): void {
185
- let response: JsonRpcResponse;
186
- try {
187
- response = JSON.parse(line) as JsonRpcResponse;
188
- } catch {
189
- return; // Ignore malformed lines
190
- }
191
-
192
- // Event (streaming notification from observer)
193
- if (response.event) {
194
- this.emit("ax-event", response.event);
195
- return;
196
- }
197
-
198
- // Response to a pending request
199
- const pending = this.pending.get(response.id);
200
- if (!pending) return;
201
-
202
- this.pending.delete(response.id);
203
- clearTimeout(pending.timer);
204
-
205
- if (response.error) {
206
- pending.reject(new Error(response.error.message));
207
- } else {
208
- pending.resolve(response.result);
209
- }
210
- }
211
-
212
- private async restart(): Promise<void> {
213
- if (this.restarting) return;
214
- this.restarting = true;
215
-
216
- // Reject all pending requests
217
- for (const [id, pending] of this.pending) {
218
- clearTimeout(pending.timer);
219
- pending.reject(new Error("Bridge process crashed, restarting"));
220
- this.pending.delete(id);
221
- }
222
-
223
- try {
224
- if (this.process) {
225
- this.process.kill();
226
- this.process = null;
227
- }
228
- await this.spawn();
229
- this.emit("restart");
230
- } finally {
231
- this.restarting = false;
232
- }
233
- }
234
- }
235
-
236
- /**
237
- * @deprecated Use BridgeClient instead. This alias exists for backward compatibility.
238
- */
239
- export const MacOSBridgeClient = BridgeClient;