screenhand 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +458 -93
  2. package/dist/.audit-log.jsonl +55 -0
  3. package/dist/.screenhand/memory/.lock +1 -0
  4. package/dist/.screenhand/memory/actions.jsonl +85 -0
  5. package/dist/.screenhand/memory/errors.jsonl +5 -0
  6. package/dist/.screenhand/memory/errors.jsonl.bak +4 -0
  7. package/dist/.screenhand/memory/state.json +35 -0
  8. package/dist/.screenhand/memory/state.json.bak +35 -0
  9. package/dist/.screenhand/memory/strategies.jsonl +12 -0
  10. package/dist/agent/cli.js +73 -0
  11. package/dist/agent/loop.js +258 -0
  12. package/dist/config.js +9 -0
  13. package/dist/index.js +56 -0
  14. package/dist/logging/timeline-logger.js +29 -0
  15. package/dist/mcp/mcp-stdio-server.js +448 -0
  16. package/dist/mcp/server.js +347 -0
  17. package/dist/mcp-desktop.js +2731 -0
  18. package/dist/mcp-entry.js +59 -0
  19. package/dist/memory/recall.js +160 -0
  20. package/dist/memory/research.js +98 -0
  21. package/dist/memory/seeds.js +89 -0
  22. package/dist/memory/session.js +161 -0
  23. package/dist/memory/store.js +391 -0
  24. package/dist/memory/types.js +4 -0
  25. package/dist/monitor/codex-monitor.js +377 -0
  26. package/dist/monitor/task-queue.js +84 -0
  27. package/dist/monitor/types.js +49 -0
  28. package/dist/native/bridge-client.js +174 -0
  29. package/dist/native/macos-bridge-client.js +5 -0
  30. package/dist/npm-publish-helper.js +117 -0
  31. package/dist/npm-token-cdp.js +113 -0
  32. package/dist/npm-token-create.js +135 -0
  33. package/dist/npm-token-finish.js +126 -0
  34. package/dist/playbook/engine.js +193 -0
  35. package/dist/playbook/index.js +4 -0
  36. package/dist/playbook/recorder.js +519 -0
  37. package/dist/playbook/runner.js +392 -0
  38. package/dist/playbook/store.js +166 -0
  39. package/dist/playbook/types.js +4 -0
  40. package/dist/runtime/accessibility-adapter.js +377 -0
  41. package/dist/runtime/app-adapter.js +48 -0
  42. package/dist/runtime/applescript-adapter.js +283 -0
  43. package/dist/runtime/ax-role-map.js +80 -0
  44. package/dist/runtime/browser-adapter.js +36 -0
  45. package/dist/runtime/cdp-chrome-adapter.js +505 -0
  46. package/dist/runtime/composite-adapter.js +205 -0
  47. package/dist/runtime/executor.js +250 -0
  48. package/dist/runtime/locator-cache.js +12 -0
  49. package/dist/runtime/planning-loop.js +47 -0
  50. package/dist/runtime/service.js +372 -0
  51. package/dist/runtime/session-manager.js +28 -0
  52. package/dist/runtime/state-observer.js +105 -0
  53. package/dist/runtime/vision-adapter.js +208 -0
  54. package/dist/scripts/codex-monitor-daemon.js +335 -0
  55. package/dist/scripts/supervisor-daemon.js +272 -0
  56. package/dist/scripts/worker-daemon.js +228 -0
  57. package/dist/src/agent/cli.js +82 -0
  58. package/dist/src/agent/loop.js +274 -0
  59. package/{src/config.ts → dist/src/config.js} +5 -10
  60. package/{src/index.ts → dist/src/index.js} +32 -52
  61. package/dist/src/jobs/manager.js +237 -0
  62. package/dist/src/jobs/runner.js +683 -0
  63. package/dist/src/jobs/store.js +102 -0
  64. package/dist/src/jobs/types.js +30 -0
  65. package/dist/src/jobs/worker.js +97 -0
  66. package/dist/src/logging/timeline-logger.js +45 -0
  67. package/dist/src/mcp/mcp-stdio-server.js +464 -0
  68. package/dist/src/mcp/server.js +363 -0
  69. package/dist/src/mcp-entry.js +60 -0
  70. package/dist/src/memory/recall.js +170 -0
  71. package/dist/src/memory/research.js +104 -0
  72. package/dist/src/memory/seeds.js +101 -0
  73. package/dist/src/memory/service.js +421 -0
  74. package/dist/src/memory/session.js +169 -0
  75. package/dist/src/memory/store.js +422 -0
  76. package/dist/src/memory/types.js +17 -0
  77. package/dist/src/monitor/codex-monitor.js +382 -0
  78. package/dist/src/monitor/task-queue.js +97 -0
  79. package/dist/src/monitor/types.js +62 -0
  80. package/dist/src/native/bridge-client.js +190 -0
  81. package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
  82. package/dist/src/playbook/engine.js +201 -0
  83. package/dist/src/playbook/index.js +20 -0
  84. package/dist/src/playbook/recorder.js +535 -0
  85. package/dist/src/playbook/runner.js +408 -0
  86. package/dist/src/playbook/store.js +183 -0
  87. package/dist/src/playbook/types.js +17 -0
  88. package/dist/src/runtime/accessibility-adapter.js +393 -0
  89. package/dist/src/runtime/app-adapter.js +64 -0
  90. package/dist/src/runtime/applescript-adapter.js +299 -0
  91. package/dist/src/runtime/ax-role-map.js +96 -0
  92. package/dist/src/runtime/browser-adapter.js +52 -0
  93. package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
  94. package/dist/src/runtime/composite-adapter.js +221 -0
  95. package/dist/src/runtime/execution-contract.js +159 -0
  96. package/dist/src/runtime/executor.js +266 -0
  97. package/{src/runtime/locator-cache.ts → dist/src/runtime/locator-cache.js} +10 -15
  98. package/dist/src/runtime/planning-loop.js +63 -0
  99. package/dist/src/runtime/service.js +388 -0
  100. package/dist/src/runtime/session-manager.js +60 -0
  101. package/dist/src/runtime/state-observer.js +121 -0
  102. package/dist/src/runtime/vision-adapter.js +224 -0
  103. package/dist/src/supervisor/locks.js +186 -0
  104. package/dist/src/supervisor/supervisor.js +403 -0
  105. package/dist/src/supervisor/types.js +30 -0
  106. package/dist/src/test-mcp-protocol.js +154 -0
  107. package/dist/src/types.js +17 -0
  108. package/dist/src/util/atomic-write.js +118 -0
  109. package/dist/test-mcp-protocol.js +138 -0
  110. package/dist/types.js +1 -0
  111. package/package.json +18 -4
  112. package/.claude/commands/automate.md +0 -28
  113. package/.claude/commands/debug-ui.md +0 -19
  114. package/.claude/commands/screenshot.md +0 -15
  115. package/.github/FUNDING.yml +0 -1
  116. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
  117. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  118. package/.mcp.json +0 -8
  119. package/DESKTOP_MCP_GUIDE.md +0 -92
  120. package/SECURITY.md +0 -44
  121. package/docs/architecture.md +0 -47
  122. package/install-skills.sh +0 -19
  123. package/mcp-bridge.ts +0 -271
  124. package/mcp-desktop.ts +0 -1221
  125. package/native/macos-bridge/Package.swift +0 -21
  126. package/native/macos-bridge/Sources/AccessibilityBridge.swift +0 -261
  127. package/native/macos-bridge/Sources/AppManagement.swift +0 -129
  128. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +0 -242
  129. package/native/macos-bridge/Sources/ObserverBridge.swift +0 -120
  130. package/native/macos-bridge/Sources/VisionBridge.swift +0 -80
  131. package/native/macos-bridge/Sources/main.swift +0 -345
  132. package/native/windows-bridge/AppManagement.cs +0 -234
  133. package/native/windows-bridge/InputBridge.cs +0 -436
  134. package/native/windows-bridge/Program.cs +0 -265
  135. package/native/windows-bridge/ScreenCapture.cs +0 -329
  136. package/native/windows-bridge/UIAutomationBridge.cs +0 -571
  137. package/native/windows-bridge/WindowsBridge.csproj +0 -17
  138. package/playbooks/devpost.json +0 -186
  139. package/playbooks/instagram.json +0 -41
  140. package/playbooks/instagram_v2.json +0 -201
  141. package/playbooks/x_v1.json +0 -211
  142. package/scripts/devpost-live-loop.mjs +0 -421
  143. package/src/logging/timeline-logger.ts +0 -55
  144. package/src/mcp/server.ts +0 -449
  145. package/src/memory/recall.ts +0 -191
  146. package/src/memory/research.ts +0 -146
  147. package/src/memory/seeds.ts +0 -123
  148. package/src/memory/session.ts +0 -201
  149. package/src/memory/store.ts +0 -434
  150. package/src/memory/types.ts +0 -69
  151. package/src/native/bridge-client.ts +0 -239
  152. package/src/runtime/accessibility-adapter.ts +0 -487
  153. package/src/runtime/app-adapter.ts +0 -169
  154. package/src/runtime/applescript-adapter.ts +0 -376
  155. package/src/runtime/ax-role-map.ts +0 -102
  156. package/src/runtime/browser-adapter.ts +0 -129
  157. package/src/runtime/cdp-chrome-adapter.ts +0 -676
  158. package/src/runtime/composite-adapter.ts +0 -274
  159. package/src/runtime/executor.ts +0 -396
  160. package/src/runtime/planning-loop.ts +0 -81
  161. package/src/runtime/service.ts +0 -448
  162. package/src/runtime/session-manager.ts +0 -50
  163. package/src/runtime/state-observer.ts +0 -136
  164. package/src/runtime/vision-adapter.ts +0 -297
  165. package/src/types.ts +0 -297
  166. package/tests/bridge-client.test.ts +0 -176
  167. package/tests/browser-stealth.test.ts +0 -210
  168. package/tests/composite-adapter.test.ts +0 -64
  169. package/tests/mcp-server.test.ts +0 -151
  170. package/tests/memory-recall.test.ts +0 -339
  171. package/tests/memory-research.test.ts +0 -159
  172. package/tests/memory-seeds.test.ts +0 -120
  173. package/tests/memory-store.test.ts +0 -392
  174. package/tests/types.test.ts +0 -92
  175. package/tsconfig.check.json +0 -17
  176. package/tsconfig.json +0 -19
  177. package/vitest.config.ts +0 -8
@@ -0,0 +1,102 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * JobStore — atomic JSON persistence for jobs.
19
+ *
20
+ * All jobs are cached in memory. Writes are sync + atomic (temp+rename).
21
+ * File: jobs.json (array of Job objects).
22
+ */
23
+ import fs from "node:fs";
24
+ import path from "node:path";
25
+ import { writeFileAtomicSync, readJsonWithRecovery } from "../util/atomic-write.js";
26
+ const MAX_COMPLETED_JOBS = 200;
27
+ export class JobStore {
28
+ filePath;
29
+ jobs = [];
30
+ initialized = false;
31
+ constructor(dir) {
32
+ fs.mkdirSync(dir, { recursive: true });
33
+ this.filePath = path.join(dir, "jobs.json");
34
+ }
35
+ init() {
36
+ if (this.initialized)
37
+ return;
38
+ this.initialized = true;
39
+ this.jobs = readJsonWithRecovery(this.filePath) ?? [];
40
+ }
41
+ /** Get all jobs, optionally filtered by state. */
42
+ list(state) {
43
+ if (state)
44
+ return this.jobs.filter((j) => j.state === state);
45
+ return [...this.jobs];
46
+ }
47
+ /** Get a single job by ID. */
48
+ get(id) {
49
+ return this.jobs.find((j) => j.id === id);
50
+ }
51
+ /** Insert a new job. */
52
+ add(job) {
53
+ this.jobs.push(job);
54
+ this.persist();
55
+ }
56
+ /** Update an existing job in place, then persist. */
57
+ update(id, patch) {
58
+ const idx = this.jobs.findIndex((j) => j.id === id);
59
+ if (idx < 0)
60
+ return undefined;
61
+ this.jobs[idx] = { ...this.jobs[idx], ...patch, updatedAt: new Date().toISOString() };
62
+ this.persist();
63
+ return this.jobs[idx];
64
+ }
65
+ /** Remove a job by ID. */
66
+ remove(id) {
67
+ const before = this.jobs.length;
68
+ this.jobs = this.jobs.filter((j) => j.id !== id);
69
+ if (this.jobs.length < before) {
70
+ this.persist();
71
+ return true;
72
+ }
73
+ return false;
74
+ }
75
+ /** Evict old completed/failed jobs beyond the cap. */
76
+ prune() {
77
+ const terminal = this.jobs
78
+ .filter((j) => j.state === "done" || j.state === "failed")
79
+ .sort((a, b) => new Date(a.updatedAt).getTime() - new Date(b.updatedAt).getTime());
80
+ if (terminal.length <= MAX_COMPLETED_JOBS)
81
+ return 0;
82
+ const evictCount = terminal.length - MAX_COMPLETED_JOBS;
83
+ const evictIds = new Set(terminal.slice(0, evictCount).map((j) => j.id));
84
+ this.jobs = this.jobs.filter((j) => !evictIds.has(j.id));
85
+ this.persist();
86
+ return evictCount;
87
+ }
88
+ /** Next queued job by priority (lower number = higher priority), then creation order. */
89
+ nextQueued() {
90
+ return this.jobs
91
+ .filter((j) => j.state === "queued")
92
+ .sort((a, b) => a.priority - b.priority || new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime())[0];
93
+ }
94
+ persist() {
95
+ try {
96
+ writeFileAtomicSync(this.filePath, JSON.stringify(this.jobs, null, 2));
97
+ }
98
+ catch {
99
+ // Non-critical — in-memory cache is authoritative
100
+ }
101
+ }
102
+ }
@@ -0,0 +1,30 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * Job layer types — persistent multi-step automation jobs
19
+ * with state machine, playbook resume, and supervisor integration.
20
+ */
21
+ export const JOB_STATES = ["queued", "running", "blocked", "waiting_human", "done", "failed"];
22
+ /** Transition rules: state → allowed next states */
23
+ export const VALID_TRANSITIONS = {
24
+ queued: ["running", "failed"],
25
+ running: ["blocked", "waiting_human", "done", "failed"],
26
+ blocked: ["running", "waiting_human", "failed"],
27
+ waiting_human: ["running", "failed"],
28
+ done: [], // terminal
29
+ failed: ["queued"], // can re-queue
30
+ };
@@ -0,0 +1,97 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ /**
18
+ * JobWorker — persistent state for the worker daemon.
19
+ *
20
+ * The actual daemon runs as a separate process (scripts/worker-daemon.ts).
21
+ * This module provides the state types and filesystem persistence
22
+ * shared between the daemon and the MCP tools that query/control it.
23
+ *
24
+ * State directory: ~/.screenhand/worker/
25
+ * Files: state.json, worker.pid, worker.log
26
+ */
27
+ import fs from "node:fs";
28
+ import path from "node:path";
29
+ import os from "node:os";
30
+ import { writeFileAtomicSync, readJsonWithRecovery } from "../util/atomic-write.js";
31
+ /** Default worker directory — used by MCP tools and the daemon. */
32
+ export const WORKER_DIR = path.join(os.homedir(), ".screenhand", "worker");
33
+ export const WORKER_PID_FILE = path.join(WORKER_DIR, "worker.pid");
34
+ export const WORKER_LOG_FILE = path.join(WORKER_DIR, "worker.log");
35
+ function stateFile(dir) {
36
+ return path.join(dir, "state.json");
37
+ }
38
+ function pidFile(dir) {
39
+ return path.join(dir, "worker.pid");
40
+ }
41
+ /** Read the persisted worker state from disk. */
42
+ export function readWorkerStatus(dir = WORKER_DIR) {
43
+ return readJsonWithRecovery(stateFile(dir));
44
+ }
45
+ /** Write worker state to disk atomically. */
46
+ export function writeWorkerStatus(status, dir = WORKER_DIR) {
47
+ fs.mkdirSync(dir, { recursive: true });
48
+ writeFileAtomicSync(stateFile(dir), JSON.stringify(status, null, 2));
49
+ }
50
+ /** Check if the worker daemon is alive by reading its PID file. */
51
+ export function getWorkerDaemonPid(dir = WORKER_DIR) {
52
+ try {
53
+ const pf = pidFile(dir);
54
+ if (!fs.existsSync(pf))
55
+ return null;
56
+ const pid = Number(fs.readFileSync(pf, "utf-8").trim());
57
+ if (isNaN(pid) || pid <= 0)
58
+ return null;
59
+ // Check if process is alive (signal 0 = test existence)
60
+ process.kill(pid, 0);
61
+ return pid;
62
+ }
63
+ catch {
64
+ return null;
65
+ }
66
+ }
67
+ /** Get a live status: reads persisted state + validates PID is alive. */
68
+ export function getWorkerLiveStatus(dir = WORKER_DIR) {
69
+ const persisted = readWorkerStatus(dir);
70
+ const pid = getWorkerDaemonPid(dir);
71
+ if (!persisted) {
72
+ return {
73
+ pid: null,
74
+ running: false,
75
+ startedAt: null,
76
+ pollMs: 3000,
77
+ maxJobs: 0,
78
+ jobsProcessed: 0,
79
+ jobsDone: 0,
80
+ jobsFailed: 0,
81
+ jobsBlocked: 0,
82
+ lastJobId: null,
83
+ lastJobState: null,
84
+ uptimeMs: 0,
85
+ recentResults: [],
86
+ };
87
+ }
88
+ // If PID file says alive but process is dead, mark as not running
89
+ if (persisted.running && pid === null) {
90
+ return { ...persisted, running: false, pid: null };
91
+ }
92
+ // Update uptime from startedAt
93
+ if (persisted.running && persisted.startedAt) {
94
+ persisted.uptimeMs = Date.now() - new Date(persisted.startedAt).getTime();
95
+ }
96
+ return { ...persisted, pid };
97
+ }
@@ -0,0 +1,45 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export class TimelineLogger {
18
+ timeline = [];
19
+ start(action, sessionId) {
20
+ return {
21
+ action,
22
+ sessionId,
23
+ startedAt: new Date().toISOString(),
24
+ locateMs: 0,
25
+ actMs: 0,
26
+ verifyMs: 0,
27
+ retries: 0,
28
+ };
29
+ }
30
+ finish(telemetry, status) {
31
+ const finishedAt = new Date().toISOString();
32
+ const totalMs = new Date(finishedAt).getTime() - new Date(telemetry.startedAt).getTime();
33
+ const finalized = {
34
+ ...telemetry,
35
+ finishedAt,
36
+ totalMs,
37
+ status,
38
+ };
39
+ this.timeline.push(finalized);
40
+ return finalized;
41
+ }
42
+ getRecent(limit = 50) {
43
+ return this.timeline.slice(-limit);
44
+ }
45
+ }