screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -36,11 +36,19 @@ export function writeFileAtomicSync(filePath, data) {
36
36
  try {
37
37
  fs.writeFileSync(tmp, data, { mode: 0o644 });
38
38
  // Back up current file before overwriting (ignore if it doesn't exist yet)
39
+ // V3: Check for symlinks before backup to prevent data exfiltration
39
40
  try {
40
- fs.copyFileSync(filePath, filePath + ".bak");
41
+ const stat = fs.lstatSync(filePath);
42
+ if (stat.isSymbolicLink()) {
43
+ // Target is a symlink — skip backup and remove the symlink before writing
44
+ fs.unlinkSync(filePath);
45
+ }
46
+ else {
47
+ fs.copyFileSync(filePath, filePath + ".bak");
48
+ }
41
49
  }
42
50
  catch {
43
- // No existing file to back up — fine
51
+ // No existing file to back up (ENOENT) — fine
44
52
  }
45
53
  fs.renameSync(tmp, filePath);
46
54
  }
@@ -70,10 +78,17 @@ export function writeFileAtomic(filePath, data, callback) {
70
78
  return;
71
79
  }
72
80
  // Back up current file (best-effort, sync is fine for a copy)
81
+ // V3: Check for symlinks before backup to prevent data exfiltration
73
82
  try {
74
- fs.copyFileSync(filePath, filePath + ".bak");
83
+ const stat = fs.lstatSync(filePath);
84
+ if (stat.isSymbolicLink()) {
85
+ fs.unlinkSync(filePath);
86
+ }
87
+ else {
88
+ fs.copyFileSync(filePath, filePath + ".bak");
89
+ }
75
90
  }
76
- catch { /* ignore */ }
91
+ catch { /* ignore — ENOENT means no file to back up */ }
77
92
  fs.rename(tmp, filePath, (renameErr) => {
78
93
  if (renameErr) {
79
94
  try {
@@ -0,0 +1,146 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /**
4
+ * Shared sanitization utilities for redacting sensitive data from tool outputs.
5
+ * Used by both the world model (state persistence) and MCP tool responses.
6
+ */
7
+ import { execSync } from "node:child_process";
8
+ import os from "node:os";
9
+ const MAX_STRING_LENGTH = 1000;
10
+ const ALLOWED_URL_PROTOCOLS = new Set(["http:", "https:", "about:", "chrome:", "chrome-extension:"]);
11
+ const SENSITIVE_URL_PARAMS = new Set([
12
+ "code", "token", "access_token", "refresh_token", "id_token",
13
+ "secret", "password", "key", "api_key", "apikey", "auth",
14
+ "session", "session_id", "sessionid", "state", "nonce",
15
+ ]);
16
+ const SENSITIVE_LABEL_PATTERNS = [
17
+ // email:password in window titles (e.g. "user@example.com:P@ssw0rd! - Chrome")
18
+ [/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+:[^\s]+/g, "[CREDENTIALS_REDACTED]"],
19
+ // Bearer tokens
20
+ [/Bearer\s+[A-Za-z0-9\-._~+/]+=*/g, "[BEARER_REDACTED]"],
21
+ ];
22
+ /**
23
+ * Sanitize untrusted strings: truncate + strip control characters.
24
+ */
25
+ export function sanitizeString(s) {
26
+ // eslint-disable-next-line no-control-regex
27
+ const stripped = s.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
28
+ return stripped.length > MAX_STRING_LENGTH
29
+ ? stripped.slice(0, MAX_STRING_LENGTH)
30
+ : stripped;
31
+ }
32
+ /**
33
+ * Sanitize a URL: validate protocol + redact sensitive query params.
34
+ */
35
+ export function sanitizeUrl(url) {
36
+ try {
37
+ const parsed = new URL(url);
38
+ if (!ALLOWED_URL_PROTOCOLS.has(parsed.protocol))
39
+ return "about:blocked";
40
+ let redacted = false;
41
+ for (const paramName of parsed.searchParams.keys()) {
42
+ if (SENSITIVE_URL_PARAMS.has(paramName.toLowerCase())) {
43
+ parsed.searchParams.set(paramName, "[REDACTED]");
44
+ redacted = true;
45
+ }
46
+ }
47
+ return redacted ? parsed.toString() : url;
48
+ }
49
+ catch {
50
+ return "about:blocked";
51
+ }
52
+ }
53
+ /**
54
+ * Check if a string looks like a token/key (32+ chars, 3+ character classes).
55
+ */
56
+ function looksLikeToken(s) {
57
+ if (s.length < 32)
58
+ return false;
59
+ const hasUpper = /[A-Z]/.test(s);
60
+ const hasLower = /[a-z]/.test(s);
61
+ const hasDigit = /[0-9]/.test(s);
62
+ const hasSpecial = /[\-._~+/]/.test(s);
63
+ const classes = [hasUpper, hasLower, hasDigit, hasSpecial].filter(Boolean).length;
64
+ return classes >= 3;
65
+ }
66
+ /**
67
+ * Redact sensitive patterns from labels/titles: credentials, bearer tokens, long token strings.
68
+ */
69
+ export function redactSensitiveLabel(label) {
70
+ let result = label;
71
+ for (const [pattern, replacement] of SENSITIVE_LABEL_PATTERNS) {
72
+ result = result.replace(pattern, replacement);
73
+ }
74
+ result = result.replace(/[A-Za-z0-9\-._~+/]{32,}={0,2}/g, (match) => looksLikeToken(match) ? "[TOKEN_REDACTED]" : match);
75
+ return result;
76
+ }
77
+ /**
78
+ * Redact PII patterns from text for persistence paths (Option C: redact on write, not on read).
79
+ * Catches: email addresses, phone numbers, and the local user's name parts.
80
+ * Applied to: memory actions/strategies, playbook exports, timeline logs.
81
+ * NOT applied to: live tool responses (ocr, ui_tree, screenshot, browser_dom, world_state).
82
+ */
83
+ export function redactPII(text) {
84
+ let result = text;
85
+ // Email addresses
86
+ result = result.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, "[EMAIL_REDACTED]");
87
+ // Phone numbers — international and domestic formats
88
+ result = result.replace(/(?:\+\d{1,3}[\s\-]?)?\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4}/g, "[PHONE_REDACTED]");
89
+ // Local user name parts (from OS)
90
+ result = redactUsername(result);
91
+ // Credential patterns already handled by redactSensitiveLabel
92
+ result = redactSensitiveLabel(result);
93
+ return result;
94
+ }
95
+ /**
96
+ * Redact the macOS username from strings (e.g. menu labels like "Log Out username").
97
+ */
98
+ let _cachedNameParts = null;
99
+ /** Collect all name parts to redact: short username, full name from id -F, home dir name. */
100
+ function getNameParts() {
101
+ if (_cachedNameParts !== null)
102
+ return _cachedNameParts;
103
+ const parts = new Set();
104
+ // Short username from env
105
+ const username = process.env.USER || process.env.USERNAME || "";
106
+ if (username.length >= 2)
107
+ parts.add(username);
108
+ // Home directory basename (e.g. /Users/khushi → khushi)
109
+ try {
110
+ const home = os.homedir();
111
+ const homeName = home.split("/").pop() || "";
112
+ if (homeName.length >= 2)
113
+ parts.add(homeName);
114
+ }
115
+ catch { /* ignore */ }
116
+ // Full display name from id -F (macOS only)
117
+ try {
118
+ const fullName = execSync("id -F", { encoding: "utf-8", timeout: 2000, stdio: ["pipe", "pipe", "pipe"] }).trim();
119
+ if (fullName.length >= 2) {
120
+ parts.add(fullName);
121
+ // Also add individual name words (e.g. "khushi goyal" → "khushi", "goyal")
122
+ for (const word of fullName.split(/\s+/)) {
123
+ if (word.length >= 2)
124
+ parts.add(word);
125
+ }
126
+ }
127
+ }
128
+ catch { /* not macOS or id -F unavailable */ }
129
+ // Sort longest first so longer matches take priority
130
+ _cachedNameParts = [...parts].sort((a, b) => b.length - a.length);
131
+ return _cachedNameParts;
132
+ }
133
+ /**
134
+ * Redact the macOS username, full display name, and individual name parts from strings.
135
+ * Also catches common patterns like "Log Out <name>" where the name may not match env vars.
136
+ */
137
+ export function redactUsername(text) {
138
+ let result = text;
139
+ for (const part of getNameParts()) {
140
+ result = result.replaceAll(part, "[USER]");
141
+ }
142
+ // Catch "Log Out <name>" pattern — macOS always formats this as "Log Out <full display name>"
143
+ // The name part may already be partially redacted (e.g. "[USER] goyal"), so match everything after "Log Out "
144
+ result = result.replace(/Log Out [^\n:]+/g, "Log Out [USER]");
145
+ return result;
146
+ }