screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,52 @@
1
+ {
2
+ "id": "linkedin",
3
+ "name": "LinkedIn Automation",
4
+ "description": "Battle-tested playbook for LinkedIn browser automation via CDP. Covers posting, liking, commenting, connecting, messaging, search, and profile management.",
5
+ "platform": "linkedin",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*linkedin.com*"
9
+ ],
10
+ "tags": [
11
+ "linkedin",
12
+ "social",
13
+ "professional",
14
+ "browser",
15
+ "cdp"
16
+ ],
17
+ "successCount": 0,
18
+ "failCount": 0,
19
+ "steps": [
20
+ {
21
+ "action": "navigate",
22
+ "url": "https://www.linkedin.com/feed/",
23
+ "description": "Open LinkedIn feed"
24
+ },
25
+ {
26
+ "action": "wait",
27
+ "ms": 2000,
28
+ "description": "Wait for feed to load"
29
+ },
30
+ {
31
+ "action": "extract",
32
+ "target": ".feed-shared-update-v2",
33
+ "format": "text",
34
+ "description": "Extract visible posts"
35
+ },
36
+ {
37
+ "action": "scroll",
38
+ "direction": "down",
39
+ "amount": 5,
40
+ "description": "Scroll for more posts"
41
+ },
42
+ {
43
+ "action": "wait",
44
+ "ms": 1500,
45
+ "description": "Wait for lazy-loaded posts"
46
+ },
47
+ {
48
+ "action": "screenshot",
49
+ "description": "Capture feed state"
50
+ }
51
+ ]
52
+ }
@@ -0,0 +1,43 @@
1
+ {
2
+ "id": "n8n",
3
+ "name": "n8n Workflow Automation",
4
+ "description": "Battle-tested playbook for n8n self-hosted workflow automation via CDP. Covers workflow creation, node management, execution, publishing, credentials, settings, and the visual workflow editor.",
5
+ "platform": "n8n",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*n8n*",
9
+ "*automation.*"
10
+ ],
11
+ "tags": [
12
+ "n8n",
13
+ "workflow",
14
+ "automation",
15
+ "browser",
16
+ "cdp",
17
+ "self-hosted"
18
+ ],
19
+ "successCount": 0,
20
+ "failCount": 0,
21
+ "steps": [
22
+ {
23
+ "action": "navigate",
24
+ "url": "{baseUrl}/home/workflows",
25
+ "description": "Open n8n workflows home"
26
+ },
27
+ {
28
+ "action": "wait",
29
+ "ms": 2000,
30
+ "description": "Wait for dashboard to load"
31
+ },
32
+ {
33
+ "action": "extract",
34
+ "target": "[data-test-id='tab-WorkflowsView']",
35
+ "format": "text",
36
+ "description": "Extract workflows list"
37
+ },
38
+ {
39
+ "action": "screenshot",
40
+ "description": "Capture n8n dashboard state"
41
+ }
42
+ ]
43
+ }
@@ -0,0 +1,52 @@
1
+ {
2
+ "id": "reddit",
3
+ "name": "Reddit Automation",
4
+ "description": "Battle-tested playbook for Reddit browser automation via CDP. Covers feed browsing, upvote/downvote, comment, create post, search, join subreddit, and profile viewing. Reddit uses shadow DOM extensively (shreddit-post, faceplate-textarea-input) and Lexical editor for text input.",
5
+ "platform": "reddit",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*reddit.com*"
9
+ ],
10
+ "tags": [
11
+ "reddit",
12
+ "social",
13
+ "browser",
14
+ "cdp",
15
+ "shadow-dom"
16
+ ],
17
+ "successCount": 0,
18
+ "failCount": 0,
19
+ "steps": [
20
+ {
21
+ "action": "navigate",
22
+ "url": "https://www.reddit.com/",
23
+ "description": "Open Reddit home feed"
24
+ },
25
+ {
26
+ "action": "wait",
27
+ "ms": 2000,
28
+ "description": "Wait for feed to load"
29
+ },
30
+ {
31
+ "action": "extract",
32
+ "target": "shreddit-post",
33
+ "format": "attributes",
34
+ "description": "Extract post titles, authors, scores from shreddit-post attributes"
35
+ },
36
+ {
37
+ "action": "scroll",
38
+ "direction": "down",
39
+ "amount": 5,
40
+ "description": "Scroll for more posts"
41
+ },
42
+ {
43
+ "action": "wait",
44
+ "ms": 1500,
45
+ "description": "Wait for lazy-loaded posts"
46
+ },
47
+ {
48
+ "action": "screenshot",
49
+ "description": "Capture current feed state"
50
+ }
51
+ ]
52
+ }
@@ -0,0 +1,59 @@
1
+ {
2
+ "id": "threads",
3
+ "name": "Threads Automation",
4
+ "description": "Battle-tested playbook for Threads (threads.com) browser automation via CDP. Covers feed, like, reply, repost, share, create post, search, follow/unfollow, and profile viewing.",
5
+ "platform": "threads",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*threads.com*",
9
+ "*threads.net*"
10
+ ],
11
+ "tags": [
12
+ "threads",
13
+ "social",
14
+ "browser",
15
+ "cdp",
16
+ "meta"
17
+ ],
18
+ "successCount": 0,
19
+ "failCount": 0,
20
+ "steps": [
21
+ {
22
+ "action": "navigate",
23
+ "url": "https://www.threads.com/",
24
+ "description": "Open Threads home feed"
25
+ },
26
+ {
27
+ "action": "wait",
28
+ "ms": 1500,
29
+ "description": "Wait for feed to load"
30
+ },
31
+ {
32
+ "action": "extract",
33
+ "target": "div[data-pressable-container='true']",
34
+ "format": "text",
35
+ "description": "Extract first visible post content"
36
+ },
37
+ {
38
+ "action": "scroll",
39
+ "direction": "down",
40
+ "amount": 5,
41
+ "description": "Scroll down to see more posts"
42
+ },
43
+ {
44
+ "action": "wait",
45
+ "ms": 1000,
46
+ "description": "Wait for new posts to load"
47
+ },
48
+ {
49
+ "action": "extract",
50
+ "target": "div[data-pressable-container='true']",
51
+ "format": "text",
52
+ "description": "Extract posts after scroll"
53
+ },
54
+ {
55
+ "action": "screenshot",
56
+ "description": "Capture current feed state"
57
+ }
58
+ ]
59
+ }
@@ -0,0 +1,59 @@
1
+ {
2
+ "id": "x-twitter",
3
+ "name": "X/Twitter Automation",
4
+ "description": "Battle-tested playbook for X/Twitter browser automation via CDP. Covers feed, like, reply, retweet, bookmark, create post, search, follow/unfollow, and profile viewing. X has excellent data-testid attributes for reliable selectors.",
5
+ "platform": "x-twitter",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*x.com*",
9
+ "*twitter.com*"
10
+ ],
11
+ "tags": [
12
+ "x",
13
+ "twitter",
14
+ "social",
15
+ "browser",
16
+ "cdp"
17
+ ],
18
+ "successCount": 0,
19
+ "failCount": 0,
20
+ "steps": [
21
+ {
22
+ "action": "navigate",
23
+ "url": "https://x.com/home",
24
+ "description": "Open X home timeline"
25
+ },
26
+ {
27
+ "action": "wait",
28
+ "ms": 2000,
29
+ "description": "Wait for timeline to load"
30
+ },
31
+ {
32
+ "action": "extract",
33
+ "target": "article[data-testid='tweet']",
34
+ "format": "text",
35
+ "description": "Extract first visible tweets"
36
+ },
37
+ {
38
+ "action": "scroll",
39
+ "direction": "down",
40
+ "amount": 5,
41
+ "description": "Scroll down for more tweets"
42
+ },
43
+ {
44
+ "action": "wait",
45
+ "ms": 1000,
46
+ "description": "Wait for new tweets to load"
47
+ },
48
+ {
49
+ "action": "extract",
50
+ "target": "article[data-testid='tweet']",
51
+ "format": "text",
52
+ "description": "Extract tweets after scroll"
53
+ },
54
+ {
55
+ "action": "screenshot",
56
+ "description": "Capture current timeline state"
57
+ }
58
+ ]
59
+ }
@@ -0,0 +1,59 @@
1
+ {
2
+ "id": "youtube",
3
+ "name": "YouTube Automation",
4
+ "description": "Battle-tested playbook for YouTube browser automation via CDP. Covers channel management, video upload, search, like, comment, subscribe, and YouTube Studio operations.",
5
+ "platform": "youtube",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*youtube.com*",
9
+ "*studio.youtube.com*"
10
+ ],
11
+ "tags": [
12
+ "youtube",
13
+ "video",
14
+ "browser",
15
+ "cdp",
16
+ "google"
17
+ ],
18
+ "successCount": 0,
19
+ "failCount": 0,
20
+ "steps": [
21
+ {
22
+ "action": "navigate",
23
+ "url": "https://www.youtube.com/",
24
+ "description": "Open YouTube home feed"
25
+ },
26
+ {
27
+ "action": "wait",
28
+ "ms": 2000,
29
+ "description": "Wait for feed to load"
30
+ },
31
+ {
32
+ "action": "extract",
33
+ "target": "ytd-rich-item-renderer",
34
+ "format": "text",
35
+ "description": "Extract visible video cards"
36
+ },
37
+ {
38
+ "action": "scroll",
39
+ "direction": "down",
40
+ "amount": 5,
41
+ "description": "Scroll for more videos"
42
+ },
43
+ {
44
+ "action": "wait",
45
+ "ms": 1000,
46
+ "description": "Wait for lazy-loaded videos"
47
+ },
48
+ {
49
+ "action": "extract",
50
+ "target": "ytd-rich-item-renderer",
51
+ "format": "text",
52
+ "description": "Extract videos after scroll"
53
+ },
54
+ {
55
+ "action": "screenshot",
56
+ "description": "Capture current feed state"
57
+ }
58
+ ]
59
+ }