screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,445 @@
1
+ {
2
+ "id": "canva-screenhand-carousel",
3
+ "name": "Canva — ScreenHand Carousel Rewrite",
4
+ "description": "Rewrite the currently open 4-page Canva tech carousel template into the ScreenHand marketing carousel. Assumes the Canva editor is already open in Chrome on a 4-page template similar to 'Black Purple Simple Tech Carousel Instagram Post'. Rewrites the main copy, updates small SH brand labels, and performs a few optional footer cleanups.",
5
+ "platform": "canva",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "https://www.canva.com/design/*/edit*"
9
+ ],
10
+ "tags": [
11
+ "canva",
12
+ "carousel",
13
+ "screenhand",
14
+ "marketing",
15
+ "template",
16
+ "chrome",
17
+ "cdp"
18
+ ],
19
+ "successCount": 0,
20
+ "failCount": 0,
21
+ "preconditions": [
22
+ "location.hostname.includes('canva.com')",
23
+ "location.pathname.includes('/design/')",
24
+ "document.querySelectorAll('.wjS_DQ').length >= 4"
25
+ ],
26
+ "flows": {
27
+ "rewrite_screenhand_copy": {
28
+ "steps": [
29
+ "Install Canva helper in page context",
30
+ "Open each target text box in edit mode",
31
+ "Select existing text and type ScreenHand copy",
32
+ "Press Escape after each edit to return to canvas mode",
33
+ "Optionally normalize a few visible footer labels"
34
+ ],
35
+ "guards": [
36
+ "The Canva editor must already be open in Chrome",
37
+ "The target design must have at least 4 pages",
38
+ "This playbook expects the same template geometry used during capture"
39
+ ],
40
+ "why": "Canva text editing is not a normal input flow. Reliable replay requires entering the in-canvas ql-editor first, then typing only after edit mode is confirmed."
41
+ }
42
+ },
43
+ "errors": [
44
+ {
45
+ "error": "Center clicks land on Canva's floating toolbar instead of the text box",
46
+ "context": "Editing large headline blocks inside the canvas",
47
+ "solution": "This playbook double-clicks lower inside the target box and waits for the ql-editor to appear before typing.",
48
+ "severity": "high"
49
+ },
50
+ {
51
+ "error": "Typing before ql-editor exists does nothing",
52
+ "context": "Canva canvas mode vs true text-edit mode",
53
+ "solution": "Each open step polls for .ql-editor[contenteditable='true'] and selects its contents before browser_type runs.",
54
+ "severity": "high"
55
+ }
56
+ ],
57
+ "steps": [
58
+ {
59
+ "action": "browser_js",
60
+ "description": "Install Canva carousel helper and verify a 4-page editor is open",
61
+ "code": "(() => { const norm = (s) => (s || '').toLowerCase().replace(/\\s+/g, ' ').trim(); const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms)); const getScroller = () => Array.from(document.querySelectorAll('*')).find(el => { const s = getComputedStyle(el); return (s.overflowY === 'scroll' || s.overflowY === 'auto') && el.scrollHeight > el.clientHeight + 500; }) || null; const getPages = () => Array.from(document.querySelectorAll('.wjS_DQ')); const getPage = (pageNum) => getPages()[pageNum - 1] || null; const waitForEditor = async (timeoutMs = 4000) => { const start = Date.now(); while (Date.now() - start < timeoutMs) { const editor = document.querySelector('.ql-editor[contenteditable=\"true\"]'); if (editor) return editor; await wait(50); } return null; }; const selectEditorText = () => { const editor = document.querySelector('.ql-editor[contenteditable=\"true\"]'); if (!editor) throw new Error('No active ql-editor'); const range = document.createRange(); range.selectNodeContents(editor); const sel = window.getSelection(); sel.removeAllRanges(); sel.addRange(range); editor.focus(); return (editor.innerText || editor.textContent || '').replace(/\\s+/g, ' ').trim(); }; const fireClickSequence = (x, y) => { const target = document.elementFromPoint(x, y) || document.body; const emit = (type, detail) => { const Ctor = type.startsWith('pointer') ? PointerEvent : MouseEvent; target.dispatchEvent(new Ctor(type, { bubbles: true, cancelable: true, composed: true, clientX: x, clientY: y, screenX: x, screenY: y, detail, button: 0, buttons: 1, pointerType: 'mouse' })); }; emit('pointerdown', 1); emit('mousedown', 1); emit('pointerup', 1); emit('mouseup', 1); emit('click', 1); emit('pointerdown', 2); emit('mousedown', 2); emit('pointerup', 2); emit('mouseup', 2); emit('click', 2); emit('dblclick', 2); return target.tagName; }; const scrollPage = async (pageNum) => { const scroller = getScroller(); const page = getPage(pageNum); if (!scroller || !page) throw new Error(`Unable to find Canva scroller/page ${pageNum}`); scroller.scrollTop = page.offsetTop; await wait(250); return page.getBoundingClientRect(); }; const openByPointOnPage = async (pageNum, relX, relY) => { const pageRect = await scrollPage(pageNum); const x = Math.round(pageRect.x + relX); const y = Math.round(pageRect.y + relY); fireClickSequence(x, y); const editor = await waitForEditor(); if (!editor) throw new Error(`Failed to enter editor on page ${pageNum} at ${relX},${relY}`); return { page: pageNum, mode: 'point', selectedText: selectEditorText() }; }; const openByTextOnPage = async (pageNum, query, index = 0, yBias = 0.82) => { const pageRect = await scrollPage(pageNum); const nq = norm(query); const matches = Array.from(document.querySelectorAll('body *')).map(el => { const text = (el.innerText || el.textContent || '').replace(/\\s+/g, ' ').trim(); const rect = el.getBoundingClientRect(); return { el, text, ntext: norm(text), rect, area: rect.width * rect.height }; }).filter(m => m.ntext.includes(nq) && m.rect.width > 3 && m.rect.height > 3 && m.rect.top >= pageRect.top - 24 && m.rect.bottom <= pageRect.bottom + 24 && m.rect.left >= pageRect.left - 24 && m.rect.right <= pageRect.right + 24).sort((a, b) => a.area - b.area || a.rect.y - b.rect.y); const match = matches[index]; if (!match) throw new Error(`No match for \"${query}\" on page ${pageNum}`); const x = Math.round(match.rect.left + Math.min(Math.max(match.rect.width * 0.35, 12), match.rect.width - 12)); const y = Math.round(match.rect.top + Math.min(Math.max(match.rect.height * yBias, 8), match.rect.height - 4)); fireClickSequence(x, y); const editor = await waitForEditor(); if (!editor) throw new Error(`Failed to enter editor for \"${query}\" on page ${pageNum}`); return { page: pageNum, mode: 'text', query, selectedText: selectEditorText(), clicked: { x, y } }; }; window.__screenhandCanvaCarousel = { norm, wait, getScroller, getPages, getPage, scrollPage, waitForEditor, selectEditorText, openByPointOnPage, openByTextOnPage }; if (getPages().length < 4) throw new Error(`Expected at least 4 Canva pages, found ${getPages().length}`); return { ok: true, pages: getPages().length }; })()"
62
+ },
63
+ {
64
+ "action": "cdp_key_event",
65
+ "description": "Exit any active text editor before starting",
66
+ "keyEvent": {
67
+ "key": "Escape",
68
+ "code": "Escape",
69
+ "windowsVirtualKeyCode": 27
70
+ },
71
+ "optional": true
72
+ },
73
+
74
+ {
75
+ "action": "browser_js",
76
+ "description": "Page 1: open the main headline box by fixed point",
77
+ "code": "window.__screenhandCanvaCarousel.openByPointOnPage(1, 116, 220)"
78
+ },
79
+ {
80
+ "action": "browser_type",
81
+ "description": "Page 1: set headline to SCREENHAND",
82
+ "target": { "selector": ".ql-editor" },
83
+ "text": "SCREENHAND"
84
+ },
85
+ {
86
+ "action": "cdp_key_event",
87
+ "description": "Leave page 1 headline edit mode",
88
+ "keyEvent": {
89
+ "key": "Escape",
90
+ "code": "Escape",
91
+ "windowsVirtualKeyCode": 27
92
+ }
93
+ },
94
+ {
95
+ "action": "browser_js",
96
+ "description": "Page 1: open the subtitle box by placeholder text",
97
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(1, 'ddd', 0, 0.82)"
98
+ },
99
+ {
100
+ "action": "browser_type",
101
+ "description": "Page 1: set subtitle to MCP AGENTS",
102
+ "target": { "selector": ".ql-editor" },
103
+ "text": "MCP AGENTS"
104
+ },
105
+ {
106
+ "action": "cdp_key_event",
107
+ "description": "Leave page 1 subtitle edit mode",
108
+ "keyEvent": {
109
+ "key": "Escape",
110
+ "code": "Escape",
111
+ "windowsVirtualKeyCode": 27
112
+ }
113
+ },
114
+ {
115
+ "action": "browser_js",
116
+ "description": "Page 1: open the small brand label",
117
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(1, 'brocelle tech', 0, 0.75)"
118
+ },
119
+ {
120
+ "action": "browser_type",
121
+ "description": "Page 1: replace the small brand with SH",
122
+ "target": { "selector": ".ql-editor" },
123
+ "text": "SH"
124
+ },
125
+ {
126
+ "action": "cdp_key_event",
127
+ "description": "Leave page 1 brand edit mode",
128
+ "keyEvent": {
129
+ "key": "Escape",
130
+ "code": "Escape",
131
+ "windowsVirtualKeyCode": 27
132
+ }
133
+ },
134
+ {
135
+ "action": "browser_js",
136
+ "description": "Page 1: open the left footer label",
137
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(1, 'sdf', 0, 0.6)",
138
+ "optional": true
139
+ },
140
+ {
141
+ "action": "browser_type",
142
+ "description": "Page 1: set left footer to screenhand.com",
143
+ "target": { "selector": ".ql-editor" },
144
+ "text": "screenhand.com",
145
+ "optional": true
146
+ },
147
+ {
148
+ "action": "cdp_key_event",
149
+ "description": "Leave page 1 footer edit mode",
150
+ "keyEvent": {
151
+ "key": "Escape",
152
+ "code": "Escape",
153
+ "windowsVirtualKeyCode": 27
154
+ },
155
+ "optional": true
156
+ },
157
+
158
+ {
159
+ "action": "browser_js",
160
+ "description": "Page 2: open the top headline",
161
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(2, 'let me tell you', 0, 0.82)"
162
+ },
163
+ {
164
+ "action": "browser_type",
165
+ "description": "Page 2: set top headline to NATIVE DESKTOP",
166
+ "target": { "selector": ".ql-editor" },
167
+ "text": "NATIVE DESKTOP"
168
+ },
169
+ {
170
+ "action": "cdp_key_event",
171
+ "description": "Leave page 2 headline edit mode",
172
+ "keyEvent": {
173
+ "key": "Escape",
174
+ "code": "Escape",
175
+ "windowsVirtualKeyCode": 27
176
+ }
177
+ },
178
+ {
179
+ "action": "browser_js",
180
+ "description": "Page 2: open the lower headline",
181
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(2, 'check the next slide!', 0, 0.82)"
182
+ },
183
+ {
184
+ "action": "browser_type",
185
+ "description": "Page 2: set lower headline to CONTROL FOR AI",
186
+ "target": { "selector": ".ql-editor" },
187
+ "text": "CONTROL FOR AI"
188
+ },
189
+ {
190
+ "action": "cdp_key_event",
191
+ "description": "Leave page 2 lower headline edit mode",
192
+ "keyEvent": {
193
+ "key": "Escape",
194
+ "code": "Escape",
195
+ "windowsVirtualKeyCode": 27
196
+ }
197
+ },
198
+ {
199
+ "action": "browser_js",
200
+ "description": "Page 2: open the small brand label",
201
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(2, 'brocelle tech', 0, 0.75)",
202
+ "optional": true
203
+ },
204
+ {
205
+ "action": "browser_type",
206
+ "description": "Page 2: replace the small brand with SH",
207
+ "target": { "selector": ".ql-editor" },
208
+ "text": "SH",
209
+ "optional": true
210
+ },
211
+ {
212
+ "action": "cdp_key_event",
213
+ "description": "Leave page 2 brand edit mode",
214
+ "keyEvent": {
215
+ "key": "Escape",
216
+ "code": "Escape",
217
+ "windowsVirtualKeyCode": 27
218
+ },
219
+ "optional": true
220
+ },
221
+ {
222
+ "action": "browser_js",
223
+ "description": "Page 2: open the left footer website",
224
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(2, 'www.reallygreatsite.com', 0, 0.6)",
225
+ "optional": true
226
+ },
227
+ {
228
+ "action": "browser_type",
229
+ "description": "Page 2: set left footer website to screenhand.com",
230
+ "target": { "selector": ".ql-editor" },
231
+ "text": "screenhand.com",
232
+ "optional": true
233
+ },
234
+ {
235
+ "action": "cdp_key_event",
236
+ "description": "Leave page 2 footer edit mode",
237
+ "keyEvent": {
238
+ "key": "Escape",
239
+ "code": "Escape",
240
+ "windowsVirtualKeyCode": 27
241
+ },
242
+ "optional": true
243
+ },
244
+
245
+ {
246
+ "action": "browser_js",
247
+ "description": "Page 3: open the first feature bullet",
248
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(3, 'the global ai market', 0, 0.82)"
249
+ },
250
+ {
251
+ "action": "browser_type",
252
+ "description": "Page 3: set first feature bullet",
253
+ "target": { "selector": ".ql-editor" },
254
+ "text": "88 TOOLS FOR DESKTOP CONTROL"
255
+ },
256
+ {
257
+ "action": "cdp_key_event",
258
+ "description": "Leave page 3 first bullet edit mode",
259
+ "keyEvent": {
260
+ "key": "Escape",
261
+ "code": "Escape",
262
+ "windowsVirtualKeyCode": 27
263
+ }
264
+ },
265
+ {
266
+ "action": "browser_js",
267
+ "description": "Page 3: open the second feature bullet",
268
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(3, 'ai usage among desktop workers', 0, 0.82)"
269
+ },
270
+ {
271
+ "action": "browser_type",
272
+ "description": "Page 3: set second feature bullet",
273
+ "target": { "selector": ".ql-editor" },
274
+ "text": "ACCESSIBILITY, CDP + OCR"
275
+ },
276
+ {
277
+ "action": "cdp_key_event",
278
+ "description": "Leave page 3 second bullet edit mode",
279
+ "keyEvent": {
280
+ "key": "Escape",
281
+ "code": "Escape",
282
+ "windowsVirtualKeyCode": 27
283
+ }
284
+ },
285
+ {
286
+ "action": "browser_js",
287
+ "description": "Page 3: open the third feature bullet",
288
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(3, 'an estimated 85 million jobs', 0, 0.82)"
289
+ },
290
+ {
291
+ "action": "browser_type",
292
+ "description": "Page 3: set third feature bullet",
293
+ "target": { "selector": ".ql-editor" },
294
+ "text": "MEMORY, JOBS + PLAYBOOKS"
295
+ },
296
+ {
297
+ "action": "cdp_key_event",
298
+ "description": "Leave page 3 third bullet edit mode",
299
+ "keyEvent": {
300
+ "key": "Escape",
301
+ "code": "Escape",
302
+ "windowsVirtualKeyCode": 27
303
+ }
304
+ },
305
+ {
306
+ "action": "browser_js",
307
+ "description": "Page 3: open the small brand label",
308
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(3, 'brocelle tech', 0, 0.75)",
309
+ "optional": true
310
+ },
311
+ {
312
+ "action": "browser_type",
313
+ "description": "Page 3: replace the small brand with SH",
314
+ "target": { "selector": ".ql-editor" },
315
+ "text": "SH",
316
+ "optional": true
317
+ },
318
+ {
319
+ "action": "cdp_key_event",
320
+ "description": "Leave page 3 brand edit mode",
321
+ "keyEvent": {
322
+ "key": "Escape",
323
+ "code": "Escape",
324
+ "windowsVirtualKeyCode": 27
325
+ },
326
+ "optional": true
327
+ },
328
+ {
329
+ "action": "browser_js",
330
+ "description": "Page 3: open the left footer website",
331
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(3, 'www.reallygreatsite.com', 0, 0.6)",
332
+ "optional": true
333
+ },
334
+ {
335
+ "action": "browser_type",
336
+ "description": "Page 3: set left footer website to screenhand.com",
337
+ "target": { "selector": ".ql-editor" },
338
+ "text": "screenhand.com",
339
+ "optional": true
340
+ },
341
+ {
342
+ "action": "cdp_key_event",
343
+ "description": "Leave page 3 footer edit mode",
344
+ "keyEvent": {
345
+ "key": "Escape",
346
+ "code": "Escape",
347
+ "windowsVirtualKeyCode": 27
348
+ },
349
+ "optional": true
350
+ },
351
+
352
+ {
353
+ "action": "browser_js",
354
+ "description": "Page 4: open the main CTA headline",
355
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(4, 'still refusing to adapt to ai?', 0, 0.82)"
356
+ },
357
+ {
358
+ "action": "browser_type",
359
+ "description": "Page 4: set CTA headline to REAL DESKTOP HANDS",
360
+ "target": { "selector": ".ql-editor" },
361
+ "text": "REAL DESKTOP HANDS"
362
+ },
363
+ {
364
+ "action": "cdp_key_event",
365
+ "description": "Leave page 4 main CTA edit mode",
366
+ "keyEvent": {
367
+ "key": "Escape",
368
+ "code": "Escape",
369
+ "windowsVirtualKeyCode": 27
370
+ }
371
+ },
372
+ {
373
+ "action": "browser_js",
374
+ "description": "Page 4: open the lower CTA line",
375
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(4, 'follow for more information!', 0, 0.82)"
376
+ },
377
+ {
378
+ "action": "browser_type",
379
+ "description": "Page 4: set lower CTA line to SCREENHAND.COM",
380
+ "target": { "selector": ".ql-editor" },
381
+ "text": "SCREENHAND.COM"
382
+ },
383
+ {
384
+ "action": "cdp_key_event",
385
+ "description": "Leave page 4 lower CTA edit mode",
386
+ "keyEvent": {
387
+ "key": "Escape",
388
+ "code": "Escape",
389
+ "windowsVirtualKeyCode": 27
390
+ }
391
+ },
392
+ {
393
+ "action": "browser_js",
394
+ "description": "Page 4: open the small brand label",
395
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(4, 'brocelle tech', 0, 0.75)",
396
+ "optional": true
397
+ },
398
+ {
399
+ "action": "browser_type",
400
+ "description": "Page 4: replace the small brand with SH",
401
+ "target": { "selector": ".ql-editor" },
402
+ "text": "SH",
403
+ "optional": true
404
+ },
405
+ {
406
+ "action": "cdp_key_event",
407
+ "description": "Leave page 4 brand edit mode",
408
+ "keyEvent": {
409
+ "key": "Escape",
410
+ "code": "Escape",
411
+ "windowsVirtualKeyCode": 27
412
+ },
413
+ "optional": true
414
+ },
415
+ {
416
+ "action": "browser_js",
417
+ "description": "Page 4: open the left footer website",
418
+ "code": "window.__screenhandCanvaCarousel.openByTextOnPage(4, 'www.reallygreatsite.com', 0, 0.6)",
419
+ "optional": true
420
+ },
421
+ {
422
+ "action": "browser_type",
423
+ "description": "Page 4: set left footer website to screenhand.com",
424
+ "target": { "selector": ".ql-editor" },
425
+ "text": "screenhand.com",
426
+ "optional": true
427
+ },
428
+ {
429
+ "action": "cdp_key_event",
430
+ "description": "Leave page 4 footer edit mode",
431
+ "keyEvent": {
432
+ "key": "Escape",
433
+ "code": "Escape",
434
+ "windowsVirtualKeyCode": 27
435
+ },
436
+ "optional": true
437
+ },
438
+
439
+ {
440
+ "action": "screenshot",
441
+ "description": "Capture the final Canva state",
442
+ "optional": true
443
+ }
444
+ ]
445
+ }
@@ -0,0 +1,76 @@
1
+ {
2
+ "id": "codex-desktop",
3
+ "name": "Codex Desktop — Full Prompt Cycle",
4
+ "description": "Full Codex automation cycle: new thread → submit prompt → wait for completion → read response. Supports follow-up prompts by repeating the submit+wait+read sequence.",
5
+ "platform": "codex-desktop",
6
+ "version": "2.0.0",
7
+ "urlPatterns": ["app://-/index.html*"],
8
+ "tags": ["codex", "openai", "desktop", "electron", "cdp"],
9
+ "cdpPort": 9333,
10
+ "successCount": 1,
11
+ "failCount": 0,
12
+ "preconditions": [
13
+ "!!document.querySelector('.ProseMirror')"
14
+ ],
15
+ "steps": [
16
+ {
17
+ "action": "wait",
18
+ "ms": 500,
19
+ "description": "Wait for Codex UI to settle"
20
+ },
21
+
22
+ {
23
+ "action": "browser_js",
24
+ "code": "document.querySelector('.ProseMirror').focus(); 'focused'",
25
+ "description": "Focus the ProseMirror editor"
26
+ },
27
+ {
28
+ "action": "browser_js",
29
+ "code": "var pm = document.querySelector('.ProseMirror'); pm.innerHTML = '<p><br></p>'; pm.focus(); 'cleared'",
30
+ "description": "Clear editor content"
31
+ },
32
+ {
33
+ "action": "wait",
34
+ "ms": 200,
35
+ "description": "Brief pause after clearing"
36
+ },
37
+ {
38
+ "action": "browser_js",
39
+ "code": "document.querySelector('.ProseMirror').focus(); document.execCommand('insertText', false, '{PROMPT_TEXT}'); 'typed'",
40
+ "description": "Type prompt text via execCommand (ProseMirror-compatible)"
41
+ },
42
+ {
43
+ "action": "wait",
44
+ "ms": 300,
45
+ "description": "Wait for ProseMirror to process input"
46
+ },
47
+ {
48
+ "action": "cdp_key_event",
49
+ "keyEvent": {
50
+ "key": "Enter",
51
+ "code": "Enter",
52
+ "modifiers": 4,
53
+ "windowsVirtualKeyCode": 13
54
+ },
55
+ "description": "Submit with Cmd+Enter"
56
+ },
57
+
58
+ {
59
+ "action": "browser_js",
60
+ "code": "(() => { const start = Date.now(); const poll = () => { const body = document.body.innerText; if (body.includes('Ask for follow-up changes') || body.includes('Hand off') || body.includes('Worked for')) return 'done'; if (Date.now() - start > 120000) return 'timeout_120s'; return null; }; return new Promise(resolve => { const check = () => { const r = poll(); if (r) resolve(r); else setTimeout(check, 2000); }; check(); }); })()",
61
+ "description": "Wait for Codex to finish (polls for 'Ask for follow-up changes' or 'Hand off', max 120s)"
62
+ },
63
+
64
+ {
65
+ "action": "browser_js",
66
+ "code": "(() => { const msgs = document.querySelectorAll('[class*=message], [class*=thread], [class*=content]'); let text = ''; for (const m of msgs) { text += m.textContent + '\\n'; } if (text.trim().length > 0) return text.substring(0, 4000); return document.body.innerText.substring(0, 4000); })()",
67
+ "description": "Read Codex response from the thread"
68
+ },
69
+
70
+ {
71
+ "action": "screenshot",
72
+ "description": "Capture final state",
73
+ "optional": true
74
+ }
75
+ ]
76
+ }
@@ -0,0 +1,122 @@
1
+ {
2
+ "id": "competitor-research-stack",
3
+ "name": "Competitor Research Stack",
4
+ "description": "Run a simple cross-platform competitor research sequence across Google Search, Google Ads Transparency, and Meta Ad Library.",
5
+ "platform": "competitor-research-stack",
6
+ "version": "1.0.0",
7
+ "urlPatterns": [
8
+ "*google.com/search*",
9
+ "*adstransparency.google.com*",
10
+ "*facebook.com/ads/library*"
11
+ ],
12
+ "tags": [
13
+ "competitor-research",
14
+ "meta",
15
+ "google",
16
+ "browser",
17
+ "cdp"
18
+ ],
19
+ "successCount": 0,
20
+ "failCount": 0,
21
+ "flows": {
22
+ "full_stack": {
23
+ "steps": [
24
+ "Discover category players on Google Search",
25
+ "Validate active Google paid inventory with a domain lookup",
26
+ "Validate active Meta campaign families with advertiser lookup"
27
+ ],
28
+ "guards": [
29
+ "QUERY should be a market phrase",
30
+ "DOMAIN should be the landing-page domain",
31
+ "ADVERTISER should be the Meta page name"
32
+ ],
33
+ "why": "This sequence separates discovery from ad-library validation."
34
+ }
35
+ },
36
+ "errors": [
37
+ {
38
+ "error": "none of the sources expose true conversion metrics",
39
+ "context": "All three sources are outside-in validation tools",
40
+ "solution": "Use repetition, longevity, and landing-page alignment as proxy signals.",
41
+ "severity": "medium"
42
+ }
43
+ ],
44
+ "steps": [
45
+ {
46
+ "action": "navigate",
47
+ "url": "https://www.google.com/search?q={QUERY}",
48
+ "description": "Open the discovery query on Google Search",
49
+ "verify": "textarea#APjFqb",
50
+ "verifyTimeoutMs": 15000
51
+ },
52
+ {
53
+ "action": "wait",
54
+ "ms": 1200,
55
+ "description": "Wait for Google results"
56
+ },
57
+ {
58
+ "action": "browser_js",
59
+ "code": "(() => { const headings = [...document.querySelectorAll('h3')].map((el) => (el.textContent || '').trim()).filter(Boolean).slice(0, 8); return { query: `{QUERY}`, url: location.href, headings }; })()",
60
+ "description": "Extract top Google competitor headings"
61
+ },
62
+ {
63
+ "action": "navigate",
64
+ "url": "https://adstransparency.google.com/?region=IN&domain={DOMAIN}",
65
+ "description": "Open the Google Ads Transparency domain lookup",
66
+ "verify": "input.input.input-area",
67
+ "verifyTimeoutMs": 15000
68
+ },
69
+ {
70
+ "action": "wait",
71
+ "ms": 1500,
72
+ "description": "Wait for Google Ads Transparency results"
73
+ },
74
+ {
75
+ "action": "browser_js",
76
+ "code": "(() => { const adCount = document.body.innerText.match(/\\b\\d+ ads\\b/i)?.[0] || null; const creativeLinks = [...document.querySelectorAll(\"a[href*='/advertiser/'][href*='/creative/']\")].map((el) => el.href).slice(0, 10); return { domain: `{DOMAIN}`, url: location.href, adCount, creativeLinks }; })()",
77
+ "description": "Extract Google ad count and creative links"
78
+ },
79
+ {
80
+ "action": "navigate",
81
+ "url": "https://www.facebook.com/ads/library/?active_status=active&ad_type=all&country=IN&is_targeted_country=false&media_type=all&search_type=keyword_unordered&sort_data[direction]=desc&sort_data[mode]=total_impressions",
82
+ "description": "Open Meta Ad Library in India all-ads mode",
83
+ "verify": "input[placeholder='Search by keyword or advertiser']",
84
+ "verifyTimeoutMs": 15000
85
+ },
86
+ {
87
+ "action": "wait",
88
+ "ms": 1500,
89
+ "description": "Let Meta Ad Library settle"
90
+ },
91
+ {
92
+ "action": "browser_js",
93
+ "code": "(() => { const input = document.querySelector(\"input[placeholder='Search by keyword or advertiser']\"); if (!(input instanceof HTMLInputElement)) throw new Error('Meta Ad Library search input not found'); const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set; if (!setter) throw new Error('HTMLInputElement value setter unavailable'); setter.call(input, `{ADVERTISER}`); input.dispatchEvent(new Event('input', { bubbles: true })); input.dispatchEvent(new Event('change', { bubbles: true })); input.focus(); return { typed: input.value }; })()",
94
+ "description": "Type the Meta advertiser page name"
95
+ },
96
+ {
97
+ "action": "wait",
98
+ "ms": 1500,
99
+ "description": "Wait for Meta advertiser suggestions"
100
+ },
101
+ {
102
+ "action": "browser_js",
103
+ "code": "(() => { const wanted = `{ADVERTISER}`.trim().toLowerCase(); const option = [...document.querySelectorAll(\"li[id^='pageID:'], [role='option']\")].find((el) => (el.innerText || '').toLowerCase().includes(wanted)); if (!option) throw new Error(`Advertiser option not found: ${wanted}`); option.click(); return { clicked: (option.innerText || '').trim() }; })()",
104
+ "description": "Open the matching Meta advertiser page"
105
+ },
106
+ {
107
+ "action": "wait",
108
+ "ms": 2500,
109
+ "description": "Wait for Meta advertiser results"
110
+ },
111
+ {
112
+ "action": "browser_js",
113
+ "code": "(() => { const text = document.body.innerText; const results = text.match(/~\\d+ results/)?.[0] || null; const snippets = text.split(/Active\\nLibrary ID:/).slice(1, 5).map((chunk) => (`Library ID:${chunk}`).slice(0, 1200)); return { advertiser: `{ADVERTISER}`, url: location.href, results, snippets }; })()",
114
+ "description": "Extract Meta ad count and visible snippets"
115
+ },
116
+ {
117
+ "action": "screenshot",
118
+ "description": "Capture the final Meta state",
119
+ "optional": true
120
+ }
121
+ ]
122
+ }