screenhand 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +165 -446
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +3615 -400
  4. package/dist/scripts/export-help-center.js +112 -0
  5. package/dist/scripts/marketing-loop.js +117 -0
  6. package/dist/scripts/observer-daemon.js +288 -0
  7. package/dist/scripts/orchestrator-daemon.js +399 -0
  8. package/dist/scripts/threads-campaign.js +208 -0
  9. package/dist/src/community/fetcher.js +109 -0
  10. package/dist/src/community/index.js +6 -0
  11. package/dist/src/community/publisher.js +191 -0
  12. package/dist/src/community/remote-api.js +121 -0
  13. package/dist/src/community/types.js +3 -0
  14. package/dist/src/community/validator.js +95 -0
  15. package/dist/src/context-tracker.js +489 -0
  16. package/dist/src/ingestion/coverage-auditor.js +233 -0
  17. package/dist/src/ingestion/doc-parser.js +164 -0
  18. package/dist/src/ingestion/index.js +8 -0
  19. package/dist/src/ingestion/menu-scanner.js +152 -0
  20. package/dist/src/ingestion/reference-merger.js +186 -0
  21. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  22. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  23. package/dist/src/ingestion/types.js +3 -0
  24. package/dist/src/jobs/manager.js +82 -14
  25. package/dist/src/jobs/runner.js +138 -15
  26. package/dist/src/learning/engine.js +356 -0
  27. package/dist/src/learning/index.js +9 -0
  28. package/dist/src/learning/locator-policy.js +120 -0
  29. package/dist/src/learning/pattern-policy.js +89 -0
  30. package/dist/src/learning/recovery-policy.js +116 -0
  31. package/dist/src/learning/sensor-policy.js +115 -0
  32. package/dist/src/learning/timing-model.js +204 -0
  33. package/dist/src/learning/topology-policy.js +90 -0
  34. package/dist/src/learning/types.js +9 -0
  35. package/dist/src/logging/timeline-logger.js +4 -1
  36. package/dist/src/memory/playbook-seeds.js +200 -0
  37. package/dist/src/memory/recall.js +60 -8
  38. package/dist/src/memory/service.js +30 -5
  39. package/dist/src/memory/store.js +34 -5
  40. package/dist/src/native/bridge-client.js +253 -31
  41. package/dist/src/observer/state.js +199 -0
  42. package/dist/src/observer/types.js +43 -0
  43. package/dist/src/orchestrator/state.js +68 -0
  44. package/dist/src/orchestrator/types.js +22 -0
  45. package/dist/src/perception/ax-source.js +162 -0
  46. package/dist/src/perception/cdp-source.js +162 -0
  47. package/dist/src/perception/coordinator.js +771 -0
  48. package/dist/src/perception/frame-differ.js +287 -0
  49. package/dist/src/perception/index.js +22 -0
  50. package/dist/src/perception/manager.js +199 -0
  51. package/dist/src/perception/types.js +47 -0
  52. package/dist/src/perception/vision-source.js +399 -0
  53. package/dist/src/planner/deterministic.js +298 -0
  54. package/dist/src/planner/executor.js +870 -0
  55. package/dist/src/planner/goal-store.js +92 -0
  56. package/dist/src/planner/index.js +21 -0
  57. package/dist/src/planner/planner.js +520 -0
  58. package/dist/src/planner/tool-registry.js +71 -0
  59. package/dist/src/planner/types.js +22 -0
  60. package/dist/src/platform/explorer.js +213 -0
  61. package/dist/src/platform/help-center-markdown.js +527 -0
  62. package/dist/src/platform/learner.js +257 -0
  63. package/dist/src/playbook/engine.js +296 -11
  64. package/dist/src/playbook/mcp-recorder.js +204 -0
  65. package/dist/src/playbook/recorder.js +3 -2
  66. package/dist/src/playbook/runner.js +1 -1
  67. package/dist/src/playbook/store.js +139 -10
  68. package/dist/src/recovery/detectors.js +156 -0
  69. package/dist/src/recovery/engine.js +327 -0
  70. package/dist/src/recovery/index.js +20 -0
  71. package/dist/src/recovery/strategies.js +274 -0
  72. package/dist/src/recovery/types.js +20 -0
  73. package/dist/src/runtime/accessibility-adapter.js +55 -18
  74. package/dist/src/runtime/applescript-adapter.js +8 -2
  75. package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
  76. package/dist/src/runtime/executor.js +23 -3
  77. package/dist/src/runtime/locator-cache.js +24 -2
  78. package/dist/src/runtime/service.js +59 -15
  79. package/dist/src/runtime/session-manager.js +4 -1
  80. package/dist/src/runtime/vision-adapter.js +2 -1
  81. package/dist/src/state/app-map-types.js +72 -0
  82. package/dist/src/state/app-map.js +1974 -0
  83. package/dist/src/state/entity-tracker.js +108 -0
  84. package/dist/src/state/fusion.js +96 -0
  85. package/dist/src/state/index.js +21 -0
  86. package/dist/src/state/ladder-generator.js +236 -0
  87. package/dist/src/state/persistence.js +156 -0
  88. package/dist/src/state/types.js +17 -0
  89. package/dist/src/state/world-model.js +1456 -0
  90. package/dist/src/util/atomic-write.js +19 -4
  91. package/dist/src/util/sanitize.js +146 -0
  92. package/dist-app-maps/com.figma.Desktop.json +959 -0
  93. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  94. package/dist-app-maps/notion.id.json +2831 -0
  95. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  96. package/dist-playbooks/codex-desktop.json +76 -0
  97. package/dist-playbooks/competitor-research-stack.json +122 -0
  98. package/dist-playbooks/davinci-color-grade.json +153 -0
  99. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  100. package/dist-playbooks/davinci-render.json +114 -0
  101. package/dist-playbooks/devto.json +52 -0
  102. package/dist-playbooks/discord.json +41 -0
  103. package/dist-playbooks/google-flow-create-project.json +59 -0
  104. package/dist-playbooks/google-flow-edit-image.json +90 -0
  105. package/dist-playbooks/google-flow-edit-video.json +90 -0
  106. package/dist-playbooks/google-flow-generate-image.json +68 -0
  107. package/dist-playbooks/google-flow-generate-video.json +191 -0
  108. package/dist-playbooks/google-flow-open-project.json +48 -0
  109. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  110. package/dist-playbooks/google-flow-search-assets.json +64 -0
  111. package/dist-playbooks/instagram.json +57 -0
  112. package/dist-playbooks/linkedin.json +52 -0
  113. package/dist-playbooks/n8n.json +43 -0
  114. package/dist-playbooks/reddit.json +52 -0
  115. package/dist-playbooks/threads.json +59 -0
  116. package/dist-playbooks/x-twitter.json +59 -0
  117. package/dist-playbooks/youtube.json +59 -0
  118. package/dist-references/canva.json +646 -0
  119. package/dist-references/codex-desktop.json +305 -0
  120. package/dist-references/davinci-resolve-keyboard.json +594 -0
  121. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  122. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  123. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  124. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  125. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  126. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  127. package/dist-references/devpost.json +186 -0
  128. package/dist-references/devto.json +317 -0
  129. package/dist-references/discord.json +549 -0
  130. package/dist-references/figma.json +1186 -0
  131. package/dist-references/finder.json +146 -0
  132. package/dist-references/google-ads-transparency.json +95 -0
  133. package/dist-references/google-flow.json +649 -0
  134. package/dist-references/instagram.json +341 -0
  135. package/dist-references/linkedin.json +324 -0
  136. package/dist-references/meta-ad-library.json +86 -0
  137. package/dist-references/n8n.json +387 -0
  138. package/dist-references/notes.json +27 -0
  139. package/dist-references/notion.json +163 -0
  140. package/dist-references/reddit.json +341 -0
  141. package/dist-references/threads.json +337 -0
  142. package/dist-references/x-twitter.json +403 -0
  143. package/dist-references/youtube.json +373 -0
  144. package/native/macos-bridge/Package.swift +22 -0
  145. package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
  146. package/native/macos-bridge/Sources/AppManagement.swift +339 -0
  147. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
  148. package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
  149. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  150. package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
  151. package/native/macos-bridge/Sources/main.swift +498 -0
  152. package/native/windows-bridge/AppManagement.cs +234 -0
  153. package/native/windows-bridge/InputBridge.cs +436 -0
  154. package/native/windows-bridge/Program.cs +270 -0
  155. package/native/windows-bridge/ScreenCapture.cs +453 -0
  156. package/native/windows-bridge/UIAutomationBridge.cs +571 -0
  157. package/native/windows-bridge/WindowsBridge.csproj +17 -0
  158. package/package.json +12 -1
  159. package/scripts/postinstall.cjs +127 -0
  160. package/dist/.audit-log.jsonl +0 -55
  161. package/dist/.screenhand/memory/.lock +0 -1
  162. package/dist/.screenhand/memory/actions.jsonl +0 -85
  163. package/dist/.screenhand/memory/errors.jsonl +0 -5
  164. package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
  165. package/dist/.screenhand/memory/state.json +0 -35
  166. package/dist/.screenhand/memory/state.json.bak +0 -35
  167. package/dist/.screenhand/memory/strategies.jsonl +0 -12
  168. package/dist/agent/cli.js +0 -73
  169. package/dist/agent/loop.js +0 -258
  170. package/dist/config.js +0 -9
  171. package/dist/index.js +0 -56
  172. package/dist/logging/timeline-logger.js +0 -29
  173. package/dist/mcp/mcp-stdio-server.js +0 -448
  174. package/dist/mcp/server.js +0 -347
  175. package/dist/mcp-entry.js +0 -59
  176. package/dist/memory/recall.js +0 -160
  177. package/dist/memory/research.js +0 -98
  178. package/dist/memory/seeds.js +0 -89
  179. package/dist/memory/session.js +0 -161
  180. package/dist/memory/store.js +0 -391
  181. package/dist/memory/types.js +0 -4
  182. package/dist/monitor/codex-monitor.js +0 -377
  183. package/dist/monitor/task-queue.js +0 -84
  184. package/dist/monitor/types.js +0 -49
  185. package/dist/native/bridge-client.js +0 -174
  186. package/dist/native/macos-bridge-client.js +0 -5
  187. package/dist/npm-publish-helper.js +0 -117
  188. package/dist/npm-token-cdp.js +0 -113
  189. package/dist/npm-token-create.js +0 -135
  190. package/dist/npm-token-finish.js +0 -126
  191. package/dist/playbook/engine.js +0 -193
  192. package/dist/playbook/index.js +0 -4
  193. package/dist/playbook/recorder.js +0 -519
  194. package/dist/playbook/runner.js +0 -392
  195. package/dist/playbook/store.js +0 -166
  196. package/dist/playbook/types.js +0 -4
  197. package/dist/runtime/accessibility-adapter.js +0 -377
  198. package/dist/runtime/app-adapter.js +0 -48
  199. package/dist/runtime/applescript-adapter.js +0 -283
  200. package/dist/runtime/ax-role-map.js +0 -80
  201. package/dist/runtime/browser-adapter.js +0 -36
  202. package/dist/runtime/cdp-chrome-adapter.js +0 -505
  203. package/dist/runtime/composite-adapter.js +0 -205
  204. package/dist/runtime/executor.js +0 -250
  205. package/dist/runtime/locator-cache.js +0 -12
  206. package/dist/runtime/planning-loop.js +0 -47
  207. package/dist/runtime/service.js +0 -372
  208. package/dist/runtime/session-manager.js +0 -28
  209. package/dist/runtime/state-observer.js +0 -105
  210. package/dist/runtime/vision-adapter.js +0 -208
  211. package/dist/test-mcp-protocol.js +0 -138
  212. package/dist/types.js +0 -1
@@ -0,0 +1,186 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ import * as fs from "node:fs";
4
+ import * as path from "node:path";
5
+ import { writeFileAtomicSync } from "../util/atomic-write.js";
6
+ import { shortcutsToReferenceFormat } from "./shortcut-extractor.js";
7
+ /**
8
+ * ReferenceMerger — merges ingested knowledge into existing reference files.
9
+ * Creates new reference files when no matching file exists.
10
+ */
11
+ export class ReferenceMerger {
12
+ referencesDir;
13
+ constructor(referencesDir) {
14
+ this.referencesDir = referencesDir;
15
+ }
16
+ /**
17
+ * Merge shortcuts from a menu scan into the reference file.
18
+ */
19
+ mergeMenuScan(scan) {
20
+ const ref = this.loadOrCreate(scan.bundleId, scan.appName);
21
+ const { shortcuts: scannedShortcuts } = this.menuScanToShortcuts(scan);
22
+ let added = 0;
23
+ let updated = 0;
24
+ if (!ref.shortcuts)
25
+ ref.shortcuts = {};
26
+ for (const [category, entries] of Object.entries(scannedShortcuts)) {
27
+ if (!ref.shortcuts[category]) {
28
+ ref.shortcuts[category] = {};
29
+ }
30
+ for (const [name, keys] of Object.entries(entries)) {
31
+ if (!ref.shortcuts[category][name]) {
32
+ added++;
33
+ }
34
+ else if (ref.shortcuts[category][name] !== keys) {
35
+ updated++;
36
+ }
37
+ ref.shortcuts[category][name] = keys;
38
+ }
39
+ }
40
+ const filePath = this.save(ref);
41
+ return { filePath, added, updated };
42
+ }
43
+ /**
44
+ * Merge shortcuts from parsed documentation.
45
+ */
46
+ mergeDocShortcuts(shortcuts, bundleId, appName) {
47
+ const ref = this.loadOrCreate(bundleId, appName);
48
+ const formatted = shortcutsToReferenceFormat(shortcuts);
49
+ let added = 0;
50
+ let updated = 0;
51
+ if (!ref.shortcuts)
52
+ ref.shortcuts = {};
53
+ for (const [category, entries] of Object.entries(formatted)) {
54
+ if (!ref.shortcuts[category]) {
55
+ ref.shortcuts[category] = {};
56
+ }
57
+ for (const [name, keys] of Object.entries(entries)) {
58
+ if (!ref.shortcuts[category][name]) {
59
+ added++;
60
+ }
61
+ else if (ref.shortcuts[category][name] !== keys) {
62
+ updated++;
63
+ }
64
+ ref.shortcuts[category][name] = keys;
65
+ }
66
+ }
67
+ const filePath = this.save(ref);
68
+ return { filePath, added, updated };
69
+ }
70
+ /**
71
+ * Merge flows from parsed documentation.
72
+ */
73
+ mergeDocFlows(docResult, bundleId, appName) {
74
+ const ref = this.loadOrCreate(bundleId, appName);
75
+ if (!ref.flows)
76
+ ref.flows = {};
77
+ let added = 0;
78
+ for (const flow of docResult.flows) {
79
+ const key = flow.name
80
+ .toLowerCase()
81
+ .replace(/[^a-z0-9]+/g, "_")
82
+ .replace(/^_|_$/g, "");
83
+ if (!ref.flows[key]) {
84
+ const parsed = flow.steps
85
+ .filter((s) => s.tool)
86
+ .map((s) => ({ tool: s.tool, params: s.params ?? {} }));
87
+ ref.flows[key] = {
88
+ steps: flow.steps.map((s) => s.description),
89
+ description: flow.name,
90
+ ...(parsed.length > 0 ? { _parsed: parsed } : {}),
91
+ };
92
+ added++;
93
+ }
94
+ }
95
+ const filePath = this.save(ref);
96
+ return { filePath, added };
97
+ }
98
+ /**
99
+ * Merge errors/solutions into reference.
100
+ */
101
+ mergeErrors(errors, bundleId, appName) {
102
+ const ref = this.loadOrCreate(bundleId, appName);
103
+ if (!ref.errors)
104
+ ref.errors = [];
105
+ let added = 0;
106
+ const existingErrors = new Set(ref.errors.map((e) => e.error.toLowerCase()));
107
+ for (const err of errors) {
108
+ if (!existingErrors.has(err.error.toLowerCase())) {
109
+ ref.errors.push(err);
110
+ existingErrors.add(err.error.toLowerCase());
111
+ added++;
112
+ }
113
+ }
114
+ const filePath = this.save(ref);
115
+ return { filePath, added };
116
+ }
117
+ /**
118
+ * Load existing reference file for a bundleId, or create a new one.
119
+ */
120
+ loadOrCreate(bundleId, appName) {
121
+ // Search for existing file by bundleId
122
+ try {
123
+ const files = fs.readdirSync(this.referencesDir);
124
+ for (const file of files) {
125
+ if (!file.endsWith(".json"))
126
+ continue;
127
+ try {
128
+ const raw = fs.readFileSync(path.join(this.referencesDir, file), "utf-8");
129
+ const ref = JSON.parse(raw);
130
+ if (ref.bundleId === bundleId)
131
+ return ref;
132
+ }
133
+ catch {
134
+ /* skip malformed */
135
+ }
136
+ }
137
+ }
138
+ catch {
139
+ /* dir doesn't exist */
140
+ }
141
+ // Create new reference
142
+ const platform = appName.toLowerCase().replace(/\s+/g, "-");
143
+ return {
144
+ id: platform,
145
+ name: `${appName} — Auto-Generated Reference`,
146
+ platform,
147
+ bundleId,
148
+ shortcuts: {},
149
+ selectors: {},
150
+ flows: {},
151
+ errors: [],
152
+ };
153
+ }
154
+ save(ref) {
155
+ fs.mkdirSync(this.referencesDir, { recursive: true });
156
+ const filePath = path.join(this.referencesDir, `${ref.id}.json`);
157
+ writeFileAtomicSync(filePath, JSON.stringify(ref, null, 2) + "\n");
158
+ return filePath;
159
+ }
160
+ menuScanToShortcuts(scan) {
161
+ const shortcuts = {};
162
+ for (const topMenu of scan.menuTree) {
163
+ const category = topMenu.title;
164
+ if (!shortcuts[category])
165
+ shortcuts[category] = {};
166
+ const items = this.flattenMenuNode(topMenu, []);
167
+ for (const item of items) {
168
+ if (item.shortcut) {
169
+ shortcuts[category][item.label] = item.shortcut;
170
+ }
171
+ }
172
+ }
173
+ return { shortcuts };
174
+ }
175
+ flattenMenuNode(node, parentPath) {
176
+ const items = [];
177
+ const path = [...parentPath, node.title];
178
+ if (node.shortcut) {
179
+ items.push({ label: path.slice(1).join(" > "), shortcut: node.shortcut });
180
+ }
181
+ for (const child of node.children ?? []) {
182
+ items.push(...this.flattenMenuNode(child, path));
183
+ }
184
+ return items;
185
+ }
186
+ }
@@ -0,0 +1,180 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /**
4
+ * ShortcutExtractor — parses keyboard shortcut lists from various formats
5
+ * (HTML tables, plain text lists, markdown) into structured data.
6
+ */
7
+ /**
8
+ * Parse shortcuts from an HTML table.
9
+ * Expects columns like: Action/Name | Shortcut/Keys
10
+ */
11
+ export function parseShortcutsFromHTML(html) {
12
+ const shortcuts = [];
13
+ // Extract table rows
14
+ const tableRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
15
+ const cellRegex = /<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi;
16
+ let tableMatch;
17
+ while ((tableMatch = tableRegex.exec(html)) !== null) {
18
+ const row = tableMatch[1];
19
+ const cells = [];
20
+ let cellMatch;
21
+ while ((cellMatch = cellRegex.exec(row)) !== null) {
22
+ // Strip HTML tags and normalize whitespace
23
+ const text = cellMatch[1]
24
+ .replace(/<[^>]+>/g, "")
25
+ .replace(/&amp;/g, "&")
26
+ .replace(/&lt;/g, "<")
27
+ .replace(/&gt;/g, ">")
28
+ .replace(/&nbsp;/g, " ")
29
+ .replace(/&#\d+;/g, "")
30
+ .trim();
31
+ cells.push(text);
32
+ }
33
+ if (cells.length >= 2 && cells[0] && cells[1]) {
34
+ // Skip header rows
35
+ const lower0 = cells[0].toLowerCase();
36
+ if (lower0 === "action" ||
37
+ lower0 === "command" ||
38
+ lower0 === "shortcut" ||
39
+ lower0 === "name" ||
40
+ lower0 === "function") {
41
+ continue;
42
+ }
43
+ shortcuts.push({
44
+ name: cells[0],
45
+ keys: normalizeKeys(cells[1]),
46
+ context: cells.length > 2 ? cells[2] : undefined,
47
+ });
48
+ }
49
+ }
50
+ return shortcuts;
51
+ }
52
+ /**
53
+ * Parse shortcuts from plain text lines.
54
+ * Supports formats:
55
+ * - "Action: Cmd+K"
56
+ * - "Action — Cmd+K"
57
+ * - "Action\tCmd+K"
58
+ * - "Cmd+K Action"
59
+ */
60
+ export function parseShortcutsFromText(text) {
61
+ const shortcuts = [];
62
+ const lines = text.split("\n");
63
+ let currentCategory;
64
+ for (const rawLine of lines) {
65
+ const line = rawLine.trim();
66
+ if (!line)
67
+ continue;
68
+ // Detect category headers (all caps or ending with colon, no shortcut key)
69
+ if ((line.endsWith(":") && !SHORTCUT_PATTERN.test(line)) ||
70
+ (line === line.toUpperCase() && line.length > 3 && !SHORTCUT_PATTERN.test(line))) {
71
+ currentCategory = line.replace(/:$/, "").trim();
72
+ continue;
73
+ }
74
+ // Try "Name: Keys" or "Name — Keys" or "Name\tKeys"
75
+ const separatorMatch = line.match(/^(.+?)(?:\s*[:—–\-|]\s*|\t)(.+)$/);
76
+ if (separatorMatch && SHORTCUT_PATTERN.test(separatorMatch[2])) {
77
+ shortcuts.push({
78
+ name: separatorMatch[1].trim(),
79
+ keys: normalizeKeys(separatorMatch[2].trim()),
80
+ category: currentCategory,
81
+ });
82
+ continue;
83
+ }
84
+ // Try "Keys Name" (shortcut first)
85
+ const keysFirstMatch = line.match(/^((?:Cmd|Ctrl|Alt|Option|Shift|Meta|⌘|⌃|⌥|⇧)[+\s].+?)\s{2,}(.+)$/i);
86
+ if (keysFirstMatch) {
87
+ shortcuts.push({
88
+ name: keysFirstMatch[2].trim(),
89
+ keys: normalizeKeys(keysFirstMatch[1].trim()),
90
+ category: currentCategory,
91
+ });
92
+ }
93
+ }
94
+ return shortcuts;
95
+ }
96
+ /**
97
+ * Parse shortcuts from markdown format.
98
+ */
99
+ export function parseShortcutsFromMarkdown(md) {
100
+ const shortcuts = [];
101
+ const lines = md.split("\n");
102
+ let currentCategory;
103
+ for (const rawLine of lines) {
104
+ const line = rawLine.trim();
105
+ // Category from heading
106
+ const headingMatch = line.match(/^#{1,3}\s+(.+)$/);
107
+ if (headingMatch) {
108
+ currentCategory = headingMatch[1].trim();
109
+ continue;
110
+ }
111
+ // Table row: | Name | Keys |
112
+ const tableMatch = line.match(/^\|(.+)\|(.+)\|/);
113
+ if (tableMatch) {
114
+ const name = tableMatch[1].trim();
115
+ const keys = tableMatch[2].trim();
116
+ if (name &&
117
+ keys &&
118
+ name !== "---" &&
119
+ !name.startsWith("-") &&
120
+ SHORTCUT_PATTERN.test(keys)) {
121
+ shortcuts.push({
122
+ name,
123
+ keys: normalizeKeys(keys),
124
+ category: currentCategory,
125
+ });
126
+ }
127
+ continue;
128
+ }
129
+ // List item: - Name: Keys or * Name — Keys
130
+ const listMatch = line.match(/^[*\-+]\s+(.+?)(?:\s*[:—–]\s*)(.+)$/);
131
+ if (listMatch && SHORTCUT_PATTERN.test(listMatch[2])) {
132
+ shortcuts.push({
133
+ name: listMatch[1].trim(),
134
+ keys: normalizeKeys(listMatch[2].trim()),
135
+ category: currentCategory,
136
+ });
137
+ continue;
138
+ }
139
+ // Inline code: `Cmd+K` — Description
140
+ const codeMatch = line.match(/`([^`]+)`\s*[-—:]\s*(.+)/);
141
+ if (codeMatch && SHORTCUT_PATTERN.test(codeMatch[1])) {
142
+ shortcuts.push({
143
+ name: codeMatch[2].trim(),
144
+ keys: normalizeKeys(codeMatch[1].trim()),
145
+ category: currentCategory,
146
+ });
147
+ }
148
+ }
149
+ return shortcuts;
150
+ }
151
+ /**
152
+ * Convert parsed shortcuts to reference JSON shortcuts format.
153
+ */
154
+ export function shortcutsToReferenceFormat(shortcuts) {
155
+ const result = {};
156
+ for (const sc of shortcuts) {
157
+ const category = sc.category ?? "general";
158
+ if (!result[category])
159
+ result[category] = {};
160
+ result[category][sc.name] = sc.keys;
161
+ }
162
+ return result;
163
+ }
164
+ // Pattern to detect shortcut-like strings
165
+ const SHORTCUT_PATTERN = /(?:Cmd|Ctrl|Alt|Option|Shift|Meta|⌘|⌃|⌥|⇧|Command|Control)[+\s]/i;
166
+ /**
167
+ * Normalize keyboard shortcut notation.
168
+ * Converts symbols to names: ⌘→Cmd, ⌃→Ctrl, ⌥→Option, ⇧→Shift
169
+ */
170
+ function normalizeKeys(keys) {
171
+ return keys
172
+ .replace(/⌘/g, "Cmd")
173
+ .replace(/⌃/g, "Ctrl")
174
+ .replace(/⌥/g, "Option")
175
+ .replace(/⇧/g, "Shift")
176
+ .replace(/Command/gi, "Cmd")
177
+ .replace(/Control/gi, "Ctrl")
178
+ .replace(/\s*\+\s*/g, "+")
179
+ .trim();
180
+ }
@@ -0,0 +1,170 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ // Patterns that indicate an action step in a tutorial
4
+ const ACTION_PATTERNS = [
5
+ /\b(?:click|tap|press|hit)\s+(?:on\s+)?(?:the\s+)?(.+?)(?:\s+button|\s+tab|\s+menu|\s+icon)?(?:\.|,|$)/i,
6
+ /\b(?:go to|navigate to|open|switch to)\s+(.+?)(?:\.|,|$)/i,
7
+ /\b(?:select|choose|pick)\s+(.+?)(?:\.|,|$)/i,
8
+ /\b(?:type|enter|input|write)\s+(.+?)(?:\.|,|$)/i,
9
+ /\b(?:drag|move)\s+(.+?)\s+to\s+(.+?)(?:\.|,|$)/i,
10
+ /\b(?:right[- ]?click|double[- ]?click)\s+(?:on\s+)?(.+?)(?:\.|,|$)/i,
11
+ /\bpress\s+((?:cmd|ctrl|alt|shift|command|control|option)[+\s].+?)(?:\.|,|$)/i,
12
+ /\b(?:scroll|zoom)\s+(.+?)(?:\.|,|$)/i,
13
+ /\b(?:set|change|adjust)\s+(.+?)\s+to\s+(.+?)(?:\.|,|$)/i,
14
+ ];
15
+ // Words that indicate non-action segments (skip these)
16
+ const SKIP_PATTERNS = [
17
+ /\bhey guys\b/i,
18
+ /\bsubscribe\b/i,
19
+ /\blike (?:this|the) video\b/i,
20
+ /\bcomment below\b/i,
21
+ /\bsponsor/i,
22
+ /\bwhat's up\b/i,
23
+ /\bhello everyone\b/i,
24
+ /\bwelcome (?:back|to)\b/i,
25
+ /\blet me know\b/i,
26
+ /\bcheck out\b/i,
27
+ /\blink in (?:the )?description\b/i,
28
+ ];
29
+ /**
30
+ * TutorialExtractor — extracts structured playbook steps from video
31
+ * transcripts (typically YouTube captions/subtitles).
32
+ */
33
+ export class TutorialExtractor {
34
+ /**
35
+ * Extract action steps from a transcript.
36
+ */
37
+ extract(segments, title, platform) {
38
+ const steps = [];
39
+ for (const segment of segments) {
40
+ // Skip filler/promo segments
41
+ if (SKIP_PATTERNS.some((p) => p.test(segment.text)))
42
+ continue;
43
+ const parsedSteps = this.parseSegment(segment.text);
44
+ for (const step of parsedSteps) {
45
+ // Deduplicate consecutive identical steps
46
+ const last = steps[steps.length - 1];
47
+ if (last && last.description === step.description)
48
+ continue;
49
+ steps.push(step);
50
+ }
51
+ }
52
+ return {
53
+ title,
54
+ platform,
55
+ steps,
56
+ rawSegments: segments.length,
57
+ actionSegments: steps.length,
58
+ extractedAt: new Date().toISOString(),
59
+ };
60
+ }
61
+ /**
62
+ * Convert extracted steps to a playbook-ready format.
63
+ */
64
+ toPlaybookSteps(result) {
65
+ return result.steps
66
+ .filter((s) => s.tool)
67
+ .map((step) => ({
68
+ action: step.tool === "key" ? "press" : step.tool === "type_text" ? "type" : "click",
69
+ tool: step.tool,
70
+ params: step.params ?? {},
71
+ description: step.description,
72
+ postcondition: step.postcondition,
73
+ }));
74
+ }
75
+ /**
76
+ * Parse a single transcript segment for action steps.
77
+ */
78
+ parseSegment(text) {
79
+ const steps = [];
80
+ // Split on sentence boundaries
81
+ const sentences = text.split(/(?<=[.!?])\s+|(?:,\s+(?:then|and then|next|after that)\s+)/i);
82
+ for (const sentence of sentences) {
83
+ const trimmed = sentence.trim();
84
+ if (trimmed.length < 5)
85
+ continue;
86
+ for (const pattern of ACTION_PATTERNS) {
87
+ const match = trimmed.match(pattern);
88
+ if (match) {
89
+ const step = this.mapToStep(trimmed, match);
90
+ if (step) {
91
+ steps.push(step);
92
+ break; // Only take first matching pattern per sentence
93
+ }
94
+ }
95
+ }
96
+ }
97
+ return steps;
98
+ }
99
+ /**
100
+ * Map a matched action to a ParsedFlowStep with tool and params.
101
+ */
102
+ mapToStep(fullText, match) {
103
+ const lower = fullText.toLowerCase();
104
+ const target = match[1]?.trim();
105
+ if (!target || target.length < 2)
106
+ return null;
107
+ // Determine tool and params
108
+ if (lower.includes("type") || lower.includes("enter") || lower.includes("input")) {
109
+ return {
110
+ description: fullText,
111
+ tool: "type_text",
112
+ params: { text: target },
113
+ };
114
+ }
115
+ if (lower.includes("press") && /(?:cmd|ctrl|alt|shift|command|control|option)/i.test(target)) {
116
+ return {
117
+ description: fullText,
118
+ tool: "key",
119
+ params: { combo: this.normalizeShortcut(target) },
120
+ };
121
+ }
122
+ if (lower.includes("drag") || lower.includes("move")) {
123
+ return {
124
+ description: fullText,
125
+ tool: "click_text",
126
+ params: { text: target },
127
+ };
128
+ }
129
+ if (lower.includes("scroll") || lower.includes("zoom")) {
130
+ return {
131
+ description: fullText,
132
+ tool: "scroll",
133
+ params: { direction: lower.includes("down") ? "down" : lower.includes("up") ? "up" : "down" },
134
+ };
135
+ }
136
+ if (lower.includes("menu") || target.includes(">")) {
137
+ const pathParts = target
138
+ .split(/\s*>\s*/)
139
+ .map((s) => s.trim())
140
+ .filter(Boolean);
141
+ if (pathParts.length >= 2) {
142
+ return {
143
+ description: fullText,
144
+ tool: "menu_click",
145
+ params: { menuPath: pathParts.join("/") },
146
+ };
147
+ }
148
+ }
149
+ if (lower.includes("set") || lower.includes("change") || lower.includes("adjust")) {
150
+ return {
151
+ description: fullText,
152
+ tool: "click_text",
153
+ params: { text: target },
154
+ };
155
+ }
156
+ // Default: click on the target
157
+ return {
158
+ description: fullText,
159
+ tool: "click_text",
160
+ params: { text: target },
161
+ };
162
+ }
163
+ normalizeShortcut(keys) {
164
+ return keys
165
+ .replace(/Command/gi, "Cmd")
166
+ .replace(/Control/gi, "Ctrl")
167
+ .replace(/\s+/g, "+")
168
+ .trim();
169
+ }
170
+ }
@@ -0,0 +1,3 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ export {};
@@ -67,9 +67,41 @@ export class JobManager {
67
67
  startedAt: null,
68
68
  completedAt: null,
69
69
  };
70
+ if (opts.chainId)
71
+ job.chainId = opts.chainId;
72
+ if (opts.dependsOn) {
73
+ // Validate dependency exists to prevent permanently stuck jobs
74
+ const dep = this.store.get(opts.dependsOn);
75
+ if (!dep) {
76
+ throw new Error(`Dependency job ${opts.dependsOn} does not exist`);
77
+ }
78
+ job.dependsOn = opts.dependsOn;
79
+ }
80
+ if (opts.vars)
81
+ job.vars = opts.vars;
70
82
  this.store.add(job);
71
83
  return job;
72
84
  }
85
+ /**
86
+ * Create a chain of linked jobs. Returns all created jobs.
87
+ * Each job depends on the previous one. Variables from prior job outputs
88
+ * are automatically passed forward using {jobId.outputKey} syntax.
89
+ */
90
+ createChain(opts) {
91
+ const chainId = opts.chainId ?? "chain_" + Date.now().toString(36) + "_" + Math.random().toString(36).slice(2, 8);
92
+ const created = [];
93
+ let prevId;
94
+ for (const jobOpts of opts.jobs) {
95
+ const job = this.create({
96
+ ...jobOpts,
97
+ chainId,
98
+ ...(prevId ? { dependsOn: prevId } : {}),
99
+ });
100
+ created.push(job);
101
+ prevId = job.id;
102
+ }
103
+ return created;
104
+ }
73
105
  // ── State transitions ───────────────────────────
74
106
  transition(id, to, opts) {
75
107
  const job = this.store.get(id);
@@ -111,7 +143,7 @@ export class JobManager {
111
143
  return updated ?? { error: `Failed to update job ${id}` };
112
144
  }
113
145
  // ── Step tracking ───────────────────────────────
114
- /** Mark a step as completed and advance lastStep. */
146
+ /** Mark a step as completed and advance lastStep. Optionally capture output. */
115
147
  completeStep(jobId, stepIndex, opts) {
116
148
  const job = this.store.get(jobId);
117
149
  if (!job)
@@ -125,14 +157,30 @@ export class JobManager {
125
157
  step.completedAt = new Date().toISOString();
126
158
  if (opts?.durationMs !== undefined)
127
159
  step.durationMs = opts.durationMs;
128
- const newLastStep = Math.max(job.lastStep, stepIndex);
129
- return this.store.update(jobId, { lastStep: newLastStep, steps: job.steps }) ?? { error: "Update failed" };
160
+ if (opts?.output !== undefined)
161
+ step.output = opts.output;
162
+ // Also store in job-level outputs for cross-job variable passing
163
+ const patch = { lastStep: Math.max(job.lastStep, stepIndex), steps: job.steps };
164
+ if (opts?.output !== undefined) {
165
+ const outputs = job.outputs ?? {};
166
+ outputs[String(stepIndex)] = opts.output;
167
+ // Also store by step description if available (friendlier key)
168
+ // Include step index to prevent collisions from similar descriptions
169
+ if (step.description) {
170
+ const key = step.description.replace(/[^a-zA-Z0-9_]/g, "_").substring(0, 50);
171
+ outputs[`${key}_${stepIndex}`] = opts.output;
172
+ }
173
+ patch.outputs = outputs;
174
+ }
175
+ return this.store.update(jobId, patch) ?? { error: "Update failed" };
130
176
  }
131
177
  /** Mark a step as failed. Does NOT transition the job — caller decides (retry vs block vs fail). */
132
178
  failStep(jobId, stepIndex, error) {
133
179
  const job = this.store.get(jobId);
134
180
  if (!job)
135
181
  return { error: `Job ${jobId} not found` };
182
+ if (job.state !== "running")
183
+ return { error: `Job is not running (state=${job.state})` };
136
184
  const step = job.steps[stepIndex];
137
185
  if (!step)
138
186
  return { error: `Step ${stepIndex} does not exist` };
@@ -145,6 +193,8 @@ export class JobManager {
145
193
  const job = this.store.get(jobId);
146
194
  if (!job)
147
195
  return { error: `Job ${jobId} not found` };
196
+ if (job.state !== "running")
197
+ return { error: `Job is not running (state=${job.state})` };
148
198
  const step = job.steps[stepIndex];
149
199
  if (!step)
150
200
  return { error: `Step ${stepIndex} does not exist` };
@@ -171,18 +221,36 @@ export class JobManager {
171
221
  list(state) {
172
222
  return this.store.list(state);
173
223
  }
174
- /** Pop the next queued job and transition it to running. */
224
+ /** Pop the next queued job and transition it to running. Skips jobs whose dependency isn't done yet. */
175
225
  dequeue(sessionId) {
176
- const next = this.store.nextQueued();
177
- if (!next)
178
- return null;
179
- const opts = {};
180
- if (sessionId !== undefined)
181
- opts.sessionId = sessionId;
182
- const result = this.transition(next.id, "running", opts);
183
- if ("error" in result)
184
- return null;
185
- return result;
226
+ const queued = this.store.list("queued")
227
+ .sort((a, b) => a.priority - b.priority || new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
228
+ for (const candidate of queued) {
229
+ // Check dependency — skip if dependent job isn't done
230
+ if (candidate.dependsOn) {
231
+ const dep = this.store.get(candidate.dependsOn);
232
+ if (!dep || dep.state !== "done")
233
+ continue;
234
+ // Resolve variables from dependency outputs
235
+ if (dep.outputs && candidate.vars) {
236
+ for (const [key, val] of Object.entries(candidate.vars)) {
237
+ // {prev.outputKey} → look up from dependency's outputs
238
+ const match = val.match(/^\{prev\.(.+)\}$/);
239
+ if (match?.[1] && dep.outputs[match[1]]) {
240
+ candidate.vars[key] = dep.outputs[match[1]];
241
+ }
242
+ }
243
+ this.store.update(candidate.id, { vars: candidate.vars });
244
+ }
245
+ }
246
+ const opts = {};
247
+ if (sessionId !== undefined)
248
+ opts.sessionId = sessionId;
249
+ const result = this.transition(candidate.id, "running", opts);
250
+ if (!("error" in result))
251
+ return result;
252
+ }
253
+ return null;
186
254
  }
187
255
  summary() {
188
256
  const all = this.store.list();