screenhand 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/README.md +193 -109
  2. package/bin/darwin-arm64/macos-bridge +0 -0
  3. package/dist/mcp-desktop.js +5876 -0
  4. package/dist/scripts/codex-monitor-daemon.js +335 -0
  5. package/dist/scripts/export-help-center.js +112 -0
  6. package/dist/scripts/marketing-loop.js +117 -0
  7. package/dist/scripts/observer-daemon.js +288 -0
  8. package/dist/scripts/orchestrator-daemon.js +399 -0
  9. package/dist/scripts/supervisor-daemon.js +272 -0
  10. package/dist/scripts/threads-campaign.js +208 -0
  11. package/dist/scripts/worker-daemon.js +228 -0
  12. package/dist/src/agent/cli.js +82 -0
  13. package/dist/src/agent/loop.js +274 -0
  14. package/dist/src/community/fetcher.js +109 -0
  15. package/dist/src/community/index.js +6 -0
  16. package/dist/src/community/publisher.js +191 -0
  17. package/dist/src/community/remote-api.js +121 -0
  18. package/dist/src/community/types.js +3 -0
  19. package/dist/src/community/validator.js +95 -0
  20. package/{src/config.ts → dist/src/config.js} +5 -10
  21. package/dist/src/context-tracker.js +489 -0
  22. package/{src/index.ts → dist/src/index.js} +32 -52
  23. package/dist/src/ingestion/coverage-auditor.js +233 -0
  24. package/dist/src/ingestion/doc-parser.js +164 -0
  25. package/dist/src/ingestion/index.js +8 -0
  26. package/dist/src/ingestion/menu-scanner.js +152 -0
  27. package/dist/src/ingestion/reference-merger.js +186 -0
  28. package/dist/src/ingestion/shortcut-extractor.js +180 -0
  29. package/dist/src/ingestion/tutorial-extractor.js +170 -0
  30. package/dist/src/ingestion/types.js +3 -0
  31. package/dist/src/jobs/manager.js +305 -0
  32. package/dist/src/jobs/runner.js +806 -0
  33. package/dist/src/jobs/store.js +102 -0
  34. package/dist/src/jobs/types.js +30 -0
  35. package/dist/src/jobs/worker.js +97 -0
  36. package/dist/src/learning/engine.js +356 -0
  37. package/dist/src/learning/index.js +9 -0
  38. package/dist/src/learning/locator-policy.js +120 -0
  39. package/dist/src/learning/pattern-policy.js +89 -0
  40. package/dist/src/learning/recovery-policy.js +116 -0
  41. package/dist/src/learning/sensor-policy.js +115 -0
  42. package/dist/src/learning/timing-model.js +204 -0
  43. package/dist/src/learning/topology-policy.js +90 -0
  44. package/dist/src/learning/types.js +9 -0
  45. package/dist/src/logging/timeline-logger.js +48 -0
  46. package/dist/src/mcp/mcp-stdio-server.js +464 -0
  47. package/dist/src/mcp/server.js +363 -0
  48. package/dist/src/mcp-entry.js +60 -0
  49. package/dist/src/memory/playbook-seeds.js +200 -0
  50. package/dist/src/memory/recall.js +222 -0
  51. package/dist/src/memory/research.js +104 -0
  52. package/dist/src/memory/seeds.js +101 -0
  53. package/dist/src/memory/service.js +446 -0
  54. package/dist/src/memory/session.js +169 -0
  55. package/dist/src/memory/store.js +451 -0
  56. package/{src/runtime/locator-cache.ts → dist/src/memory/types.js} +1 -17
  57. package/dist/src/monitor/codex-monitor.js +382 -0
  58. package/dist/src/monitor/task-queue.js +97 -0
  59. package/dist/src/monitor/types.js +62 -0
  60. package/dist/src/native/bridge-client.js +412 -0
  61. package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js} +0 -1
  62. package/dist/src/observer/state.js +199 -0
  63. package/dist/src/observer/types.js +43 -0
  64. package/dist/src/orchestrator/state.js +68 -0
  65. package/dist/src/orchestrator/types.js +22 -0
  66. package/dist/src/perception/ax-source.js +162 -0
  67. package/dist/src/perception/cdp-source.js +162 -0
  68. package/dist/src/perception/coordinator.js +771 -0
  69. package/dist/src/perception/frame-differ.js +287 -0
  70. package/dist/src/perception/index.js +22 -0
  71. package/dist/src/perception/manager.js +199 -0
  72. package/dist/src/perception/types.js +47 -0
  73. package/dist/src/perception/vision-source.js +399 -0
  74. package/dist/src/planner/deterministic.js +298 -0
  75. package/dist/src/planner/executor.js +870 -0
  76. package/dist/src/planner/goal-store.js +92 -0
  77. package/dist/src/planner/index.js +21 -0
  78. package/dist/src/planner/planner.js +520 -0
  79. package/dist/src/planner/tool-registry.js +71 -0
  80. package/dist/src/planner/types.js +22 -0
  81. package/dist/src/platform/explorer.js +213 -0
  82. package/dist/src/platform/help-center-markdown.js +527 -0
  83. package/dist/src/platform/learner.js +257 -0
  84. package/dist/src/playbook/engine.js +486 -0
  85. package/dist/src/playbook/index.js +20 -0
  86. package/dist/src/playbook/mcp-recorder.js +204 -0
  87. package/dist/src/playbook/recorder.js +536 -0
  88. package/dist/src/playbook/runner.js +408 -0
  89. package/dist/src/playbook/store.js +312 -0
  90. package/dist/src/playbook/types.js +17 -0
  91. package/dist/src/recovery/detectors.js +156 -0
  92. package/dist/src/recovery/engine.js +327 -0
  93. package/dist/src/recovery/index.js +20 -0
  94. package/dist/src/recovery/strategies.js +274 -0
  95. package/dist/src/recovery/types.js +20 -0
  96. package/dist/src/runtime/accessibility-adapter.js +430 -0
  97. package/dist/src/runtime/app-adapter.js +64 -0
  98. package/dist/src/runtime/applescript-adapter.js +305 -0
  99. package/dist/src/runtime/ax-role-map.js +96 -0
  100. package/dist/src/runtime/browser-adapter.js +52 -0
  101. package/dist/src/runtime/cdp-chrome-adapter.js +521 -0
  102. package/dist/src/runtime/composite-adapter.js +221 -0
  103. package/dist/src/runtime/execution-contract.js +159 -0
  104. package/dist/src/runtime/executor.js +286 -0
  105. package/dist/src/runtime/locator-cache.js +50 -0
  106. package/dist/src/runtime/planning-loop.js +63 -0
  107. package/dist/src/runtime/service.js +432 -0
  108. package/dist/src/runtime/session-manager.js +63 -0
  109. package/dist/src/runtime/state-observer.js +121 -0
  110. package/dist/src/runtime/vision-adapter.js +225 -0
  111. package/dist/src/state/app-map-types.js +72 -0
  112. package/dist/src/state/app-map.js +1974 -0
  113. package/dist/src/state/entity-tracker.js +108 -0
  114. package/dist/src/state/fusion.js +96 -0
  115. package/dist/src/state/index.js +21 -0
  116. package/dist/src/state/ladder-generator.js +236 -0
  117. package/dist/src/state/persistence.js +156 -0
  118. package/dist/src/state/types.js +17 -0
  119. package/dist/src/state/world-model.js +1456 -0
  120. package/dist/src/supervisor/locks.js +186 -0
  121. package/dist/src/supervisor/supervisor.js +403 -0
  122. package/dist/src/supervisor/types.js +30 -0
  123. package/dist/src/test-mcp-protocol.js +154 -0
  124. package/dist/src/types.js +17 -0
  125. package/dist/src/util/atomic-write.js +133 -0
  126. package/dist/src/util/sanitize.js +146 -0
  127. package/dist-app-maps/com.figma.Desktop.json +959 -0
  128. package/dist-app-maps/com.hnc.Discord.json +1146 -0
  129. package/dist-app-maps/notion.id.json +2831 -0
  130. package/dist-playbooks/canva-screenhand-carousel.json +445 -0
  131. package/dist-playbooks/codex-desktop.json +76 -0
  132. package/dist-playbooks/competitor-research-stack.json +122 -0
  133. package/dist-playbooks/davinci-color-grade.json +153 -0
  134. package/dist-playbooks/davinci-edit-timeline.json +162 -0
  135. package/dist-playbooks/davinci-render.json +114 -0
  136. package/dist-playbooks/devto.json +52 -0
  137. package/dist-playbooks/discord.json +41 -0
  138. package/dist-playbooks/google-flow-create-project.json +59 -0
  139. package/dist-playbooks/google-flow-edit-image.json +90 -0
  140. package/dist-playbooks/google-flow-edit-video.json +90 -0
  141. package/dist-playbooks/google-flow-generate-image.json +68 -0
  142. package/dist-playbooks/google-flow-generate-video.json +191 -0
  143. package/dist-playbooks/google-flow-open-project.json +48 -0
  144. package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
  145. package/dist-playbooks/google-flow-search-assets.json +64 -0
  146. package/dist-playbooks/instagram.json +57 -0
  147. package/dist-playbooks/linkedin.json +52 -0
  148. package/dist-playbooks/n8n.json +43 -0
  149. package/dist-playbooks/reddit.json +52 -0
  150. package/dist-playbooks/threads.json +59 -0
  151. package/dist-playbooks/x-twitter.json +59 -0
  152. package/dist-playbooks/youtube.json +59 -0
  153. package/dist-references/canva.json +646 -0
  154. package/dist-references/codex-desktop.json +305 -0
  155. package/dist-references/davinci-resolve-keyboard.json +594 -0
  156. package/dist-references/davinci-resolve-menu-map.json +1139 -0
  157. package/dist-references/davinci-resolve-menus-batch1.json +116 -0
  158. package/dist-references/davinci-resolve-menus-batch2.json +372 -0
  159. package/dist-references/davinci-resolve-menus-batch3.json +330 -0
  160. package/dist-references/davinci-resolve-menus-batch4.json +297 -0
  161. package/dist-references/davinci-resolve-shortcuts.json +333 -0
  162. package/dist-references/devto.json +317 -0
  163. package/dist-references/discord.json +549 -0
  164. package/dist-references/figma.json +1186 -0
  165. package/dist-references/finder.json +146 -0
  166. package/dist-references/google-ads-transparency.json +95 -0
  167. package/dist-references/google-flow.json +649 -0
  168. package/dist-references/instagram.json +341 -0
  169. package/dist-references/linkedin.json +324 -0
  170. package/dist-references/meta-ad-library.json +86 -0
  171. package/dist-references/n8n.json +387 -0
  172. package/dist-references/notes.json +27 -0
  173. package/dist-references/notion.json +163 -0
  174. package/dist-references/reddit.json +341 -0
  175. package/dist-references/threads.json +337 -0
  176. package/dist-references/x-twitter.json +403 -0
  177. package/dist-references/youtube.json +373 -0
  178. package/native/macos-bridge/Package.swift +1 -0
  179. package/native/macos-bridge/Sources/AccessibilityBridge.swift +257 -36
  180. package/native/macos-bridge/Sources/AppManagement.swift +212 -2
  181. package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +348 -53
  182. package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
  183. package/native/macos-bridge/Sources/VisionBridge.swift +165 -7
  184. package/native/macos-bridge/Sources/main.swift +169 -16
  185. package/native/windows-bridge/Program.cs +5 -0
  186. package/native/windows-bridge/ScreenCapture.cs +124 -0
  187. package/package.json +29 -4
  188. package/scripts/postinstall.cjs +127 -0
  189. package/.claude/commands/automate.md +0 -28
  190. package/.claude/commands/debug-ui.md +0 -19
  191. package/.claude/commands/screenshot.md +0 -15
  192. package/.github/FUNDING.yml +0 -1
  193. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -27
  194. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  195. package/.mcp.json +0 -8
  196. package/DESKTOP_MCP_GUIDE.md +0 -92
  197. package/SECURITY.md +0 -44
  198. package/docs/architecture.md +0 -47
  199. package/install-skills.sh +0 -19
  200. package/mcp-bridge.ts +0 -271
  201. package/mcp-desktop.ts +0 -1221
  202. package/playbooks/instagram.json +0 -41
  203. package/playbooks/instagram_v2.json +0 -201
  204. package/playbooks/x_v1.json +0 -211
  205. package/scripts/devpost-live-loop.mjs +0 -421
  206. package/src/logging/timeline-logger.ts +0 -55
  207. package/src/mcp/server.ts +0 -449
  208. package/src/memory/recall.ts +0 -191
  209. package/src/memory/research.ts +0 -146
  210. package/src/memory/seeds.ts +0 -123
  211. package/src/memory/session.ts +0 -201
  212. package/src/memory/store.ts +0 -434
  213. package/src/memory/types.ts +0 -69
  214. package/src/native/bridge-client.ts +0 -239
  215. package/src/runtime/accessibility-adapter.ts +0 -487
  216. package/src/runtime/app-adapter.ts +0 -169
  217. package/src/runtime/applescript-adapter.ts +0 -376
  218. package/src/runtime/ax-role-map.ts +0 -102
  219. package/src/runtime/browser-adapter.ts +0 -129
  220. package/src/runtime/cdp-chrome-adapter.ts +0 -676
  221. package/src/runtime/composite-adapter.ts +0 -274
  222. package/src/runtime/executor.ts +0 -396
  223. package/src/runtime/planning-loop.ts +0 -81
  224. package/src/runtime/service.ts +0 -448
  225. package/src/runtime/session-manager.ts +0 -50
  226. package/src/runtime/state-observer.ts +0 -136
  227. package/src/runtime/vision-adapter.ts +0 -297
  228. package/src/types.ts +0 -297
  229. package/tests/bridge-client.test.ts +0 -176
  230. package/tests/browser-stealth.test.ts +0 -210
  231. package/tests/composite-adapter.test.ts +0 -64
  232. package/tests/mcp-server.test.ts +0 -151
  233. package/tests/memory-recall.test.ts +0 -339
  234. package/tests/memory-research.test.ts +0 -159
  235. package/tests/memory-seeds.test.ts +0 -120
  236. package/tests/memory-store.test.ts +0 -392
  237. package/tests/types.test.ts +0 -92
  238. package/tsconfig.check.json +0 -17
  239. package/tsconfig.json +0 -19
  240. package/vitest.config.ts +0 -8
  241. /package/{playbooks → dist-references}/devpost.json +0 -0
@@ -0,0 +1,108 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ import crypto from "node:crypto";
4
+ const MAX_POSITIONS = 10;
5
+ const PROXIMITY_THRESHOLD = 50; // pixels
6
+ /**
7
+ * EntityTracker — persistent cross-frame identity for UI elements.
8
+ *
9
+ * A panel that moves 10px between frames keeps the same entityId
10
+ * instead of getting a new stableId (which includes quantized position).
11
+ * Matches by label similarity + position proximity.
12
+ */
13
+ export class EntityTracker {
14
+ entities = new Map();
15
+ /**
16
+ * Match an incoming element to an existing entity, or create a new one.
17
+ * Fuzzy match by label (exact) + window scope + position proximity (within threshold).
18
+ */
19
+ matchOrCreate(type, label, position, windowId) {
20
+ const now = new Date().toISOString();
21
+ // Try to find an existing entity with the same label, window, and nearby position
22
+ for (const entity of this.entities.values()) {
23
+ if (entity.type !== type || entity.label !== label)
24
+ continue;
25
+ if (entity.windowId !== windowId)
26
+ continue;
27
+ // Check position proximity against last known position
28
+ const lastPos = entity.positions[entity.positions.length - 1];
29
+ if (lastPos) {
30
+ const dx = Math.abs(lastPos.x - position.x);
31
+ const dy = Math.abs(lastPos.y - position.y);
32
+ if (dx <= PROXIMITY_THRESHOLD && dy <= PROXIMITY_THRESHOLD) {
33
+ // Match found — update position history
34
+ entity.lastSeen = now;
35
+ entity.positions.push({ x: position.x, y: position.y, timestamp: now });
36
+ if (entity.positions.length > MAX_POSITIONS) {
37
+ entity.positions = entity.positions.slice(-MAX_POSITIONS);
38
+ }
39
+ return entity;
40
+ }
41
+ }
42
+ else {
43
+ // No position history (e.g. after rehydrate with empty positions) —
44
+ // match by label+type+window alone, and seed the position
45
+ entity.lastSeen = now;
46
+ entity.positions.push({ x: position.x, y: position.y, timestamp: now });
47
+ return entity;
48
+ }
49
+ }
50
+ // No match — create new entity
51
+ const entityId = crypto.randomUUID();
52
+ const entity = {
53
+ entityId,
54
+ type,
55
+ label,
56
+ windowId,
57
+ stableIds: [],
58
+ firstSeen: now,
59
+ lastSeen: now,
60
+ positions: [{ x: position.x, y: position.y, timestamp: now }],
61
+ properties: {},
62
+ };
63
+ this.entities.set(entityId, entity);
64
+ return entity;
65
+ }
66
+ /**
67
+ * Get entities filtered by type.
68
+ */
69
+ getByType(type) {
70
+ return [...this.entities.values()].filter((e) => e.type === type);
71
+ }
72
+ /**
73
+ * Get all tracked entities.
74
+ */
75
+ getAll() {
76
+ return new Map(this.entities);
77
+ }
78
+ /**
79
+ * Remove entities not seen within maxAgeMs.
80
+ */
81
+ pruneStale(maxAgeMs) {
82
+ const cutoff = Date.now() - maxAgeMs;
83
+ let pruned = 0;
84
+ for (const [id, entity] of this.entities) {
85
+ if (new Date(entity.lastSeen).getTime() < cutoff) {
86
+ this.entities.delete(id);
87
+ pruned++;
88
+ }
89
+ }
90
+ return pruned;
91
+ }
92
+ /**
93
+ * Rehydrate internal state from persisted entities.
94
+ * Clears current state first to avoid duplication.
95
+ */
96
+ rehydrate(entities) {
97
+ this.entities.clear();
98
+ for (const [id, entity] of entities) {
99
+ this.entities.set(id, { ...entity, positions: [...entity.positions] });
100
+ }
101
+ }
102
+ /**
103
+ * Clear all entities.
104
+ */
105
+ clear() {
106
+ this.entities.clear();
107
+ }
108
+ }
@@ -0,0 +1,96 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ /** Clamp confidence to [0, 1], replacing NaN/Infinity with a safe default. */
4
+ function sanitizeConfidence(c) {
5
+ if (isNaN(c) || !isFinite(c))
6
+ return 0.5;
7
+ return Math.max(0, Math.min(1, c));
8
+ }
9
+ /**
10
+ * FusionPipeline — unified ingestion point that queues source updates
11
+ * and flushes them into the world model in timestamp order.
12
+ *
13
+ * Benefits over direct ingest calls:
14
+ * - Deduplication by source+windowId (keeps latest per source+window)
15
+ * - Timestamp-ordered processing
16
+ * - Learning-adaptive confidence via LearningEngine
17
+ */
18
+ const MAX_QUEUE_SIZE = 100;
19
+ export class FusionPipeline {
20
+ queue = [];
21
+ learningEngine = null;
22
+ setLearningEngine(engine) {
23
+ this.learningEngine = engine;
24
+ }
25
+ /**
26
+ * Enqueue a source update for batch processing.
27
+ */
28
+ enqueue(update) {
29
+ // Sanitize confidence: clamp to [0, 1], replace NaN with 0.5
30
+ update.confidence = sanitizeConfidence(update.confidence);
31
+ // Deduplicate: if there's already an update for the same source+windowId,
32
+ // keep the newer one (by timestamp)
33
+ const existingIdx = this.queue.findIndex((u) => u.source === update.source && u.windowId === update.windowId);
34
+ if (existingIdx >= 0) {
35
+ const existing = this.queue[existingIdx];
36
+ if (update.timestamp >= existing.timestamp) {
37
+ this.queue[existingIdx] = update;
38
+ }
39
+ // else: existing is newer, drop the incoming
40
+ return;
41
+ }
42
+ // Cap queue size to prevent unbounded growth during slow flushes
43
+ if (this.queue.length >= MAX_QUEUE_SIZE) {
44
+ // Drop oldest non-deduplicated entry
45
+ this.queue.shift();
46
+ }
47
+ this.queue.push(update);
48
+ }
49
+ /**
50
+ * Process all queued updates in timestamp order and flush into the world model.
51
+ */
52
+ flush(worldModel) {
53
+ if (this.queue.length === 0)
54
+ return;
55
+ // Sort by timestamp
56
+ this.queue.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
57
+ for (const update of this.queue) {
58
+ // Resolve adaptive confidence and apply it to the update
59
+ update.confidence = this.resolveConfidence(update);
60
+ if (update.source === "ax" && update.axTree && update.appContext) {
61
+ worldModel.ingestAXTree(update.windowId, update.axTree, update.appContext, update.confidence);
62
+ }
63
+ else if (update.source === "ocr" && update.ocrRegions) {
64
+ worldModel.ingestOCRRegions(update.windowId, update.ocrRegions, update.confidence);
65
+ }
66
+ else if (update.source === "cdp" && update.cdpSnapshot) {
67
+ worldModel.ingestCDPSnapshot(update.cdpSnapshot.bundleId, update.cdpSnapshot.url, update.cdpSnapshot.title, update.windowId);
68
+ }
69
+ }
70
+ this.queue = [];
71
+ }
72
+ /**
73
+ * Get the number of queued updates.
74
+ */
75
+ get size() {
76
+ return this.queue.length;
77
+ }
78
+ /**
79
+ * Resolve source confidence — use learning engine if available,
80
+ * otherwise fall back to hardcoded defaults.
81
+ */
82
+ resolveConfidence(update) {
83
+ if (!this.learningEngine)
84
+ return sanitizeConfidence(update.confidence);
85
+ // Get bundleId from the update context
86
+ const bundleId = update.appContext?.bundleId ?? update.cdpSnapshot?.bundleId;
87
+ if (!bundleId)
88
+ return sanitizeConfidence(update.confidence);
89
+ const ranked = this.learningEngine.rankSensors(bundleId);
90
+ const match = ranked.find((r) => r.sourceType === update.source);
91
+ if (match && match.score > 0) {
92
+ return sanitizeConfidence(match.score);
93
+ }
94
+ return sanitizeConfidence(update.confidence);
95
+ }
96
+ }
@@ -0,0 +1,21 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export { WorldModel } from "./world-model.js";
18
+ export { EntityTracker } from "./entity-tracker.js";
19
+ export { AppMap } from "./app-map.js";
20
+ export { DEFAULT_APP_MAP_CONFIG, RATING_GRADES, RATING_FACTOR_WEIGHTS, GRADE_THRESHOLDS, ratingToString, } from "./app-map-types.js";
21
+ export { generateLadderFromReference } from "./ladder-generator.js";
@@ -0,0 +1,236 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ // ── Keyword sets for level assignment ────────────────────────────
4
+ const BEGINNER_KEYWORDS = new Set([
5
+ "navigation", "browse", "basic", "search", "header", "page", "nav", "home",
6
+ "page_header", "quick_find",
7
+ ]);
8
+ const PRO_KEYWORDS = new Set([
9
+ "settings", "create", "views", "sort", "filter", "template", "new", "sidebar",
10
+ "create_new", "slash_commands", "notification", "preferences",
11
+ ]);
12
+ const EXPERT_KEYWORDS = new Set([
13
+ "database", "admin", "moderation", "permissions", "automation", "advanced",
14
+ "server", "import", "export", "workflow", "security", "form",
15
+ ]);
16
+ const GRANDMASTER_KEYWORDS = new Set([
17
+ "ai", "api", "integration", "custom", "governance", "crisis", "identity",
18
+ "billing", "orchestrat", "pipeline",
19
+ ]);
20
+ // ── Stop words to exclude from signal keywords ───────────────────
21
+ const STOP_WORDS = new Set([
22
+ "the", "a", "an", "in", "to", "or", "and", "at", "for", "of", "is", "on",
23
+ "by", "it", "if", "be", "as", "this", "that", "with", "from", "then",
24
+ "via", "e.g.", "etc", "note", "use",
25
+ ]);
26
+ // ── Selector groups to skip (not real features) ──────────────────
27
+ const SKIP_GROUPS = new Set(["auto_discovered"]);
28
+ // ── Main: Generate ladder from reference ─────────────────────────
29
+ export function generateLadderFromReference(ref) {
30
+ const features = [];
31
+ const signals = {};
32
+ const selectorGroups = ref.selectors ?? {};
33
+ const flows = ref.flows ?? {};
34
+ // Minimum threshold: need at least 2 meaningful selector groups
35
+ const meaningfulGroups = Object.keys(selectorGroups).filter(k => !SKIP_GROUPS.has(k));
36
+ if (meaningfulGroups.length < 2 && Object.keys(flows).length < 2) {
37
+ return { ladder: [], signals: {}, hash: computeHash(ref) };
38
+ }
39
+ // Track which flow names are already covered by selector groups
40
+ const coveredFlows = new Set();
41
+ // ── Step 1: Features from selector groups ──────────────────────
42
+ for (const groupName of meaningfulGroups) {
43
+ const sels = selectorGroups[groupName];
44
+ const selCount = Object.keys(sels).length;
45
+ const level = assignLevel(groupName, selCount, false, 0);
46
+ const weight = assignWeight(groupName, selCount, level);
47
+ const critical = selCount >= 6 || weight === 3;
48
+ const featureId = groupName;
49
+ const description = describeFeature(groupName, sels);
50
+ features.push({ id: featureId, description, level, weight, critical });
51
+ // Generate signal keywords from selector keys and values
52
+ const keywords = extractKeywordsFromSelectors(groupName, sels);
53
+ signals[featureId] = keywords;
54
+ // Mark related flows as covered
55
+ for (const flowName of Object.keys(flows)) {
56
+ if (flowNamesRelated(groupName, flowName)) {
57
+ coveredFlows.add(flowName);
58
+ // Merge flow keywords into this feature's signals
59
+ const flowKw = extractKeywordsFromFlow(flowName, flows[flowName]);
60
+ signals[featureId] = deduplicateArray([...signals[featureId], ...flowKw]);
61
+ }
62
+ }
63
+ }
64
+ // ── Step 2: Additional features from uncovered flows ───────────
65
+ for (const [flowName, flow] of Object.entries(flows)) {
66
+ if (coveredFlows.has(flowName))
67
+ continue;
68
+ const stepCount = flow.steps?.length ?? 0;
69
+ const guardCount = flow.guards?.length ?? 0;
70
+ const level = assignLevel(flowName, 0, true, stepCount + guardCount);
71
+ const weight = assignWeight(flowName, stepCount, level);
72
+ const critical = guardCount >= 2 || weight === 3;
73
+ const featureId = flowName;
74
+ const description = flow.steps?.[0] ?? `${flowName.replace(/_/g, " ")} workflow`;
75
+ features.push({ id: featureId, description, level, weight, critical });
76
+ const keywords = extractKeywordsFromFlow(flowName, flow);
77
+ signals[featureId] = keywords;
78
+ }
79
+ // ── Step 3: Sort by level progression ──────────────────────────
80
+ const levelOrder = {
81
+ beginner: 0, pro: 1, expert: 2, grandmaster: 3,
82
+ };
83
+ features.sort((a, b) => levelOrder[a.level] - levelOrder[b.level]);
84
+ return { ladder: features, signals, hash: computeHash(ref) };
85
+ }
86
+ // ── Level assignment heuristics ──────────────────────────────────
87
+ function assignLevel(name, selectorCount, isFlow, complexity) {
88
+ const nameLower = name.toLowerCase();
89
+ const parts = nameLower.split(/[_\s-]+/);
90
+ // Check grandmaster first (most specific)
91
+ if (parts.some(p => GRANDMASTER_KEYWORDS.has(p)))
92
+ return "grandmaster";
93
+ if (parts.some(p => EXPERT_KEYWORDS.has(p)))
94
+ return "expert";
95
+ if (parts.some(p => PRO_KEYWORDS.has(p)))
96
+ return "pro";
97
+ if (parts.some(p => BEGINNER_KEYWORDS.has(p)))
98
+ return "beginner";
99
+ // Fallback based on complexity
100
+ if (isFlow) {
101
+ if (complexity >= 8)
102
+ return "expert";
103
+ if (complexity >= 5)
104
+ return "pro";
105
+ return "beginner";
106
+ }
107
+ // Selector group: more selectors = more complex
108
+ if (selectorCount >= 8)
109
+ return "expert";
110
+ if (selectorCount >= 4)
111
+ return "pro";
112
+ return "beginner";
113
+ }
114
+ // ── Weight assignment ────────────────────────────────────────────
115
+ function assignWeight(name, complexity, level) {
116
+ if (level === "grandmaster")
117
+ return 3;
118
+ if (level === "expert" && complexity >= 6)
119
+ return 3;
120
+ if (level === "expert" || level === "pro")
121
+ return 2;
122
+ return 1;
123
+ }
124
+ // ── Description generation ───────────────────────────────────────
125
+ function describeFeature(groupName, sels) {
126
+ const humanName = groupName.replace(/_/g, " ");
127
+ const selNames = Object.keys(sels).slice(0, 3).map(k => k.replace(/_/g, " ")).join(", ");
128
+ return `${humanName}: ${selNames}`;
129
+ }
130
+ // ── Keyword extraction from selectors ────────────────────────────
131
+ function extractKeywordsFromSelectors(groupName, sels) {
132
+ const keywords = [];
133
+ // Add group name parts
134
+ for (const part of groupName.split("_")) {
135
+ if (!STOP_WORDS.has(part.toLowerCase()) && part.length > 1) {
136
+ keywords.push(part.toLowerCase());
137
+ }
138
+ }
139
+ // Add selector key parts
140
+ for (const key of Object.keys(sels)) {
141
+ for (const part of key.split("_")) {
142
+ const lower = part.toLowerCase();
143
+ if (!STOP_WORDS.has(lower) && lower.length > 2) {
144
+ keywords.push(lower);
145
+ }
146
+ }
147
+ }
148
+ // Extract significant words from selector values
149
+ for (const val of Object.values(sels)) {
150
+ // Extract quoted text: 'text' or "text"
151
+ const quoted = val.match(/['"]([^'"]{2,20})['"]/g);
152
+ if (quoted) {
153
+ for (const q of quoted) {
154
+ const clean = q.replace(/['"]/g, "").toLowerCase();
155
+ if (!STOP_WORDS.has(clean))
156
+ keywords.push(clean);
157
+ }
158
+ }
159
+ // Extract shortcut keys like Cmd+K
160
+ const shortcuts = val.match(/Cmd\+\S+/gi);
161
+ if (shortcuts) {
162
+ for (const s of shortcuts)
163
+ keywords.push(s.toLowerCase());
164
+ }
165
+ }
166
+ return deduplicateArray(keywords);
167
+ }
168
+ // ── Keyword extraction from flows ────────────────────────────────
169
+ function extractKeywordsFromFlow(flowName, flow) {
170
+ const keywords = [];
171
+ // Flow name parts
172
+ for (const part of flowName.split("_")) {
173
+ if (!STOP_WORDS.has(part.toLowerCase()) && part.length > 1) {
174
+ keywords.push(part.toLowerCase());
175
+ }
176
+ }
177
+ // Extract from steps
178
+ if (flow.steps) {
179
+ for (const step of flow.steps) {
180
+ // Extract quoted text
181
+ const quoted = step.match(/['"]([^'"]{2,25})['"]/g);
182
+ if (quoted) {
183
+ for (const q of quoted) {
184
+ const clean = q.replace(/['"]/g, "").toLowerCase();
185
+ if (!STOP_WORDS.has(clean) && clean.length > 2)
186
+ keywords.push(clean);
187
+ }
188
+ }
189
+ // Extract shortcut keys
190
+ const shortcuts = step.match(/Cmd\+\S+/gi);
191
+ if (shortcuts) {
192
+ for (const s of shortcuts)
193
+ keywords.push(s.toLowerCase());
194
+ }
195
+ // Extract action verbs and significant nouns
196
+ const words = step.split(/\s+/);
197
+ for (const w of words) {
198
+ const lower = w.toLowerCase().replace(/[^a-z0-9+]/g, "");
199
+ if (lower.length > 3 && !STOP_WORDS.has(lower)) {
200
+ keywords.push(lower);
201
+ }
202
+ }
203
+ }
204
+ }
205
+ return deduplicateArray(keywords);
206
+ }
207
+ // ── Flow-to-selector group name matching ─────────────────────────
208
+ function flowNamesRelated(groupName, flowName) {
209
+ const gParts = new Set(groupName.split("_"));
210
+ const fParts = flowName.split("_");
211
+ // At least 50% of flow name parts match group name parts
212
+ const matches = fParts.filter(p => gParts.has(p)).length;
213
+ if (matches >= Math.ceil(fParts.length * 0.5))
214
+ return true;
215
+ // Direct substring match
216
+ if (groupName.includes(flowName) || flowName.includes(groupName))
217
+ return true;
218
+ return false;
219
+ }
220
+ // ── Hash computation for cache invalidation ──────────────────────
221
+ function computeHash(ref) {
222
+ const keys = [
223
+ ...Object.keys(ref.selectors ?? {}).sort(),
224
+ ...Object.keys(ref.flows ?? {}).sort(),
225
+ ].join("|");
226
+ // Simple string hash (djb2)
227
+ let hash = 5381;
228
+ for (let i = 0; i < keys.length; i++) {
229
+ hash = ((hash << 5) + hash + keys.charCodeAt(i)) | 0;
230
+ }
231
+ return `ref_${Math.abs(hash).toString(36)}`;
232
+ }
233
+ // ── Utilities ────────────────────────────────────────────────────
234
+ function deduplicateArray(arr) {
235
+ return [...new Set(arr)];
236
+ }
@@ -0,0 +1,156 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ import fs from "node:fs";
18
+ import os from "node:os";
19
+ import path from "node:path";
20
+ import { writeFileAtomicSync, readJsonWithRecovery } from "../util/atomic-write.js";
21
+ const DEFAULT_STATE_DIR = path.join(os.homedir(), ".screenhand", "state");
22
+ function stateFilePath(stateDir, sessionId) {
23
+ // Sanitize sessionId to prevent path traversal
24
+ const safeId = sessionId.replace(/[^a-zA-Z0-9_\-]/g, "_");
25
+ const filePath = path.join(stateDir, `${safeId}.json`);
26
+ // Double-check the resolved path stays inside stateDir
27
+ const resolved = path.resolve(filePath);
28
+ if (!resolved.startsWith(path.resolve(stateDir))) {
29
+ return path.join(stateDir, "invalid_session.json");
30
+ }
31
+ return filePath;
32
+ }
33
+ export function worldStateToJSON(state) {
34
+ const serialized = {
35
+ windows: Object.fromEntries(Array.from(state.windows.entries()).map(([id, win]) => [
36
+ String(id),
37
+ {
38
+ ...win,
39
+ controls: Object.fromEntries(win.controls),
40
+ },
41
+ ])),
42
+ focusedWindowId: state.focusedWindowId,
43
+ focusedApp: state.focusedApp,
44
+ activeDialogs: state.activeDialogs.map((d) => ({
45
+ ...d,
46
+ controls: Object.fromEntries(d.controls),
47
+ })),
48
+ appDomains: Object.fromEntries(state.appDomains),
49
+ lastFullScan: state.lastFullScan,
50
+ sessionId: state.sessionId,
51
+ expectedPostcondition: state.expectedPostcondition,
52
+ updatedAt: state.updatedAt,
53
+ confidence: state.confidence,
54
+ pendingGoal: state.pendingGoal,
55
+ recentTransitions: state.recentTransitions,
56
+ trackedEntities: Object.fromEntries(state.trackedEntities),
57
+ };
58
+ return JSON.stringify(serialized);
59
+ }
60
+ export function worldStateFromJSON(json) {
61
+ const windows = new Map();
62
+ for (const [idStr, win] of Object.entries(json.windows ?? {})) {
63
+ windows.set(Number(idStr), {
64
+ ...win,
65
+ controls: new Map(Object.entries(win.controls)),
66
+ // Defaults for new WindowState fields (backwards compat)
67
+ focusedElement: win.focusedElement ?? null,
68
+ visibleControls: win.visibleControls ?? [],
69
+ dialogStack: win.dialogStack ?? [],
70
+ scrollPosition: win.scrollPosition ?? null,
71
+ lastAXScanAt: win.lastAXScanAt ?? null,
72
+ lastCDPScanAt: win.lastCDPScanAt ?? null,
73
+ lastOCRAt: win.lastOCRAt ?? null,
74
+ lastScreenshotHash: win.lastScreenshotHash ?? null,
75
+ });
76
+ }
77
+ const activeDialogs = (json.activeDialogs ?? []).map((d) => ({
78
+ ...d,
79
+ controls: new Map(Object.entries(d.controls)),
80
+ // Defaults for new DialogState fields (backwards compat)
81
+ message: d.message ?? null,
82
+ buttons: d.buttons ?? [],
83
+ source: d.source ?? "ax",
84
+ }));
85
+ const appDomains = new Map(Object.entries(json.appDomains ?? {}));
86
+ return {
87
+ windows,
88
+ focusedWindowId: json.focusedWindowId,
89
+ focusedApp: json.focusedApp,
90
+ activeDialogs,
91
+ appDomains,
92
+ lastFullScan: json.lastFullScan,
93
+ sessionId: json.sessionId,
94
+ expectedPostcondition: json.expectedPostcondition ?? null,
95
+ updatedAt: json.updatedAt ?? json.lastFullScan,
96
+ confidence: json.confidence ?? 1.0,
97
+ pendingGoal: json.pendingGoal ?? null,
98
+ recentTransitions: json.recentTransitions ?? [],
99
+ trackedEntities: new Map(Object.entries(json.trackedEntities ?? {})),
100
+ };
101
+ }
102
+ export function saveWorldState(state, stateDir) {
103
+ const dir = stateDir ?? DEFAULT_STATE_DIR;
104
+ fs.mkdirSync(dir, { recursive: true });
105
+ const filePath = stateFilePath(dir, state.sessionId);
106
+ writeFileAtomicSync(filePath, worldStateToJSON(state));
107
+ }
108
+ export function loadWorldState(sessionId, stateDir) {
109
+ const dir = stateDir ?? DEFAULT_STATE_DIR;
110
+ const filePath = stateFilePath(dir, sessionId);
111
+ const raw = readJsonWithRecovery(filePath);
112
+ if (!raw)
113
+ return null;
114
+ return worldStateFromJSON(raw);
115
+ }
116
+ export class DebouncedPersister {
117
+ timer = null;
118
+ pending = null;
119
+ saveFn;
120
+ constructor(debounceMs, saveFn) {
121
+ this.saveFn = saveFn ?? ((s) => saveWorldState(s));
122
+ if (debounceMs <= 0) {
123
+ // Zero debounce = no-op persister (for tests)
124
+ this.schedule = () => { };
125
+ this.flush = () => { };
126
+ return;
127
+ }
128
+ // Store debounceMs for use in schedule
129
+ const ms = debounceMs;
130
+ this.schedule = (state) => {
131
+ this.pending = state;
132
+ if (this.timer)
133
+ return;
134
+ this.timer = setTimeout(() => {
135
+ this.timer = null;
136
+ if (this.pending) {
137
+ this.saveFn(this.pending);
138
+ this.pending = null;
139
+ }
140
+ }, ms);
141
+ };
142
+ }
143
+ schedule(_state) {
144
+ // Overridden in constructor when debounceMs > 0
145
+ }
146
+ flush() {
147
+ if (this.timer) {
148
+ clearTimeout(this.timer);
149
+ this.timer = null;
150
+ }
151
+ if (this.pending) {
152
+ this.saveFn(this.pending);
153
+ this.pending = null;
154
+ }
155
+ }
156
+ }
@@ -0,0 +1,17 @@
1
+ // Copyright (C) 2025 Clazro Technology Private Limited
2
+ // SPDX-License-Identifier: AGPL-3.0-only
3
+ //
4
+ // This file is part of ScreenHand.
5
+ //
6
+ // ScreenHand is free software: you can redistribute it and/or modify
7
+ // it under the terms of the GNU Affero General Public License as
8
+ // published by the Free Software Foundation, version 3.
9
+ //
10
+ // ScreenHand is distributed in the hope that it will be useful,
11
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ // GNU Affero General Public License for more details.
14
+ //
15
+ // You should have received a copy of the GNU Affero General Public License
16
+ // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
17
+ export {};