vellum 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/README.md +15 -2
  2. package/bun.lock +5 -2
  3. package/package.json +4 -2
  4. package/scripts/capture-x-graphql.ts +562 -0
  5. package/scripts/ipc/check-swift-decoder-drift.ts +2 -1
  6. package/scripts/test.sh +5 -0
  7. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +161 -34
  8. package/src/__tests__/account-registry.test.ts +2 -1
  9. package/src/__tests__/agent-heartbeat-service.test.ts +250 -0
  10. package/src/__tests__/app-bundler.test.ts +12 -33
  11. package/src/__tests__/asset-materialize-tool.test.ts +16 -15
  12. package/src/__tests__/asset-search-tool.test.ts +23 -22
  13. package/src/__tests__/attachments-store.test.ts +56 -127
  14. package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +5 -4
  15. package/src/__tests__/browser-skill-endstate.test.ts +5 -8
  16. package/src/__tests__/call-bridge.test.ts +385 -0
  17. package/src/__tests__/call-constants.test.ts +40 -0
  18. package/src/__tests__/call-orchestrator.test.ts +454 -0
  19. package/src/__tests__/call-recovery.test.ts +518 -0
  20. package/src/__tests__/call-routes-http.test.ts +459 -0
  21. package/src/__tests__/call-state-machine.test.ts +143 -0
  22. package/src/__tests__/call-state.test.ts +133 -0
  23. package/src/__tests__/call-store.test.ts +691 -0
  24. package/src/__tests__/cli-discover.test.ts +1 -1
  25. package/src/__tests__/commit-message-enrichment-service.test.ts +550 -0
  26. package/src/__tests__/compaction.benchmark.test.ts +176 -0
  27. package/src/__tests__/computer-use-tools.test.ts +250 -0
  28. package/src/__tests__/config-schema.test.ts +348 -3
  29. package/src/__tests__/conflict-store.test.ts +2 -1
  30. package/src/__tests__/contacts-tools.test.ts +331 -0
  31. package/src/__tests__/conversation-store.test.ts +30 -32
  32. package/src/__tests__/credential-security-invariants.test.ts +4 -0
  33. package/src/__tests__/date-context.test.ts +373 -0
  34. package/src/__tests__/db-schedule-syntax-migration.test.ts +129 -0
  35. package/src/__tests__/doordash-session.test.ts +9 -0
  36. package/src/__tests__/fixtures/media-reuse-fixtures.ts +3 -3
  37. package/src/__tests__/followup-tools.test.ts +303 -0
  38. package/src/__tests__/handlers-twitter-config.test.ts +718 -0
  39. package/src/__tests__/intent-routing.test.ts +64 -57
  40. package/src/__tests__/ipc-roundtrip.benchmark.test.ts +237 -0
  41. package/src/__tests__/ipc-snapshot.test.ts +96 -28
  42. package/src/__tests__/llm-usage-store.test.ts +3 -8
  43. package/src/__tests__/media-generate-image.test.ts +1 -1
  44. package/src/__tests__/media-reuse-story.e2e.test.ts +7 -7
  45. package/src/__tests__/memory-retrieval.benchmark.test.ts +430 -0
  46. package/src/__tests__/parallel-tool.benchmark.test.ts +294 -0
  47. package/src/__tests__/playbook-tools.test.ts +342 -0
  48. package/src/__tests__/profile-compiler.test.ts +2 -1
  49. package/src/__tests__/provider-streaming.benchmark.test.ts +773 -0
  50. package/src/__tests__/recurrence-engine-rruleset.test.ts +78 -0
  51. package/src/__tests__/recurrence-engine.test.ts +69 -0
  52. package/src/__tests__/recurrence-types.test.ts +71 -0
  53. package/src/__tests__/registry.test.ts +17 -10
  54. package/src/__tests__/relay-server.test.ts +633 -0
  55. package/src/__tests__/reminder-store.test.ts +6 -3
  56. package/src/__tests__/reminder.test.ts +43 -77
  57. package/src/__tests__/run-orchestrator-assistant-events.test.ts +222 -0
  58. package/src/__tests__/run-orchestrator.test.ts +7 -7
  59. package/src/__tests__/runtime-attachment-metadata.test.ts +19 -20
  60. package/src/__tests__/runtime-runs-http.test.ts +5 -23
  61. package/src/__tests__/runtime-runs.test.ts +11 -11
  62. package/src/__tests__/schedule-store.test.ts +482 -0
  63. package/src/__tests__/schedule-tools.test.ts +700 -0
  64. package/src/__tests__/scheduler-recurrence.test.ts +329 -0
  65. package/src/__tests__/server-history-render.test.ts +14 -13
  66. package/src/__tests__/session-error.test.ts +28 -0
  67. package/src/__tests__/session-init.benchmark.test.ts +462 -0
  68. package/src/__tests__/session-queue.test.ts +89 -16
  69. package/src/__tests__/session-runtime-assembly.test.ts +161 -0
  70. package/src/__tests__/session-surfaces-task-progress.test.ts +104 -0
  71. package/src/__tests__/signup-e2e.test.ts +2 -1
  72. package/src/__tests__/skill-projection.benchmark.test.ts +328 -0
  73. package/src/__tests__/skill-script-runner.test.ts +159 -0
  74. package/src/__tests__/speaker-identification.test.ts +52 -0
  75. package/src/__tests__/subagent-manager-notify.test.ts +42 -10
  76. package/src/__tests__/subagent-tools.test.ts +141 -41
  77. package/src/__tests__/task-compiler.test.ts +2 -1
  78. package/src/__tests__/task-runner.test.ts +2 -1
  79. package/src/__tests__/task-scheduler.test.ts +2 -1
  80. package/src/__tests__/task-tools.test.ts +49 -56
  81. package/src/__tests__/tool-audit-listener.test.ts +1 -0
  82. package/src/__tests__/tool-domain-event-publisher.test.ts +2 -0
  83. package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +500 -0
  84. package/src/__tests__/tool-executor.test.ts +13 -17
  85. package/src/__tests__/turn-commit.test.ts +273 -2
  86. package/src/__tests__/twilio-provider.test.ts +143 -0
  87. package/src/__tests__/twilio-routes.test.ts +789 -0
  88. package/src/__tests__/twitter-auth-handler.test.ts +581 -0
  89. package/src/__tests__/view-image-tool.test.ts +217 -0
  90. package/src/__tests__/workspace-git-service.test.ts +403 -0
  91. package/src/__tests__/workspace-heartbeat-service.test.ts +141 -2
  92. package/src/agent-heartbeat/agent-heartbeat-service.ts +155 -0
  93. package/src/bundler/app-bundler.ts +35 -14
  94. package/src/calls/call-bridge.ts +95 -0
  95. package/src/calls/call-constants.ts +48 -0
  96. package/src/calls/call-domain.ts +276 -0
  97. package/src/calls/call-orchestrator.ts +390 -0
  98. package/src/calls/call-recovery.ts +207 -0
  99. package/src/calls/call-state-machine.ts +68 -0
  100. package/src/calls/call-state.ts +64 -0
  101. package/src/calls/call-store.ts +416 -0
  102. package/src/calls/relay-server.ts +335 -0
  103. package/src/calls/speaker-identification.ts +213 -0
  104. package/src/calls/twilio-config.ts +34 -0
  105. package/src/calls/twilio-provider.ts +173 -0
  106. package/src/calls/twilio-routes.ts +250 -0
  107. package/src/calls/types.ts +37 -0
  108. package/src/calls/voice-provider.ts +14 -0
  109. package/src/cli/config-commands.ts +334 -0
  110. package/src/cli/core-commands.ts +776 -0
  111. package/src/cli/doordash.ts +256 -25
  112. package/src/cli/ipc-client.ts +82 -0
  113. package/src/cli/map.ts +246 -0
  114. package/src/cli/twitter.ts +575 -0
  115. package/src/cli.ts +7 -5
  116. package/src/commands/__tests__/cc-command-registry.test.ts +319 -0
  117. package/src/commands/cc-command-registry.ts +209 -0
  118. package/src/config/bundled-skills/contacts/SKILL.md +39 -0
  119. package/src/config/bundled-skills/contacts/TOOLS.json +122 -0
  120. package/src/config/bundled-skills/contacts/tools/contact-merge.ts +9 -0
  121. package/src/config/bundled-skills/contacts/tools/contact-search.ts +9 -0
  122. package/src/config/bundled-skills/contacts/tools/contact-upsert.ts +9 -0
  123. package/src/config/bundled-skills/document/SKILL.md +18 -0
  124. package/src/config/bundled-skills/document/TOOLS.json +53 -0
  125. package/src/config/bundled-skills/document/tools/document-create.ts +9 -0
  126. package/src/config/bundled-skills/document/tools/document-update.ts +9 -0
  127. package/src/config/bundled-skills/doordash/SKILL.md +163 -0
  128. package/src/config/bundled-skills/followups/SKILL.md +32 -0
  129. package/src/config/bundled-skills/followups/TOOLS.json +100 -0
  130. package/src/config/bundled-skills/followups/tools/followup-create.ts +9 -0
  131. package/src/config/bundled-skills/followups/tools/followup-list.ts +9 -0
  132. package/src/config/bundled-skills/followups/tools/followup-resolve.ts +9 -0
  133. package/src/config/bundled-skills/image-studio/TOOLS.json +2 -2
  134. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +2 -24
  135. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +2 -1
  136. package/src/config/bundled-skills/playbooks/SKILL.md +31 -0
  137. package/src/config/bundled-skills/playbooks/TOOLS.json +126 -0
  138. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +9 -0
  139. package/src/config/bundled-skills/playbooks/tools/playbook-delete.ts +9 -0
  140. package/src/config/bundled-skills/playbooks/tools/playbook-list.ts +9 -0
  141. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +9 -0
  142. package/src/config/bundled-skills/reminder/SKILL.md +20 -0
  143. package/src/config/bundled-skills/reminder/TOOLS.json +67 -0
  144. package/src/config/bundled-skills/reminder/tools/reminder-cancel.ts +9 -0
  145. package/src/config/bundled-skills/reminder/tools/reminder-create.ts +9 -0
  146. package/src/config/bundled-skills/reminder/tools/reminder-list.ts +9 -0
  147. package/src/config/bundled-skills/schedule/SKILL.md +74 -0
  148. package/src/config/bundled-skills/schedule/TOOLS.json +135 -0
  149. package/src/config/bundled-skills/schedule/tools/schedule-create.ts +9 -0
  150. package/src/config/bundled-skills/schedule/tools/schedule-delete.ts +9 -0
  151. package/src/config/bundled-skills/schedule/tools/schedule-list.ts +9 -0
  152. package/src/config/bundled-skills/schedule/tools/schedule-update.ts +9 -0
  153. package/src/config/bundled-skills/subagent/SKILL.md +25 -0
  154. package/src/config/bundled-skills/subagent/TOOLS.json +107 -0
  155. package/src/config/bundled-skills/subagent/tools/subagent-abort.ts +9 -0
  156. package/src/config/bundled-skills/subagent/tools/subagent-message.ts +9 -0
  157. package/src/config/bundled-skills/subagent/tools/subagent-read.ts +9 -0
  158. package/src/config/bundled-skills/subagent/tools/subagent-spawn.ts +9 -0
  159. package/src/config/bundled-skills/subagent/tools/subagent-status.ts +9 -0
  160. package/src/config/bundled-skills/tasks/SKILL.md +28 -0
  161. package/src/config/bundled-skills/tasks/TOOLS.json +256 -0
  162. package/src/config/bundled-skills/tasks/tools/task-delete.ts +9 -0
  163. package/src/config/bundled-skills/tasks/tools/task-list-add.ts +9 -0
  164. package/src/config/bundled-skills/tasks/tools/task-list-remove.ts +9 -0
  165. package/src/config/bundled-skills/tasks/tools/task-list-show.ts +9 -0
  166. package/src/config/bundled-skills/tasks/tools/task-list-update.ts +9 -0
  167. package/src/config/bundled-skills/tasks/tools/task-list.ts +9 -0
  168. package/src/config/bundled-skills/tasks/tools/task-run.ts +9 -0
  169. package/src/config/bundled-skills/tasks/tools/task-save.ts +9 -0
  170. package/src/config/bundled-skills/twitter/SKILL.md +134 -0
  171. package/src/config/bundled-skills/watcher/SKILL.md +27 -0
  172. package/src/config/bundled-skills/watcher/TOOLS.json +147 -0
  173. package/src/config/bundled-skills/watcher/tools/watcher-create.ts +9 -0
  174. package/src/config/bundled-skills/watcher/tools/watcher-delete.ts +9 -0
  175. package/src/config/bundled-skills/watcher/tools/watcher-digest.ts +9 -0
  176. package/src/config/bundled-skills/watcher/tools/watcher-list.ts +9 -0
  177. package/src/config/bundled-skills/watcher/tools/watcher-update.ts +9 -0
  178. package/src/config/defaults.ts +44 -0
  179. package/src/config/loader.ts +4 -1
  180. package/src/config/schema.ts +218 -1
  181. package/src/config/system-prompt.ts +100 -6
  182. package/src/config/templates/IDENTITY.md +7 -0
  183. package/src/config/types.ts +5 -0
  184. package/src/contacts/contact-store.ts +4 -4
  185. package/src/daemon/assistant-attachments.ts +10 -0
  186. package/src/daemon/classifier.ts +3 -1
  187. package/src/daemon/computer-use-session.ts +3 -1
  188. package/src/daemon/date-context.ts +136 -0
  189. package/src/daemon/handlers/apps.ts +16 -1
  190. package/src/daemon/handlers/browser.ts +54 -0
  191. package/src/daemon/handlers/computer-use.ts +7 -1
  192. package/src/daemon/handlers/config.ts +192 -4
  193. package/src/daemon/handlers/diagnostics.ts +5 -1
  194. package/src/daemon/handlers/documents.ts +18 -29
  195. package/src/daemon/handlers/home-base.ts +5 -1
  196. package/src/daemon/handlers/index.ts +40 -271
  197. package/src/daemon/handlers/misc.ts +9 -1
  198. package/src/daemon/handlers/publish.ts +6 -1
  199. package/src/daemon/handlers/sessions.ts +65 -12
  200. package/src/daemon/handlers/shared.ts +36 -1
  201. package/src/daemon/handlers/signing.ts +37 -0
  202. package/src/daemon/handlers/skills.ts +20 -6
  203. package/src/daemon/handlers/subagents.ts +8 -3
  204. package/src/daemon/handlers/twitter-auth.ts +169 -0
  205. package/src/daemon/handlers/work-items.ts +495 -39
  206. package/src/daemon/ipc-contract-inventory.json +40 -4
  207. package/src/daemon/ipc-contract.ts +185 -37
  208. package/src/daemon/ipc-protocol.ts +7 -2
  209. package/src/daemon/lifecycle.ts +48 -5
  210. package/src/daemon/main.ts +10 -4
  211. package/src/daemon/ride-shotgun-handler.ts +74 -10
  212. package/src/daemon/server.ts +144 -29
  213. package/src/daemon/session-agent-loop.ts +887 -0
  214. package/src/daemon/session-attachments.ts +28 -5
  215. package/src/daemon/session-error.ts +24 -3
  216. package/src/daemon/session-lifecycle.ts +147 -0
  217. package/src/daemon/session-media-retry.ts +147 -0
  218. package/src/daemon/session-messaging.ts +145 -0
  219. package/src/daemon/session-notifiers.ts +164 -0
  220. package/src/daemon/session-process.ts +2 -2
  221. package/src/daemon/session-queue-manager.ts +1 -0
  222. package/src/daemon/session-runtime-assembly.ts +52 -0
  223. package/src/daemon/session-skill-tools.ts +124 -5
  224. package/src/daemon/session-slash.ts +3 -0
  225. package/src/daemon/session-surfaces.ts +77 -2
  226. package/src/daemon/session-tool-setup.ts +222 -2
  227. package/src/daemon/session-usage.ts +0 -2
  228. package/src/daemon/session.ts +114 -1365
  229. package/src/daemon/video-thumbnail.ts +60 -0
  230. package/src/doordash/client.ts +121 -27
  231. package/src/doordash/queries.ts +1 -2
  232. package/src/export/formatter.ts +3 -1
  233. package/src/followups/followup-store.ts +4 -2
  234. package/src/followups/types.ts +6 -0
  235. package/src/hooks/templates.ts +1 -1
  236. package/src/index.ts +32 -1151
  237. package/src/media/gemini-image-service.ts +1 -1
  238. package/src/memory/attachments-store.ts +28 -83
  239. package/src/memory/channel-delivery-store.ts +7 -21
  240. package/src/memory/clarification-resolver.ts +6 -5
  241. package/src/memory/contradiction-checker.ts +3 -2
  242. package/src/memory/conversation-key-store.ts +10 -29
  243. package/src/memory/conversation-store.ts +2 -1
  244. package/src/memory/db.ts +362 -2
  245. package/src/memory/entity-extractor.ts +6 -3
  246. package/src/memory/items-extractor.ts +5 -4
  247. package/src/memory/jobs-store.ts +3 -2
  248. package/src/memory/llm-usage-store.ts +1 -2
  249. package/src/memory/runs-store.ts +1 -2
  250. package/src/memory/schema.ts +65 -2
  251. package/src/messaging/style-analyzer.ts +3 -2
  252. package/src/messaging/thread-summarizer.ts +8 -12
  253. package/src/messaging/triage-engine.ts +4 -2
  254. package/src/providers/openrouter/client.ts +20 -0
  255. package/src/providers/registry.ts +8 -0
  256. package/src/runtime/http-server.ts +277 -25
  257. package/src/runtime/http-types.ts +0 -2
  258. package/src/runtime/routes/attachment-routes.ts +5 -6
  259. package/src/runtime/routes/call-routes.ts +140 -0
  260. package/src/runtime/routes/channel-routes.ts +12 -19
  261. package/src/runtime/routes/conversation-routes.ts +5 -9
  262. package/src/runtime/routes/run-routes.ts +4 -8
  263. package/src/runtime/run-orchestrator.ts +39 -6
  264. package/src/schedule/recurrence-engine.ts +138 -0
  265. package/src/schedule/recurrence-types.ts +67 -0
  266. package/src/schedule/schedule-store.ts +102 -57
  267. package/src/schedule/scheduler.ts +9 -6
  268. package/src/security/oauth2.ts +29 -4
  269. package/src/security/secret-allowlist.ts +46 -0
  270. package/src/skills/clawhub.ts +1 -1
  271. package/src/subagent/manager.ts +40 -8
  272. package/src/swarm/backend-claude-code.ts +64 -9
  273. package/src/swarm/worker-prompts.ts +2 -1
  274. package/src/tasks/SPEC.md +34 -28
  275. package/src/tasks/ephemeral-permissions.ts +16 -7
  276. package/src/tasks/task-compiler.ts +5 -4
  277. package/src/tasks/task-runner.ts +10 -5
  278. package/src/tasks/task-scheduler.ts +1 -1
  279. package/src/tasks/tool-sanitizer.ts +36 -0
  280. package/src/tools/assets/search.ts +4 -4
  281. package/src/tools/browser/api-map.ts +220 -0
  282. package/src/tools/browser/auto-navigate.ts +270 -0
  283. package/src/tools/browser/browser-execution.ts +2 -1
  284. package/src/tools/browser/browser-manager.ts +2 -2
  285. package/src/tools/browser/network-recorder.ts +5 -4
  286. package/src/tools/browser/x-auto-navigate.ts +207 -0
  287. package/src/tools/calls/call-end.ts +67 -0
  288. package/src/tools/calls/call-start.ts +73 -0
  289. package/src/tools/calls/call-status.ts +81 -0
  290. package/src/tools/claude-code/claude-code.ts +77 -11
  291. package/src/tools/contacts/contact-merge.ts +46 -78
  292. package/src/tools/contacts/contact-search.ts +35 -79
  293. package/src/tools/contacts/contact-upsert.ts +35 -108
  294. package/src/tools/credentials/vault.ts +21 -5
  295. package/src/tools/document/document-tool.ts +71 -144
  296. package/src/tools/executor.ts +129 -10
  297. package/src/tools/followups/followup_create.ts +46 -88
  298. package/src/tools/followups/followup_list.ts +34 -74
  299. package/src/tools/followups/followup_resolve.ts +31 -66
  300. package/src/tools/host-terminal/cli-discover.ts +2 -1
  301. package/src/tools/host-terminal/host-shell.ts +10 -0
  302. package/src/tools/memory/handlers.ts +5 -4
  303. package/src/tools/network/__tests__/web-search.test.ts +427 -0
  304. package/src/tools/network/script-proxy/__tests__/logging.test.ts +248 -0
  305. package/src/tools/network/script-proxy/__tests__/policy.test.ts +234 -0
  306. package/src/tools/network/script-proxy/__tests__/router.test.ts +76 -0
  307. package/src/tools/network/web-fetch.ts +18 -6
  308. package/src/tools/playbooks/index.ts +4 -5
  309. package/src/tools/playbooks/playbook-create.ts +3 -47
  310. package/src/tools/playbooks/playbook-delete.ts +1 -25
  311. package/src/tools/playbooks/playbook-list.ts +1 -28
  312. package/src/tools/playbooks/playbook-update.ts +3 -51
  313. package/src/tools/registry.ts +2 -4
  314. package/src/tools/reminder/reminder.ts +5 -78
  315. package/src/tools/schedule/create.ts +69 -74
  316. package/src/tools/schedule/delete.ts +21 -47
  317. package/src/tools/schedule/list.ts +55 -74
  318. package/src/tools/schedule/update.ts +77 -84
  319. package/src/tools/subagent/abort.ts +29 -58
  320. package/src/tools/subagent/message.ts +30 -63
  321. package/src/tools/subagent/read.ts +53 -84
  322. package/src/tools/subagent/spawn.ts +43 -82
  323. package/src/tools/subagent/status.ts +42 -71
  324. package/src/tools/swarm/delegate.ts +2 -1
  325. package/src/tools/tasks/index.ts +8 -6
  326. package/src/tools/tasks/task-delete.ts +69 -56
  327. package/src/tools/tasks/task-list.ts +31 -52
  328. package/src/tools/tasks/task-run.ts +74 -102
  329. package/src/tools/tasks/task-save.ts +33 -65
  330. package/src/tools/tasks/work-item-enqueue.ts +192 -134
  331. package/src/tools/tasks/work-item-list.ts +33 -78
  332. package/src/tools/tasks/work-item-remove.ts +60 -0
  333. package/src/tools/tasks/work-item-update.ts +114 -0
  334. package/src/tools/terminal/backends/native.ts +3 -1
  335. package/src/tools/tool-manifest.ts +20 -74
  336. package/src/tools/types.ts +6 -0
  337. package/src/tools/ui-surface/definitions.ts +6 -1
  338. package/src/tools/watch/screen-watch.ts +3 -1
  339. package/src/tools/watcher/create.ts +52 -98
  340. package/src/tools/watcher/delete.ts +20 -46
  341. package/src/tools/watcher/digest.ts +36 -70
  342. package/src/tools/watcher/list.ts +49 -79
  343. package/src/tools/watcher/update.ts +45 -91
  344. package/src/twitter/client.ts +690 -0
  345. package/src/twitter/session.ts +91 -0
  346. package/src/usage/types.ts +0 -1
  347. package/src/util/truncate.ts +6 -0
  348. package/src/watcher/providers/slack.ts +2 -1
  349. package/src/watcher/watcher-store.ts +3 -2
  350. package/src/work-items/work-item-store.ts +236 -2
  351. package/src/workspace/commit-message-enrichment-service.ts +284 -0
  352. package/src/workspace/commit-message-provider.ts +95 -0
  353. package/src/workspace/git-service.ts +272 -52
  354. package/src/workspace/heartbeat-service.ts +70 -13
  355. package/src/workspace/provider-commit-message-generator.ts +242 -0
  356. package/src/workspace/turn-commit.ts +100 -51
  357. package/src/tools/contacts/index.ts +0 -4
  358. package/src/tools/document/index.ts +0 -5
  359. package/src/tools/followups/index.ts +0 -3
  360. package/src/tools/subagent/index.ts +0 -5
  361. /package/src/__tests__/{memory-context-benchmark.test.ts → memory-context-benchmark.benchmark.test.ts} +0 -0
@@ -200,30 +200,75 @@ function buildTaskScheduleReminderRoutingSection(): string {
200
200
  '',
201
201
  'These three systems serve different purposes. Choose the right one based on user intent:',
202
202
  '',
203
- '### Task Queue (task_list_add / task_list_show)',
203
+ '### Task Queue (task_list_add / task_list_show / task_list_update / task_list_remove)',
204
204
  'For tracking things the user wants to do or remember. Use when the user says:',
205
205
  '- "Add to my tasks", "add to my queue", "put this on my task list"',
206
206
  '- "Track this", "I need to do X", "queue this up"',
207
207
  '- Any request to add a one-off item to their personal to-do list',
208
208
  '',
209
+ 'To modify an existing task, use `task_list_update`:',
210
+ '- "Bump the priority on X", "make X high priority", "move this up"',
211
+ '- "Change the status of X", "mark X as done"',
212
+ '- "Update the notes on X"',
213
+ 'Do NOT use `task_list_add` for updates — it will detect duplicates and suggest using `task_list_update` instead.',
214
+ '',
215
+ 'To remove a task from the queue, use `task_list_remove`:',
216
+ '- "Remove X from my tasks", "delete that task", "clean up the duplicate"',
217
+ '- "Take this off the list", "drop this task"',
218
+ '',
209
219
  'You can create ad-hoc work items by providing just a `title` to `task_list_add` — no existing task template is needed. A lightweight template is auto-created behind the scenes. For reusable task definitions with templates and input schemas, use `task_save` first.',
210
220
  '',
221
+ '**IMPORTANT:** When you call `task_list_show`, the Tasks window opens automatically on the client. Do NOT also create a separate surface/UI (via `ui_show` or `app_create`) to display the task queue. Doing so causes duplicate Task Queue windows. Just call `task_list_show` and let the native window handle the presentation.',
222
+ '',
211
223
  '### Schedules (schedule_create / schedule_list / schedule_update / schedule_delete)',
212
- 'For recurring automated jobs that run on a cron schedule. Use ONLY when the user explicitly wants:',
224
+ 'For recurring automated jobs that run on a recurrence schedule (cron or RRULE). Use ONLY when the user explicitly wants:',
213
225
  '- Recurring automation: "every day at 9am", "weekly on Mondays", "every hour"',
226
+ '- Complex recurrence patterns: "every other Tuesday", "last weekday of the month" (use RRULE)',
227
+ '- Bounded recurrence: "every day for 30 days", "weekly until March" (RRULE with COUNT or UNTIL)',
214
228
  '- Periodic background tasks: "check my email every morning", "run this report weekly"',
215
229
  '',
216
- '### Reminders (reminder)',
230
+ '#### RRULE Set Constructs',
231
+ 'When building RRULE expressions, these set lines are supported:',
232
+ '- **RRULE** — one or more recurrence rules (multiple RRULE lines form a union of occurrences)',
233
+ '- **RDATE** — add one-off dates that are not covered by the RRULE pattern',
234
+ '- **EXDATE** — exclude specific dates from the recurrence set',
235
+ '- **EXRULE** — exclude an entire series of dates defined by a recurrence pattern',
236
+ '',
237
+ 'Exclusions (EXDATE, EXRULE) take precedence over inclusions (RRULE, RDATE). All RRULE expressions must include a DTSTART line and at least one RRULE or RDATE inclusion.',
238
+ '',
239
+ '### Reminders (reminder_create / reminder_list / reminder_cancel)',
217
240
  'For one-time time-triggered notifications. Use ONLY when the user wants:',
218
241
  '- A notification at a specific future time: "remind me at 3pm", "remind me in 2 hours"',
219
242
  '- A timed alert, not a tracked task',
220
243
  '',
221
244
  '### Common mistakes to avoid',
222
- '- "Add this to my tasks" → task_list_add (NOT schedule_create or reminder)',
245
+ '- "Add this to my tasks" → task_list_add (NOT schedule_create or reminder_create)',
223
246
  '- "What\'s on my task list?" → task_list_show (NOT schedule_list)',
224
247
  '- "Remind me to buy groceries" without a time → task_list_add (it\'s a task, not a timed reminder)',
225
- '- "Remind me at 5pm to buy groceries" → reminder (explicit time trigger)',
226
- '- "Check my inbox every morning at 8am" → schedule_create (recurring automation)',
248
+ '- "Remind me at 5pm to buy groceries" → reminder_create (explicit time trigger)',
249
+ '- "Check my inbox every morning at 8am" → schedule_create (recurring automation, cron)',
250
+ '- "Every other Tuesday at 10am" → schedule_create (recurring automation, RRULE)',
251
+ '- "Every weekday except holidays" → schedule_create (RRULE with EXDATE for exclusions)',
252
+ '- "Daily for the next 30 days" → schedule_create (RRULE with COUNT=30)',
253
+ '- "Bump priority on X" → task_list_update (NOT task_list_add)',
254
+ '- "Move this up" / "change this task priority" → task_list_update (NOT task_list_add)',
255
+ '- "Mark X as done" → task_list_update (NOT task_list_add)',
256
+ '- "Remove X from my tasks" → task_list_remove (NOT task_list_update)',
257
+ '- "Delete that task" / "clean up the duplicate" → task_list_remove',
258
+ '',
259
+ '### Entity type routing: work items vs task templates',
260
+ '',
261
+ 'There are two entity types with separate ID spaces:',
262
+ '- **Work items** (the user\'s task queue) — managed by task_list_add, task_list_show, task_list_update, task_list_remove',
263
+ '- **Task templates** (reusable definitions) — managed by task_save, task_list, task_run, task_delete',
264
+ '',
265
+ 'Do NOT pass a work item ID to a task template tool or vice versa:',
266
+ '- Deleting a work item from the queue → task_list_remove (NOT task_delete)',
267
+ '- Deleting a task template → task_delete (NOT task_list_remove)',
268
+ '- Running a task template → task_run with task_id (NOT a work item ID)',
269
+ '- Updating a work item → task_list_update with work_item_id (NOT a task template ID)',
270
+ '',
271
+ 'If an error says "entity mismatch", read the corrective action and selector fields it provides to pick the right tool.',
227
272
  ].join('\n');
228
273
  }
229
274
 
@@ -277,6 +322,7 @@ function buildDynamicUiSection(): string {
277
322
  '- **Tool auto-emissions** (e.g. `get_weather`): handled automatically — do nothing extra',
278
323
  '- **Predefined domain data** (flights, stocks): `ui_show` with `surface_type: "dynamic_page"` and domain component classes',
279
324
  '- **Simple structured data** (key-value, table, list): `ui_show` with `card`/`table`/`list`/`form` surface_type',
325
+ '- **Multi-step tasks** (ordering food, booking, purchasing, multi-phase workflows): `ui_show` with `card` surface_type + `data.template: "task_progress"` (see below)',
280
326
  '- **Interactive apps only**: `app_create` (calculators, dashboards, tools - NOT text content)',
281
327
  '',
282
328
  '### Loading app tools',
@@ -379,6 +425,51 @@ function buildDynamicUiSection(): string {
379
425
  '- **Data review/selection**: Use a `table` surface with selectable rows',
380
426
  '',
381
427
  'Interactive surfaces provide a better user experience than asking your user to type their choice. Only fall back to plain text when the interaction is conversational or doesn\'t fit a structured format.',
428
+ '',
429
+ '### Task progress for multi-step workflows',
430
+ 'When executing a multi-step task (booking a flight, purchasing, research with multiple phases), show live progress using the `task_progress` card template. Note: DoorDash ordering auto-emits a task_progress card on the first `vellum doordash` command (surface ID `doordash-progress`), so you only need `ui_update` for DoorDash flows.',
431
+ '',
432
+ '1. **Before starting**, call `ui_show` with `surface_type: "card"` and put `template: "task_progress"` inside `data`:',
433
+ ' ```json',
434
+ ' {',
435
+ ' "surface_type": "card",',
436
+ ' "data": {',
437
+ ' "title": "Booking Flight",',
438
+ ' "body": "",',
439
+ ' "template": "task_progress",',
440
+ ' "templateData": {',
441
+ ' "title": "Booking Flight",',
442
+ ' "status": "in_progress",',
443
+ ' "steps": [',
444
+ ' { "label": "Search flights", "status": "in_progress" },',
445
+ ' { "label": "Compare options", "status": "pending" },',
446
+ ' { "label": "Select flight", "status": "pending" },',
447
+ ' { "label": "Complete booking", "status": "pending" }',
448
+ ' ]',
449
+ ' }',
450
+ ' }',
451
+ ' }',
452
+ ' ```',
453
+ '2. **As each step completes**, call `ui_update` with the same surface ID and patch `data.templateData` (not top-level `status`/`steps`):',
454
+ ' ```json',
455
+ ' {',
456
+ ' "surface_id": "<surface-id>",',
457
+ ' "data": {',
458
+ ' "templateData": {',
459
+ ' "status": "in_progress",',
460
+ ' "steps": [',
461
+ ' { "label": "Search flights", "status": "completed", "detail": "Found 12 options" },',
462
+ ' { "label": "Compare options", "status": "in_progress" },',
463
+ ' { "label": "Select flight", "status": "pending" },',
464
+ ' { "label": "Complete booking", "status": "pending" }',
465
+ ' ]',
466
+ ' }',
467
+ ' }',
468
+ ' }',
469
+ ' ```',
470
+ '3. **On completion**, set `data.templateData.status` to `"completed"`. On failure, set it to `"failed"` and mark the failing step accordingly.',
471
+ '',
472
+ 'Use this for ANY multi-step workflow where the user benefits from seeing structured progress instead of just "Running a command...".',
382
473
  ].join('\n');
383
474
  }
384
475
 
@@ -891,6 +982,9 @@ function buildDynamicSkillWorkflowSection(): string {
891
982
  '',
892
983
  '### Browser Skill Prerequisite',
893
984
  'If you need browser capabilities (navigating web pages, clicking elements, extracting content) and `browser_*` tools are not available, load the "browser" skill first using `skill_load`.',
985
+ '',
986
+ '### X (Twitter) Skill',
987
+ 'When the user asks to post, reply, or interact with X/Twitter, load the "twitter" skill using `skill_load`. Do NOT use computer-use or the browser skill for X — the X skill provides CLI commands (`vellum x post`, `vellum x reply`) that are faster and more reliable.',
894
988
  ].join('\n');
895
989
  }
896
990
 
@@ -14,5 +14,12 @@ _ This file defines who you are. Fill it in during your first conversation. Make
14
14
  - **Emoji:** [Choose one that matches your personality.]
15
15
  - **Style tendency:** [Will be filled in by the evolution system based on personality]
16
16
  - **Role:** Personal AI assistant
17
+ - **Home:** Local (~/.vellum/workspace)
18
+
19
+ _ Home describes where this assistant lives. Format examples:
20
+ _ Local (path): Local (~/.vellum/workspace)
21
+ _ GCP: GCP (project: my-project, zone: us-central1-a, instance: vellum-abc)
22
+ _ AWS: AWS (project: my-project, region: us-east-1, instance: vellum-abc)
23
+ _ Custom: Custom (ip: 192.168.1.100, port: 8080)
17
24
 
18
25
  The user can change their emoji at any time — just update this file when they ask.
@@ -27,6 +27,11 @@ export type {
27
27
  SkillEntryConfig,
28
28
  SkillsLoadConfig,
29
29
  SkillsInstallConfig,
30
+ AgentHeartbeatConfig,
30
31
  SwarmConfig,
31
32
  SkillsConfig,
33
+ WorkspaceGitConfig,
34
+ CallsConfig,
35
+ CallsDisclosureConfig,
36
+ CallsSafetyConfig,
32
37
  } from './schema.js';
@@ -70,7 +70,7 @@ export function upsertContact(params: {
70
70
  responseExpectation?: string | null;
71
71
  preferredTone?: string | null;
72
72
  channels?: Array<{ type: string; address: string; isPrimary?: boolean }>;
73
- }): ContactWithChannels {
73
+ }): ContactWithChannels & { created: boolean } {
74
74
  const db = getDb();
75
75
  const now = Date.now();
76
76
 
@@ -96,7 +96,7 @@ export function upsertContact(params: {
96
96
  syncChannels(contactId, params.channels, now);
97
97
  }
98
98
 
99
- return getContact(contactId)!;
99
+ return { ...getContact(contactId)!, created: false };
100
100
  }
101
101
  }
102
102
 
@@ -124,7 +124,7 @@ export function upsertContact(params: {
124
124
  .run();
125
125
 
126
126
  syncChannels(contactId, params.channels, now);
127
- return getContact(contactId)!;
127
+ return { ...getContact(contactId)!, created: false };
128
128
  }
129
129
  }
130
130
  }
@@ -148,7 +148,7 @@ export function upsertContact(params: {
148
148
  syncChannels(contactId, params.channels, now);
149
149
  }
150
150
 
151
- return getContact(contactId)!;
151
+ return { ...getContact(contactId)!, created: true };
152
152
  }
153
153
 
154
154
  /**
@@ -313,6 +313,16 @@ export function drainDirectiveDisplayBuffer(buffer: string): DirectiveDisplayDra
313
313
 
314
314
  if (!isValidDirective) {
315
315
  emitText += tag;
316
+ } else {
317
+ // Only trim the trailing newline when the directive occupied its own
318
+ // line (preceded by \n and followed by \n or \r\n). We intentionally
319
+ // do NOT trim when nextChar is undefined (end-of-buffer) because in
320
+ // streaming mode more data may arrive in the next chunk — eagerly
321
+ // trimming would merge words across the directive boundary.
322
+ const nextChar = buffer[end + 2];
323
+ if (emitText.endsWith('\n') && (nextChar === '\n' || nextChar === '\r')) {
324
+ emitText = emitText.slice(0, -1);
325
+ }
316
326
  }
317
327
 
318
328
  cursor = end + 2;
@@ -4,6 +4,8 @@ import { getLogger } from '../util/logger.js';
4
4
 
5
5
  const log = getLogger('classifier');
6
6
 
7
+ const CLASSIFICATION_TIMEOUT_MS = 5000;
8
+
7
9
  export type InteractionType = 'computer_use' | 'text_qa';
8
10
 
9
11
  /**
@@ -53,7 +55,7 @@ export async function classifyInteraction(task: string, source?: 'voice' | 'text
53
55
  messages: [{ role: 'user' as const, content: task }],
54
56
  }),
55
57
  new Promise<never>((_, reject) =>
56
- setTimeout(() => reject(new Error('Classification timeout')), 5000),
58
+ setTimeout(() => reject(new Error('Classification timeout')), CLASSIFICATION_TIMEOUT_MS),
57
59
  ),
58
60
  ]);
59
61
 
@@ -21,7 +21,7 @@ import { registerSkillTools } from '../tools/registry.js';
21
21
  import { buildComputerUseSystemPrompt } from '../config/computer-use-prompt.js';
22
22
  import { getSandboxWorkingDir } from '../util/platform.js';
23
23
  import { getConfig } from '../config/loader.js';
24
- import { projectSkillTools, resetSkillToolProjection } from './session-skill-tools.js';
24
+ import { projectSkillTools, resetSkillToolProjection, type SkillProjectionCache } from './session-skill-tools.js';
25
25
  import { getLogger } from '../util/logger.js';
26
26
 
27
27
  const log = getLogger('computer-use-session');
@@ -60,6 +60,7 @@ export class ComputerUseSession {
60
60
  private readonly onTerminal?: (sessionId: string) => void;
61
61
  private readonly preactivatedSkillIds: string[];
62
62
  private readonly skillProjectionState = new Map<string, string>();
63
+ private readonly skillProjectionCache: SkillProjectionCache = {};
63
64
 
64
65
  private state: SessionState = 'idle';
65
66
  private stepCount = 0;
@@ -235,6 +236,7 @@ export class ComputerUseSession {
235
236
  const projection = projectSkillTools([], {
236
237
  preactivatedSkillIds: this.preactivatedSkillIds,
237
238
  previouslyActiveSkillIds: this.skillProjectionState,
239
+ cache: this.skillProjectionCache,
238
240
  });
239
241
 
240
242
  if (projection.toolDefinitions.length === 0) {
@@ -0,0 +1,136 @@
1
+ /**
2
+ * Temporal context formatter for future weekday/weekend grounding.
3
+ *
4
+ * Produces a compact, deterministic payload describing the current date,
5
+ * upcoming weekend/work-week windows, and a short horizon of labelled
6
+ * future dates. Intended for runtime injection into the model context.
7
+ */
8
+
9
+ export interface TemporalContextOptions {
10
+ /** Override current time (epoch ms) for deterministic tests. */
11
+ nowMs?: number;
12
+ /** IANA timezone (e.g. "America/New_York"). Defaults to host timezone. */
13
+ timeZone?: string;
14
+ /** Number of future days to list (default 14, hard-capped at 14). */
15
+ horizonDays?: number;
16
+ }
17
+
18
+ const MAX_OUTPUT_CHARS = 1500;
19
+ const MAX_HORIZON_ENTRIES = 14;
20
+
21
+ const WEEKDAY_NAMES = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'] as const;
22
+
23
+ /**
24
+ * Get the local date parts for a given instant in the specified timezone.
25
+ */
26
+ function localDateParts(date: Date, timeZone: string): { year: number; month: number; day: number; weekday: number } {
27
+ const fmt = new Intl.DateTimeFormat('en-US', {
28
+ timeZone,
29
+ year: 'numeric',
30
+ month: '2-digit',
31
+ day: '2-digit',
32
+ weekday: 'short',
33
+ });
34
+ const parts = fmt.formatToParts(date);
35
+ const get = (t: string) => parts.find((p) => p.type === t)?.value ?? '';
36
+ // Weekday as 0-6 (Sun-Sat)
37
+ const weekdayShort = get('weekday');
38
+ const weekdayMap: Record<string, number> = { Sun: 0, Mon: 1, Tue: 2, Wed: 3, Thu: 4, Fri: 5, Sat: 6 };
39
+ return {
40
+ year: parseInt(get('year'), 10),
41
+ month: parseInt(get('month'), 10),
42
+ day: parseInt(get('day'), 10),
43
+ weekday: weekdayMap[weekdayShort] ?? 0,
44
+ };
45
+ }
46
+
47
+ /**
48
+ * Format a Date as YYYY-MM-DD in the given timezone.
49
+ */
50
+ function formatLocalDate(date: Date, timeZone: string): string {
51
+ const p = localDateParts(date, timeZone);
52
+ return `${p.year}-${String(p.month).padStart(2, '0')}-${String(p.day).padStart(2, '0')}`;
53
+ }
54
+
55
+ /**
56
+ * Advance a date by `days` calendar days in the given timezone.
57
+ *
58
+ * Computes the local date, adds days to the day component, then anchors
59
+ * the result at noon local time to avoid DST-transition edge cases.
60
+ */
61
+ function addDays(date: Date, days: number, timeZone: string): Date {
62
+ const parts = localDateParts(date, timeZone);
63
+ // Use Date.UTC for calendar overflow (e.g. Jan 32 → Feb 1).
64
+ const ref = new Date(Date.UTC(parts.year, parts.month - 1, parts.day + days));
65
+ const tY = ref.getUTCFullYear();
66
+ const tM = ref.getUTCMonth() + 1;
67
+ const tD = ref.getUTCDate();
68
+ // Noon UTC covers UTC-12 through ~UTC+11. For far-east timezones
69
+ // (UTC+12/+13/+14) noon UTC is already the next local day, so fall
70
+ // back to midnight UTC which resolves correctly there.
71
+ const noonUTC = new Date(Date.UTC(tY, tM - 1, tD, 12, 0, 0));
72
+ const r = localDateParts(noonUTC, timeZone);
73
+ if (r.year === tY && r.month === tM && r.day === tD) {
74
+ return noonUTC;
75
+ }
76
+ return new Date(Date.UTC(tY, tM - 1, tD, 0, 0, 0));
77
+ }
78
+
79
+ /**
80
+ * Build a compact temporal context string for model injection.
81
+ *
82
+ * Output is hard-capped at {@link MAX_OUTPUT_CHARS} characters and
83
+ * {@link MAX_HORIZON_ENTRIES} horizon entries.
84
+ */
85
+ export function buildTemporalContext(options: TemporalContextOptions = {}): string {
86
+ const now = new Date(options.nowMs ?? Date.now());
87
+ const timeZone = options.timeZone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
88
+ const horizonDays = Math.min(options.horizonDays ?? MAX_HORIZON_ENTRIES, MAX_HORIZON_ENTRIES);
89
+
90
+ const todayParts = localDateParts(now, timeZone);
91
+ const todayStr = formatLocalDate(now, timeZone);
92
+ const todayWeekday = WEEKDAY_NAMES[todayParts.weekday];
93
+
94
+ // ── Next weekend (Saturday-Sunday) ──
95
+ const daysUntilSaturday = (6 - todayParts.weekday + 7) % 7 || 7;
96
+ const nextSaturday = addDays(now, daysUntilSaturday, timeZone);
97
+ const nextSunday = addDays(now, daysUntilSaturday + 1, timeZone);
98
+
99
+ // ── Next work week (Monday-Friday) ──
100
+ const daysUntilMonday = (1 - todayParts.weekday + 7) % 7 || 7;
101
+ const nextMonday = addDays(now, daysUntilMonday, timeZone);
102
+ const nextFriday = addDays(now, daysUntilMonday + 4, timeZone);
103
+
104
+ // ── Horizon list ──
105
+ const horizonLines: string[] = [];
106
+ for (let i = 1; i <= horizonDays; i++) {
107
+ const futureDate = addDays(now, i, timeZone);
108
+ const futureParts = localDateParts(futureDate, timeZone);
109
+ const label = WEEKDAY_NAMES[futureParts.weekday];
110
+ horizonLines.push(` ${formatLocalDate(futureDate, timeZone)} ${label}`);
111
+ }
112
+
113
+ const lines = [
114
+ `<temporal_context>`,
115
+ `Today: ${todayStr} (${todayWeekday})`,
116
+ `Timezone: ${timeZone}`,
117
+ ``,
118
+ `Week definitions: work week = Monday–Friday, weekend = Saturday–Sunday`,
119
+ ``,
120
+ `Next weekend: ${formatLocalDate(nextSaturday, timeZone)} – ${formatLocalDate(nextSunday, timeZone)}`,
121
+ `Next work week: ${formatLocalDate(nextMonday, timeZone)} – ${formatLocalDate(nextFriday, timeZone)}`,
122
+ ``,
123
+ `Upcoming dates:`,
124
+ ...horizonLines,
125
+ `</temporal_context>`,
126
+ ];
127
+
128
+ let output = lines.join('\n');
129
+
130
+ // Hard cap: truncate if somehow over budget (shouldn't happen with 14 entries).
131
+ if (output.length > MAX_OUTPUT_CHARS) {
132
+ output = output.slice(0, MAX_OUTPUT_CHARS - 25) + '\n</temporal_context>';
133
+ }
134
+
135
+ return output;
136
+ }
@@ -18,7 +18,7 @@ import type {
18
18
  AppUpdatePreviewRequest,
19
19
  UiSurfaceShow,
20
20
  } from '../ipc-protocol.js';
21
- import { log, compareSemver, createSigningCallback, type HandlerContext } from './shared.js';
21
+ import { log, compareSemver, createSigningCallback, defineHandlers, type HandlerContext } from './shared.js';
22
22
 
23
23
  export function handleAppDataRequest(
24
24
  msg: AppDataRequest,
@@ -444,3 +444,18 @@ export function handleGalleryInstall(
444
444
  });
445
445
  }
446
446
  }
447
+
448
+ export const appHandlers = defineHandlers({
449
+ app_data_request: handleAppDataRequest,
450
+ app_open_request: handleAppOpenRequest,
451
+ app_update_preview: handleAppUpdatePreview,
452
+ app_preview_request: handleAppPreview,
453
+ apps_list: (_msg, socket, ctx) => handleAppsList(socket, ctx),
454
+ shared_apps_list: (_msg, socket, ctx) => handleSharedAppsList(socket, ctx),
455
+ shared_app_delete: handleSharedAppDelete,
456
+ fork_shared_app: handleForkSharedApp,
457
+ share_app_cloud: handleShareAppCloud,
458
+ bundle_app: handleBundleApp,
459
+ gallery_list: (_msg, socket, ctx) => handleGalleryList(socket, ctx),
460
+ gallery_install: handleGalleryInstall,
461
+ });
@@ -0,0 +1,54 @@
1
+ import { browserManager } from '../../tools/browser/browser-manager.js';
2
+ import { log, defineHandlers } from './shared.js';
3
+
4
+ export const browserHandlers = defineHandlers({
5
+ browser_cdp_response: (msg) => {
6
+ browserManager.resolveCDPResponse(msg.sessionId, msg.success, msg.declined);
7
+ },
8
+
9
+ browser_user_click: async (msg) => {
10
+ try {
11
+ const page = await browserManager.getOrCreateSessionPage(msg.sessionId);
12
+ const viewport = await page.evaluate('(() => ({ vw: window.innerWidth, vh: window.innerHeight }))()') as { vw: number; vh: number };
13
+ const scale = Math.min(1280 / viewport.vw, 960 / viewport.vh);
14
+ const pageX = msg.x / scale;
15
+ const pageY = msg.y / scale;
16
+ const options: Record<string, unknown> = {};
17
+ if (msg.button === 'right') options.button = 'right';
18
+ if (msg.doubleClick) options.clickCount = 2;
19
+ await page.mouse.click(pageX, pageY, options);
20
+ } catch (err) {
21
+ log.warn({ err, sessionId: msg.sessionId }, 'Failed to forward user click');
22
+ }
23
+ },
24
+
25
+ browser_user_scroll: async (msg) => {
26
+ try {
27
+ const page = await browserManager.getOrCreateSessionPage(msg.sessionId);
28
+ await page.mouse.wheel(msg.deltaX, msg.deltaY);
29
+ } catch (err) {
30
+ log.warn({ err, sessionId: msg.sessionId }, 'Failed to forward user scroll');
31
+ }
32
+ },
33
+
34
+ browser_user_keypress: async (msg) => {
35
+ try {
36
+ const page = await browserManager.getOrCreateSessionPage(msg.sessionId);
37
+ const combo = msg.modifiers?.length ? [...msg.modifiers, msg.key].join('+') : msg.key;
38
+ await page.keyboard.press(combo);
39
+ } catch (err) {
40
+ log.warn({ err, sessionId: msg.sessionId }, 'Failed to forward user keypress');
41
+ }
42
+ },
43
+
44
+ browser_interactive_mode: (msg, socket, ctx) => {
45
+ log.info({ sessionId: msg.sessionId, enabled: msg.enabled }, 'Interactive mode toggled');
46
+ browserManager.setInteractiveMode(msg.sessionId, msg.enabled);
47
+ ctx.send(socket, {
48
+ type: 'browser_interactive_mode_changed',
49
+ sessionId: msg.sessionId,
50
+ surfaceId: msg.surfaceId,
51
+ enabled: msg.enabled,
52
+ });
53
+ },
54
+ });
@@ -10,7 +10,7 @@ import type {
10
10
  CuObservation,
11
11
  ServerMessage,
12
12
  } from '../ipc-protocol.js';
13
- import { log, type HandlerContext } from './shared.js';
13
+ import { log, defineHandlers, type HandlerContext } from './shared.js';
14
14
 
15
15
  const cuObservationSequenceBySession = new Map<string, number>();
16
16
 
@@ -179,3 +179,9 @@ export async function handleCuObservation(
179
179
  log.error({ err, sessionId: msg.sessionId }, 'Error handling CU observation');
180
180
  });
181
181
  }
182
+
183
+ export const computerUseHandlers = defineHandlers({
184
+ cu_session_create: handleCuSessionCreate,
185
+ cu_session_abort: handleCuSessionAbort,
186
+ cu_observation: handleCuObservation,
187
+ });