specrails-desktop 2.29.1 → 2.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (544) hide show
  1. package/client/dist/assets/ActivityFeedPage-ClpD5CFj.js +1 -0
  2. package/client/dist/assets/AgentBrowserCapture-CSPs9Ofc.js +1 -0
  3. package/client/dist/assets/AgentModeCodePane-DwcyklA5.js +2 -0
  4. package/client/dist/assets/AgentModeJobsPane-C69bqlMa.js +1 -0
  5. package/client/dist/assets/AgentsPage-pRiRr9-W.js +86 -0
  6. package/client/dist/assets/AnalyticsPage-C14exKsC.js +1 -0
  7. package/client/dist/assets/BarChart-nBOMNfiy.js +33 -0
  8. package/client/dist/assets/CodePage-CK76JKm9.js +2 -0
  9. package/client/dist/assets/DesktopAnalyticsPage-Cnv0f3XQ.js +1 -0
  10. package/client/dist/assets/DocsDialog-CHUm_Ym0.js +11 -0
  11. package/client/dist/assets/DocsPage-COI-oDok.js +11 -0
  12. package/client/dist/assets/ExportDropdown-BrtfWzEG.js +1 -0
  13. package/client/dist/assets/IntegrationsPage-Bs00_tl4.js +1 -0
  14. package/client/dist/assets/InteractiveJobComposer-CypM7R1F.js +19 -0
  15. package/client/dist/assets/JobDetailModal-B_W0H9dw.js +1 -0
  16. package/client/dist/assets/JobDetailPage-CeeXOoKS.js +1 -0
  17. package/client/dist/assets/JobsPage-CwhJG9jw.js +1 -0
  18. package/client/dist/assets/LoopBuilderPage-C51t1WOM.js +7 -0
  19. package/client/dist/assets/LoopPreviewModal-Y2HgtIJu.js +1 -0
  20. package/client/dist/assets/LoopsPage-Bvn1Kw5V.js +1 -0
  21. package/client/dist/assets/MinimizedChatsContext-DiFvZDPW.js +1 -0
  22. package/client/dist/assets/PluginsPage-BGPRJpdf.js +2 -0
  23. package/client/dist/assets/ProjectSettingsDialog-D_GTyGLc.js +1 -0
  24. package/client/dist/assets/TemplatePreviewModal-szwBhd7h.js +1 -0
  25. package/client/dist/assets/TicketDetailModal-9UEyK974.js +14 -0
  26. package/client/dist/assets/Trans-B_vKZ-Hb.js +1 -0
  27. package/client/dist/assets/activity-DiyjsgQV.js +1 -0
  28. package/client/dist/assets/addon-image-BRmmRHgQ.js +1 -0
  29. package/client/dist/assets/addon-ligatures-qIZUPvRV.js +2 -0
  30. package/client/dist/assets/{addon-webgl-DMbGeI_u.js → addon-webgl-C2d7lriQ.js} +3 -3
  31. package/client/dist/assets/addspec-Rw8uVBC1.js +1 -0
  32. package/client/dist/assets/agent-4rmXoo_p.js +1 -0
  33. package/client/dist/assets/agent-BxZuNGih.js +1 -0
  34. package/client/dist/assets/agent-C3uXOYv_.js +1 -0
  35. package/client/dist/assets/agent-CA8Y4wbw.js +1 -0
  36. package/client/dist/assets/agent-CASKrhG12.js +1 -0
  37. package/client/dist/assets/agent-Cv-jud5n2.js +1 -0
  38. package/client/dist/assets/agent-DitUxYh2.js +1 -0
  39. package/client/dist/assets/agent-Doow4X7o.js +1 -0
  40. package/client/dist/assets/agents-DhG7mg6-.js +1 -0
  41. package/client/dist/assets/agentstudio-BIUe1Im2.js +1 -0
  42. package/client/dist/assets/aiedit-iIjz2_Qk.js +1 -0
  43. package/client/dist/assets/analytics-DZ8Aoq3I.js +1 -0
  44. package/client/dist/assets/api-B2M7CtDw.js +1 -0
  45. package/client/dist/assets/arrow-left-CpIR4j7G.js +1 -0
  46. package/client/dist/assets/arrow-up-right-9DoMn7fD.js +1 -0
  47. package/client/dist/assets/attachments-B53PG733.js +1 -0
  48. package/client/dist/assets/brain-DI9lHkKY.js +1 -0
  49. package/client/dist/assets/browser-CXUcnTMI.js +1 -0
  50. package/client/dist/assets/button-BUIIBQPT.js +1 -0
  51. package/client/dist/assets/chat-BGD7MXAy.js +1 -0
  52. package/client/dist/assets/chevron-right-CViMxucY.js +1 -0
  53. package/client/dist/assets/chunk-KS7C4IRE-Cx5sqopX.js +3 -0
  54. package/client/dist/assets/circle-x-CMYsN4ZH.js +1 -0
  55. package/client/dist/assets/code-BDXEM4ml.js +1 -0
  56. package/client/dist/assets/commands-DyUZQ1FI.js +1 -0
  57. package/client/dist/assets/common-CD1QHriA2.js +1 -0
  58. package/client/dist/assets/common-CwwPaRxx.js +1 -0
  59. package/client/dist/assets/common-DVQuhs-m.js +1 -0
  60. package/client/dist/assets/common-Dg2jaX7d.js +1 -0
  61. package/client/dist/assets/common-DuBwj6tK.js +1 -0
  62. package/client/dist/assets/common-MRj9MHcc.js +1 -0
  63. package/client/dist/assets/common-O4ZpnMLp.js +1 -0
  64. package/client/dist/assets/common-fyeF0v7C2.js +1 -0
  65. package/client/dist/assets/createLucideIcon-Bjbbp_KS.js +1 -0
  66. package/client/dist/assets/{css.worker-Wv5dxAWO.js → css.worker-yKcq13f2.js} +13 -13
  67. package/client/dist/assets/{cssMode-DBHbkBvI.js → cssMode-BM4z9OX0.js} +1 -1
  68. package/client/dist/assets/dashboard-B_NkSHYm.js +1 -0
  69. package/client/dist/assets/dashboard-ClBb_3pq.js +1 -0
  70. package/client/dist/assets/dashboard-CpCmJvMB.js +1 -0
  71. package/client/dist/assets/dashboard-Czohu35O2.js +1 -0
  72. package/client/dist/assets/dashboard-DcjbD0UK.js +1 -0
  73. package/client/dist/assets/dashboard-Dvx5F3we.js +1 -0
  74. package/client/dist/assets/dashboard-R1bOPjBh2.js +1 -0
  75. package/client/dist/assets/dashboard-owcDucDi.js +1 -0
  76. package/client/dist/assets/dialog-CzIekCw0.js +45 -0
  77. package/client/dist/assets/dist-Cez5CRkX.js +1 -0
  78. package/client/dist/assets/dist-bVqfYmby.js +1 -0
  79. package/client/dist/assets/{dist-js-CgpP6tKI.js → dist-js-CcBCbb32.js} +1 -1
  80. package/client/dist/assets/{dist-js-DHiN-A_E.js → dist-js-VwztRkGj.js} +1 -1
  81. package/client/dist/assets/download-CFHyyHcw.js +1 -0
  82. package/client/dist/assets/editor.api2-4ndOupAA.js +872 -0
  83. package/client/dist/assets/{editor.main-C9J2ikJK.js → editor.main-DxrmGfoq.js} +2 -2
  84. package/client/dist/assets/editor.worker-tytyn86m.js +26 -0
  85. package/client/dist/assets/explore-DgBACn39.js +1 -0
  86. package/client/dist/assets/external-link-BcU5EWZp.js +1 -0
  87. package/client/dist/assets/file-text-CFT-xXKN.js +1 -0
  88. package/client/dist/assets/formatDistanceToNow-ByHIauTQ.js +1 -0
  89. package/client/dist/assets/{freemarker2-J5iIIIK-.js → freemarker2-CELlTDJ2.js} +1 -1
  90. package/client/dist/assets/getTimezoneOffsetInMilliseconds-BxHAksaU.js +1 -0
  91. package/client/dist/assets/git-branch-BRM0F5GN.js +1 -0
  92. package/client/dist/assets/{handlebars-DSw1l_tw.js → handlebars-NEjHJ7o_.js} +1 -1
  93. package/client/dist/assets/{html-h2-wE1Lj.js → html-CA10zN5A.js} +1 -1
  94. package/client/dist/assets/{html.worker-CQP8QQsS.js → html.worker-DSJ9zmd7.js} +12 -12
  95. package/client/dist/assets/{htmlMode-Cc_ij7lW.js → htmlMode-89rPYiyq.js} +1 -1
  96. package/client/dist/assets/i18n-CgaRxpII.js +2 -0
  97. package/client/dist/assets/index-BflYQpWm.css +2 -0
  98. package/client/dist/assets/index-C1_-BTXy.js +72 -0
  99. package/client/dist/assets/integrations-mB-7zoyT.js +1 -0
  100. package/client/dist/assets/{javascript-C1q3xv6N.js → javascript-B1gDmwzS.js} +1 -1
  101. package/client/dist/assets/jira-CmKWERE0.js +1 -0
  102. package/client/dist/assets/jira-api-BgFIr8eD.js +1 -0
  103. package/client/dist/assets/job-time-BCL7wee9.js +1 -0
  104. package/client/dist/assets/jobs-B-sDHEay2.js +1 -0
  105. package/client/dist/assets/jobs-BCtGfgUA.js +1 -0
  106. package/client/dist/assets/jobs-BHR_EnDI.js +1 -0
  107. package/client/dist/assets/jobs-BMbZJjJJ.js +1 -0
  108. package/client/dist/assets/jobs-CVFD8rar.js +1 -0
  109. package/client/dist/assets/jobs-DoSaFzAa.js +1 -0
  110. package/client/dist/assets/jobs-DzSIR8V62.js +1 -0
  111. package/client/dist/assets/jobs-lk1_Dq1U.js +1 -0
  112. package/client/dist/assets/json.worker-Oa9UrkKE.js +58 -0
  113. package/client/dist/assets/{jsonMode-CeaCdEhr.js → jsonMode-xHLMfzhF.js} +1 -1
  114. package/client/dist/assets/jsx-runtime-zu2_FqZY.js +1 -0
  115. package/client/dist/assets/lib-C7Mhm-gT.js +7 -0
  116. package/client/dist/assets/lib-CGdyxIWg.js +29 -0
  117. package/client/dist/assets/{liquid-eFsVK0V-.js → liquid-BY6hCUXG.js} +1 -1
  118. package/client/dist/assets/loader-circle-DcygOQpC.js +1 -0
  119. package/client/dist/assets/loops-BzfXWEwO.js +1 -0
  120. package/client/dist/assets/{lspLanguageFeatures-DT3-bbhn.js → lspLanguageFeatures-ira2aEJh.js} +1 -1
  121. package/client/dist/assets/mcp-uDkXYu4L.js +1 -0
  122. package/client/dist/assets/{mdx-Fuwnh1cX.js → mdx-BNx0xr3N.js} +1 -1
  123. package/client/dist/assets/monaco.contribution-Bt92R3Vw.js +2 -0
  124. package/client/dist/assets/nav-myFaXRHD.js +1 -0
  125. package/client/dist/assets/pencil-C9RkrIPZ.js +1 -0
  126. package/client/dist/assets/play-DNlrawbS.js +1 -0
  127. package/client/dist/assets/plus-CsnjfkmX.js +1 -0
  128. package/client/dist/assets/preload-helper-Czpn1I53.js +1 -0
  129. package/client/dist/assets/{python-BgoEs6Cj.js → python-DLdbtmNw.js} +1 -1
  130. package/client/dist/assets/{razor-CM1sPMMq.js → razor-B9gdhAbR.js} +1 -1
  131. package/client/dist/assets/react-dom-DiWzBDP9.js +1 -0
  132. package/client/dist/assets/rolldown-runtime-CNC7AqOf.js +1 -0
  133. package/client/dist/assets/save-28sIlCdr.js +1 -0
  134. package/client/dist/assets/search-Cq_gE3Ik.js +1 -0
  135. package/client/dist/assets/settings-B0ypw9uw.js +1 -0
  136. package/client/dist/assets/setup-CmK87LuS.js +1 -0
  137. package/client/dist/assets/sparkles-DrY8hb0X.js +1 -0
  138. package/client/dist/assets/specs-DII0C8gw.js +1 -0
  139. package/client/dist/assets/spending-C4p06nen.js +1 -0
  140. package/client/dist/assets/terminal-BFRZnMPP.js +1 -0
  141. package/client/dist/assets/tickets-B8BuAaj5.js +1 -0
  142. package/client/dist/assets/tickets-B_yh4qQk.js +1 -0
  143. package/client/dist/assets/tickets-Bs37scwY2.js +1 -0
  144. package/client/dist/assets/tickets-CFk2n2Or.js +1 -0
  145. package/client/dist/assets/tickets-DQp7JS2R2.js +1 -0
  146. package/client/dist/assets/tickets-D_yDquEw.js +1 -0
  147. package/client/dist/assets/tickets-DpmdE8tv2.js +1 -0
  148. package/client/dist/assets/tickets-pAUJkDTu.js +1 -0
  149. package/client/dist/assets/trash-2-BOU3WdZZ.js +1 -0
  150. package/client/dist/assets/triangle-alert-H9T4jAmh.js +1 -0
  151. package/client/dist/assets/{ts.worker-METxwbDZ.js → ts.worker-BtjBFNHX.js} +48 -33
  152. package/client/dist/assets/{tsMode-BmT8WogI.js → tsMode-DEjSThU9.js} +1 -1
  153. package/client/dist/assets/{typescript-CQrQPsoW.js → typescript-CRRnD2Ts.js} +1 -1
  154. package/client/dist/assets/upload-DYMjAQ6M.js +1 -0
  155. package/client/dist/assets/useBrowserCaptureSession-2OqX0vap.js +1 -0
  156. package/client/dist/assets/useDesktop-X8YB9xzv.js +1 -0
  157. package/client/dist/assets/useSharedWebSocket-DaPh7QBL.js +1 -0
  158. package/client/dist/assets/utils-BC4DHlz2.js +1 -0
  159. package/client/dist/assets/value-s-gzBeyF.js +1 -0
  160. package/client/dist/assets/{workers-U0UXyLUf.js → workers-uOo7WLaa.js} +1 -1
  161. package/client/dist/assets/x-CBMI9ejn.js +1 -0
  162. package/client/dist/assets/{xml-WRxJs8_I.js → xml-EDREH2X7.js} +1 -1
  163. package/client/dist/assets/{yaml-CjpSTjJU.js → yaml-BjaDVX1E.js} +1 -1
  164. package/client/dist/assets/zap-CHo3aCD1.js +1 -0
  165. package/client/dist/index.html +73 -32
  166. package/docs/guide/en/integrations/5-mcp-server.md +2 -2
  167. package/docs/internals/safe-pr-review-flow.md +190 -132
  168. package/docs/mcp.md +15 -5
  169. package/package.json +11 -7
  170. package/server/dist/active-pr-continuation.js +16 -1
  171. package/server/dist/agent-chat-manager.js +360 -181
  172. package/server/dist/agent-cwd-manager.js +52 -8
  173. package/server/dist/agent-generator.js +90 -48
  174. package/server/dist/agent-mcp-config.js +69 -36
  175. package/server/dist/agent-operator-prompt.js +3 -2
  176. package/server/dist/agent-refine-manager.js +47 -6
  177. package/server/dist/agent-tier.js +13 -11
  178. package/server/dist/artifact-registry.js +33 -2
  179. package/server/dist/auth.js +5 -4
  180. package/server/dist/browser-capture-manager.js +92 -29
  181. package/server/dist/chat-manager.js +302 -46
  182. package/server/dist/contract-refine-runner.js +19 -6
  183. package/server/dist/db.js +665 -8
  184. package/server/dist/desktop-router.js +1 -2
  185. package/server/dist/headroom-manager.js +1310 -121
  186. package/server/dist/headroom-routing.js +120 -2
  187. package/server/dist/index.js +48 -13
  188. package/server/dist/interactive-job-session.js +237 -82
  189. package/server/dist/jira/jira-adf.js +10 -0
  190. package/server/dist/jira/jira-db.js +15 -3
  191. package/server/dist/jira/jira-sync-manager.js +182 -13
  192. package/server/dist/job-listing.js +140 -0
  193. package/server/dist/job-spawn-idempotency.js +67 -0
  194. package/server/dist/loop-run-manager.js +508 -98
  195. package/server/dist/loop-runs-store.js +266 -13
  196. package/server/dist/mcp/agent-capability.js +71 -0
  197. package/server/dist/mcp/guide.js +5 -2
  198. package/server/dist/mcp/mcp-server.js +5 -10
  199. package/server/dist/mcp/mcp-token.js +5 -5
  200. package/server/dist/mcp/tools/jobs.js +20 -13
  201. package/server/dist/mcp/tools/types.js +20 -42
  202. package/server/dist/mobile/mobile-router.js +1 -1
  203. package/server/dist/pr-lifecycle.js +77 -0
  204. package/server/dist/pr-publisher.js +94 -27
  205. package/server/dist/process-admission.js +82 -0
  206. package/server/dist/project-registry.js +572 -174
  207. package/server/dist/project-router-git.js +13 -2
  208. package/server/dist/project-router-jobs.js +111 -55
  209. package/server/dist/project-router-tickets.js +32 -2
  210. package/server/dist/providers/claude-adapter.js +21 -13
  211. package/server/dist/queue-manager.js +2332 -623
  212. package/server/dist/rail-isolated-launch.js +1137 -290
  213. package/server/dist/rail-isolation.js +4 -2
  214. package/server/dist/rail-pr-decision.js +740 -266
  215. package/server/dist/rail-pr-delivery.js +2 -1
  216. package/server/dist/rail-pr-store.js +317 -18
  217. package/server/dist/rail-pr-ticket-effects.js +328 -0
  218. package/server/dist/rail-worktree-release.js +18 -2
  219. package/server/dist/rails-router.js +149 -119
  220. package/server/dist/rails-store.js +106 -3
  221. package/server/dist/ticket-store.js +23 -1
  222. package/server/dist/transient-children.js +13 -0
  223. package/server/dist/util/cli-prompt.js +42 -8
  224. package/server/dist/util/secure-token-file.js +84 -0
  225. package/server/dist/worktree-manager.js +6 -5
  226. package/client/dist/assets/ActivityFeedPage-D8RKRRoM.js +0 -1
  227. package/client/dist/assets/AgentBrowserCapture-BUHPHNSN.js +0 -1
  228. package/client/dist/assets/AgentModeCodePane-GZc8bNGT.js +0 -2
  229. package/client/dist/assets/AgentModeJobsPane-DWtQqj3k.js +0 -1
  230. package/client/dist/assets/AgentsPage-CKPtHkRP.js +0 -86
  231. package/client/dist/assets/AnalyticsPage-CZW8c77k.js +0 -1
  232. package/client/dist/assets/BarChart-DugEoe2q.js +0 -33
  233. package/client/dist/assets/CodePage-CQ-uKETG.js +0 -2
  234. package/client/dist/assets/DesktopAnalyticsPage-Bgo4SyCH.js +0 -1
  235. package/client/dist/assets/DocsDialog-DxpWw_La.js +0 -11
  236. package/client/dist/assets/DocsPage-CmcOkK19.js +0 -11
  237. package/client/dist/assets/ExportDropdown-DvTS2wc7.js +0 -1
  238. package/client/dist/assets/IntegrationsPage-B72WxJ3Z.js +0 -1
  239. package/client/dist/assets/InteractiveJobComposer-B5gh4Qoy.js +0 -19
  240. package/client/dist/assets/JobDetailModal-D-BeqsJ6.js +0 -1
  241. package/client/dist/assets/JobDetailPage-DwdozwLO.js +0 -1
  242. package/client/dist/assets/JobsPage-BDkinklt.js +0 -1
  243. package/client/dist/assets/LoopBuilderPage-CclJ3VZA.js +0 -7
  244. package/client/dist/assets/LoopPreviewModal-Dw4UgoLu.js +0 -1
  245. package/client/dist/assets/LoopsPage-D7ac4M24.js +0 -1
  246. package/client/dist/assets/PluginsPage-wdC-qa40.js +0 -2
  247. package/client/dist/assets/ProjectSettingsDialog-BlTlRtGI.js +0 -1
  248. package/client/dist/assets/TemplatePreviewModal-CeMx2Red.js +0 -1
  249. package/client/dist/assets/activity-Dbl71S-E.js +0 -1
  250. package/client/dist/assets/addon-image-CwoZj17F.js +0 -1
  251. package/client/dist/assets/addon-ligatures-CWDrED80.js +0 -2
  252. package/client/dist/assets/addspec-DrXqP8ip.js +0 -1
  253. package/client/dist/assets/agent-B8BXmlUQ2.js +0 -1
  254. package/client/dist/assets/agent-BEXYxDbs.js +0 -1
  255. package/client/dist/assets/agent-BYc3-6Ln2.js +0 -1
  256. package/client/dist/assets/agent-BbJorPdT.js +0 -1
  257. package/client/dist/assets/agent-CewFImgL.js +0 -1
  258. package/client/dist/assets/agent-DfhYoUmp.js +0 -1
  259. package/client/dist/assets/agent-TK_rrK4d.js +0 -1
  260. package/client/dist/assets/agent-bHjA9R_j2.js +0 -1
  261. package/client/dist/assets/agents-CGDn_3kB.js +0 -1
  262. package/client/dist/assets/agentstudio-Dxe_94PK.js +0 -1
  263. package/client/dist/assets/aiedit-DrfHQnlz.js +0 -1
  264. package/client/dist/assets/analytics-CxDYcBl9.js +0 -1
  265. package/client/dist/assets/attachments-DvxQbBTC.js +0 -1
  266. package/client/dist/assets/brain-C92rjjYP.js +0 -1
  267. package/client/dist/assets/browser-DM8G151q.js +0 -1
  268. package/client/dist/assets/chat-0dg5hOo_.js +0 -1
  269. package/client/dist/assets/chunk-BNv3lrIs.js +0 -1
  270. package/client/dist/assets/code-B62fClk-.js +0 -1
  271. package/client/dist/assets/commands-DCJ-R9E4.js +0 -1
  272. package/client/dist/assets/common-BM3LFLCE2.js +0 -1
  273. package/client/dist/assets/common-Bwafujnc2.js +0 -1
  274. package/client/dist/assets/common-C-vU-j-I.js +0 -1
  275. package/client/dist/assets/common-C5HKSbbz.js +0 -1
  276. package/client/dist/assets/common-DXrmj6WD.js +0 -1
  277. package/client/dist/assets/common-DhWTkocg.js +0 -1
  278. package/client/dist/assets/common-DiPTT7V_2.js +0 -1
  279. package/client/dist/assets/common-DmqBPwZG.js +0 -1
  280. package/client/dist/assets/dashboard-B5ang-ZS.js +0 -1
  281. package/client/dist/assets/dashboard-CHHcOWVL2.js +0 -1
  282. package/client/dist/assets/dashboard-DBD4wPGY2.js +0 -1
  283. package/client/dist/assets/dashboard-DBuLfWo9.js +0 -1
  284. package/client/dist/assets/dashboard-DSnztwWH.js +0 -1
  285. package/client/dist/assets/dashboard-D_aP_InU.js +0 -1
  286. package/client/dist/assets/dashboard-DmBcdl1B.js +0 -1
  287. package/client/dist/assets/dashboard-RIcUKY1Y2.js +0 -1
  288. package/client/dist/assets/editor.api2-C3B0tp2x.js +0 -872
  289. package/client/dist/assets/editor.worker-Bd9IXS8d.js +0 -26
  290. package/client/dist/assets/explore-_XyU5e7R.js +0 -1
  291. package/client/dist/assets/index-BVbgKXUu.css +0 -2
  292. package/client/dist/assets/index-DyY1QyIn.js +0 -158
  293. package/client/dist/assets/integrations-By80EBkf.js +0 -1
  294. package/client/dist/assets/jira-DeIjwCK3.js +0 -1
  295. package/client/dist/assets/jobs-1GV2-4zd2.js +0 -1
  296. package/client/dist/assets/jobs-26F9gGcE.js +0 -1
  297. package/client/dist/assets/jobs-BgQzB0Sx2.js +0 -1
  298. package/client/dist/assets/jobs-BqWui9_f.js +0 -1
  299. package/client/dist/assets/jobs-CE8qeIYj.js +0 -1
  300. package/client/dist/assets/jobs-DrYmsKy5.js +0 -1
  301. package/client/dist/assets/jobs-DzeuASRf.js +0 -1
  302. package/client/dist/assets/jobs-Wh7dn28q2.js +0 -1
  303. package/client/dist/assets/json.worker-DzV-CpCQ.js +0 -58
  304. package/client/dist/assets/lib-0PFDyMxJ.js +0 -7
  305. package/client/dist/assets/loops-CO6q75W8.js +0 -1
  306. package/client/dist/assets/mcp-HwBbQDk0.js +0 -1
  307. package/client/dist/assets/monaco.contribution-Desq7oq4.js +0 -2
  308. package/client/dist/assets/nav-BTg8s-cD.js +0 -1
  309. package/client/dist/assets/preload-helper-DSXbuxSR.js +0 -1
  310. package/client/dist/assets/settings-cJNynGIZ.js +0 -1
  311. package/client/dist/assets/setup-D2CmgAro.js +0 -1
  312. package/client/dist/assets/specs-PGIOzMtc.js +0 -1
  313. package/client/dist/assets/terminal-CFt56A5f.js +0 -1
  314. package/client/dist/assets/tickets-B7zT4IRf.js +0 -1
  315. package/client/dist/assets/tickets-BHv3ewvq.js +0 -1
  316. package/client/dist/assets/tickets-BoXom7l8.js +0 -1
  317. package/client/dist/assets/tickets-C3b8cHhd2.js +0 -1
  318. package/client/dist/assets/tickets-CFXnJsYF.js +0 -1
  319. package/client/dist/assets/tickets-CWYFvx1W2.js +0 -1
  320. package/client/dist/assets/tickets-DTHo9c3C.js +0 -1
  321. package/client/dist/assets/tickets-SuX_HkK4.js +0 -1
  322. package/client/dist/assets/upload-CmR4h9H1.js +0 -1
  323. package/client/dist/assets/value-JA3bf47v.js +0 -1
  324. /package/client/dist/assets/{abap-D0E4FfVW.js → abap-08VXUWAP.js} +0 -0
  325. /package/client/dist/assets/{activity--iN4Ad2R2.js → activity-B-rp4h_v.js} +0 -0
  326. /package/client/dist/assets/{activity-6hM_ZSg9.js → activity-C6kJpm9q2.js} +0 -0
  327. /package/client/dist/assets/{activity-CIOLxGvh.js → activity-CGx4kb_j.js} +0 -0
  328. /package/client/dist/assets/{activity-BZNfOdiS.js → activity-CnoX2bH7.js} +0 -0
  329. /package/client/dist/assets/{activity-BuVo5sPB.js → activity-DAyekdlt2.js} +0 -0
  330. /package/client/dist/assets/{activity-D2r2h3SE2.js → activity-DCxtpT7j.js} +0 -0
  331. /package/client/dist/assets/{activity-DtxrAUMo2.js → activity-JzwRaWPD.js} +0 -0
  332. /package/client/dist/assets/{addspec-BMtLZLW62.js → addspec-BXjvkh5q.js} +0 -0
  333. /package/client/dist/assets/{addspec-CRuvWn8d.js → addspec-C4bwrmpi.js} +0 -0
  334. /package/client/dist/assets/{addspec-D-JLDsUV.js → addspec-CArHXUDM.js} +0 -0
  335. /package/client/dist/assets/{addspec-mgPOt5q02.js → addspec-DTKJcXiR.js} +0 -0
  336. /package/client/dist/assets/{addspec-DfhDbg5c.js → addspec-DbNBbrha2.js} +0 -0
  337. /package/client/dist/assets/{addspec-CP0HRuxY.js → addspec-htHGZHOe2.js} +0 -0
  338. /package/client/dist/assets/{addspec-bsNDdqRm2.js → addspec-lr_LP6jk.js} +0 -0
  339. /package/client/dist/assets/{agents-78bTZGZh.js → agents-6ZOJXWVk2.js} +0 -0
  340. /package/client/dist/assets/{agents-DDtCJ5M02.js → agents-BZ0XQKjg.js} +0 -0
  341. /package/client/dist/assets/{agents-CWo-KyAB.js → agents-BqWklnHR.js} +0 -0
  342. /package/client/dist/assets/{agents-BUdEi3-N.js → agents-Bz-thHkB2.js} +0 -0
  343. /package/client/dist/assets/{agents-bvyRLczL.js → agents-CJclElx6.js} +0 -0
  344. /package/client/dist/assets/{agents-BpoUaoW92.js → agents-D_l9L0VA.js} +0 -0
  345. /package/client/dist/assets/{agents-BLhWCmDS2.js → agents-u_C0jPB5.js} +0 -0
  346. /package/client/dist/assets/{agentstudio-BMQxc_bz.js → agentstudio-BF19hbXj.js} +0 -0
  347. /package/client/dist/assets/{agentstudio-C_Jq8Tl4.js → agentstudio-BLGvxWBx2.js} +0 -0
  348. /package/client/dist/assets/{agentstudio-Dtq3dGYk.js → agentstudio-D9UKiWTj2.js} +0 -0
  349. /package/client/dist/assets/{agentstudio-CWgXG5W52.js → agentstudio-Drs37nIM.js} +0 -0
  350. /package/client/dist/assets/{agentstudio-sI27TpBu2.js → agentstudio-nEnKkcXM.js} +0 -0
  351. /package/client/dist/assets/{agentstudio-jKMTdT70.js → agentstudio-pyFcSBKy.js} +0 -0
  352. /package/client/dist/assets/{agentstudio-COZriimI2.js → agentstudio-zfTzQh3U.js} +0 -0
  353. /package/client/dist/assets/{aiedit-hknZlly82.js → aiedit-B0KUbpB6.js} +0 -0
  354. /package/client/dist/assets/{aiedit-eMN4eQPF.js → aiedit-BoG24nlg2.js} +0 -0
  355. /package/client/dist/assets/{aiedit-pc5JDUq1.js → aiedit-CjlSRQVP.js} +0 -0
  356. /package/client/dist/assets/{aiedit-B03pD_Na.js → aiedit-DCjA7jLJ.js} +0 -0
  357. /package/client/dist/assets/{aiedit-6CHdb_iv2.js → aiedit-DNXxq_Dr.js} +0 -0
  358. /package/client/dist/assets/{aiedit-igmw-ZoZ2.js → aiedit-DQ-0_pvE.js} +0 -0
  359. /package/client/dist/assets/{aiedit-BtRqRxML.js → aiedit-olnv3QSt2.js} +0 -0
  360. /package/client/dist/assets/{analytics-CCprAqk3.js → analytics-BC7mwR4s.js} +0 -0
  361. /package/client/dist/assets/{analytics-CR1TJqUi2.js → analytics-BiNg8edP.js} +0 -0
  362. /package/client/dist/assets/{analytics-Bj-SmKHa.js → analytics-C0s_T2GN.js} +0 -0
  363. /package/client/dist/assets/{analytics-C5dpiS0o.js → analytics-CdsAPAwZ2.js} +0 -0
  364. /package/client/dist/assets/{analytics-BWMUZ2Pq2.js → analytics-Cr1_q-lC.js} +0 -0
  365. /package/client/dist/assets/{analytics-DfNdXR0y2.js → analytics-IEsKZvpW2.js} +0 -0
  366. /package/client/dist/assets/{analytics-rmHuJv0V.js → analytics-SRPf84OT.js} +0 -0
  367. /package/client/dist/assets/{apex-D_X_8J4d.js → apex-BWPQTe0t.js} +0 -0
  368. /package/client/dist/assets/{atom-one-dark-BMDPFwRZ.js → atom-one-dark-qiGhVQxg.js} +0 -0
  369. /package/client/dist/assets/{attachments-BgBNSqQd.js → attachments-D0wXXt9T2.js} +0 -0
  370. /package/client/dist/assets/{attachments-C9GTTJsx2.js → attachments-DenIjuVI.js} +0 -0
  371. /package/client/dist/assets/{attachments-BjyMmrwE.js → attachments-DhO7LPfy.js} +0 -0
  372. /package/client/dist/assets/{attachments-DPCmkFZu2.js → attachments-DoH5w0x4.js} +0 -0
  373. /package/client/dist/assets/{attachments-DCZOL1RU.js → attachments-Q9yNtiEM.js} +0 -0
  374. /package/client/dist/assets/{attachments-rNn1ekRy2.js → attachments-V_WZAYXx2.js} +0 -0
  375. /package/client/dist/assets/{attachments-DL-bPy9E.js → attachments-eaeScpjM.js} +0 -0
  376. /package/client/dist/assets/{auth-DpZq5p1e.js → auth-D_5bQv5W.js} +0 -0
  377. /package/client/dist/assets/{azcli-PZLnZ3jH.js → azcli-Bc_sGQ0U.js} +0 -0
  378. /package/client/dist/assets/{bat-KlBOxpO4.js → bat-i0X4ZdIN.js} +0 -0
  379. /package/client/dist/assets/{bicep-CgG9oF3V.js → bicep-B5-_aFwp.js} +0 -0
  380. /package/client/dist/assets/{browser-BPkhILSJ.js → browser-C25DkQiO.js} +0 -0
  381. /package/client/dist/assets/{browser-BqLqJVVT.js → browser-CiJ7-L32.js} +0 -0
  382. /package/client/dist/assets/{browser-rWKj-RR_2.js → browser-CvgDySd9.js} +0 -0
  383. /package/client/dist/assets/{browser-Cvg-QHvd2.js → browser-CyR1Kp3m2.js} +0 -0
  384. /package/client/dist/assets/{browser-FRv8P02E.js → browser-DwZljcRh2.js} +0 -0
  385. /package/client/dist/assets/{browser-Cc0iDqUi.js → browser-DxP-alBK.js} +0 -0
  386. /package/client/dist/assets/{browser-BijVYG0Y2.js → browser-rRCwi7Y7.js} +0 -0
  387. /package/client/dist/assets/{cameligo-6R45j6Ph.js → cameligo-DMUM7wLl.js} +0 -0
  388. /package/client/dist/assets/{chat-Bom6bqK3.js → chat-2rIB8HhT.js} +0 -0
  389. /package/client/dist/assets/{chat-vylxqR0P2.js → chat-BR8oe53x2.js} +0 -0
  390. /package/client/dist/assets/{chat-Dnwg-NpQ.js → chat-Bua03tty.js} +0 -0
  391. /package/client/dist/assets/{chat-BgL3zv6x2.js → chat-CZpAQan1.js} +0 -0
  392. /package/client/dist/assets/{chat-DsbwJhus.js → chat-CdY5SGvS.js} +0 -0
  393. /package/client/dist/assets/{chat-Bh3NdgJs.js → chat-DZyu3ImY2.js} +0 -0
  394. /package/client/dist/assets/{chat-DdwZjT8K2.js → chat-D_I7nU5K.js} +0 -0
  395. /package/client/dist/assets/{clojure-CLAAMO7J.js → clojure-Cm7r79vr.js} +0 -0
  396. /package/client/dist/assets/{clsx-CNmW0Nip.js → clsx-CjueKrWZ.js} +0 -0
  397. /package/client/dist/assets/{code-UPVvmAKM2.js → code-76gS0IdL.js} +0 -0
  398. /package/client/dist/assets/{code-BRcDlze52.js → code-C1VwxlML.js} +0 -0
  399. /package/client/dist/assets/{code-CeQ0FZpC.js → code-CnMZyJpr.js} +0 -0
  400. /package/client/dist/assets/{code-DgeHz9CB.js → code-D20ef9Cf2.js} +0 -0
  401. /package/client/dist/assets/{code-DzC1rMk12.js → code-DdOXaWRY.js} +0 -0
  402. /package/client/dist/assets/{code-CwnBPPyA.js → code-Duohuf9l.js} +0 -0
  403. /package/client/dist/assets/{code-BdMy3Ffq.js → code-geHmknB92.js} +0 -0
  404. /package/client/dist/assets/{coffee-AruSygo-.js → coffee-Ba7i2nA0.js} +0 -0
  405. /package/client/dist/assets/{commands-DkU0qliq.js → commands-B9RZO4qS.js} +0 -0
  406. /package/client/dist/assets/{commands-u-4eEHCN.js → commands-BEwbAjtD2.js} +0 -0
  407. /package/client/dist/assets/{commands-JWMfGDmr.js → commands-Bbxu8kLU.js} +0 -0
  408. /package/client/dist/assets/{commands-C4Y3C-PF2.js → commands-CDU3qv68.js} +0 -0
  409. /package/client/dist/assets/{commands-xHDDGfcP2.js → commands-CzezgtQj.js} +0 -0
  410. /package/client/dist/assets/{commands-CSDs9q_2.js → commands-DTQBF_q52.js} +0 -0
  411. /package/client/dist/assets/{commands-RHOchN4n2.js → commands-hOtKuOXc.js} +0 -0
  412. /package/client/dist/assets/{cpp-DEE5r4Pw.js → cpp-C7h46wYY.js} +0 -0
  413. /package/client/dist/assets/{csharp-BVXJUvbK.js → csharp-BKxtCVv1.js} +0 -0
  414. /package/client/dist/assets/{csp-cXOF0JoV.js → csp-bTuwJoIa.js} +0 -0
  415. /package/client/dist/assets/{css-C_zQuAoy.js → css-DIMkf-bt.js} +0 -0
  416. /package/client/dist/assets/{cypher-t95m_Q-O.js → cypher-CVaqCwHa.js} +0 -0
  417. /package/client/dist/assets/{dart-B4r1Tg-e.js → dart-onAF5SnQ.js} +0 -0
  418. /package/client/dist/assets/{dockerfile-6mt7zHQw.js → dockerfile-DZFCIeNp.js} +0 -0
  419. /package/client/dist/assets/{ecl-DEY_Fz_Q.js → ecl-D05T4iGw.js} +0 -0
  420. /package/client/dist/assets/{elixir-R6-q1c5t.js → elixir-6RTg0lbw.js} +0 -0
  421. /package/client/dist/assets/{explore-B7fblXBb2.js → explore-BG99VGpL.js} +0 -0
  422. /package/client/dist/assets/{explore-CNBzrEPS.js → explore-C0TjprJW2.js} +0 -0
  423. /package/client/dist/assets/{explore-C0k8HrkN.js → explore-C2dgEo7q.js} +0 -0
  424. /package/client/dist/assets/{explore-BHtEpoNN2.js → explore-C8NDHbob.js} +0 -0
  425. /package/client/dist/assets/{explore-Dahc1lOx.js → explore-CWWgzQGR.js} +0 -0
  426. /package/client/dist/assets/{explore-D_mWBYCS2.js → explore-CcQXpdtS.js} +0 -0
  427. /package/client/dist/assets/{explore-UvW1jEMa.js → explore-DsXAgnmJ2.js} +0 -0
  428. /package/client/dist/assets/{flow9-CNQvHBOt.js → flow9-C5_-GSwl.js} +0 -0
  429. /package/client/dist/assets/{format-command-CuWuRvka.js → format-command-Dfa1o2d-.js} +0 -0
  430. /package/client/dist/assets/{fsharp-vPZkFXwv.js → fsharp-C8Ef5oNN.js} +0 -0
  431. /package/client/dist/assets/{go-KUzBMYJ_.js → go-C-y9NEjX.js} +0 -0
  432. /package/client/dist/assets/{graphql-C3W0rCsD.js → graphql-fmXr3nnJ.js} +0 -0
  433. /package/client/dist/assets/{hcl-BOOMFSNX.js → hcl-CpzslTdj.js} +0 -0
  434. /package/client/dist/assets/{ini-CFat1WJr.js → ini-sBoK_t0W.js} +0 -0
  435. /package/client/dist/assets/{integrations-TAcYNh0b.js → integrations-CH0zy1Mw2.js} +0 -0
  436. /package/client/dist/assets/{integrations-C1ThkTC52.js → integrations-CIAjhd7L.js} +0 -0
  437. /package/client/dist/assets/{integrations-2YcNyAl3.js → integrations-CZdwOe8A.js} +0 -0
  438. /package/client/dist/assets/{integrations-BXCvXUlG2.js → integrations-Ct7omMld.js} +0 -0
  439. /package/client/dist/assets/{integrations-Crk2rFhL2.js → integrations-D6MSxy3-.js} +0 -0
  440. /package/client/dist/assets/{integrations-Dn4JiGtW.js → integrations-DbZa-Zyc.js} +0 -0
  441. /package/client/dist/assets/{integrations-D9YrJ_Ep.js → integrations-b8pH-vfl2.js} +0 -0
  442. /package/client/dist/assets/{java-Dee-CNAM.js → java-BEtHBSE6.js} +0 -0
  443. /package/client/dist/assets/{jira-D5efgwp_.js → jira-BySfUuQZ.js} +0 -0
  444. /package/client/dist/assets/{jira-bDs7Im2C.js → jira-Byw2C0F8.js} +0 -0
  445. /package/client/dist/assets/{jira-BHAKtc4B.js → jira-CNeYi06w2.js} +0 -0
  446. /package/client/dist/assets/{jira-Be369RIR2.js → jira-D11gPhxz.js} +0 -0
  447. /package/client/dist/assets/{jira-CPpgA-5z2.js → jira-D38Gj7AT.js} +0 -0
  448. /package/client/dist/assets/{jira-uNH1gETL2.js → jira-gAvOyije.js} +0 -0
  449. /package/client/dist/assets/{jira-DS7ownDy.js → jira-pRaL7JQo2.js} +0 -0
  450. /package/client/dist/assets/{julia-CGNlLYuU.js → julia-Bri6UV-V.js} +0 -0
  451. /package/client/dist/assets/{kotlin-CY7lhknJ.js → kotlin-BOotOW0E.js} +0 -0
  452. /package/client/dist/assets/{less-D0KN7hfJ.js → less-B9JPFI3C.js} +0 -0
  453. /package/client/dist/assets/{lexon-CuRJufPs.js → lexon-CfSJPG6W.js} +0 -0
  454. /package/client/dist/assets/{loops-LIgO_-eM.js → loops-CPpgMapU.js} +0 -0
  455. /package/client/dist/assets/{loops-DJNhp17p2.js → loops-CWWbKuVL.js} +0 -0
  456. /package/client/dist/assets/{loops-D6D3XHjj.js → loops-Crc9It8o2.js} +0 -0
  457. /package/client/dist/assets/{loops-CvOWsi5r.js → loops-DH2hZcbK2.js} +0 -0
  458. /package/client/dist/assets/{loops-vhJdbcz92.js → loops-DS5TmvIU2.js} +0 -0
  459. /package/client/dist/assets/{loops-K5_HXfJa.js → loops-P2f0cN8-.js} +0 -0
  460. /package/client/dist/assets/{loops-BvqQ-bHR2.js → loops-fJV-9hH7.js} +0 -0
  461. /package/client/dist/assets/{lua-DHvH62Zf.js → lua-CsQS60Ue.js} +0 -0
  462. /package/client/dist/assets/{m3-tlthQ8Fo.js → m3-D-oSqn_W.js} +0 -0
  463. /package/client/dist/assets/{markdown-Dbx1LgUZ.js → markdown-Cimd5fb3.js} +0 -0
  464. /package/client/dist/assets/{mcp-Dcl-XxkR.js → mcp-Czd_iyp02.js} +0 -0
  465. /package/client/dist/assets/{mcp-XJNRfxVp2.js → mcp-D2pc5GGx.js} +0 -0
  466. /package/client/dist/assets/{mcp-sRmWE8WJ.js → mcp-D4ZjtIx32.js} +0 -0
  467. /package/client/dist/assets/{mcp-DtbxuH3V2.js → mcp-DIfBUv2D2.js} +0 -0
  468. /package/client/dist/assets/{mcp-H-gnQUdN.js → mcp-DsTwmjST.js} +0 -0
  469. /package/client/dist/assets/{mcp-N1nzp8oL.js → mcp-SMkn8boG.js} +0 -0
  470. /package/client/dist/assets/{mcp-CjVKXAhD2.js → mcp-fSfiVYAc.js} +0 -0
  471. /package/client/dist/assets/{mips-D_O3IUz_.js → mips-CIPQ_RoX.js} +0 -0
  472. /package/client/dist/assets/{msdax-C_ZmCnRt.js → msdax-DauUninz.js} +0 -0
  473. /package/client/dist/assets/{mysql-C-dCEYSX.js → mysql-SOo6toE5.js} +0 -0
  474. /package/client/dist/assets/{nav-BElxljIF2.js → nav-BRJBxaGt2.js} +0 -0
  475. /package/client/dist/assets/{nav-COu2E_-L.js → nav-CAGKv3_F.js} +0 -0
  476. /package/client/dist/assets/{nav-BXO7PJJG2.js → nav-CNskRJKQ2.js} +0 -0
  477. /package/client/dist/assets/{nav-CW9pxWfC2.js → nav-DY93irBL.js} +0 -0
  478. /package/client/dist/assets/{nav-CXHUA4uf.js → nav-DhjvYgqt.js} +0 -0
  479. /package/client/dist/assets/{nav-DdHF5VWJ.js → nav-Gb-r4wop.js} +0 -0
  480. /package/client/dist/assets/{nav-C8kOuGh8.js → nav-TvumAFD22.js} +0 -0
  481. /package/client/dist/assets/{objective-c-DZY9Aqxd.js → objective-c-FvmIjYaQ.js} +0 -0
  482. /package/client/dist/assets/{pascal-BVHLXoor.js → pascal-DrH0SRf2.js} +0 -0
  483. /package/client/dist/assets/{pascaligo-XdG_6u3k.js → pascaligo-D-ptJ9y-.js} +0 -0
  484. /package/client/dist/assets/{perl-b9QV0BcD.js → perl-oz_6vUea.js} +0 -0
  485. /package/client/dist/assets/{pgsql-Du5FDOgT.js → pgsql-DTj74zXo.js} +0 -0
  486. /package/client/dist/assets/{php-WJD0z2Z_.js → php-nr791fC2.js} +0 -0
  487. /package/client/dist/assets/{pla-CbciTf2w.js → pla-CopQ2nXW.js} +0 -0
  488. /package/client/dist/assets/{postiats-C1QxmvVq.js → postiats-43DmfD33.js} +0 -0
  489. /package/client/dist/assets/{powerquery-BEYDhPLZ.js → powerquery-D3hlyOfw.js} +0 -0
  490. /package/client/dist/assets/{powershell-CfaYw4XL.js → powershell-DmHpPYUd.js} +0 -0
  491. /package/client/dist/assets/{protobuf-pwZEDOYa.js → protobuf-C531GsRP.js} +0 -0
  492. /package/client/dist/assets/{pug-5ftqSPVe.js → pug-Z5eAx3Zn.js} +0 -0
  493. /package/client/dist/assets/{qsharp-uI6qZMq4.js → qsharp-DkqhCAOL.js} +0 -0
  494. /package/client/dist/assets/{r-BK54P1EG.js → r-BwWrilGY.js} +0 -0
  495. /package/client/dist/assets/{redis-DI0yd7kN.js → redis-ClamHrr6.js} +0 -0
  496. /package/client/dist/assets/{redshift-Dov26eyI.js → redshift-DT7zqm-g.js} +0 -0
  497. /package/client/dist/assets/{restructuredtext-QqJ_1dC-.js → restructuredtext-BYgofb2h.js} +0 -0
  498. /package/client/dist/assets/{ruby-5WjveeVb.js → ruby-DezsRK8O.js} +0 -0
  499. /package/client/dist/assets/{rust-BzVSfqFH.js → rust-DdL9SqIa.js} +0 -0
  500. /package/client/dist/assets/{sb-DMIUfT-3.js → sb-CcwsVR0C.js} +0 -0
  501. /package/client/dist/assets/{scala-CSKA7lqt.js → scala-DHpiXF5c.js} +0 -0
  502. /package/client/dist/assets/{scheme-CojOP2Qc.js → scheme-BeGwcela.js} +0 -0
  503. /package/client/dist/assets/{scss-DoBZGJG6.js → scss-gp-XZpBa.js} +0 -0
  504. /package/client/dist/assets/{settings-1w1O5Og82.js → settings-B9ilOSYO.js} +0 -0
  505. /package/client/dist/assets/{settings-Jc46vUhd.js → settings-BqlWHRHA.js} +0 -0
  506. /package/client/dist/assets/{settings-BSihkr0e.js → settings-D8dlqt4d.js} +0 -0
  507. /package/client/dist/assets/{settings-BiDOB3hV2.js → settings-DAZqjX2a2.js} +0 -0
  508. /package/client/dist/assets/{settings-DNL68KTE.js → settings-DRh39v1S2.js} +0 -0
  509. /package/client/dist/assets/{settings-BoubJhDP2.js → settings-Mj1OJbfd2.js} +0 -0
  510. /package/client/dist/assets/{settings-CB-EZ_W8.js → settings-lRReaSYm.js} +0 -0
  511. /package/client/dist/assets/{setup-D84y3wx-.js → setup-CGZjy2IF2.js} +0 -0
  512. /package/client/dist/assets/{setup-BuQNKztt2.js → setup-CXa0Sx16.js} +0 -0
  513. /package/client/dist/assets/{setup-BwMovM9z2.js → setup-D2ktKKMF2.js} +0 -0
  514. /package/client/dist/assets/{setup-DU0ULX8q2.js → setup-DG2H7ha5.js} +0 -0
  515. /package/client/dist/assets/{setup-BXpw07r-.js → setup-DK5bHDUq2.js} +0 -0
  516. /package/client/dist/assets/{setup-7J8TjG4O.js → setup-DrWlIQTZ.js} +0 -0
  517. /package/client/dist/assets/{setup-BiduSX9d.js → setup-rzHUVgAu.js} +0 -0
  518. /package/client/dist/assets/{shell-YOaE4udl.js → shell-CC2rA5mh.js} +0 -0
  519. /package/client/dist/assets/{solidity-l3jSyVLu.js → solidity-BEEn4gHE.js} +0 -0
  520. /package/client/dist/assets/{sophia-BhSGTaQJ.js → sophia-CRfGWb83.js} +0 -0
  521. /package/client/dist/assets/{sparql-qhk6QD9w.js → sparql-D_Lu-MrJ.js} +0 -0
  522. /package/client/dist/assets/{specs-CWtKcVDC.js → specs-BcyxaCwf2.js} +0 -0
  523. /package/client/dist/assets/{specs-DTplt2XM.js → specs-C1q9geAi.js} +0 -0
  524. /package/client/dist/assets/{specs-PozUNhqb.js → specs-CLZKysq-2.js} +0 -0
  525. /package/client/dist/assets/{specs-C9Sd7BhH2.js → specs-DZMS9o54.js} +0 -0
  526. /package/client/dist/assets/{specs-D2clrt2c2.js → specs-Dm6zUI-R.js} +0 -0
  527. /package/client/dist/assets/{specs-CBDvEYKv.js → specs-V3TvhKLX.js} +0 -0
  528. /package/client/dist/assets/{specs-CIPr9reA2.js → specs-dV3Mq5Mu2.js} +0 -0
  529. /package/client/dist/assets/{sql-DoO3y21-.js → sql-NEE52Syq.js} +0 -0
  530. /package/client/dist/assets/{st-ChZAY8oG.js → st-DbInun42.js} +0 -0
  531. /package/client/dist/assets/{swift-D290Tr7_.js → swift-Bxkupp3x.js} +0 -0
  532. /package/client/dist/assets/{systemverilog-DsxzgJ-o.js → systemverilog-Bz4Y3fRF.js} +0 -0
  533. /package/client/dist/assets/{tcl-BKDUHAAR.js → tcl-DISqw1ZD.js} +0 -0
  534. /package/client/dist/assets/{terminal-De2Brd23.js → terminal-BZiIu3rf.js} +0 -0
  535. /package/client/dist/assets/{terminal-cqfd3EDV.js → terminal-C2NAd7BW2.js} +0 -0
  536. /package/client/dist/assets/{terminal-WSAvHXoU.js → terminal-IahE4LW92.js} +0 -0
  537. /package/client/dist/assets/{terminal-Z7KsDM_n.js → terminal-QpXZU2Bn2.js} +0 -0
  538. /package/client/dist/assets/{terminal-xGbCZJHg.js → terminal-ScjyXNxQ.js} +0 -0
  539. /package/client/dist/assets/{terminal-BRm03iFA2.js → terminal-d7GadXbN.js} +0 -0
  540. /package/client/dist/assets/{terminal-DxQWCqSD2.js → terminal-tITa81iz.js} +0 -0
  541. /package/client/dist/assets/{twig-UpZQaM8G.js → twig-De2hgUGE.js} +0 -0
  542. /package/client/dist/assets/{typespec-DzaKvN8i.js → typespec-B8J7ngcE.js} +0 -0
  543. /package/client/dist/assets/{vb-CZpbb7tk.js → vb-DV3o63ZY.js} +0 -0
  544. /package/client/dist/assets/{wgsl-CNt3OIvl.js → wgsl-DpFanUEy.js} +0 -0
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.QueueManager = exports.FREESTYLE_COMMAND_RE = exports.JobAlreadyTerminalError = exports.JobNotFoundError = exports.CodexNotFoundError = exports.ClaudeNotFoundError = exports.DEFAULT_ZOMBIE_TIMEOUT_MS = void 0;
6
+ exports.QueueManager = exports.FREESTYLE_COMMAND_RE = exports.InvalidJobDependencyError = exports.JobAlreadyTerminalError = exports.JobNotFoundError = exports.CodexNotFoundError = exports.ClaudeNotFoundError = exports.DEFAULT_ZOMBIE_TIMEOUT_MS = void 0;
7
7
  exports.buildTelemetryEnv = buildTelemetryEnv;
8
8
  exports.projectSupportsProfiles = projectSupportsProfiles;
9
9
  exports.distributeIntEvenly = distributeIntEvenly;
@@ -143,6 +143,33 @@ function distributeIntEvenly(total, n) {
143
143
  }
144
144
  return out;
145
145
  }
146
+ function maxNullable(left, right) {
147
+ const safeLeft = typeof left === 'number' && Number.isFinite(left) && left >= 0 ? left : null;
148
+ const safeRight = typeof right === 'number' && Number.isFinite(right) && right >= 0 ? right : null;
149
+ if (safeLeft == null)
150
+ return safeRight;
151
+ if (safeRight == null)
152
+ return safeLeft;
153
+ return Math.max(safeLeft, safeRight);
154
+ }
155
+ function sanitizeRecoveredResult(result) {
156
+ const safe = { ...result };
157
+ const numeric = [
158
+ 'tokens_in', 'tokens_out', 'tokens_cache_read', 'tokens_cache_create',
159
+ 'total_cost_usd', 'num_turns', 'duration_ms', 'duration_api_ms',
160
+ ];
161
+ for (const key of numeric) {
162
+ const value = safe[key];
163
+ if (value !== undefined && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) {
164
+ delete safe[key];
165
+ }
166
+ }
167
+ if (safe.model !== undefined && typeof safe.model !== 'string')
168
+ delete safe.model;
169
+ if (safe.session_id !== undefined && typeof safe.session_id !== 'string')
170
+ delete safe.session_id;
171
+ return safe;
172
+ }
146
173
  const LOG_BUFFER_MAX = 5000;
147
174
  const LOG_BUFFER_DROP = 1000;
148
175
  exports.DEFAULT_ZOMBIE_TIMEOUT_MS = 1_800_000; // 30 minutes
@@ -175,6 +202,13 @@ class JobAlreadyTerminalError extends Error {
175
202
  }
176
203
  }
177
204
  exports.JobAlreadyTerminalError = JobAlreadyTerminalError;
205
+ class InvalidJobDependencyError extends Error {
206
+ constructor(message) {
207
+ super(message);
208
+ this.name = 'InvalidJobDependencyError';
209
+ }
210
+ }
211
+ exports.InvalidJobDependencyError = InvalidJobDependencyError;
178
212
  // ─── Helpers ──────────────────────────────────────────────────────────────────
179
213
  const TERMINAL_STATUSES = new Set(['completed', 'failed', 'canceled', 'zombie_terminated', 'skipped']);
180
214
  /** Match an Freestyle rail command: `/specrails:freestyle #5 …` (or `/sr:…`). */
@@ -189,6 +223,7 @@ class QueueManager {
189
223
  _killTimer;
190
224
  _cancelingJobs;
191
225
  _zombieJobs;
226
+ _persistenceFailedJobs;
192
227
  _broadcast;
193
228
  _db;
194
229
  _logBuffer;
@@ -199,6 +234,12 @@ class QueueManager {
199
234
  /** Set by shutdown(); once disposed the manager spawns no new jobs and never
200
235
  * touches the (now possibly closed) DB from late child 'close' callbacks. */
201
236
  _disposed;
237
+ /** Startup projection/capture failed and must be rebuilt before admissions
238
+ * or resume may proceed. Cleared only by a complete restore pass. */
239
+ _restoreBlocked;
240
+ /** Invalidates async pre-spawn work across shutdown. A job captures the
241
+ * generation at start and must still own the active slot before spawning. */
242
+ _lifecycleGeneration;
202
243
  _getCostAlertThreshold;
203
244
  _getDesktopDailyBudget;
204
245
  _adapter;
@@ -208,6 +249,11 @@ class QueueManager {
208
249
  * fallback model name stamped onto the ai_invocations row. */
209
250
  _resolvedModel;
210
251
  _onJobFinished;
252
+ /** Project-owned durable effects that are part of every queue admission
253
+ * (for example, recovery/rail ownership). Invoked after queued_jobs has
254
+ * been written but inside the same outer transaction, before any
255
+ * route-specific admission claim. */
256
+ _onJobAdmission;
211
257
  _onBudgetExceeded;
212
258
  /** Project ID used for OTEL resource attributes (Super mode only) */
213
259
  _projectId;
@@ -215,21 +261,20 @@ class QueueManager {
215
261
  _desktopPort;
216
262
  /** Project slug used for per-job profile snapshots (Super mode only) */
217
263
  _projectSlug;
218
- /** Pending profile selection keyed by jobId — read at spawn time */
264
+ /** Pending profile selection keyed by jobId — read at spawn time. Map
265
+ * absence means default resolution; a present null forces legacy mode. */
219
266
  _jobProfileSelection;
220
- /** Pending per-job provider override keyed by jobId — read at spawn time.
221
- * In-memory only (mirrors _jobProfileSelection): a queued job that survives a
222
- * restart falls back to the project's primary provider. */
267
+ /** Pending per-job provider override keyed by jobId — restart-durable while
268
+ * queued and consumed only after durable promotion. */
223
269
  _jobProviderSelection;
224
- /** Resolved adapter id per RUNNING job, captured at `_startJob` time AFTER the
225
- * per-job override has been consumed-and-deleted from `_jobProviderSelection`.
270
+ /** Resolved adapter id per RUNNING job, captured at `_startJob` time.
226
271
  * Read by `_forceFailUnkillableJob` (and any other terminal path that runs
227
272
  * without a child exit) to stamp `ai_invocations.provider` with the provider
228
- * the child ACTUALLY ran on — the override map is empty by then. Cleared with
273
+ * the child ACTUALLY ran on. Cleared with
229
274
  * the other per-job maps at every teardown. In-memory only. */
230
275
  _jobResolvedProvider;
231
- /** Pending per-job model override keyed by jobId — read at spawn time.
232
- * In-memory only (mirrors _jobProviderSelection). */
276
+ /** Pending per-job model override keyed by jobId — restart-durable while
277
+ * queued and consumed only after durable promotion. */
233
278
  _jobModelSelection;
234
279
  /** Pre-spawn working-tree snapshot refs keyed by jobId — read at exit time
235
280
  * by the Code-Explorer provenance hook. Cleared on job exit. */
@@ -241,8 +286,8 @@ class QueueManager {
241
286
  /** Per-job openspec PATH shim dir (relocated claude rails only). Cleaned up on
242
287
  * job exit. In-memory map of jobId → shim dir. */
243
288
  _openspecShims = new Map();
244
- /** Pending per-job interactive flag keyed by jobId — read at spawn time.
245
- * In-memory only (mirrors _jobModelSelection). */
289
+ /** Pending per-job interactive override keyed by jobId. Map absence means the
290
+ * spawn-time default; present false/true are both restart-durable. */
246
291
  _jobInteractiveSelection;
247
292
  /** Per-job PR-delivery mode (safe-pr-workflow), captured ONCE at spawn time by
248
293
  * the SAME `isRailPrDeliveryEnabled()` read that injects
@@ -265,12 +310,27 @@ class QueueManager {
265
310
  * `_disposed` and the whole job's spend is lost (COST-ACCOUNTING-AUDIT CRIT-3).
266
311
  * Cleared on every terminal path. In-memory only. */
267
312
  _jobLiveAccounting;
313
+ _jobReaders;
268
314
  /** Jobs terminated by `_forceFailUnkillableJob` (SIGKILL-escalation failure)
269
315
  * whose surviving child's `close` may still fire `_onJobExit` later. Guards
270
316
  * against a duplicate ai_invocations row + a double `_onJobFinished`; a late
271
317
  * close that carries REAL cost replaces the no-cost placeholder rows
272
318
  * (COST-ACCOUNTING-AUDIT LOW-6). In-memory only. */
273
319
  _forceFailedRowJobs;
320
+ /** Children that survived a failed SIGKILL escalation. They no longer own the
321
+ * runnable slot, but shutdown must still attempt to terminate them. */
322
+ _forceFailedProcesses;
323
+ /** Child exits whose terminal transaction failed. They deliberately retain
324
+ * the active slot until shutdown/restart recovery succeeds. */
325
+ _terminalPersistenceBlockedJobs;
326
+ _pendingLateReconciliations;
327
+ /** Last terminal state emitted per pipeline. Descendant skipping and parent
328
+ * settlement can both evaluate the same pipeline in one call stack; this
329
+ * keeps the externally visible transition exactly-once. */
330
+ _emittedPipelineStatuses;
331
+ /** Test seam for the sole async pre-spawn dependency. Production resolves the
332
+ * implementation lazily to keep the plugin subsystem optional. */
333
+ _resolvePluginsForSpawn;
274
334
  constructor(broadcast, db, commands, cwd, options) {
275
335
  this._queue = [];
276
336
  this._jobs = new Map();
@@ -280,6 +340,7 @@ class QueueManager {
280
340
  this._killTimer = null;
281
341
  this._cancelingJobs = new Set();
282
342
  this._zombieJobs = new Set();
343
+ this._persistenceFailedJobs = new Set();
283
344
  this._broadcast = broadcast;
284
345
  this._db = db ?? null;
285
346
  this._logBuffer = [];
@@ -287,12 +348,16 @@ class QueueManager {
287
348
  this._cwd = cwd;
288
349
  this._inactivityTimer = null;
289
350
  this._disposed = false;
351
+ this._restoreBlocked = false;
352
+ this._lifecycleGeneration = 0;
290
353
  this._getCostAlertThreshold = options?.getCostAlertThreshold ?? null;
291
354
  this._getDesktopDailyBudget = options?.getDesktopDailyBudget ?? null;
292
355
  this._adapter = (0, providers_1.getAdapter)(options?.provider ?? 'claude');
293
356
  this._resolvedModel = options?.resolvedModel ?? null;
294
357
  this._onJobFinished = options?.onJobFinished ?? null;
358
+ this._onJobAdmission = options?.onJobAdmission ?? null;
295
359
  this._onBudgetExceeded = options?.onBudgetExceeded ?? null;
360
+ this._resolvePluginsForSpawn = options?.resolvePluginsForSpawn ?? null;
296
361
  this._projectId = options?.projectId ?? null;
297
362
  this._desktopPort = options?.desktopPort ?? 4200;
298
363
  this._projectSlug = options?.projectSlug ?? null;
@@ -306,7 +371,12 @@ class QueueManager {
306
371
  this._jobPrDelivery = new Map();
307
372
  this._interactiveSessions = new Map();
308
373
  this._jobLiveAccounting = new Map();
374
+ this._jobReaders = new Map();
309
375
  this._forceFailedRowJobs = new Set();
376
+ this._forceFailedProcesses = new Map();
377
+ this._terminalPersistenceBlockedJobs = new Set();
378
+ this._pendingLateReconciliations = new Map();
379
+ this._emittedPipelineStatuses = new Map();
310
380
  const envTimeout = process.env.WM_ZOMBIE_TIMEOUT_MS !== undefined
311
381
  ? parseInt(process.env.WM_ZOMBIE_TIMEOUT_MS, 10)
312
382
  : null;
@@ -338,9 +408,17 @@ class QueueManager {
338
408
  * is closed (e.g. in ProjectRegistry.removeProject) and on graceful shutdown.
339
409
  */
340
410
  shutdown() {
341
- if (this._disposed)
342
- return;
411
+ if (this._disposed) {
412
+ if (!this._db)
413
+ return true;
414
+ // A previous removal attempt may have stopped children but failed to
415
+ // stage their terminal intent. Re-enter the idempotent shutdown path with
416
+ // the retained ownership/maps so storage repair can converge it.
417
+ this._disposed = false;
418
+ return this.shutdown();
419
+ }
343
420
  this._disposed = true;
421
+ this._lifecycleGeneration += 1;
344
422
  if (this._inactivityTimer !== null) {
345
423
  clearTimeout(this._inactivityTimer);
346
424
  this._inactivityTimer = null;
@@ -349,8 +427,14 @@ class QueueManager {
349
427
  clearTimeout(this._killTimer);
350
428
  this._killTimer = null;
351
429
  }
352
- const proc = this._activeProcess;
353
- if (proc && proc.pid) {
430
+ const ownedProcesses = new Set();
431
+ if (this._activeProcess)
432
+ ownedProcesses.add(this._activeProcess);
433
+ for (const proc of this._forceFailedProcesses.values())
434
+ ownedProcesses.add(proc);
435
+ for (const proc of ownedProcesses) {
436
+ if (!proc.pid)
437
+ continue;
354
438
  const pid = proc.pid;
355
439
  try {
356
440
  (0, win_spawn_1.treeKillSafe)(pid, 'SIGTERM', () => { });
@@ -370,9 +454,40 @@ class QueueManager {
370
454
  // the treeKill'd child's later 'close' hits `if (this._disposed) return` in
371
455
  // _onJobExit, so without this flush the whole job's spend is lost
372
456
  // (COST-ACCOUNTING-AUDIT CRIT-3 / HIGH-1). Best-effort — never throws.
373
- this._flushInFlightAccounting();
457
+ for (const jobId of Array.from(this._forceFailedRowJobs)) {
458
+ const live = this._jobLiveAccounting.get(jobId);
459
+ this._reconcileForceFailedJobExit(jobId, null, live?.events ?? [], live?.adapter ?? this._adapter, live?.model);
460
+ }
461
+ for (const [jobId, pending] of Array.from(this._pendingLateReconciliations)) {
462
+ this._reconcileForceFailedJobExit(jobId, pending.code, pending.adapterEvents, pending.adapter, pending.spawnedModel);
463
+ }
464
+ if (!this._flushInFlightAccounting())
465
+ return false;
466
+ try {
467
+ // A child may have closed through the disposed guard after an earlier
468
+ // staging failure, leaving only its durable RUNNING row. Capture that row
469
+ // from raw events before declaring a retried removal safe.
470
+ this._captureOrphanRecoveries();
471
+ this._resumeOrphanRecoveries();
472
+ }
473
+ catch (err) {
474
+ console.error('[queue-manager] shutdown orphan capture failed:', err);
475
+ return false;
476
+ }
477
+ for (const readers of this._jobReaders.values()) {
478
+ try {
479
+ readers.stdout.close();
480
+ }
481
+ catch { /* best-effort */ }
482
+ try {
483
+ readers.stderr.close();
484
+ }
485
+ catch { /* best-effort */ }
486
+ }
487
+ this._jobReaders.clear();
374
488
  this._activeProcess = null;
375
489
  this._activeJobId = null;
490
+ this._forceFailedProcesses.clear();
376
491
  // Tear down any resident interactive sessions (SIGTERM their children) so
377
492
  // teardown orphans no persistent claude process. dispose() does not settle
378
493
  // (the aborted row was already written by _flushInFlightAccounting).
@@ -390,26 +505,172 @@ class QueueManager {
390
505
  this._jobPrDelivery.clear();
391
506
  this._jobLiveAccounting.clear();
392
507
  this._openspecShims.clear();
393
- // Drop the DB reference last so any in-flight 'close' callback sees null
394
- // and skips all DB work via the existing `if (this._db)` guards.
395
- this._db = null;
508
+ // Project removal must retain this durable DB until every critical outbox
509
+ // effect converges. App shutdown may still close the connection externally;
510
+ // the file/outbox survives for next startup.
511
+ const recoveryComplete = this._terminalRecoveryComplete();
512
+ if (recoveryComplete)
513
+ this._db = null;
514
+ return recoveryComplete;
515
+ }
516
+ _terminalRecoveryComplete() {
517
+ const db = this._db;
518
+ if (!db)
519
+ return true;
520
+ try {
521
+ const rows = db.prepare(`
522
+ SELECT job_id, payload, accounting_completed, callback_completed, terminal_completed
523
+ FROM orphan_job_recovery
524
+ `).all();
525
+ const running = db.prepare(`SELECT 1 FROM jobs WHERE status = 'running' AND owner = 'queue' LIMIT 1`).get();
526
+ if (running)
527
+ return false;
528
+ for (const row of rows) {
529
+ const { payload } = this._decodeRecoveryPayload(row.job_id, row.payload);
530
+ if (row.accounting_completed === 0 ||
531
+ row.callback_completed === 0 ||
532
+ row.terminal_completed === 0 ||
533
+ payload.awaitingLateReconciliation)
534
+ return false;
535
+ }
536
+ return true;
537
+ }
538
+ catch (err) {
539
+ console.error('[queue-manager] terminal recovery completion check failed:', err);
540
+ return false;
541
+ }
542
+ }
543
+ /** Atomically terminalize a job and create its immutable recovery intent.
544
+ * No accounting, ticket/rail callback, dependent mutation or user-visible
545
+ * terminal broadcast may happen before this transaction commits. */
546
+ _stageTerminalIntent(job, input) {
547
+ const db = this._db;
548
+ if (!db)
549
+ throw new Error('Cannot stage a terminal job without its project database');
550
+ const result = input.result ?? {};
551
+ const payload = {
552
+ id: job.id,
553
+ command: job.command,
554
+ ticketIds: this._extractTicketIds(job.command),
555
+ pipelineId: job.pipelineId,
556
+ startedAt: job.startedAt ?? input.finishedAt,
557
+ finishedAt: input.finishedAt,
558
+ provider: input.provider,
559
+ model: result.model ?? null,
560
+ tokensIn: result.tokens_in ?? null,
561
+ tokensOut: result.tokens_out ?? null,
562
+ tokensCacheRead: result.tokens_cache_read ?? null,
563
+ tokensCacheCreate: result.tokens_cache_create ?? null,
564
+ totalCostUsd: result.total_cost_usd ?? null,
565
+ totalCostUsdEstimated: result.total_cost_usd_estimated == null
566
+ ? 0
567
+ : (result.total_cost_usd_estimated ? 1 : 0),
568
+ numTurns: result.num_turns ?? null,
569
+ durationMs: result.duration_ms ?? null,
570
+ durationApiMs: result.duration_api_ms ?? null,
571
+ sessionId: result.session_id ?? null,
572
+ descendants: input.descendants ?? (input.status === 'completed' ? [] : this._snapshotRecoveryDescendants(job.id)),
573
+ causalOwnership: job.causalOwnership === true,
574
+ terminalStatus: input.status,
575
+ invocationStatus: input.invocationStatus,
576
+ ticketCompletionStatus: input.ticketCompletionStatus,
577
+ exitCode: input.exitCode,
578
+ awaitingLateReconciliation: input.awaitingLateReconciliation,
579
+ };
580
+ const encoded = JSON.stringify(this._decodeRecoveryPayload(job.id, JSON.stringify(payload)).payload);
581
+ const stage = db.transaction(() => {
582
+ // Works for both a running row and an async pre-spawn admission. The
583
+ // nested promotion transaction removes queued_jobs in the same commit.
584
+ (0, db_1.createJob)(db, {
585
+ id: job.id,
586
+ command: job.command,
587
+ started_at: payload.startedAt,
588
+ provider: payload.provider,
589
+ priority: job.priority,
590
+ depends_on_job_id: input.dependsOnJobId === undefined
591
+ ? job.dependsOnJobId
592
+ : input.dependsOnJobId,
593
+ pipeline_id: job.pipelineId,
594
+ interactive: input.interactive,
595
+ causal_ownership: job.causalOwnership === true,
596
+ });
597
+ (0, db_1.finishJob)(db, job.id, {
598
+ exit_code: input.exitCode ?? -1,
599
+ status: input.status,
600
+ tokens_in: payload.tokensIn ?? undefined,
601
+ tokens_out: payload.tokensOut ?? undefined,
602
+ tokens_cache_read: payload.tokensCacheRead ?? undefined,
603
+ tokens_cache_create: payload.tokensCacheCreate ?? undefined,
604
+ total_cost_usd: payload.totalCostUsd ?? undefined,
605
+ total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
606
+ num_turns: payload.numTurns ?? undefined,
607
+ model: payload.model ?? undefined,
608
+ duration_ms: payload.durationMs ?? undefined,
609
+ duration_api_ms: payload.durationApiMs ?? undefined,
610
+ session_id: payload.sessionId ?? undefined,
611
+ });
612
+ db.prepare(`
613
+ INSERT INTO orphan_job_recovery (
614
+ job_id, payload, accounting_completed, callback_completed, terminal_completed
615
+ ) VALUES (?, ?, ?, ?, ?)
616
+ ON CONFLICT(job_id) DO NOTHING
617
+ `).run(job.id, encoded, input.accountingCompleted || !this._projectId ? 1 : 0, this._onJobFinished ? 0 : 1, input.terminalCompleted ? 1 : 0);
618
+ const intent = db.prepare(`SELECT payload FROM orphan_job_recovery WHERE job_id = ?`)
619
+ .get(job.id);
620
+ if (!intent || intent.payload !== encoded) {
621
+ throw new Error(`A different terminal recovery intent already owns job ${job.id}`);
622
+ }
623
+ const persisted = db.prepare(`SELECT status FROM jobs WHERE id = ?`).get(job.id);
624
+ if (persisted?.status !== input.status) {
625
+ throw new Error(`Failed to persist terminal status ${input.status} for ${job.id}`);
626
+ }
627
+ });
628
+ stage();
629
+ this._terminalPersistenceBlockedJobs.delete(job.id);
630
+ return payload;
631
+ }
632
+ /** Graceful shutdown uses the same terminal intent as ordinary exits. */
633
+ _stageGracefulAbort(job, payload, interactive) {
634
+ this._stageTerminalIntent(job, {
635
+ status: 'failed',
636
+ invocationStatus: 'aborted',
637
+ provider: payload.provider,
638
+ finishedAt: payload.finishedAt,
639
+ exitCode: -1,
640
+ interactive,
641
+ result: {
642
+ tokens_in: payload.tokensIn ?? undefined,
643
+ tokens_out: payload.tokensOut ?? undefined,
644
+ tokens_cache_read: payload.tokensCacheRead ?? undefined,
645
+ tokens_cache_create: payload.tokensCacheCreate ?? undefined,
646
+ total_cost_usd: payload.totalCostUsd ?? undefined,
647
+ total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
648
+ num_turns: payload.numTurns ?? undefined,
649
+ model: payload.model ?? undefined,
650
+ duration_ms: payload.durationMs ?? undefined,
651
+ duration_api_ms: payload.durationApiMs ?? undefined,
652
+ session_id: payload.sessionId ?? undefined,
653
+ },
654
+ descendants: payload.descendants,
655
+ });
656
+ job.status = 'failed';
657
+ job.finishedAt = payload.finishedAt;
658
+ job.exitCode = -1;
396
659
  }
397
660
  /**
398
- * Write an aborted ai_invocations row (+ persist onto the jobs row) for every
399
- * job still in flight when the manager is torn down (shutdown / project
400
- * removal). For a non-interactive rail the cost is estimated from the live
401
- * `adapterEvents` captured so far (foundation contract); for an interactive
402
- * session it is the accumulated per-turn spend plus any folded in-flight turn.
403
- * Called once from shutdown() BEFORE `_db` is nulled. Best-effort per job.
661
+ * Stage every in-flight job before shutdown/project removal, then immediately
662
+ * drain the callback and terminal checkpoints while the DB and project owners
663
+ * are still alive. Any failed critical callback remains durable for restart.
404
664
  */
405
665
  _flushInFlightAccounting() {
406
666
  const db = this._db;
407
667
  const projectId = this._projectId;
408
668
  if (!db || !projectId)
409
- return;
669
+ return true;
670
+ let staged = true;
410
671
  // ── Active non-interactive rail ──────────────────────────────────────────
411
672
  const activeJobId = this._activeJobId;
412
- if (activeJobId) {
673
+ if (activeJobId && !this._interactiveSessions.has(activeJobId)) {
413
674
  const job = this._jobs.get(activeJobId);
414
675
  if (job && job.status === 'running') {
415
676
  try {
@@ -419,39 +680,30 @@ class QueueManager {
419
680
  : { result: {}, estimated: false };
420
681
  const provider = live?.adapter.id ?? this._jobResolvedProvider.get(activeJobId) ?? this._adapter.id;
421
682
  const finishedAt = new Date().toISOString();
422
- job.status = 'failed';
423
- job.finishedAt = finishedAt;
424
- try {
425
- (0, db_1.finishJob)(db, activeJobId, {
426
- exit_code: -1,
427
- status: 'failed',
428
- tokens_in: normalised.tokens_in,
429
- tokens_out: normalised.tokens_out,
430
- tokens_cache_read: normalised.tokens_cache_read,
431
- tokens_cache_create: normalised.tokens_cache_create,
432
- total_cost_usd: normalised.total_cost_usd,
433
- total_cost_usd_estimated: estimated,
434
- num_turns: normalised.num_turns,
435
- model: normalised.model,
436
- duration_ms: normalised.duration_ms,
437
- duration_api_ms: normalised.duration_api_ms,
438
- session_id: normalised.session_id,
439
- });
440
- }
441
- catch { /* DB may be mid-close */ }
442
- this._recordJobInvocations({
443
- jobId: activeJobId,
444
- provider,
445
- status: 'aborted',
683
+ this._stageGracefulAbort(job, {
684
+ id: activeJobId,
685
+ command: job.command,
686
+ ticketIds: this._extractTicketIds(job.command),
687
+ pipelineId: job.pipelineId,
446
688
  startedAt: job.startedAt ?? finishedAt,
447
689
  finishedAt,
448
- ticketIds: this._extractTicketIds(job.command),
449
- estimated,
450
- result: normalised,
451
- });
690
+ provider,
691
+ model: normalised.model ?? null,
692
+ tokensIn: normalised.tokens_in ?? null,
693
+ tokensOut: normalised.tokens_out ?? null,
694
+ tokensCacheRead: normalised.tokens_cache_read ?? null,
695
+ tokensCacheCreate: normalised.tokens_cache_create ?? null,
696
+ totalCostUsd: normalised.total_cost_usd ?? null,
697
+ totalCostUsdEstimated: estimated ? 1 : 0,
698
+ numTurns: normalised.num_turns ?? null,
699
+ durationMs: normalised.duration_ms ?? null,
700
+ durationApiMs: normalised.duration_api_ms ?? null,
701
+ sessionId: normalised.session_id ?? null,
702
+ }, false);
452
703
  this._broadcast({ type: 'spending.invalidated', projectId });
453
704
  }
454
705
  catch (err) {
706
+ staged = false;
455
707
  console.error('[queue-manager] shutdown flush (active job) failed:', err);
456
708
  }
457
709
  }
@@ -464,41 +716,55 @@ class QueueManager {
464
716
  try {
465
717
  const snap = session.snapshotForAbort();
466
718
  const finishedAt = new Date().toISOString();
467
- job.status = 'failed';
468
- job.finishedAt = finishedAt;
469
- try {
470
- (0, db_1.finalizeInteractiveJob)(db, jobId, 'failed');
471
- }
472
- catch { /* DB may be mid-close */ }
473
- this._recordJobInvocations({
474
- jobId,
475
- provider: 'claude',
476
- status: 'aborted',
719
+ const provider = this._jobResolvedProvider.get(jobId) ?? 'claude';
720
+ this._stageGracefulAbort(job, {
721
+ id: jobId,
722
+ command: job.command,
723
+ ticketIds: this._extractTicketIds(job.command),
724
+ pipelineId: job.pipelineId,
477
725
  startedAt: job.startedAt ?? finishedAt,
478
726
  finishedAt,
479
- ticketIds: this._extractTicketIds(job.command),
480
- estimated: snap.estimated,
481
- result: {
482
- tokens_in: snap.totals.tokens_in,
483
- tokens_out: snap.totals.tokens_out,
484
- tokens_cache_read: snap.totals.tokens_cache_read,
485
- tokens_cache_create: snap.totals.tokens_cache_create,
486
- total_cost_usd: snap.totals.total_cost_usd,
487
- num_turns: snap.totals.num_turns,
488
- model: snap.model ?? undefined,
489
- session_id: snap.sessionId ?? undefined,
490
- duration_ms: snap.activeDurationMs,
491
- },
492
- });
727
+ provider,
728
+ model: snap.model ?? null,
729
+ tokensIn: snap.totals.tokens_in,
730
+ tokensOut: snap.totals.tokens_out,
731
+ tokensCacheRead: snap.totals.tokens_cache_read,
732
+ tokensCacheCreate: snap.totals.tokens_cache_create,
733
+ totalCostUsd: snap.totals.total_cost_usd,
734
+ totalCostUsdEstimated: snap.estimated ? 1 : 0,
735
+ numTurns: snap.totals.num_turns,
736
+ durationMs: snap.activeDurationMs,
737
+ durationApiMs: null,
738
+ sessionId: snap.sessionId ?? null,
739
+ }, true);
493
740
  this._broadcast({ type: 'spending.invalidated', projectId });
494
741
  }
495
742
  catch (err) {
743
+ staged = false;
496
744
  console.error('[queue-manager] shutdown flush (interactive job) failed:', err);
497
745
  }
498
746
  }
747
+ // Complete local ticket/rail and dependent invariants now (important for
748
+ // project removal, where this DB may never reopen). Failed critical effects
749
+ // stay in orphan_job_recovery and are replayed on the next project load.
750
+ try {
751
+ this._resumeOrphanRecoveries();
752
+ }
753
+ catch (err) {
754
+ console.error('[queue-manager] shutdown recovery replay failed:', err);
755
+ }
756
+ return staged;
499
757
  }
500
758
  // ─── Public API ─────────────────────────────────────────────────────────────
501
- enqueue(command, priorityOrOpts, opts) {
759
+ _assertMutable() {
760
+ if (this._disposed)
761
+ throw new Error('Queue manager is shutting down');
762
+ }
763
+ enqueue(command, priorityOrOpts, opts, durableAdmission) {
764
+ this._assertMutable();
765
+ if (this._restoreBlocked) {
766
+ throw new Error('Queue recovery is incomplete; retry resume after repairing storage');
767
+ }
502
768
  // Support both: enqueue(cmd, priority, opts) and enqueue(cmd, opts)
503
769
  let priority = 'normal';
504
770
  let resolvedOpts = opts;
@@ -508,6 +774,20 @@ class QueueManager {
508
774
  else if (priorityOrOpts && typeof priorityOrOpts === 'object') {
509
775
  resolvedOpts = priorityOrOpts;
510
776
  }
777
+ const rawDependsOnJobId = resolvedOpts?.dependsOnJobId;
778
+ let dependsOnJobId = null;
779
+ if (rawDependsOnJobId !== undefined) {
780
+ if (typeof rawDependsOnJobId !== 'string' || !rawDependsOnJobId.trim()) {
781
+ throw new InvalidJobDependencyError('dependsOnJobId must be a non-empty string');
782
+ }
783
+ dependsOnJobId = rawDependsOnJobId.trim();
784
+ const parentStatus = this._getDependencyStatus(dependsOnJobId);
785
+ if (parentStatus &&
786
+ TERMINAL_STATUSES.has(parentStatus) &&
787
+ parentStatus !== 'completed') {
788
+ throw new InvalidJobDependencyError(`Cannot depend on job ${dependsOnJobId} because it is ${parentStatus}`);
789
+ }
790
+ }
511
791
  // Resolve the adapter for THIS job: the per-job provider override when set
512
792
  // and installed, else the project's primary provider. The binary check
513
793
  // below probes the chosen provider's CLI.
@@ -526,7 +806,19 @@ class QueueManager {
526
806
  // none has been declared; the adapter's id surfaces in the error.
527
807
  throw new Error(`${enqueueAdapter.binary} binary not found`);
528
808
  }
529
- const id = (0, ids_1.newId)();
809
+ if ((durableAdmission || this._onJobAdmission) && !this._db) {
810
+ throw new Error('Durable enqueue admission requires a project database');
811
+ }
812
+ const id = durableAdmission?.jobId ?? (0, ids_1.newId)();
813
+ if (typeof id !== 'string' || !id.trim()) {
814
+ throw new Error('Durable enqueue admission requires a non-empty jobId');
815
+ }
816
+ if (this._jobs.has(id)) {
817
+ throw new Error(`Job ${id} is already admitted`);
818
+ }
819
+ if (dependsOnJobId === id) {
820
+ throw new InvalidJobDependencyError('A job cannot depend on itself');
821
+ }
530
822
  const job = {
531
823
  id,
532
824
  command,
@@ -536,21 +828,22 @@ class QueueManager {
536
828
  startedAt: null,
537
829
  finishedAt: null,
538
830
  exitCode: null,
539
- dependsOnJobId: resolvedOpts?.dependsOnJobId ?? null,
831
+ dependsOnJobId,
540
832
  pipelineId: resolvedOpts?.pipelineId ?? null,
541
833
  skipReason: null,
542
834
  resultText: null,
835
+ causalOwnership: false,
543
836
  };
544
837
  this._jobs.set(id, job);
545
838
  // Record profile selection (if provided) so spawn time can pick it up.
546
839
  // `undefined` means "use default resolution"; `null` means "force legacy".
547
- if (resolvedOpts && 'profileName' in resolvedOpts) {
548
- this._jobProfileSelection.set(id, resolvedOpts.profileName ?? null);
549
- }
550
- // Record per-job provider override so _startJob resolves the right adapter.
551
- if (resolvedOpts?.provider) {
552
- this._jobProviderSelection.set(id, resolvedOpts.provider);
840
+ if (resolvedOpts?.profileName !== undefined) {
841
+ this._jobProfileSelection.set(id, resolvedOpts.profileName);
553
842
  }
843
+ // Pin the adapter resolved at admission even when no override was supplied.
844
+ // A restart or a long pre-spawn await must not reinterpret this job under a
845
+ // project default that changed after the user submitted it.
846
+ this._jobProviderSelection.set(id, enqueueAdapter.id);
554
847
  // Record per-job model override (e.g. freestyle model picker).
555
848
  if (resolvedOpts?.model) {
556
849
  this._jobModelSelection.set(id, resolvedOpts.model);
@@ -574,12 +867,40 @@ class QueueManager {
574
867
  }
575
868
  this._queue.splice(insertIdx, 0, id);
576
869
  this._recomputePositions();
577
- this._persistJob(job);
870
+ try {
871
+ // Admission is not successful until it is durable. A best-effort write
872
+ // here would acknowledge work that disappears on the next process crash.
873
+ if (durableAdmission || this._onJobAdmission) {
874
+ const db = this._db;
875
+ const commitAdmission = db.transaction(() => {
876
+ this._persistQueuedState(true);
877
+ this._onJobAdmission?.(db, job);
878
+ durableAdmission?.commit(db, job);
879
+ });
880
+ commitAdmission();
881
+ }
882
+ else {
883
+ this._persistQueuedState(true);
884
+ }
885
+ }
886
+ catch (err) {
887
+ const idx = this._queue.indexOf(id);
888
+ if (idx !== -1)
889
+ this._queue.splice(idx, 1);
890
+ this._jobs.delete(id);
891
+ this._jobProfileSelection.delete(id);
892
+ this._jobProviderSelection.delete(id);
893
+ this._jobModelSelection.delete(id);
894
+ this._jobInteractiveSelection.delete(id);
895
+ this._recomputePositions();
896
+ throw err;
897
+ }
578
898
  this._broadcastQueueState();
579
899
  this._drainQueue();
580
900
  return job;
581
901
  }
582
902
  cancel(jobId) {
903
+ this._assertMutable();
583
904
  const job = this._jobs.get(jobId);
584
905
  if (!job) {
585
906
  throw new JobNotFoundError();
@@ -587,38 +908,20 @@ class QueueManager {
587
908
  if (TERMINAL_STATUSES.has(job.status)) {
588
909
  throw new JobAlreadyTerminalError();
589
910
  }
590
- if (job.status === 'queued') {
591
- const idx = this._queue.indexOf(jobId);
592
- if (idx !== -1) {
593
- this._queue.splice(idx, 1);
594
- }
595
- job.status = 'canceled';
596
- job.finishedAt = new Date().toISOString();
597
- // B47: a queued job's per-job selection entries are consumed only when the
598
- // job STARTS (_resolveJobAdapter et al.). Cancelling it while queued means
599
- // it never starts, so drop them here to avoid leaking map entries forever.
600
- this._jobProviderSelection.delete(jobId);
601
- this._jobResolvedProvider.delete(jobId);
602
- this._jobModelSelection.delete(jobId);
603
- this._jobProfileSelection.delete(jobId);
604
- this._jobInteractiveSelection.delete(jobId);
605
- this._skipDependents(jobId, `Parent job ${jobId} was canceled`);
606
- this._recomputePositions();
607
- this._persistJob(job);
608
- this._broadcastQueueState();
609
- // M20: a queued cancel never reached _onJobFinished (only the running path
610
- // does, via _kill→_onJobExit), so a rail-launched queued job left its
611
- // railJobs entry stuck 'running' forever and dropped the job.canceled
612
- // webhook + rail.job_completed broadcast. Fire the callback here too; it is
613
- // exit-status-driven and idempotent on tickets.
614
- if (this._onJobFinished) {
615
- try {
616
- this._onJobFinished(jobId, 'canceled', undefined);
617
- }
618
- catch (err) {
619
- console.error(`[QueueManager] onJobFinished(canceled) failed for ${jobId}: ${err.message}`);
620
- }
621
- }
911
+ if (this._terminalPersistenceBlockedJobs.has(jobId)) {
912
+ throw new Error(`Job ${jobId} is awaiting durable terminal recovery`);
913
+ }
914
+ // `_startJob` marks the in-memory job running before its async plugin
915
+ // verification finishes, while the durable admission intentionally remains
916
+ // in queued_jobs until spawn. Treat that reserved/no-child window as
917
+ // cancelable pre-start work: a Set-only cancel intent would be lost by a
918
+ // process crash and the command would run after restart.
919
+ const isPreparingToSpawn = job.status === 'running' &&
920
+ this._activeJobId === jobId &&
921
+ this._activeProcess === null &&
922
+ !this._interactiveSessions.has(jobId);
923
+ if (job.status === 'queued' || isPreparingToSpawn) {
924
+ this._cancelBeforeSpawn(job, isPreparingToSpawn);
622
925
  return 'canceled';
623
926
  }
624
927
  // job.status === 'running'
@@ -634,21 +937,233 @@ class QueueManager {
634
937
  this._kill(jobId);
635
938
  return 'canceling';
636
939
  }
940
+ /**
941
+ * Cancel queued or asynchronously-preparing work as one durable state change.
942
+ * The parent cancellation, recursive dependent skips, queued-row removals and
943
+ * the surviving queue positions commit together. In-memory state is restored
944
+ * exactly when any write fails, so the API never reports a cancellation whose
945
+ * dependency chain can later resurrect from a partial SQLite commit.
946
+ */
947
+ _cancelBeforeSpawn(job, wasPreparingToSpawn) {
948
+ const previousQueue = [...this._queue];
949
+ const previousActiveJobId = this._activeJobId;
950
+ const snapshots = new Map();
951
+ const snapshot = (candidate) => {
952
+ if (snapshots.has(candidate.id))
953
+ return;
954
+ snapshots.set(candidate.id, {
955
+ status: candidate.status,
956
+ queuePosition: candidate.queuePosition,
957
+ startedAt: candidate.startedAt,
958
+ finishedAt: candidate.finishedAt,
959
+ exitCode: candidate.exitCode,
960
+ skipReason: candidate.skipReason,
961
+ });
962
+ };
963
+ snapshot(job);
964
+ const canceledAt = new Date().toISOString();
965
+ const parentIndex = this._queue.indexOf(job.id);
966
+ if (parentIndex !== -1)
967
+ this._queue.splice(parentIndex, 1);
968
+ job.status = 'canceled';
969
+ job.finishedAt = canceledAt;
970
+ job.exitCode = null;
971
+ job.queuePosition = null;
972
+ const skipped = [];
973
+ const stageDependentSkips = (parentJobId, reason) => {
974
+ const children = Array.from(this._jobs.values()).filter((candidate) => candidate.dependsOnJobId === parentJobId && candidate.status === 'queued');
975
+ for (const child of children) {
976
+ snapshot(child);
977
+ const index = this._queue.indexOf(child.id);
978
+ if (index !== -1)
979
+ this._queue.splice(index, 1);
980
+ child.status = 'skipped';
981
+ child.finishedAt = canceledAt;
982
+ child.queuePosition = null;
983
+ child.skipReason = reason;
984
+ skipped.push(child);
985
+ stageDependentSkips(child.id, `Parent job ${child.id} was skipped`);
986
+ }
987
+ };
988
+ stageDependentSkips(job.id, `Parent job ${job.id} was canceled`);
989
+ this._recomputePositions();
990
+ try {
991
+ if (this._db) {
992
+ const db = this._db;
993
+ const persistCancellation = db.transaction(() => {
994
+ // Persist a canceled tombstone even for work that never reached the
995
+ // provider. Besides making the cancellation auditable, this satisfies
996
+ // jobs.depends_on_job_id while descendant skipped rows are inserted;
997
+ // using canceledAt for both timestamps truthfully records zero runtime.
998
+ const persistedParent = job.dependsOnJobId
999
+ ? db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(job.dependsOnJobId)
1000
+ : null;
1001
+ this._stageTerminalIntent(job, {
1002
+ status: 'canceled',
1003
+ invocationStatus: 'aborted',
1004
+ provider: this._jobResolvedProvider.get(job.id)
1005
+ ?? this._jobProviderSelection.get(job.id)
1006
+ ?? this._adapter.id,
1007
+ finishedAt: canceledAt,
1008
+ exitCode: -1,
1009
+ accountingCompleted: true,
1010
+ // A not-yet-started parent lives only in queued_jobs and therefore
1011
+ // cannot satisfy jobs.depends_on_job_id's FK. Terminal history does
1012
+ // not need to retain that unresolved edge; descendants of THIS job
1013
+ // still reference the canceled tombstone inserted here.
1014
+ dependsOnJobId: persistedParent ? job.dependsOnJobId : null,
1015
+ descendants: skipped.map((child) => ({
1016
+ id: child.id,
1017
+ command: child.command,
1018
+ parentId: child.dependsOnJobId ?? job.id,
1019
+ pipelineId: child.pipelineId,
1020
+ priority: child.priority,
1021
+ causalOwnership: child.causalOwnership === true,
1022
+ })),
1023
+ });
1024
+ for (const child of skipped) {
1025
+ const exists = db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(child.id);
1026
+ if (exists) {
1027
+ (0, db_1.skipJob)(db, child.id, child.skipReason ?? `Parent job ${job.id} was canceled`);
1028
+ }
1029
+ else {
1030
+ db.prepare(`INSERT INTO jobs (
1031
+ id, command, started_at, status, skip_reason, finished_at,
1032
+ depends_on_job_id, pipeline_id, causal_ownership
1033
+ ) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?)`).run(child.id, child.command, child.finishedAt ?? canceledAt, child.skipReason, child.finishedAt ?? canceledAt, child.dependsOnJobId, child.pipelineId, child.causalOwnership === true ? 1 : 0);
1034
+ }
1035
+ (0, db_1.deleteQueuedJob)(db, child.id);
1036
+ }
1037
+ // Nested better-sqlite3 transactions use a savepoint; every surviving
1038
+ // position therefore shares the cancellation's outer commit.
1039
+ this._persistQueuedState(true);
1040
+ });
1041
+ persistCancellation();
1042
+ }
1043
+ }
1044
+ catch (err) {
1045
+ this._queue = previousQueue;
1046
+ this._activeJobId = previousActiveJobId;
1047
+ for (const [id, previous] of snapshots) {
1048
+ const candidate = this._jobs.get(id);
1049
+ if (candidate)
1050
+ Object.assign(candidate, previous);
1051
+ }
1052
+ this._recomputePositions();
1053
+ throw err;
1054
+ }
1055
+ if (wasPreparingToSpawn && this._activeJobId === job.id) {
1056
+ // Invalidates every `_canContinueStart` check when the pending await
1057
+ // resolves; no provider process can be created after this point.
1058
+ this._activeJobId = null;
1059
+ }
1060
+ this._cancelingJobs.delete(job.id);
1061
+ const terminalIds = [job.id, ...skipped.map((child) => child.id)];
1062
+ for (const id of terminalIds) {
1063
+ this._jobProviderSelection.delete(id);
1064
+ this._jobResolvedProvider.delete(id);
1065
+ this._jobModelSelection.delete(id);
1066
+ this._jobProfileSelection.delete(id);
1067
+ this._jobInteractiveSelection.delete(id);
1068
+ this._jobExecution.delete(id);
1069
+ this._snapshotRefs.delete(id);
1070
+ this._jobPrDelivery.delete(id);
1071
+ this._jobLiveAccounting.delete(id);
1072
+ this._cleanupOpenspecShim(id);
1073
+ }
1074
+ let accountingReady = true;
1075
+ if (this._db) {
1076
+ accountingReady = this._resumeOrphanRecoveries();
1077
+ }
1078
+ else {
1079
+ for (const child of skipped) {
1080
+ try {
1081
+ this._onJobFinished?.(child.id, 'skipped', undefined);
1082
+ }
1083
+ catch (err) {
1084
+ console.error(`[QueueManager] onJobFinished(skipped) failed for ${child.id}: ${err.message}`);
1085
+ }
1086
+ }
1087
+ try {
1088
+ this._onJobFinished?.(job.id, 'canceled', undefined);
1089
+ }
1090
+ catch (err) {
1091
+ console.error(`[QueueManager] onJobFinished(canceled) failed for ${job.id}: ${err.message}`);
1092
+ }
1093
+ const affectedPipelines = new Set();
1094
+ if (job.pipelineId)
1095
+ affectedPipelines.add(job.pipelineId);
1096
+ for (const child of skipped) {
1097
+ if (child.pipelineId)
1098
+ affectedPipelines.add(child.pipelineId);
1099
+ }
1100
+ for (const pipelineId of affectedPipelines)
1101
+ this._checkPipelineStatus(pipelineId);
1102
+ }
1103
+ if (!accountingReady) {
1104
+ this._paused = true;
1105
+ this._persistQueueState();
1106
+ }
1107
+ this._broadcastQueueState();
1108
+ if (accountingReady)
1109
+ this._drainQueue();
1110
+ }
637
1111
  pause() {
1112
+ this._assertMutable();
638
1113
  this._paused = true;
639
1114
  this._persistQueueState();
640
1115
  this._broadcastQueueState();
641
1116
  }
642
1117
  resume() {
1118
+ this._assertMutable();
1119
+ if (this._restoreBlocked) {
1120
+ this._restoreFromDb();
1121
+ if (this._restoreBlocked) {
1122
+ this._paused = true;
1123
+ this._persistQueueState();
1124
+ this._broadcastQueueState();
1125
+ return;
1126
+ }
1127
+ }
1128
+ for (const [jobId, pending] of Array.from(this._pendingLateReconciliations)) {
1129
+ this._reconcileForceFailedJobExit(jobId, pending.code, pending.adapterEvents, pending.adapter, pending.spawnedModel);
1130
+ }
1131
+ if (this._pendingLateReconciliations.size > 0) {
1132
+ this._paused = true;
1133
+ this._persistQueueState();
1134
+ this._broadcastQueueState();
1135
+ return;
1136
+ }
1137
+ if (this._terminalPersistenceBlockedJobs.size > 0) {
1138
+ this._paused = true;
1139
+ this._persistQueueState();
1140
+ this._broadcastQueueState();
1141
+ return;
1142
+ }
1143
+ if (!this._resumeOrphanRecoveries()) {
1144
+ // A provider run whose ledger is still pending must remain ahead of new
1145
+ // admissions; otherwise budget enforcement can undercount spend.
1146
+ this._paused = true;
1147
+ this._persistQueueState();
1148
+ this._broadcastQueueState();
1149
+ return;
1150
+ }
643
1151
  this._paused = false;
644
1152
  this._persistQueueState();
645
1153
  this._broadcastQueueState();
646
1154
  this._drainQueue();
647
1155
  }
648
1156
  reorder(jobIds) {
1157
+ this._assertMutable();
1158
+ if (jobIds.some((id) => typeof id !== 'string')) {
1159
+ throw new Error('jobIds must contain only string IDs');
1160
+ }
1161
+ if (jobIds.length !== this._queue.length) {
1162
+ throw new Error('jobIds must contain exactly the IDs of all currently-queued jobs');
1163
+ }
649
1164
  const queuedSet = new Set(this._queue);
650
1165
  const incomingSet = new Set(jobIds);
651
- if (queuedSet.size !== incomingSet.size) {
1166
+ if (incomingSet.size !== jobIds.length || queuedSet.size !== this._queue.length) {
652
1167
  throw new Error('jobIds must contain exactly the IDs of all currently-queued jobs');
653
1168
  }
654
1169
  for (const id of jobIds) {
@@ -656,25 +1171,37 @@ class QueueManager {
656
1171
  throw new Error(`Job ${id} is not in queued state`);
657
1172
  }
658
1173
  }
1174
+ for (let index = 1; index < jobIds.length; index += 1) {
1175
+ const previous = this._jobs.get(jobIds[index - 1]);
1176
+ const current = this._jobs.get(jobIds[index]);
1177
+ if (previous && current &&
1178
+ types_1.PRIORITY_WEIGHT[previous.priority] < types_1.PRIORITY_WEIGHT[current.priority]) {
1179
+ throw new Error('Cannot reorder jobs across priority levels; update priority first');
1180
+ }
1181
+ }
1182
+ const previousQueue = [...this._queue];
659
1183
  this._queue = [...jobIds];
660
1184
  this._recomputePositions();
661
- if (this._db) {
662
- for (const id of jobIds) {
663
- const job = this._jobs.get(id);
664
- if (job) {
665
- this._persistJob(job);
666
- }
667
- }
1185
+ try {
1186
+ this._persistQueuedState(true);
1187
+ }
1188
+ catch (err) {
1189
+ this._queue = previousQueue;
1190
+ this._recomputePositions();
1191
+ throw err;
668
1192
  }
669
1193
  this._broadcastQueueState();
670
1194
  }
671
1195
  updatePriority(jobId, priority) {
1196
+ this._assertMutable();
672
1197
  const job = this._jobs.get(jobId);
673
1198
  if (!job)
674
1199
  throw new JobNotFoundError();
675
1200
  if (job.status !== 'queued') {
676
1201
  throw new Error('Can only change priority of queued jobs');
677
1202
  }
1203
+ const previousPriority = job.priority;
1204
+ const previousQueue = [...this._queue];
678
1205
  job.priority = priority;
679
1206
  // Remove from queue and re-insert at correct position
680
1207
  const idx = this._queue.indexOf(jobId);
@@ -691,7 +1218,15 @@ class QueueManager {
691
1218
  }
692
1219
  this._queue.splice(insertIdx, 0, jobId);
693
1220
  this._recomputePositions();
694
- this._persistJob(job);
1221
+ try {
1222
+ this._persistQueuedState(true);
1223
+ }
1224
+ catch (err) {
1225
+ job.priority = previousPriority;
1226
+ this._queue = previousQueue;
1227
+ this._recomputePositions();
1228
+ throw err;
1229
+ }
695
1230
  this._broadcastQueueState();
696
1231
  }
697
1232
  getJobs() {
@@ -721,6 +1256,7 @@ class QueueManager {
721
1256
  * active turn). Returns false when the job isn't an active interactive
722
1257
  * session (unknown / already finalized / not interactive). */
723
1258
  sendInteractiveTurn(jobId, text) {
1259
+ this._assertMutable();
724
1260
  const session = this._interactiveSessions.get(jobId);
725
1261
  if (!session)
726
1262
  return false;
@@ -730,6 +1266,7 @@ class QueueManager {
730
1266
  * the settle path stamps the summed totals + 'completed' status. Returns false
731
1267
  * when the job isn't an active interactive session. */
732
1268
  finalizeInteractive(jobId) {
1269
+ this._assertMutable();
733
1270
  const session = this._interactiveSessions.get(jobId);
734
1271
  if (!session)
735
1272
  return false;
@@ -978,11 +1515,39 @@ class QueueManager {
978
1515
  return;
979
1516
  if (this._queue.length === 0)
980
1517
  return;
1518
+ // Defense in depth for state restored from old/corrupt builds or mutated by
1519
+ // an invalid caller: only unique, presently-queued jobs may reach start.
1520
+ // In particular, a duplicated id must not relaunch after its first run
1521
+ // becomes terminal.
1522
+ const seen = new Set();
1523
+ const normalizedQueue = [];
1524
+ const removedJobIds = [];
1525
+ for (const id of this._queue) {
1526
+ if (seen.has(id))
1527
+ continue;
1528
+ seen.add(id);
1529
+ const candidate = this._jobs.get(id);
1530
+ if (candidate?.status === 'queued')
1531
+ normalizedQueue.push(id);
1532
+ else
1533
+ removedJobIds.push(id);
1534
+ }
1535
+ if (normalizedQueue.length !== this._queue.length) {
1536
+ this._queue = normalizedQueue;
1537
+ this._recomputePositions();
1538
+ this._persistQueuedState(false, removedJobIds);
1539
+ if (this._queue.length === 0)
1540
+ return;
1541
+ }
1542
+ // App/project budgets are durable policies, not merely post-job alerts.
1543
+ // Re-check before reserving a slot so a different project that crossed the
1544
+ // app-wide cap cannot be followed by a fresh spawn from this queue.
1545
+ this._enforceDailyBudget();
1546
+ if (this._paused)
1547
+ return;
981
1548
  const readyIndex = this._queue.findIndex(id => {
982
1549
  const job = this._jobs.get(id);
983
- if (!job)
984
- return true;
985
- return this._isDependencyMet(job);
1550
+ return job?.status === 'queued' && this._isDependencyMet(job);
986
1551
  });
987
1552
  if (readyIndex === -1)
988
1553
  return;
@@ -995,6 +1560,9 @@ class QueueManager {
995
1560
  // clobbered so cancel/zombie-kill hits the wrong child.
996
1561
  this._activeJobId = nextJobId;
997
1562
  this._recomputePositions();
1563
+ // Keep every still-queued position in sync while preserving the selected
1564
+ // job's durable admission until createJob promotes it just before spawn.
1565
+ this._persistQueuedState(false, [], nextJobId);
998
1566
  void this._startJob(nextJobId).catch((err) => {
999
1567
  console.error(`[QueueManager] _startJob(${nextJobId}) threw before spawn: ${err?.message}`);
1000
1568
  // Only release if we never established a child (else _onJobExit owns cleanup).
@@ -1002,9 +1570,10 @@ class QueueManager {
1002
1570
  // Stamp the job terminal — _onJobExit never runs without a child, so the
1003
1571
  // job would otherwise wedge 'running' forever and never fire
1004
1572
  // onJobFinished (rail/webhook never settle) and leak its per-job maps.
1005
- this._failWedgedJob(nextJobId, err?.message ?? 'startup failure');
1006
1573
  this._activeJobId = null;
1007
- this._drainQueue();
1574
+ const terminalPersisted = this._failWedgedJob(nextJobId, err?.message ?? 'startup failure');
1575
+ if (terminalPersisted)
1576
+ this._drainQueue();
1008
1577
  }
1009
1578
  });
1010
1579
  }
@@ -1016,54 +1585,105 @@ class QueueManager {
1016
1585
  */
1017
1586
  _failWedgedJob(jobId, reason) {
1018
1587
  const job = this._jobs.get(jobId);
1588
+ const previousJobState = job ? {
1589
+ status: job.status,
1590
+ queuePosition: job.queuePosition,
1591
+ startedAt: job.startedAt,
1592
+ finishedAt: job.finishedAt,
1593
+ exitCode: job.exitCode,
1594
+ skipReason: job.skipReason,
1595
+ } : null;
1019
1596
  // The wedge can land before _startJob set status='running' (an early throw in
1020
1597
  // execution/agent resolution) OR after it. Either way the job is terminal-
1021
1598
  // failed now — capture a finished_at to drive the invocation row regardless.
1022
1599
  const finishedAt = new Date().toISOString();
1023
- if (job && job.status === 'running') {
1600
+ if (job && !TERMINAL_STATUSES.has(job.status)) {
1024
1601
  job.status = 'failed';
1602
+ job.startedAt ??= finishedAt;
1025
1603
  job.finishedAt = finishedAt;
1026
1604
  }
1605
+ // Side effects are valid only after the terminal row and queued-admission
1606
+ // removal commit together. In particular, a failed promotion must leave
1607
+ // queued_jobs intact for restart instead of manufacturing accounting and a
1608
+ // completion callback for a terminal state SQLite never accepted.
1609
+ let terminalPersisted = !this._db && !!job;
1027
1610
  if (this._db) {
1028
1611
  try {
1029
- (0, db_1.finishJob)(this._db, jobId, { exit_code: -1, status: 'failed' });
1612
+ if (!job)
1613
+ throw new Error(`Missing in-memory job ${jobId}`);
1614
+ const durationMs = job.startedAt
1615
+ ? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
1616
+ : undefined;
1617
+ this._stageTerminalIntent(job, {
1618
+ status: 'failed',
1619
+ invocationStatus: 'failed',
1620
+ provider: this._jobResolvedProvider.get(jobId)
1621
+ ?? this._jobProviderSelection.get(jobId)
1622
+ ?? this._adapter.id,
1623
+ finishedAt,
1624
+ exitCode: -1,
1625
+ result: { duration_ms: durationMs },
1626
+ });
1627
+ terminalPersisted = true;
1030
1628
  }
1031
- catch {
1032
- /* DB may be closed mid-shutdown — never throw from the drain catch */
1033
- }
1034
- // ai_invocations capture (surface='job', failed) so a startup-failed job
1035
- // still counts toward totalRuns/failureRate on Analytics. _onJobExit never
1036
- // runs without a child, so this is the ONLY place the row can be written.
1037
- // No token/cost data was ever finalised (the spawn never produced output).
1038
- // BUG-ANALYTICS-01's limitation applies to the provider stamp here too: by
1039
- // this point the per-job override is consumed, so we read the resolved
1040
- // provider captured at _startJob (set right after adapter resolution; may
1041
- // be absent if the throw preceded it) with _adapter.id as the fallback.
1042
- if (this._projectId && job) {
1629
+ catch (err) {
1630
+ // The transaction rollback preserves queued_jobs (or the pre-existing
1631
+ // running row) for durable recovery. Do not delete it in a later
1632
+ // best-effort cleanup and do not publish false terminal side effects.
1633
+ console.error(`[queue-manager] startup failure persistence failed for ${jobId}:`, err);
1634
+ }
1635
+ }
1636
+ if (!terminalPersisted) {
1637
+ let durableQueued = false;
1638
+ if (this._db) {
1043
1639
  try {
1044
- const ticketIds = this._extractTicketIds(job.command);
1045
- const durationMs = job.startedAt
1046
- ? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
1047
- : undefined;
1048
- (0, ai_invocations_1.recordInvocation)(this._db, {
1049
- id: (0, crypto_1.randomUUID)(),
1050
- project_id: this._projectId,
1051
- provider: this._jobResolvedProvider.get(jobId) ?? this._adapter.id,
1052
- surface: 'job',
1053
- surface_ref_id: jobId,
1054
- ticket_id: ticketIds[0] ?? null,
1055
- status: 'failed',
1056
- started_at: job.startedAt ?? finishedAt,
1057
- finished_at: finishedAt,
1058
- total_cost_usd_estimated: false,
1059
- duration_ms: durationMs,
1060
- });
1061
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
1640
+ durableQueued = !!this._db.prepare('SELECT 1 FROM queued_jobs WHERE id = ?').get(jobId);
1062
1641
  }
1063
- catch (err) {
1064
- console.error('[queue-manager] recordInvocation (wedged) failed:', err);
1642
+ catch {
1643
+ // If even the read fails, fail closed below by pausing the queue.
1644
+ }
1645
+ }
1646
+ if (job && previousJobState) {
1647
+ Object.assign(job, previousJobState);
1648
+ if (durableQueued) {
1649
+ // The provider never acquired the admission. Restore the same
1650
+ // pre-execution semantics in memory and put it back at the front, but
1651
+ // pause instead of immediately retrying a persistently-broken DB.
1652
+ job.status = 'queued';
1653
+ job.startedAt = null;
1654
+ job.finishedAt = null;
1655
+ job.exitCode = null;
1656
+ job.skipReason = null;
1657
+ if (!this._queue.includes(jobId))
1658
+ this._queue.unshift(jobId);
1659
+ this._recomputePositions();
1660
+ this._persistQueuedState();
1661
+ }
1662
+ else {
1663
+ // Promotion already consumed queued_jobs, so there is no admission to
1664
+ // retry. Retain ownership of the childless RUNNING row and make
1665
+ // resume fail closed until shutdown/startup recovery stages it.
1666
+ this._activeJobId = jobId;
1667
+ this._terminalPersistenceBlockedJobs.add(jobId);
1065
1668
  }
1066
1669
  }
1670
+ // Make the degraded state visible and stable. A user can explicitly
1671
+ // resume after repairing storage; enqueue cannot silently spin the same
1672
+ // failed promotion in a tight loop.
1673
+ this._paused = true;
1674
+ this._persistQueueState();
1675
+ const resolvedProvider = this._jobResolvedProvider.get(jobId);
1676
+ if (resolvedProvider)
1677
+ this._jobProviderSelection.set(jobId, resolvedProvider);
1678
+ this._jobResolvedProvider.delete(jobId);
1679
+ this._jobExecution.delete(jobId);
1680
+ this._snapshotRefs.delete(jobId);
1681
+ this._jobPrDelivery.delete(jobId);
1682
+ this._jobLiveAccounting.delete(jobId);
1683
+ this._cleanupOpenspecShim(jobId);
1684
+ this._broadcastQueueState();
1685
+ console.error(`[QueueManager] job ${jobId} remains durably ${durableQueued ? 'queued' : 'unsettled'} after startup failure: ${reason}`);
1686
+ return false;
1067
1687
  }
1068
1688
  // Clear per-job selection/snapshot maps (none were consumed by a spawn).
1069
1689
  this._jobExecution.delete(jobId);
@@ -1078,15 +1698,29 @@ class QueueManager {
1078
1698
  // A wedged job may have had its openspec shim materialised already (the
1079
1699
  // wedge can land after _startJob's shim setup). Mirror the settle path.
1080
1700
  this._cleanupOpenspecShim(jobId);
1081
- try {
1082
- this._onJobFinished?.(jobId, 'failed', undefined);
1701
+ let accountingReady = true;
1702
+ if (this._db) {
1703
+ accountingReady = this._resumeOrphanRecoveries();
1704
+ if (!accountingReady) {
1705
+ this._paused = true;
1706
+ this._persistQueueState();
1707
+ }
1083
1708
  }
1084
- catch {
1085
- /* onJobFinished is best-effort */
1709
+ else if (terminalPersisted) {
1710
+ try {
1711
+ this._onJobFinished?.(jobId, 'failed', undefined);
1712
+ }
1713
+ catch {
1714
+ /* onJobFinished is best-effort */
1715
+ }
1716
+ this._skipDependents(jobId, `Parent job ${jobId} failed`);
1717
+ if (job?.pipelineId)
1718
+ this._checkPipelineStatus(job.pipelineId);
1086
1719
  }
1087
1720
  this._persistQueueState();
1088
1721
  this._broadcastQueueState();
1089
1722
  console.error(`[QueueManager] job ${jobId} failed before spawn: ${reason}`);
1723
+ return accountingReady;
1090
1724
  }
1091
1725
  /**
1092
1726
  * Remove the per-job openspec PATH shim (relocated claude rails only) from
@@ -1138,12 +1772,12 @@ class QueueManager {
1138
1772
  }
1139
1773
  /**
1140
1774
  * Resolve the adapter for a job at spawn time: the per-job provider override
1141
- * (consumed from `_jobProviderSelection`) when present and registered, else
1142
- * the project's primary adapter. Consuming the entry keeps the map bounded.
1775
+ * when present and registered, else the project's primary adapter. Pending
1776
+ * selections remain intact until createJob durably promotes the admission;
1777
+ * a pre-promotion failure can therefore be retried without semantic drift.
1143
1778
  */
1144
1779
  _resolveJobAdapter(jobId) {
1145
1780
  const override = this._jobProviderSelection.get(jobId);
1146
- this._jobProviderSelection.delete(jobId);
1147
1781
  if (override) {
1148
1782
  try {
1149
1783
  return (0, providers_1.getAdapter)(override);
@@ -1154,6 +1788,15 @@ class QueueManager {
1154
1788
  }
1155
1789
  return this._adapter;
1156
1790
  }
1791
+ /** Consume restart-durable pre-spawn selections only after queued_jobs has
1792
+ * been atomically promoted to jobs. Runtime state already holds every value
1793
+ * needed by the child at that point. */
1794
+ _consumePendingSelections(jobId) {
1795
+ this._jobProviderSelection.delete(jobId);
1796
+ this._jobModelSelection.delete(jobId);
1797
+ this._jobProfileSelection.delete(jobId);
1798
+ this._jobInteractiveSelection.delete(jobId);
1799
+ }
1157
1800
  /**
1158
1801
  * Resolve the relocate-artifacts execution context for this manager's project.
1159
1802
  * The gate: relocated only when a registry entry exists AND core has populated
@@ -1207,21 +1850,23 @@ class QueueManager {
1207
1850
  */
1208
1851
  _startInteractiveJob(jobId, job, adapter, spec, firstPrompt, settleMode) {
1209
1852
  if (this._db) {
1210
- try {
1211
- (0, db_1.createJob)(this._db, {
1212
- id: jobId,
1213
- command: job.command,
1214
- started_at: job.startedAt,
1215
- priority: job.priority,
1216
- depends_on_job_id: job.dependsOnJobId,
1217
- pipeline_id: job.pipelineId,
1218
- interactive: true,
1219
- });
1220
- }
1221
- catch (err) {
1222
- console.error('[queue-manager] createJob (interactive) failed:', err);
1223
- }
1853
+ // Promotion is the execution boundary. Propagate any failure to
1854
+ // `_drainQueue` before constructing the session: starting a child while
1855
+ // its queued admission remains replayable would duplicate the command on
1856
+ // the next process restart.
1857
+ (0, db_1.createJob)(this._db, {
1858
+ id: jobId,
1859
+ command: job.command,
1860
+ started_at: job.startedAt,
1861
+ provider: adapter.id,
1862
+ priority: job.priority,
1863
+ depends_on_job_id: job.dependsOnJobId,
1864
+ pipeline_id: job.pipelineId,
1865
+ interactive: true,
1866
+ causal_ownership: job.causalOwnership === true,
1867
+ });
1224
1868
  }
1869
+ this._consumePendingSelections(jobId);
1225
1870
  const session = new interactive_job_session_1.InteractiveJobSession({
1226
1871
  jobId,
1227
1872
  projectId: this._projectId ?? '',
@@ -1249,36 +1894,40 @@ class QueueManager {
1249
1894
  * drains the queue.
1250
1895
  */
1251
1896
  _settleInteractiveJob(jobId, info) {
1252
- this._interactiveSessions.delete(jobId);
1253
- if (this._activeJobId === jobId) {
1254
- this._activeProcess = null;
1255
- this._activeJobId = null;
1256
- }
1257
- // Consume the pre-spawn provenance snapshot + the resolved execution
1258
- // context BEFORE any early return (mirrors _onJobExit) so a disposed/
1259
- // unknown-job settle can never leak the map entries.
1897
+ // Read terminal context first, but do not consume it until the durable
1898
+ // terminal intent commits. If storage rejects the transition, shutdown can
1899
+ // retry from the still-owned session instead of losing in-flight usage.
1260
1900
  const snapshot = this._snapshotRefs.get(jobId);
1261
- this._snapshotRefs.delete(jobId);
1262
1901
  const jobExecution = this._jobExecution.get(jobId);
1263
- this._jobExecution.delete(jobId);
1264
1902
  const provenanceRepoDir = jobExecution?.repoDir ?? this._cwd;
1265
- this._jobResolvedProvider.delete(jobId);
1903
+ const provider = this._jobResolvedProvider.get(jobId) ?? 'claude';
1266
1904
  // Consume the spawn-captured PR-delivery mode (before any early return so a
1267
1905
  // disposed/unknown-job settle can never leak the entry). Decides whether a
1268
1906
  // COMPLETED job's tickets park at on_review (ask-first) or done (legacy).
1269
1907
  const prDelivery = this._jobPrDelivery.get(jobId) ?? false;
1270
- this._jobPrDelivery.delete(jobId);
1271
- // Clean up the per-job openspec PATH shim (relocated claude rails only).
1272
- this._cleanupOpenspecShim(jobId);
1273
- if (this._disposed)
1908
+ if (this._disposed) {
1909
+ this._interactiveSessions.delete(jobId);
1910
+ this._snapshotRefs.delete(jobId);
1911
+ this._jobExecution.delete(jobId);
1912
+ this._jobResolvedProvider.delete(jobId);
1913
+ this._jobPrDelivery.delete(jobId);
1914
+ this._cleanupOpenspecShim(jobId);
1274
1915
  return;
1916
+ }
1275
1917
  const job = this._jobs.get(jobId);
1276
1918
  if (!job) {
1919
+ this._interactiveSessions.delete(jobId);
1920
+ this._snapshotRefs.delete(jobId);
1921
+ this._jobExecution.delete(jobId);
1922
+ this._jobResolvedProvider.delete(jobId);
1923
+ this._jobPrDelivery.delete(jobId);
1924
+ this._cleanupOpenspecShim(jobId);
1925
+ if (this._activeJobId === jobId)
1926
+ this._activeJobId = null;
1277
1927
  this._drainQueue();
1278
1928
  return;
1279
1929
  }
1280
1930
  const wasCanceling = this._cancelingJobs.has(jobId);
1281
- this._cancelingJobs.delete(jobId);
1282
1931
  // Zero-work strictness: a session whose WHOLE life consumed no model work
1283
1932
  // (the claude CLI's synthetic `Unknown command:` result frame — num_turns
1284
1933
  // 0, no assistant events, zero usage tokens) settles FAILED even on a
@@ -1291,59 +1940,77 @@ class QueueManager {
1291
1940
  : info.reason === 'finalized' && !info.zeroWork
1292
1941
  ? 'completed'
1293
1942
  : 'failed';
1943
+ const finishedAt = new Date().toISOString();
1944
+ const exitCode = info.reason === 'finalized' && finalStatus !== 'failed' ? 0 : 1;
1945
+ const invStatus = finalStatus === 'completed'
1946
+ ? 'success'
1947
+ : finalStatus === 'canceled'
1948
+ ? 'aborted'
1949
+ : 'failed';
1950
+ const totals = info.totals;
1951
+ const result = {
1952
+ tokens_in: totals.tokens_in,
1953
+ tokens_out: totals.tokens_out,
1954
+ tokens_cache_read: totals.tokens_cache_read,
1955
+ tokens_cache_create: totals.tokens_cache_create,
1956
+ total_cost_usd: totals.total_cost_usd,
1957
+ total_cost_usd_estimated: info.estimated,
1958
+ num_turns: totals.num_turns,
1959
+ model: info.model ?? undefined,
1960
+ session_id: info.sessionId ?? undefined,
1961
+ duration_ms: info.activeDurationMs,
1962
+ };
1963
+ if (this._db) {
1964
+ try {
1965
+ this._stageTerminalIntent(job, {
1966
+ status: finalStatus,
1967
+ invocationStatus: invStatus,
1968
+ provider,
1969
+ finishedAt,
1970
+ exitCode,
1971
+ interactive: true,
1972
+ result,
1973
+ ticketCompletionStatus: finalStatus === 'completed'
1974
+ ? (prDelivery ? 'on_review' : 'done')
1975
+ : undefined,
1976
+ });
1977
+ }
1978
+ catch (err) {
1979
+ // The session has stopped, but its durable row intentionally remains
1980
+ // running. Reserve the slot and pause so neither resume nor enqueue can
1981
+ // overtake an unaccounted terminal transition; startup/shutdown recovery
1982
+ // can safely retry it from that state.
1983
+ this._activeProcess = null;
1984
+ this._activeJobId = jobId;
1985
+ this._terminalPersistenceBlockedJobs.add(jobId);
1986
+ this._paused = true;
1987
+ this._persistQueueState();
1988
+ this._broadcastQueueState();
1989
+ console.error(`[queue-manager] interactive terminal staging failed for ${jobId}:`, err);
1990
+ return;
1991
+ }
1992
+ }
1993
+ this._interactiveSessions.delete(jobId);
1994
+ if (this._activeJobId === jobId) {
1995
+ this._activeProcess = null;
1996
+ this._activeJobId = null;
1997
+ }
1998
+ this._snapshotRefs.delete(jobId);
1999
+ this._jobExecution.delete(jobId);
2000
+ this._jobResolvedProvider.delete(jobId);
2001
+ this._jobPrDelivery.delete(jobId);
2002
+ this._cancelingJobs.delete(jobId);
2003
+ this._cleanupOpenspecShim(jobId);
1294
2004
  job.status = finalStatus;
1295
- job.finishedAt = new Date().toISOString();
1296
- job.exitCode = info.reason === 'finalized' && finalStatus !== 'failed' ? 0 : 1;
2005
+ job.finishedAt = finishedAt;
2006
+ job.exitCode = exitCode;
1297
2007
  // Result text for output chaining between dependent pipeline steps — the
1298
2008
  // same field the one-shot path captures from its last `result` event.
1299
2009
  if (info.resultText != null) {
1300
2010
  job.resultText = info.resultText;
1301
2011
  }
1302
- const totals = info.totals;
2012
+ let accountingReady = true;
1303
2013
  if (this._db) {
1304
- try {
1305
- (0, db_1.finalizeInteractiveJob)(this._db, jobId, finalStatus);
1306
- }
1307
- catch (err) {
1308
- console.error('[queue-manager] finalizeInteractiveJob failed:', err);
1309
- }
1310
- if (this._projectId) {
1311
- try {
1312
- const invStatus = finalStatus === 'completed'
1313
- ? 'success'
1314
- : finalStatus === 'canceled'
1315
- ? 'aborted'
1316
- : 'failed';
1317
- const ticketIds = this._extractTicketIds(job.command);
1318
- this._recordJobInvocations({
1319
- jobId,
1320
- provider: 'claude',
1321
- status: invStatus,
1322
- startedAt: job.startedAt ?? new Date().toISOString(),
1323
- finishedAt: job.finishedAt,
1324
- ticketIds,
1325
- // CRIT-4: true when a mid-turn finalize/crash folded an in-flight
1326
- // turn's rate-card-priced usage into the accumulated totals.
1327
- estimated: info.estimated,
1328
- result: {
1329
- tokens_in: totals.tokens_in,
1330
- tokens_out: totals.tokens_out,
1331
- tokens_cache_read: totals.tokens_cache_read,
1332
- tokens_cache_create: totals.tokens_cache_create,
1333
- total_cost_usd: totals.total_cost_usd,
1334
- num_turns: totals.num_turns,
1335
- model: info.model ?? undefined,
1336
- session_id: info.sessionId ?? undefined,
1337
- // LOW-15: sum of active turn wall-segments, not finished−started.
1338
- duration_ms: info.activeDurationMs,
1339
- },
1340
- });
1341
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
1342
- }
1343
- catch (err) {
1344
- console.error('[queue-manager] recordInvocation (interactive) failed:', err);
1345
- }
1346
- }
1347
2014
  // Code-Explorer post-settle provenance hook — the interactive lifecycle
1348
2015
  // equivalent of _onJobExit's post-exit diff (pre-spawn snapshot taken in
1349
2016
  // _startJob's interactive branch, against the REPO dir, never the
@@ -1355,52 +2022,63 @@ class QueueManager {
1355
2022
  if (totals.total_cost_usd > 0 && finalStatus === 'completed') {
1356
2023
  this._emitCostAlerts(jobId, totals.total_cost_usd);
1357
2024
  }
2025
+ accountingReady = this._resumeOrphanRecoveries();
1358
2026
  this._enforceDailyBudget();
1359
2027
  }
1360
- this._persistJob(job);
1361
- this._broadcast({
1362
- type: 'job.finalized',
1363
- projectId: this._projectId ?? '',
1364
- jobId,
1365
- status: finalStatus,
1366
- totals,
1367
- timestamp: new Date().toISOString(),
1368
- });
1369
- this._broadcastQueueState();
1370
- if (this._onJobFinished) {
2028
+ if (!this._db) {
1371
2029
  try {
1372
2030
  if (finalStatus === 'completed') {
1373
- // Thread the spawn-captured PR-delivery mode: under the ask-first
1374
- // methodology a completed job's tickets park at on_review, never done.
1375
- // Failure statuses keep the legacy 3-arg call shape (the field is
1376
- // completion-only).
1377
- this._onJobFinished(jobId, finalStatus, totals.total_cost_usd, {
2031
+ this._onJobFinished?.(jobId, finalStatus, totals.total_cost_usd, {
1378
2032
  ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
1379
2033
  });
1380
2034
  }
1381
2035
  else {
1382
- this._onJobFinished(jobId, finalStatus, totals.total_cost_usd);
2036
+ this._onJobFinished?.(jobId, finalStatus, totals.total_cost_usd);
1383
2037
  }
1384
2038
  }
1385
2039
  catch (err) {
1386
2040
  console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
1387
2041
  }
2042
+ if (finalStatus !== 'completed') {
2043
+ this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
2044
+ }
2045
+ if (job.pipelineId)
2046
+ this._checkPipelineStatus(job.pipelineId);
1388
2047
  }
1389
- // Dependent-job + pipeline bookkeeping (mirrors _onJobExit) — chained /
1390
- // template jobs flow through the interactive settle path by default now.
1391
- if (finalStatus !== 'completed') {
1392
- this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
1393
- }
1394
- if (job.pipelineId) {
1395
- this._checkPipelineStatus(job.pipelineId);
2048
+ if (!accountingReady) {
2049
+ this._paused = true;
2050
+ this._persistQueueState();
1396
2051
  }
1397
- this._drainQueue();
2052
+ this._persistJob(job);
2053
+ this._broadcast({
2054
+ type: 'job.finalized',
2055
+ projectId: this._projectId ?? '',
2056
+ jobId,
2057
+ status: finalStatus,
2058
+ totals,
2059
+ timestamp: new Date().toISOString(),
2060
+ });
2061
+ this._broadcastQueueState();
2062
+ if (accountingReady)
2063
+ this._drainQueue();
2064
+ }
2065
+ /** True only while this async start still belongs to the live manager and its
2066
+ * synchronously-reserved queue slot. The generation closes the shutdown race;
2067
+ * the slot check also protects against future replacement/cancel paths. */
2068
+ _canContinueStart(jobId, lifecycleGeneration) {
2069
+ return (!this._disposed &&
2070
+ this._lifecycleGeneration === lifecycleGeneration &&
2071
+ this._activeJobId === jobId);
1398
2072
  }
1399
2073
  async _startJob(jobId) {
2074
+ const lifecycleGeneration = this._lifecycleGeneration;
2075
+ if (!this._canContinueStart(jobId, lifecycleGeneration))
2076
+ return;
1400
2077
  const job = this._jobs.get(jobId);
1401
- if (!job) {
2078
+ if (!job || job.status !== 'queued') {
1402
2079
  // Job vanished between the synchronous slot reservation in _drainQueue and
1403
- // here — release the reserved slot and move on (A3).
2080
+ // here (or became non-queued through a defensive race) — release the
2081
+ // reserved slot and move on. Never relaunch a terminal/running id.
1404
2082
  if (this._activeJobId === jobId)
1405
2083
  this._activeJobId = null;
1406
2084
  this._drainQueue();
@@ -1411,9 +2089,13 @@ class QueueManager {
1411
2089
  // plugins, result parsing, ai_invocations.provider) flows from `adapter`.
1412
2090
  const adapter = this._resolveJobAdapter(jobId);
1413
2091
  // Remember the provider this job actually runs on, for terminal paths that
1414
- // fire without a child 'close' (e.g. _forceFailUnkillableJob) where the
1415
- // per-job override has already been consumed from _jobProviderSelection.
2092
+ // fire without a child 'close' (e.g. _forceFailUnkillableJob). The pending
2093
+ // map remains durable until promotion, then this resolved map takes over.
1416
2094
  this._jobResolvedProvider.set(jobId, adapter.id);
2095
+ // Pin the adapter that was actually resolved. If any later pre-promotion
2096
+ // step fails, the durable retry must not switch providers because project
2097
+ // defaults or an invalid override fallback changed in the meantime.
2098
+ this._jobProviderSelection.set(jobId, adapter.id);
1417
2099
  // Relocate-artifacts gate: resolve cwd/repoDir/env for this spawn. Legacy
1418
2100
  // projects get cwd = project.path + empty env (byte-identical to today);
1419
2101
  // relocated projects get cwd = workspace + SPECRAILS_REPO_DIR. Captured per
@@ -1471,11 +2153,10 @@ class QueueManager {
1471
2153
  // today); codex/gemini always take the legacy one-shot spawn below.
1472
2154
  // EnqueueOptions.interactive is a per-job OVERRIDE: false forces legacy,
1473
2155
  // true forces interactive where capable, undefined = default ON. Derived
1474
- // HERE (spawn time), not at enqueue, so a queued job that survives a server
1475
- // restart (selection map lost) still spawns interactive.
2156
+ // HERE (spawn time), not at enqueue, so legacy queued rows with no explicit
2157
+ // selection still receive the current default after restart.
1476
2158
  const isFreestyle = adapter.id === 'claude' && exports.FREESTYLE_COMMAND_RE.test(commandToRun);
1477
2159
  const interactiveOverride = this._jobInteractiveSelection.get(jobId);
1478
- this._jobInteractiveSelection.delete(jobId);
1479
2160
  const spawnInteractive = (0, feature_flags_1.isInteractiveJobsEnabled)() &&
1480
2161
  adapter.capabilities.persistentStdin &&
1481
2162
  (interactiveOverride ?? true);
@@ -1591,7 +2272,6 @@ class QueueManager {
1591
2272
  // Per-job model override (consumed once) takes precedence — used by the
1592
2273
  // freestyle model picker so the user can choose haiku/sonnet/opus per launch.
1593
2274
  const modelOverride = this._jobModelSelection.get(jobId);
1594
- this._jobModelSelection.delete(jobId);
1595
2275
  const railModel = modelOverride
1596
2276
  ? modelOverride
1597
2277
  : adapter.id === 'claude' && this._db
@@ -1622,7 +2302,6 @@ class QueueManager {
1622
2302
  if (adapter.capabilities.profileEnvSupport && this._projectId && this._projectSlug && this._cwd) {
1623
2303
  try {
1624
2304
  const selection = this._jobProfileSelection.get(jobId); // undefined|null|string
1625
- this._jobProfileSelection.delete(jobId);
1626
2305
  // When relocated, core + `.specrails/profiles` live in the workspace
1627
2306
  // (execution.cwd); legacy reads from the repo (execution.cwd === repo).
1628
2307
  const coreSupports = projectSupportsProfiles(execution.cwd);
@@ -1699,13 +2378,27 @@ class QueueManager {
1699
2378
  let pluginSnapshotPath = null;
1700
2379
  if (adapter.mcpRegistration === 'project-json' && this._projectId && this._projectSlug && this._cwd) {
1701
2380
  try {
1702
- const { resolvePluginsForSpawn, snapshotPluginsForJob } = require('./plugins/rail-integration');
2381
+ let resolver = this._resolvePluginsForSpawn;
2382
+ let snapshotter = null;
2383
+ if (!resolver) {
2384
+ const pluginIntegration = require('./plugins/rail-integration');
2385
+ resolver = pluginIntegration.resolvePluginsForSpawn;
2386
+ snapshotter = pluginIntegration.snapshotPluginsForJob;
2387
+ }
1703
2388
  // Relocated ⇒ `.mcp.json`/plugin state live in the workspace (execution.cwd).
1704
- const resolution = await resolvePluginsForSpawn(execution.cwd, this._projectId, jobId);
2389
+ const resolution = await resolver(execution.cwd, this._projectId, jobId);
2390
+ // shutdown() may have run while plugin verification was awaiting. Never
2391
+ // snapshot, broadcast, or spawn for a manager generation that no longer
2392
+ // owns this active slot.
2393
+ if (!this._canContinueStart(jobId, lifecycleGeneration))
2394
+ return;
1705
2395
  pluginActive = resolution.active;
1706
2396
  pluginDegraded = resolution.degraded;
1707
2397
  if (pluginActive.length > 0 || pluginDegraded.length > 0) {
1708
- pluginSnapshotPath = snapshotPluginsForJob(this._projectSlug, jobId, this._projectId, pluginActive, pluginDegraded);
2398
+ // The injected resolver normally returns an empty test fixture. Load
2399
+ // the snapshotter lazily only if its result actually needs one.
2400
+ snapshotter ??= require('./plugins/rail-integration').snapshotPluginsForJob;
2401
+ pluginSnapshotPath = snapshotter(this._projectSlug, jobId, this._projectId, pluginActive, pluginDegraded);
1709
2402
  }
1710
2403
  for (const d of pluginDegraded) {
1711
2404
  this._broadcast({
@@ -1722,6 +2415,9 @@ class QueueManager {
1722
2415
  console.warn(`[queue-manager] plugin resolution failed for job ${jobId}: ${err.message}`);
1723
2416
  }
1724
2417
  }
2418
+ // Covers resolver throws as well as future awaits added to the block above.
2419
+ if (!this._canContinueStart(jobId, lifecycleGeneration))
2420
+ return;
1725
2421
  if (pluginActive.length > 0 && pluginSnapshotPath) {
1726
2422
  spawnEnv = {
1727
2423
  ...spawnEnv,
@@ -1818,6 +2514,8 @@ class QueueManager {
1818
2514
  // provenance is captured around the SESSION lifecycle exactly like the
1819
2515
  // one-shot path: pre-spawn snapshot here, diff at settle.
1820
2516
  if (spawnInteractive) {
2517
+ if (!this._canContinueStart(jobId, lifecycleGeneration))
2518
+ return;
1821
2519
  if ((0, feature_flags_1.isCodeExplorerEnabled)()) {
1822
2520
  try {
1823
2521
  const snap = (0, file_provenance_1.snapshotWorkingTree)(execution.repoDir);
@@ -1860,6 +2558,25 @@ class QueueManager {
1860
2558
  console.warn(`[queue-manager] provenance snapshot failed: ${err.message}`);
1861
2559
  }
1862
2560
  }
2561
+ // Durably promote queued → running BEFORE the provider process exists. If
2562
+ // spawn throws, the normal _failWedgedJob path marks this row failed; if the
2563
+ // app crashes after this point, startup recovers a running orphan instead of
2564
+ // replaying the queued admission alongside a possibly-live old child.
2565
+ if (!this._canContinueStart(jobId, lifecycleGeneration))
2566
+ return;
2567
+ if (this._db) {
2568
+ (0, db_1.createJob)(this._db, {
2569
+ id: jobId,
2570
+ command: job.command,
2571
+ started_at: job.startedAt,
2572
+ provider: adapter.id,
2573
+ priority: job.priority,
2574
+ depends_on_job_id: job.dependsOnJobId,
2575
+ pipeline_id: job.pipelineId,
2576
+ causal_ownership: job.causalOwnership === true,
2577
+ });
2578
+ }
2579
+ this._consumePendingSelections(jobId);
1863
2580
  // spawnAiCli reroutes multi-line argv values through stdin on Windows.
1864
2581
  const child = (0, cli_prompt_1.spawnAiCli)(binary, args, {
1865
2582
  env: spawnEnv,
@@ -1887,11 +2604,51 @@ class QueueManager {
1887
2604
  // Start zombie detection timer. Reset on any raw data from the process.
1888
2605
  // Using 'data' events (not readline 'line') ensures the timer resets
1889
2606
  // synchronously in test environments with fake timers.
1890
- this._resetZombieTimer();
1891
- child.stdout.on('data', () => { this._resetZombieTimer(); });
1892
- child.stderr.on('data', () => { this._resetZombieTimer(); });
2607
+ this._resetZombieTimer(jobId);
2608
+ child.stdout.on('data', () => { this._resetZombieTimer(jobId); });
2609
+ child.stderr.on('data', () => { this._resetZombieTimer(jobId); });
1893
2610
  let eventSeq = 0;
1894
2611
  let lastResultEvent = null;
2612
+ let rawPersistenceFailed = false;
2613
+ const persistEvent = (event, critical = true) => {
2614
+ const seq = eventSeq++;
2615
+ const db = this._db;
2616
+ // A force-failed child may continue emitting after its durable job became
2617
+ // terminal (and its history may already be deleted). Keep parsing usage
2618
+ // in memory for late reconciliation, but never write child events through
2619
+ // a terminal/missing FK or let storage failure escape readline.
2620
+ if (!db)
2621
+ return { seq, persisted: true };
2622
+ if (this._jobs.get(jobId)?.status !== 'running' && !this._forceFailedRowJobs.has(jobId))
2623
+ return { seq, persisted: false };
2624
+ try {
2625
+ (0, db_1.appendEvent)(db, jobId, seq, event);
2626
+ }
2627
+ catch (err) {
2628
+ console.error(`[queue-manager] event persistence failed for ${jobId}:`, err);
2629
+ if (critical && !rawPersistenceFailed) {
2630
+ rawPersistenceFailed = true;
2631
+ this._paused = true;
2632
+ this._persistQueueState();
2633
+ this._broadcastQueueState();
2634
+ if (this._activeJobId === jobId) {
2635
+ this._persistenceFailedJobs.add(jobId);
2636
+ this._kill(jobId);
2637
+ }
2638
+ else {
2639
+ const surviving = this._forceFailedProcesses.get(jobId);
2640
+ if (surviving?.pid) {
2641
+ try {
2642
+ (0, win_spawn_1.treeKillSafe)(surviving.pid, 'SIGTERM', () => { });
2643
+ }
2644
+ catch { /* best-effort */ }
2645
+ }
2646
+ }
2647
+ }
2648
+ return { seq, persisted: false };
2649
+ }
2650
+ return { seq, persisted: true };
2651
+ };
1895
2652
  // Accumulator of parsed AdapterEvent for finaliseInvocationResult on close.
1896
2653
  const adapterEvents = [];
1897
2654
  // Expose the (growing) accumulator + adapter/model so shutdown() can flush an
@@ -1909,26 +2666,22 @@ class QueueManager {
1909
2666
  model: railModel,
1910
2667
  });
1911
2668
  }
1912
- if (this._db) {
1913
- (0, db_1.createJob)(this._db, {
1914
- id: jobId,
1915
- command: job.command,
1916
- started_at: job.startedAt,
1917
- priority: job.priority,
1918
- depends_on_job_id: job.dependsOnJobId,
1919
- pipeline_id: job.pipelineId,
1920
- });
1921
- }
1922
2669
  // ── Batched broadcast for high-frequency messages (log + event) ──────
1923
2670
  // Collects messages and flushes every ~80ms instead of one WS send per line.
1924
2671
  const pendingBroadcast = [];
1925
2672
  let flushTimer = null;
1926
2673
  const FLUSH_INTERVAL_MS = 80;
1927
2674
  const batchedBroadcast = (msg) => {
2675
+ if (this._disposed)
2676
+ return;
1928
2677
  pendingBroadcast.push(msg);
1929
2678
  if (!flushTimer) {
1930
2679
  flushTimer = setTimeout(() => {
1931
2680
  flushTimer = null;
2681
+ if (this._disposed) {
2682
+ pendingBroadcast.length = 0;
2683
+ return;
2684
+ }
1932
2685
  const batch = pendingBroadcast.splice(0);
1933
2686
  for (const m of batch)
1934
2687
  this._broadcast(m);
@@ -1941,6 +2694,8 @@ class QueueManager {
1941
2694
  flushTimer = null;
1942
2695
  }
1943
2696
  const batch = pendingBroadcast.splice(0);
2697
+ if (this._disposed)
2698
+ return;
1944
2699
  for (const m of batch)
1945
2700
  this._broadcast(m);
1946
2701
  };
@@ -1960,7 +2715,10 @@ class QueueManager {
1960
2715
  };
1961
2716
  const stdoutReader = (0, readline_1.createInterface)({ input: child.stdout, crlfDelay: Infinity });
1962
2717
  const stderrReader = (0, readline_1.createInterface)({ input: child.stderr, crlfDelay: Infinity });
2718
+ this._jobReaders.set(jobId, { stdout: stdoutReader, stderr: stderrReader });
1963
2719
  stdoutReader.on('line', (line) => {
2720
+ if (this._disposed)
2721
+ return;
1964
2722
  let parsed = null;
1965
2723
  try {
1966
2724
  parsed = JSON.parse(line);
@@ -1978,13 +2736,13 @@ class QueueManager {
1978
2736
  }
1979
2737
  if (parsed) {
1980
2738
  const eventType = parsed.type ?? 'unknown';
1981
- if (this._db) {
1982
- (0, db_1.appendEvent)(this._db, jobId, eventSeq++, {
1983
- event_type: eventType,
1984
- source: 'stdout',
1985
- payload: line,
1986
- });
1987
- }
2739
+ const persisted = persistEvent({
2740
+ event_type: eventType,
2741
+ source: 'stdout',
2742
+ payload: line,
2743
+ });
2744
+ if (!persisted.persisted)
2745
+ return;
1988
2746
  batchedBroadcast({
1989
2747
  type: 'event',
1990
2748
  jobId,
@@ -1992,31 +2750,27 @@ class QueueManager {
1992
2750
  source: 'stdout',
1993
2751
  payload: line,
1994
2752
  timestamp: new Date().toISOString(),
1995
- seq: eventSeq - 1,
2753
+ seq: persisted.seq,
1996
2754
  });
1997
2755
  if (eventType === 'result') {
1998
2756
  lastResultEvent = parsed;
1999
2757
  }
2000
2758
  const displayText = (0, stream_display_1.extractDisplayText)(parsed);
2001
2759
  if (displayText !== null) {
2002
- if (this._db) {
2003
- (0, db_1.appendEvent)(this._db, jobId, eventSeq++, {
2004
- event_type: 'log',
2005
- source: 'stdout',
2006
- payload: JSON.stringify({ line: displayText }),
2007
- });
2008
- }
2760
+ persistEvent({
2761
+ event_type: 'log',
2762
+ source: 'stdout',
2763
+ payload: JSON.stringify({ line: displayText }),
2764
+ }, false);
2009
2765
  emitLine('stdout', displayText);
2010
2766
  }
2011
2767
  }
2012
2768
  else {
2013
- if (this._db) {
2014
- (0, db_1.appendEvent)(this._db, jobId, eventSeq++, {
2015
- event_type: 'log',
2016
- source: 'stdout',
2017
- payload: JSON.stringify({ line }),
2018
- });
2019
- }
2769
+ persistEvent({
2770
+ event_type: 'log',
2771
+ source: 'stdout',
2772
+ payload: JSON.stringify({ line }),
2773
+ }, false);
2020
2774
  // For adapters whose stream is JSONL (claude, codex), a non-parseable
2021
2775
  // line is unexpected noise. For future plain-text adapters this is
2022
2776
  // their normal output. emitLine surfaces it either way.
@@ -2029,13 +2783,13 @@ class QueueManager {
2029
2783
  }
2030
2784
  });
2031
2785
  stderrReader.on('line', (line) => {
2032
- if (this._db) {
2033
- (0, db_1.appendEvent)(this._db, jobId, eventSeq++, {
2034
- event_type: 'log',
2035
- source: 'stderr',
2036
- payload: JSON.stringify({ line }),
2037
- });
2038
- }
2786
+ if (this._disposed)
2787
+ return;
2788
+ persistEvent({
2789
+ event_type: 'log',
2790
+ source: 'stderr',
2791
+ payload: JSON.stringify({ line }),
2792
+ }, false);
2039
2793
  emitLine('stderr', line);
2040
2794
  });
2041
2795
  child.on('close', (code) => {
@@ -2055,38 +2809,41 @@ class QueueManager {
2055
2809
  /** Per-job adapter resolved in _startJob; defaults to the project primary
2056
2810
  * for any caller that does not thread it (none today). */
2057
2811
  adapter = this._adapter) {
2058
- this._clearZombieTimer();
2059
- if (this._killTimer !== null) {
2060
- clearTimeout(this._killTimer);
2061
- this._killTimer = null;
2812
+ // Timers belong to the job currently holding the single execution slot.
2813
+ // A late close from a previously force-failed, unkillable child must not
2814
+ // disarm the successor's inactivity watchdog or SIGKILL escalation.
2815
+ if (this._activeJobId === jobId) {
2816
+ this._clearZombieTimer();
2817
+ if (this._killTimer !== null) {
2818
+ clearTimeout(this._killTimer);
2819
+ this._killTimer = null;
2820
+ }
2062
2821
  }
2063
- // Reclaim the pre-spawn snapshot unconditionally, BEFORE any early return,
2064
- // so a disposed/unknown job can't leak its entry in _snapshotRefs (the git
2065
- // stash commit it references is dangling and git-GC'd on its own).
2822
+ // Read terminal context now, but consume it only after the terminal row and
2823
+ // recovery intent commit. A failed disk write must leave enough ownership
2824
+ // for graceful/startup recovery to retry the exact same run.
2066
2825
  const snapshot = this._snapshotRefs.get(jobId);
2067
- this._snapshotRefs.delete(jobId);
2068
- // Relocate-artifacts: the repo dir this job snapshotted against (= repoDir,
2069
- // never the workspace). Falls back to this._cwd for jobs spawned before this
2070
- // map existed (e.g. restored-from-db) so provenance still targets the repo.
2071
2826
  const jobExecution = this._jobExecution.get(jobId);
2072
- this._jobExecution.delete(jobId);
2073
- // Release the resolved-provider entry on the normal child-exit path (the
2074
- // adapter is threaded into _onJobExit directly, so this is pure cleanup).
2075
- this._jobResolvedProvider.delete(jobId);
2076
- // Reclaim the live-accounting handle unconditionally (shutdown flush no longer
2077
- // needs it once the child has exited).
2078
- this._jobLiveAccounting.delete(jobId);
2079
- // Consume the spawn-captured PR-delivery mode (before any early return so a
2080
- // disposed/unknown-job exit can never leak the entry). Decides whether a
2081
- // COMPLETED job's tickets park at on_review (ask-first) or done (legacy).
2082
2827
  const prDelivery = this._jobPrDelivery.get(jobId) ?? false;
2083
- this._jobPrDelivery.delete(jobId);
2084
2828
  const provenanceRepoDir = jobExecution?.repoDir ?? this._cwd;
2085
- // Clean up the per-job openspec PATH shim (relocated claude rails only),
2086
- // BEFORE any early return, so a disposed/unknown-job exit can't leak the
2087
- // in-memory map entry or the on-disk chmod-700 dir. Mirrors the interactive
2088
- // settle path (the dominant non-interactive rail path lives here).
2089
- this._cleanupOpenspecShim(jobId);
2829
+ const consumeTerminalContext = () => {
2830
+ const readers = this._jobReaders.get(jobId);
2831
+ try {
2832
+ readers?.stdout.close();
2833
+ }
2834
+ catch { /* best-effort */ }
2835
+ try {
2836
+ readers?.stderr.close();
2837
+ }
2838
+ catch { /* best-effort */ }
2839
+ this._jobReaders.delete(jobId);
2840
+ this._snapshotRefs.delete(jobId);
2841
+ this._jobExecution.delete(jobId);
2842
+ this._jobResolvedProvider.delete(jobId);
2843
+ this._jobLiveAccounting.delete(jobId);
2844
+ this._jobPrDelivery.delete(jobId);
2845
+ this._cleanupOpenspecShim(jobId);
2846
+ };
2090
2847
  // A3: release the active slot for THIS job before any early return, so a
2091
2848
  // disposed/unknown-job exit can never leave the slot reserved (which would
2092
2849
  // wedge the queue). Guarded by identity in case a stale exit fires late.
@@ -2097,11 +2854,16 @@ class QueueManager {
2097
2854
  // The manager was torn down (e.g. project removed) while the child was
2098
2855
  // still running. The DB may be closed; skip all bookkeeping to avoid an
2099
2856
  // uncaught throw inside this EventEmitter 'close' listener.
2100
- if (this._disposed)
2857
+ if (this._disposed) {
2858
+ consumeTerminalContext();
2101
2859
  return;
2860
+ }
2102
2861
  const job = this._jobs.get(jobId);
2103
- if (!job)
2862
+ if (!job) {
2863
+ consumeTerminalContext();
2864
+ this._drainQueue();
2104
2865
  return;
2866
+ }
2105
2867
  // LOW-6: this is the LATE close of a job already force-failed by
2106
2868
  // _forceFailUnkillableJob (SIGKILL escalation failed, then the child died on
2107
2869
  // its own). The terminal side-effects (status, onJobFinished, dependents,
@@ -2109,15 +2871,18 @@ class QueueManager {
2109
2871
  // that replaces the no-cost placeholder row(s) IF this close captured real
2110
2872
  // spend — never a duplicate row nor a second onJobFinished.
2111
2873
  if (this._forceFailedRowJobs.has(jobId)) {
2874
+ consumeTerminalContext();
2112
2875
  this._reconcileForceFailedJobExit(jobId, code, adapterEvents, adapter, spawnedModel);
2113
2876
  return;
2114
2877
  }
2115
2878
  const wasZombie = this._zombieJobs.has(jobId);
2116
2879
  const wasCanceling = this._cancelingJobs.has(jobId);
2117
- this._zombieJobs.delete(jobId);
2118
- this._cancelingJobs.delete(jobId);
2880
+ const wasPersistenceFailed = this._persistenceFailedJobs.has(jobId);
2119
2881
  let finalStatus;
2120
- if (wasZombie) {
2882
+ if (wasPersistenceFailed) {
2883
+ finalStatus = 'failed';
2884
+ }
2885
+ else if (wasZombie) {
2121
2886
  finalStatus = 'zombie_terminated';
2122
2887
  }
2123
2888
  else if (wasCanceling) {
@@ -2129,72 +2894,69 @@ class QueueManager {
2129
2894
  else {
2130
2895
  finalStatus = 'failed';
2131
2896
  }
2132
- job.status = finalStatus;
2133
- job.finishedAt = new Date().toISOString();
2134
- job.exitCode = code;
2135
- // Capture result text for output chaining between pipeline steps
2136
- if (lastResultEvent && typeof lastResultEvent.result === 'string') {
2137
- job.resultText = lastResultEvent.result;
2138
- }
2139
- // (_activeProcess/_activeJobId already released above, before the early
2140
- // returns, so the slot is freed on every exit path — A3.)
2897
+ // Adapter-driven finalisation must happen before staging so the immutable
2898
+ // intent owns the same usage that lands on jobs and ai_invocations.
2899
+ const { result: normalised, estimated } = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
2900
+ const tokenData = lastResultEvent || adapterEvents.length > 0
2901
+ ? {
2902
+ tokens_in: normalised.tokens_in,
2903
+ tokens_out: normalised.tokens_out,
2904
+ tokens_cache_read: normalised.tokens_cache_read,
2905
+ tokens_cache_create: normalised.tokens_cache_create,
2906
+ total_cost_usd: normalised.total_cost_usd,
2907
+ total_cost_usd_estimated: estimated,
2908
+ num_turns: normalised.num_turns,
2909
+ model: normalised.model,
2910
+ duration_ms: normalised.duration_ms,
2911
+ duration_api_ms: normalised.duration_api_ms,
2912
+ session_id: normalised.session_id,
2913
+ }
2914
+ : {};
2915
+ const finishedAt = new Date().toISOString();
2916
+ const invStatus = finalStatus === 'completed'
2917
+ ? 'success'
2918
+ : (finalStatus === 'canceled' || finalStatus === 'zombie_terminated')
2919
+ ? 'aborted'
2920
+ : 'failed';
2141
2921
  if (this._db) {
2142
- // Adapter-driven result finalisation handles tokens, cost (or pricing-
2143
- // table estimate for non-native-cost providers), and session_id stamping.
2144
- const { result: normalised, estimated } = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
2145
- const tokenData = lastResultEvent || adapterEvents.length > 0
2146
- ? {
2147
- tokens_in: normalised.tokens_in,
2148
- tokens_out: normalised.tokens_out,
2149
- tokens_cache_read: normalised.tokens_cache_read,
2150
- tokens_cache_create: normalised.tokens_cache_create,
2151
- total_cost_usd: normalised.total_cost_usd,
2152
- total_cost_usd_estimated: estimated,
2153
- num_turns: normalised.num_turns,
2154
- model: normalised.model,
2155
- duration_ms: normalised.duration_ms,
2156
- duration_api_ms: normalised.duration_api_ms,
2157
- session_id: normalised.session_id,
2158
- }
2159
- : {};
2160
2922
  try {
2161
- (0, db_1.finishJob)(this._db, jobId, {
2162
- exit_code: code ?? -1,
2923
+ this._stageTerminalIntent(job, {
2163
2924
  status: finalStatus,
2164
- ...tokenData,
2925
+ invocationStatus: invStatus,
2926
+ provider: adapter.id,
2927
+ finishedAt,
2928
+ exitCode: code,
2929
+ result: tokenData,
2930
+ ticketCompletionStatus: finalStatus === 'completed'
2931
+ ? (prDelivery ? 'on_review' : 'done')
2932
+ : undefined,
2165
2933
  });
2166
2934
  }
2167
2935
  catch (err) {
2168
- // Defense-in-depth: the DB may have been closed underneath us mid-job.
2169
- // Never let a write throw uncaught inside the child 'close' listener.
2170
- console.error('[queue-manager] finishJob failed (db unavailable?):', err);
2171
- }
2172
- // ai_invocations capture (surface='job'). One row per job exit, or one row
2173
- // per extracted ticket for a multi-ticket batch (MED-7).
2174
- if (this._projectId) {
2175
- try {
2176
- const invStatus = finalStatus === 'completed'
2177
- ? 'success'
2178
- : (finalStatus === 'canceled' || finalStatus === 'zombie_terminated')
2179
- ? 'aborted'
2180
- : 'failed';
2181
- const ticketIds = this._extractTicketIds(job.command);
2182
- this._recordJobInvocations({
2183
- jobId,
2184
- provider: adapter.id,
2185
- status: invStatus,
2186
- startedAt: job.startedAt ?? new Date().toISOString(),
2187
- finishedAt: job.finishedAt,
2188
- ticketIds,
2189
- estimated,
2190
- result: normalised,
2191
- });
2192
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
2193
- }
2194
- catch (err) {
2195
- console.error('[queue-manager] recordInvocation failed:', err);
2196
- }
2936
+ // Keep the durable row running and reserve the now-childless slot. This
2937
+ // is an explicit fail-stop: no later job may overtake unaccounted work.
2938
+ this._activeProcess = null;
2939
+ this._activeJobId = jobId;
2940
+ this._terminalPersistenceBlockedJobs.add(jobId);
2941
+ this._paused = true;
2942
+ this._persistQueueState();
2943
+ this._broadcastQueueState();
2944
+ console.error(`[queue-manager] terminal staging failed for ${jobId}:`, err);
2945
+ return;
2197
2946
  }
2947
+ }
2948
+ consumeTerminalContext();
2949
+ this._zombieJobs.delete(jobId);
2950
+ this._cancelingJobs.delete(jobId);
2951
+ this._persistenceFailedJobs.delete(jobId);
2952
+ job.status = finalStatus;
2953
+ job.finishedAt = finishedAt;
2954
+ job.exitCode = code;
2955
+ if (lastResultEvent && typeof lastResultEvent.result === 'string') {
2956
+ job.resultText = lastResultEvent.result;
2957
+ }
2958
+ let accountingReady = true;
2959
+ if (this._db) {
2198
2960
  // Code-Explorer post-exit provenance hook. Diffs the working tree against
2199
2961
  // the pre-spawn snapshot and inserts one row per touched path. Gated by
2200
2962
  // SPECRAILS_CODE_EXPLORER (re-checked at each completion so the flag can
@@ -2212,6 +2974,7 @@ class QueueManager {
2212
2974
  if (jobCost != null && finalStatus === 'completed') {
2213
2975
  this._emitCostAlerts(jobId, jobCost);
2214
2976
  }
2977
+ accountingReady = this._resumeOrphanRecoveries();
2215
2978
  // ─── Daily-budget enforcement (MED-5) ───────────────────────────────────
2216
2979
  // Runs on EVERY terminal exit, not just completed jobs: a failed/aborted
2217
2980
  // claude run still emits real cost (error_max_turns etc.), so a day of
@@ -2224,45 +2987,32 @@ class QueueManager {
2224
2987
  }
2225
2988
  else {
2226
2989
  emitLine('stdout', `[process exited with code ${code ?? 'unknown'}]`);
2227
- }
2228
- // Notify webhook handler (if any) about job completion/failure/cancellation.
2229
- // zombie_terminated is included so a timed-out rail job still releases its
2230
- // tickets (revert/flag) and clears its in-memory railJobs entry instead of
2231
- // wedging the rail card in 'running' until a server restart.
2232
- if (this._onJobFinished &&
2233
- (finalStatus === 'completed' || finalStatus === 'failed' || finalStatus === 'canceled' || finalStatus === 'zombie_terminated')) {
2234
- let costUsd;
2235
2990
  try {
2236
- costUsd = this._db
2237
- ? this._db.prepare('SELECT total_cost_usd FROM jobs WHERE id = ?').get(jobId)?.total_cost_usd ?? undefined
2238
- : undefined;
2991
+ if (finalStatus === 'completed') {
2992
+ this._onJobFinished?.(jobId, finalStatus, normalised.total_cost_usd, {
2993
+ ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
2994
+ });
2995
+ }
2996
+ else {
2997
+ this._onJobFinished?.(jobId, finalStatus, normalised.total_cost_usd);
2998
+ }
2239
2999
  }
2240
3000
  catch (err) {
2241
- console.error('[queue-manager] cost read for webhook failed (db unavailable?):', err);
2242
- }
2243
- if (finalStatus === 'completed') {
2244
- // Thread the spawn-captured PR-delivery mode: under the ask-first
2245
- // methodology a completed job's tickets park at on_review, never done.
2246
- // Failure statuses keep the legacy 3-arg call shape (the field is
2247
- // completion-only).
2248
- this._onJobFinished(jobId, finalStatus, costUsd ?? undefined, {
2249
- ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
2250
- });
3001
+ console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
2251
3002
  }
2252
- else {
2253
- this._onJobFinished(jobId, finalStatus, costUsd ?? undefined);
3003
+ if (finalStatus !== 'completed') {
3004
+ this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
2254
3005
  }
3006
+ if (job.pipelineId)
3007
+ this._checkPipelineStatus(job.pipelineId);
2255
3008
  }
2256
- // Handle dependent jobs: skip them if parent did not complete successfully
2257
- if (finalStatus !== 'completed') {
2258
- this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
2259
- }
2260
- // Check pipeline status
2261
- if (job.pipelineId) {
2262
- this._checkPipelineStatus(job.pipelineId);
3009
+ if (!accountingReady) {
3010
+ this._paused = true;
3011
+ this._persistQueueState();
2263
3012
  }
2264
3013
  this._broadcastQueueState();
2265
- this._drainQueue();
3014
+ if (accountingReady)
3015
+ this._drainQueue();
2266
3016
  }
2267
3017
  /**
2268
3018
  * Code-Explorer provenance recording, shared by the one-shot exit path
@@ -2319,18 +3069,19 @@ class QueueManager {
2319
3069
  console.error('[queue-manager] cost-alert bookkeeping failed (db unavailable?):', err);
2320
3070
  }
2321
3071
  }
2322
- _resetZombieTimer() {
3072
+ _resetZombieTimer(ownerJobId) {
2323
3073
  if (this._zombieTimeoutMs <= 0)
2324
3074
  return;
3075
+ const jobId = ownerJobId ?? this._activeJobId;
3076
+ if (!jobId || this._activeJobId !== jobId)
3077
+ return;
2325
3078
  if (this._inactivityTimer !== null) {
2326
3079
  clearTimeout(this._inactivityTimer);
2327
3080
  }
2328
- const jobId = this._activeJobId;
2329
- if (!jobId)
2330
- return;
2331
3081
  this._inactivityTimer = setTimeout(() => {
2332
3082
  this._inactivityTimer = null;
2333
- this._onZombieDetected(jobId);
3083
+ if (this._activeJobId === jobId)
3084
+ this._onZombieDetected(jobId);
2334
3085
  }, this._zombieTimeoutMs);
2335
3086
  }
2336
3087
  _clearZombieTimer() {
@@ -2421,52 +3172,60 @@ class QueueManager {
2421
3172
  _forceFailUnkillableJob(jobId) {
2422
3173
  const job = this._jobs.get(jobId);
2423
3174
  const isRunning = !!job && job.status === 'running';
3175
+ const survivingProcess = this._activeJobId === jobId ? this._activeProcess : null;
3176
+ let accountingReady = true;
2424
3177
  if (job && isRunning) {
2425
- job.status = 'failed';
2426
- job.finishedAt = new Date().toISOString();
2427
- job.exitCode = -1;
3178
+ const finishedAt = new Date().toISOString();
3179
+ const live = this._jobLiveAccounting.get(jobId);
3180
+ const { result: normalised, estimated } = live
3181
+ ? (0, result_event_1.finaliseInvocationResult)(live.adapter, live.events, { fallbackModel: live.model })
3182
+ : { result: {}, estimated: false };
3183
+ const durationMs = normalised.duration_ms ?? (job.startedAt
3184
+ ? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
3185
+ : undefined);
3186
+ const provider = live?.adapter.id ?? this._jobResolvedProvider.get(jobId) ?? this._adapter.id;
2428
3187
  if (this._db) {
2429
3188
  try {
2430
- (0, db_1.finishJob)(this._db, jobId, { exit_code: -1, status: 'failed' });
2431
- }
2432
- catch {
2433
- /* DB may be closed mid-shutdown — never throw from the kill callback */
2434
- }
2435
- }
2436
- // ai_invocations capture (surface='job', aborted) so the failed rail still
2437
- // shows on Analytics with no token/cost data (none was finalised). LOW-6:
2438
- // the surviving child's `close` may still fire `_onJobExit` later — mark the
2439
- // job so that late close replaces this no-cost placeholder with the real
2440
- // captured cost instead of inserting a SECOND row / re-firing onJobFinished.
2441
- if (this._db && this._projectId) {
2442
- try {
2443
- const ticketIds = this._extractTicketIds(job.command);
2444
- const durationMs = job.startedAt
2445
- ? new Date(job.finishedAt).getTime() - new Date(job.startedAt).getTime()
2446
- : undefined;
2447
- this._recordJobInvocations({
2448
- jobId,
2449
- // Stamp the provider the child ACTUALLY ran on (per-job override
2450
- // already consumed from _jobProviderSelection); _adapter.id is the
2451
- // final fallback for jobs with no resolved-provider entry.
2452
- provider: this._jobResolvedProvider.get(jobId) ?? this._adapter.id,
2453
- status: 'aborted',
2454
- startedAt: job.startedAt ?? new Date().toISOString(),
2455
- finishedAt: job.finishedAt,
2456
- ticketIds,
2457
- estimated: false,
2458
- result: { duration_ms: durationMs },
3189
+ this._stageTerminalIntent(job, {
3190
+ status: 'failed',
3191
+ invocationStatus: 'aborted',
3192
+ provider,
3193
+ finishedAt,
3194
+ exitCode: -1,
3195
+ awaitingLateReconciliation: true,
3196
+ result: {
3197
+ ...normalised,
3198
+ total_cost_usd_estimated: estimated,
3199
+ duration_ms: durationMs,
3200
+ },
2459
3201
  });
2460
- this._forceFailedRowJobs.add(jobId);
2461
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
2462
3202
  }
2463
3203
  catch (err) {
2464
- console.error('[queue-manager] recordInvocation (unkillable) failed:', err);
3204
+ // The child still exists and continues to own the active slot. Leave
3205
+ // every map intact and fail-stop the queue so a later close or
3206
+ // shutdown can retry the terminal transaction with fuller usage.
3207
+ this._terminalPersistenceBlockedJobs.add(jobId);
3208
+ this._paused = true;
3209
+ this._persistQueueState();
3210
+ this._broadcastQueueState();
3211
+ console.error(`[queue-manager] unkillable terminal staging failed for ${jobId}:`, err);
3212
+ return;
2465
3213
  }
2466
3214
  }
3215
+ job.status = 'failed';
3216
+ job.finishedAt = finishedAt;
3217
+ job.exitCode = -1;
3218
+ // A surviving child's late close only reconciles richer usage. The
3219
+ // durable callback/dependency effects below must never be fired twice.
3220
+ this._forceFailedRowJobs.add(jobId);
3221
+ if (survivingProcess)
3222
+ this._forceFailedProcesses.set(jobId, survivingProcess);
3223
+ if (this._db)
3224
+ accountingReady = this._resumeOrphanRecoveries();
2467
3225
  }
2468
- // Clear ALL per-job maps + the git-stash snapshot + the openspec shim so a
2469
- // surviving child cannot leak memory/disk (mirrors _onJobExit/_failWedgedJob).
3226
+ // Clear completed lifecycle maps, but retain the live adapter-event
3227
+ // accumulator until late close/shutdown so usage emitted by the surviving
3228
+ // child can still be reconciled.
2470
3229
  this._snapshotRefs.delete(jobId);
2471
3230
  this._jobExecution.delete(jobId);
2472
3231
  this._jobModelSelection.delete(jobId);
@@ -2475,24 +3234,31 @@ class QueueManager {
2475
3234
  this._jobResolvedProvider.delete(jobId);
2476
3235
  this._jobInteractiveSelection.delete(jobId);
2477
3236
  this._jobPrDelivery.delete(jobId);
2478
- this._jobLiveAccounting.delete(jobId);
2479
3237
  this._cleanupOpenspecShim(jobId);
2480
3238
  this._activeProcess = null;
2481
3239
  this._activeJobId = null;
2482
3240
  this._cancelingJobs.delete(jobId);
2483
3241
  this._zombieJobs.delete(jobId);
2484
- // Fire the rail/ticket completion callback so status reverts/flags and the
2485
- // budget/webhook/Jira write-back path runs — the whole point of the fix.
2486
- if (isRunning) {
3242
+ // DB-backed terminal effects are delivered by the outbox. Keep the direct
3243
+ // path only for ephemeral managers that have no durable project database.
3244
+ if (isRunning && !this._db) {
2487
3245
  try {
2488
3246
  this._onJobFinished?.(jobId, 'failed', undefined);
2489
3247
  }
2490
3248
  catch (err) {
2491
3249
  console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
2492
3250
  }
3251
+ this._skipDependents(jobId, `Parent job ${jobId} failed`);
3252
+ if (job?.pipelineId)
3253
+ this._checkPipelineStatus(job.pipelineId);
3254
+ }
3255
+ if (!accountingReady) {
3256
+ this._paused = true;
3257
+ this._persistQueueState();
2493
3258
  }
2494
3259
  this._broadcastQueueState();
2495
- this._drainQueue();
3260
+ if (accountingReady)
3261
+ this._drainQueue();
2496
3262
  }
2497
3263
  /**
2498
3264
  * Cost-only reconciliation for the LATE `close` of a job already force-failed
@@ -2503,62 +3269,113 @@ class QueueManager {
2503
3269
  * real cost; otherwise the placeholder is left untouched.
2504
3270
  */
2505
3271
  _reconcileForceFailedJobExit(jobId, code, adapterEvents, adapter, spawnedModel) {
2506
- this._forceFailedRowJobs.delete(jobId);
2507
3272
  const job = this._jobs.get(jobId);
2508
- // Remove the job now that its terminal handling is fully settled — a further
2509
- // stray close then hits the `if (!job) return` guard in _onJobExit.
2510
- this._jobs.delete(jobId);
2511
- this._jobLiveAccounting.delete(jobId);
2512
- if (!this._db || !this._projectId || !job)
3273
+ this._forceFailedProcesses.delete(jobId);
3274
+ // Ephemeral managers have no second durable history projection. Retain both
3275
+ // the failed tombstone and the late-close guard; a duplicate close remains
3276
+ // a no-op instead of making the job disappear or re-firing callbacks.
3277
+ if (!this._db || !job)
2513
3278
  return;
3279
+ this._pendingLateReconciliations.set(jobId, {
3280
+ code,
3281
+ adapterEvents: [...adapterEvents],
3282
+ adapter,
3283
+ spawnedModel,
3284
+ });
2514
3285
  try {
2515
- const { result: normalised, estimated } = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
3286
+ const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
3287
+ const normalised = sanitizeRecoveredResult(finalised.result);
3288
+ const estimated = finalised.estimated;
2516
3289
  const hasRealSpend = (normalised.total_cost_usd ?? 0) > 0 ||
2517
3290
  (normalised.tokens_in ?? 0) > 0 ||
2518
3291
  (normalised.tokens_out ?? 0) > 0 ||
2519
3292
  (normalised.tokens_cache_read ?? 0) > 0 ||
2520
3293
  (normalised.tokens_cache_create ?? 0) > 0;
2521
- if (!hasRealSpend)
2522
- return; // placeholder stands; nothing real to record.
2523
- // Replace the placeholder row(s) with the real captured spend. Delete the
2524
- // plain-jobId row and any per-ticket split rows, then re-record.
2525
- this._db.prepare(`DELETE FROM ai_invocations WHERE surface_ref_id = ? OR surface_ref_id LIKE ?`).run(jobId, `${jobId}#t%`);
2526
- const finalStatus = code === 0 ? 'success' : 'failed';
2527
- const ticketIds = this._extractTicketIds(job.command);
2528
- this._recordJobInvocations({
2529
- jobId,
2530
- provider: adapter.id,
2531
- status: finalStatus,
2532
- startedAt: job.startedAt ?? new Date().toISOString(),
2533
- finishedAt: job.finishedAt ?? new Date().toISOString(),
2534
- ticketIds,
2535
- estimated,
2536
- result: normalised,
3294
+ const finalStatus = code === 0
3295
+ ? 'success'
3296
+ : code === null
3297
+ ? 'aborted'
3298
+ : 'failed';
3299
+ const reconcile = this._db.transaction(() => {
3300
+ const recovery = this._db.prepare(`
3301
+ SELECT payload, accounting_completed
3302
+ FROM orphan_job_recovery WHERE job_id = ?
3303
+ `).get(jobId);
3304
+ if (!recovery) {
3305
+ throw new Error(`Missing late-reconciliation intent for ${jobId}`);
3306
+ }
3307
+ const { payload } = this._decodeRecoveryPayload(jobId, recovery.payload);
3308
+ payload.awaitingLateReconciliation = false;
3309
+ if (hasRealSpend) {
3310
+ Object.assign(payload, {
3311
+ provider: adapter.id,
3312
+ model: normalised.model ?? null,
3313
+ tokensIn: normalised.tokens_in ?? null,
3314
+ tokensOut: normalised.tokens_out ?? null,
3315
+ tokensCacheRead: normalised.tokens_cache_read ?? null,
3316
+ tokensCacheCreate: normalised.tokens_cache_create ?? null,
3317
+ totalCostUsd: normalised.total_cost_usd ?? null,
3318
+ totalCostUsdEstimated: estimated ? 1 : 0,
3319
+ numTurns: normalised.num_turns ?? null,
3320
+ durationMs: normalised.duration_ms ?? null,
3321
+ durationApiMs: normalised.duration_api_ms ?? null,
3322
+ sessionId: normalised.session_id ?? null,
3323
+ invocationStatus: finalStatus,
3324
+ });
3325
+ }
3326
+ const validatedPayload = this._decodeRecoveryPayload(jobId, JSON.stringify(payload)).payload;
3327
+ // Convert the already-completed partial accounting checkpoint back into
3328
+ // a durable pending intent before removing its placeholder. If the real
3329
+ // ledger insert later fails, restart/resume owns the full late payload.
3330
+ const shouldReaccount = hasRealSpend && !!this._projectId;
3331
+ this._db.prepare(`
3332
+ UPDATE orphan_job_recovery
3333
+ SET payload = ?, accounting_completed = ?
3334
+ WHERE job_id = ?
3335
+ `).run(JSON.stringify(validatedPayload), shouldReaccount ? 0 : recovery.accounting_completed, jobId);
3336
+ if (shouldReaccount) {
3337
+ this._db.prepare(`DELETE FROM ai_invocations WHERE surface_ref_id = ? OR surface_ref_id LIKE ?`).run(jobId, `${jobId}#t%`);
3338
+ }
3339
+ if (hasRealSpend) {
3340
+ // Reflect the real cost on the jobs row too (it was force-failed with
3341
+ // a partial snapshot while the child was still alive).
3342
+ (0, db_1.finishJob)(this._db, jobId, {
3343
+ exit_code: code ?? -1,
3344
+ status: 'failed',
3345
+ tokens_in: normalised.tokens_in,
3346
+ tokens_out: normalised.tokens_out,
3347
+ tokens_cache_read: normalised.tokens_cache_read,
3348
+ tokens_cache_create: normalised.tokens_cache_create,
3349
+ total_cost_usd: normalised.total_cost_usd,
3350
+ total_cost_usd_estimated: estimated,
3351
+ num_turns: normalised.num_turns,
3352
+ model: normalised.model,
3353
+ duration_ms: normalised.duration_ms,
3354
+ duration_api_ms: normalised.duration_api_ms,
3355
+ session_id: normalised.session_id,
3356
+ });
3357
+ }
2537
3358
  });
2538
- // Reflect the real cost on the jobs row too (it was stamped failed w/o cost).
2539
- try {
2540
- (0, db_1.finishJob)(this._db, jobId, {
2541
- exit_code: code ?? -1,
2542
- status: 'failed',
2543
- tokens_in: normalised.tokens_in,
2544
- tokens_out: normalised.tokens_out,
2545
- tokens_cache_read: normalised.tokens_cache_read,
2546
- tokens_cache_create: normalised.tokens_cache_create,
2547
- total_cost_usd: normalised.total_cost_usd,
2548
- total_cost_usd_estimated: estimated,
2549
- num_turns: normalised.num_turns,
2550
- model: normalised.model,
2551
- duration_ms: normalised.duration_ms,
2552
- duration_api_ms: normalised.duration_api_ms,
2553
- session_id: normalised.session_id,
2554
- });
2555
- }
2556
- catch (err) {
2557
- console.error('[queue-manager] force-fail reconcile finishJob failed:', err);
3359
+ reconcile();
3360
+ this._forceFailedRowJobs.delete(jobId);
3361
+ this._jobLiveAccounting.delete(jobId);
3362
+ this._pendingLateReconciliations.delete(jobId);
3363
+ this._terminalPersistenceBlockedJobs.delete(jobId);
3364
+ const accountingReady = this._resumeOrphanRecoveries();
3365
+ if (!accountingReady) {
3366
+ this._paused = true;
3367
+ this._persistQueueState();
3368
+ this._broadcastQueueState();
2558
3369
  }
2559
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
2560
3370
  }
2561
3371
  catch (err) {
3372
+ // Keep the guard + tombstone so this runtime never treats a duplicate
3373
+ // event as a new terminal job. No successor is admitted after an unknown
3374
+ // late spend until the operator retries/restarts with healthy storage.
3375
+ this._terminalPersistenceBlockedJobs.add(jobId);
3376
+ this._paused = true;
3377
+ this._persistQueueState();
3378
+ this._broadcastQueueState();
2562
3379
  console.error('[queue-manager] force-fail reconcile failed:', err);
2563
3380
  }
2564
3381
  }
@@ -2571,19 +3388,84 @@ class QueueManager {
2571
3388
  timestamp: new Date().toISOString(),
2572
3389
  });
2573
3390
  }
2574
- _persistJob(job) {
3391
+ /** Materialise the exact restart semantics for one unstarted admission.
3392
+ * Map presence matters for profile and interactive: null/false are explicit
3393
+ * choices and must never collapse into the absent/default state. */
3394
+ _queuedJobRecord(job) {
3395
+ const hasProfileSelection = this._jobProfileSelection.has(job.id);
3396
+ const hasInteractiveSelection = this._jobInteractiveSelection.has(job.id);
3397
+ return {
3398
+ id: job.id,
3399
+ command: job.command,
3400
+ queue_position: job.queuePosition,
3401
+ priority: job.priority,
3402
+ depends_on_job_id: job.dependsOnJobId,
3403
+ pipeline_id: job.pipelineId,
3404
+ provider: this._jobProviderSelection.get(job.id) ?? null,
3405
+ model: this._jobModelSelection.get(job.id) ?? null,
3406
+ profile_name: hasProfileSelection
3407
+ ? (this._jobProfileSelection.get(job.id) ?? null)
3408
+ : null,
3409
+ profile_selection_set: hasProfileSelection,
3410
+ interactive: hasInteractiveSelection
3411
+ ? this._jobInteractiveSelection.get(job.id)
3412
+ : null,
3413
+ causal_ownership: job.causalOwnership === true,
3414
+ };
3415
+ }
3416
+ _persistJob(job, strict = false) {
2575
3417
  if (!this._db)
2576
3418
  return;
2577
- // For queued jobs, we use the DB to store queue position and priority for startup restore.
2578
- // We only upsert queue_position + priority + dependency fields — the rest is handled by createJob/finishJob.
2579
- // Since this method is called for all status transitions, we use a flexible upsert
2580
- // that only touches queue_position, priority, and dependency fields (for queued jobs) — other fields are
2581
- // managed by the existing createJob/finishJob API.
2582
3419
  try {
2583
- this._db.prepare(`UPDATE jobs SET queue_position = ?, priority = ?, depends_on_job_id = ?, pipeline_id = ? WHERE id = ?`).run(job.queuePosition ?? null, job.priority, job.dependsOnJobId ?? null, job.pipelineId ?? null, job.id);
3420
+ if (job.status === 'queued') {
3421
+ (0, db_1.upsertQueuedJob)(this._db, this._queuedJobRecord(job));
3422
+ }
3423
+ else if (TERMINAL_STATUSES.has(job.status)) {
3424
+ (0, db_1.deleteQueuedJob)(this._db, job.id);
3425
+ }
2584
3426
  }
2585
- catch {
2586
- // Job may not exist in DB yet
3427
+ catch (err) {
3428
+ if (strict)
3429
+ throw err;
3430
+ // Persistence remains best-effort for callers racing project teardown.
3431
+ }
3432
+ }
3433
+ /** Persist the complete in-memory queue order in one transaction. Explicit
3434
+ * removals cover terminal-before-start paths while leaving the active
3435
+ * pre-spawn admission intact until createJob atomically promotes it. */
3436
+ _persistQueuedState(strict = false, removedJobIds = [], reservedJobId) {
3437
+ const db = this._db;
3438
+ if (!db)
3439
+ return;
3440
+ try {
3441
+ const persist = db.transaction(() => {
3442
+ for (const id of removedJobIds)
3443
+ (0, db_1.deleteQueuedJob)(db, id);
3444
+ if (reservedJobId) {
3445
+ const reserved = this._jobs.get(reservedJobId);
3446
+ if (reserved) {
3447
+ (0, db_1.upsertQueuedJob)(db, {
3448
+ ...this._queuedJobRecord(reserved),
3449
+ // A selected-but-not-yet-spawned admission must restore ahead of
3450
+ // the remaining queue; using 0 also avoids duplicate position 1.
3451
+ queue_position: 0,
3452
+ });
3453
+ }
3454
+ }
3455
+ for (const id of this._queue) {
3456
+ const job = this._jobs.get(id);
3457
+ if (!job || job.status !== 'queued')
3458
+ continue;
3459
+ (0, db_1.upsertQueuedJob)(db, this._queuedJobRecord(job));
3460
+ }
3461
+ });
3462
+ persist();
3463
+ }
3464
+ catch (err) {
3465
+ if (strict)
3466
+ throw err;
3467
+ // Best-effort for internal teardown/drain paths; admissions and public
3468
+ // queue mutations call this in strict mode and roll back in memory.
2587
3469
  }
2588
3470
  }
2589
3471
  _persistQueueState() {
@@ -2599,57 +3481,26 @@ class QueueManager {
2599
3481
  _restoreFromDb() {
2600
3482
  if (!this._db)
2601
3483
  return;
3484
+ let recoveryAccountingReady = false;
3485
+ const restoredIds = [];
2602
3486
  try {
2603
- // Backfill an aborted ai_invocations row for every job orphaned 'running'
2604
- // by an UNGRACEFUL crash (no shutdown() flush ran). Each row carries
2605
- // whatever spend the jobs row accumulated — interactive per-turn writes
2606
- // hold real cost; a non-interactive rail whose finishJob never ran holds
2607
- // NULL — so the run still counts toward totalRuns/failureRate and an
2608
- // interactive session's cost is not lost from Analytics
2609
- // (COST-ACCOUNTING-AUDIT CRIT-3 / HIGH-1, crash path). Runs BEFORE the
2610
- // status flip so we can read the pre-fail token/cost columns.
2611
- if (this._projectId) {
2612
- try {
2613
- const orphans = this._db.prepare(`SELECT id, command, started_at, model, tokens_in, tokens_out, tokens_cache_read,
2614
- tokens_cache_create, total_cost_usd, total_cost_usd_estimated, num_turns,
2615
- duration_ms, duration_api_ms, session_id
2616
- FROM jobs WHERE status = 'running'`).all();
2617
- const finishedAt = new Date().toISOString();
2618
- for (const o of orphans) {
2619
- this._recordJobInvocations({
2620
- jobId: o.id,
2621
- provider: this._adapter.id,
2622
- status: 'aborted',
2623
- startedAt: o.started_at ?? finishedAt,
2624
- finishedAt,
2625
- ticketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(o.command),
2626
- estimated: !!o.total_cost_usd_estimated,
2627
- result: {
2628
- tokens_in: o.tokens_in ?? undefined,
2629
- tokens_out: o.tokens_out ?? undefined,
2630
- tokens_cache_read: o.tokens_cache_read ?? undefined,
2631
- tokens_cache_create: o.tokens_cache_create ?? undefined,
2632
- total_cost_usd: o.total_cost_usd ?? undefined,
2633
- num_turns: o.num_turns ?? undefined,
2634
- model: o.model ?? undefined,
2635
- session_id: o.session_id ?? undefined,
2636
- duration_ms: o.duration_ms ?? undefined,
2637
- duration_api_ms: o.duration_api_ms ?? undefined,
2638
- },
2639
- });
2640
- }
2641
- if (orphans.length > 0) {
2642
- this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
2643
- }
2644
- }
2645
- catch (err) {
2646
- console.error('[queue-manager] restore backfill failed:', err);
2647
- }
2648
- }
2649
- // Fail any jobs that were running when the server last shut down
2650
- this._db.prepare(`UPDATE jobs SET status = 'failed', finished_at = CURRENT_TIMESTAMP WHERE status = 'running'`).run();
2651
- // Restore queued jobs in order (priority DESC then queue_position ASC)
2652
- const rows = this._db.prepare(`SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id FROM jobs WHERE status = 'queued' ORDER BY queue_position ASC`).all();
3487
+ this._captureOrphanRecoveries();
3488
+ // Restore queued jobs by their durable positions, then re-assert the
3489
+ // priority invariant. `reorder` only permits movement within a priority
3490
+ // band, so this sort is also a defensive repair for older/corrupt rows.
3491
+ const rows = this._db.prepare(`SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id,
3492
+ provider, model, profile_name, profile_selection_set, interactive,
3493
+ causal_ownership
3494
+ FROM queued_jobs
3495
+ UNION ALL
3496
+ SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id,
3497
+ NULL AS provider, NULL AS model, NULL AS profile_name,
3498
+ 0 AS profile_selection_set, NULL AS interactive,
3499
+ causal_ownership
3500
+ FROM jobs
3501
+ WHERE status = 'queued'
3502
+ AND NOT EXISTS (SELECT 1 FROM queued_jobs WHERE queued_jobs.id = jobs.id)
3503
+ ORDER BY queue_position ASC, id ASC`).all();
2653
3504
  for (const row of rows) {
2654
3505
  const priority = (types_1.VALID_PRIORITIES.has(row.priority ?? '') ? row.priority : 'normal');
2655
3506
  const job = {
@@ -2665,42 +3516,97 @@ class QueueManager {
2665
3516
  pipelineId: row.pipeline_id ?? null,
2666
3517
  skipReason: null,
2667
3518
  resultText: null,
3519
+ causalOwnership: row.causal_ownership === 1,
2668
3520
  };
2669
3521
  this._jobs.set(row.id, job);
2670
3522
  this._queue.push(row.id);
3523
+ restoredIds.push(row.id);
3524
+ if (row.provider) {
3525
+ this._jobProviderSelection.set(row.id, row.provider);
3526
+ }
3527
+ if (row.model !== null) {
3528
+ this._jobModelSelection.set(row.id, row.model);
3529
+ }
3530
+ if (row.profile_selection_set === 1) {
3531
+ this._jobProfileSelection.set(row.id, row.profile_name);
3532
+ }
3533
+ if (row.interactive === 0 || row.interactive === 1) {
3534
+ this._jobInteractiveSelection.set(row.id, row.interactive === 1);
3535
+ }
2671
3536
  }
2672
- // Re-sort queue by priority (higher first), preserving FIFO within same level
3537
+ // Higher priority always executes first; Array#sort is stable, preserving
3538
+ // queue_position order within each band.
2673
3539
  this._queue.sort((a, b) => {
2674
3540
  const jobA = this._jobs.get(a);
2675
3541
  const jobB = this._jobs.get(b);
2676
3542
  return types_1.PRIORITY_WEIGHT[jobB.priority] - types_1.PRIORITY_WEIGHT[jobA.priority];
2677
3543
  });
3544
+ // Queue-terminal recovery must run AFTER queued rows are materialised in
3545
+ // memory: the normal `_skipDependents` invariant walks `_jobs`/`_queue`.
3546
+ // Its own durable checkpoint keeps this replay-safe if startup crashes
3547
+ // before, during, or immediately after the recursive skip.
3548
+ recoveryAccountingReady = this._resumeOrphanRecoveries();
2678
3549
  this._recomputePositions();
2679
3550
  // Restore pause state
2680
3551
  const pauseRow = this._db.prepare(`SELECT value FROM queue_state WHERE key = 'paused'`).get();
2681
- this._paused = pauseRow?.value === 'true';
3552
+ this._paused = pauseRow?.value === 'true' || !recoveryAccountingReady;
3553
+ if (!recoveryAccountingReady)
3554
+ this._persistQueueState();
3555
+ this._restoreBlocked = false;
2682
3556
  }
2683
- catch {
2684
- // DB may not have queue_state table yet — ignore
3557
+ catch (err) {
3558
+ // Discard any partial in-memory projection from this attempt. Admissions
3559
+ // are rejected while blocked, so every id here belongs exclusively to the
3560
+ // failed restore and can be rebuilt deterministically on resume.
3561
+ const restored = new Set(restoredIds);
3562
+ this._queue = this._queue.filter((id) => !restored.has(id));
3563
+ for (const id of restored) {
3564
+ this._jobs.delete(id);
3565
+ this._jobProfileSelection.delete(id);
3566
+ this._jobProviderSelection.delete(id);
3567
+ this._jobResolvedProvider.delete(id);
3568
+ this._jobModelSelection.delete(id);
3569
+ this._jobInteractiveSelection.delete(id);
3570
+ }
3571
+ this._restoreBlocked = true;
3572
+ // Startup cannot safely admit provider work when orphan capture/replay or
3573
+ // queue restoration itself failed. Persist the fail-stop when possible;
3574
+ // the user can retry after repairing the database.
3575
+ this._paused = true;
3576
+ this._persistQueueState();
3577
+ console.error('[queue-manager] durable queue recovery failed:', err);
2685
3578
  }
2686
3579
  // Kick off any restored queued jobs that are ready to run
2687
- this._drainQueue();
3580
+ if (!this._restoreBlocked)
3581
+ this._drainQueue();
2688
3582
  }
2689
- _isDependencyMet(job) {
2690
- if (!job.dependsOnJobId)
2691
- return true;
2692
- const parent = this._jobs.get(job.dependsOnJobId);
3583
+ /** Resolve a parent across live state, execution history and the durable
3584
+ * pre-start queue. A missing string id intentionally remains `null`: the
3585
+ * established QueueManager contract treats external/nonexistent parents as
3586
+ * already satisfied. */
3587
+ _getDependencyStatus(jobId) {
3588
+ const parent = this._jobs.get(jobId);
2693
3589
  if (parent)
2694
- return parent.status === 'completed';
3590
+ return parent.status;
2695
3591
  if (this._db) {
2696
- const row = this._db.prepare('SELECT status FROM jobs WHERE id = ?').get(job.dependsOnJobId);
2697
- if (row)
2698
- return row.status === 'completed';
3592
+ const history = this._db.prepare('SELECT status FROM jobs WHERE id = ?').get(jobId);
3593
+ if (history)
3594
+ return history.status;
3595
+ const queued = this._db.prepare('SELECT 1 FROM queued_jobs WHERE id = ?').get(jobId);
3596
+ if (queued)
3597
+ return 'queued';
2699
3598
  }
2700
- return true;
3599
+ return null;
3600
+ }
3601
+ _isDependencyMet(job) {
3602
+ if (!job.dependsOnJobId)
3603
+ return true;
3604
+ const parentStatus = this._getDependencyStatus(job.dependsOnJobId);
3605
+ return parentStatus === null || parentStatus === 'completed';
2701
3606
  }
2702
3607
  _skipDependents(parentJobId, reason) {
2703
3608
  const toSkip = [];
3609
+ const affectedPipelines = new Set();
2704
3610
  for (const [id, job] of this._jobs) {
2705
3611
  if (job.dependsOnJobId === parentJobId && job.status === 'queued') {
2706
3612
  toSkip.push(id);
@@ -2716,18 +3622,782 @@ class QueueManager {
2716
3622
  job.status = 'skipped';
2717
3623
  job.finishedAt = new Date().toISOString();
2718
3624
  job.skipReason = reason;
3625
+ if (job.pipelineId)
3626
+ affectedPipelines.add(job.pipelineId);
2719
3627
  if (this._db) {
2720
3628
  // Ensure the job row exists before updating (queued jobs may not have been persisted via createJob yet)
2721
3629
  const exists = this._db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(id);
2722
3630
  if (!exists) {
2723
- this._db.prepare(`INSERT INTO jobs (id, command, started_at, status, skip_reason, finished_at, depends_on_job_id, pipeline_id) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?)`).run(id, job.command, job.finishedAt, reason, job.finishedAt, job.dependsOnJobId, job.pipelineId);
3631
+ this._db.prepare(`INSERT INTO jobs (id, command, started_at, status, skip_reason, finished_at, depends_on_job_id, pipeline_id, causal_ownership) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?)`).run(id, job.command, job.finishedAt, reason, job.finishedAt, job.dependsOnJobId, job.pipelineId, job.causalOwnership === true ? 1 : 0);
2724
3632
  }
2725
3633
  else {
2726
3634
  (0, db_1.skipJob)(this._db, id, reason);
2727
3635
  }
3636
+ (0, db_1.deleteQueuedJob)(this._db, id);
3637
+ }
3638
+ try {
3639
+ this._onJobFinished?.(id, 'skipped', undefined);
3640
+ }
3641
+ catch (err) {
3642
+ console.error(`[QueueManager] onJobFinished(skipped) failed for ${id}: ${err.message}`);
2728
3643
  }
2729
3644
  this._skipDependents(id, `Parent job ${id} was skipped`);
2730
3645
  }
3646
+ this._recomputePositions();
3647
+ this._persistQueuedState();
3648
+ for (const pipelineId of affectedPipelines)
3649
+ this._checkPipelineStatus(pipelineId);
3650
+ }
3651
+ _snapshotRecoveryDescendants(parentJobId) {
3652
+ const db = this._db;
3653
+ if (!db)
3654
+ return [];
3655
+ const rows = db.prepare(`
3656
+ SELECT id, command, depends_on_job_id, pipeline_id, priority, causal_ownership
3657
+ FROM queued_jobs
3658
+ UNION ALL
3659
+ SELECT id, command, depends_on_job_id, pipeline_id, priority, causal_ownership
3660
+ FROM jobs
3661
+ WHERE status = 'queued'
3662
+ AND NOT EXISTS (SELECT 1 FROM queued_jobs WHERE queued_jobs.id = jobs.id)
3663
+ `).all();
3664
+ const children = new Map();
3665
+ for (const row of rows) {
3666
+ if (!row.depends_on_job_id)
3667
+ continue;
3668
+ const list = children.get(row.depends_on_job_id) ?? [];
3669
+ list.push(row);
3670
+ children.set(row.depends_on_job_id, list);
3671
+ }
3672
+ const result = [];
3673
+ const seen = new Set();
3674
+ const visit = (parentId) => {
3675
+ for (const row of children.get(parentId) ?? []) {
3676
+ if (seen.has(row.id))
3677
+ continue;
3678
+ seen.add(row.id);
3679
+ result.push({
3680
+ id: row.id,
3681
+ command: row.command,
3682
+ parentId,
3683
+ pipelineId: row.pipeline_id,
3684
+ priority: types_1.VALID_PRIORITIES.has(row.priority ?? '')
3685
+ ? row.priority
3686
+ : 'normal',
3687
+ causalOwnership: row.causal_ownership === 1,
3688
+ });
3689
+ visit(row.id);
3690
+ }
3691
+ };
3692
+ visit(parentJobId);
3693
+ return result;
3694
+ }
3695
+ /** Apply the immutable descendant snapshot even if history retention already
3696
+ * deleted the parent and nulled every live FK. Rows that became terminal by a
3697
+ * newer user action are left untouched. */
3698
+ _skipRecoveredDescendants(parentJobId, descendants, parentStatus = 'failed') {
3699
+ const db = this._db;
3700
+ if (!db)
3701
+ return;
3702
+ for (const descendant of descendants) {
3703
+ const admission = db.prepare(`SELECT 1 FROM queued_jobs WHERE id = ?`).get(descendant.id);
3704
+ const history = db.prepare(`SELECT status FROM jobs WHERE id = ?`).get(descendant.id);
3705
+ const inMemory = this._jobs.get(descendant.id);
3706
+ const stillQueued = !!admission || history?.status === 'queued' || inMemory?.status === 'queued';
3707
+ // Queued cancellation terminalizes descendants in the same transaction
3708
+ // as the root intent. Their callbacks are nevertheless owned by this
3709
+ // checkpoint, so an already-skipped snapshot still needs delivery. A
3710
+ // different terminal status means a newer action won and is left alone.
3711
+ const alreadySkipped = history?.status === 'skipped' || inMemory?.status === 'skipped';
3712
+ if (!stillQueued && !alreadySkipped)
3713
+ continue;
3714
+ const finishedAt = new Date().toISOString();
3715
+ const reason = `Parent job ${descendant.parentId === parentJobId ? parentJobId : descendant.parentId} ${descendant.parentId === parentJobId ? parentStatus : 'was skipped'}`;
3716
+ if (stillQueued) {
3717
+ const index = this._queue.indexOf(descendant.id);
3718
+ if (index !== -1)
3719
+ this._queue.splice(index, 1);
3720
+ if (inMemory) {
3721
+ inMemory.status = 'skipped';
3722
+ inMemory.finishedAt = finishedAt;
3723
+ inMemory.skipReason = reason;
3724
+ inMemory.queuePosition = null;
3725
+ }
3726
+ if (history) {
3727
+ (0, db_1.skipJob)(db, descendant.id, reason);
3728
+ }
3729
+ else {
3730
+ const parentStillExists = db.prepare(`SELECT 1 FROM jobs WHERE id = ?`).get(descendant.parentId);
3731
+ db.prepare(`
3732
+ INSERT INTO jobs (
3733
+ id, command, started_at, status, priority, skip_reason, finished_at,
3734
+ depends_on_job_id, pipeline_id, causal_ownership
3735
+ ) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?, ?)
3736
+ `).run(descendant.id, descendant.command, finishedAt, descendant.priority, reason, finishedAt, parentStillExists ? descendant.parentId : null, descendant.pipelineId, descendant.causalOwnership === true ? 1 : 0);
3737
+ }
3738
+ (0, db_1.deleteQueuedJob)(db, descendant.id);
3739
+ }
3740
+ this._onJobFinished?.(descendant.id, 'skipped', undefined, {
3741
+ recoveryReplay: true,
3742
+ recoveryCommand: descendant.command,
3743
+ recoveryTicketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(descendant.command),
3744
+ recoveryDurationMs: null,
3745
+ recoveryCausalOwnership: descendant.causalOwnership === true,
3746
+ });
3747
+ }
3748
+ this._recomputePositions();
3749
+ this._persistQueuedState(true);
3750
+ }
3751
+ /** Rebuild the best durable usage frontier from raw provider frames. For
3752
+ * interactive Claude sessions each result is a per-turn token snapshot but a
3753
+ * cumulative cost/turn snapshot, so segments must be folded exactly like the
3754
+ * live session; assistant-only tail frames are one estimated in-flight turn. */
3755
+ _recoverPersistedJobUsage(jobId, providerId, fallbackModel, interactive) {
3756
+ const db = this._db;
3757
+ if (!db)
3758
+ return { result: {}, estimated: false, authoritative: false };
3759
+ let adapter;
3760
+ try {
3761
+ adapter = (0, providers_1.getAdapter)(providerId);
3762
+ }
3763
+ catch {
3764
+ adapter = this._adapter;
3765
+ }
3766
+ const parse = (row) => {
3767
+ try {
3768
+ return adapter.parseStreamLine(row.payload);
3769
+ }
3770
+ catch (err) {
3771
+ console.warn(`[queue-manager] ignored malformed durable event for ${jobId}:`, err);
3772
+ return null;
3773
+ }
3774
+ };
3775
+ const makeUsageAccumulator = () => {
3776
+ const totals = {
3777
+ tokens_in: 0,
3778
+ tokens_out: 0,
3779
+ tokens_cache_read: 0,
3780
+ tokens_cache_create: 0,
3781
+ total_cost_usd: 0,
3782
+ num_turns: 0,
3783
+ duration_ms: 0,
3784
+ duration_api_ms: 0,
3785
+ };
3786
+ let model = fallbackModel ?? undefined;
3787
+ let sessionId;
3788
+ let estimated = false;
3789
+ const snapshots = new Map();
3790
+ const MAX_SNAPSHOT_KEYS = 4_096;
3791
+ const add = (event, seq) => {
3792
+ const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], { fallbackModel: model });
3793
+ const result = sanitizeRecoveredResult(finalised.result);
3794
+ const raw = event.kind === 'other' ? event.raw : null;
3795
+ const message = raw && typeof raw.message === 'object' && raw.message
3796
+ ? raw.message
3797
+ : null;
3798
+ const directMessageId = event.messageId;
3799
+ const messageId = directMessageId ?? (message && typeof message.id === 'string' ? message.id : undefined);
3800
+ const stableId = messageId ? `message:${messageId}` : `${event.kind}:${seq}`;
3801
+ const previous = snapshots.get(stableId) ?? {};
3802
+ const addDelta = (key) => {
3803
+ const current = result[key];
3804
+ if (typeof current !== 'number')
3805
+ return;
3806
+ const prior = previous[key];
3807
+ totals[key] += Math.max(0, current - (typeof prior === 'number' ? prior : 0));
3808
+ };
3809
+ addDelta('tokens_in');
3810
+ addDelta('tokens_out');
3811
+ addDelta('tokens_cache_read');
3812
+ addDelta('tokens_cache_create');
3813
+ addDelta('total_cost_usd');
3814
+ addDelta('num_turns');
3815
+ addDelta('duration_ms');
3816
+ addDelta('duration_api_ms');
3817
+ model = result.model ?? model;
3818
+ sessionId = result.session_id ?? sessionId;
3819
+ estimated = estimated || finalised.estimated;
3820
+ if (!snapshots.has(stableId) && snapshots.size >= MAX_SNAPSHOT_KEYS) {
3821
+ snapshots.delete(snapshots.keys().next().value);
3822
+ }
3823
+ // Frames for one provider message are contiguous; retain a bounded LRU
3824
+ // of recent snapshots for retransmission/delta dedupe without letting a
3825
+ // malicious transcript allocate one map entry per event forever.
3826
+ snapshots.delete(stableId);
3827
+ snapshots.set(stableId, { ...previous, ...result });
3828
+ };
3829
+ const result = () => {
3830
+ const hasUsage = Object.values(totals).some((value) => value > 0);
3831
+ return {
3832
+ result: hasUsage ? { ...totals, model, session_id: sessionId } : {},
3833
+ estimated,
3834
+ };
3835
+ };
3836
+ return { add, result };
3837
+ };
3838
+ if (!interactive) {
3839
+ const accumulator = makeUsageAccumulator();
3840
+ let lastValidResult = null;
3841
+ const events = db.prepare(`
3842
+ SELECT seq, event_type, payload FROM events
3843
+ WHERE job_id = ? AND source = 'stdout' AND event_type != 'log'
3844
+ ORDER BY seq, id
3845
+ `).iterate(jobId);
3846
+ for (const row of events) {
3847
+ const event = parse(row);
3848
+ if (event?.kind === 'result') {
3849
+ const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], {
3850
+ fallbackModel: fallbackModel ?? undefined,
3851
+ });
3852
+ lastValidResult = {
3853
+ ...finalised,
3854
+ result: sanitizeRecoveredResult(finalised.result),
3855
+ };
3856
+ }
3857
+ else if (event) {
3858
+ accumulator.add(event, row.seq);
3859
+ }
3860
+ }
3861
+ const fallback = accumulator.result();
3862
+ if (lastValidResult) {
3863
+ const result = { ...fallback.result };
3864
+ for (const [key, value] of Object.entries(lastValidResult.result)) {
3865
+ // Provider normalisers intentionally retain optional keys with an
3866
+ // undefined value. Only concrete terminal fields are authoritative;
3867
+ // an omitted field must not erase recoverable assistant evidence.
3868
+ if (value !== undefined) {
3869
+ result[key] = value;
3870
+ }
3871
+ }
3872
+ return {
3873
+ result,
3874
+ // A native terminal cost is authoritative. When the terminal frame
3875
+ // omits cost, preserve whether the assistant-frame backfill was an
3876
+ // estimate instead of silently relabelling it as exact.
3877
+ estimated: result.total_cost_usd === lastValidResult.result.total_cost_usd &&
3878
+ lastValidResult.result.total_cost_usd !== undefined
3879
+ ? lastValidResult.estimated
3880
+ : (lastValidResult.estimated || fallback.estimated),
3881
+ authoritative: true,
3882
+ };
3883
+ }
3884
+ return { ...fallback, authoritative: false };
3885
+ }
3886
+ const totals = {
3887
+ tokens_in: 0,
3888
+ tokens_out: 0,
3889
+ tokens_cache_read: 0,
3890
+ tokens_cache_create: 0,
3891
+ total_cost_usd: 0,
3892
+ num_turns: 0,
3893
+ duration_ms: 0,
3894
+ duration_api_ms: 0,
3895
+ };
3896
+ let model = fallbackModel ?? undefined;
3897
+ let sessionId;
3898
+ let baselineCost = 0;
3899
+ let baselineTurns = 0;
3900
+ let estimated = false;
3901
+ let lastResultSeq = -1;
3902
+ const resultRows = db.prepare(`
3903
+ SELECT seq, event_type, payload FROM events
3904
+ WHERE job_id = ? AND source = 'stdout' AND event_type = 'result'
3905
+ ORDER BY seq, id
3906
+ `).iterate(jobId);
3907
+ for (const row of resultRows) {
3908
+ const event = parse(row);
3909
+ if (event?.kind !== 'result')
3910
+ continue;
3911
+ const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], { fallbackModel: model });
3912
+ const result = sanitizeRecoveredResult(finalised.result);
3913
+ totals.tokens_in += result.tokens_in ?? 0;
3914
+ totals.tokens_out += result.tokens_out ?? 0;
3915
+ totals.tokens_cache_read += result.tokens_cache_read ?? 0;
3916
+ totals.tokens_cache_create += result.tokens_cache_create ?? 0;
3917
+ const cumulativeCost = result.total_cost_usd ?? baselineCost;
3918
+ totals.total_cost_usd += Math.max(0, cumulativeCost - baselineCost);
3919
+ baselineCost = cumulativeCost;
3920
+ const cumulativeTurns = result.num_turns ?? (baselineTurns + 1);
3921
+ totals.num_turns += Math.max(0, cumulativeTurns - baselineTurns);
3922
+ baselineTurns = cumulativeTurns;
3923
+ totals.duration_ms += result.duration_ms ?? 0;
3924
+ totals.duration_api_ms += result.duration_api_ms ?? 0;
3925
+ model = result.model ?? model;
3926
+ sessionId = result.session_id ?? sessionId;
3927
+ estimated = estimated || finalised.estimated;
3928
+ lastResultSeq = row.seq;
3929
+ }
3930
+ const tail = makeUsageAccumulator();
3931
+ const tailRows = db.prepare(`
3932
+ SELECT seq, event_type, payload FROM events
3933
+ WHERE job_id = ? AND source = 'stdout' AND event_type != 'log' AND seq > ?
3934
+ ORDER BY seq, id
3935
+ `).iterate(jobId, lastResultSeq);
3936
+ for (const row of tailRows) {
3937
+ const event = parse(row);
3938
+ if (event && event.kind !== 'result')
3939
+ tail.add(event, row.seq);
3940
+ }
3941
+ const tailUsage = tail.result();
3942
+ const tailResult = tailUsage.result;
3943
+ const hasTailUsage = [
3944
+ tailResult.tokens_in,
3945
+ tailResult.tokens_out,
3946
+ tailResult.tokens_cache_read,
3947
+ tailResult.tokens_cache_create,
3948
+ tailResult.total_cost_usd,
3949
+ ].some((value) => typeof value === 'number' && value > 0);
3950
+ if (hasTailUsage) {
3951
+ totals.tokens_in += tailResult.tokens_in ?? 0;
3952
+ totals.tokens_out += tailResult.tokens_out ?? 0;
3953
+ totals.tokens_cache_read += tailResult.tokens_cache_read ?? 0;
3954
+ totals.tokens_cache_create += tailResult.tokens_cache_create ?? 0;
3955
+ totals.total_cost_usd += tailResult.total_cost_usd ?? 0;
3956
+ totals.num_turns += tailResult.num_turns ?? 1;
3957
+ totals.duration_ms += tailResult.duration_ms ?? 0;
3958
+ totals.duration_api_ms += tailResult.duration_api_ms ?? 0;
3959
+ model = tailResult.model ?? model;
3960
+ sessionId = tailResult.session_id ?? sessionId;
3961
+ estimated = estimated || tailUsage.estimated;
3962
+ }
3963
+ const hasRecoveredUsage = Object.values(totals).some((value) => value > 0);
3964
+ return {
3965
+ result: hasRecoveredUsage
3966
+ ? {
3967
+ ...totals,
3968
+ model,
3969
+ session_id: sessionId,
3970
+ }
3971
+ : {},
3972
+ estimated,
3973
+ authoritative: false,
3974
+ };
3975
+ }
3976
+ _decodeRecoveryPayload(jobId, encoded) {
3977
+ const payload = JSON.parse(encoded);
3978
+ if (!payload || typeof payload !== 'object')
3979
+ throw new Error('payload is not an object');
3980
+ if (typeof payload.command !== 'string')
3981
+ throw new Error('payload command is invalid');
3982
+ // Ticket ownership is derived from the immutable command, never trusted
3983
+ // from a legacy/corrupt payload. This also repairs pre-fix [0]/overflow ids.
3984
+ payload.ticketIds = (0, ticket_store_1.extractTicketIdsFromCommand)(payload.command);
3985
+ payload.totalCostUsdEstimated ??= 0;
3986
+ const terminalStatus = payload.terminalStatus ?? 'failed';
3987
+ const invocationStatus = payload.invocationStatus ?? 'aborted';
3988
+ const nullableNumber = (value) => value == null || (typeof value === 'number' && Number.isFinite(value) && value >= 0);
3989
+ if (payload.id !== jobId ||
3990
+ typeof payload.startedAt !== 'string' ||
3991
+ typeof payload.finishedAt !== 'string' ||
3992
+ typeof payload.provider !== 'string' || !payload.provider ||
3993
+ (payload.pipelineId !== null && typeof payload.pipelineId !== 'string') ||
3994
+ !payload.ticketIds.every((id) => Number.isSafeInteger(id) && id > 0) ||
3995
+ !nullableNumber(payload.tokensIn) ||
3996
+ !nullableNumber(payload.tokensOut) ||
3997
+ !nullableNumber(payload.tokensCacheRead) ||
3998
+ !nullableNumber(payload.tokensCacheCreate) ||
3999
+ !nullableNumber(payload.totalCostUsd) ||
4000
+ !nullableNumber(payload.numTurns) ||
4001
+ !nullableNumber(payload.durationMs) ||
4002
+ !nullableNumber(payload.durationApiMs) ||
4003
+ (payload.totalCostUsdEstimated !== 0 && payload.totalCostUsdEstimated !== 1) ||
4004
+ (payload.exitCode != null && !Number.isInteger(payload.exitCode)) ||
4005
+ (payload.model !== null && typeof payload.model !== 'string') ||
4006
+ (payload.sessionId !== null && typeof payload.sessionId !== 'string') ||
4007
+ (payload.causalOwnership !== undefined && typeof payload.causalOwnership !== 'boolean') ||
4008
+ (payload.awaitingLateReconciliation !== undefined && typeof payload.awaitingLateReconciliation !== 'boolean') ||
4009
+ (payload.ticketCompletionStatus !== undefined && !['done', 'on_review'].includes(payload.ticketCompletionStatus)) ||
4010
+ !TERMINAL_STATUSES.has(terminalStatus) ||
4011
+ !['success', 'failed', 'aborted'].includes(invocationStatus) ||
4012
+ (payload.descendants !== undefined && (!Array.isArray(payload.descendants) ||
4013
+ !payload.descendants.every((descendant) => descendant && typeof descendant.id === 'string' &&
4014
+ typeof descendant.command === 'string' &&
4015
+ typeof descendant.parentId === 'string' &&
4016
+ (descendant.pipelineId === null || typeof descendant.pipelineId === 'string') &&
4017
+ types_1.VALID_PRIORITIES.has(descendant.priority) &&
4018
+ (descendant.causalOwnership === undefined || typeof descendant.causalOwnership === 'boolean'))))) {
4019
+ throw new Error('payload identity or fields are invalid');
4020
+ }
4021
+ return { payload, terminalStatus, invocationStatus };
4022
+ }
4023
+ /**
4024
+ * Atomically convert every crash-orphaned RUNNING job into a durable recovery
4025
+ * intent and a failed job. The outbox snapshot is committed in the SAME
4026
+ * SQLite transaction as the status flip, so a process death can leave either
4027
+ * the original running row or a replayable failed row — never an untracked
4028
+ * half-recovery.
4029
+ */
4030
+ _captureOrphanRecoveries() {
4031
+ const db = this._db;
4032
+ if (!db)
4033
+ return;
4034
+ const capture = db.transaction(() => {
4035
+ // A server restart proves that no prior-process `close` listener can
4036
+ // still enrich a force-failed job. Release those holds and retain the
4037
+ // partial usage already checkpointed as the best available evidence.
4038
+ const heldRecoveries = db.prepare(`
4039
+ SELECT recovery.job_id, recovery.payload, recovery.accounting_completed,
4040
+ jobs.provider, jobs.model, jobs.interactive
4041
+ FROM orphan_job_recovery AS recovery
4042
+ LEFT JOIN jobs ON jobs.id = recovery.job_id
4043
+ `).all();
4044
+ const releaseHold = db.prepare(`
4045
+ UPDATE orphan_job_recovery
4046
+ SET payload = ?, accounting_completed = ?
4047
+ WHERE job_id = ?
4048
+ `);
4049
+ for (const held of heldRecoveries) {
4050
+ let decoded;
4051
+ try {
4052
+ decoded = this._decodeRecoveryPayload(held.job_id, held.payload);
4053
+ }
4054
+ catch (err) {
4055
+ console.error(`[queue-manager] invalid held recovery payload for ${held.job_id}:`, err);
4056
+ throw err;
4057
+ }
4058
+ const { payload, terminalStatus } = decoded;
4059
+ if (!payload.awaitingLateReconciliation)
4060
+ continue;
4061
+ const recovered = this._recoverPersistedJobUsage(held.job_id, held.provider ?? payload.provider ?? this._adapter.id, held.model ?? payload.model, held.interactive === 1);
4062
+ const hasRecoveredUsage = [
4063
+ recovered.result.tokens_in,
4064
+ recovered.result.tokens_out,
4065
+ recovered.result.tokens_cache_read,
4066
+ recovered.result.tokens_cache_create,
4067
+ recovered.result.total_cost_usd,
4068
+ ].some((value) => typeof value === 'number' && value > 0);
4069
+ const hasRecoveredEvidence = hasRecoveredUsage || recovered.authoritative;
4070
+ if (hasRecoveredEvidence) {
4071
+ if (recovered.authoritative) {
4072
+ payload.model = recovered.result.model ?? null;
4073
+ payload.tokensIn = recovered.result.tokens_in ?? null;
4074
+ payload.tokensOut = recovered.result.tokens_out ?? null;
4075
+ payload.tokensCacheRead = recovered.result.tokens_cache_read ?? null;
4076
+ payload.tokensCacheCreate = recovered.result.tokens_cache_create ?? null;
4077
+ payload.totalCostUsd = recovered.result.total_cost_usd ?? null;
4078
+ payload.totalCostUsdEstimated = recovered.estimated ? 1 : 0;
4079
+ payload.numTurns = recovered.result.num_turns ?? null;
4080
+ payload.durationMs = recovered.result.duration_ms ?? null;
4081
+ payload.durationApiMs = recovered.result.duration_api_ms ?? null;
4082
+ payload.sessionId = recovered.result.session_id ?? null;
4083
+ }
4084
+ else {
4085
+ payload.model = recovered.result.model ?? payload.model;
4086
+ payload.tokensIn = maxNullable(payload.tokensIn, recovered.result.tokens_in);
4087
+ payload.tokensOut = maxNullable(payload.tokensOut, recovered.result.tokens_out);
4088
+ payload.tokensCacheRead = maxNullable(payload.tokensCacheRead, recovered.result.tokens_cache_read);
4089
+ payload.tokensCacheCreate = maxNullable(payload.tokensCacheCreate, recovered.result.tokens_cache_create);
4090
+ payload.totalCostUsd = maxNullable(payload.totalCostUsd, recovered.result.total_cost_usd);
4091
+ payload.totalCostUsdEstimated = (payload.totalCostUsdEstimated || recovered.estimated) ? 1 : 0;
4092
+ payload.numTurns = maxNullable(payload.numTurns, recovered.result.num_turns);
4093
+ payload.durationMs = maxNullable(payload.durationMs, recovered.result.duration_ms);
4094
+ payload.durationApiMs = maxNullable(payload.durationApiMs, recovered.result.duration_api_ms);
4095
+ payload.sessionId = recovered.result.session_id ?? payload.sessionId;
4096
+ }
4097
+ this._decodeRecoveryPayload(held.job_id, JSON.stringify(payload));
4098
+ (0, db_1.finishJob)(db, held.job_id, {
4099
+ exit_code: payload.exitCode ?? -1,
4100
+ status: terminalStatus,
4101
+ tokens_in: payload.tokensIn ?? undefined,
4102
+ tokens_out: payload.tokensOut ?? undefined,
4103
+ tokens_cache_read: payload.tokensCacheRead ?? undefined,
4104
+ tokens_cache_create: payload.tokensCacheCreate ?? undefined,
4105
+ total_cost_usd: payload.totalCostUsd ?? undefined,
4106
+ total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
4107
+ num_turns: payload.numTurns ?? undefined,
4108
+ model: payload.model ?? undefined,
4109
+ duration_ms: payload.durationMs ?? undefined,
4110
+ duration_api_ms: payload.durationApiMs ?? undefined,
4111
+ session_id: payload.sessionId ?? undefined,
4112
+ });
4113
+ }
4114
+ payload.awaitingLateReconciliation = false;
4115
+ const shouldReaccount = hasRecoveredEvidence && !!this._projectId;
4116
+ if (shouldReaccount) {
4117
+ db.prepare(`DELETE FROM ai_invocations WHERE surface_ref_id = ? OR surface_ref_id LIKE ?`).run(held.job_id, `${held.job_id}#t%`);
4118
+ }
4119
+ releaseHold.run(JSON.stringify(payload), shouldReaccount ? 0 : held.accounting_completed, held.job_id);
4120
+ }
4121
+ const orphans = db.prepare(`SELECT id, command, pipeline_id, provider, started_at, model, tokens_in, tokens_out, tokens_cache_read,
4122
+ tokens_cache_create, total_cost_usd, total_cost_usd_estimated, num_turns,
4123
+ duration_ms, duration_api_ms, session_id, causal_ownership, interactive
4124
+ FROM jobs WHERE status = 'running' AND owner = 'queue'`).all();
4125
+ if (orphans.length === 0)
4126
+ return;
4127
+ const finishedAt = new Date().toISOString();
4128
+ const insert = db.prepare(`INSERT OR IGNORE INTO orphan_job_recovery (job_id, payload)
4129
+ VALUES (?, ?)`);
4130
+ for (const orphan of orphans) {
4131
+ const finalised = this._recoverPersistedJobUsage(orphan.id, orphan.provider ?? this._adapter.id, orphan.model, orphan.interactive === 1);
4132
+ const recovered = finalised.result;
4133
+ const recoveredEstimated = finalised.estimated;
4134
+ const choose = (persisted, raw) => finalised.authoritative ? (raw ?? null) : maxNullable(persisted, raw);
4135
+ const payload = {
4136
+ id: orphan.id,
4137
+ command: orphan.command,
4138
+ ticketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(orphan.command),
4139
+ pipelineId: orphan.pipeline_id,
4140
+ startedAt: orphan.started_at ?? finishedAt,
4141
+ finishedAt,
4142
+ // Old rows predate jobs.provider; their only honest fallback is the
4143
+ // project's primary adapter. New rows preserve per-job overrides.
4144
+ provider: orphan.provider ?? this._adapter.id,
4145
+ model: finalised.authoritative
4146
+ ? (recovered.model ?? null)
4147
+ : (recovered.model ?? orphan.model ?? null),
4148
+ tokensIn: choose(orphan.tokens_in, recovered.tokens_in),
4149
+ tokensOut: choose(orphan.tokens_out, recovered.tokens_out),
4150
+ tokensCacheRead: choose(orphan.tokens_cache_read, recovered.tokens_cache_read),
4151
+ tokensCacheCreate: choose(orphan.tokens_cache_create, recovered.tokens_cache_create),
4152
+ totalCostUsd: choose(orphan.total_cost_usd, recovered.total_cost_usd),
4153
+ totalCostUsdEstimated: finalised.authoritative
4154
+ ? (recoveredEstimated ? 1 : 0)
4155
+ : ((orphan.total_cost_usd_estimated || recoveredEstimated) ? 1 : 0),
4156
+ numTurns: choose(orphan.num_turns, recovered.num_turns),
4157
+ durationMs: choose(orphan.duration_ms, recovered.duration_ms),
4158
+ durationApiMs: choose(orphan.duration_api_ms, recovered.duration_api_ms),
4159
+ sessionId: finalised.authoritative
4160
+ ? (recovered.session_id ?? null)
4161
+ : (recovered.session_id ?? orphan.session_id ?? null),
4162
+ descendants: this._snapshotRecoveryDescendants(orphan.id),
4163
+ causalOwnership: orphan.causal_ownership === 1,
4164
+ terminalStatus: 'failed',
4165
+ invocationStatus: 'aborted',
4166
+ exitCode: -1,
4167
+ };
4168
+ const encoded = JSON.stringify(this._decodeRecoveryPayload(orphan.id, JSON.stringify(payload)).payload);
4169
+ insert.run(orphan.id, encoded);
4170
+ (0, db_1.finishJob)(db, orphan.id, {
4171
+ exit_code: -1,
4172
+ status: 'failed',
4173
+ tokens_in: payload.tokensIn ?? undefined,
4174
+ tokens_out: payload.tokensOut ?? undefined,
4175
+ tokens_cache_read: payload.tokensCacheRead ?? undefined,
4176
+ tokens_cache_create: payload.tokensCacheCreate ?? undefined,
4177
+ total_cost_usd: payload.totalCostUsd ?? undefined,
4178
+ total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
4179
+ num_turns: payload.numTurns ?? undefined,
4180
+ model: payload.model ?? undefined,
4181
+ duration_ms: payload.durationMs ?? undefined,
4182
+ duration_api_ms: payload.durationApiMs ?? undefined,
4183
+ session_id: payload.sessionId ?? undefined,
4184
+ });
4185
+ }
4186
+ });
4187
+ capture();
4188
+ }
4189
+ /**
4190
+ * Drain the orphan outbox with an independent durable checkpoint per effect.
4191
+ * Accounting runs in the same transaction as its checkpoint, which makes it
4192
+ * exactly-once across arbitrary process death. The domain callback is a
4193
+ * conventional at-least-once outbox delivery: its DB mutations and checkpoint
4194
+ * share a transaction, and every replay carries the same stable job id so its
4195
+ * idempotent ticket/rail/Jira mutations can safely converge after a crash. A
4196
+ * third checkpoint replays the normal dependent-skip/pipeline invariants only
4197
+ * after queued rows have been restored into `_jobs` and `_queue`.
4198
+ */
4199
+ _resumeOrphanRecoveries() {
4200
+ try {
4201
+ return this._resumeOrphanRecoveriesUnsafe();
4202
+ }
4203
+ catch (err) {
4204
+ // A durable effect remains pending by construction. Surface a fail-stop
4205
+ // result instead of letting an EventEmitter close handler throw uncaught
4206
+ // or allowing later provider work to overtake unknown accounting.
4207
+ console.error('[queue-manager] terminal recovery drain failed:', err);
4208
+ return false;
4209
+ }
4210
+ }
4211
+ _resumeOrphanRecoveriesUnsafe() {
4212
+ const db = this._db;
4213
+ if (!db)
4214
+ return true;
4215
+ const rows = db.prepare(`SELECT job_id, payload, accounting_completed, callback_completed, terminal_completed
4216
+ FROM orphan_job_recovery
4217
+ WHERE accounting_completed = 0 OR callback_completed = 0 OR terminal_completed = 0
4218
+ ORDER BY created_at, job_id`).all();
4219
+ let spendingChanged = false;
4220
+ let invalidRecovery = false;
4221
+ for (const row of rows) {
4222
+ let payload;
4223
+ let terminalStatus;
4224
+ let invocationStatus;
4225
+ try {
4226
+ ({ payload, terminalStatus, invocationStatus } = this._decodeRecoveryPayload(row.job_id, row.payload));
4227
+ }
4228
+ catch (err) {
4229
+ console.error(`[queue-manager] invalid orphan recovery payload for ${row.job_id}:`, err);
4230
+ invalidRecovery = true;
4231
+ continue;
4232
+ }
4233
+ if (row.accounting_completed === 0 && !this._projectId) {
4234
+ db.prepare(`UPDATE orphan_job_recovery SET accounting_completed = 1 WHERE job_id = ?`)
4235
+ .run(row.job_id);
4236
+ }
4237
+ else if (row.accounting_completed === 0 && this._projectId) {
4238
+ try {
4239
+ const recordAccounting = db.transaction(() => {
4240
+ const pending = db.prepare(`SELECT accounting_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
4241
+ if (!pending || pending.accounting_completed !== 0)
4242
+ return false;
4243
+ this._recordJobInvocations({
4244
+ jobId: payload.id,
4245
+ provider: payload.provider,
4246
+ status: invocationStatus,
4247
+ startedAt: payload.startedAt,
4248
+ finishedAt: payload.finishedAt,
4249
+ ticketIds: payload.ticketIds,
4250
+ estimated: !!payload.totalCostUsdEstimated,
4251
+ result: {
4252
+ tokens_in: payload.tokensIn ?? undefined,
4253
+ tokens_out: payload.tokensOut ?? undefined,
4254
+ tokens_cache_read: payload.tokensCacheRead ?? undefined,
4255
+ tokens_cache_create: payload.tokensCacheCreate ?? undefined,
4256
+ total_cost_usd: payload.totalCostUsd ?? undefined,
4257
+ num_turns: payload.numTurns ?? undefined,
4258
+ model: payload.model ?? undefined,
4259
+ session_id: payload.sessionId ?? undefined,
4260
+ duration_ms: payload.durationMs ?? undefined,
4261
+ duration_api_ms: payload.durationApiMs ?? undefined,
4262
+ },
4263
+ });
4264
+ db.prepare(`UPDATE orphan_job_recovery SET accounting_completed = 1 WHERE job_id = ?`).run(row.job_id);
4265
+ return true;
4266
+ });
4267
+ spendingChanged = recordAccounting() || spendingChanged;
4268
+ }
4269
+ catch (err) {
4270
+ console.error(`[queue-manager] orphan accounting replay failed for ${row.job_id}:`, err);
4271
+ }
4272
+ }
4273
+ if (row.callback_completed === 0 && !this._onJobFinished) {
4274
+ db.prepare(`UPDATE orphan_job_recovery SET callback_completed = 1 WHERE job_id = ?`)
4275
+ .run(row.job_id);
4276
+ }
4277
+ else if (row.callback_completed === 0 && this._onJobFinished) {
4278
+ try {
4279
+ const deliverCallback = db.transaction(() => {
4280
+ const pending = db.prepare(`SELECT callback_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
4281
+ if (!pending || pending.callback_completed !== 0)
4282
+ return;
4283
+ this._onJobFinished?.(payload.id, terminalStatus, payload.totalCostUsd ?? undefined, {
4284
+ recoveryReplay: true,
4285
+ recoveryCommand: payload.command,
4286
+ recoveryTicketIds: payload.ticketIds,
4287
+ recoveryDurationMs: payload.durationMs,
4288
+ recoveryCausalOwnership: payload.causalOwnership === true,
4289
+ ...(terminalStatus === 'completed' && payload.ticketCompletionStatus
4290
+ ? { ticketCompletionStatus: payload.ticketCompletionStatus }
4291
+ : {}),
4292
+ });
4293
+ db.prepare(`UPDATE orphan_job_recovery SET callback_completed = 1 WHERE job_id = ?`).run(row.job_id);
4294
+ });
4295
+ deliverCallback();
4296
+ }
4297
+ catch (err) {
4298
+ console.error(`[queue-manager] orphan outcome replay failed for ${row.job_id}: ${err.message}`);
4299
+ }
4300
+ }
4301
+ if (row.terminal_completed === 0) {
4302
+ // `_skipDependents` updates the in-memory projection before its SQL.
4303
+ // SQLite can roll the transaction back, so retain an equally atomic
4304
+ // rollback image; otherwise a transient checkpoint failure removes
4305
+ // durable queued work from this process until another restart.
4306
+ const queueBefore = [...this._queue];
4307
+ const jobsBefore = new Map(Array.from(this._jobs, ([id, job]) => [id, {
4308
+ status: job.status,
4309
+ finishedAt: job.finishedAt,
4310
+ skipReason: job.skipReason,
4311
+ queuePosition: job.queuePosition,
4312
+ }]));
4313
+ try {
4314
+ const replayTerminalInvariants = db.transaction(() => {
4315
+ const pending = db.prepare(`SELECT terminal_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
4316
+ if (!pending || pending.terminal_completed !== 0)
4317
+ return false;
4318
+ // Reuse the exact recursive terminal semantics used by live failed
4319
+ // jobs. At this point startup has already restored every queued row
4320
+ // into memory, so descendants are persisted as skipped and removed
4321
+ // from the runnable queue just as they are on a normal exit.
4322
+ // Compatibility with intents written before descendant snapshots:
4323
+ // materialise once and reuse it for both recovery-safe callbacks
4324
+ // and cross-pipeline status evaluation.
4325
+ const recoveryDescendants = Array.isArray(payload.descendants)
4326
+ ? payload.descendants
4327
+ : this._snapshotRecoveryDescendants(payload.id);
4328
+ if (terminalStatus !== 'completed') {
4329
+ this._skipRecoveredDescendants(payload.id, recoveryDescendants, terminalStatus);
4330
+ }
4331
+ // Descendants may intentionally belong to different pipelines.
4332
+ // Recompute every affected pipeline before checkpointing so a
4333
+ // crash replays any missed notification instead of orphaning the
4334
+ // child's pipeline in a stale running state.
4335
+ const affectedPipelines = new Set();
4336
+ if (payload.pipelineId)
4337
+ affectedPipelines.add(payload.pipelineId);
4338
+ for (const descendant of recoveryDescendants) {
4339
+ if (descendant.pipelineId)
4340
+ affectedPipelines.add(descendant.pipelineId);
4341
+ }
4342
+ for (const pipelineId of affectedPipelines) {
4343
+ this._checkPersistedPipelineStatus(pipelineId);
4344
+ }
4345
+ db.prepare(`UPDATE orphan_job_recovery SET terminal_completed = 1 WHERE job_id = ?`).run(row.job_id);
4346
+ return true;
4347
+ });
4348
+ replayTerminalInvariants();
4349
+ }
4350
+ catch (err) {
4351
+ this._queue = queueBefore;
4352
+ for (const [id, before] of jobsBefore) {
4353
+ const job = this._jobs.get(id);
4354
+ if (!job)
4355
+ continue;
4356
+ job.status = before.status;
4357
+ job.finishedAt = before.finishedAt;
4358
+ job.skipReason = before.skipReason;
4359
+ job.queuePosition = before.queuePosition;
4360
+ }
4361
+ console.error(`[queue-manager] orphan terminal replay failed for ${row.job_id}: ${err.message}`);
4362
+ }
4363
+ }
4364
+ }
4365
+ // A crash before this cleanup is harmless: completed rows are skipped on
4366
+ // the next startup and removed then.
4367
+ const completedRows = db.prepare(`
4368
+ SELECT job_id, payload FROM orphan_job_recovery
4369
+ WHERE accounting_completed = 1
4370
+ AND callback_completed = 1
4371
+ AND terminal_completed = 1
4372
+ `).all();
4373
+ const removeCompleted = db.prepare(`DELETE FROM orphan_job_recovery WHERE job_id = ?`);
4374
+ const cleanupCompleted = db.transaction(() => {
4375
+ for (const completed of completedRows) {
4376
+ let payload;
4377
+ try {
4378
+ payload = this._decodeRecoveryPayload(completed.job_id, completed.payload).payload;
4379
+ }
4380
+ catch (err) {
4381
+ throw new Error(`Invalid completed recovery payload for ${completed.job_id}: ${err.message}`);
4382
+ }
4383
+ if (payload.awaitingLateReconciliation)
4384
+ continue;
4385
+ removeCompleted.run(completed.job_id);
4386
+ }
4387
+ });
4388
+ cleanupCompleted();
4389
+ if (spendingChanged && this._projectId) {
4390
+ try {
4391
+ this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
4392
+ }
4393
+ catch { /* advisory */ }
4394
+ }
4395
+ const pendingCritical = db.prepare(`
4396
+ SELECT 1 FROM orphan_job_recovery
4397
+ WHERE accounting_completed = 0 OR terminal_completed = 0
4398
+ LIMIT 1
4399
+ `).get();
4400
+ return !pendingCritical && !invalidRecovery;
2731
4401
  }
2732
4402
  _checkPipelineStatus(pipelineId) {
2733
4403
  const pipelineJobs = Array.from(this._jobs.values()).filter(j => j.pipelineId === pipelineId);
@@ -2736,11 +4406,50 @@ class QueueManager {
2736
4406
  const allDone = pipelineJobs.every(j => j.status === 'completed');
2737
4407
  const anyFailed = pipelineJobs.some(j => j.status === 'failed' || j.status === 'skipped' || j.status === 'canceled' || j.status === 'zombie_terminated');
2738
4408
  const anyPending = pipelineJobs.some(j => j.status === 'queued' || j.status === 'running');
2739
- if (allDone) {
2740
- this._broadcast({ type: 'pipeline_status', pipelineId, status: 'completed' });
4409
+ const status = allDone ? 'completed' : (anyFailed && !anyPending ? 'failed' : null);
4410
+ if (!status) {
4411
+ // A caller may append work to an existing pipeline id. Its next terminal
4412
+ // transition is new and must be observable again.
4413
+ this._emittedPipelineStatuses.delete(pipelineId);
4414
+ return;
2741
4415
  }
2742
- else if (anyFailed && !anyPending) {
2743
- this._broadcast({ type: 'pipeline_status', pipelineId, status: 'failed' });
4416
+ if (this._emittedPipelineStatuses.get(pipelineId) === status)
4417
+ return;
4418
+ this._broadcast({ type: 'pipeline_status', pipelineId, status });
4419
+ this._emittedPipelineStatuses.set(pipelineId, status);
4420
+ }
4421
+ /**
4422
+ * Recovery counterpart of `_checkPipelineStatus`. Terminal parents are not
4423
+ * restored into the in-memory job map, so startup must evaluate the complete
4424
+ * persisted pipeline rather than the queued-only `_jobs` view.
4425
+ */
4426
+ _checkPersistedPipelineStatus(pipelineId) {
4427
+ const db = this._db;
4428
+ if (!db)
4429
+ return;
4430
+ const rows = db.prepare(`SELECT status FROM jobs WHERE pipeline_id = ?
4431
+ UNION ALL
4432
+ SELECT 'queued' AS status FROM queued_jobs
4433
+ WHERE pipeline_id = ?
4434
+ AND NOT EXISTS (SELECT 1 FROM jobs WHERE jobs.id = queued_jobs.id)`).all(pipelineId, pipelineId);
4435
+ if (rows.length === 0)
4436
+ return;
4437
+ const allDone = rows.every((job) => job.status === 'completed');
4438
+ const anyFailed = rows.some((job) => job.status === 'failed' || job.status === 'skipped' || job.status === 'canceled' || job.status === 'zombie_terminated');
4439
+ const anyPending = rows.some((job) => job.status === 'queued' || job.status === 'running');
4440
+ try {
4441
+ const status = allDone ? 'completed' : (anyFailed && !anyPending ? 'failed' : null);
4442
+ if (!status) {
4443
+ this._emittedPipelineStatuses.delete(pipelineId);
4444
+ return;
4445
+ }
4446
+ if (this._emittedPipelineStatuses.get(pipelineId) === status)
4447
+ return;
4448
+ this._broadcast({ type: 'pipeline_status', pipelineId, status });
4449
+ this._emittedPipelineStatuses.set(pipelineId, status);
4450
+ }
4451
+ catch {
4452
+ // Startup broadcasts are advisory; persisted terminal state is authoritative.
2744
4453
  }
2745
4454
  }
2746
4455
  _recomputePositions() {