specrails-desktop 2.29.1 → 2.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/dist/assets/ActivityFeedPage-ClpD5CFj.js +1 -0
- package/client/dist/assets/AgentBrowserCapture-CSPs9Ofc.js +1 -0
- package/client/dist/assets/AgentModeCodePane-DwcyklA5.js +2 -0
- package/client/dist/assets/AgentModeJobsPane-C69bqlMa.js +1 -0
- package/client/dist/assets/AgentsPage-pRiRr9-W.js +86 -0
- package/client/dist/assets/AnalyticsPage-C14exKsC.js +1 -0
- package/client/dist/assets/BarChart-nBOMNfiy.js +33 -0
- package/client/dist/assets/CodePage-CK76JKm9.js +2 -0
- package/client/dist/assets/DesktopAnalyticsPage-Cnv0f3XQ.js +1 -0
- package/client/dist/assets/DocsDialog-CHUm_Ym0.js +11 -0
- package/client/dist/assets/DocsPage-COI-oDok.js +11 -0
- package/client/dist/assets/ExportDropdown-BrtfWzEG.js +1 -0
- package/client/dist/assets/IntegrationsPage-Bs00_tl4.js +1 -0
- package/client/dist/assets/InteractiveJobComposer-CypM7R1F.js +19 -0
- package/client/dist/assets/JobDetailModal-B_W0H9dw.js +1 -0
- package/client/dist/assets/JobDetailPage-CeeXOoKS.js +1 -0
- package/client/dist/assets/JobsPage-CwhJG9jw.js +1 -0
- package/client/dist/assets/LoopBuilderPage-C51t1WOM.js +7 -0
- package/client/dist/assets/LoopPreviewModal-Y2HgtIJu.js +1 -0
- package/client/dist/assets/LoopsPage-Bvn1Kw5V.js +1 -0
- package/client/dist/assets/MinimizedChatsContext-DiFvZDPW.js +1 -0
- package/client/dist/assets/PluginsPage-BGPRJpdf.js +2 -0
- package/client/dist/assets/ProjectSettingsDialog-D_GTyGLc.js +1 -0
- package/client/dist/assets/TemplatePreviewModal-szwBhd7h.js +1 -0
- package/client/dist/assets/TicketDetailModal-9UEyK974.js +14 -0
- package/client/dist/assets/Trans-B_vKZ-Hb.js +1 -0
- package/client/dist/assets/activity-DiyjsgQV.js +1 -0
- package/client/dist/assets/addon-image-BRmmRHgQ.js +1 -0
- package/client/dist/assets/addon-ligatures-qIZUPvRV.js +2 -0
- package/client/dist/assets/{addon-webgl-DMbGeI_u.js → addon-webgl-C2d7lriQ.js} +3 -3
- package/client/dist/assets/addspec-Rw8uVBC1.js +1 -0
- package/client/dist/assets/agent-4rmXoo_p.js +1 -0
- package/client/dist/assets/agent-BxZuNGih.js +1 -0
- package/client/dist/assets/agent-C3uXOYv_.js +1 -0
- package/client/dist/assets/agent-CA8Y4wbw.js +1 -0
- package/client/dist/assets/agent-CASKrhG12.js +1 -0
- package/client/dist/assets/agent-Cv-jud5n2.js +1 -0
- package/client/dist/assets/agent-DitUxYh2.js +1 -0
- package/client/dist/assets/agent-Doow4X7o.js +1 -0
- package/client/dist/assets/agents-DhG7mg6-.js +1 -0
- package/client/dist/assets/agentstudio-BIUe1Im2.js +1 -0
- package/client/dist/assets/aiedit-iIjz2_Qk.js +1 -0
- package/client/dist/assets/analytics-DZ8Aoq3I.js +1 -0
- package/client/dist/assets/api-B2M7CtDw.js +1 -0
- package/client/dist/assets/arrow-left-CpIR4j7G.js +1 -0
- package/client/dist/assets/arrow-up-right-9DoMn7fD.js +1 -0
- package/client/dist/assets/attachments-B53PG733.js +1 -0
- package/client/dist/assets/brain-DI9lHkKY.js +1 -0
- package/client/dist/assets/browser-CXUcnTMI.js +1 -0
- package/client/dist/assets/button-BUIIBQPT.js +1 -0
- package/client/dist/assets/chat-BGD7MXAy.js +1 -0
- package/client/dist/assets/chevron-right-CViMxucY.js +1 -0
- package/client/dist/assets/chunk-KS7C4IRE-Cx5sqopX.js +3 -0
- package/client/dist/assets/circle-x-CMYsN4ZH.js +1 -0
- package/client/dist/assets/code-BDXEM4ml.js +1 -0
- package/client/dist/assets/commands-DyUZQ1FI.js +1 -0
- package/client/dist/assets/common-CD1QHriA2.js +1 -0
- package/client/dist/assets/common-CwwPaRxx.js +1 -0
- package/client/dist/assets/common-DVQuhs-m.js +1 -0
- package/client/dist/assets/common-Dg2jaX7d.js +1 -0
- package/client/dist/assets/common-DuBwj6tK.js +1 -0
- package/client/dist/assets/common-MRj9MHcc.js +1 -0
- package/client/dist/assets/common-O4ZpnMLp.js +1 -0
- package/client/dist/assets/common-fyeF0v7C2.js +1 -0
- package/client/dist/assets/createLucideIcon-Bjbbp_KS.js +1 -0
- package/client/dist/assets/{css.worker-Wv5dxAWO.js → css.worker-yKcq13f2.js} +13 -13
- package/client/dist/assets/{cssMode-DBHbkBvI.js → cssMode-BM4z9OX0.js} +1 -1
- package/client/dist/assets/dashboard-B_NkSHYm.js +1 -0
- package/client/dist/assets/dashboard-ClBb_3pq.js +1 -0
- package/client/dist/assets/dashboard-CpCmJvMB.js +1 -0
- package/client/dist/assets/dashboard-Czohu35O2.js +1 -0
- package/client/dist/assets/dashboard-DcjbD0UK.js +1 -0
- package/client/dist/assets/dashboard-Dvx5F3we.js +1 -0
- package/client/dist/assets/dashboard-R1bOPjBh2.js +1 -0
- package/client/dist/assets/dashboard-owcDucDi.js +1 -0
- package/client/dist/assets/dialog-CzIekCw0.js +45 -0
- package/client/dist/assets/dist-Cez5CRkX.js +1 -0
- package/client/dist/assets/dist-bVqfYmby.js +1 -0
- package/client/dist/assets/{dist-js-CgpP6tKI.js → dist-js-CcBCbb32.js} +1 -1
- package/client/dist/assets/{dist-js-DHiN-A_E.js → dist-js-VwztRkGj.js} +1 -1
- package/client/dist/assets/download-CFHyyHcw.js +1 -0
- package/client/dist/assets/editor.api2-4ndOupAA.js +872 -0
- package/client/dist/assets/{editor.main-C9J2ikJK.js → editor.main-DxrmGfoq.js} +2 -2
- package/client/dist/assets/editor.worker-tytyn86m.js +26 -0
- package/client/dist/assets/explore-DgBACn39.js +1 -0
- package/client/dist/assets/external-link-BcU5EWZp.js +1 -0
- package/client/dist/assets/file-text-CFT-xXKN.js +1 -0
- package/client/dist/assets/formatDistanceToNow-ByHIauTQ.js +1 -0
- package/client/dist/assets/{freemarker2-J5iIIIK-.js → freemarker2-CELlTDJ2.js} +1 -1
- package/client/dist/assets/getTimezoneOffsetInMilliseconds-BxHAksaU.js +1 -0
- package/client/dist/assets/git-branch-BRM0F5GN.js +1 -0
- package/client/dist/assets/{handlebars-DSw1l_tw.js → handlebars-NEjHJ7o_.js} +1 -1
- package/client/dist/assets/{html-h2-wE1Lj.js → html-CA10zN5A.js} +1 -1
- package/client/dist/assets/{html.worker-CQP8QQsS.js → html.worker-DSJ9zmd7.js} +12 -12
- package/client/dist/assets/{htmlMode-Cc_ij7lW.js → htmlMode-89rPYiyq.js} +1 -1
- package/client/dist/assets/i18n-CgaRxpII.js +2 -0
- package/client/dist/assets/index-BflYQpWm.css +2 -0
- package/client/dist/assets/index-C1_-BTXy.js +72 -0
- package/client/dist/assets/integrations-mB-7zoyT.js +1 -0
- package/client/dist/assets/{javascript-C1q3xv6N.js → javascript-B1gDmwzS.js} +1 -1
- package/client/dist/assets/jira-CmKWERE0.js +1 -0
- package/client/dist/assets/jira-api-BgFIr8eD.js +1 -0
- package/client/dist/assets/job-time-BCL7wee9.js +1 -0
- package/client/dist/assets/jobs-B-sDHEay2.js +1 -0
- package/client/dist/assets/jobs-BCtGfgUA.js +1 -0
- package/client/dist/assets/jobs-BHR_EnDI.js +1 -0
- package/client/dist/assets/jobs-BMbZJjJJ.js +1 -0
- package/client/dist/assets/jobs-CVFD8rar.js +1 -0
- package/client/dist/assets/jobs-DoSaFzAa.js +1 -0
- package/client/dist/assets/jobs-DzSIR8V62.js +1 -0
- package/client/dist/assets/jobs-lk1_Dq1U.js +1 -0
- package/client/dist/assets/json.worker-Oa9UrkKE.js +58 -0
- package/client/dist/assets/{jsonMode-CeaCdEhr.js → jsonMode-xHLMfzhF.js} +1 -1
- package/client/dist/assets/jsx-runtime-zu2_FqZY.js +1 -0
- package/client/dist/assets/lib-C7Mhm-gT.js +7 -0
- package/client/dist/assets/lib-CGdyxIWg.js +29 -0
- package/client/dist/assets/{liquid-eFsVK0V-.js → liquid-BY6hCUXG.js} +1 -1
- package/client/dist/assets/loader-circle-DcygOQpC.js +1 -0
- package/client/dist/assets/loops-BzfXWEwO.js +1 -0
- package/client/dist/assets/{lspLanguageFeatures-DT3-bbhn.js → lspLanguageFeatures-ira2aEJh.js} +1 -1
- package/client/dist/assets/mcp-uDkXYu4L.js +1 -0
- package/client/dist/assets/{mdx-Fuwnh1cX.js → mdx-BNx0xr3N.js} +1 -1
- package/client/dist/assets/monaco.contribution-Bt92R3Vw.js +2 -0
- package/client/dist/assets/nav-myFaXRHD.js +1 -0
- package/client/dist/assets/pencil-C9RkrIPZ.js +1 -0
- package/client/dist/assets/play-DNlrawbS.js +1 -0
- package/client/dist/assets/plus-CsnjfkmX.js +1 -0
- package/client/dist/assets/preload-helper-Czpn1I53.js +1 -0
- package/client/dist/assets/{python-BgoEs6Cj.js → python-DLdbtmNw.js} +1 -1
- package/client/dist/assets/{razor-CM1sPMMq.js → razor-B9gdhAbR.js} +1 -1
- package/client/dist/assets/react-dom-DiWzBDP9.js +1 -0
- package/client/dist/assets/rolldown-runtime-CNC7AqOf.js +1 -0
- package/client/dist/assets/save-28sIlCdr.js +1 -0
- package/client/dist/assets/search-Cq_gE3Ik.js +1 -0
- package/client/dist/assets/settings-B0ypw9uw.js +1 -0
- package/client/dist/assets/setup-CmK87LuS.js +1 -0
- package/client/dist/assets/sparkles-DrY8hb0X.js +1 -0
- package/client/dist/assets/specs-DII0C8gw.js +1 -0
- package/client/dist/assets/spending-C4p06nen.js +1 -0
- package/client/dist/assets/terminal-BFRZnMPP.js +1 -0
- package/client/dist/assets/tickets-B8BuAaj5.js +1 -0
- package/client/dist/assets/tickets-B_yh4qQk.js +1 -0
- package/client/dist/assets/tickets-Bs37scwY2.js +1 -0
- package/client/dist/assets/tickets-CFk2n2Or.js +1 -0
- package/client/dist/assets/tickets-DQp7JS2R2.js +1 -0
- package/client/dist/assets/tickets-D_yDquEw.js +1 -0
- package/client/dist/assets/tickets-DpmdE8tv2.js +1 -0
- package/client/dist/assets/tickets-pAUJkDTu.js +1 -0
- package/client/dist/assets/trash-2-BOU3WdZZ.js +1 -0
- package/client/dist/assets/triangle-alert-H9T4jAmh.js +1 -0
- package/client/dist/assets/{ts.worker-METxwbDZ.js → ts.worker-BtjBFNHX.js} +48 -33
- package/client/dist/assets/{tsMode-BmT8WogI.js → tsMode-DEjSThU9.js} +1 -1
- package/client/dist/assets/{typescript-CQrQPsoW.js → typescript-CRRnD2Ts.js} +1 -1
- package/client/dist/assets/upload-DYMjAQ6M.js +1 -0
- package/client/dist/assets/useBrowserCaptureSession-2OqX0vap.js +1 -0
- package/client/dist/assets/useDesktop-X8YB9xzv.js +1 -0
- package/client/dist/assets/useSharedWebSocket-DaPh7QBL.js +1 -0
- package/client/dist/assets/utils-BC4DHlz2.js +1 -0
- package/client/dist/assets/value-s-gzBeyF.js +1 -0
- package/client/dist/assets/{workers-U0UXyLUf.js → workers-uOo7WLaa.js} +1 -1
- package/client/dist/assets/x-CBMI9ejn.js +1 -0
- package/client/dist/assets/{xml-WRxJs8_I.js → xml-EDREH2X7.js} +1 -1
- package/client/dist/assets/{yaml-CjpSTjJU.js → yaml-BjaDVX1E.js} +1 -1
- package/client/dist/assets/zap-CHo3aCD1.js +1 -0
- package/client/dist/index.html +73 -32
- package/docs/guide/en/integrations/5-mcp-server.md +2 -2
- package/docs/internals/safe-pr-review-flow.md +190 -132
- package/docs/mcp.md +15 -5
- package/package.json +11 -7
- package/server/dist/active-pr-continuation.js +16 -1
- package/server/dist/agent-chat-manager.js +360 -181
- package/server/dist/agent-cwd-manager.js +52 -8
- package/server/dist/agent-generator.js +90 -48
- package/server/dist/agent-mcp-config.js +69 -36
- package/server/dist/agent-operator-prompt.js +3 -2
- package/server/dist/agent-refine-manager.js +47 -6
- package/server/dist/agent-tier.js +13 -11
- package/server/dist/artifact-registry.js +33 -2
- package/server/dist/auth.js +5 -4
- package/server/dist/browser-capture-manager.js +92 -29
- package/server/dist/chat-manager.js +302 -46
- package/server/dist/contract-refine-runner.js +19 -6
- package/server/dist/db.js +665 -8
- package/server/dist/desktop-router.js +1 -2
- package/server/dist/headroom-manager.js +1310 -121
- package/server/dist/headroom-routing.js +120 -2
- package/server/dist/index.js +48 -13
- package/server/dist/interactive-job-session.js +237 -82
- package/server/dist/jira/jira-adf.js +10 -0
- package/server/dist/jira/jira-db.js +15 -3
- package/server/dist/jira/jira-sync-manager.js +182 -13
- package/server/dist/job-listing.js +140 -0
- package/server/dist/job-spawn-idempotency.js +67 -0
- package/server/dist/loop-run-manager.js +508 -98
- package/server/dist/loop-runs-store.js +266 -13
- package/server/dist/mcp/agent-capability.js +71 -0
- package/server/dist/mcp/guide.js +5 -2
- package/server/dist/mcp/mcp-server.js +5 -10
- package/server/dist/mcp/mcp-token.js +5 -5
- package/server/dist/mcp/tools/jobs.js +20 -13
- package/server/dist/mcp/tools/types.js +20 -42
- package/server/dist/mobile/mobile-router.js +1 -1
- package/server/dist/pr-lifecycle.js +77 -0
- package/server/dist/pr-publisher.js +94 -27
- package/server/dist/process-admission.js +82 -0
- package/server/dist/project-registry.js +572 -174
- package/server/dist/project-router-git.js +13 -2
- package/server/dist/project-router-jobs.js +111 -55
- package/server/dist/project-router-tickets.js +32 -2
- package/server/dist/providers/claude-adapter.js +21 -13
- package/server/dist/queue-manager.js +2332 -623
- package/server/dist/rail-isolated-launch.js +1137 -290
- package/server/dist/rail-isolation.js +4 -2
- package/server/dist/rail-pr-decision.js +740 -266
- package/server/dist/rail-pr-delivery.js +2 -1
- package/server/dist/rail-pr-store.js +317 -18
- package/server/dist/rail-pr-ticket-effects.js +328 -0
- package/server/dist/rail-worktree-release.js +18 -2
- package/server/dist/rails-router.js +149 -119
- package/server/dist/rails-store.js +106 -3
- package/server/dist/ticket-store.js +23 -1
- package/server/dist/transient-children.js +13 -0
- package/server/dist/util/cli-prompt.js +42 -8
- package/server/dist/util/secure-token-file.js +84 -0
- package/server/dist/worktree-manager.js +6 -5
- package/client/dist/assets/ActivityFeedPage-D8RKRRoM.js +0 -1
- package/client/dist/assets/AgentBrowserCapture-BUHPHNSN.js +0 -1
- package/client/dist/assets/AgentModeCodePane-GZc8bNGT.js +0 -2
- package/client/dist/assets/AgentModeJobsPane-DWtQqj3k.js +0 -1
- package/client/dist/assets/AgentsPage-CKPtHkRP.js +0 -86
- package/client/dist/assets/AnalyticsPage-CZW8c77k.js +0 -1
- package/client/dist/assets/BarChart-DugEoe2q.js +0 -33
- package/client/dist/assets/CodePage-CQ-uKETG.js +0 -2
- package/client/dist/assets/DesktopAnalyticsPage-Bgo4SyCH.js +0 -1
- package/client/dist/assets/DocsDialog-DxpWw_La.js +0 -11
- package/client/dist/assets/DocsPage-CmcOkK19.js +0 -11
- package/client/dist/assets/ExportDropdown-DvTS2wc7.js +0 -1
- package/client/dist/assets/IntegrationsPage-B72WxJ3Z.js +0 -1
- package/client/dist/assets/InteractiveJobComposer-B5gh4Qoy.js +0 -19
- package/client/dist/assets/JobDetailModal-D-BeqsJ6.js +0 -1
- package/client/dist/assets/JobDetailPage-DwdozwLO.js +0 -1
- package/client/dist/assets/JobsPage-BDkinklt.js +0 -1
- package/client/dist/assets/LoopBuilderPage-CclJ3VZA.js +0 -7
- package/client/dist/assets/LoopPreviewModal-Dw4UgoLu.js +0 -1
- package/client/dist/assets/LoopsPage-D7ac4M24.js +0 -1
- package/client/dist/assets/PluginsPage-wdC-qa40.js +0 -2
- package/client/dist/assets/ProjectSettingsDialog-BlTlRtGI.js +0 -1
- package/client/dist/assets/TemplatePreviewModal-CeMx2Red.js +0 -1
- package/client/dist/assets/activity-Dbl71S-E.js +0 -1
- package/client/dist/assets/addon-image-CwoZj17F.js +0 -1
- package/client/dist/assets/addon-ligatures-CWDrED80.js +0 -2
- package/client/dist/assets/addspec-DrXqP8ip.js +0 -1
- package/client/dist/assets/agent-B8BXmlUQ2.js +0 -1
- package/client/dist/assets/agent-BEXYxDbs.js +0 -1
- package/client/dist/assets/agent-BYc3-6Ln2.js +0 -1
- package/client/dist/assets/agent-BbJorPdT.js +0 -1
- package/client/dist/assets/agent-CewFImgL.js +0 -1
- package/client/dist/assets/agent-DfhYoUmp.js +0 -1
- package/client/dist/assets/agent-TK_rrK4d.js +0 -1
- package/client/dist/assets/agent-bHjA9R_j2.js +0 -1
- package/client/dist/assets/agents-CGDn_3kB.js +0 -1
- package/client/dist/assets/agentstudio-Dxe_94PK.js +0 -1
- package/client/dist/assets/aiedit-DrfHQnlz.js +0 -1
- package/client/dist/assets/analytics-CxDYcBl9.js +0 -1
- package/client/dist/assets/attachments-DvxQbBTC.js +0 -1
- package/client/dist/assets/brain-C92rjjYP.js +0 -1
- package/client/dist/assets/browser-DM8G151q.js +0 -1
- package/client/dist/assets/chat-0dg5hOo_.js +0 -1
- package/client/dist/assets/chunk-BNv3lrIs.js +0 -1
- package/client/dist/assets/code-B62fClk-.js +0 -1
- package/client/dist/assets/commands-DCJ-R9E4.js +0 -1
- package/client/dist/assets/common-BM3LFLCE2.js +0 -1
- package/client/dist/assets/common-Bwafujnc2.js +0 -1
- package/client/dist/assets/common-C-vU-j-I.js +0 -1
- package/client/dist/assets/common-C5HKSbbz.js +0 -1
- package/client/dist/assets/common-DXrmj6WD.js +0 -1
- package/client/dist/assets/common-DhWTkocg.js +0 -1
- package/client/dist/assets/common-DiPTT7V_2.js +0 -1
- package/client/dist/assets/common-DmqBPwZG.js +0 -1
- package/client/dist/assets/dashboard-B5ang-ZS.js +0 -1
- package/client/dist/assets/dashboard-CHHcOWVL2.js +0 -1
- package/client/dist/assets/dashboard-DBD4wPGY2.js +0 -1
- package/client/dist/assets/dashboard-DBuLfWo9.js +0 -1
- package/client/dist/assets/dashboard-DSnztwWH.js +0 -1
- package/client/dist/assets/dashboard-D_aP_InU.js +0 -1
- package/client/dist/assets/dashboard-DmBcdl1B.js +0 -1
- package/client/dist/assets/dashboard-RIcUKY1Y2.js +0 -1
- package/client/dist/assets/editor.api2-C3B0tp2x.js +0 -872
- package/client/dist/assets/editor.worker-Bd9IXS8d.js +0 -26
- package/client/dist/assets/explore-_XyU5e7R.js +0 -1
- package/client/dist/assets/index-BVbgKXUu.css +0 -2
- package/client/dist/assets/index-DyY1QyIn.js +0 -158
- package/client/dist/assets/integrations-By80EBkf.js +0 -1
- package/client/dist/assets/jira-DeIjwCK3.js +0 -1
- package/client/dist/assets/jobs-1GV2-4zd2.js +0 -1
- package/client/dist/assets/jobs-26F9gGcE.js +0 -1
- package/client/dist/assets/jobs-BgQzB0Sx2.js +0 -1
- package/client/dist/assets/jobs-BqWui9_f.js +0 -1
- package/client/dist/assets/jobs-CE8qeIYj.js +0 -1
- package/client/dist/assets/jobs-DrYmsKy5.js +0 -1
- package/client/dist/assets/jobs-DzeuASRf.js +0 -1
- package/client/dist/assets/jobs-Wh7dn28q2.js +0 -1
- package/client/dist/assets/json.worker-DzV-CpCQ.js +0 -58
- package/client/dist/assets/lib-0PFDyMxJ.js +0 -7
- package/client/dist/assets/loops-CO6q75W8.js +0 -1
- package/client/dist/assets/mcp-HwBbQDk0.js +0 -1
- package/client/dist/assets/monaco.contribution-Desq7oq4.js +0 -2
- package/client/dist/assets/nav-BTg8s-cD.js +0 -1
- package/client/dist/assets/preload-helper-DSXbuxSR.js +0 -1
- package/client/dist/assets/settings-cJNynGIZ.js +0 -1
- package/client/dist/assets/setup-D2CmgAro.js +0 -1
- package/client/dist/assets/specs-PGIOzMtc.js +0 -1
- package/client/dist/assets/terminal-CFt56A5f.js +0 -1
- package/client/dist/assets/tickets-B7zT4IRf.js +0 -1
- package/client/dist/assets/tickets-BHv3ewvq.js +0 -1
- package/client/dist/assets/tickets-BoXom7l8.js +0 -1
- package/client/dist/assets/tickets-C3b8cHhd2.js +0 -1
- package/client/dist/assets/tickets-CFXnJsYF.js +0 -1
- package/client/dist/assets/tickets-CWYFvx1W2.js +0 -1
- package/client/dist/assets/tickets-DTHo9c3C.js +0 -1
- package/client/dist/assets/tickets-SuX_HkK4.js +0 -1
- package/client/dist/assets/upload-CmR4h9H1.js +0 -1
- package/client/dist/assets/value-JA3bf47v.js +0 -1
- /package/client/dist/assets/{abap-D0E4FfVW.js → abap-08VXUWAP.js} +0 -0
- /package/client/dist/assets/{activity--iN4Ad2R2.js → activity-B-rp4h_v.js} +0 -0
- /package/client/dist/assets/{activity-6hM_ZSg9.js → activity-C6kJpm9q2.js} +0 -0
- /package/client/dist/assets/{activity-CIOLxGvh.js → activity-CGx4kb_j.js} +0 -0
- /package/client/dist/assets/{activity-BZNfOdiS.js → activity-CnoX2bH7.js} +0 -0
- /package/client/dist/assets/{activity-BuVo5sPB.js → activity-DAyekdlt2.js} +0 -0
- /package/client/dist/assets/{activity-D2r2h3SE2.js → activity-DCxtpT7j.js} +0 -0
- /package/client/dist/assets/{activity-DtxrAUMo2.js → activity-JzwRaWPD.js} +0 -0
- /package/client/dist/assets/{addspec-BMtLZLW62.js → addspec-BXjvkh5q.js} +0 -0
- /package/client/dist/assets/{addspec-CRuvWn8d.js → addspec-C4bwrmpi.js} +0 -0
- /package/client/dist/assets/{addspec-D-JLDsUV.js → addspec-CArHXUDM.js} +0 -0
- /package/client/dist/assets/{addspec-mgPOt5q02.js → addspec-DTKJcXiR.js} +0 -0
- /package/client/dist/assets/{addspec-DfhDbg5c.js → addspec-DbNBbrha2.js} +0 -0
- /package/client/dist/assets/{addspec-CP0HRuxY.js → addspec-htHGZHOe2.js} +0 -0
- /package/client/dist/assets/{addspec-bsNDdqRm2.js → addspec-lr_LP6jk.js} +0 -0
- /package/client/dist/assets/{agents-78bTZGZh.js → agents-6ZOJXWVk2.js} +0 -0
- /package/client/dist/assets/{agents-DDtCJ5M02.js → agents-BZ0XQKjg.js} +0 -0
- /package/client/dist/assets/{agents-CWo-KyAB.js → agents-BqWklnHR.js} +0 -0
- /package/client/dist/assets/{agents-BUdEi3-N.js → agents-Bz-thHkB2.js} +0 -0
- /package/client/dist/assets/{agents-bvyRLczL.js → agents-CJclElx6.js} +0 -0
- /package/client/dist/assets/{agents-BpoUaoW92.js → agents-D_l9L0VA.js} +0 -0
- /package/client/dist/assets/{agents-BLhWCmDS2.js → agents-u_C0jPB5.js} +0 -0
- /package/client/dist/assets/{agentstudio-BMQxc_bz.js → agentstudio-BF19hbXj.js} +0 -0
- /package/client/dist/assets/{agentstudio-C_Jq8Tl4.js → agentstudio-BLGvxWBx2.js} +0 -0
- /package/client/dist/assets/{agentstudio-Dtq3dGYk.js → agentstudio-D9UKiWTj2.js} +0 -0
- /package/client/dist/assets/{agentstudio-CWgXG5W52.js → agentstudio-Drs37nIM.js} +0 -0
- /package/client/dist/assets/{agentstudio-sI27TpBu2.js → agentstudio-nEnKkcXM.js} +0 -0
- /package/client/dist/assets/{agentstudio-jKMTdT70.js → agentstudio-pyFcSBKy.js} +0 -0
- /package/client/dist/assets/{agentstudio-COZriimI2.js → agentstudio-zfTzQh3U.js} +0 -0
- /package/client/dist/assets/{aiedit-hknZlly82.js → aiedit-B0KUbpB6.js} +0 -0
- /package/client/dist/assets/{aiedit-eMN4eQPF.js → aiedit-BoG24nlg2.js} +0 -0
- /package/client/dist/assets/{aiedit-pc5JDUq1.js → aiedit-CjlSRQVP.js} +0 -0
- /package/client/dist/assets/{aiedit-B03pD_Na.js → aiedit-DCjA7jLJ.js} +0 -0
- /package/client/dist/assets/{aiedit-6CHdb_iv2.js → aiedit-DNXxq_Dr.js} +0 -0
- /package/client/dist/assets/{aiedit-igmw-ZoZ2.js → aiedit-DQ-0_pvE.js} +0 -0
- /package/client/dist/assets/{aiedit-BtRqRxML.js → aiedit-olnv3QSt2.js} +0 -0
- /package/client/dist/assets/{analytics-CCprAqk3.js → analytics-BC7mwR4s.js} +0 -0
- /package/client/dist/assets/{analytics-CR1TJqUi2.js → analytics-BiNg8edP.js} +0 -0
- /package/client/dist/assets/{analytics-Bj-SmKHa.js → analytics-C0s_T2GN.js} +0 -0
- /package/client/dist/assets/{analytics-C5dpiS0o.js → analytics-CdsAPAwZ2.js} +0 -0
- /package/client/dist/assets/{analytics-BWMUZ2Pq2.js → analytics-Cr1_q-lC.js} +0 -0
- /package/client/dist/assets/{analytics-DfNdXR0y2.js → analytics-IEsKZvpW2.js} +0 -0
- /package/client/dist/assets/{analytics-rmHuJv0V.js → analytics-SRPf84OT.js} +0 -0
- /package/client/dist/assets/{apex-D_X_8J4d.js → apex-BWPQTe0t.js} +0 -0
- /package/client/dist/assets/{atom-one-dark-BMDPFwRZ.js → atom-one-dark-qiGhVQxg.js} +0 -0
- /package/client/dist/assets/{attachments-BgBNSqQd.js → attachments-D0wXXt9T2.js} +0 -0
- /package/client/dist/assets/{attachments-C9GTTJsx2.js → attachments-DenIjuVI.js} +0 -0
- /package/client/dist/assets/{attachments-BjyMmrwE.js → attachments-DhO7LPfy.js} +0 -0
- /package/client/dist/assets/{attachments-DPCmkFZu2.js → attachments-DoH5w0x4.js} +0 -0
- /package/client/dist/assets/{attachments-DCZOL1RU.js → attachments-Q9yNtiEM.js} +0 -0
- /package/client/dist/assets/{attachments-rNn1ekRy2.js → attachments-V_WZAYXx2.js} +0 -0
- /package/client/dist/assets/{attachments-DL-bPy9E.js → attachments-eaeScpjM.js} +0 -0
- /package/client/dist/assets/{auth-DpZq5p1e.js → auth-D_5bQv5W.js} +0 -0
- /package/client/dist/assets/{azcli-PZLnZ3jH.js → azcli-Bc_sGQ0U.js} +0 -0
- /package/client/dist/assets/{bat-KlBOxpO4.js → bat-i0X4ZdIN.js} +0 -0
- /package/client/dist/assets/{bicep-CgG9oF3V.js → bicep-B5-_aFwp.js} +0 -0
- /package/client/dist/assets/{browser-BPkhILSJ.js → browser-C25DkQiO.js} +0 -0
- /package/client/dist/assets/{browser-BqLqJVVT.js → browser-CiJ7-L32.js} +0 -0
- /package/client/dist/assets/{browser-rWKj-RR_2.js → browser-CvgDySd9.js} +0 -0
- /package/client/dist/assets/{browser-Cvg-QHvd2.js → browser-CyR1Kp3m2.js} +0 -0
- /package/client/dist/assets/{browser-FRv8P02E.js → browser-DwZljcRh2.js} +0 -0
- /package/client/dist/assets/{browser-Cc0iDqUi.js → browser-DxP-alBK.js} +0 -0
- /package/client/dist/assets/{browser-BijVYG0Y2.js → browser-rRCwi7Y7.js} +0 -0
- /package/client/dist/assets/{cameligo-6R45j6Ph.js → cameligo-DMUM7wLl.js} +0 -0
- /package/client/dist/assets/{chat-Bom6bqK3.js → chat-2rIB8HhT.js} +0 -0
- /package/client/dist/assets/{chat-vylxqR0P2.js → chat-BR8oe53x2.js} +0 -0
- /package/client/dist/assets/{chat-Dnwg-NpQ.js → chat-Bua03tty.js} +0 -0
- /package/client/dist/assets/{chat-BgL3zv6x2.js → chat-CZpAQan1.js} +0 -0
- /package/client/dist/assets/{chat-DsbwJhus.js → chat-CdY5SGvS.js} +0 -0
- /package/client/dist/assets/{chat-Bh3NdgJs.js → chat-DZyu3ImY2.js} +0 -0
- /package/client/dist/assets/{chat-DdwZjT8K2.js → chat-D_I7nU5K.js} +0 -0
- /package/client/dist/assets/{clojure-CLAAMO7J.js → clojure-Cm7r79vr.js} +0 -0
- /package/client/dist/assets/{clsx-CNmW0Nip.js → clsx-CjueKrWZ.js} +0 -0
- /package/client/dist/assets/{code-UPVvmAKM2.js → code-76gS0IdL.js} +0 -0
- /package/client/dist/assets/{code-BRcDlze52.js → code-C1VwxlML.js} +0 -0
- /package/client/dist/assets/{code-CeQ0FZpC.js → code-CnMZyJpr.js} +0 -0
- /package/client/dist/assets/{code-DgeHz9CB.js → code-D20ef9Cf2.js} +0 -0
- /package/client/dist/assets/{code-DzC1rMk12.js → code-DdOXaWRY.js} +0 -0
- /package/client/dist/assets/{code-CwnBPPyA.js → code-Duohuf9l.js} +0 -0
- /package/client/dist/assets/{code-BdMy3Ffq.js → code-geHmknB92.js} +0 -0
- /package/client/dist/assets/{coffee-AruSygo-.js → coffee-Ba7i2nA0.js} +0 -0
- /package/client/dist/assets/{commands-DkU0qliq.js → commands-B9RZO4qS.js} +0 -0
- /package/client/dist/assets/{commands-u-4eEHCN.js → commands-BEwbAjtD2.js} +0 -0
- /package/client/dist/assets/{commands-JWMfGDmr.js → commands-Bbxu8kLU.js} +0 -0
- /package/client/dist/assets/{commands-C4Y3C-PF2.js → commands-CDU3qv68.js} +0 -0
- /package/client/dist/assets/{commands-xHDDGfcP2.js → commands-CzezgtQj.js} +0 -0
- /package/client/dist/assets/{commands-CSDs9q_2.js → commands-DTQBF_q52.js} +0 -0
- /package/client/dist/assets/{commands-RHOchN4n2.js → commands-hOtKuOXc.js} +0 -0
- /package/client/dist/assets/{cpp-DEE5r4Pw.js → cpp-C7h46wYY.js} +0 -0
- /package/client/dist/assets/{csharp-BVXJUvbK.js → csharp-BKxtCVv1.js} +0 -0
- /package/client/dist/assets/{csp-cXOF0JoV.js → csp-bTuwJoIa.js} +0 -0
- /package/client/dist/assets/{css-C_zQuAoy.js → css-DIMkf-bt.js} +0 -0
- /package/client/dist/assets/{cypher-t95m_Q-O.js → cypher-CVaqCwHa.js} +0 -0
- /package/client/dist/assets/{dart-B4r1Tg-e.js → dart-onAF5SnQ.js} +0 -0
- /package/client/dist/assets/{dockerfile-6mt7zHQw.js → dockerfile-DZFCIeNp.js} +0 -0
- /package/client/dist/assets/{ecl-DEY_Fz_Q.js → ecl-D05T4iGw.js} +0 -0
- /package/client/dist/assets/{elixir-R6-q1c5t.js → elixir-6RTg0lbw.js} +0 -0
- /package/client/dist/assets/{explore-B7fblXBb2.js → explore-BG99VGpL.js} +0 -0
- /package/client/dist/assets/{explore-CNBzrEPS.js → explore-C0TjprJW2.js} +0 -0
- /package/client/dist/assets/{explore-C0k8HrkN.js → explore-C2dgEo7q.js} +0 -0
- /package/client/dist/assets/{explore-BHtEpoNN2.js → explore-C8NDHbob.js} +0 -0
- /package/client/dist/assets/{explore-Dahc1lOx.js → explore-CWWgzQGR.js} +0 -0
- /package/client/dist/assets/{explore-D_mWBYCS2.js → explore-CcQXpdtS.js} +0 -0
- /package/client/dist/assets/{explore-UvW1jEMa.js → explore-DsXAgnmJ2.js} +0 -0
- /package/client/dist/assets/{flow9-CNQvHBOt.js → flow9-C5_-GSwl.js} +0 -0
- /package/client/dist/assets/{format-command-CuWuRvka.js → format-command-Dfa1o2d-.js} +0 -0
- /package/client/dist/assets/{fsharp-vPZkFXwv.js → fsharp-C8Ef5oNN.js} +0 -0
- /package/client/dist/assets/{go-KUzBMYJ_.js → go-C-y9NEjX.js} +0 -0
- /package/client/dist/assets/{graphql-C3W0rCsD.js → graphql-fmXr3nnJ.js} +0 -0
- /package/client/dist/assets/{hcl-BOOMFSNX.js → hcl-CpzslTdj.js} +0 -0
- /package/client/dist/assets/{ini-CFat1WJr.js → ini-sBoK_t0W.js} +0 -0
- /package/client/dist/assets/{integrations-TAcYNh0b.js → integrations-CH0zy1Mw2.js} +0 -0
- /package/client/dist/assets/{integrations-C1ThkTC52.js → integrations-CIAjhd7L.js} +0 -0
- /package/client/dist/assets/{integrations-2YcNyAl3.js → integrations-CZdwOe8A.js} +0 -0
- /package/client/dist/assets/{integrations-BXCvXUlG2.js → integrations-Ct7omMld.js} +0 -0
- /package/client/dist/assets/{integrations-Crk2rFhL2.js → integrations-D6MSxy3-.js} +0 -0
- /package/client/dist/assets/{integrations-Dn4JiGtW.js → integrations-DbZa-Zyc.js} +0 -0
- /package/client/dist/assets/{integrations-D9YrJ_Ep.js → integrations-b8pH-vfl2.js} +0 -0
- /package/client/dist/assets/{java-Dee-CNAM.js → java-BEtHBSE6.js} +0 -0
- /package/client/dist/assets/{jira-D5efgwp_.js → jira-BySfUuQZ.js} +0 -0
- /package/client/dist/assets/{jira-bDs7Im2C.js → jira-Byw2C0F8.js} +0 -0
- /package/client/dist/assets/{jira-BHAKtc4B.js → jira-CNeYi06w2.js} +0 -0
- /package/client/dist/assets/{jira-Be369RIR2.js → jira-D11gPhxz.js} +0 -0
- /package/client/dist/assets/{jira-CPpgA-5z2.js → jira-D38Gj7AT.js} +0 -0
- /package/client/dist/assets/{jira-uNH1gETL2.js → jira-gAvOyije.js} +0 -0
- /package/client/dist/assets/{jira-DS7ownDy.js → jira-pRaL7JQo2.js} +0 -0
- /package/client/dist/assets/{julia-CGNlLYuU.js → julia-Bri6UV-V.js} +0 -0
- /package/client/dist/assets/{kotlin-CY7lhknJ.js → kotlin-BOotOW0E.js} +0 -0
- /package/client/dist/assets/{less-D0KN7hfJ.js → less-B9JPFI3C.js} +0 -0
- /package/client/dist/assets/{lexon-CuRJufPs.js → lexon-CfSJPG6W.js} +0 -0
- /package/client/dist/assets/{loops-LIgO_-eM.js → loops-CPpgMapU.js} +0 -0
- /package/client/dist/assets/{loops-DJNhp17p2.js → loops-CWWbKuVL.js} +0 -0
- /package/client/dist/assets/{loops-D6D3XHjj.js → loops-Crc9It8o2.js} +0 -0
- /package/client/dist/assets/{loops-CvOWsi5r.js → loops-DH2hZcbK2.js} +0 -0
- /package/client/dist/assets/{loops-vhJdbcz92.js → loops-DS5TmvIU2.js} +0 -0
- /package/client/dist/assets/{loops-K5_HXfJa.js → loops-P2f0cN8-.js} +0 -0
- /package/client/dist/assets/{loops-BvqQ-bHR2.js → loops-fJV-9hH7.js} +0 -0
- /package/client/dist/assets/{lua-DHvH62Zf.js → lua-CsQS60Ue.js} +0 -0
- /package/client/dist/assets/{m3-tlthQ8Fo.js → m3-D-oSqn_W.js} +0 -0
- /package/client/dist/assets/{markdown-Dbx1LgUZ.js → markdown-Cimd5fb3.js} +0 -0
- /package/client/dist/assets/{mcp-Dcl-XxkR.js → mcp-Czd_iyp02.js} +0 -0
- /package/client/dist/assets/{mcp-XJNRfxVp2.js → mcp-D2pc5GGx.js} +0 -0
- /package/client/dist/assets/{mcp-sRmWE8WJ.js → mcp-D4ZjtIx32.js} +0 -0
- /package/client/dist/assets/{mcp-DtbxuH3V2.js → mcp-DIfBUv2D2.js} +0 -0
- /package/client/dist/assets/{mcp-H-gnQUdN.js → mcp-DsTwmjST.js} +0 -0
- /package/client/dist/assets/{mcp-N1nzp8oL.js → mcp-SMkn8boG.js} +0 -0
- /package/client/dist/assets/{mcp-CjVKXAhD2.js → mcp-fSfiVYAc.js} +0 -0
- /package/client/dist/assets/{mips-D_O3IUz_.js → mips-CIPQ_RoX.js} +0 -0
- /package/client/dist/assets/{msdax-C_ZmCnRt.js → msdax-DauUninz.js} +0 -0
- /package/client/dist/assets/{mysql-C-dCEYSX.js → mysql-SOo6toE5.js} +0 -0
- /package/client/dist/assets/{nav-BElxljIF2.js → nav-BRJBxaGt2.js} +0 -0
- /package/client/dist/assets/{nav-COu2E_-L.js → nav-CAGKv3_F.js} +0 -0
- /package/client/dist/assets/{nav-BXO7PJJG2.js → nav-CNskRJKQ2.js} +0 -0
- /package/client/dist/assets/{nav-CW9pxWfC2.js → nav-DY93irBL.js} +0 -0
- /package/client/dist/assets/{nav-CXHUA4uf.js → nav-DhjvYgqt.js} +0 -0
- /package/client/dist/assets/{nav-DdHF5VWJ.js → nav-Gb-r4wop.js} +0 -0
- /package/client/dist/assets/{nav-C8kOuGh8.js → nav-TvumAFD22.js} +0 -0
- /package/client/dist/assets/{objective-c-DZY9Aqxd.js → objective-c-FvmIjYaQ.js} +0 -0
- /package/client/dist/assets/{pascal-BVHLXoor.js → pascal-DrH0SRf2.js} +0 -0
- /package/client/dist/assets/{pascaligo-XdG_6u3k.js → pascaligo-D-ptJ9y-.js} +0 -0
- /package/client/dist/assets/{perl-b9QV0BcD.js → perl-oz_6vUea.js} +0 -0
- /package/client/dist/assets/{pgsql-Du5FDOgT.js → pgsql-DTj74zXo.js} +0 -0
- /package/client/dist/assets/{php-WJD0z2Z_.js → php-nr791fC2.js} +0 -0
- /package/client/dist/assets/{pla-CbciTf2w.js → pla-CopQ2nXW.js} +0 -0
- /package/client/dist/assets/{postiats-C1QxmvVq.js → postiats-43DmfD33.js} +0 -0
- /package/client/dist/assets/{powerquery-BEYDhPLZ.js → powerquery-D3hlyOfw.js} +0 -0
- /package/client/dist/assets/{powershell-CfaYw4XL.js → powershell-DmHpPYUd.js} +0 -0
- /package/client/dist/assets/{protobuf-pwZEDOYa.js → protobuf-C531GsRP.js} +0 -0
- /package/client/dist/assets/{pug-5ftqSPVe.js → pug-Z5eAx3Zn.js} +0 -0
- /package/client/dist/assets/{qsharp-uI6qZMq4.js → qsharp-DkqhCAOL.js} +0 -0
- /package/client/dist/assets/{r-BK54P1EG.js → r-BwWrilGY.js} +0 -0
- /package/client/dist/assets/{redis-DI0yd7kN.js → redis-ClamHrr6.js} +0 -0
- /package/client/dist/assets/{redshift-Dov26eyI.js → redshift-DT7zqm-g.js} +0 -0
- /package/client/dist/assets/{restructuredtext-QqJ_1dC-.js → restructuredtext-BYgofb2h.js} +0 -0
- /package/client/dist/assets/{ruby-5WjveeVb.js → ruby-DezsRK8O.js} +0 -0
- /package/client/dist/assets/{rust-BzVSfqFH.js → rust-DdL9SqIa.js} +0 -0
- /package/client/dist/assets/{sb-DMIUfT-3.js → sb-CcwsVR0C.js} +0 -0
- /package/client/dist/assets/{scala-CSKA7lqt.js → scala-DHpiXF5c.js} +0 -0
- /package/client/dist/assets/{scheme-CojOP2Qc.js → scheme-BeGwcela.js} +0 -0
- /package/client/dist/assets/{scss-DoBZGJG6.js → scss-gp-XZpBa.js} +0 -0
- /package/client/dist/assets/{settings-1w1O5Og82.js → settings-B9ilOSYO.js} +0 -0
- /package/client/dist/assets/{settings-Jc46vUhd.js → settings-BqlWHRHA.js} +0 -0
- /package/client/dist/assets/{settings-BSihkr0e.js → settings-D8dlqt4d.js} +0 -0
- /package/client/dist/assets/{settings-BiDOB3hV2.js → settings-DAZqjX2a2.js} +0 -0
- /package/client/dist/assets/{settings-DNL68KTE.js → settings-DRh39v1S2.js} +0 -0
- /package/client/dist/assets/{settings-BoubJhDP2.js → settings-Mj1OJbfd2.js} +0 -0
- /package/client/dist/assets/{settings-CB-EZ_W8.js → settings-lRReaSYm.js} +0 -0
- /package/client/dist/assets/{setup-D84y3wx-.js → setup-CGZjy2IF2.js} +0 -0
- /package/client/dist/assets/{setup-BuQNKztt2.js → setup-CXa0Sx16.js} +0 -0
- /package/client/dist/assets/{setup-BwMovM9z2.js → setup-D2ktKKMF2.js} +0 -0
- /package/client/dist/assets/{setup-DU0ULX8q2.js → setup-DG2H7ha5.js} +0 -0
- /package/client/dist/assets/{setup-BXpw07r-.js → setup-DK5bHDUq2.js} +0 -0
- /package/client/dist/assets/{setup-7J8TjG4O.js → setup-DrWlIQTZ.js} +0 -0
- /package/client/dist/assets/{setup-BiduSX9d.js → setup-rzHUVgAu.js} +0 -0
- /package/client/dist/assets/{shell-YOaE4udl.js → shell-CC2rA5mh.js} +0 -0
- /package/client/dist/assets/{solidity-l3jSyVLu.js → solidity-BEEn4gHE.js} +0 -0
- /package/client/dist/assets/{sophia-BhSGTaQJ.js → sophia-CRfGWb83.js} +0 -0
- /package/client/dist/assets/{sparql-qhk6QD9w.js → sparql-D_Lu-MrJ.js} +0 -0
- /package/client/dist/assets/{specs-CWtKcVDC.js → specs-BcyxaCwf2.js} +0 -0
- /package/client/dist/assets/{specs-DTplt2XM.js → specs-C1q9geAi.js} +0 -0
- /package/client/dist/assets/{specs-PozUNhqb.js → specs-CLZKysq-2.js} +0 -0
- /package/client/dist/assets/{specs-C9Sd7BhH2.js → specs-DZMS9o54.js} +0 -0
- /package/client/dist/assets/{specs-D2clrt2c2.js → specs-Dm6zUI-R.js} +0 -0
- /package/client/dist/assets/{specs-CBDvEYKv.js → specs-V3TvhKLX.js} +0 -0
- /package/client/dist/assets/{specs-CIPr9reA2.js → specs-dV3Mq5Mu2.js} +0 -0
- /package/client/dist/assets/{sql-DoO3y21-.js → sql-NEE52Syq.js} +0 -0
- /package/client/dist/assets/{st-ChZAY8oG.js → st-DbInun42.js} +0 -0
- /package/client/dist/assets/{swift-D290Tr7_.js → swift-Bxkupp3x.js} +0 -0
- /package/client/dist/assets/{systemverilog-DsxzgJ-o.js → systemverilog-Bz4Y3fRF.js} +0 -0
- /package/client/dist/assets/{tcl-BKDUHAAR.js → tcl-DISqw1ZD.js} +0 -0
- /package/client/dist/assets/{terminal-De2Brd23.js → terminal-BZiIu3rf.js} +0 -0
- /package/client/dist/assets/{terminal-cqfd3EDV.js → terminal-C2NAd7BW2.js} +0 -0
- /package/client/dist/assets/{terminal-WSAvHXoU.js → terminal-IahE4LW92.js} +0 -0
- /package/client/dist/assets/{terminal-Z7KsDM_n.js → terminal-QpXZU2Bn2.js} +0 -0
- /package/client/dist/assets/{terminal-xGbCZJHg.js → terminal-ScjyXNxQ.js} +0 -0
- /package/client/dist/assets/{terminal-BRm03iFA2.js → terminal-d7GadXbN.js} +0 -0
- /package/client/dist/assets/{terminal-DxQWCqSD2.js → terminal-tITa81iz.js} +0 -0
- /package/client/dist/assets/{twig-UpZQaM8G.js → twig-De2hgUGE.js} +0 -0
- /package/client/dist/assets/{typespec-DzaKvN8i.js → typespec-B8J7ngcE.js} +0 -0
- /package/client/dist/assets/{vb-CZpbb7tk.js → vb-DV3o63ZY.js} +0 -0
- /package/client/dist/assets/{wgsl-CNt3OIvl.js → wgsl-DpFanUEy.js} +0 -0
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.QueueManager = exports.FREESTYLE_COMMAND_RE = exports.JobAlreadyTerminalError = exports.JobNotFoundError = exports.CodexNotFoundError = exports.ClaudeNotFoundError = exports.DEFAULT_ZOMBIE_TIMEOUT_MS = void 0;
|
|
6
|
+
exports.QueueManager = exports.FREESTYLE_COMMAND_RE = exports.InvalidJobDependencyError = exports.JobAlreadyTerminalError = exports.JobNotFoundError = exports.CodexNotFoundError = exports.ClaudeNotFoundError = exports.DEFAULT_ZOMBIE_TIMEOUT_MS = void 0;
|
|
7
7
|
exports.buildTelemetryEnv = buildTelemetryEnv;
|
|
8
8
|
exports.projectSupportsProfiles = projectSupportsProfiles;
|
|
9
9
|
exports.distributeIntEvenly = distributeIntEvenly;
|
|
@@ -143,6 +143,33 @@ function distributeIntEvenly(total, n) {
|
|
|
143
143
|
}
|
|
144
144
|
return out;
|
|
145
145
|
}
|
|
146
|
+
function maxNullable(left, right) {
|
|
147
|
+
const safeLeft = typeof left === 'number' && Number.isFinite(left) && left >= 0 ? left : null;
|
|
148
|
+
const safeRight = typeof right === 'number' && Number.isFinite(right) && right >= 0 ? right : null;
|
|
149
|
+
if (safeLeft == null)
|
|
150
|
+
return safeRight;
|
|
151
|
+
if (safeRight == null)
|
|
152
|
+
return safeLeft;
|
|
153
|
+
return Math.max(safeLeft, safeRight);
|
|
154
|
+
}
|
|
155
|
+
function sanitizeRecoveredResult(result) {
|
|
156
|
+
const safe = { ...result };
|
|
157
|
+
const numeric = [
|
|
158
|
+
'tokens_in', 'tokens_out', 'tokens_cache_read', 'tokens_cache_create',
|
|
159
|
+
'total_cost_usd', 'num_turns', 'duration_ms', 'duration_api_ms',
|
|
160
|
+
];
|
|
161
|
+
for (const key of numeric) {
|
|
162
|
+
const value = safe[key];
|
|
163
|
+
if (value !== undefined && (typeof value !== 'number' || !Number.isFinite(value) || value < 0)) {
|
|
164
|
+
delete safe[key];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (safe.model !== undefined && typeof safe.model !== 'string')
|
|
168
|
+
delete safe.model;
|
|
169
|
+
if (safe.session_id !== undefined && typeof safe.session_id !== 'string')
|
|
170
|
+
delete safe.session_id;
|
|
171
|
+
return safe;
|
|
172
|
+
}
|
|
146
173
|
const LOG_BUFFER_MAX = 5000;
|
|
147
174
|
const LOG_BUFFER_DROP = 1000;
|
|
148
175
|
exports.DEFAULT_ZOMBIE_TIMEOUT_MS = 1_800_000; // 30 minutes
|
|
@@ -175,6 +202,13 @@ class JobAlreadyTerminalError extends Error {
|
|
|
175
202
|
}
|
|
176
203
|
}
|
|
177
204
|
exports.JobAlreadyTerminalError = JobAlreadyTerminalError;
|
|
205
|
+
class InvalidJobDependencyError extends Error {
|
|
206
|
+
constructor(message) {
|
|
207
|
+
super(message);
|
|
208
|
+
this.name = 'InvalidJobDependencyError';
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
exports.InvalidJobDependencyError = InvalidJobDependencyError;
|
|
178
212
|
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
|
179
213
|
const TERMINAL_STATUSES = new Set(['completed', 'failed', 'canceled', 'zombie_terminated', 'skipped']);
|
|
180
214
|
/** Match an Freestyle rail command: `/specrails:freestyle #5 …` (or `/sr:…`). */
|
|
@@ -189,6 +223,7 @@ class QueueManager {
|
|
|
189
223
|
_killTimer;
|
|
190
224
|
_cancelingJobs;
|
|
191
225
|
_zombieJobs;
|
|
226
|
+
_persistenceFailedJobs;
|
|
192
227
|
_broadcast;
|
|
193
228
|
_db;
|
|
194
229
|
_logBuffer;
|
|
@@ -199,6 +234,12 @@ class QueueManager {
|
|
|
199
234
|
/** Set by shutdown(); once disposed the manager spawns no new jobs and never
|
|
200
235
|
* touches the (now possibly closed) DB from late child 'close' callbacks. */
|
|
201
236
|
_disposed;
|
|
237
|
+
/** Startup projection/capture failed and must be rebuilt before admissions
|
|
238
|
+
* or resume may proceed. Cleared only by a complete restore pass. */
|
|
239
|
+
_restoreBlocked;
|
|
240
|
+
/** Invalidates async pre-spawn work across shutdown. A job captures the
|
|
241
|
+
* generation at start and must still own the active slot before spawning. */
|
|
242
|
+
_lifecycleGeneration;
|
|
202
243
|
_getCostAlertThreshold;
|
|
203
244
|
_getDesktopDailyBudget;
|
|
204
245
|
_adapter;
|
|
@@ -208,6 +249,11 @@ class QueueManager {
|
|
|
208
249
|
* fallback model name stamped onto the ai_invocations row. */
|
|
209
250
|
_resolvedModel;
|
|
210
251
|
_onJobFinished;
|
|
252
|
+
/** Project-owned durable effects that are part of every queue admission
|
|
253
|
+
* (for example, recovery/rail ownership). Invoked after queued_jobs has
|
|
254
|
+
* been written but inside the same outer transaction, before any
|
|
255
|
+
* route-specific admission claim. */
|
|
256
|
+
_onJobAdmission;
|
|
211
257
|
_onBudgetExceeded;
|
|
212
258
|
/** Project ID used for OTEL resource attributes (Super mode only) */
|
|
213
259
|
_projectId;
|
|
@@ -215,21 +261,20 @@ class QueueManager {
|
|
|
215
261
|
_desktopPort;
|
|
216
262
|
/** Project slug used for per-job profile snapshots (Super mode only) */
|
|
217
263
|
_projectSlug;
|
|
218
|
-
/** Pending profile selection keyed by jobId — read at spawn time
|
|
264
|
+
/** Pending profile selection keyed by jobId — read at spawn time. Map
|
|
265
|
+
* absence means default resolution; a present null forces legacy mode. */
|
|
219
266
|
_jobProfileSelection;
|
|
220
|
-
/** Pending per-job provider override keyed by jobId —
|
|
221
|
-
*
|
|
222
|
-
* restart falls back to the project's primary provider. */
|
|
267
|
+
/** Pending per-job provider override keyed by jobId — restart-durable while
|
|
268
|
+
* queued and consumed only after durable promotion. */
|
|
223
269
|
_jobProviderSelection;
|
|
224
|
-
/** Resolved adapter id per RUNNING job, captured at `_startJob` time
|
|
225
|
-
* per-job override has been consumed-and-deleted from `_jobProviderSelection`.
|
|
270
|
+
/** Resolved adapter id per RUNNING job, captured at `_startJob` time.
|
|
226
271
|
* Read by `_forceFailUnkillableJob` (and any other terminal path that runs
|
|
227
272
|
* without a child exit) to stamp `ai_invocations.provider` with the provider
|
|
228
|
-
* the child ACTUALLY ran on
|
|
273
|
+
* the child ACTUALLY ran on. Cleared with
|
|
229
274
|
* the other per-job maps at every teardown. In-memory only. */
|
|
230
275
|
_jobResolvedProvider;
|
|
231
|
-
/** Pending per-job model override keyed by jobId —
|
|
232
|
-
*
|
|
276
|
+
/** Pending per-job model override keyed by jobId — restart-durable while
|
|
277
|
+
* queued and consumed only after durable promotion. */
|
|
233
278
|
_jobModelSelection;
|
|
234
279
|
/** Pre-spawn working-tree snapshot refs keyed by jobId — read at exit time
|
|
235
280
|
* by the Code-Explorer provenance hook. Cleared on job exit. */
|
|
@@ -241,8 +286,8 @@ class QueueManager {
|
|
|
241
286
|
/** Per-job openspec PATH shim dir (relocated claude rails only). Cleaned up on
|
|
242
287
|
* job exit. In-memory map of jobId → shim dir. */
|
|
243
288
|
_openspecShims = new Map();
|
|
244
|
-
/** Pending per-job interactive
|
|
245
|
-
*
|
|
289
|
+
/** Pending per-job interactive override keyed by jobId. Map absence means the
|
|
290
|
+
* spawn-time default; present false/true are both restart-durable. */
|
|
246
291
|
_jobInteractiveSelection;
|
|
247
292
|
/** Per-job PR-delivery mode (safe-pr-workflow), captured ONCE at spawn time by
|
|
248
293
|
* the SAME `isRailPrDeliveryEnabled()` read that injects
|
|
@@ -265,12 +310,27 @@ class QueueManager {
|
|
|
265
310
|
* `_disposed` and the whole job's spend is lost (COST-ACCOUNTING-AUDIT CRIT-3).
|
|
266
311
|
* Cleared on every terminal path. In-memory only. */
|
|
267
312
|
_jobLiveAccounting;
|
|
313
|
+
_jobReaders;
|
|
268
314
|
/** Jobs terminated by `_forceFailUnkillableJob` (SIGKILL-escalation failure)
|
|
269
315
|
* whose surviving child's `close` may still fire `_onJobExit` later. Guards
|
|
270
316
|
* against a duplicate ai_invocations row + a double `_onJobFinished`; a late
|
|
271
317
|
* close that carries REAL cost replaces the no-cost placeholder rows
|
|
272
318
|
* (COST-ACCOUNTING-AUDIT LOW-6). In-memory only. */
|
|
273
319
|
_forceFailedRowJobs;
|
|
320
|
+
/** Children that survived a failed SIGKILL escalation. They no longer own the
|
|
321
|
+
* runnable slot, but shutdown must still attempt to terminate them. */
|
|
322
|
+
_forceFailedProcesses;
|
|
323
|
+
/** Child exits whose terminal transaction failed. They deliberately retain
|
|
324
|
+
* the active slot until shutdown/restart recovery succeeds. */
|
|
325
|
+
_terminalPersistenceBlockedJobs;
|
|
326
|
+
_pendingLateReconciliations;
|
|
327
|
+
/** Last terminal state emitted per pipeline. Descendant skipping and parent
|
|
328
|
+
* settlement can both evaluate the same pipeline in one call stack; this
|
|
329
|
+
* keeps the externally visible transition exactly-once. */
|
|
330
|
+
_emittedPipelineStatuses;
|
|
331
|
+
/** Test seam for the sole async pre-spawn dependency. Production resolves the
|
|
332
|
+
* implementation lazily to keep the plugin subsystem optional. */
|
|
333
|
+
_resolvePluginsForSpawn;
|
|
274
334
|
constructor(broadcast, db, commands, cwd, options) {
|
|
275
335
|
this._queue = [];
|
|
276
336
|
this._jobs = new Map();
|
|
@@ -280,6 +340,7 @@ class QueueManager {
|
|
|
280
340
|
this._killTimer = null;
|
|
281
341
|
this._cancelingJobs = new Set();
|
|
282
342
|
this._zombieJobs = new Set();
|
|
343
|
+
this._persistenceFailedJobs = new Set();
|
|
283
344
|
this._broadcast = broadcast;
|
|
284
345
|
this._db = db ?? null;
|
|
285
346
|
this._logBuffer = [];
|
|
@@ -287,12 +348,16 @@ class QueueManager {
|
|
|
287
348
|
this._cwd = cwd;
|
|
288
349
|
this._inactivityTimer = null;
|
|
289
350
|
this._disposed = false;
|
|
351
|
+
this._restoreBlocked = false;
|
|
352
|
+
this._lifecycleGeneration = 0;
|
|
290
353
|
this._getCostAlertThreshold = options?.getCostAlertThreshold ?? null;
|
|
291
354
|
this._getDesktopDailyBudget = options?.getDesktopDailyBudget ?? null;
|
|
292
355
|
this._adapter = (0, providers_1.getAdapter)(options?.provider ?? 'claude');
|
|
293
356
|
this._resolvedModel = options?.resolvedModel ?? null;
|
|
294
357
|
this._onJobFinished = options?.onJobFinished ?? null;
|
|
358
|
+
this._onJobAdmission = options?.onJobAdmission ?? null;
|
|
295
359
|
this._onBudgetExceeded = options?.onBudgetExceeded ?? null;
|
|
360
|
+
this._resolvePluginsForSpawn = options?.resolvePluginsForSpawn ?? null;
|
|
296
361
|
this._projectId = options?.projectId ?? null;
|
|
297
362
|
this._desktopPort = options?.desktopPort ?? 4200;
|
|
298
363
|
this._projectSlug = options?.projectSlug ?? null;
|
|
@@ -306,7 +371,12 @@ class QueueManager {
|
|
|
306
371
|
this._jobPrDelivery = new Map();
|
|
307
372
|
this._interactiveSessions = new Map();
|
|
308
373
|
this._jobLiveAccounting = new Map();
|
|
374
|
+
this._jobReaders = new Map();
|
|
309
375
|
this._forceFailedRowJobs = new Set();
|
|
376
|
+
this._forceFailedProcesses = new Map();
|
|
377
|
+
this._terminalPersistenceBlockedJobs = new Set();
|
|
378
|
+
this._pendingLateReconciliations = new Map();
|
|
379
|
+
this._emittedPipelineStatuses = new Map();
|
|
310
380
|
const envTimeout = process.env.WM_ZOMBIE_TIMEOUT_MS !== undefined
|
|
311
381
|
? parseInt(process.env.WM_ZOMBIE_TIMEOUT_MS, 10)
|
|
312
382
|
: null;
|
|
@@ -338,9 +408,17 @@ class QueueManager {
|
|
|
338
408
|
* is closed (e.g. in ProjectRegistry.removeProject) and on graceful shutdown.
|
|
339
409
|
*/
|
|
340
410
|
shutdown() {
|
|
341
|
-
if (this._disposed)
|
|
342
|
-
|
|
411
|
+
if (this._disposed) {
|
|
412
|
+
if (!this._db)
|
|
413
|
+
return true;
|
|
414
|
+
// A previous removal attempt may have stopped children but failed to
|
|
415
|
+
// stage their terminal intent. Re-enter the idempotent shutdown path with
|
|
416
|
+
// the retained ownership/maps so storage repair can converge it.
|
|
417
|
+
this._disposed = false;
|
|
418
|
+
return this.shutdown();
|
|
419
|
+
}
|
|
343
420
|
this._disposed = true;
|
|
421
|
+
this._lifecycleGeneration += 1;
|
|
344
422
|
if (this._inactivityTimer !== null) {
|
|
345
423
|
clearTimeout(this._inactivityTimer);
|
|
346
424
|
this._inactivityTimer = null;
|
|
@@ -349,8 +427,14 @@ class QueueManager {
|
|
|
349
427
|
clearTimeout(this._killTimer);
|
|
350
428
|
this._killTimer = null;
|
|
351
429
|
}
|
|
352
|
-
const
|
|
353
|
-
if (
|
|
430
|
+
const ownedProcesses = new Set();
|
|
431
|
+
if (this._activeProcess)
|
|
432
|
+
ownedProcesses.add(this._activeProcess);
|
|
433
|
+
for (const proc of this._forceFailedProcesses.values())
|
|
434
|
+
ownedProcesses.add(proc);
|
|
435
|
+
for (const proc of ownedProcesses) {
|
|
436
|
+
if (!proc.pid)
|
|
437
|
+
continue;
|
|
354
438
|
const pid = proc.pid;
|
|
355
439
|
try {
|
|
356
440
|
(0, win_spawn_1.treeKillSafe)(pid, 'SIGTERM', () => { });
|
|
@@ -370,9 +454,40 @@ class QueueManager {
|
|
|
370
454
|
// the treeKill'd child's later 'close' hits `if (this._disposed) return` in
|
|
371
455
|
// _onJobExit, so without this flush the whole job's spend is lost
|
|
372
456
|
// (COST-ACCOUNTING-AUDIT CRIT-3 / HIGH-1). Best-effort — never throws.
|
|
373
|
-
this.
|
|
457
|
+
for (const jobId of Array.from(this._forceFailedRowJobs)) {
|
|
458
|
+
const live = this._jobLiveAccounting.get(jobId);
|
|
459
|
+
this._reconcileForceFailedJobExit(jobId, null, live?.events ?? [], live?.adapter ?? this._adapter, live?.model);
|
|
460
|
+
}
|
|
461
|
+
for (const [jobId, pending] of Array.from(this._pendingLateReconciliations)) {
|
|
462
|
+
this._reconcileForceFailedJobExit(jobId, pending.code, pending.adapterEvents, pending.adapter, pending.spawnedModel);
|
|
463
|
+
}
|
|
464
|
+
if (!this._flushInFlightAccounting())
|
|
465
|
+
return false;
|
|
466
|
+
try {
|
|
467
|
+
// A child may have closed through the disposed guard after an earlier
|
|
468
|
+
// staging failure, leaving only its durable RUNNING row. Capture that row
|
|
469
|
+
// from raw events before declaring a retried removal safe.
|
|
470
|
+
this._captureOrphanRecoveries();
|
|
471
|
+
this._resumeOrphanRecoveries();
|
|
472
|
+
}
|
|
473
|
+
catch (err) {
|
|
474
|
+
console.error('[queue-manager] shutdown orphan capture failed:', err);
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
for (const readers of this._jobReaders.values()) {
|
|
478
|
+
try {
|
|
479
|
+
readers.stdout.close();
|
|
480
|
+
}
|
|
481
|
+
catch { /* best-effort */ }
|
|
482
|
+
try {
|
|
483
|
+
readers.stderr.close();
|
|
484
|
+
}
|
|
485
|
+
catch { /* best-effort */ }
|
|
486
|
+
}
|
|
487
|
+
this._jobReaders.clear();
|
|
374
488
|
this._activeProcess = null;
|
|
375
489
|
this._activeJobId = null;
|
|
490
|
+
this._forceFailedProcesses.clear();
|
|
376
491
|
// Tear down any resident interactive sessions (SIGTERM their children) so
|
|
377
492
|
// teardown orphans no persistent claude process. dispose() does not settle
|
|
378
493
|
// (the aborted row was already written by _flushInFlightAccounting).
|
|
@@ -390,26 +505,172 @@ class QueueManager {
|
|
|
390
505
|
this._jobPrDelivery.clear();
|
|
391
506
|
this._jobLiveAccounting.clear();
|
|
392
507
|
this._openspecShims.clear();
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
|
|
508
|
+
// Project removal must retain this durable DB until every critical outbox
|
|
509
|
+
// effect converges. App shutdown may still close the connection externally;
|
|
510
|
+
// the file/outbox survives for next startup.
|
|
511
|
+
const recoveryComplete = this._terminalRecoveryComplete();
|
|
512
|
+
if (recoveryComplete)
|
|
513
|
+
this._db = null;
|
|
514
|
+
return recoveryComplete;
|
|
515
|
+
}
|
|
516
|
+
_terminalRecoveryComplete() {
|
|
517
|
+
const db = this._db;
|
|
518
|
+
if (!db)
|
|
519
|
+
return true;
|
|
520
|
+
try {
|
|
521
|
+
const rows = db.prepare(`
|
|
522
|
+
SELECT job_id, payload, accounting_completed, callback_completed, terminal_completed
|
|
523
|
+
FROM orphan_job_recovery
|
|
524
|
+
`).all();
|
|
525
|
+
const running = db.prepare(`SELECT 1 FROM jobs WHERE status = 'running' AND owner = 'queue' LIMIT 1`).get();
|
|
526
|
+
if (running)
|
|
527
|
+
return false;
|
|
528
|
+
for (const row of rows) {
|
|
529
|
+
const { payload } = this._decodeRecoveryPayload(row.job_id, row.payload);
|
|
530
|
+
if (row.accounting_completed === 0 ||
|
|
531
|
+
row.callback_completed === 0 ||
|
|
532
|
+
row.terminal_completed === 0 ||
|
|
533
|
+
payload.awaitingLateReconciliation)
|
|
534
|
+
return false;
|
|
535
|
+
}
|
|
536
|
+
return true;
|
|
537
|
+
}
|
|
538
|
+
catch (err) {
|
|
539
|
+
console.error('[queue-manager] terminal recovery completion check failed:', err);
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
/** Atomically terminalize a job and create its immutable recovery intent.
|
|
544
|
+
* No accounting, ticket/rail callback, dependent mutation or user-visible
|
|
545
|
+
* terminal broadcast may happen before this transaction commits. */
|
|
546
|
+
_stageTerminalIntent(job, input) {
|
|
547
|
+
const db = this._db;
|
|
548
|
+
if (!db)
|
|
549
|
+
throw new Error('Cannot stage a terminal job without its project database');
|
|
550
|
+
const result = input.result ?? {};
|
|
551
|
+
const payload = {
|
|
552
|
+
id: job.id,
|
|
553
|
+
command: job.command,
|
|
554
|
+
ticketIds: this._extractTicketIds(job.command),
|
|
555
|
+
pipelineId: job.pipelineId,
|
|
556
|
+
startedAt: job.startedAt ?? input.finishedAt,
|
|
557
|
+
finishedAt: input.finishedAt,
|
|
558
|
+
provider: input.provider,
|
|
559
|
+
model: result.model ?? null,
|
|
560
|
+
tokensIn: result.tokens_in ?? null,
|
|
561
|
+
tokensOut: result.tokens_out ?? null,
|
|
562
|
+
tokensCacheRead: result.tokens_cache_read ?? null,
|
|
563
|
+
tokensCacheCreate: result.tokens_cache_create ?? null,
|
|
564
|
+
totalCostUsd: result.total_cost_usd ?? null,
|
|
565
|
+
totalCostUsdEstimated: result.total_cost_usd_estimated == null
|
|
566
|
+
? 0
|
|
567
|
+
: (result.total_cost_usd_estimated ? 1 : 0),
|
|
568
|
+
numTurns: result.num_turns ?? null,
|
|
569
|
+
durationMs: result.duration_ms ?? null,
|
|
570
|
+
durationApiMs: result.duration_api_ms ?? null,
|
|
571
|
+
sessionId: result.session_id ?? null,
|
|
572
|
+
descendants: input.descendants ?? (input.status === 'completed' ? [] : this._snapshotRecoveryDescendants(job.id)),
|
|
573
|
+
causalOwnership: job.causalOwnership === true,
|
|
574
|
+
terminalStatus: input.status,
|
|
575
|
+
invocationStatus: input.invocationStatus,
|
|
576
|
+
ticketCompletionStatus: input.ticketCompletionStatus,
|
|
577
|
+
exitCode: input.exitCode,
|
|
578
|
+
awaitingLateReconciliation: input.awaitingLateReconciliation,
|
|
579
|
+
};
|
|
580
|
+
const encoded = JSON.stringify(this._decodeRecoveryPayload(job.id, JSON.stringify(payload)).payload);
|
|
581
|
+
const stage = db.transaction(() => {
|
|
582
|
+
// Works for both a running row and an async pre-spawn admission. The
|
|
583
|
+
// nested promotion transaction removes queued_jobs in the same commit.
|
|
584
|
+
(0, db_1.createJob)(db, {
|
|
585
|
+
id: job.id,
|
|
586
|
+
command: job.command,
|
|
587
|
+
started_at: payload.startedAt,
|
|
588
|
+
provider: payload.provider,
|
|
589
|
+
priority: job.priority,
|
|
590
|
+
depends_on_job_id: input.dependsOnJobId === undefined
|
|
591
|
+
? job.dependsOnJobId
|
|
592
|
+
: input.dependsOnJobId,
|
|
593
|
+
pipeline_id: job.pipelineId,
|
|
594
|
+
interactive: input.interactive,
|
|
595
|
+
causal_ownership: job.causalOwnership === true,
|
|
596
|
+
});
|
|
597
|
+
(0, db_1.finishJob)(db, job.id, {
|
|
598
|
+
exit_code: input.exitCode ?? -1,
|
|
599
|
+
status: input.status,
|
|
600
|
+
tokens_in: payload.tokensIn ?? undefined,
|
|
601
|
+
tokens_out: payload.tokensOut ?? undefined,
|
|
602
|
+
tokens_cache_read: payload.tokensCacheRead ?? undefined,
|
|
603
|
+
tokens_cache_create: payload.tokensCacheCreate ?? undefined,
|
|
604
|
+
total_cost_usd: payload.totalCostUsd ?? undefined,
|
|
605
|
+
total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
|
|
606
|
+
num_turns: payload.numTurns ?? undefined,
|
|
607
|
+
model: payload.model ?? undefined,
|
|
608
|
+
duration_ms: payload.durationMs ?? undefined,
|
|
609
|
+
duration_api_ms: payload.durationApiMs ?? undefined,
|
|
610
|
+
session_id: payload.sessionId ?? undefined,
|
|
611
|
+
});
|
|
612
|
+
db.prepare(`
|
|
613
|
+
INSERT INTO orphan_job_recovery (
|
|
614
|
+
job_id, payload, accounting_completed, callback_completed, terminal_completed
|
|
615
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
616
|
+
ON CONFLICT(job_id) DO NOTHING
|
|
617
|
+
`).run(job.id, encoded, input.accountingCompleted || !this._projectId ? 1 : 0, this._onJobFinished ? 0 : 1, input.terminalCompleted ? 1 : 0);
|
|
618
|
+
const intent = db.prepare(`SELECT payload FROM orphan_job_recovery WHERE job_id = ?`)
|
|
619
|
+
.get(job.id);
|
|
620
|
+
if (!intent || intent.payload !== encoded) {
|
|
621
|
+
throw new Error(`A different terminal recovery intent already owns job ${job.id}`);
|
|
622
|
+
}
|
|
623
|
+
const persisted = db.prepare(`SELECT status FROM jobs WHERE id = ?`).get(job.id);
|
|
624
|
+
if (persisted?.status !== input.status) {
|
|
625
|
+
throw new Error(`Failed to persist terminal status ${input.status} for ${job.id}`);
|
|
626
|
+
}
|
|
627
|
+
});
|
|
628
|
+
stage();
|
|
629
|
+
this._terminalPersistenceBlockedJobs.delete(job.id);
|
|
630
|
+
return payload;
|
|
631
|
+
}
|
|
632
|
+
/** Graceful shutdown uses the same terminal intent as ordinary exits. */
|
|
633
|
+
_stageGracefulAbort(job, payload, interactive) {
|
|
634
|
+
this._stageTerminalIntent(job, {
|
|
635
|
+
status: 'failed',
|
|
636
|
+
invocationStatus: 'aborted',
|
|
637
|
+
provider: payload.provider,
|
|
638
|
+
finishedAt: payload.finishedAt,
|
|
639
|
+
exitCode: -1,
|
|
640
|
+
interactive,
|
|
641
|
+
result: {
|
|
642
|
+
tokens_in: payload.tokensIn ?? undefined,
|
|
643
|
+
tokens_out: payload.tokensOut ?? undefined,
|
|
644
|
+
tokens_cache_read: payload.tokensCacheRead ?? undefined,
|
|
645
|
+
tokens_cache_create: payload.tokensCacheCreate ?? undefined,
|
|
646
|
+
total_cost_usd: payload.totalCostUsd ?? undefined,
|
|
647
|
+
total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
|
|
648
|
+
num_turns: payload.numTurns ?? undefined,
|
|
649
|
+
model: payload.model ?? undefined,
|
|
650
|
+
duration_ms: payload.durationMs ?? undefined,
|
|
651
|
+
duration_api_ms: payload.durationApiMs ?? undefined,
|
|
652
|
+
session_id: payload.sessionId ?? undefined,
|
|
653
|
+
},
|
|
654
|
+
descendants: payload.descendants,
|
|
655
|
+
});
|
|
656
|
+
job.status = 'failed';
|
|
657
|
+
job.finishedAt = payload.finishedAt;
|
|
658
|
+
job.exitCode = -1;
|
|
396
659
|
}
|
|
397
660
|
/**
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
* `adapterEvents` captured so far (foundation contract); for an interactive
|
|
402
|
-
* session it is the accumulated per-turn spend plus any folded in-flight turn.
|
|
403
|
-
* Called once from shutdown() BEFORE `_db` is nulled. Best-effort per job.
|
|
661
|
+
* Stage every in-flight job before shutdown/project removal, then immediately
|
|
662
|
+
* drain the callback and terminal checkpoints while the DB and project owners
|
|
663
|
+
* are still alive. Any failed critical callback remains durable for restart.
|
|
404
664
|
*/
|
|
405
665
|
_flushInFlightAccounting() {
|
|
406
666
|
const db = this._db;
|
|
407
667
|
const projectId = this._projectId;
|
|
408
668
|
if (!db || !projectId)
|
|
409
|
-
return;
|
|
669
|
+
return true;
|
|
670
|
+
let staged = true;
|
|
410
671
|
// ── Active non-interactive rail ──────────────────────────────────────────
|
|
411
672
|
const activeJobId = this._activeJobId;
|
|
412
|
-
if (activeJobId) {
|
|
673
|
+
if (activeJobId && !this._interactiveSessions.has(activeJobId)) {
|
|
413
674
|
const job = this._jobs.get(activeJobId);
|
|
414
675
|
if (job && job.status === 'running') {
|
|
415
676
|
try {
|
|
@@ -419,39 +680,30 @@ class QueueManager {
|
|
|
419
680
|
: { result: {}, estimated: false };
|
|
420
681
|
const provider = live?.adapter.id ?? this._jobResolvedProvider.get(activeJobId) ?? this._adapter.id;
|
|
421
682
|
const finishedAt = new Date().toISOString();
|
|
422
|
-
job
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
status: 'failed',
|
|
428
|
-
tokens_in: normalised.tokens_in,
|
|
429
|
-
tokens_out: normalised.tokens_out,
|
|
430
|
-
tokens_cache_read: normalised.tokens_cache_read,
|
|
431
|
-
tokens_cache_create: normalised.tokens_cache_create,
|
|
432
|
-
total_cost_usd: normalised.total_cost_usd,
|
|
433
|
-
total_cost_usd_estimated: estimated,
|
|
434
|
-
num_turns: normalised.num_turns,
|
|
435
|
-
model: normalised.model,
|
|
436
|
-
duration_ms: normalised.duration_ms,
|
|
437
|
-
duration_api_ms: normalised.duration_api_ms,
|
|
438
|
-
session_id: normalised.session_id,
|
|
439
|
-
});
|
|
440
|
-
}
|
|
441
|
-
catch { /* DB may be mid-close */ }
|
|
442
|
-
this._recordJobInvocations({
|
|
443
|
-
jobId: activeJobId,
|
|
444
|
-
provider,
|
|
445
|
-
status: 'aborted',
|
|
683
|
+
this._stageGracefulAbort(job, {
|
|
684
|
+
id: activeJobId,
|
|
685
|
+
command: job.command,
|
|
686
|
+
ticketIds: this._extractTicketIds(job.command),
|
|
687
|
+
pipelineId: job.pipelineId,
|
|
446
688
|
startedAt: job.startedAt ?? finishedAt,
|
|
447
689
|
finishedAt,
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
690
|
+
provider,
|
|
691
|
+
model: normalised.model ?? null,
|
|
692
|
+
tokensIn: normalised.tokens_in ?? null,
|
|
693
|
+
tokensOut: normalised.tokens_out ?? null,
|
|
694
|
+
tokensCacheRead: normalised.tokens_cache_read ?? null,
|
|
695
|
+
tokensCacheCreate: normalised.tokens_cache_create ?? null,
|
|
696
|
+
totalCostUsd: normalised.total_cost_usd ?? null,
|
|
697
|
+
totalCostUsdEstimated: estimated ? 1 : 0,
|
|
698
|
+
numTurns: normalised.num_turns ?? null,
|
|
699
|
+
durationMs: normalised.duration_ms ?? null,
|
|
700
|
+
durationApiMs: normalised.duration_api_ms ?? null,
|
|
701
|
+
sessionId: normalised.session_id ?? null,
|
|
702
|
+
}, false);
|
|
452
703
|
this._broadcast({ type: 'spending.invalidated', projectId });
|
|
453
704
|
}
|
|
454
705
|
catch (err) {
|
|
706
|
+
staged = false;
|
|
455
707
|
console.error('[queue-manager] shutdown flush (active job) failed:', err);
|
|
456
708
|
}
|
|
457
709
|
}
|
|
@@ -464,41 +716,55 @@ class QueueManager {
|
|
|
464
716
|
try {
|
|
465
717
|
const snap = session.snapshotForAbort();
|
|
466
718
|
const finishedAt = new Date().toISOString();
|
|
467
|
-
|
|
468
|
-
job
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
this._recordJobInvocations({
|
|
474
|
-
jobId,
|
|
475
|
-
provider: 'claude',
|
|
476
|
-
status: 'aborted',
|
|
719
|
+
const provider = this._jobResolvedProvider.get(jobId) ?? 'claude';
|
|
720
|
+
this._stageGracefulAbort(job, {
|
|
721
|
+
id: jobId,
|
|
722
|
+
command: job.command,
|
|
723
|
+
ticketIds: this._extractTicketIds(job.command),
|
|
724
|
+
pipelineId: job.pipelineId,
|
|
477
725
|
startedAt: job.startedAt ?? finishedAt,
|
|
478
726
|
finishedAt,
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
});
|
|
727
|
+
provider,
|
|
728
|
+
model: snap.model ?? null,
|
|
729
|
+
tokensIn: snap.totals.tokens_in,
|
|
730
|
+
tokensOut: snap.totals.tokens_out,
|
|
731
|
+
tokensCacheRead: snap.totals.tokens_cache_read,
|
|
732
|
+
tokensCacheCreate: snap.totals.tokens_cache_create,
|
|
733
|
+
totalCostUsd: snap.totals.total_cost_usd,
|
|
734
|
+
totalCostUsdEstimated: snap.estimated ? 1 : 0,
|
|
735
|
+
numTurns: snap.totals.num_turns,
|
|
736
|
+
durationMs: snap.activeDurationMs,
|
|
737
|
+
durationApiMs: null,
|
|
738
|
+
sessionId: snap.sessionId ?? null,
|
|
739
|
+
}, true);
|
|
493
740
|
this._broadcast({ type: 'spending.invalidated', projectId });
|
|
494
741
|
}
|
|
495
742
|
catch (err) {
|
|
743
|
+
staged = false;
|
|
496
744
|
console.error('[queue-manager] shutdown flush (interactive job) failed:', err);
|
|
497
745
|
}
|
|
498
746
|
}
|
|
747
|
+
// Complete local ticket/rail and dependent invariants now (important for
|
|
748
|
+
// project removal, where this DB may never reopen). Failed critical effects
|
|
749
|
+
// stay in orphan_job_recovery and are replayed on the next project load.
|
|
750
|
+
try {
|
|
751
|
+
this._resumeOrphanRecoveries();
|
|
752
|
+
}
|
|
753
|
+
catch (err) {
|
|
754
|
+
console.error('[queue-manager] shutdown recovery replay failed:', err);
|
|
755
|
+
}
|
|
756
|
+
return staged;
|
|
499
757
|
}
|
|
500
758
|
// ─── Public API ─────────────────────────────────────────────────────────────
|
|
501
|
-
|
|
759
|
+
_assertMutable() {
|
|
760
|
+
if (this._disposed)
|
|
761
|
+
throw new Error('Queue manager is shutting down');
|
|
762
|
+
}
|
|
763
|
+
enqueue(command, priorityOrOpts, opts, durableAdmission) {
|
|
764
|
+
this._assertMutable();
|
|
765
|
+
if (this._restoreBlocked) {
|
|
766
|
+
throw new Error('Queue recovery is incomplete; retry resume after repairing storage');
|
|
767
|
+
}
|
|
502
768
|
// Support both: enqueue(cmd, priority, opts) and enqueue(cmd, opts)
|
|
503
769
|
let priority = 'normal';
|
|
504
770
|
let resolvedOpts = opts;
|
|
@@ -508,6 +774,20 @@ class QueueManager {
|
|
|
508
774
|
else if (priorityOrOpts && typeof priorityOrOpts === 'object') {
|
|
509
775
|
resolvedOpts = priorityOrOpts;
|
|
510
776
|
}
|
|
777
|
+
const rawDependsOnJobId = resolvedOpts?.dependsOnJobId;
|
|
778
|
+
let dependsOnJobId = null;
|
|
779
|
+
if (rawDependsOnJobId !== undefined) {
|
|
780
|
+
if (typeof rawDependsOnJobId !== 'string' || !rawDependsOnJobId.trim()) {
|
|
781
|
+
throw new InvalidJobDependencyError('dependsOnJobId must be a non-empty string');
|
|
782
|
+
}
|
|
783
|
+
dependsOnJobId = rawDependsOnJobId.trim();
|
|
784
|
+
const parentStatus = this._getDependencyStatus(dependsOnJobId);
|
|
785
|
+
if (parentStatus &&
|
|
786
|
+
TERMINAL_STATUSES.has(parentStatus) &&
|
|
787
|
+
parentStatus !== 'completed') {
|
|
788
|
+
throw new InvalidJobDependencyError(`Cannot depend on job ${dependsOnJobId} because it is ${parentStatus}`);
|
|
789
|
+
}
|
|
790
|
+
}
|
|
511
791
|
// Resolve the adapter for THIS job: the per-job provider override when set
|
|
512
792
|
// and installed, else the project's primary provider. The binary check
|
|
513
793
|
// below probes the chosen provider's CLI.
|
|
@@ -526,7 +806,19 @@ class QueueManager {
|
|
|
526
806
|
// none has been declared; the adapter's id surfaces in the error.
|
|
527
807
|
throw new Error(`${enqueueAdapter.binary} binary not found`);
|
|
528
808
|
}
|
|
529
|
-
|
|
809
|
+
if ((durableAdmission || this._onJobAdmission) && !this._db) {
|
|
810
|
+
throw new Error('Durable enqueue admission requires a project database');
|
|
811
|
+
}
|
|
812
|
+
const id = durableAdmission?.jobId ?? (0, ids_1.newId)();
|
|
813
|
+
if (typeof id !== 'string' || !id.trim()) {
|
|
814
|
+
throw new Error('Durable enqueue admission requires a non-empty jobId');
|
|
815
|
+
}
|
|
816
|
+
if (this._jobs.has(id)) {
|
|
817
|
+
throw new Error(`Job ${id} is already admitted`);
|
|
818
|
+
}
|
|
819
|
+
if (dependsOnJobId === id) {
|
|
820
|
+
throw new InvalidJobDependencyError('A job cannot depend on itself');
|
|
821
|
+
}
|
|
530
822
|
const job = {
|
|
531
823
|
id,
|
|
532
824
|
command,
|
|
@@ -536,21 +828,22 @@ class QueueManager {
|
|
|
536
828
|
startedAt: null,
|
|
537
829
|
finishedAt: null,
|
|
538
830
|
exitCode: null,
|
|
539
|
-
dependsOnJobId
|
|
831
|
+
dependsOnJobId,
|
|
540
832
|
pipelineId: resolvedOpts?.pipelineId ?? null,
|
|
541
833
|
skipReason: null,
|
|
542
834
|
resultText: null,
|
|
835
|
+
causalOwnership: false,
|
|
543
836
|
};
|
|
544
837
|
this._jobs.set(id, job);
|
|
545
838
|
// Record profile selection (if provided) so spawn time can pick it up.
|
|
546
839
|
// `undefined` means "use default resolution"; `null` means "force legacy".
|
|
547
|
-
if (resolvedOpts
|
|
548
|
-
this._jobProfileSelection.set(id, resolvedOpts.profileName
|
|
549
|
-
}
|
|
550
|
-
// Record per-job provider override so _startJob resolves the right adapter.
|
|
551
|
-
if (resolvedOpts?.provider) {
|
|
552
|
-
this._jobProviderSelection.set(id, resolvedOpts.provider);
|
|
840
|
+
if (resolvedOpts?.profileName !== undefined) {
|
|
841
|
+
this._jobProfileSelection.set(id, resolvedOpts.profileName);
|
|
553
842
|
}
|
|
843
|
+
// Pin the adapter resolved at admission even when no override was supplied.
|
|
844
|
+
// A restart or a long pre-spawn await must not reinterpret this job under a
|
|
845
|
+
// project default that changed after the user submitted it.
|
|
846
|
+
this._jobProviderSelection.set(id, enqueueAdapter.id);
|
|
554
847
|
// Record per-job model override (e.g. freestyle model picker).
|
|
555
848
|
if (resolvedOpts?.model) {
|
|
556
849
|
this._jobModelSelection.set(id, resolvedOpts.model);
|
|
@@ -574,12 +867,40 @@ class QueueManager {
|
|
|
574
867
|
}
|
|
575
868
|
this._queue.splice(insertIdx, 0, id);
|
|
576
869
|
this._recomputePositions();
|
|
577
|
-
|
|
870
|
+
try {
|
|
871
|
+
// Admission is not successful until it is durable. A best-effort write
|
|
872
|
+
// here would acknowledge work that disappears on the next process crash.
|
|
873
|
+
if (durableAdmission || this._onJobAdmission) {
|
|
874
|
+
const db = this._db;
|
|
875
|
+
const commitAdmission = db.transaction(() => {
|
|
876
|
+
this._persistQueuedState(true);
|
|
877
|
+
this._onJobAdmission?.(db, job);
|
|
878
|
+
durableAdmission?.commit(db, job);
|
|
879
|
+
});
|
|
880
|
+
commitAdmission();
|
|
881
|
+
}
|
|
882
|
+
else {
|
|
883
|
+
this._persistQueuedState(true);
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
catch (err) {
|
|
887
|
+
const idx = this._queue.indexOf(id);
|
|
888
|
+
if (idx !== -1)
|
|
889
|
+
this._queue.splice(idx, 1);
|
|
890
|
+
this._jobs.delete(id);
|
|
891
|
+
this._jobProfileSelection.delete(id);
|
|
892
|
+
this._jobProviderSelection.delete(id);
|
|
893
|
+
this._jobModelSelection.delete(id);
|
|
894
|
+
this._jobInteractiveSelection.delete(id);
|
|
895
|
+
this._recomputePositions();
|
|
896
|
+
throw err;
|
|
897
|
+
}
|
|
578
898
|
this._broadcastQueueState();
|
|
579
899
|
this._drainQueue();
|
|
580
900
|
return job;
|
|
581
901
|
}
|
|
582
902
|
cancel(jobId) {
|
|
903
|
+
this._assertMutable();
|
|
583
904
|
const job = this._jobs.get(jobId);
|
|
584
905
|
if (!job) {
|
|
585
906
|
throw new JobNotFoundError();
|
|
@@ -587,38 +908,20 @@ class QueueManager {
|
|
|
587
908
|
if (TERMINAL_STATUSES.has(job.status)) {
|
|
588
909
|
throw new JobAlreadyTerminalError();
|
|
589
910
|
}
|
|
590
|
-
if (
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
this.
|
|
601
|
-
this.
|
|
602
|
-
|
|
603
|
-
this.
|
|
604
|
-
this._jobInteractiveSelection.delete(jobId);
|
|
605
|
-
this._skipDependents(jobId, `Parent job ${jobId} was canceled`);
|
|
606
|
-
this._recomputePositions();
|
|
607
|
-
this._persistJob(job);
|
|
608
|
-
this._broadcastQueueState();
|
|
609
|
-
// M20: a queued cancel never reached _onJobFinished (only the running path
|
|
610
|
-
// does, via _kill→_onJobExit), so a rail-launched queued job left its
|
|
611
|
-
// railJobs entry stuck 'running' forever and dropped the job.canceled
|
|
612
|
-
// webhook + rail.job_completed broadcast. Fire the callback here too; it is
|
|
613
|
-
// exit-status-driven and idempotent on tickets.
|
|
614
|
-
if (this._onJobFinished) {
|
|
615
|
-
try {
|
|
616
|
-
this._onJobFinished(jobId, 'canceled', undefined);
|
|
617
|
-
}
|
|
618
|
-
catch (err) {
|
|
619
|
-
console.error(`[QueueManager] onJobFinished(canceled) failed for ${jobId}: ${err.message}`);
|
|
620
|
-
}
|
|
621
|
-
}
|
|
911
|
+
if (this._terminalPersistenceBlockedJobs.has(jobId)) {
|
|
912
|
+
throw new Error(`Job ${jobId} is awaiting durable terminal recovery`);
|
|
913
|
+
}
|
|
914
|
+
// `_startJob` marks the in-memory job running before its async plugin
|
|
915
|
+
// verification finishes, while the durable admission intentionally remains
|
|
916
|
+
// in queued_jobs until spawn. Treat that reserved/no-child window as
|
|
917
|
+
// cancelable pre-start work: a Set-only cancel intent would be lost by a
|
|
918
|
+
// process crash and the command would run after restart.
|
|
919
|
+
const isPreparingToSpawn = job.status === 'running' &&
|
|
920
|
+
this._activeJobId === jobId &&
|
|
921
|
+
this._activeProcess === null &&
|
|
922
|
+
!this._interactiveSessions.has(jobId);
|
|
923
|
+
if (job.status === 'queued' || isPreparingToSpawn) {
|
|
924
|
+
this._cancelBeforeSpawn(job, isPreparingToSpawn);
|
|
622
925
|
return 'canceled';
|
|
623
926
|
}
|
|
624
927
|
// job.status === 'running'
|
|
@@ -634,21 +937,233 @@ class QueueManager {
|
|
|
634
937
|
this._kill(jobId);
|
|
635
938
|
return 'canceling';
|
|
636
939
|
}
|
|
940
|
+
/**
|
|
941
|
+
* Cancel queued or asynchronously-preparing work as one durable state change.
|
|
942
|
+
* The parent cancellation, recursive dependent skips, queued-row removals and
|
|
943
|
+
* the surviving queue positions commit together. In-memory state is restored
|
|
944
|
+
* exactly when any write fails, so the API never reports a cancellation whose
|
|
945
|
+
* dependency chain can later resurrect from a partial SQLite commit.
|
|
946
|
+
*/
|
|
947
|
+
_cancelBeforeSpawn(job, wasPreparingToSpawn) {
|
|
948
|
+
const previousQueue = [...this._queue];
|
|
949
|
+
const previousActiveJobId = this._activeJobId;
|
|
950
|
+
const snapshots = new Map();
|
|
951
|
+
const snapshot = (candidate) => {
|
|
952
|
+
if (snapshots.has(candidate.id))
|
|
953
|
+
return;
|
|
954
|
+
snapshots.set(candidate.id, {
|
|
955
|
+
status: candidate.status,
|
|
956
|
+
queuePosition: candidate.queuePosition,
|
|
957
|
+
startedAt: candidate.startedAt,
|
|
958
|
+
finishedAt: candidate.finishedAt,
|
|
959
|
+
exitCode: candidate.exitCode,
|
|
960
|
+
skipReason: candidate.skipReason,
|
|
961
|
+
});
|
|
962
|
+
};
|
|
963
|
+
snapshot(job);
|
|
964
|
+
const canceledAt = new Date().toISOString();
|
|
965
|
+
const parentIndex = this._queue.indexOf(job.id);
|
|
966
|
+
if (parentIndex !== -1)
|
|
967
|
+
this._queue.splice(parentIndex, 1);
|
|
968
|
+
job.status = 'canceled';
|
|
969
|
+
job.finishedAt = canceledAt;
|
|
970
|
+
job.exitCode = null;
|
|
971
|
+
job.queuePosition = null;
|
|
972
|
+
const skipped = [];
|
|
973
|
+
const stageDependentSkips = (parentJobId, reason) => {
|
|
974
|
+
const children = Array.from(this._jobs.values()).filter((candidate) => candidate.dependsOnJobId === parentJobId && candidate.status === 'queued');
|
|
975
|
+
for (const child of children) {
|
|
976
|
+
snapshot(child);
|
|
977
|
+
const index = this._queue.indexOf(child.id);
|
|
978
|
+
if (index !== -1)
|
|
979
|
+
this._queue.splice(index, 1);
|
|
980
|
+
child.status = 'skipped';
|
|
981
|
+
child.finishedAt = canceledAt;
|
|
982
|
+
child.queuePosition = null;
|
|
983
|
+
child.skipReason = reason;
|
|
984
|
+
skipped.push(child);
|
|
985
|
+
stageDependentSkips(child.id, `Parent job ${child.id} was skipped`);
|
|
986
|
+
}
|
|
987
|
+
};
|
|
988
|
+
stageDependentSkips(job.id, `Parent job ${job.id} was canceled`);
|
|
989
|
+
this._recomputePositions();
|
|
990
|
+
try {
|
|
991
|
+
if (this._db) {
|
|
992
|
+
const db = this._db;
|
|
993
|
+
const persistCancellation = db.transaction(() => {
|
|
994
|
+
// Persist a canceled tombstone even for work that never reached the
|
|
995
|
+
// provider. Besides making the cancellation auditable, this satisfies
|
|
996
|
+
// jobs.depends_on_job_id while descendant skipped rows are inserted;
|
|
997
|
+
// using canceledAt for both timestamps truthfully records zero runtime.
|
|
998
|
+
const persistedParent = job.dependsOnJobId
|
|
999
|
+
? db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(job.dependsOnJobId)
|
|
1000
|
+
: null;
|
|
1001
|
+
this._stageTerminalIntent(job, {
|
|
1002
|
+
status: 'canceled',
|
|
1003
|
+
invocationStatus: 'aborted',
|
|
1004
|
+
provider: this._jobResolvedProvider.get(job.id)
|
|
1005
|
+
?? this._jobProviderSelection.get(job.id)
|
|
1006
|
+
?? this._adapter.id,
|
|
1007
|
+
finishedAt: canceledAt,
|
|
1008
|
+
exitCode: -1,
|
|
1009
|
+
accountingCompleted: true,
|
|
1010
|
+
// A not-yet-started parent lives only in queued_jobs and therefore
|
|
1011
|
+
// cannot satisfy jobs.depends_on_job_id's FK. Terminal history does
|
|
1012
|
+
// not need to retain that unresolved edge; descendants of THIS job
|
|
1013
|
+
// still reference the canceled tombstone inserted here.
|
|
1014
|
+
dependsOnJobId: persistedParent ? job.dependsOnJobId : null,
|
|
1015
|
+
descendants: skipped.map((child) => ({
|
|
1016
|
+
id: child.id,
|
|
1017
|
+
command: child.command,
|
|
1018
|
+
parentId: child.dependsOnJobId ?? job.id,
|
|
1019
|
+
pipelineId: child.pipelineId,
|
|
1020
|
+
priority: child.priority,
|
|
1021
|
+
causalOwnership: child.causalOwnership === true,
|
|
1022
|
+
})),
|
|
1023
|
+
});
|
|
1024
|
+
for (const child of skipped) {
|
|
1025
|
+
const exists = db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(child.id);
|
|
1026
|
+
if (exists) {
|
|
1027
|
+
(0, db_1.skipJob)(db, child.id, child.skipReason ?? `Parent job ${job.id} was canceled`);
|
|
1028
|
+
}
|
|
1029
|
+
else {
|
|
1030
|
+
db.prepare(`INSERT INTO jobs (
|
|
1031
|
+
id, command, started_at, status, skip_reason, finished_at,
|
|
1032
|
+
depends_on_job_id, pipeline_id, causal_ownership
|
|
1033
|
+
) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?)`).run(child.id, child.command, child.finishedAt ?? canceledAt, child.skipReason, child.finishedAt ?? canceledAt, child.dependsOnJobId, child.pipelineId, child.causalOwnership === true ? 1 : 0);
|
|
1034
|
+
}
|
|
1035
|
+
(0, db_1.deleteQueuedJob)(db, child.id);
|
|
1036
|
+
}
|
|
1037
|
+
// Nested better-sqlite3 transactions use a savepoint; every surviving
|
|
1038
|
+
// position therefore shares the cancellation's outer commit.
|
|
1039
|
+
this._persistQueuedState(true);
|
|
1040
|
+
});
|
|
1041
|
+
persistCancellation();
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
catch (err) {
|
|
1045
|
+
this._queue = previousQueue;
|
|
1046
|
+
this._activeJobId = previousActiveJobId;
|
|
1047
|
+
for (const [id, previous] of snapshots) {
|
|
1048
|
+
const candidate = this._jobs.get(id);
|
|
1049
|
+
if (candidate)
|
|
1050
|
+
Object.assign(candidate, previous);
|
|
1051
|
+
}
|
|
1052
|
+
this._recomputePositions();
|
|
1053
|
+
throw err;
|
|
1054
|
+
}
|
|
1055
|
+
if (wasPreparingToSpawn && this._activeJobId === job.id) {
|
|
1056
|
+
// Invalidates every `_canContinueStart` check when the pending await
|
|
1057
|
+
// resolves; no provider process can be created after this point.
|
|
1058
|
+
this._activeJobId = null;
|
|
1059
|
+
}
|
|
1060
|
+
this._cancelingJobs.delete(job.id);
|
|
1061
|
+
const terminalIds = [job.id, ...skipped.map((child) => child.id)];
|
|
1062
|
+
for (const id of terminalIds) {
|
|
1063
|
+
this._jobProviderSelection.delete(id);
|
|
1064
|
+
this._jobResolvedProvider.delete(id);
|
|
1065
|
+
this._jobModelSelection.delete(id);
|
|
1066
|
+
this._jobProfileSelection.delete(id);
|
|
1067
|
+
this._jobInteractiveSelection.delete(id);
|
|
1068
|
+
this._jobExecution.delete(id);
|
|
1069
|
+
this._snapshotRefs.delete(id);
|
|
1070
|
+
this._jobPrDelivery.delete(id);
|
|
1071
|
+
this._jobLiveAccounting.delete(id);
|
|
1072
|
+
this._cleanupOpenspecShim(id);
|
|
1073
|
+
}
|
|
1074
|
+
let accountingReady = true;
|
|
1075
|
+
if (this._db) {
|
|
1076
|
+
accountingReady = this._resumeOrphanRecoveries();
|
|
1077
|
+
}
|
|
1078
|
+
else {
|
|
1079
|
+
for (const child of skipped) {
|
|
1080
|
+
try {
|
|
1081
|
+
this._onJobFinished?.(child.id, 'skipped', undefined);
|
|
1082
|
+
}
|
|
1083
|
+
catch (err) {
|
|
1084
|
+
console.error(`[QueueManager] onJobFinished(skipped) failed for ${child.id}: ${err.message}`);
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
try {
|
|
1088
|
+
this._onJobFinished?.(job.id, 'canceled', undefined);
|
|
1089
|
+
}
|
|
1090
|
+
catch (err) {
|
|
1091
|
+
console.error(`[QueueManager] onJobFinished(canceled) failed for ${job.id}: ${err.message}`);
|
|
1092
|
+
}
|
|
1093
|
+
const affectedPipelines = new Set();
|
|
1094
|
+
if (job.pipelineId)
|
|
1095
|
+
affectedPipelines.add(job.pipelineId);
|
|
1096
|
+
for (const child of skipped) {
|
|
1097
|
+
if (child.pipelineId)
|
|
1098
|
+
affectedPipelines.add(child.pipelineId);
|
|
1099
|
+
}
|
|
1100
|
+
for (const pipelineId of affectedPipelines)
|
|
1101
|
+
this._checkPipelineStatus(pipelineId);
|
|
1102
|
+
}
|
|
1103
|
+
if (!accountingReady) {
|
|
1104
|
+
this._paused = true;
|
|
1105
|
+
this._persistQueueState();
|
|
1106
|
+
}
|
|
1107
|
+
this._broadcastQueueState();
|
|
1108
|
+
if (accountingReady)
|
|
1109
|
+
this._drainQueue();
|
|
1110
|
+
}
|
|
637
1111
|
pause() {
|
|
1112
|
+
this._assertMutable();
|
|
638
1113
|
this._paused = true;
|
|
639
1114
|
this._persistQueueState();
|
|
640
1115
|
this._broadcastQueueState();
|
|
641
1116
|
}
|
|
642
1117
|
resume() {
|
|
1118
|
+
this._assertMutable();
|
|
1119
|
+
if (this._restoreBlocked) {
|
|
1120
|
+
this._restoreFromDb();
|
|
1121
|
+
if (this._restoreBlocked) {
|
|
1122
|
+
this._paused = true;
|
|
1123
|
+
this._persistQueueState();
|
|
1124
|
+
this._broadcastQueueState();
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
for (const [jobId, pending] of Array.from(this._pendingLateReconciliations)) {
|
|
1129
|
+
this._reconcileForceFailedJobExit(jobId, pending.code, pending.adapterEvents, pending.adapter, pending.spawnedModel);
|
|
1130
|
+
}
|
|
1131
|
+
if (this._pendingLateReconciliations.size > 0) {
|
|
1132
|
+
this._paused = true;
|
|
1133
|
+
this._persistQueueState();
|
|
1134
|
+
this._broadcastQueueState();
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
if (this._terminalPersistenceBlockedJobs.size > 0) {
|
|
1138
|
+
this._paused = true;
|
|
1139
|
+
this._persistQueueState();
|
|
1140
|
+
this._broadcastQueueState();
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
if (!this._resumeOrphanRecoveries()) {
|
|
1144
|
+
// A provider run whose ledger is still pending must remain ahead of new
|
|
1145
|
+
// admissions; otherwise budget enforcement can undercount spend.
|
|
1146
|
+
this._paused = true;
|
|
1147
|
+
this._persistQueueState();
|
|
1148
|
+
this._broadcastQueueState();
|
|
1149
|
+
return;
|
|
1150
|
+
}
|
|
643
1151
|
this._paused = false;
|
|
644
1152
|
this._persistQueueState();
|
|
645
1153
|
this._broadcastQueueState();
|
|
646
1154
|
this._drainQueue();
|
|
647
1155
|
}
|
|
648
1156
|
reorder(jobIds) {
|
|
1157
|
+
this._assertMutable();
|
|
1158
|
+
if (jobIds.some((id) => typeof id !== 'string')) {
|
|
1159
|
+
throw new Error('jobIds must contain only string IDs');
|
|
1160
|
+
}
|
|
1161
|
+
if (jobIds.length !== this._queue.length) {
|
|
1162
|
+
throw new Error('jobIds must contain exactly the IDs of all currently-queued jobs');
|
|
1163
|
+
}
|
|
649
1164
|
const queuedSet = new Set(this._queue);
|
|
650
1165
|
const incomingSet = new Set(jobIds);
|
|
651
|
-
if (queuedSet.size !==
|
|
1166
|
+
if (incomingSet.size !== jobIds.length || queuedSet.size !== this._queue.length) {
|
|
652
1167
|
throw new Error('jobIds must contain exactly the IDs of all currently-queued jobs');
|
|
653
1168
|
}
|
|
654
1169
|
for (const id of jobIds) {
|
|
@@ -656,25 +1171,37 @@ class QueueManager {
|
|
|
656
1171
|
throw new Error(`Job ${id} is not in queued state`);
|
|
657
1172
|
}
|
|
658
1173
|
}
|
|
1174
|
+
for (let index = 1; index < jobIds.length; index += 1) {
|
|
1175
|
+
const previous = this._jobs.get(jobIds[index - 1]);
|
|
1176
|
+
const current = this._jobs.get(jobIds[index]);
|
|
1177
|
+
if (previous && current &&
|
|
1178
|
+
types_1.PRIORITY_WEIGHT[previous.priority] < types_1.PRIORITY_WEIGHT[current.priority]) {
|
|
1179
|
+
throw new Error('Cannot reorder jobs across priority levels; update priority first');
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
const previousQueue = [...this._queue];
|
|
659
1183
|
this._queue = [...jobIds];
|
|
660
1184
|
this._recomputePositions();
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
1185
|
+
try {
|
|
1186
|
+
this._persistQueuedState(true);
|
|
1187
|
+
}
|
|
1188
|
+
catch (err) {
|
|
1189
|
+
this._queue = previousQueue;
|
|
1190
|
+
this._recomputePositions();
|
|
1191
|
+
throw err;
|
|
668
1192
|
}
|
|
669
1193
|
this._broadcastQueueState();
|
|
670
1194
|
}
|
|
671
1195
|
updatePriority(jobId, priority) {
|
|
1196
|
+
this._assertMutable();
|
|
672
1197
|
const job = this._jobs.get(jobId);
|
|
673
1198
|
if (!job)
|
|
674
1199
|
throw new JobNotFoundError();
|
|
675
1200
|
if (job.status !== 'queued') {
|
|
676
1201
|
throw new Error('Can only change priority of queued jobs');
|
|
677
1202
|
}
|
|
1203
|
+
const previousPriority = job.priority;
|
|
1204
|
+
const previousQueue = [...this._queue];
|
|
678
1205
|
job.priority = priority;
|
|
679
1206
|
// Remove from queue and re-insert at correct position
|
|
680
1207
|
const idx = this._queue.indexOf(jobId);
|
|
@@ -691,7 +1218,15 @@ class QueueManager {
|
|
|
691
1218
|
}
|
|
692
1219
|
this._queue.splice(insertIdx, 0, jobId);
|
|
693
1220
|
this._recomputePositions();
|
|
694
|
-
|
|
1221
|
+
try {
|
|
1222
|
+
this._persistQueuedState(true);
|
|
1223
|
+
}
|
|
1224
|
+
catch (err) {
|
|
1225
|
+
job.priority = previousPriority;
|
|
1226
|
+
this._queue = previousQueue;
|
|
1227
|
+
this._recomputePositions();
|
|
1228
|
+
throw err;
|
|
1229
|
+
}
|
|
695
1230
|
this._broadcastQueueState();
|
|
696
1231
|
}
|
|
697
1232
|
getJobs() {
|
|
@@ -721,6 +1256,7 @@ class QueueManager {
|
|
|
721
1256
|
* active turn). Returns false when the job isn't an active interactive
|
|
722
1257
|
* session (unknown / already finalized / not interactive). */
|
|
723
1258
|
sendInteractiveTurn(jobId, text) {
|
|
1259
|
+
this._assertMutable();
|
|
724
1260
|
const session = this._interactiveSessions.get(jobId);
|
|
725
1261
|
if (!session)
|
|
726
1262
|
return false;
|
|
@@ -730,6 +1266,7 @@ class QueueManager {
|
|
|
730
1266
|
* the settle path stamps the summed totals + 'completed' status. Returns false
|
|
731
1267
|
* when the job isn't an active interactive session. */
|
|
732
1268
|
finalizeInteractive(jobId) {
|
|
1269
|
+
this._assertMutable();
|
|
733
1270
|
const session = this._interactiveSessions.get(jobId);
|
|
734
1271
|
if (!session)
|
|
735
1272
|
return false;
|
|
@@ -978,11 +1515,39 @@ class QueueManager {
|
|
|
978
1515
|
return;
|
|
979
1516
|
if (this._queue.length === 0)
|
|
980
1517
|
return;
|
|
1518
|
+
// Defense in depth for state restored from old/corrupt builds or mutated by
|
|
1519
|
+
// an invalid caller: only unique, presently-queued jobs may reach start.
|
|
1520
|
+
// In particular, a duplicated id must not relaunch after its first run
|
|
1521
|
+
// becomes terminal.
|
|
1522
|
+
const seen = new Set();
|
|
1523
|
+
const normalizedQueue = [];
|
|
1524
|
+
const removedJobIds = [];
|
|
1525
|
+
for (const id of this._queue) {
|
|
1526
|
+
if (seen.has(id))
|
|
1527
|
+
continue;
|
|
1528
|
+
seen.add(id);
|
|
1529
|
+
const candidate = this._jobs.get(id);
|
|
1530
|
+
if (candidate?.status === 'queued')
|
|
1531
|
+
normalizedQueue.push(id);
|
|
1532
|
+
else
|
|
1533
|
+
removedJobIds.push(id);
|
|
1534
|
+
}
|
|
1535
|
+
if (normalizedQueue.length !== this._queue.length) {
|
|
1536
|
+
this._queue = normalizedQueue;
|
|
1537
|
+
this._recomputePositions();
|
|
1538
|
+
this._persistQueuedState(false, removedJobIds);
|
|
1539
|
+
if (this._queue.length === 0)
|
|
1540
|
+
return;
|
|
1541
|
+
}
|
|
1542
|
+
// App/project budgets are durable policies, not merely post-job alerts.
|
|
1543
|
+
// Re-check before reserving a slot so a different project that crossed the
|
|
1544
|
+
// app-wide cap cannot be followed by a fresh spawn from this queue.
|
|
1545
|
+
this._enforceDailyBudget();
|
|
1546
|
+
if (this._paused)
|
|
1547
|
+
return;
|
|
981
1548
|
const readyIndex = this._queue.findIndex(id => {
|
|
982
1549
|
const job = this._jobs.get(id);
|
|
983
|
-
|
|
984
|
-
return true;
|
|
985
|
-
return this._isDependencyMet(job);
|
|
1550
|
+
return job?.status === 'queued' && this._isDependencyMet(job);
|
|
986
1551
|
});
|
|
987
1552
|
if (readyIndex === -1)
|
|
988
1553
|
return;
|
|
@@ -995,6 +1560,9 @@ class QueueManager {
|
|
|
995
1560
|
// clobbered so cancel/zombie-kill hits the wrong child.
|
|
996
1561
|
this._activeJobId = nextJobId;
|
|
997
1562
|
this._recomputePositions();
|
|
1563
|
+
// Keep every still-queued position in sync while preserving the selected
|
|
1564
|
+
// job's durable admission until createJob promotes it just before spawn.
|
|
1565
|
+
this._persistQueuedState(false, [], nextJobId);
|
|
998
1566
|
void this._startJob(nextJobId).catch((err) => {
|
|
999
1567
|
console.error(`[QueueManager] _startJob(${nextJobId}) threw before spawn: ${err?.message}`);
|
|
1000
1568
|
// Only release if we never established a child (else _onJobExit owns cleanup).
|
|
@@ -1002,9 +1570,10 @@ class QueueManager {
|
|
|
1002
1570
|
// Stamp the job terminal — _onJobExit never runs without a child, so the
|
|
1003
1571
|
// job would otherwise wedge 'running' forever and never fire
|
|
1004
1572
|
// onJobFinished (rail/webhook never settle) and leak its per-job maps.
|
|
1005
|
-
this._failWedgedJob(nextJobId, err?.message ?? 'startup failure');
|
|
1006
1573
|
this._activeJobId = null;
|
|
1007
|
-
this.
|
|
1574
|
+
const terminalPersisted = this._failWedgedJob(nextJobId, err?.message ?? 'startup failure');
|
|
1575
|
+
if (terminalPersisted)
|
|
1576
|
+
this._drainQueue();
|
|
1008
1577
|
}
|
|
1009
1578
|
});
|
|
1010
1579
|
}
|
|
@@ -1016,54 +1585,105 @@ class QueueManager {
|
|
|
1016
1585
|
*/
|
|
1017
1586
|
_failWedgedJob(jobId, reason) {
|
|
1018
1587
|
const job = this._jobs.get(jobId);
|
|
1588
|
+
const previousJobState = job ? {
|
|
1589
|
+
status: job.status,
|
|
1590
|
+
queuePosition: job.queuePosition,
|
|
1591
|
+
startedAt: job.startedAt,
|
|
1592
|
+
finishedAt: job.finishedAt,
|
|
1593
|
+
exitCode: job.exitCode,
|
|
1594
|
+
skipReason: job.skipReason,
|
|
1595
|
+
} : null;
|
|
1019
1596
|
// The wedge can land before _startJob set status='running' (an early throw in
|
|
1020
1597
|
// execution/agent resolution) OR after it. Either way the job is terminal-
|
|
1021
1598
|
// failed now — capture a finished_at to drive the invocation row regardless.
|
|
1022
1599
|
const finishedAt = new Date().toISOString();
|
|
1023
|
-
if (job && job.status
|
|
1600
|
+
if (job && !TERMINAL_STATUSES.has(job.status)) {
|
|
1024
1601
|
job.status = 'failed';
|
|
1602
|
+
job.startedAt ??= finishedAt;
|
|
1025
1603
|
job.finishedAt = finishedAt;
|
|
1026
1604
|
}
|
|
1605
|
+
// Side effects are valid only after the terminal row and queued-admission
|
|
1606
|
+
// removal commit together. In particular, a failed promotion must leave
|
|
1607
|
+
// queued_jobs intact for restart instead of manufacturing accounting and a
|
|
1608
|
+
// completion callback for a terminal state SQLite never accepted.
|
|
1609
|
+
let terminalPersisted = !this._db && !!job;
|
|
1027
1610
|
if (this._db) {
|
|
1028
1611
|
try {
|
|
1029
|
-
|
|
1612
|
+
if (!job)
|
|
1613
|
+
throw new Error(`Missing in-memory job ${jobId}`);
|
|
1614
|
+
const durationMs = job.startedAt
|
|
1615
|
+
? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
|
|
1616
|
+
: undefined;
|
|
1617
|
+
this._stageTerminalIntent(job, {
|
|
1618
|
+
status: 'failed',
|
|
1619
|
+
invocationStatus: 'failed',
|
|
1620
|
+
provider: this._jobResolvedProvider.get(jobId)
|
|
1621
|
+
?? this._jobProviderSelection.get(jobId)
|
|
1622
|
+
?? this._adapter.id,
|
|
1623
|
+
finishedAt,
|
|
1624
|
+
exitCode: -1,
|
|
1625
|
+
result: { duration_ms: durationMs },
|
|
1626
|
+
});
|
|
1627
|
+
terminalPersisted = true;
|
|
1030
1628
|
}
|
|
1031
|
-
catch {
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
// be absent if the throw preceded it) with _adapter.id as the fallback.
|
|
1042
|
-
if (this._projectId && job) {
|
|
1629
|
+
catch (err) {
|
|
1630
|
+
// The transaction rollback preserves queued_jobs (or the pre-existing
|
|
1631
|
+
// running row) for durable recovery. Do not delete it in a later
|
|
1632
|
+
// best-effort cleanup and do not publish false terminal side effects.
|
|
1633
|
+
console.error(`[queue-manager] startup failure persistence failed for ${jobId}:`, err);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
if (!terminalPersisted) {
|
|
1637
|
+
let durableQueued = false;
|
|
1638
|
+
if (this._db) {
|
|
1043
1639
|
try {
|
|
1044
|
-
|
|
1045
|
-
const durationMs = job.startedAt
|
|
1046
|
-
? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
|
|
1047
|
-
: undefined;
|
|
1048
|
-
(0, ai_invocations_1.recordInvocation)(this._db, {
|
|
1049
|
-
id: (0, crypto_1.randomUUID)(),
|
|
1050
|
-
project_id: this._projectId,
|
|
1051
|
-
provider: this._jobResolvedProvider.get(jobId) ?? this._adapter.id,
|
|
1052
|
-
surface: 'job',
|
|
1053
|
-
surface_ref_id: jobId,
|
|
1054
|
-
ticket_id: ticketIds[0] ?? null,
|
|
1055
|
-
status: 'failed',
|
|
1056
|
-
started_at: job.startedAt ?? finishedAt,
|
|
1057
|
-
finished_at: finishedAt,
|
|
1058
|
-
total_cost_usd_estimated: false,
|
|
1059
|
-
duration_ms: durationMs,
|
|
1060
|
-
});
|
|
1061
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
1640
|
+
durableQueued = !!this._db.prepare('SELECT 1 FROM queued_jobs WHERE id = ?').get(jobId);
|
|
1062
1641
|
}
|
|
1063
|
-
catch
|
|
1064
|
-
|
|
1642
|
+
catch {
|
|
1643
|
+
// If even the read fails, fail closed below by pausing the queue.
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
if (job && previousJobState) {
|
|
1647
|
+
Object.assign(job, previousJobState);
|
|
1648
|
+
if (durableQueued) {
|
|
1649
|
+
// The provider never acquired the admission. Restore the same
|
|
1650
|
+
// pre-execution semantics in memory and put it back at the front, but
|
|
1651
|
+
// pause instead of immediately retrying a persistently-broken DB.
|
|
1652
|
+
job.status = 'queued';
|
|
1653
|
+
job.startedAt = null;
|
|
1654
|
+
job.finishedAt = null;
|
|
1655
|
+
job.exitCode = null;
|
|
1656
|
+
job.skipReason = null;
|
|
1657
|
+
if (!this._queue.includes(jobId))
|
|
1658
|
+
this._queue.unshift(jobId);
|
|
1659
|
+
this._recomputePositions();
|
|
1660
|
+
this._persistQueuedState();
|
|
1661
|
+
}
|
|
1662
|
+
else {
|
|
1663
|
+
// Promotion already consumed queued_jobs, so there is no admission to
|
|
1664
|
+
// retry. Retain ownership of the childless RUNNING row and make
|
|
1665
|
+
// resume fail closed until shutdown/startup recovery stages it.
|
|
1666
|
+
this._activeJobId = jobId;
|
|
1667
|
+
this._terminalPersistenceBlockedJobs.add(jobId);
|
|
1065
1668
|
}
|
|
1066
1669
|
}
|
|
1670
|
+
// Make the degraded state visible and stable. A user can explicitly
|
|
1671
|
+
// resume after repairing storage; enqueue cannot silently spin the same
|
|
1672
|
+
// failed promotion in a tight loop.
|
|
1673
|
+
this._paused = true;
|
|
1674
|
+
this._persistQueueState();
|
|
1675
|
+
const resolvedProvider = this._jobResolvedProvider.get(jobId);
|
|
1676
|
+
if (resolvedProvider)
|
|
1677
|
+
this._jobProviderSelection.set(jobId, resolvedProvider);
|
|
1678
|
+
this._jobResolvedProvider.delete(jobId);
|
|
1679
|
+
this._jobExecution.delete(jobId);
|
|
1680
|
+
this._snapshotRefs.delete(jobId);
|
|
1681
|
+
this._jobPrDelivery.delete(jobId);
|
|
1682
|
+
this._jobLiveAccounting.delete(jobId);
|
|
1683
|
+
this._cleanupOpenspecShim(jobId);
|
|
1684
|
+
this._broadcastQueueState();
|
|
1685
|
+
console.error(`[QueueManager] job ${jobId} remains durably ${durableQueued ? 'queued' : 'unsettled'} after startup failure: ${reason}`);
|
|
1686
|
+
return false;
|
|
1067
1687
|
}
|
|
1068
1688
|
// Clear per-job selection/snapshot maps (none were consumed by a spawn).
|
|
1069
1689
|
this._jobExecution.delete(jobId);
|
|
@@ -1078,15 +1698,29 @@ class QueueManager {
|
|
|
1078
1698
|
// A wedged job may have had its openspec shim materialised already (the
|
|
1079
1699
|
// wedge can land after _startJob's shim setup). Mirror the settle path.
|
|
1080
1700
|
this._cleanupOpenspecShim(jobId);
|
|
1081
|
-
|
|
1082
|
-
|
|
1701
|
+
let accountingReady = true;
|
|
1702
|
+
if (this._db) {
|
|
1703
|
+
accountingReady = this._resumeOrphanRecoveries();
|
|
1704
|
+
if (!accountingReady) {
|
|
1705
|
+
this._paused = true;
|
|
1706
|
+
this._persistQueueState();
|
|
1707
|
+
}
|
|
1083
1708
|
}
|
|
1084
|
-
|
|
1085
|
-
|
|
1709
|
+
else if (terminalPersisted) {
|
|
1710
|
+
try {
|
|
1711
|
+
this._onJobFinished?.(jobId, 'failed', undefined);
|
|
1712
|
+
}
|
|
1713
|
+
catch {
|
|
1714
|
+
/* onJobFinished is best-effort */
|
|
1715
|
+
}
|
|
1716
|
+
this._skipDependents(jobId, `Parent job ${jobId} failed`);
|
|
1717
|
+
if (job?.pipelineId)
|
|
1718
|
+
this._checkPipelineStatus(job.pipelineId);
|
|
1086
1719
|
}
|
|
1087
1720
|
this._persistQueueState();
|
|
1088
1721
|
this._broadcastQueueState();
|
|
1089
1722
|
console.error(`[QueueManager] job ${jobId} failed before spawn: ${reason}`);
|
|
1723
|
+
return accountingReady;
|
|
1090
1724
|
}
|
|
1091
1725
|
/**
|
|
1092
1726
|
* Remove the per-job openspec PATH shim (relocated claude rails only) from
|
|
@@ -1138,12 +1772,12 @@ class QueueManager {
|
|
|
1138
1772
|
}
|
|
1139
1773
|
/**
|
|
1140
1774
|
* Resolve the adapter for a job at spawn time: the per-job provider override
|
|
1141
|
-
*
|
|
1142
|
-
*
|
|
1775
|
+
* when present and registered, else the project's primary adapter. Pending
|
|
1776
|
+
* selections remain intact until createJob durably promotes the admission;
|
|
1777
|
+
* a pre-promotion failure can therefore be retried without semantic drift.
|
|
1143
1778
|
*/
|
|
1144
1779
|
_resolveJobAdapter(jobId) {
|
|
1145
1780
|
const override = this._jobProviderSelection.get(jobId);
|
|
1146
|
-
this._jobProviderSelection.delete(jobId);
|
|
1147
1781
|
if (override) {
|
|
1148
1782
|
try {
|
|
1149
1783
|
return (0, providers_1.getAdapter)(override);
|
|
@@ -1154,6 +1788,15 @@ class QueueManager {
|
|
|
1154
1788
|
}
|
|
1155
1789
|
return this._adapter;
|
|
1156
1790
|
}
|
|
1791
|
+
/** Consume restart-durable pre-spawn selections only after queued_jobs has
|
|
1792
|
+
* been atomically promoted to jobs. Runtime state already holds every value
|
|
1793
|
+
* needed by the child at that point. */
|
|
1794
|
+
_consumePendingSelections(jobId) {
|
|
1795
|
+
this._jobProviderSelection.delete(jobId);
|
|
1796
|
+
this._jobModelSelection.delete(jobId);
|
|
1797
|
+
this._jobProfileSelection.delete(jobId);
|
|
1798
|
+
this._jobInteractiveSelection.delete(jobId);
|
|
1799
|
+
}
|
|
1157
1800
|
/**
|
|
1158
1801
|
* Resolve the relocate-artifacts execution context for this manager's project.
|
|
1159
1802
|
* The gate: relocated only when a registry entry exists AND core has populated
|
|
@@ -1207,21 +1850,23 @@ class QueueManager {
|
|
|
1207
1850
|
*/
|
|
1208
1851
|
_startInteractiveJob(jobId, job, adapter, spec, firstPrompt, settleMode) {
|
|
1209
1852
|
if (this._db) {
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1853
|
+
// Promotion is the execution boundary. Propagate any failure to
|
|
1854
|
+
// `_drainQueue` before constructing the session: starting a child while
|
|
1855
|
+
// its queued admission remains replayable would duplicate the command on
|
|
1856
|
+
// the next process restart.
|
|
1857
|
+
(0, db_1.createJob)(this._db, {
|
|
1858
|
+
id: jobId,
|
|
1859
|
+
command: job.command,
|
|
1860
|
+
started_at: job.startedAt,
|
|
1861
|
+
provider: adapter.id,
|
|
1862
|
+
priority: job.priority,
|
|
1863
|
+
depends_on_job_id: job.dependsOnJobId,
|
|
1864
|
+
pipeline_id: job.pipelineId,
|
|
1865
|
+
interactive: true,
|
|
1866
|
+
causal_ownership: job.causalOwnership === true,
|
|
1867
|
+
});
|
|
1224
1868
|
}
|
|
1869
|
+
this._consumePendingSelections(jobId);
|
|
1225
1870
|
const session = new interactive_job_session_1.InteractiveJobSession({
|
|
1226
1871
|
jobId,
|
|
1227
1872
|
projectId: this._projectId ?? '',
|
|
@@ -1249,36 +1894,40 @@ class QueueManager {
|
|
|
1249
1894
|
* drains the queue.
|
|
1250
1895
|
*/
|
|
1251
1896
|
_settleInteractiveJob(jobId, info) {
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
this._activeJobId = null;
|
|
1256
|
-
}
|
|
1257
|
-
// Consume the pre-spawn provenance snapshot + the resolved execution
|
|
1258
|
-
// context BEFORE any early return (mirrors _onJobExit) so a disposed/
|
|
1259
|
-
// unknown-job settle can never leak the map entries.
|
|
1897
|
+
// Read terminal context first, but do not consume it until the durable
|
|
1898
|
+
// terminal intent commits. If storage rejects the transition, shutdown can
|
|
1899
|
+
// retry from the still-owned session instead of losing in-flight usage.
|
|
1260
1900
|
const snapshot = this._snapshotRefs.get(jobId);
|
|
1261
|
-
this._snapshotRefs.delete(jobId);
|
|
1262
1901
|
const jobExecution = this._jobExecution.get(jobId);
|
|
1263
|
-
this._jobExecution.delete(jobId);
|
|
1264
1902
|
const provenanceRepoDir = jobExecution?.repoDir ?? this._cwd;
|
|
1265
|
-
this._jobResolvedProvider.
|
|
1903
|
+
const provider = this._jobResolvedProvider.get(jobId) ?? 'claude';
|
|
1266
1904
|
// Consume the spawn-captured PR-delivery mode (before any early return so a
|
|
1267
1905
|
// disposed/unknown-job settle can never leak the entry). Decides whether a
|
|
1268
1906
|
// COMPLETED job's tickets park at on_review (ask-first) or done (legacy).
|
|
1269
1907
|
const prDelivery = this._jobPrDelivery.get(jobId) ?? false;
|
|
1270
|
-
this.
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1908
|
+
if (this._disposed) {
|
|
1909
|
+
this._interactiveSessions.delete(jobId);
|
|
1910
|
+
this._snapshotRefs.delete(jobId);
|
|
1911
|
+
this._jobExecution.delete(jobId);
|
|
1912
|
+
this._jobResolvedProvider.delete(jobId);
|
|
1913
|
+
this._jobPrDelivery.delete(jobId);
|
|
1914
|
+
this._cleanupOpenspecShim(jobId);
|
|
1274
1915
|
return;
|
|
1916
|
+
}
|
|
1275
1917
|
const job = this._jobs.get(jobId);
|
|
1276
1918
|
if (!job) {
|
|
1919
|
+
this._interactiveSessions.delete(jobId);
|
|
1920
|
+
this._snapshotRefs.delete(jobId);
|
|
1921
|
+
this._jobExecution.delete(jobId);
|
|
1922
|
+
this._jobResolvedProvider.delete(jobId);
|
|
1923
|
+
this._jobPrDelivery.delete(jobId);
|
|
1924
|
+
this._cleanupOpenspecShim(jobId);
|
|
1925
|
+
if (this._activeJobId === jobId)
|
|
1926
|
+
this._activeJobId = null;
|
|
1277
1927
|
this._drainQueue();
|
|
1278
1928
|
return;
|
|
1279
1929
|
}
|
|
1280
1930
|
const wasCanceling = this._cancelingJobs.has(jobId);
|
|
1281
|
-
this._cancelingJobs.delete(jobId);
|
|
1282
1931
|
// Zero-work strictness: a session whose WHOLE life consumed no model work
|
|
1283
1932
|
// (the claude CLI's synthetic `Unknown command:` result frame — num_turns
|
|
1284
1933
|
// 0, no assistant events, zero usage tokens) settles FAILED even on a
|
|
@@ -1291,59 +1940,77 @@ class QueueManager {
|
|
|
1291
1940
|
: info.reason === 'finalized' && !info.zeroWork
|
|
1292
1941
|
? 'completed'
|
|
1293
1942
|
: 'failed';
|
|
1943
|
+
const finishedAt = new Date().toISOString();
|
|
1944
|
+
const exitCode = info.reason === 'finalized' && finalStatus !== 'failed' ? 0 : 1;
|
|
1945
|
+
const invStatus = finalStatus === 'completed'
|
|
1946
|
+
? 'success'
|
|
1947
|
+
: finalStatus === 'canceled'
|
|
1948
|
+
? 'aborted'
|
|
1949
|
+
: 'failed';
|
|
1950
|
+
const totals = info.totals;
|
|
1951
|
+
const result = {
|
|
1952
|
+
tokens_in: totals.tokens_in,
|
|
1953
|
+
tokens_out: totals.tokens_out,
|
|
1954
|
+
tokens_cache_read: totals.tokens_cache_read,
|
|
1955
|
+
tokens_cache_create: totals.tokens_cache_create,
|
|
1956
|
+
total_cost_usd: totals.total_cost_usd,
|
|
1957
|
+
total_cost_usd_estimated: info.estimated,
|
|
1958
|
+
num_turns: totals.num_turns,
|
|
1959
|
+
model: info.model ?? undefined,
|
|
1960
|
+
session_id: info.sessionId ?? undefined,
|
|
1961
|
+
duration_ms: info.activeDurationMs,
|
|
1962
|
+
};
|
|
1963
|
+
if (this._db) {
|
|
1964
|
+
try {
|
|
1965
|
+
this._stageTerminalIntent(job, {
|
|
1966
|
+
status: finalStatus,
|
|
1967
|
+
invocationStatus: invStatus,
|
|
1968
|
+
provider,
|
|
1969
|
+
finishedAt,
|
|
1970
|
+
exitCode,
|
|
1971
|
+
interactive: true,
|
|
1972
|
+
result,
|
|
1973
|
+
ticketCompletionStatus: finalStatus === 'completed'
|
|
1974
|
+
? (prDelivery ? 'on_review' : 'done')
|
|
1975
|
+
: undefined,
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
catch (err) {
|
|
1979
|
+
// The session has stopped, but its durable row intentionally remains
|
|
1980
|
+
// running. Reserve the slot and pause so neither resume nor enqueue can
|
|
1981
|
+
// overtake an unaccounted terminal transition; startup/shutdown recovery
|
|
1982
|
+
// can safely retry it from that state.
|
|
1983
|
+
this._activeProcess = null;
|
|
1984
|
+
this._activeJobId = jobId;
|
|
1985
|
+
this._terminalPersistenceBlockedJobs.add(jobId);
|
|
1986
|
+
this._paused = true;
|
|
1987
|
+
this._persistQueueState();
|
|
1988
|
+
this._broadcastQueueState();
|
|
1989
|
+
console.error(`[queue-manager] interactive terminal staging failed for ${jobId}:`, err);
|
|
1990
|
+
return;
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
this._interactiveSessions.delete(jobId);
|
|
1994
|
+
if (this._activeJobId === jobId) {
|
|
1995
|
+
this._activeProcess = null;
|
|
1996
|
+
this._activeJobId = null;
|
|
1997
|
+
}
|
|
1998
|
+
this._snapshotRefs.delete(jobId);
|
|
1999
|
+
this._jobExecution.delete(jobId);
|
|
2000
|
+
this._jobResolvedProvider.delete(jobId);
|
|
2001
|
+
this._jobPrDelivery.delete(jobId);
|
|
2002
|
+
this._cancelingJobs.delete(jobId);
|
|
2003
|
+
this._cleanupOpenspecShim(jobId);
|
|
1294
2004
|
job.status = finalStatus;
|
|
1295
|
-
job.finishedAt =
|
|
1296
|
-
job.exitCode =
|
|
2005
|
+
job.finishedAt = finishedAt;
|
|
2006
|
+
job.exitCode = exitCode;
|
|
1297
2007
|
// Result text for output chaining between dependent pipeline steps — the
|
|
1298
2008
|
// same field the one-shot path captures from its last `result` event.
|
|
1299
2009
|
if (info.resultText != null) {
|
|
1300
2010
|
job.resultText = info.resultText;
|
|
1301
2011
|
}
|
|
1302
|
-
|
|
2012
|
+
let accountingReady = true;
|
|
1303
2013
|
if (this._db) {
|
|
1304
|
-
try {
|
|
1305
|
-
(0, db_1.finalizeInteractiveJob)(this._db, jobId, finalStatus);
|
|
1306
|
-
}
|
|
1307
|
-
catch (err) {
|
|
1308
|
-
console.error('[queue-manager] finalizeInteractiveJob failed:', err);
|
|
1309
|
-
}
|
|
1310
|
-
if (this._projectId) {
|
|
1311
|
-
try {
|
|
1312
|
-
const invStatus = finalStatus === 'completed'
|
|
1313
|
-
? 'success'
|
|
1314
|
-
: finalStatus === 'canceled'
|
|
1315
|
-
? 'aborted'
|
|
1316
|
-
: 'failed';
|
|
1317
|
-
const ticketIds = this._extractTicketIds(job.command);
|
|
1318
|
-
this._recordJobInvocations({
|
|
1319
|
-
jobId,
|
|
1320
|
-
provider: 'claude',
|
|
1321
|
-
status: invStatus,
|
|
1322
|
-
startedAt: job.startedAt ?? new Date().toISOString(),
|
|
1323
|
-
finishedAt: job.finishedAt,
|
|
1324
|
-
ticketIds,
|
|
1325
|
-
// CRIT-4: true when a mid-turn finalize/crash folded an in-flight
|
|
1326
|
-
// turn's rate-card-priced usage into the accumulated totals.
|
|
1327
|
-
estimated: info.estimated,
|
|
1328
|
-
result: {
|
|
1329
|
-
tokens_in: totals.tokens_in,
|
|
1330
|
-
tokens_out: totals.tokens_out,
|
|
1331
|
-
tokens_cache_read: totals.tokens_cache_read,
|
|
1332
|
-
tokens_cache_create: totals.tokens_cache_create,
|
|
1333
|
-
total_cost_usd: totals.total_cost_usd,
|
|
1334
|
-
num_turns: totals.num_turns,
|
|
1335
|
-
model: info.model ?? undefined,
|
|
1336
|
-
session_id: info.sessionId ?? undefined,
|
|
1337
|
-
// LOW-15: sum of active turn wall-segments, not finished−started.
|
|
1338
|
-
duration_ms: info.activeDurationMs,
|
|
1339
|
-
},
|
|
1340
|
-
});
|
|
1341
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
1342
|
-
}
|
|
1343
|
-
catch (err) {
|
|
1344
|
-
console.error('[queue-manager] recordInvocation (interactive) failed:', err);
|
|
1345
|
-
}
|
|
1346
|
-
}
|
|
1347
2014
|
// Code-Explorer post-settle provenance hook — the interactive lifecycle
|
|
1348
2015
|
// equivalent of _onJobExit's post-exit diff (pre-spawn snapshot taken in
|
|
1349
2016
|
// _startJob's interactive branch, against the REPO dir, never the
|
|
@@ -1355,52 +2022,63 @@ class QueueManager {
|
|
|
1355
2022
|
if (totals.total_cost_usd > 0 && finalStatus === 'completed') {
|
|
1356
2023
|
this._emitCostAlerts(jobId, totals.total_cost_usd);
|
|
1357
2024
|
}
|
|
2025
|
+
accountingReady = this._resumeOrphanRecoveries();
|
|
1358
2026
|
this._enforceDailyBudget();
|
|
1359
2027
|
}
|
|
1360
|
-
this.
|
|
1361
|
-
this._broadcast({
|
|
1362
|
-
type: 'job.finalized',
|
|
1363
|
-
projectId: this._projectId ?? '',
|
|
1364
|
-
jobId,
|
|
1365
|
-
status: finalStatus,
|
|
1366
|
-
totals,
|
|
1367
|
-
timestamp: new Date().toISOString(),
|
|
1368
|
-
});
|
|
1369
|
-
this._broadcastQueueState();
|
|
1370
|
-
if (this._onJobFinished) {
|
|
2028
|
+
if (!this._db) {
|
|
1371
2029
|
try {
|
|
1372
2030
|
if (finalStatus === 'completed') {
|
|
1373
|
-
|
|
1374
|
-
// methodology a completed job's tickets park at on_review, never done.
|
|
1375
|
-
// Failure statuses keep the legacy 3-arg call shape (the field is
|
|
1376
|
-
// completion-only).
|
|
1377
|
-
this._onJobFinished(jobId, finalStatus, totals.total_cost_usd, {
|
|
2031
|
+
this._onJobFinished?.(jobId, finalStatus, totals.total_cost_usd, {
|
|
1378
2032
|
ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
|
|
1379
2033
|
});
|
|
1380
2034
|
}
|
|
1381
2035
|
else {
|
|
1382
|
-
this._onJobFinished(jobId, finalStatus, totals.total_cost_usd);
|
|
2036
|
+
this._onJobFinished?.(jobId, finalStatus, totals.total_cost_usd);
|
|
1383
2037
|
}
|
|
1384
2038
|
}
|
|
1385
2039
|
catch (err) {
|
|
1386
2040
|
console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
|
|
1387
2041
|
}
|
|
2042
|
+
if (finalStatus !== 'completed') {
|
|
2043
|
+
this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
|
|
2044
|
+
}
|
|
2045
|
+
if (job.pipelineId)
|
|
2046
|
+
this._checkPipelineStatus(job.pipelineId);
|
|
1388
2047
|
}
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
|
|
1393
|
-
}
|
|
1394
|
-
if (job.pipelineId) {
|
|
1395
|
-
this._checkPipelineStatus(job.pipelineId);
|
|
2048
|
+
if (!accountingReady) {
|
|
2049
|
+
this._paused = true;
|
|
2050
|
+
this._persistQueueState();
|
|
1396
2051
|
}
|
|
1397
|
-
this.
|
|
2052
|
+
this._persistJob(job);
|
|
2053
|
+
this._broadcast({
|
|
2054
|
+
type: 'job.finalized',
|
|
2055
|
+
projectId: this._projectId ?? '',
|
|
2056
|
+
jobId,
|
|
2057
|
+
status: finalStatus,
|
|
2058
|
+
totals,
|
|
2059
|
+
timestamp: new Date().toISOString(),
|
|
2060
|
+
});
|
|
2061
|
+
this._broadcastQueueState();
|
|
2062
|
+
if (accountingReady)
|
|
2063
|
+
this._drainQueue();
|
|
2064
|
+
}
|
|
2065
|
+
/** True only while this async start still belongs to the live manager and its
|
|
2066
|
+
* synchronously-reserved queue slot. The generation closes the shutdown race;
|
|
2067
|
+
* the slot check also protects against future replacement/cancel paths. */
|
|
2068
|
+
_canContinueStart(jobId, lifecycleGeneration) {
|
|
2069
|
+
return (!this._disposed &&
|
|
2070
|
+
this._lifecycleGeneration === lifecycleGeneration &&
|
|
2071
|
+
this._activeJobId === jobId);
|
|
1398
2072
|
}
|
|
1399
2073
|
async _startJob(jobId) {
|
|
2074
|
+
const lifecycleGeneration = this._lifecycleGeneration;
|
|
2075
|
+
if (!this._canContinueStart(jobId, lifecycleGeneration))
|
|
2076
|
+
return;
|
|
1400
2077
|
const job = this._jobs.get(jobId);
|
|
1401
|
-
if (!job) {
|
|
2078
|
+
if (!job || job.status !== 'queued') {
|
|
1402
2079
|
// Job vanished between the synchronous slot reservation in _drainQueue and
|
|
1403
|
-
// here
|
|
2080
|
+
// here (or became non-queued through a defensive race) — release the
|
|
2081
|
+
// reserved slot and move on. Never relaunch a terminal/running id.
|
|
1404
2082
|
if (this._activeJobId === jobId)
|
|
1405
2083
|
this._activeJobId = null;
|
|
1406
2084
|
this._drainQueue();
|
|
@@ -1411,9 +2089,13 @@ class QueueManager {
|
|
|
1411
2089
|
// plugins, result parsing, ai_invocations.provider) flows from `adapter`.
|
|
1412
2090
|
const adapter = this._resolveJobAdapter(jobId);
|
|
1413
2091
|
// Remember the provider this job actually runs on, for terminal paths that
|
|
1414
|
-
// fire without a child 'close' (e.g. _forceFailUnkillableJob)
|
|
1415
|
-
//
|
|
2092
|
+
// fire without a child 'close' (e.g. _forceFailUnkillableJob). The pending
|
|
2093
|
+
// map remains durable until promotion, then this resolved map takes over.
|
|
1416
2094
|
this._jobResolvedProvider.set(jobId, adapter.id);
|
|
2095
|
+
// Pin the adapter that was actually resolved. If any later pre-promotion
|
|
2096
|
+
// step fails, the durable retry must not switch providers because project
|
|
2097
|
+
// defaults or an invalid override fallback changed in the meantime.
|
|
2098
|
+
this._jobProviderSelection.set(jobId, adapter.id);
|
|
1417
2099
|
// Relocate-artifacts gate: resolve cwd/repoDir/env for this spawn. Legacy
|
|
1418
2100
|
// projects get cwd = project.path + empty env (byte-identical to today);
|
|
1419
2101
|
// relocated projects get cwd = workspace + SPECRAILS_REPO_DIR. Captured per
|
|
@@ -1471,11 +2153,10 @@ class QueueManager {
|
|
|
1471
2153
|
// today); codex/gemini always take the legacy one-shot spawn below.
|
|
1472
2154
|
// EnqueueOptions.interactive is a per-job OVERRIDE: false forces legacy,
|
|
1473
2155
|
// true forces interactive where capable, undefined = default ON. Derived
|
|
1474
|
-
// HERE (spawn time), not at enqueue, so
|
|
1475
|
-
//
|
|
2156
|
+
// HERE (spawn time), not at enqueue, so legacy queued rows with no explicit
|
|
2157
|
+
// selection still receive the current default after restart.
|
|
1476
2158
|
const isFreestyle = adapter.id === 'claude' && exports.FREESTYLE_COMMAND_RE.test(commandToRun);
|
|
1477
2159
|
const interactiveOverride = this._jobInteractiveSelection.get(jobId);
|
|
1478
|
-
this._jobInteractiveSelection.delete(jobId);
|
|
1479
2160
|
const spawnInteractive = (0, feature_flags_1.isInteractiveJobsEnabled)() &&
|
|
1480
2161
|
adapter.capabilities.persistentStdin &&
|
|
1481
2162
|
(interactiveOverride ?? true);
|
|
@@ -1591,7 +2272,6 @@ class QueueManager {
|
|
|
1591
2272
|
// Per-job model override (consumed once) takes precedence — used by the
|
|
1592
2273
|
// freestyle model picker so the user can choose haiku/sonnet/opus per launch.
|
|
1593
2274
|
const modelOverride = this._jobModelSelection.get(jobId);
|
|
1594
|
-
this._jobModelSelection.delete(jobId);
|
|
1595
2275
|
const railModel = modelOverride
|
|
1596
2276
|
? modelOverride
|
|
1597
2277
|
: adapter.id === 'claude' && this._db
|
|
@@ -1622,7 +2302,6 @@ class QueueManager {
|
|
|
1622
2302
|
if (adapter.capabilities.profileEnvSupport && this._projectId && this._projectSlug && this._cwd) {
|
|
1623
2303
|
try {
|
|
1624
2304
|
const selection = this._jobProfileSelection.get(jobId); // undefined|null|string
|
|
1625
|
-
this._jobProfileSelection.delete(jobId);
|
|
1626
2305
|
// When relocated, core + `.specrails/profiles` live in the workspace
|
|
1627
2306
|
// (execution.cwd); legacy reads from the repo (execution.cwd === repo).
|
|
1628
2307
|
const coreSupports = projectSupportsProfiles(execution.cwd);
|
|
@@ -1699,13 +2378,27 @@ class QueueManager {
|
|
|
1699
2378
|
let pluginSnapshotPath = null;
|
|
1700
2379
|
if (adapter.mcpRegistration === 'project-json' && this._projectId && this._projectSlug && this._cwd) {
|
|
1701
2380
|
try {
|
|
1702
|
-
|
|
2381
|
+
let resolver = this._resolvePluginsForSpawn;
|
|
2382
|
+
let snapshotter = null;
|
|
2383
|
+
if (!resolver) {
|
|
2384
|
+
const pluginIntegration = require('./plugins/rail-integration');
|
|
2385
|
+
resolver = pluginIntegration.resolvePluginsForSpawn;
|
|
2386
|
+
snapshotter = pluginIntegration.snapshotPluginsForJob;
|
|
2387
|
+
}
|
|
1703
2388
|
// Relocated ⇒ `.mcp.json`/plugin state live in the workspace (execution.cwd).
|
|
1704
|
-
const resolution = await
|
|
2389
|
+
const resolution = await resolver(execution.cwd, this._projectId, jobId);
|
|
2390
|
+
// shutdown() may have run while plugin verification was awaiting. Never
|
|
2391
|
+
// snapshot, broadcast, or spawn for a manager generation that no longer
|
|
2392
|
+
// owns this active slot.
|
|
2393
|
+
if (!this._canContinueStart(jobId, lifecycleGeneration))
|
|
2394
|
+
return;
|
|
1705
2395
|
pluginActive = resolution.active;
|
|
1706
2396
|
pluginDegraded = resolution.degraded;
|
|
1707
2397
|
if (pluginActive.length > 0 || pluginDegraded.length > 0) {
|
|
1708
|
-
|
|
2398
|
+
// The injected resolver normally returns an empty test fixture. Load
|
|
2399
|
+
// the snapshotter lazily only if its result actually needs one.
|
|
2400
|
+
snapshotter ??= require('./plugins/rail-integration').snapshotPluginsForJob;
|
|
2401
|
+
pluginSnapshotPath = snapshotter(this._projectSlug, jobId, this._projectId, pluginActive, pluginDegraded);
|
|
1709
2402
|
}
|
|
1710
2403
|
for (const d of pluginDegraded) {
|
|
1711
2404
|
this._broadcast({
|
|
@@ -1722,6 +2415,9 @@ class QueueManager {
|
|
|
1722
2415
|
console.warn(`[queue-manager] plugin resolution failed for job ${jobId}: ${err.message}`);
|
|
1723
2416
|
}
|
|
1724
2417
|
}
|
|
2418
|
+
// Covers resolver throws as well as future awaits added to the block above.
|
|
2419
|
+
if (!this._canContinueStart(jobId, lifecycleGeneration))
|
|
2420
|
+
return;
|
|
1725
2421
|
if (pluginActive.length > 0 && pluginSnapshotPath) {
|
|
1726
2422
|
spawnEnv = {
|
|
1727
2423
|
...spawnEnv,
|
|
@@ -1818,6 +2514,8 @@ class QueueManager {
|
|
|
1818
2514
|
// provenance is captured around the SESSION lifecycle exactly like the
|
|
1819
2515
|
// one-shot path: pre-spawn snapshot here, diff at settle.
|
|
1820
2516
|
if (spawnInteractive) {
|
|
2517
|
+
if (!this._canContinueStart(jobId, lifecycleGeneration))
|
|
2518
|
+
return;
|
|
1821
2519
|
if ((0, feature_flags_1.isCodeExplorerEnabled)()) {
|
|
1822
2520
|
try {
|
|
1823
2521
|
const snap = (0, file_provenance_1.snapshotWorkingTree)(execution.repoDir);
|
|
@@ -1860,6 +2558,25 @@ class QueueManager {
|
|
|
1860
2558
|
console.warn(`[queue-manager] provenance snapshot failed: ${err.message}`);
|
|
1861
2559
|
}
|
|
1862
2560
|
}
|
|
2561
|
+
// Durably promote queued → running BEFORE the provider process exists. If
|
|
2562
|
+
// spawn throws, the normal _failWedgedJob path marks this row failed; if the
|
|
2563
|
+
// app crashes after this point, startup recovers a running orphan instead of
|
|
2564
|
+
// replaying the queued admission alongside a possibly-live old child.
|
|
2565
|
+
if (!this._canContinueStart(jobId, lifecycleGeneration))
|
|
2566
|
+
return;
|
|
2567
|
+
if (this._db) {
|
|
2568
|
+
(0, db_1.createJob)(this._db, {
|
|
2569
|
+
id: jobId,
|
|
2570
|
+
command: job.command,
|
|
2571
|
+
started_at: job.startedAt,
|
|
2572
|
+
provider: adapter.id,
|
|
2573
|
+
priority: job.priority,
|
|
2574
|
+
depends_on_job_id: job.dependsOnJobId,
|
|
2575
|
+
pipeline_id: job.pipelineId,
|
|
2576
|
+
causal_ownership: job.causalOwnership === true,
|
|
2577
|
+
});
|
|
2578
|
+
}
|
|
2579
|
+
this._consumePendingSelections(jobId);
|
|
1863
2580
|
// spawnAiCli reroutes multi-line argv values through stdin on Windows.
|
|
1864
2581
|
const child = (0, cli_prompt_1.spawnAiCli)(binary, args, {
|
|
1865
2582
|
env: spawnEnv,
|
|
@@ -1887,11 +2604,51 @@ class QueueManager {
|
|
|
1887
2604
|
// Start zombie detection timer. Reset on any raw data from the process.
|
|
1888
2605
|
// Using 'data' events (not readline 'line') ensures the timer resets
|
|
1889
2606
|
// synchronously in test environments with fake timers.
|
|
1890
|
-
this._resetZombieTimer();
|
|
1891
|
-
child.stdout.on('data', () => { this._resetZombieTimer(); });
|
|
1892
|
-
child.stderr.on('data', () => { this._resetZombieTimer(); });
|
|
2607
|
+
this._resetZombieTimer(jobId);
|
|
2608
|
+
child.stdout.on('data', () => { this._resetZombieTimer(jobId); });
|
|
2609
|
+
child.stderr.on('data', () => { this._resetZombieTimer(jobId); });
|
|
1893
2610
|
let eventSeq = 0;
|
|
1894
2611
|
let lastResultEvent = null;
|
|
2612
|
+
let rawPersistenceFailed = false;
|
|
2613
|
+
const persistEvent = (event, critical = true) => {
|
|
2614
|
+
const seq = eventSeq++;
|
|
2615
|
+
const db = this._db;
|
|
2616
|
+
// A force-failed child may continue emitting after its durable job became
|
|
2617
|
+
// terminal (and its history may already be deleted). Keep parsing usage
|
|
2618
|
+
// in memory for late reconciliation, but never write child events through
|
|
2619
|
+
// a terminal/missing FK or let storage failure escape readline.
|
|
2620
|
+
if (!db)
|
|
2621
|
+
return { seq, persisted: true };
|
|
2622
|
+
if (this._jobs.get(jobId)?.status !== 'running' && !this._forceFailedRowJobs.has(jobId))
|
|
2623
|
+
return { seq, persisted: false };
|
|
2624
|
+
try {
|
|
2625
|
+
(0, db_1.appendEvent)(db, jobId, seq, event);
|
|
2626
|
+
}
|
|
2627
|
+
catch (err) {
|
|
2628
|
+
console.error(`[queue-manager] event persistence failed for ${jobId}:`, err);
|
|
2629
|
+
if (critical && !rawPersistenceFailed) {
|
|
2630
|
+
rawPersistenceFailed = true;
|
|
2631
|
+
this._paused = true;
|
|
2632
|
+
this._persistQueueState();
|
|
2633
|
+
this._broadcastQueueState();
|
|
2634
|
+
if (this._activeJobId === jobId) {
|
|
2635
|
+
this._persistenceFailedJobs.add(jobId);
|
|
2636
|
+
this._kill(jobId);
|
|
2637
|
+
}
|
|
2638
|
+
else {
|
|
2639
|
+
const surviving = this._forceFailedProcesses.get(jobId);
|
|
2640
|
+
if (surviving?.pid) {
|
|
2641
|
+
try {
|
|
2642
|
+
(0, win_spawn_1.treeKillSafe)(surviving.pid, 'SIGTERM', () => { });
|
|
2643
|
+
}
|
|
2644
|
+
catch { /* best-effort */ }
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
return { seq, persisted: false };
|
|
2649
|
+
}
|
|
2650
|
+
return { seq, persisted: true };
|
|
2651
|
+
};
|
|
1895
2652
|
// Accumulator of parsed AdapterEvent for finaliseInvocationResult on close.
|
|
1896
2653
|
const adapterEvents = [];
|
|
1897
2654
|
// Expose the (growing) accumulator + adapter/model so shutdown() can flush an
|
|
@@ -1909,26 +2666,22 @@ class QueueManager {
|
|
|
1909
2666
|
model: railModel,
|
|
1910
2667
|
});
|
|
1911
2668
|
}
|
|
1912
|
-
if (this._db) {
|
|
1913
|
-
(0, db_1.createJob)(this._db, {
|
|
1914
|
-
id: jobId,
|
|
1915
|
-
command: job.command,
|
|
1916
|
-
started_at: job.startedAt,
|
|
1917
|
-
priority: job.priority,
|
|
1918
|
-
depends_on_job_id: job.dependsOnJobId,
|
|
1919
|
-
pipeline_id: job.pipelineId,
|
|
1920
|
-
});
|
|
1921
|
-
}
|
|
1922
2669
|
// ── Batched broadcast for high-frequency messages (log + event) ──────
|
|
1923
2670
|
// Collects messages and flushes every ~80ms instead of one WS send per line.
|
|
1924
2671
|
const pendingBroadcast = [];
|
|
1925
2672
|
let flushTimer = null;
|
|
1926
2673
|
const FLUSH_INTERVAL_MS = 80;
|
|
1927
2674
|
const batchedBroadcast = (msg) => {
|
|
2675
|
+
if (this._disposed)
|
|
2676
|
+
return;
|
|
1928
2677
|
pendingBroadcast.push(msg);
|
|
1929
2678
|
if (!flushTimer) {
|
|
1930
2679
|
flushTimer = setTimeout(() => {
|
|
1931
2680
|
flushTimer = null;
|
|
2681
|
+
if (this._disposed) {
|
|
2682
|
+
pendingBroadcast.length = 0;
|
|
2683
|
+
return;
|
|
2684
|
+
}
|
|
1932
2685
|
const batch = pendingBroadcast.splice(0);
|
|
1933
2686
|
for (const m of batch)
|
|
1934
2687
|
this._broadcast(m);
|
|
@@ -1941,6 +2694,8 @@ class QueueManager {
|
|
|
1941
2694
|
flushTimer = null;
|
|
1942
2695
|
}
|
|
1943
2696
|
const batch = pendingBroadcast.splice(0);
|
|
2697
|
+
if (this._disposed)
|
|
2698
|
+
return;
|
|
1944
2699
|
for (const m of batch)
|
|
1945
2700
|
this._broadcast(m);
|
|
1946
2701
|
};
|
|
@@ -1960,7 +2715,10 @@ class QueueManager {
|
|
|
1960
2715
|
};
|
|
1961
2716
|
const stdoutReader = (0, readline_1.createInterface)({ input: child.stdout, crlfDelay: Infinity });
|
|
1962
2717
|
const stderrReader = (0, readline_1.createInterface)({ input: child.stderr, crlfDelay: Infinity });
|
|
2718
|
+
this._jobReaders.set(jobId, { stdout: stdoutReader, stderr: stderrReader });
|
|
1963
2719
|
stdoutReader.on('line', (line) => {
|
|
2720
|
+
if (this._disposed)
|
|
2721
|
+
return;
|
|
1964
2722
|
let parsed = null;
|
|
1965
2723
|
try {
|
|
1966
2724
|
parsed = JSON.parse(line);
|
|
@@ -1978,13 +2736,13 @@ class QueueManager {
|
|
|
1978
2736
|
}
|
|
1979
2737
|
if (parsed) {
|
|
1980
2738
|
const eventType = parsed.type ?? 'unknown';
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
2739
|
+
const persisted = persistEvent({
|
|
2740
|
+
event_type: eventType,
|
|
2741
|
+
source: 'stdout',
|
|
2742
|
+
payload: line,
|
|
2743
|
+
});
|
|
2744
|
+
if (!persisted.persisted)
|
|
2745
|
+
return;
|
|
1988
2746
|
batchedBroadcast({
|
|
1989
2747
|
type: 'event',
|
|
1990
2748
|
jobId,
|
|
@@ -1992,31 +2750,27 @@ class QueueManager {
|
|
|
1992
2750
|
source: 'stdout',
|
|
1993
2751
|
payload: line,
|
|
1994
2752
|
timestamp: new Date().toISOString(),
|
|
1995
|
-
seq:
|
|
2753
|
+
seq: persisted.seq,
|
|
1996
2754
|
});
|
|
1997
2755
|
if (eventType === 'result') {
|
|
1998
2756
|
lastResultEvent = parsed;
|
|
1999
2757
|
}
|
|
2000
2758
|
const displayText = (0, stream_display_1.extractDisplayText)(parsed);
|
|
2001
2759
|
if (displayText !== null) {
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
});
|
|
2008
|
-
}
|
|
2760
|
+
persistEvent({
|
|
2761
|
+
event_type: 'log',
|
|
2762
|
+
source: 'stdout',
|
|
2763
|
+
payload: JSON.stringify({ line: displayText }),
|
|
2764
|
+
}, false);
|
|
2009
2765
|
emitLine('stdout', displayText);
|
|
2010
2766
|
}
|
|
2011
2767
|
}
|
|
2012
2768
|
else {
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
});
|
|
2019
|
-
}
|
|
2769
|
+
persistEvent({
|
|
2770
|
+
event_type: 'log',
|
|
2771
|
+
source: 'stdout',
|
|
2772
|
+
payload: JSON.stringify({ line }),
|
|
2773
|
+
}, false);
|
|
2020
2774
|
// For adapters whose stream is JSONL (claude, codex), a non-parseable
|
|
2021
2775
|
// line is unexpected noise. For future plain-text adapters this is
|
|
2022
2776
|
// their normal output. emitLine surfaces it either way.
|
|
@@ -2029,13 +2783,13 @@ class QueueManager {
|
|
|
2029
2783
|
}
|
|
2030
2784
|
});
|
|
2031
2785
|
stderrReader.on('line', (line) => {
|
|
2032
|
-
if (this.
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
})
|
|
2038
|
-
}
|
|
2786
|
+
if (this._disposed)
|
|
2787
|
+
return;
|
|
2788
|
+
persistEvent({
|
|
2789
|
+
event_type: 'log',
|
|
2790
|
+
source: 'stderr',
|
|
2791
|
+
payload: JSON.stringify({ line }),
|
|
2792
|
+
}, false);
|
|
2039
2793
|
emitLine('stderr', line);
|
|
2040
2794
|
});
|
|
2041
2795
|
child.on('close', (code) => {
|
|
@@ -2055,38 +2809,41 @@ class QueueManager {
|
|
|
2055
2809
|
/** Per-job adapter resolved in _startJob; defaults to the project primary
|
|
2056
2810
|
* for any caller that does not thread it (none today). */
|
|
2057
2811
|
adapter = this._adapter) {
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2812
|
+
// Timers belong to the job currently holding the single execution slot.
|
|
2813
|
+
// A late close from a previously force-failed, unkillable child must not
|
|
2814
|
+
// disarm the successor's inactivity watchdog or SIGKILL escalation.
|
|
2815
|
+
if (this._activeJobId === jobId) {
|
|
2816
|
+
this._clearZombieTimer();
|
|
2817
|
+
if (this._killTimer !== null) {
|
|
2818
|
+
clearTimeout(this._killTimer);
|
|
2819
|
+
this._killTimer = null;
|
|
2820
|
+
}
|
|
2062
2821
|
}
|
|
2063
|
-
//
|
|
2064
|
-
//
|
|
2065
|
-
//
|
|
2822
|
+
// Read terminal context now, but consume it only after the terminal row and
|
|
2823
|
+
// recovery intent commit. A failed disk write must leave enough ownership
|
|
2824
|
+
// for graceful/startup recovery to retry the exact same run.
|
|
2066
2825
|
const snapshot = this._snapshotRefs.get(jobId);
|
|
2067
|
-
this._snapshotRefs.delete(jobId);
|
|
2068
|
-
// Relocate-artifacts: the repo dir this job snapshotted against (= repoDir,
|
|
2069
|
-
// never the workspace). Falls back to this._cwd for jobs spawned before this
|
|
2070
|
-
// map existed (e.g. restored-from-db) so provenance still targets the repo.
|
|
2071
2826
|
const jobExecution = this._jobExecution.get(jobId);
|
|
2072
|
-
this._jobExecution.delete(jobId);
|
|
2073
|
-
// Release the resolved-provider entry on the normal child-exit path (the
|
|
2074
|
-
// adapter is threaded into _onJobExit directly, so this is pure cleanup).
|
|
2075
|
-
this._jobResolvedProvider.delete(jobId);
|
|
2076
|
-
// Reclaim the live-accounting handle unconditionally (shutdown flush no longer
|
|
2077
|
-
// needs it once the child has exited).
|
|
2078
|
-
this._jobLiveAccounting.delete(jobId);
|
|
2079
|
-
// Consume the spawn-captured PR-delivery mode (before any early return so a
|
|
2080
|
-
// disposed/unknown-job exit can never leak the entry). Decides whether a
|
|
2081
|
-
// COMPLETED job's tickets park at on_review (ask-first) or done (legacy).
|
|
2082
2827
|
const prDelivery = this._jobPrDelivery.get(jobId) ?? false;
|
|
2083
|
-
this._jobPrDelivery.delete(jobId);
|
|
2084
2828
|
const provenanceRepoDir = jobExecution?.repoDir ?? this._cwd;
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2829
|
+
const consumeTerminalContext = () => {
|
|
2830
|
+
const readers = this._jobReaders.get(jobId);
|
|
2831
|
+
try {
|
|
2832
|
+
readers?.stdout.close();
|
|
2833
|
+
}
|
|
2834
|
+
catch { /* best-effort */ }
|
|
2835
|
+
try {
|
|
2836
|
+
readers?.stderr.close();
|
|
2837
|
+
}
|
|
2838
|
+
catch { /* best-effort */ }
|
|
2839
|
+
this._jobReaders.delete(jobId);
|
|
2840
|
+
this._snapshotRefs.delete(jobId);
|
|
2841
|
+
this._jobExecution.delete(jobId);
|
|
2842
|
+
this._jobResolvedProvider.delete(jobId);
|
|
2843
|
+
this._jobLiveAccounting.delete(jobId);
|
|
2844
|
+
this._jobPrDelivery.delete(jobId);
|
|
2845
|
+
this._cleanupOpenspecShim(jobId);
|
|
2846
|
+
};
|
|
2090
2847
|
// A3: release the active slot for THIS job before any early return, so a
|
|
2091
2848
|
// disposed/unknown-job exit can never leave the slot reserved (which would
|
|
2092
2849
|
// wedge the queue). Guarded by identity in case a stale exit fires late.
|
|
@@ -2097,11 +2854,16 @@ class QueueManager {
|
|
|
2097
2854
|
// The manager was torn down (e.g. project removed) while the child was
|
|
2098
2855
|
// still running. The DB may be closed; skip all bookkeeping to avoid an
|
|
2099
2856
|
// uncaught throw inside this EventEmitter 'close' listener.
|
|
2100
|
-
if (this._disposed)
|
|
2857
|
+
if (this._disposed) {
|
|
2858
|
+
consumeTerminalContext();
|
|
2101
2859
|
return;
|
|
2860
|
+
}
|
|
2102
2861
|
const job = this._jobs.get(jobId);
|
|
2103
|
-
if (!job)
|
|
2862
|
+
if (!job) {
|
|
2863
|
+
consumeTerminalContext();
|
|
2864
|
+
this._drainQueue();
|
|
2104
2865
|
return;
|
|
2866
|
+
}
|
|
2105
2867
|
// LOW-6: this is the LATE close of a job already force-failed by
|
|
2106
2868
|
// _forceFailUnkillableJob (SIGKILL escalation failed, then the child died on
|
|
2107
2869
|
// its own). The terminal side-effects (status, onJobFinished, dependents,
|
|
@@ -2109,15 +2871,18 @@ class QueueManager {
|
|
|
2109
2871
|
// that replaces the no-cost placeholder row(s) IF this close captured real
|
|
2110
2872
|
// spend — never a duplicate row nor a second onJobFinished.
|
|
2111
2873
|
if (this._forceFailedRowJobs.has(jobId)) {
|
|
2874
|
+
consumeTerminalContext();
|
|
2112
2875
|
this._reconcileForceFailedJobExit(jobId, code, adapterEvents, adapter, spawnedModel);
|
|
2113
2876
|
return;
|
|
2114
2877
|
}
|
|
2115
2878
|
const wasZombie = this._zombieJobs.has(jobId);
|
|
2116
2879
|
const wasCanceling = this._cancelingJobs.has(jobId);
|
|
2117
|
-
this.
|
|
2118
|
-
this._cancelingJobs.delete(jobId);
|
|
2880
|
+
const wasPersistenceFailed = this._persistenceFailedJobs.has(jobId);
|
|
2119
2881
|
let finalStatus;
|
|
2120
|
-
if (
|
|
2882
|
+
if (wasPersistenceFailed) {
|
|
2883
|
+
finalStatus = 'failed';
|
|
2884
|
+
}
|
|
2885
|
+
else if (wasZombie) {
|
|
2121
2886
|
finalStatus = 'zombie_terminated';
|
|
2122
2887
|
}
|
|
2123
2888
|
else if (wasCanceling) {
|
|
@@ -2129,72 +2894,69 @@ class QueueManager {
|
|
|
2129
2894
|
else {
|
|
2130
2895
|
finalStatus = 'failed';
|
|
2131
2896
|
}
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2897
|
+
// Adapter-driven finalisation must happen before staging so the immutable
|
|
2898
|
+
// intent owns the same usage that lands on jobs and ai_invocations.
|
|
2899
|
+
const { result: normalised, estimated } = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
|
|
2900
|
+
const tokenData = lastResultEvent || adapterEvents.length > 0
|
|
2901
|
+
? {
|
|
2902
|
+
tokens_in: normalised.tokens_in,
|
|
2903
|
+
tokens_out: normalised.tokens_out,
|
|
2904
|
+
tokens_cache_read: normalised.tokens_cache_read,
|
|
2905
|
+
tokens_cache_create: normalised.tokens_cache_create,
|
|
2906
|
+
total_cost_usd: normalised.total_cost_usd,
|
|
2907
|
+
total_cost_usd_estimated: estimated,
|
|
2908
|
+
num_turns: normalised.num_turns,
|
|
2909
|
+
model: normalised.model,
|
|
2910
|
+
duration_ms: normalised.duration_ms,
|
|
2911
|
+
duration_api_ms: normalised.duration_api_ms,
|
|
2912
|
+
session_id: normalised.session_id,
|
|
2913
|
+
}
|
|
2914
|
+
: {};
|
|
2915
|
+
const finishedAt = new Date().toISOString();
|
|
2916
|
+
const invStatus = finalStatus === 'completed'
|
|
2917
|
+
? 'success'
|
|
2918
|
+
: (finalStatus === 'canceled' || finalStatus === 'zombie_terminated')
|
|
2919
|
+
? 'aborted'
|
|
2920
|
+
: 'failed';
|
|
2141
2921
|
if (this._db) {
|
|
2142
|
-
// Adapter-driven result finalisation handles tokens, cost (or pricing-
|
|
2143
|
-
// table estimate for non-native-cost providers), and session_id stamping.
|
|
2144
|
-
const { result: normalised, estimated } = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
|
|
2145
|
-
const tokenData = lastResultEvent || adapterEvents.length > 0
|
|
2146
|
-
? {
|
|
2147
|
-
tokens_in: normalised.tokens_in,
|
|
2148
|
-
tokens_out: normalised.tokens_out,
|
|
2149
|
-
tokens_cache_read: normalised.tokens_cache_read,
|
|
2150
|
-
tokens_cache_create: normalised.tokens_cache_create,
|
|
2151
|
-
total_cost_usd: normalised.total_cost_usd,
|
|
2152
|
-
total_cost_usd_estimated: estimated,
|
|
2153
|
-
num_turns: normalised.num_turns,
|
|
2154
|
-
model: normalised.model,
|
|
2155
|
-
duration_ms: normalised.duration_ms,
|
|
2156
|
-
duration_api_ms: normalised.duration_api_ms,
|
|
2157
|
-
session_id: normalised.session_id,
|
|
2158
|
-
}
|
|
2159
|
-
: {};
|
|
2160
2922
|
try {
|
|
2161
|
-
|
|
2162
|
-
exit_code: code ?? -1,
|
|
2923
|
+
this._stageTerminalIntent(job, {
|
|
2163
2924
|
status: finalStatus,
|
|
2164
|
-
|
|
2925
|
+
invocationStatus: invStatus,
|
|
2926
|
+
provider: adapter.id,
|
|
2927
|
+
finishedAt,
|
|
2928
|
+
exitCode: code,
|
|
2929
|
+
result: tokenData,
|
|
2930
|
+
ticketCompletionStatus: finalStatus === 'completed'
|
|
2931
|
+
? (prDelivery ? 'on_review' : 'done')
|
|
2932
|
+
: undefined,
|
|
2165
2933
|
});
|
|
2166
2934
|
}
|
|
2167
2935
|
catch (err) {
|
|
2168
|
-
//
|
|
2169
|
-
//
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
: (finalStatus === 'canceled' || finalStatus === 'zombie_terminated')
|
|
2179
|
-
? 'aborted'
|
|
2180
|
-
: 'failed';
|
|
2181
|
-
const ticketIds = this._extractTicketIds(job.command);
|
|
2182
|
-
this._recordJobInvocations({
|
|
2183
|
-
jobId,
|
|
2184
|
-
provider: adapter.id,
|
|
2185
|
-
status: invStatus,
|
|
2186
|
-
startedAt: job.startedAt ?? new Date().toISOString(),
|
|
2187
|
-
finishedAt: job.finishedAt,
|
|
2188
|
-
ticketIds,
|
|
2189
|
-
estimated,
|
|
2190
|
-
result: normalised,
|
|
2191
|
-
});
|
|
2192
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
2193
|
-
}
|
|
2194
|
-
catch (err) {
|
|
2195
|
-
console.error('[queue-manager] recordInvocation failed:', err);
|
|
2196
|
-
}
|
|
2936
|
+
// Keep the durable row running and reserve the now-childless slot. This
|
|
2937
|
+
// is an explicit fail-stop: no later job may overtake unaccounted work.
|
|
2938
|
+
this._activeProcess = null;
|
|
2939
|
+
this._activeJobId = jobId;
|
|
2940
|
+
this._terminalPersistenceBlockedJobs.add(jobId);
|
|
2941
|
+
this._paused = true;
|
|
2942
|
+
this._persistQueueState();
|
|
2943
|
+
this._broadcastQueueState();
|
|
2944
|
+
console.error(`[queue-manager] terminal staging failed for ${jobId}:`, err);
|
|
2945
|
+
return;
|
|
2197
2946
|
}
|
|
2947
|
+
}
|
|
2948
|
+
consumeTerminalContext();
|
|
2949
|
+
this._zombieJobs.delete(jobId);
|
|
2950
|
+
this._cancelingJobs.delete(jobId);
|
|
2951
|
+
this._persistenceFailedJobs.delete(jobId);
|
|
2952
|
+
job.status = finalStatus;
|
|
2953
|
+
job.finishedAt = finishedAt;
|
|
2954
|
+
job.exitCode = code;
|
|
2955
|
+
if (lastResultEvent && typeof lastResultEvent.result === 'string') {
|
|
2956
|
+
job.resultText = lastResultEvent.result;
|
|
2957
|
+
}
|
|
2958
|
+
let accountingReady = true;
|
|
2959
|
+
if (this._db) {
|
|
2198
2960
|
// Code-Explorer post-exit provenance hook. Diffs the working tree against
|
|
2199
2961
|
// the pre-spawn snapshot and inserts one row per touched path. Gated by
|
|
2200
2962
|
// SPECRAILS_CODE_EXPLORER (re-checked at each completion so the flag can
|
|
@@ -2212,6 +2974,7 @@ class QueueManager {
|
|
|
2212
2974
|
if (jobCost != null && finalStatus === 'completed') {
|
|
2213
2975
|
this._emitCostAlerts(jobId, jobCost);
|
|
2214
2976
|
}
|
|
2977
|
+
accountingReady = this._resumeOrphanRecoveries();
|
|
2215
2978
|
// ─── Daily-budget enforcement (MED-5) ───────────────────────────────────
|
|
2216
2979
|
// Runs on EVERY terminal exit, not just completed jobs: a failed/aborted
|
|
2217
2980
|
// claude run still emits real cost (error_max_turns etc.), so a day of
|
|
@@ -2224,45 +2987,32 @@ class QueueManager {
|
|
|
2224
2987
|
}
|
|
2225
2988
|
else {
|
|
2226
2989
|
emitLine('stdout', `[process exited with code ${code ?? 'unknown'}]`);
|
|
2227
|
-
}
|
|
2228
|
-
// Notify webhook handler (if any) about job completion/failure/cancellation.
|
|
2229
|
-
// zombie_terminated is included so a timed-out rail job still releases its
|
|
2230
|
-
// tickets (revert/flag) and clears its in-memory railJobs entry instead of
|
|
2231
|
-
// wedging the rail card in 'running' until a server restart.
|
|
2232
|
-
if (this._onJobFinished &&
|
|
2233
|
-
(finalStatus === 'completed' || finalStatus === 'failed' || finalStatus === 'canceled' || finalStatus === 'zombie_terminated')) {
|
|
2234
|
-
let costUsd;
|
|
2235
2990
|
try {
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2991
|
+
if (finalStatus === 'completed') {
|
|
2992
|
+
this._onJobFinished?.(jobId, finalStatus, normalised.total_cost_usd, {
|
|
2993
|
+
ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
else {
|
|
2997
|
+
this._onJobFinished?.(jobId, finalStatus, normalised.total_cost_usd);
|
|
2998
|
+
}
|
|
2239
2999
|
}
|
|
2240
3000
|
catch (err) {
|
|
2241
|
-
console.error(
|
|
2242
|
-
}
|
|
2243
|
-
if (finalStatus === 'completed') {
|
|
2244
|
-
// Thread the spawn-captured PR-delivery mode: under the ask-first
|
|
2245
|
-
// methodology a completed job's tickets park at on_review, never done.
|
|
2246
|
-
// Failure statuses keep the legacy 3-arg call shape (the field is
|
|
2247
|
-
// completion-only).
|
|
2248
|
-
this._onJobFinished(jobId, finalStatus, costUsd ?? undefined, {
|
|
2249
|
-
ticketCompletionStatus: prDelivery ? 'on_review' : 'done',
|
|
2250
|
-
});
|
|
3001
|
+
console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
|
|
2251
3002
|
}
|
|
2252
|
-
|
|
2253
|
-
this.
|
|
3003
|
+
if (finalStatus !== 'completed') {
|
|
3004
|
+
this._skipDependents(jobId, `Parent job ${jobId} ${finalStatus}`);
|
|
2254
3005
|
}
|
|
3006
|
+
if (job.pipelineId)
|
|
3007
|
+
this._checkPipelineStatus(job.pipelineId);
|
|
2255
3008
|
}
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
this.
|
|
2259
|
-
}
|
|
2260
|
-
// Check pipeline status
|
|
2261
|
-
if (job.pipelineId) {
|
|
2262
|
-
this._checkPipelineStatus(job.pipelineId);
|
|
3009
|
+
if (!accountingReady) {
|
|
3010
|
+
this._paused = true;
|
|
3011
|
+
this._persistQueueState();
|
|
2263
3012
|
}
|
|
2264
3013
|
this._broadcastQueueState();
|
|
2265
|
-
|
|
3014
|
+
if (accountingReady)
|
|
3015
|
+
this._drainQueue();
|
|
2266
3016
|
}
|
|
2267
3017
|
/**
|
|
2268
3018
|
* Code-Explorer provenance recording, shared by the one-shot exit path
|
|
@@ -2319,18 +3069,19 @@ class QueueManager {
|
|
|
2319
3069
|
console.error('[queue-manager] cost-alert bookkeeping failed (db unavailable?):', err);
|
|
2320
3070
|
}
|
|
2321
3071
|
}
|
|
2322
|
-
_resetZombieTimer() {
|
|
3072
|
+
_resetZombieTimer(ownerJobId) {
|
|
2323
3073
|
if (this._zombieTimeoutMs <= 0)
|
|
2324
3074
|
return;
|
|
3075
|
+
const jobId = ownerJobId ?? this._activeJobId;
|
|
3076
|
+
if (!jobId || this._activeJobId !== jobId)
|
|
3077
|
+
return;
|
|
2325
3078
|
if (this._inactivityTimer !== null) {
|
|
2326
3079
|
clearTimeout(this._inactivityTimer);
|
|
2327
3080
|
}
|
|
2328
|
-
const jobId = this._activeJobId;
|
|
2329
|
-
if (!jobId)
|
|
2330
|
-
return;
|
|
2331
3081
|
this._inactivityTimer = setTimeout(() => {
|
|
2332
3082
|
this._inactivityTimer = null;
|
|
2333
|
-
this.
|
|
3083
|
+
if (this._activeJobId === jobId)
|
|
3084
|
+
this._onZombieDetected(jobId);
|
|
2334
3085
|
}, this._zombieTimeoutMs);
|
|
2335
3086
|
}
|
|
2336
3087
|
_clearZombieTimer() {
|
|
@@ -2421,52 +3172,60 @@ class QueueManager {
|
|
|
2421
3172
|
_forceFailUnkillableJob(jobId) {
|
|
2422
3173
|
const job = this._jobs.get(jobId);
|
|
2423
3174
|
const isRunning = !!job && job.status === 'running';
|
|
3175
|
+
const survivingProcess = this._activeJobId === jobId ? this._activeProcess : null;
|
|
3176
|
+
let accountingReady = true;
|
|
2424
3177
|
if (job && isRunning) {
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
3178
|
+
const finishedAt = new Date().toISOString();
|
|
3179
|
+
const live = this._jobLiveAccounting.get(jobId);
|
|
3180
|
+
const { result: normalised, estimated } = live
|
|
3181
|
+
? (0, result_event_1.finaliseInvocationResult)(live.adapter, live.events, { fallbackModel: live.model })
|
|
3182
|
+
: { result: {}, estimated: false };
|
|
3183
|
+
const durationMs = normalised.duration_ms ?? (job.startedAt
|
|
3184
|
+
? new Date(finishedAt).getTime() - new Date(job.startedAt).getTime()
|
|
3185
|
+
: undefined);
|
|
3186
|
+
const provider = live?.adapter.id ?? this._jobResolvedProvider.get(jobId) ?? this._adapter.id;
|
|
2428
3187
|
if (this._db) {
|
|
2429
3188
|
try {
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
try {
|
|
2443
|
-
const ticketIds = this._extractTicketIds(job.command);
|
|
2444
|
-
const durationMs = job.startedAt
|
|
2445
|
-
? new Date(job.finishedAt).getTime() - new Date(job.startedAt).getTime()
|
|
2446
|
-
: undefined;
|
|
2447
|
-
this._recordJobInvocations({
|
|
2448
|
-
jobId,
|
|
2449
|
-
// Stamp the provider the child ACTUALLY ran on (per-job override
|
|
2450
|
-
// already consumed from _jobProviderSelection); _adapter.id is the
|
|
2451
|
-
// final fallback for jobs with no resolved-provider entry.
|
|
2452
|
-
provider: this._jobResolvedProvider.get(jobId) ?? this._adapter.id,
|
|
2453
|
-
status: 'aborted',
|
|
2454
|
-
startedAt: job.startedAt ?? new Date().toISOString(),
|
|
2455
|
-
finishedAt: job.finishedAt,
|
|
2456
|
-
ticketIds,
|
|
2457
|
-
estimated: false,
|
|
2458
|
-
result: { duration_ms: durationMs },
|
|
3189
|
+
this._stageTerminalIntent(job, {
|
|
3190
|
+
status: 'failed',
|
|
3191
|
+
invocationStatus: 'aborted',
|
|
3192
|
+
provider,
|
|
3193
|
+
finishedAt,
|
|
3194
|
+
exitCode: -1,
|
|
3195
|
+
awaitingLateReconciliation: true,
|
|
3196
|
+
result: {
|
|
3197
|
+
...normalised,
|
|
3198
|
+
total_cost_usd_estimated: estimated,
|
|
3199
|
+
duration_ms: durationMs,
|
|
3200
|
+
},
|
|
2459
3201
|
});
|
|
2460
|
-
this._forceFailedRowJobs.add(jobId);
|
|
2461
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
2462
3202
|
}
|
|
2463
3203
|
catch (err) {
|
|
2464
|
-
|
|
3204
|
+
// The child still exists and continues to own the active slot. Leave
|
|
3205
|
+
// every map intact and fail-stop the queue so a later close or
|
|
3206
|
+
// shutdown can retry the terminal transaction with fuller usage.
|
|
3207
|
+
this._terminalPersistenceBlockedJobs.add(jobId);
|
|
3208
|
+
this._paused = true;
|
|
3209
|
+
this._persistQueueState();
|
|
3210
|
+
this._broadcastQueueState();
|
|
3211
|
+
console.error(`[queue-manager] unkillable terminal staging failed for ${jobId}:`, err);
|
|
3212
|
+
return;
|
|
2465
3213
|
}
|
|
2466
3214
|
}
|
|
3215
|
+
job.status = 'failed';
|
|
3216
|
+
job.finishedAt = finishedAt;
|
|
3217
|
+
job.exitCode = -1;
|
|
3218
|
+
// A surviving child's late close only reconciles richer usage. The
|
|
3219
|
+
// durable callback/dependency effects below must never be fired twice.
|
|
3220
|
+
this._forceFailedRowJobs.add(jobId);
|
|
3221
|
+
if (survivingProcess)
|
|
3222
|
+
this._forceFailedProcesses.set(jobId, survivingProcess);
|
|
3223
|
+
if (this._db)
|
|
3224
|
+
accountingReady = this._resumeOrphanRecoveries();
|
|
2467
3225
|
}
|
|
2468
|
-
// Clear
|
|
2469
|
-
//
|
|
3226
|
+
// Clear completed lifecycle maps, but retain the live adapter-event
|
|
3227
|
+
// accumulator until late close/shutdown so usage emitted by the surviving
|
|
3228
|
+
// child can still be reconciled.
|
|
2470
3229
|
this._snapshotRefs.delete(jobId);
|
|
2471
3230
|
this._jobExecution.delete(jobId);
|
|
2472
3231
|
this._jobModelSelection.delete(jobId);
|
|
@@ -2475,24 +3234,31 @@ class QueueManager {
|
|
|
2475
3234
|
this._jobResolvedProvider.delete(jobId);
|
|
2476
3235
|
this._jobInteractiveSelection.delete(jobId);
|
|
2477
3236
|
this._jobPrDelivery.delete(jobId);
|
|
2478
|
-
this._jobLiveAccounting.delete(jobId);
|
|
2479
3237
|
this._cleanupOpenspecShim(jobId);
|
|
2480
3238
|
this._activeProcess = null;
|
|
2481
3239
|
this._activeJobId = null;
|
|
2482
3240
|
this._cancelingJobs.delete(jobId);
|
|
2483
3241
|
this._zombieJobs.delete(jobId);
|
|
2484
|
-
//
|
|
2485
|
-
//
|
|
2486
|
-
if (isRunning) {
|
|
3242
|
+
// DB-backed terminal effects are delivered by the outbox. Keep the direct
|
|
3243
|
+
// path only for ephemeral managers that have no durable project database.
|
|
3244
|
+
if (isRunning && !this._db) {
|
|
2487
3245
|
try {
|
|
2488
3246
|
this._onJobFinished?.(jobId, 'failed', undefined);
|
|
2489
3247
|
}
|
|
2490
3248
|
catch (err) {
|
|
2491
3249
|
console.error(`[QueueManager] onJobFinished failed for ${jobId}: ${err.message}`);
|
|
2492
3250
|
}
|
|
3251
|
+
this._skipDependents(jobId, `Parent job ${jobId} failed`);
|
|
3252
|
+
if (job?.pipelineId)
|
|
3253
|
+
this._checkPipelineStatus(job.pipelineId);
|
|
3254
|
+
}
|
|
3255
|
+
if (!accountingReady) {
|
|
3256
|
+
this._paused = true;
|
|
3257
|
+
this._persistQueueState();
|
|
2493
3258
|
}
|
|
2494
3259
|
this._broadcastQueueState();
|
|
2495
|
-
|
|
3260
|
+
if (accountingReady)
|
|
3261
|
+
this._drainQueue();
|
|
2496
3262
|
}
|
|
2497
3263
|
/**
|
|
2498
3264
|
* Cost-only reconciliation for the LATE `close` of a job already force-failed
|
|
@@ -2503,62 +3269,113 @@ class QueueManager {
|
|
|
2503
3269
|
* real cost; otherwise the placeholder is left untouched.
|
|
2504
3270
|
*/
|
|
2505
3271
|
_reconcileForceFailedJobExit(jobId, code, adapterEvents, adapter, spawnedModel) {
|
|
2506
|
-
this._forceFailedRowJobs.delete(jobId);
|
|
2507
3272
|
const job = this._jobs.get(jobId);
|
|
2508
|
-
|
|
2509
|
-
//
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
if (!this._db || !
|
|
3273
|
+
this._forceFailedProcesses.delete(jobId);
|
|
3274
|
+
// Ephemeral managers have no second durable history projection. Retain both
|
|
3275
|
+
// the failed tombstone and the late-close guard; a duplicate close remains
|
|
3276
|
+
// a no-op instead of making the job disappear or re-firing callbacks.
|
|
3277
|
+
if (!this._db || !job)
|
|
2513
3278
|
return;
|
|
3279
|
+
this._pendingLateReconciliations.set(jobId, {
|
|
3280
|
+
code,
|
|
3281
|
+
adapterEvents: [...adapterEvents],
|
|
3282
|
+
adapter,
|
|
3283
|
+
spawnedModel,
|
|
3284
|
+
});
|
|
2514
3285
|
try {
|
|
2515
|
-
const
|
|
3286
|
+
const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, adapterEvents, { fallbackModel: spawnedModel });
|
|
3287
|
+
const normalised = sanitizeRecoveredResult(finalised.result);
|
|
3288
|
+
const estimated = finalised.estimated;
|
|
2516
3289
|
const hasRealSpend = (normalised.total_cost_usd ?? 0) > 0 ||
|
|
2517
3290
|
(normalised.tokens_in ?? 0) > 0 ||
|
|
2518
3291
|
(normalised.tokens_out ?? 0) > 0 ||
|
|
2519
3292
|
(normalised.tokens_cache_read ?? 0) > 0 ||
|
|
2520
3293
|
(normalised.tokens_cache_create ?? 0) > 0;
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
const
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
3294
|
+
const finalStatus = code === 0
|
|
3295
|
+
? 'success'
|
|
3296
|
+
: code === null
|
|
3297
|
+
? 'aborted'
|
|
3298
|
+
: 'failed';
|
|
3299
|
+
const reconcile = this._db.transaction(() => {
|
|
3300
|
+
const recovery = this._db.prepare(`
|
|
3301
|
+
SELECT payload, accounting_completed
|
|
3302
|
+
FROM orphan_job_recovery WHERE job_id = ?
|
|
3303
|
+
`).get(jobId);
|
|
3304
|
+
if (!recovery) {
|
|
3305
|
+
throw new Error(`Missing late-reconciliation intent for ${jobId}`);
|
|
3306
|
+
}
|
|
3307
|
+
const { payload } = this._decodeRecoveryPayload(jobId, recovery.payload);
|
|
3308
|
+
payload.awaitingLateReconciliation = false;
|
|
3309
|
+
if (hasRealSpend) {
|
|
3310
|
+
Object.assign(payload, {
|
|
3311
|
+
provider: adapter.id,
|
|
3312
|
+
model: normalised.model ?? null,
|
|
3313
|
+
tokensIn: normalised.tokens_in ?? null,
|
|
3314
|
+
tokensOut: normalised.tokens_out ?? null,
|
|
3315
|
+
tokensCacheRead: normalised.tokens_cache_read ?? null,
|
|
3316
|
+
tokensCacheCreate: normalised.tokens_cache_create ?? null,
|
|
3317
|
+
totalCostUsd: normalised.total_cost_usd ?? null,
|
|
3318
|
+
totalCostUsdEstimated: estimated ? 1 : 0,
|
|
3319
|
+
numTurns: normalised.num_turns ?? null,
|
|
3320
|
+
durationMs: normalised.duration_ms ?? null,
|
|
3321
|
+
durationApiMs: normalised.duration_api_ms ?? null,
|
|
3322
|
+
sessionId: normalised.session_id ?? null,
|
|
3323
|
+
invocationStatus: finalStatus,
|
|
3324
|
+
});
|
|
3325
|
+
}
|
|
3326
|
+
const validatedPayload = this._decodeRecoveryPayload(jobId, JSON.stringify(payload)).payload;
|
|
3327
|
+
// Convert the already-completed partial accounting checkpoint back into
|
|
3328
|
+
// a durable pending intent before removing its placeholder. If the real
|
|
3329
|
+
// ledger insert later fails, restart/resume owns the full late payload.
|
|
3330
|
+
const shouldReaccount = hasRealSpend && !!this._projectId;
|
|
3331
|
+
this._db.prepare(`
|
|
3332
|
+
UPDATE orphan_job_recovery
|
|
3333
|
+
SET payload = ?, accounting_completed = ?
|
|
3334
|
+
WHERE job_id = ?
|
|
3335
|
+
`).run(JSON.stringify(validatedPayload), shouldReaccount ? 0 : recovery.accounting_completed, jobId);
|
|
3336
|
+
if (shouldReaccount) {
|
|
3337
|
+
this._db.prepare(`DELETE FROM ai_invocations WHERE surface_ref_id = ? OR surface_ref_id LIKE ?`).run(jobId, `${jobId}#t%`);
|
|
3338
|
+
}
|
|
3339
|
+
if (hasRealSpend) {
|
|
3340
|
+
// Reflect the real cost on the jobs row too (it was force-failed with
|
|
3341
|
+
// a partial snapshot while the child was still alive).
|
|
3342
|
+
(0, db_1.finishJob)(this._db, jobId, {
|
|
3343
|
+
exit_code: code ?? -1,
|
|
3344
|
+
status: 'failed',
|
|
3345
|
+
tokens_in: normalised.tokens_in,
|
|
3346
|
+
tokens_out: normalised.tokens_out,
|
|
3347
|
+
tokens_cache_read: normalised.tokens_cache_read,
|
|
3348
|
+
tokens_cache_create: normalised.tokens_cache_create,
|
|
3349
|
+
total_cost_usd: normalised.total_cost_usd,
|
|
3350
|
+
total_cost_usd_estimated: estimated,
|
|
3351
|
+
num_turns: normalised.num_turns,
|
|
3352
|
+
model: normalised.model,
|
|
3353
|
+
duration_ms: normalised.duration_ms,
|
|
3354
|
+
duration_api_ms: normalised.duration_api_ms,
|
|
3355
|
+
session_id: normalised.session_id,
|
|
3356
|
+
});
|
|
3357
|
+
}
|
|
2537
3358
|
});
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
total_cost_usd_estimated: estimated,
|
|
2549
|
-
num_turns: normalised.num_turns,
|
|
2550
|
-
model: normalised.model,
|
|
2551
|
-
duration_ms: normalised.duration_ms,
|
|
2552
|
-
duration_api_ms: normalised.duration_api_ms,
|
|
2553
|
-
session_id: normalised.session_id,
|
|
2554
|
-
});
|
|
2555
|
-
}
|
|
2556
|
-
catch (err) {
|
|
2557
|
-
console.error('[queue-manager] force-fail reconcile finishJob failed:', err);
|
|
3359
|
+
reconcile();
|
|
3360
|
+
this._forceFailedRowJobs.delete(jobId);
|
|
3361
|
+
this._jobLiveAccounting.delete(jobId);
|
|
3362
|
+
this._pendingLateReconciliations.delete(jobId);
|
|
3363
|
+
this._terminalPersistenceBlockedJobs.delete(jobId);
|
|
3364
|
+
const accountingReady = this._resumeOrphanRecoveries();
|
|
3365
|
+
if (!accountingReady) {
|
|
3366
|
+
this._paused = true;
|
|
3367
|
+
this._persistQueueState();
|
|
3368
|
+
this._broadcastQueueState();
|
|
2558
3369
|
}
|
|
2559
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
2560
3370
|
}
|
|
2561
3371
|
catch (err) {
|
|
3372
|
+
// Keep the guard + tombstone so this runtime never treats a duplicate
|
|
3373
|
+
// event as a new terminal job. No successor is admitted after an unknown
|
|
3374
|
+
// late spend until the operator retries/restarts with healthy storage.
|
|
3375
|
+
this._terminalPersistenceBlockedJobs.add(jobId);
|
|
3376
|
+
this._paused = true;
|
|
3377
|
+
this._persistQueueState();
|
|
3378
|
+
this._broadcastQueueState();
|
|
2562
3379
|
console.error('[queue-manager] force-fail reconcile failed:', err);
|
|
2563
3380
|
}
|
|
2564
3381
|
}
|
|
@@ -2571,19 +3388,84 @@ class QueueManager {
|
|
|
2571
3388
|
timestamp: new Date().toISOString(),
|
|
2572
3389
|
});
|
|
2573
3390
|
}
|
|
2574
|
-
|
|
3391
|
+
/** Materialise the exact restart semantics for one unstarted admission.
|
|
3392
|
+
* Map presence matters for profile and interactive: null/false are explicit
|
|
3393
|
+
* choices and must never collapse into the absent/default state. */
|
|
3394
|
+
_queuedJobRecord(job) {
|
|
3395
|
+
const hasProfileSelection = this._jobProfileSelection.has(job.id);
|
|
3396
|
+
const hasInteractiveSelection = this._jobInteractiveSelection.has(job.id);
|
|
3397
|
+
return {
|
|
3398
|
+
id: job.id,
|
|
3399
|
+
command: job.command,
|
|
3400
|
+
queue_position: job.queuePosition,
|
|
3401
|
+
priority: job.priority,
|
|
3402
|
+
depends_on_job_id: job.dependsOnJobId,
|
|
3403
|
+
pipeline_id: job.pipelineId,
|
|
3404
|
+
provider: this._jobProviderSelection.get(job.id) ?? null,
|
|
3405
|
+
model: this._jobModelSelection.get(job.id) ?? null,
|
|
3406
|
+
profile_name: hasProfileSelection
|
|
3407
|
+
? (this._jobProfileSelection.get(job.id) ?? null)
|
|
3408
|
+
: null,
|
|
3409
|
+
profile_selection_set: hasProfileSelection,
|
|
3410
|
+
interactive: hasInteractiveSelection
|
|
3411
|
+
? this._jobInteractiveSelection.get(job.id)
|
|
3412
|
+
: null,
|
|
3413
|
+
causal_ownership: job.causalOwnership === true,
|
|
3414
|
+
};
|
|
3415
|
+
}
|
|
3416
|
+
_persistJob(job, strict = false) {
|
|
2575
3417
|
if (!this._db)
|
|
2576
3418
|
return;
|
|
2577
|
-
// For queued jobs, we use the DB to store queue position and priority for startup restore.
|
|
2578
|
-
// We only upsert queue_position + priority + dependency fields — the rest is handled by createJob/finishJob.
|
|
2579
|
-
// Since this method is called for all status transitions, we use a flexible upsert
|
|
2580
|
-
// that only touches queue_position, priority, and dependency fields (for queued jobs) — other fields are
|
|
2581
|
-
// managed by the existing createJob/finishJob API.
|
|
2582
3419
|
try {
|
|
2583
|
-
|
|
3420
|
+
if (job.status === 'queued') {
|
|
3421
|
+
(0, db_1.upsertQueuedJob)(this._db, this._queuedJobRecord(job));
|
|
3422
|
+
}
|
|
3423
|
+
else if (TERMINAL_STATUSES.has(job.status)) {
|
|
3424
|
+
(0, db_1.deleteQueuedJob)(this._db, job.id);
|
|
3425
|
+
}
|
|
2584
3426
|
}
|
|
2585
|
-
catch {
|
|
2586
|
-
|
|
3427
|
+
catch (err) {
|
|
3428
|
+
if (strict)
|
|
3429
|
+
throw err;
|
|
3430
|
+
// Persistence remains best-effort for callers racing project teardown.
|
|
3431
|
+
}
|
|
3432
|
+
}
|
|
3433
|
+
/** Persist the complete in-memory queue order in one transaction. Explicit
|
|
3434
|
+
* removals cover terminal-before-start paths while leaving the active
|
|
3435
|
+
* pre-spawn admission intact until createJob atomically promotes it. */
|
|
3436
|
+
_persistQueuedState(strict = false, removedJobIds = [], reservedJobId) {
|
|
3437
|
+
const db = this._db;
|
|
3438
|
+
if (!db)
|
|
3439
|
+
return;
|
|
3440
|
+
try {
|
|
3441
|
+
const persist = db.transaction(() => {
|
|
3442
|
+
for (const id of removedJobIds)
|
|
3443
|
+
(0, db_1.deleteQueuedJob)(db, id);
|
|
3444
|
+
if (reservedJobId) {
|
|
3445
|
+
const reserved = this._jobs.get(reservedJobId);
|
|
3446
|
+
if (reserved) {
|
|
3447
|
+
(0, db_1.upsertQueuedJob)(db, {
|
|
3448
|
+
...this._queuedJobRecord(reserved),
|
|
3449
|
+
// A selected-but-not-yet-spawned admission must restore ahead of
|
|
3450
|
+
// the remaining queue; using 0 also avoids duplicate position 1.
|
|
3451
|
+
queue_position: 0,
|
|
3452
|
+
});
|
|
3453
|
+
}
|
|
3454
|
+
}
|
|
3455
|
+
for (const id of this._queue) {
|
|
3456
|
+
const job = this._jobs.get(id);
|
|
3457
|
+
if (!job || job.status !== 'queued')
|
|
3458
|
+
continue;
|
|
3459
|
+
(0, db_1.upsertQueuedJob)(db, this._queuedJobRecord(job));
|
|
3460
|
+
}
|
|
3461
|
+
});
|
|
3462
|
+
persist();
|
|
3463
|
+
}
|
|
3464
|
+
catch (err) {
|
|
3465
|
+
if (strict)
|
|
3466
|
+
throw err;
|
|
3467
|
+
// Best-effort for internal teardown/drain paths; admissions and public
|
|
3468
|
+
// queue mutations call this in strict mode and roll back in memory.
|
|
2587
3469
|
}
|
|
2588
3470
|
}
|
|
2589
3471
|
_persistQueueState() {
|
|
@@ -2599,57 +3481,26 @@ class QueueManager {
|
|
|
2599
3481
|
_restoreFromDb() {
|
|
2600
3482
|
if (!this._db)
|
|
2601
3483
|
return;
|
|
3484
|
+
let recoveryAccountingReady = false;
|
|
3485
|
+
const restoredIds = [];
|
|
2602
3486
|
try {
|
|
2603
|
-
|
|
2604
|
-
//
|
|
2605
|
-
//
|
|
2606
|
-
//
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
jobId: o.id,
|
|
2621
|
-
provider: this._adapter.id,
|
|
2622
|
-
status: 'aborted',
|
|
2623
|
-
startedAt: o.started_at ?? finishedAt,
|
|
2624
|
-
finishedAt,
|
|
2625
|
-
ticketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(o.command),
|
|
2626
|
-
estimated: !!o.total_cost_usd_estimated,
|
|
2627
|
-
result: {
|
|
2628
|
-
tokens_in: o.tokens_in ?? undefined,
|
|
2629
|
-
tokens_out: o.tokens_out ?? undefined,
|
|
2630
|
-
tokens_cache_read: o.tokens_cache_read ?? undefined,
|
|
2631
|
-
tokens_cache_create: o.tokens_cache_create ?? undefined,
|
|
2632
|
-
total_cost_usd: o.total_cost_usd ?? undefined,
|
|
2633
|
-
num_turns: o.num_turns ?? undefined,
|
|
2634
|
-
model: o.model ?? undefined,
|
|
2635
|
-
session_id: o.session_id ?? undefined,
|
|
2636
|
-
duration_ms: o.duration_ms ?? undefined,
|
|
2637
|
-
duration_api_ms: o.duration_api_ms ?? undefined,
|
|
2638
|
-
},
|
|
2639
|
-
});
|
|
2640
|
-
}
|
|
2641
|
-
if (orphans.length > 0) {
|
|
2642
|
-
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
2643
|
-
}
|
|
2644
|
-
}
|
|
2645
|
-
catch (err) {
|
|
2646
|
-
console.error('[queue-manager] restore backfill failed:', err);
|
|
2647
|
-
}
|
|
2648
|
-
}
|
|
2649
|
-
// Fail any jobs that were running when the server last shut down
|
|
2650
|
-
this._db.prepare(`UPDATE jobs SET status = 'failed', finished_at = CURRENT_TIMESTAMP WHERE status = 'running'`).run();
|
|
2651
|
-
// Restore queued jobs in order (priority DESC then queue_position ASC)
|
|
2652
|
-
const rows = this._db.prepare(`SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id FROM jobs WHERE status = 'queued' ORDER BY queue_position ASC`).all();
|
|
3487
|
+
this._captureOrphanRecoveries();
|
|
3488
|
+
// Restore queued jobs by their durable positions, then re-assert the
|
|
3489
|
+
// priority invariant. `reorder` only permits movement within a priority
|
|
3490
|
+
// band, so this sort is also a defensive repair for older/corrupt rows.
|
|
3491
|
+
const rows = this._db.prepare(`SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id,
|
|
3492
|
+
provider, model, profile_name, profile_selection_set, interactive,
|
|
3493
|
+
causal_ownership
|
|
3494
|
+
FROM queued_jobs
|
|
3495
|
+
UNION ALL
|
|
3496
|
+
SELECT id, command, queue_position, priority, depends_on_job_id, pipeline_id,
|
|
3497
|
+
NULL AS provider, NULL AS model, NULL AS profile_name,
|
|
3498
|
+
0 AS profile_selection_set, NULL AS interactive,
|
|
3499
|
+
causal_ownership
|
|
3500
|
+
FROM jobs
|
|
3501
|
+
WHERE status = 'queued'
|
|
3502
|
+
AND NOT EXISTS (SELECT 1 FROM queued_jobs WHERE queued_jobs.id = jobs.id)
|
|
3503
|
+
ORDER BY queue_position ASC, id ASC`).all();
|
|
2653
3504
|
for (const row of rows) {
|
|
2654
3505
|
const priority = (types_1.VALID_PRIORITIES.has(row.priority ?? '') ? row.priority : 'normal');
|
|
2655
3506
|
const job = {
|
|
@@ -2665,42 +3516,97 @@ class QueueManager {
|
|
|
2665
3516
|
pipelineId: row.pipeline_id ?? null,
|
|
2666
3517
|
skipReason: null,
|
|
2667
3518
|
resultText: null,
|
|
3519
|
+
causalOwnership: row.causal_ownership === 1,
|
|
2668
3520
|
};
|
|
2669
3521
|
this._jobs.set(row.id, job);
|
|
2670
3522
|
this._queue.push(row.id);
|
|
3523
|
+
restoredIds.push(row.id);
|
|
3524
|
+
if (row.provider) {
|
|
3525
|
+
this._jobProviderSelection.set(row.id, row.provider);
|
|
3526
|
+
}
|
|
3527
|
+
if (row.model !== null) {
|
|
3528
|
+
this._jobModelSelection.set(row.id, row.model);
|
|
3529
|
+
}
|
|
3530
|
+
if (row.profile_selection_set === 1) {
|
|
3531
|
+
this._jobProfileSelection.set(row.id, row.profile_name);
|
|
3532
|
+
}
|
|
3533
|
+
if (row.interactive === 0 || row.interactive === 1) {
|
|
3534
|
+
this._jobInteractiveSelection.set(row.id, row.interactive === 1);
|
|
3535
|
+
}
|
|
2671
3536
|
}
|
|
2672
|
-
//
|
|
3537
|
+
// Higher priority always executes first; Array#sort is stable, preserving
|
|
3538
|
+
// queue_position order within each band.
|
|
2673
3539
|
this._queue.sort((a, b) => {
|
|
2674
3540
|
const jobA = this._jobs.get(a);
|
|
2675
3541
|
const jobB = this._jobs.get(b);
|
|
2676
3542
|
return types_1.PRIORITY_WEIGHT[jobB.priority] - types_1.PRIORITY_WEIGHT[jobA.priority];
|
|
2677
3543
|
});
|
|
3544
|
+
// Queue-terminal recovery must run AFTER queued rows are materialised in
|
|
3545
|
+
// memory: the normal `_skipDependents` invariant walks `_jobs`/`_queue`.
|
|
3546
|
+
// Its own durable checkpoint keeps this replay-safe if startup crashes
|
|
3547
|
+
// before, during, or immediately after the recursive skip.
|
|
3548
|
+
recoveryAccountingReady = this._resumeOrphanRecoveries();
|
|
2678
3549
|
this._recomputePositions();
|
|
2679
3550
|
// Restore pause state
|
|
2680
3551
|
const pauseRow = this._db.prepare(`SELECT value FROM queue_state WHERE key = 'paused'`).get();
|
|
2681
|
-
this._paused = pauseRow?.value === 'true';
|
|
3552
|
+
this._paused = pauseRow?.value === 'true' || !recoveryAccountingReady;
|
|
3553
|
+
if (!recoveryAccountingReady)
|
|
3554
|
+
this._persistQueueState();
|
|
3555
|
+
this._restoreBlocked = false;
|
|
2682
3556
|
}
|
|
2683
|
-
catch {
|
|
2684
|
-
//
|
|
3557
|
+
catch (err) {
|
|
3558
|
+
// Discard any partial in-memory projection from this attempt. Admissions
|
|
3559
|
+
// are rejected while blocked, so every id here belongs exclusively to the
|
|
3560
|
+
// failed restore and can be rebuilt deterministically on resume.
|
|
3561
|
+
const restored = new Set(restoredIds);
|
|
3562
|
+
this._queue = this._queue.filter((id) => !restored.has(id));
|
|
3563
|
+
for (const id of restored) {
|
|
3564
|
+
this._jobs.delete(id);
|
|
3565
|
+
this._jobProfileSelection.delete(id);
|
|
3566
|
+
this._jobProviderSelection.delete(id);
|
|
3567
|
+
this._jobResolvedProvider.delete(id);
|
|
3568
|
+
this._jobModelSelection.delete(id);
|
|
3569
|
+
this._jobInteractiveSelection.delete(id);
|
|
3570
|
+
}
|
|
3571
|
+
this._restoreBlocked = true;
|
|
3572
|
+
// Startup cannot safely admit provider work when orphan capture/replay or
|
|
3573
|
+
// queue restoration itself failed. Persist the fail-stop when possible;
|
|
3574
|
+
// the user can retry after repairing the database.
|
|
3575
|
+
this._paused = true;
|
|
3576
|
+
this._persistQueueState();
|
|
3577
|
+
console.error('[queue-manager] durable queue recovery failed:', err);
|
|
2685
3578
|
}
|
|
2686
3579
|
// Kick off any restored queued jobs that are ready to run
|
|
2687
|
-
this.
|
|
3580
|
+
if (!this._restoreBlocked)
|
|
3581
|
+
this._drainQueue();
|
|
2688
3582
|
}
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
3583
|
+
/** Resolve a parent across live state, execution history and the durable
|
|
3584
|
+
* pre-start queue. A missing string id intentionally remains `null`: the
|
|
3585
|
+
* established QueueManager contract treats external/nonexistent parents as
|
|
3586
|
+
* already satisfied. */
|
|
3587
|
+
_getDependencyStatus(jobId) {
|
|
3588
|
+
const parent = this._jobs.get(jobId);
|
|
2693
3589
|
if (parent)
|
|
2694
|
-
return parent.status
|
|
3590
|
+
return parent.status;
|
|
2695
3591
|
if (this._db) {
|
|
2696
|
-
const
|
|
2697
|
-
if (
|
|
2698
|
-
return
|
|
3592
|
+
const history = this._db.prepare('SELECT status FROM jobs WHERE id = ?').get(jobId);
|
|
3593
|
+
if (history)
|
|
3594
|
+
return history.status;
|
|
3595
|
+
const queued = this._db.prepare('SELECT 1 FROM queued_jobs WHERE id = ?').get(jobId);
|
|
3596
|
+
if (queued)
|
|
3597
|
+
return 'queued';
|
|
2699
3598
|
}
|
|
2700
|
-
return
|
|
3599
|
+
return null;
|
|
3600
|
+
}
|
|
3601
|
+
_isDependencyMet(job) {
|
|
3602
|
+
if (!job.dependsOnJobId)
|
|
3603
|
+
return true;
|
|
3604
|
+
const parentStatus = this._getDependencyStatus(job.dependsOnJobId);
|
|
3605
|
+
return parentStatus === null || parentStatus === 'completed';
|
|
2701
3606
|
}
|
|
2702
3607
|
_skipDependents(parentJobId, reason) {
|
|
2703
3608
|
const toSkip = [];
|
|
3609
|
+
const affectedPipelines = new Set();
|
|
2704
3610
|
for (const [id, job] of this._jobs) {
|
|
2705
3611
|
if (job.dependsOnJobId === parentJobId && job.status === 'queued') {
|
|
2706
3612
|
toSkip.push(id);
|
|
@@ -2716,18 +3622,782 @@ class QueueManager {
|
|
|
2716
3622
|
job.status = 'skipped';
|
|
2717
3623
|
job.finishedAt = new Date().toISOString();
|
|
2718
3624
|
job.skipReason = reason;
|
|
3625
|
+
if (job.pipelineId)
|
|
3626
|
+
affectedPipelines.add(job.pipelineId);
|
|
2719
3627
|
if (this._db) {
|
|
2720
3628
|
// Ensure the job row exists before updating (queued jobs may not have been persisted via createJob yet)
|
|
2721
3629
|
const exists = this._db.prepare('SELECT 1 FROM jobs WHERE id = ?').get(id);
|
|
2722
3630
|
if (!exists) {
|
|
2723
|
-
this._db.prepare(`INSERT INTO jobs (id, command, started_at, status, skip_reason, finished_at, depends_on_job_id, pipeline_id) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?)`).run(id, job.command, job.finishedAt, reason, job.finishedAt, job.dependsOnJobId, job.pipelineId);
|
|
3631
|
+
this._db.prepare(`INSERT INTO jobs (id, command, started_at, status, skip_reason, finished_at, depends_on_job_id, pipeline_id, causal_ownership) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?)`).run(id, job.command, job.finishedAt, reason, job.finishedAt, job.dependsOnJobId, job.pipelineId, job.causalOwnership === true ? 1 : 0);
|
|
2724
3632
|
}
|
|
2725
3633
|
else {
|
|
2726
3634
|
(0, db_1.skipJob)(this._db, id, reason);
|
|
2727
3635
|
}
|
|
3636
|
+
(0, db_1.deleteQueuedJob)(this._db, id);
|
|
3637
|
+
}
|
|
3638
|
+
try {
|
|
3639
|
+
this._onJobFinished?.(id, 'skipped', undefined);
|
|
3640
|
+
}
|
|
3641
|
+
catch (err) {
|
|
3642
|
+
console.error(`[QueueManager] onJobFinished(skipped) failed for ${id}: ${err.message}`);
|
|
2728
3643
|
}
|
|
2729
3644
|
this._skipDependents(id, `Parent job ${id} was skipped`);
|
|
2730
3645
|
}
|
|
3646
|
+
this._recomputePositions();
|
|
3647
|
+
this._persistQueuedState();
|
|
3648
|
+
for (const pipelineId of affectedPipelines)
|
|
3649
|
+
this._checkPipelineStatus(pipelineId);
|
|
3650
|
+
}
|
|
3651
|
+
_snapshotRecoveryDescendants(parentJobId) {
|
|
3652
|
+
const db = this._db;
|
|
3653
|
+
if (!db)
|
|
3654
|
+
return [];
|
|
3655
|
+
const rows = db.prepare(`
|
|
3656
|
+
SELECT id, command, depends_on_job_id, pipeline_id, priority, causal_ownership
|
|
3657
|
+
FROM queued_jobs
|
|
3658
|
+
UNION ALL
|
|
3659
|
+
SELECT id, command, depends_on_job_id, pipeline_id, priority, causal_ownership
|
|
3660
|
+
FROM jobs
|
|
3661
|
+
WHERE status = 'queued'
|
|
3662
|
+
AND NOT EXISTS (SELECT 1 FROM queued_jobs WHERE queued_jobs.id = jobs.id)
|
|
3663
|
+
`).all();
|
|
3664
|
+
const children = new Map();
|
|
3665
|
+
for (const row of rows) {
|
|
3666
|
+
if (!row.depends_on_job_id)
|
|
3667
|
+
continue;
|
|
3668
|
+
const list = children.get(row.depends_on_job_id) ?? [];
|
|
3669
|
+
list.push(row);
|
|
3670
|
+
children.set(row.depends_on_job_id, list);
|
|
3671
|
+
}
|
|
3672
|
+
const result = [];
|
|
3673
|
+
const seen = new Set();
|
|
3674
|
+
const visit = (parentId) => {
|
|
3675
|
+
for (const row of children.get(parentId) ?? []) {
|
|
3676
|
+
if (seen.has(row.id))
|
|
3677
|
+
continue;
|
|
3678
|
+
seen.add(row.id);
|
|
3679
|
+
result.push({
|
|
3680
|
+
id: row.id,
|
|
3681
|
+
command: row.command,
|
|
3682
|
+
parentId,
|
|
3683
|
+
pipelineId: row.pipeline_id,
|
|
3684
|
+
priority: types_1.VALID_PRIORITIES.has(row.priority ?? '')
|
|
3685
|
+
? row.priority
|
|
3686
|
+
: 'normal',
|
|
3687
|
+
causalOwnership: row.causal_ownership === 1,
|
|
3688
|
+
});
|
|
3689
|
+
visit(row.id);
|
|
3690
|
+
}
|
|
3691
|
+
};
|
|
3692
|
+
visit(parentJobId);
|
|
3693
|
+
return result;
|
|
3694
|
+
}
|
|
3695
|
+
/** Apply the immutable descendant snapshot even if history retention already
|
|
3696
|
+
* deleted the parent and nulled every live FK. Rows that became terminal by a
|
|
3697
|
+
* newer user action are left untouched. */
|
|
3698
|
+
_skipRecoveredDescendants(parentJobId, descendants, parentStatus = 'failed') {
|
|
3699
|
+
const db = this._db;
|
|
3700
|
+
if (!db)
|
|
3701
|
+
return;
|
|
3702
|
+
for (const descendant of descendants) {
|
|
3703
|
+
const admission = db.prepare(`SELECT 1 FROM queued_jobs WHERE id = ?`).get(descendant.id);
|
|
3704
|
+
const history = db.prepare(`SELECT status FROM jobs WHERE id = ?`).get(descendant.id);
|
|
3705
|
+
const inMemory = this._jobs.get(descendant.id);
|
|
3706
|
+
const stillQueued = !!admission || history?.status === 'queued' || inMemory?.status === 'queued';
|
|
3707
|
+
// Queued cancellation terminalizes descendants in the same transaction
|
|
3708
|
+
// as the root intent. Their callbacks are nevertheless owned by this
|
|
3709
|
+
// checkpoint, so an already-skipped snapshot still needs delivery. A
|
|
3710
|
+
// different terminal status means a newer action won and is left alone.
|
|
3711
|
+
const alreadySkipped = history?.status === 'skipped' || inMemory?.status === 'skipped';
|
|
3712
|
+
if (!stillQueued && !alreadySkipped)
|
|
3713
|
+
continue;
|
|
3714
|
+
const finishedAt = new Date().toISOString();
|
|
3715
|
+
const reason = `Parent job ${descendant.parentId === parentJobId ? parentJobId : descendant.parentId} ${descendant.parentId === parentJobId ? parentStatus : 'was skipped'}`;
|
|
3716
|
+
if (stillQueued) {
|
|
3717
|
+
const index = this._queue.indexOf(descendant.id);
|
|
3718
|
+
if (index !== -1)
|
|
3719
|
+
this._queue.splice(index, 1);
|
|
3720
|
+
if (inMemory) {
|
|
3721
|
+
inMemory.status = 'skipped';
|
|
3722
|
+
inMemory.finishedAt = finishedAt;
|
|
3723
|
+
inMemory.skipReason = reason;
|
|
3724
|
+
inMemory.queuePosition = null;
|
|
3725
|
+
}
|
|
3726
|
+
if (history) {
|
|
3727
|
+
(0, db_1.skipJob)(db, descendant.id, reason);
|
|
3728
|
+
}
|
|
3729
|
+
else {
|
|
3730
|
+
const parentStillExists = db.prepare(`SELECT 1 FROM jobs WHERE id = ?`).get(descendant.parentId);
|
|
3731
|
+
db.prepare(`
|
|
3732
|
+
INSERT INTO jobs (
|
|
3733
|
+
id, command, started_at, status, priority, skip_reason, finished_at,
|
|
3734
|
+
depends_on_job_id, pipeline_id, causal_ownership
|
|
3735
|
+
) VALUES (?, ?, ?, 'skipped', ?, ?, ?, ?, ?, ?)
|
|
3736
|
+
`).run(descendant.id, descendant.command, finishedAt, descendant.priority, reason, finishedAt, parentStillExists ? descendant.parentId : null, descendant.pipelineId, descendant.causalOwnership === true ? 1 : 0);
|
|
3737
|
+
}
|
|
3738
|
+
(0, db_1.deleteQueuedJob)(db, descendant.id);
|
|
3739
|
+
}
|
|
3740
|
+
this._onJobFinished?.(descendant.id, 'skipped', undefined, {
|
|
3741
|
+
recoveryReplay: true,
|
|
3742
|
+
recoveryCommand: descendant.command,
|
|
3743
|
+
recoveryTicketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(descendant.command),
|
|
3744
|
+
recoveryDurationMs: null,
|
|
3745
|
+
recoveryCausalOwnership: descendant.causalOwnership === true,
|
|
3746
|
+
});
|
|
3747
|
+
}
|
|
3748
|
+
this._recomputePositions();
|
|
3749
|
+
this._persistQueuedState(true);
|
|
3750
|
+
}
|
|
3751
|
+
/** Rebuild the best durable usage frontier from raw provider frames. For
|
|
3752
|
+
* interactive Claude sessions each result is a per-turn token snapshot but a
|
|
3753
|
+
* cumulative cost/turn snapshot, so segments must be folded exactly like the
|
|
3754
|
+
* live session; assistant-only tail frames are one estimated in-flight turn. */
|
|
3755
|
+
_recoverPersistedJobUsage(jobId, providerId, fallbackModel, interactive) {
|
|
3756
|
+
const db = this._db;
|
|
3757
|
+
if (!db)
|
|
3758
|
+
return { result: {}, estimated: false, authoritative: false };
|
|
3759
|
+
let adapter;
|
|
3760
|
+
try {
|
|
3761
|
+
adapter = (0, providers_1.getAdapter)(providerId);
|
|
3762
|
+
}
|
|
3763
|
+
catch {
|
|
3764
|
+
adapter = this._adapter;
|
|
3765
|
+
}
|
|
3766
|
+
const parse = (row) => {
|
|
3767
|
+
try {
|
|
3768
|
+
return adapter.parseStreamLine(row.payload);
|
|
3769
|
+
}
|
|
3770
|
+
catch (err) {
|
|
3771
|
+
console.warn(`[queue-manager] ignored malformed durable event for ${jobId}:`, err);
|
|
3772
|
+
return null;
|
|
3773
|
+
}
|
|
3774
|
+
};
|
|
3775
|
+
const makeUsageAccumulator = () => {
|
|
3776
|
+
const totals = {
|
|
3777
|
+
tokens_in: 0,
|
|
3778
|
+
tokens_out: 0,
|
|
3779
|
+
tokens_cache_read: 0,
|
|
3780
|
+
tokens_cache_create: 0,
|
|
3781
|
+
total_cost_usd: 0,
|
|
3782
|
+
num_turns: 0,
|
|
3783
|
+
duration_ms: 0,
|
|
3784
|
+
duration_api_ms: 0,
|
|
3785
|
+
};
|
|
3786
|
+
let model = fallbackModel ?? undefined;
|
|
3787
|
+
let sessionId;
|
|
3788
|
+
let estimated = false;
|
|
3789
|
+
const snapshots = new Map();
|
|
3790
|
+
const MAX_SNAPSHOT_KEYS = 4_096;
|
|
3791
|
+
const add = (event, seq) => {
|
|
3792
|
+
const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], { fallbackModel: model });
|
|
3793
|
+
const result = sanitizeRecoveredResult(finalised.result);
|
|
3794
|
+
const raw = event.kind === 'other' ? event.raw : null;
|
|
3795
|
+
const message = raw && typeof raw.message === 'object' && raw.message
|
|
3796
|
+
? raw.message
|
|
3797
|
+
: null;
|
|
3798
|
+
const directMessageId = event.messageId;
|
|
3799
|
+
const messageId = directMessageId ?? (message && typeof message.id === 'string' ? message.id : undefined);
|
|
3800
|
+
const stableId = messageId ? `message:${messageId}` : `${event.kind}:${seq}`;
|
|
3801
|
+
const previous = snapshots.get(stableId) ?? {};
|
|
3802
|
+
const addDelta = (key) => {
|
|
3803
|
+
const current = result[key];
|
|
3804
|
+
if (typeof current !== 'number')
|
|
3805
|
+
return;
|
|
3806
|
+
const prior = previous[key];
|
|
3807
|
+
totals[key] += Math.max(0, current - (typeof prior === 'number' ? prior : 0));
|
|
3808
|
+
};
|
|
3809
|
+
addDelta('tokens_in');
|
|
3810
|
+
addDelta('tokens_out');
|
|
3811
|
+
addDelta('tokens_cache_read');
|
|
3812
|
+
addDelta('tokens_cache_create');
|
|
3813
|
+
addDelta('total_cost_usd');
|
|
3814
|
+
addDelta('num_turns');
|
|
3815
|
+
addDelta('duration_ms');
|
|
3816
|
+
addDelta('duration_api_ms');
|
|
3817
|
+
model = result.model ?? model;
|
|
3818
|
+
sessionId = result.session_id ?? sessionId;
|
|
3819
|
+
estimated = estimated || finalised.estimated;
|
|
3820
|
+
if (!snapshots.has(stableId) && snapshots.size >= MAX_SNAPSHOT_KEYS) {
|
|
3821
|
+
snapshots.delete(snapshots.keys().next().value);
|
|
3822
|
+
}
|
|
3823
|
+
// Frames for one provider message are contiguous; retain a bounded LRU
|
|
3824
|
+
// of recent snapshots for retransmission/delta dedupe without letting a
|
|
3825
|
+
// malicious transcript allocate one map entry per event forever.
|
|
3826
|
+
snapshots.delete(stableId);
|
|
3827
|
+
snapshots.set(stableId, { ...previous, ...result });
|
|
3828
|
+
};
|
|
3829
|
+
const result = () => {
|
|
3830
|
+
const hasUsage = Object.values(totals).some((value) => value > 0);
|
|
3831
|
+
return {
|
|
3832
|
+
result: hasUsage ? { ...totals, model, session_id: sessionId } : {},
|
|
3833
|
+
estimated,
|
|
3834
|
+
};
|
|
3835
|
+
};
|
|
3836
|
+
return { add, result };
|
|
3837
|
+
};
|
|
3838
|
+
if (!interactive) {
|
|
3839
|
+
const accumulator = makeUsageAccumulator();
|
|
3840
|
+
let lastValidResult = null;
|
|
3841
|
+
const events = db.prepare(`
|
|
3842
|
+
SELECT seq, event_type, payload FROM events
|
|
3843
|
+
WHERE job_id = ? AND source = 'stdout' AND event_type != 'log'
|
|
3844
|
+
ORDER BY seq, id
|
|
3845
|
+
`).iterate(jobId);
|
|
3846
|
+
for (const row of events) {
|
|
3847
|
+
const event = parse(row);
|
|
3848
|
+
if (event?.kind === 'result') {
|
|
3849
|
+
const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], {
|
|
3850
|
+
fallbackModel: fallbackModel ?? undefined,
|
|
3851
|
+
});
|
|
3852
|
+
lastValidResult = {
|
|
3853
|
+
...finalised,
|
|
3854
|
+
result: sanitizeRecoveredResult(finalised.result),
|
|
3855
|
+
};
|
|
3856
|
+
}
|
|
3857
|
+
else if (event) {
|
|
3858
|
+
accumulator.add(event, row.seq);
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
const fallback = accumulator.result();
|
|
3862
|
+
if (lastValidResult) {
|
|
3863
|
+
const result = { ...fallback.result };
|
|
3864
|
+
for (const [key, value] of Object.entries(lastValidResult.result)) {
|
|
3865
|
+
// Provider normalisers intentionally retain optional keys with an
|
|
3866
|
+
// undefined value. Only concrete terminal fields are authoritative;
|
|
3867
|
+
// an omitted field must not erase recoverable assistant evidence.
|
|
3868
|
+
if (value !== undefined) {
|
|
3869
|
+
result[key] = value;
|
|
3870
|
+
}
|
|
3871
|
+
}
|
|
3872
|
+
return {
|
|
3873
|
+
result,
|
|
3874
|
+
// A native terminal cost is authoritative. When the terminal frame
|
|
3875
|
+
// omits cost, preserve whether the assistant-frame backfill was an
|
|
3876
|
+
// estimate instead of silently relabelling it as exact.
|
|
3877
|
+
estimated: result.total_cost_usd === lastValidResult.result.total_cost_usd &&
|
|
3878
|
+
lastValidResult.result.total_cost_usd !== undefined
|
|
3879
|
+
? lastValidResult.estimated
|
|
3880
|
+
: (lastValidResult.estimated || fallback.estimated),
|
|
3881
|
+
authoritative: true,
|
|
3882
|
+
};
|
|
3883
|
+
}
|
|
3884
|
+
return { ...fallback, authoritative: false };
|
|
3885
|
+
}
|
|
3886
|
+
const totals = {
|
|
3887
|
+
tokens_in: 0,
|
|
3888
|
+
tokens_out: 0,
|
|
3889
|
+
tokens_cache_read: 0,
|
|
3890
|
+
tokens_cache_create: 0,
|
|
3891
|
+
total_cost_usd: 0,
|
|
3892
|
+
num_turns: 0,
|
|
3893
|
+
duration_ms: 0,
|
|
3894
|
+
duration_api_ms: 0,
|
|
3895
|
+
};
|
|
3896
|
+
let model = fallbackModel ?? undefined;
|
|
3897
|
+
let sessionId;
|
|
3898
|
+
let baselineCost = 0;
|
|
3899
|
+
let baselineTurns = 0;
|
|
3900
|
+
let estimated = false;
|
|
3901
|
+
let lastResultSeq = -1;
|
|
3902
|
+
const resultRows = db.prepare(`
|
|
3903
|
+
SELECT seq, event_type, payload FROM events
|
|
3904
|
+
WHERE job_id = ? AND source = 'stdout' AND event_type = 'result'
|
|
3905
|
+
ORDER BY seq, id
|
|
3906
|
+
`).iterate(jobId);
|
|
3907
|
+
for (const row of resultRows) {
|
|
3908
|
+
const event = parse(row);
|
|
3909
|
+
if (event?.kind !== 'result')
|
|
3910
|
+
continue;
|
|
3911
|
+
const finalised = (0, result_event_1.finaliseInvocationResult)(adapter, [event], { fallbackModel: model });
|
|
3912
|
+
const result = sanitizeRecoveredResult(finalised.result);
|
|
3913
|
+
totals.tokens_in += result.tokens_in ?? 0;
|
|
3914
|
+
totals.tokens_out += result.tokens_out ?? 0;
|
|
3915
|
+
totals.tokens_cache_read += result.tokens_cache_read ?? 0;
|
|
3916
|
+
totals.tokens_cache_create += result.tokens_cache_create ?? 0;
|
|
3917
|
+
const cumulativeCost = result.total_cost_usd ?? baselineCost;
|
|
3918
|
+
totals.total_cost_usd += Math.max(0, cumulativeCost - baselineCost);
|
|
3919
|
+
baselineCost = cumulativeCost;
|
|
3920
|
+
const cumulativeTurns = result.num_turns ?? (baselineTurns + 1);
|
|
3921
|
+
totals.num_turns += Math.max(0, cumulativeTurns - baselineTurns);
|
|
3922
|
+
baselineTurns = cumulativeTurns;
|
|
3923
|
+
totals.duration_ms += result.duration_ms ?? 0;
|
|
3924
|
+
totals.duration_api_ms += result.duration_api_ms ?? 0;
|
|
3925
|
+
model = result.model ?? model;
|
|
3926
|
+
sessionId = result.session_id ?? sessionId;
|
|
3927
|
+
estimated = estimated || finalised.estimated;
|
|
3928
|
+
lastResultSeq = row.seq;
|
|
3929
|
+
}
|
|
3930
|
+
const tail = makeUsageAccumulator();
|
|
3931
|
+
const tailRows = db.prepare(`
|
|
3932
|
+
SELECT seq, event_type, payload FROM events
|
|
3933
|
+
WHERE job_id = ? AND source = 'stdout' AND event_type != 'log' AND seq > ?
|
|
3934
|
+
ORDER BY seq, id
|
|
3935
|
+
`).iterate(jobId, lastResultSeq);
|
|
3936
|
+
for (const row of tailRows) {
|
|
3937
|
+
const event = parse(row);
|
|
3938
|
+
if (event && event.kind !== 'result')
|
|
3939
|
+
tail.add(event, row.seq);
|
|
3940
|
+
}
|
|
3941
|
+
const tailUsage = tail.result();
|
|
3942
|
+
const tailResult = tailUsage.result;
|
|
3943
|
+
const hasTailUsage = [
|
|
3944
|
+
tailResult.tokens_in,
|
|
3945
|
+
tailResult.tokens_out,
|
|
3946
|
+
tailResult.tokens_cache_read,
|
|
3947
|
+
tailResult.tokens_cache_create,
|
|
3948
|
+
tailResult.total_cost_usd,
|
|
3949
|
+
].some((value) => typeof value === 'number' && value > 0);
|
|
3950
|
+
if (hasTailUsage) {
|
|
3951
|
+
totals.tokens_in += tailResult.tokens_in ?? 0;
|
|
3952
|
+
totals.tokens_out += tailResult.tokens_out ?? 0;
|
|
3953
|
+
totals.tokens_cache_read += tailResult.tokens_cache_read ?? 0;
|
|
3954
|
+
totals.tokens_cache_create += tailResult.tokens_cache_create ?? 0;
|
|
3955
|
+
totals.total_cost_usd += tailResult.total_cost_usd ?? 0;
|
|
3956
|
+
totals.num_turns += tailResult.num_turns ?? 1;
|
|
3957
|
+
totals.duration_ms += tailResult.duration_ms ?? 0;
|
|
3958
|
+
totals.duration_api_ms += tailResult.duration_api_ms ?? 0;
|
|
3959
|
+
model = tailResult.model ?? model;
|
|
3960
|
+
sessionId = tailResult.session_id ?? sessionId;
|
|
3961
|
+
estimated = estimated || tailUsage.estimated;
|
|
3962
|
+
}
|
|
3963
|
+
const hasRecoveredUsage = Object.values(totals).some((value) => value > 0);
|
|
3964
|
+
return {
|
|
3965
|
+
result: hasRecoveredUsage
|
|
3966
|
+
? {
|
|
3967
|
+
...totals,
|
|
3968
|
+
model,
|
|
3969
|
+
session_id: sessionId,
|
|
3970
|
+
}
|
|
3971
|
+
: {},
|
|
3972
|
+
estimated,
|
|
3973
|
+
authoritative: false,
|
|
3974
|
+
};
|
|
3975
|
+
}
|
|
3976
|
+
_decodeRecoveryPayload(jobId, encoded) {
|
|
3977
|
+
const payload = JSON.parse(encoded);
|
|
3978
|
+
if (!payload || typeof payload !== 'object')
|
|
3979
|
+
throw new Error('payload is not an object');
|
|
3980
|
+
if (typeof payload.command !== 'string')
|
|
3981
|
+
throw new Error('payload command is invalid');
|
|
3982
|
+
// Ticket ownership is derived from the immutable command, never trusted
|
|
3983
|
+
// from a legacy/corrupt payload. This also repairs pre-fix [0]/overflow ids.
|
|
3984
|
+
payload.ticketIds = (0, ticket_store_1.extractTicketIdsFromCommand)(payload.command);
|
|
3985
|
+
payload.totalCostUsdEstimated ??= 0;
|
|
3986
|
+
const terminalStatus = payload.terminalStatus ?? 'failed';
|
|
3987
|
+
const invocationStatus = payload.invocationStatus ?? 'aborted';
|
|
3988
|
+
const nullableNumber = (value) => value == null || (typeof value === 'number' && Number.isFinite(value) && value >= 0);
|
|
3989
|
+
if (payload.id !== jobId ||
|
|
3990
|
+
typeof payload.startedAt !== 'string' ||
|
|
3991
|
+
typeof payload.finishedAt !== 'string' ||
|
|
3992
|
+
typeof payload.provider !== 'string' || !payload.provider ||
|
|
3993
|
+
(payload.pipelineId !== null && typeof payload.pipelineId !== 'string') ||
|
|
3994
|
+
!payload.ticketIds.every((id) => Number.isSafeInteger(id) && id > 0) ||
|
|
3995
|
+
!nullableNumber(payload.tokensIn) ||
|
|
3996
|
+
!nullableNumber(payload.tokensOut) ||
|
|
3997
|
+
!nullableNumber(payload.tokensCacheRead) ||
|
|
3998
|
+
!nullableNumber(payload.tokensCacheCreate) ||
|
|
3999
|
+
!nullableNumber(payload.totalCostUsd) ||
|
|
4000
|
+
!nullableNumber(payload.numTurns) ||
|
|
4001
|
+
!nullableNumber(payload.durationMs) ||
|
|
4002
|
+
!nullableNumber(payload.durationApiMs) ||
|
|
4003
|
+
(payload.totalCostUsdEstimated !== 0 && payload.totalCostUsdEstimated !== 1) ||
|
|
4004
|
+
(payload.exitCode != null && !Number.isInteger(payload.exitCode)) ||
|
|
4005
|
+
(payload.model !== null && typeof payload.model !== 'string') ||
|
|
4006
|
+
(payload.sessionId !== null && typeof payload.sessionId !== 'string') ||
|
|
4007
|
+
(payload.causalOwnership !== undefined && typeof payload.causalOwnership !== 'boolean') ||
|
|
4008
|
+
(payload.awaitingLateReconciliation !== undefined && typeof payload.awaitingLateReconciliation !== 'boolean') ||
|
|
4009
|
+
(payload.ticketCompletionStatus !== undefined && !['done', 'on_review'].includes(payload.ticketCompletionStatus)) ||
|
|
4010
|
+
!TERMINAL_STATUSES.has(terminalStatus) ||
|
|
4011
|
+
!['success', 'failed', 'aborted'].includes(invocationStatus) ||
|
|
4012
|
+
(payload.descendants !== undefined && (!Array.isArray(payload.descendants) ||
|
|
4013
|
+
!payload.descendants.every((descendant) => descendant && typeof descendant.id === 'string' &&
|
|
4014
|
+
typeof descendant.command === 'string' &&
|
|
4015
|
+
typeof descendant.parentId === 'string' &&
|
|
4016
|
+
(descendant.pipelineId === null || typeof descendant.pipelineId === 'string') &&
|
|
4017
|
+
types_1.VALID_PRIORITIES.has(descendant.priority) &&
|
|
4018
|
+
(descendant.causalOwnership === undefined || typeof descendant.causalOwnership === 'boolean'))))) {
|
|
4019
|
+
throw new Error('payload identity or fields are invalid');
|
|
4020
|
+
}
|
|
4021
|
+
return { payload, terminalStatus, invocationStatus };
|
|
4022
|
+
}
|
|
4023
|
+
/**
|
|
4024
|
+
* Atomically convert every crash-orphaned RUNNING job into a durable recovery
|
|
4025
|
+
* intent and a failed job. The outbox snapshot is committed in the SAME
|
|
4026
|
+
* SQLite transaction as the status flip, so a process death can leave either
|
|
4027
|
+
* the original running row or a replayable failed row — never an untracked
|
|
4028
|
+
* half-recovery.
|
|
4029
|
+
*/
|
|
4030
|
+
_captureOrphanRecoveries() {
|
|
4031
|
+
const db = this._db;
|
|
4032
|
+
if (!db)
|
|
4033
|
+
return;
|
|
4034
|
+
const capture = db.transaction(() => {
|
|
4035
|
+
// A server restart proves that no prior-process `close` listener can
|
|
4036
|
+
// still enrich a force-failed job. Release those holds and retain the
|
|
4037
|
+
// partial usage already checkpointed as the best available evidence.
|
|
4038
|
+
const heldRecoveries = db.prepare(`
|
|
4039
|
+
SELECT recovery.job_id, recovery.payload, recovery.accounting_completed,
|
|
4040
|
+
jobs.provider, jobs.model, jobs.interactive
|
|
4041
|
+
FROM orphan_job_recovery AS recovery
|
|
4042
|
+
LEFT JOIN jobs ON jobs.id = recovery.job_id
|
|
4043
|
+
`).all();
|
|
4044
|
+
const releaseHold = db.prepare(`
|
|
4045
|
+
UPDATE orphan_job_recovery
|
|
4046
|
+
SET payload = ?, accounting_completed = ?
|
|
4047
|
+
WHERE job_id = ?
|
|
4048
|
+
`);
|
|
4049
|
+
for (const held of heldRecoveries) {
|
|
4050
|
+
let decoded;
|
|
4051
|
+
try {
|
|
4052
|
+
decoded = this._decodeRecoveryPayload(held.job_id, held.payload);
|
|
4053
|
+
}
|
|
4054
|
+
catch (err) {
|
|
4055
|
+
console.error(`[queue-manager] invalid held recovery payload for ${held.job_id}:`, err);
|
|
4056
|
+
throw err;
|
|
4057
|
+
}
|
|
4058
|
+
const { payload, terminalStatus } = decoded;
|
|
4059
|
+
if (!payload.awaitingLateReconciliation)
|
|
4060
|
+
continue;
|
|
4061
|
+
const recovered = this._recoverPersistedJobUsage(held.job_id, held.provider ?? payload.provider ?? this._adapter.id, held.model ?? payload.model, held.interactive === 1);
|
|
4062
|
+
const hasRecoveredUsage = [
|
|
4063
|
+
recovered.result.tokens_in,
|
|
4064
|
+
recovered.result.tokens_out,
|
|
4065
|
+
recovered.result.tokens_cache_read,
|
|
4066
|
+
recovered.result.tokens_cache_create,
|
|
4067
|
+
recovered.result.total_cost_usd,
|
|
4068
|
+
].some((value) => typeof value === 'number' && value > 0);
|
|
4069
|
+
const hasRecoveredEvidence = hasRecoveredUsage || recovered.authoritative;
|
|
4070
|
+
if (hasRecoveredEvidence) {
|
|
4071
|
+
if (recovered.authoritative) {
|
|
4072
|
+
payload.model = recovered.result.model ?? null;
|
|
4073
|
+
payload.tokensIn = recovered.result.tokens_in ?? null;
|
|
4074
|
+
payload.tokensOut = recovered.result.tokens_out ?? null;
|
|
4075
|
+
payload.tokensCacheRead = recovered.result.tokens_cache_read ?? null;
|
|
4076
|
+
payload.tokensCacheCreate = recovered.result.tokens_cache_create ?? null;
|
|
4077
|
+
payload.totalCostUsd = recovered.result.total_cost_usd ?? null;
|
|
4078
|
+
payload.totalCostUsdEstimated = recovered.estimated ? 1 : 0;
|
|
4079
|
+
payload.numTurns = recovered.result.num_turns ?? null;
|
|
4080
|
+
payload.durationMs = recovered.result.duration_ms ?? null;
|
|
4081
|
+
payload.durationApiMs = recovered.result.duration_api_ms ?? null;
|
|
4082
|
+
payload.sessionId = recovered.result.session_id ?? null;
|
|
4083
|
+
}
|
|
4084
|
+
else {
|
|
4085
|
+
payload.model = recovered.result.model ?? payload.model;
|
|
4086
|
+
payload.tokensIn = maxNullable(payload.tokensIn, recovered.result.tokens_in);
|
|
4087
|
+
payload.tokensOut = maxNullable(payload.tokensOut, recovered.result.tokens_out);
|
|
4088
|
+
payload.tokensCacheRead = maxNullable(payload.tokensCacheRead, recovered.result.tokens_cache_read);
|
|
4089
|
+
payload.tokensCacheCreate = maxNullable(payload.tokensCacheCreate, recovered.result.tokens_cache_create);
|
|
4090
|
+
payload.totalCostUsd = maxNullable(payload.totalCostUsd, recovered.result.total_cost_usd);
|
|
4091
|
+
payload.totalCostUsdEstimated = (payload.totalCostUsdEstimated || recovered.estimated) ? 1 : 0;
|
|
4092
|
+
payload.numTurns = maxNullable(payload.numTurns, recovered.result.num_turns);
|
|
4093
|
+
payload.durationMs = maxNullable(payload.durationMs, recovered.result.duration_ms);
|
|
4094
|
+
payload.durationApiMs = maxNullable(payload.durationApiMs, recovered.result.duration_api_ms);
|
|
4095
|
+
payload.sessionId = recovered.result.session_id ?? payload.sessionId;
|
|
4096
|
+
}
|
|
4097
|
+
this._decodeRecoveryPayload(held.job_id, JSON.stringify(payload));
|
|
4098
|
+
(0, db_1.finishJob)(db, held.job_id, {
|
|
4099
|
+
exit_code: payload.exitCode ?? -1,
|
|
4100
|
+
status: terminalStatus,
|
|
4101
|
+
tokens_in: payload.tokensIn ?? undefined,
|
|
4102
|
+
tokens_out: payload.tokensOut ?? undefined,
|
|
4103
|
+
tokens_cache_read: payload.tokensCacheRead ?? undefined,
|
|
4104
|
+
tokens_cache_create: payload.tokensCacheCreate ?? undefined,
|
|
4105
|
+
total_cost_usd: payload.totalCostUsd ?? undefined,
|
|
4106
|
+
total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
|
|
4107
|
+
num_turns: payload.numTurns ?? undefined,
|
|
4108
|
+
model: payload.model ?? undefined,
|
|
4109
|
+
duration_ms: payload.durationMs ?? undefined,
|
|
4110
|
+
duration_api_ms: payload.durationApiMs ?? undefined,
|
|
4111
|
+
session_id: payload.sessionId ?? undefined,
|
|
4112
|
+
});
|
|
4113
|
+
}
|
|
4114
|
+
payload.awaitingLateReconciliation = false;
|
|
4115
|
+
const shouldReaccount = hasRecoveredEvidence && !!this._projectId;
|
|
4116
|
+
if (shouldReaccount) {
|
|
4117
|
+
db.prepare(`DELETE FROM ai_invocations WHERE surface_ref_id = ? OR surface_ref_id LIKE ?`).run(held.job_id, `${held.job_id}#t%`);
|
|
4118
|
+
}
|
|
4119
|
+
releaseHold.run(JSON.stringify(payload), shouldReaccount ? 0 : held.accounting_completed, held.job_id);
|
|
4120
|
+
}
|
|
4121
|
+
const orphans = db.prepare(`SELECT id, command, pipeline_id, provider, started_at, model, tokens_in, tokens_out, tokens_cache_read,
|
|
4122
|
+
tokens_cache_create, total_cost_usd, total_cost_usd_estimated, num_turns,
|
|
4123
|
+
duration_ms, duration_api_ms, session_id, causal_ownership, interactive
|
|
4124
|
+
FROM jobs WHERE status = 'running' AND owner = 'queue'`).all();
|
|
4125
|
+
if (orphans.length === 0)
|
|
4126
|
+
return;
|
|
4127
|
+
const finishedAt = new Date().toISOString();
|
|
4128
|
+
const insert = db.prepare(`INSERT OR IGNORE INTO orphan_job_recovery (job_id, payload)
|
|
4129
|
+
VALUES (?, ?)`);
|
|
4130
|
+
for (const orphan of orphans) {
|
|
4131
|
+
const finalised = this._recoverPersistedJobUsage(orphan.id, orphan.provider ?? this._adapter.id, orphan.model, orphan.interactive === 1);
|
|
4132
|
+
const recovered = finalised.result;
|
|
4133
|
+
const recoveredEstimated = finalised.estimated;
|
|
4134
|
+
const choose = (persisted, raw) => finalised.authoritative ? (raw ?? null) : maxNullable(persisted, raw);
|
|
4135
|
+
const payload = {
|
|
4136
|
+
id: orphan.id,
|
|
4137
|
+
command: orphan.command,
|
|
4138
|
+
ticketIds: (0, ticket_store_1.extractTicketIdsFromCommand)(orphan.command),
|
|
4139
|
+
pipelineId: orphan.pipeline_id,
|
|
4140
|
+
startedAt: orphan.started_at ?? finishedAt,
|
|
4141
|
+
finishedAt,
|
|
4142
|
+
// Old rows predate jobs.provider; their only honest fallback is the
|
|
4143
|
+
// project's primary adapter. New rows preserve per-job overrides.
|
|
4144
|
+
provider: orphan.provider ?? this._adapter.id,
|
|
4145
|
+
model: finalised.authoritative
|
|
4146
|
+
? (recovered.model ?? null)
|
|
4147
|
+
: (recovered.model ?? orphan.model ?? null),
|
|
4148
|
+
tokensIn: choose(orphan.tokens_in, recovered.tokens_in),
|
|
4149
|
+
tokensOut: choose(orphan.tokens_out, recovered.tokens_out),
|
|
4150
|
+
tokensCacheRead: choose(orphan.tokens_cache_read, recovered.tokens_cache_read),
|
|
4151
|
+
tokensCacheCreate: choose(orphan.tokens_cache_create, recovered.tokens_cache_create),
|
|
4152
|
+
totalCostUsd: choose(orphan.total_cost_usd, recovered.total_cost_usd),
|
|
4153
|
+
totalCostUsdEstimated: finalised.authoritative
|
|
4154
|
+
? (recoveredEstimated ? 1 : 0)
|
|
4155
|
+
: ((orphan.total_cost_usd_estimated || recoveredEstimated) ? 1 : 0),
|
|
4156
|
+
numTurns: choose(orphan.num_turns, recovered.num_turns),
|
|
4157
|
+
durationMs: choose(orphan.duration_ms, recovered.duration_ms),
|
|
4158
|
+
durationApiMs: choose(orphan.duration_api_ms, recovered.duration_api_ms),
|
|
4159
|
+
sessionId: finalised.authoritative
|
|
4160
|
+
? (recovered.session_id ?? null)
|
|
4161
|
+
: (recovered.session_id ?? orphan.session_id ?? null),
|
|
4162
|
+
descendants: this._snapshotRecoveryDescendants(orphan.id),
|
|
4163
|
+
causalOwnership: orphan.causal_ownership === 1,
|
|
4164
|
+
terminalStatus: 'failed',
|
|
4165
|
+
invocationStatus: 'aborted',
|
|
4166
|
+
exitCode: -1,
|
|
4167
|
+
};
|
|
4168
|
+
const encoded = JSON.stringify(this._decodeRecoveryPayload(orphan.id, JSON.stringify(payload)).payload);
|
|
4169
|
+
insert.run(orphan.id, encoded);
|
|
4170
|
+
(0, db_1.finishJob)(db, orphan.id, {
|
|
4171
|
+
exit_code: -1,
|
|
4172
|
+
status: 'failed',
|
|
4173
|
+
tokens_in: payload.tokensIn ?? undefined,
|
|
4174
|
+
tokens_out: payload.tokensOut ?? undefined,
|
|
4175
|
+
tokens_cache_read: payload.tokensCacheRead ?? undefined,
|
|
4176
|
+
tokens_cache_create: payload.tokensCacheCreate ?? undefined,
|
|
4177
|
+
total_cost_usd: payload.totalCostUsd ?? undefined,
|
|
4178
|
+
total_cost_usd_estimated: !!payload.totalCostUsdEstimated,
|
|
4179
|
+
num_turns: payload.numTurns ?? undefined,
|
|
4180
|
+
model: payload.model ?? undefined,
|
|
4181
|
+
duration_ms: payload.durationMs ?? undefined,
|
|
4182
|
+
duration_api_ms: payload.durationApiMs ?? undefined,
|
|
4183
|
+
session_id: payload.sessionId ?? undefined,
|
|
4184
|
+
});
|
|
4185
|
+
}
|
|
4186
|
+
});
|
|
4187
|
+
capture();
|
|
4188
|
+
}
|
|
4189
|
+
/**
|
|
4190
|
+
* Drain the orphan outbox with an independent durable checkpoint per effect.
|
|
4191
|
+
* Accounting runs in the same transaction as its checkpoint, which makes it
|
|
4192
|
+
* exactly-once across arbitrary process death. The domain callback is a
|
|
4193
|
+
* conventional at-least-once outbox delivery: its DB mutations and checkpoint
|
|
4194
|
+
* share a transaction, and every replay carries the same stable job id so its
|
|
4195
|
+
* idempotent ticket/rail/Jira mutations can safely converge after a crash. A
|
|
4196
|
+
* third checkpoint replays the normal dependent-skip/pipeline invariants only
|
|
4197
|
+
* after queued rows have been restored into `_jobs` and `_queue`.
|
|
4198
|
+
*/
|
|
4199
|
+
_resumeOrphanRecoveries() {
|
|
4200
|
+
try {
|
|
4201
|
+
return this._resumeOrphanRecoveriesUnsafe();
|
|
4202
|
+
}
|
|
4203
|
+
catch (err) {
|
|
4204
|
+
// A durable effect remains pending by construction. Surface a fail-stop
|
|
4205
|
+
// result instead of letting an EventEmitter close handler throw uncaught
|
|
4206
|
+
// or allowing later provider work to overtake unknown accounting.
|
|
4207
|
+
console.error('[queue-manager] terminal recovery drain failed:', err);
|
|
4208
|
+
return false;
|
|
4209
|
+
}
|
|
4210
|
+
}
|
|
4211
|
+
_resumeOrphanRecoveriesUnsafe() {
|
|
4212
|
+
const db = this._db;
|
|
4213
|
+
if (!db)
|
|
4214
|
+
return true;
|
|
4215
|
+
const rows = db.prepare(`SELECT job_id, payload, accounting_completed, callback_completed, terminal_completed
|
|
4216
|
+
FROM orphan_job_recovery
|
|
4217
|
+
WHERE accounting_completed = 0 OR callback_completed = 0 OR terminal_completed = 0
|
|
4218
|
+
ORDER BY created_at, job_id`).all();
|
|
4219
|
+
let spendingChanged = false;
|
|
4220
|
+
let invalidRecovery = false;
|
|
4221
|
+
for (const row of rows) {
|
|
4222
|
+
let payload;
|
|
4223
|
+
let terminalStatus;
|
|
4224
|
+
let invocationStatus;
|
|
4225
|
+
try {
|
|
4226
|
+
({ payload, terminalStatus, invocationStatus } = this._decodeRecoveryPayload(row.job_id, row.payload));
|
|
4227
|
+
}
|
|
4228
|
+
catch (err) {
|
|
4229
|
+
console.error(`[queue-manager] invalid orphan recovery payload for ${row.job_id}:`, err);
|
|
4230
|
+
invalidRecovery = true;
|
|
4231
|
+
continue;
|
|
4232
|
+
}
|
|
4233
|
+
if (row.accounting_completed === 0 && !this._projectId) {
|
|
4234
|
+
db.prepare(`UPDATE orphan_job_recovery SET accounting_completed = 1 WHERE job_id = ?`)
|
|
4235
|
+
.run(row.job_id);
|
|
4236
|
+
}
|
|
4237
|
+
else if (row.accounting_completed === 0 && this._projectId) {
|
|
4238
|
+
try {
|
|
4239
|
+
const recordAccounting = db.transaction(() => {
|
|
4240
|
+
const pending = db.prepare(`SELECT accounting_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
|
|
4241
|
+
if (!pending || pending.accounting_completed !== 0)
|
|
4242
|
+
return false;
|
|
4243
|
+
this._recordJobInvocations({
|
|
4244
|
+
jobId: payload.id,
|
|
4245
|
+
provider: payload.provider,
|
|
4246
|
+
status: invocationStatus,
|
|
4247
|
+
startedAt: payload.startedAt,
|
|
4248
|
+
finishedAt: payload.finishedAt,
|
|
4249
|
+
ticketIds: payload.ticketIds,
|
|
4250
|
+
estimated: !!payload.totalCostUsdEstimated,
|
|
4251
|
+
result: {
|
|
4252
|
+
tokens_in: payload.tokensIn ?? undefined,
|
|
4253
|
+
tokens_out: payload.tokensOut ?? undefined,
|
|
4254
|
+
tokens_cache_read: payload.tokensCacheRead ?? undefined,
|
|
4255
|
+
tokens_cache_create: payload.tokensCacheCreate ?? undefined,
|
|
4256
|
+
total_cost_usd: payload.totalCostUsd ?? undefined,
|
|
4257
|
+
num_turns: payload.numTurns ?? undefined,
|
|
4258
|
+
model: payload.model ?? undefined,
|
|
4259
|
+
session_id: payload.sessionId ?? undefined,
|
|
4260
|
+
duration_ms: payload.durationMs ?? undefined,
|
|
4261
|
+
duration_api_ms: payload.durationApiMs ?? undefined,
|
|
4262
|
+
},
|
|
4263
|
+
});
|
|
4264
|
+
db.prepare(`UPDATE orphan_job_recovery SET accounting_completed = 1 WHERE job_id = ?`).run(row.job_id);
|
|
4265
|
+
return true;
|
|
4266
|
+
});
|
|
4267
|
+
spendingChanged = recordAccounting() || spendingChanged;
|
|
4268
|
+
}
|
|
4269
|
+
catch (err) {
|
|
4270
|
+
console.error(`[queue-manager] orphan accounting replay failed for ${row.job_id}:`, err);
|
|
4271
|
+
}
|
|
4272
|
+
}
|
|
4273
|
+
if (row.callback_completed === 0 && !this._onJobFinished) {
|
|
4274
|
+
db.prepare(`UPDATE orphan_job_recovery SET callback_completed = 1 WHERE job_id = ?`)
|
|
4275
|
+
.run(row.job_id);
|
|
4276
|
+
}
|
|
4277
|
+
else if (row.callback_completed === 0 && this._onJobFinished) {
|
|
4278
|
+
try {
|
|
4279
|
+
const deliverCallback = db.transaction(() => {
|
|
4280
|
+
const pending = db.prepare(`SELECT callback_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
|
|
4281
|
+
if (!pending || pending.callback_completed !== 0)
|
|
4282
|
+
return;
|
|
4283
|
+
this._onJobFinished?.(payload.id, terminalStatus, payload.totalCostUsd ?? undefined, {
|
|
4284
|
+
recoveryReplay: true,
|
|
4285
|
+
recoveryCommand: payload.command,
|
|
4286
|
+
recoveryTicketIds: payload.ticketIds,
|
|
4287
|
+
recoveryDurationMs: payload.durationMs,
|
|
4288
|
+
recoveryCausalOwnership: payload.causalOwnership === true,
|
|
4289
|
+
...(terminalStatus === 'completed' && payload.ticketCompletionStatus
|
|
4290
|
+
? { ticketCompletionStatus: payload.ticketCompletionStatus }
|
|
4291
|
+
: {}),
|
|
4292
|
+
});
|
|
4293
|
+
db.prepare(`UPDATE orphan_job_recovery SET callback_completed = 1 WHERE job_id = ?`).run(row.job_id);
|
|
4294
|
+
});
|
|
4295
|
+
deliverCallback();
|
|
4296
|
+
}
|
|
4297
|
+
catch (err) {
|
|
4298
|
+
console.error(`[queue-manager] orphan outcome replay failed for ${row.job_id}: ${err.message}`);
|
|
4299
|
+
}
|
|
4300
|
+
}
|
|
4301
|
+
if (row.terminal_completed === 0) {
|
|
4302
|
+
// `_skipDependents` updates the in-memory projection before its SQL.
|
|
4303
|
+
// SQLite can roll the transaction back, so retain an equally atomic
|
|
4304
|
+
// rollback image; otherwise a transient checkpoint failure removes
|
|
4305
|
+
// durable queued work from this process until another restart.
|
|
4306
|
+
const queueBefore = [...this._queue];
|
|
4307
|
+
const jobsBefore = new Map(Array.from(this._jobs, ([id, job]) => [id, {
|
|
4308
|
+
status: job.status,
|
|
4309
|
+
finishedAt: job.finishedAt,
|
|
4310
|
+
skipReason: job.skipReason,
|
|
4311
|
+
queuePosition: job.queuePosition,
|
|
4312
|
+
}]));
|
|
4313
|
+
try {
|
|
4314
|
+
const replayTerminalInvariants = db.transaction(() => {
|
|
4315
|
+
const pending = db.prepare(`SELECT terminal_completed FROM orphan_job_recovery WHERE job_id = ?`).get(row.job_id);
|
|
4316
|
+
if (!pending || pending.terminal_completed !== 0)
|
|
4317
|
+
return false;
|
|
4318
|
+
// Reuse the exact recursive terminal semantics used by live failed
|
|
4319
|
+
// jobs. At this point startup has already restored every queued row
|
|
4320
|
+
// into memory, so descendants are persisted as skipped and removed
|
|
4321
|
+
// from the runnable queue just as they are on a normal exit.
|
|
4322
|
+
// Compatibility with intents written before descendant snapshots:
|
|
4323
|
+
// materialise once and reuse it for both recovery-safe callbacks
|
|
4324
|
+
// and cross-pipeline status evaluation.
|
|
4325
|
+
const recoveryDescendants = Array.isArray(payload.descendants)
|
|
4326
|
+
? payload.descendants
|
|
4327
|
+
: this._snapshotRecoveryDescendants(payload.id);
|
|
4328
|
+
if (terminalStatus !== 'completed') {
|
|
4329
|
+
this._skipRecoveredDescendants(payload.id, recoveryDescendants, terminalStatus);
|
|
4330
|
+
}
|
|
4331
|
+
// Descendants may intentionally belong to different pipelines.
|
|
4332
|
+
// Recompute every affected pipeline before checkpointing so a
|
|
4333
|
+
// crash replays any missed notification instead of orphaning the
|
|
4334
|
+
// child's pipeline in a stale running state.
|
|
4335
|
+
const affectedPipelines = new Set();
|
|
4336
|
+
if (payload.pipelineId)
|
|
4337
|
+
affectedPipelines.add(payload.pipelineId);
|
|
4338
|
+
for (const descendant of recoveryDescendants) {
|
|
4339
|
+
if (descendant.pipelineId)
|
|
4340
|
+
affectedPipelines.add(descendant.pipelineId);
|
|
4341
|
+
}
|
|
4342
|
+
for (const pipelineId of affectedPipelines) {
|
|
4343
|
+
this._checkPersistedPipelineStatus(pipelineId);
|
|
4344
|
+
}
|
|
4345
|
+
db.prepare(`UPDATE orphan_job_recovery SET terminal_completed = 1 WHERE job_id = ?`).run(row.job_id);
|
|
4346
|
+
return true;
|
|
4347
|
+
});
|
|
4348
|
+
replayTerminalInvariants();
|
|
4349
|
+
}
|
|
4350
|
+
catch (err) {
|
|
4351
|
+
this._queue = queueBefore;
|
|
4352
|
+
for (const [id, before] of jobsBefore) {
|
|
4353
|
+
const job = this._jobs.get(id);
|
|
4354
|
+
if (!job)
|
|
4355
|
+
continue;
|
|
4356
|
+
job.status = before.status;
|
|
4357
|
+
job.finishedAt = before.finishedAt;
|
|
4358
|
+
job.skipReason = before.skipReason;
|
|
4359
|
+
job.queuePosition = before.queuePosition;
|
|
4360
|
+
}
|
|
4361
|
+
console.error(`[queue-manager] orphan terminal replay failed for ${row.job_id}: ${err.message}`);
|
|
4362
|
+
}
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
// A crash before this cleanup is harmless: completed rows are skipped on
|
|
4366
|
+
// the next startup and removed then.
|
|
4367
|
+
const completedRows = db.prepare(`
|
|
4368
|
+
SELECT job_id, payload FROM orphan_job_recovery
|
|
4369
|
+
WHERE accounting_completed = 1
|
|
4370
|
+
AND callback_completed = 1
|
|
4371
|
+
AND terminal_completed = 1
|
|
4372
|
+
`).all();
|
|
4373
|
+
const removeCompleted = db.prepare(`DELETE FROM orphan_job_recovery WHERE job_id = ?`);
|
|
4374
|
+
const cleanupCompleted = db.transaction(() => {
|
|
4375
|
+
for (const completed of completedRows) {
|
|
4376
|
+
let payload;
|
|
4377
|
+
try {
|
|
4378
|
+
payload = this._decodeRecoveryPayload(completed.job_id, completed.payload).payload;
|
|
4379
|
+
}
|
|
4380
|
+
catch (err) {
|
|
4381
|
+
throw new Error(`Invalid completed recovery payload for ${completed.job_id}: ${err.message}`);
|
|
4382
|
+
}
|
|
4383
|
+
if (payload.awaitingLateReconciliation)
|
|
4384
|
+
continue;
|
|
4385
|
+
removeCompleted.run(completed.job_id);
|
|
4386
|
+
}
|
|
4387
|
+
});
|
|
4388
|
+
cleanupCompleted();
|
|
4389
|
+
if (spendingChanged && this._projectId) {
|
|
4390
|
+
try {
|
|
4391
|
+
this._broadcast({ type: 'spending.invalidated', projectId: this._projectId });
|
|
4392
|
+
}
|
|
4393
|
+
catch { /* advisory */ }
|
|
4394
|
+
}
|
|
4395
|
+
const pendingCritical = db.prepare(`
|
|
4396
|
+
SELECT 1 FROM orphan_job_recovery
|
|
4397
|
+
WHERE accounting_completed = 0 OR terminal_completed = 0
|
|
4398
|
+
LIMIT 1
|
|
4399
|
+
`).get();
|
|
4400
|
+
return !pendingCritical && !invalidRecovery;
|
|
2731
4401
|
}
|
|
2732
4402
|
_checkPipelineStatus(pipelineId) {
|
|
2733
4403
|
const pipelineJobs = Array.from(this._jobs.values()).filter(j => j.pipelineId === pipelineId);
|
|
@@ -2736,11 +4406,50 @@ class QueueManager {
|
|
|
2736
4406
|
const allDone = pipelineJobs.every(j => j.status === 'completed');
|
|
2737
4407
|
const anyFailed = pipelineJobs.some(j => j.status === 'failed' || j.status === 'skipped' || j.status === 'canceled' || j.status === 'zombie_terminated');
|
|
2738
4408
|
const anyPending = pipelineJobs.some(j => j.status === 'queued' || j.status === 'running');
|
|
2739
|
-
|
|
2740
|
-
|
|
4409
|
+
const status = allDone ? 'completed' : (anyFailed && !anyPending ? 'failed' : null);
|
|
4410
|
+
if (!status) {
|
|
4411
|
+
// A caller may append work to an existing pipeline id. Its next terminal
|
|
4412
|
+
// transition is new and must be observable again.
|
|
4413
|
+
this._emittedPipelineStatuses.delete(pipelineId);
|
|
4414
|
+
return;
|
|
2741
4415
|
}
|
|
2742
|
-
|
|
2743
|
-
|
|
4416
|
+
if (this._emittedPipelineStatuses.get(pipelineId) === status)
|
|
4417
|
+
return;
|
|
4418
|
+
this._broadcast({ type: 'pipeline_status', pipelineId, status });
|
|
4419
|
+
this._emittedPipelineStatuses.set(pipelineId, status);
|
|
4420
|
+
}
|
|
4421
|
+
/**
|
|
4422
|
+
* Recovery counterpart of `_checkPipelineStatus`. Terminal parents are not
|
|
4423
|
+
* restored into the in-memory job map, so startup must evaluate the complete
|
|
4424
|
+
* persisted pipeline rather than the queued-only `_jobs` view.
|
|
4425
|
+
*/
|
|
4426
|
+
_checkPersistedPipelineStatus(pipelineId) {
|
|
4427
|
+
const db = this._db;
|
|
4428
|
+
if (!db)
|
|
4429
|
+
return;
|
|
4430
|
+
const rows = db.prepare(`SELECT status FROM jobs WHERE pipeline_id = ?
|
|
4431
|
+
UNION ALL
|
|
4432
|
+
SELECT 'queued' AS status FROM queued_jobs
|
|
4433
|
+
WHERE pipeline_id = ?
|
|
4434
|
+
AND NOT EXISTS (SELECT 1 FROM jobs WHERE jobs.id = queued_jobs.id)`).all(pipelineId, pipelineId);
|
|
4435
|
+
if (rows.length === 0)
|
|
4436
|
+
return;
|
|
4437
|
+
const allDone = rows.every((job) => job.status === 'completed');
|
|
4438
|
+
const anyFailed = rows.some((job) => job.status === 'failed' || job.status === 'skipped' || job.status === 'canceled' || job.status === 'zombie_terminated');
|
|
4439
|
+
const anyPending = rows.some((job) => job.status === 'queued' || job.status === 'running');
|
|
4440
|
+
try {
|
|
4441
|
+
const status = allDone ? 'completed' : (anyFailed && !anyPending ? 'failed' : null);
|
|
4442
|
+
if (!status) {
|
|
4443
|
+
this._emittedPipelineStatuses.delete(pipelineId);
|
|
4444
|
+
return;
|
|
4445
|
+
}
|
|
4446
|
+
if (this._emittedPipelineStatuses.get(pipelineId) === status)
|
|
4447
|
+
return;
|
|
4448
|
+
this._broadcast({ type: 'pipeline_status', pipelineId, status });
|
|
4449
|
+
this._emittedPipelineStatuses.set(pipelineId, status);
|
|
4450
|
+
}
|
|
4451
|
+
catch {
|
|
4452
|
+
// Startup broadcasts are advisory; persisted terminal state is authoritative.
|
|
2744
4453
|
}
|
|
2745
4454
|
}
|
|
2746
4455
|
_recomputePositions() {
|