squidclaw 3.0.2 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{accounts-LA308FHj.js → accounts-CK_sHUyT.js} +2 -2
- package/dist/{accounts-CL_NXliB.js → accounts-CkF7YwoF.js} +17 -17
- package/dist/{accounts-F7tGwezI.js → accounts-DbloMfwT.js} +2 -2
- package/dist/{active-listener-DJv1FZqf.js → active-listener-AepfNSUY.js} +2 -2
- package/dist/{agents-DdixSPs3.js → agents-JnnOlm2G.js} +5 -5
- package/dist/{agents.config-Cn_vTN1v.js → agents.config-BeGeS2jv.js} +1 -1
- package/dist/{agents.config-C6KTwnde.js → agents.config-DHJBQ7uA.js} +1 -1
- package/dist/{plugin-sdk/api-key-rotation-DE4gr5YM.js → api-key-rotation-BHFJiYbw.js} +2 -2
- package/dist/{audio-preflight-AEM744TY.js → audio-preflight-BTYxAJjy.js} +32 -32
- package/dist/{audio-preflight-DpCWFB4z.js → audio-preflight-Dkl6Z32z.js} +4 -4
- package/dist/{audio-transcription-runner-CItniQDZ.js → audio-transcription-runner-DBkDgluo.js} +12 -12
- package/dist/{audio-transcription-runner-B2BdTEps.js → audio-transcription-runner-Gi_h5HEE.js} +1 -1
- package/dist/{audit-membership-runtime-w23FnNAN.js → audit-membership-runtime-DyLj-uhz.js} +4 -4
- package/dist/{auth-choice-7WVhiM9J.js → auth-choice-DQbCl-4F.js} +2 -2
- package/dist/{auth-choice-17U1cGPH.js → auth-choice-xwYK6txn.js} +2 -2
- package/dist/{banner-CJTrU-HC.js → banner-BxibaqUz.js} +1 -1
- package/dist/build-info.json +3 -3
- package/dist/bundled/boot-md/handler.js +51 -51
- package/dist/bundled/bootstrap-extra-files/handler.js +6 -6
- package/dist/bundled/command-logger/handler.js +2 -2
- package/dist/bundled/session-memory/handler.js +51 -51
- package/dist/canvas-host/a2ui/.bundle.hash +1 -1
- package/dist/{channel-activity-DPrXawu4.js → channel-activity-C5kTj83_.js} +3 -3
- package/dist/{channel-options-x47KAgAV.js → channel-options-BKsCYdHu.js} +1 -1
- package/dist/{channel-options-BM7IEY5X.js → channel-options-FdCN4cFo.js} +1 -1
- package/dist/{channel-web-B48pVgke.js → channel-web-BFUPrpIe.js} +1 -1
- package/dist/{channel-web-HM1q5FP_.js → channel-web-BkYtM8H5.js} +1 -1
- package/dist/{channels-cli-iPiD6449.js → channels-cli-BxhfVD-R.js} +7 -7
- package/dist/{channels-cli-y66ZCzYf.js → channels-cli-CZzGaGvG.js} +7 -7
- package/dist/{chrome-BQDCalPp.js → chrome-CAd6FQEn.js} +8 -8
- package/dist/{chrome-BFfAGQtd.js → chrome-pBkBuWci.js} +26 -26
- package/dist/cli/daemon-cli.js +1 -1
- package/dist/{cli-C-neGkM4.js → cli-GSev2Q95.js} +2 -2
- package/dist/{cli-BzpBs_KI.js → cli-bXiYaLre.js} +2 -2
- package/dist/{command-registry-BL80-JCV.js → command-registry-CvgCFxfY.js} +9 -9
- package/dist/{commands-registry-C2t2bcZ6.js → commands-registry-D6ZOTo1C.js} +4 -4
- package/dist/{completion-cli-CJ_L_gYr.js → completion-cli-BGM1V6EN.js} +2 -2
- package/dist/{completion-cli-Bi2s1mq8.js → completion-cli-OrgUDc2S.js} +1 -1
- package/dist/{config-cli-FhKX7MOY.js → config-cli-DbBvjvpS.js} +1 -1
- package/dist/{config-cli-BVpzzhoY.js → config-cli-uIP4r17f.js} +1 -1
- package/dist/{configure-ChiLGQo6.js → configure-BGvoAfbs.js} +6 -6
- package/dist/{configure-t9fm4x9H.js → configure-BOsTXjBw.js} +6 -6
- package/dist/{daemon-cli-DK8fQgAw.js → daemon-cli-CNi-QjEX.js} +1 -1
- package/dist/{daemon-cli-CO09shIt.js → daemon-cli-DN2TnjHQ.js} +1 -1
- package/dist/{deliver-aL8yOYS1.js → deliver-BJuiq0GS.js} +1 -1
- package/dist/{deliver-neVJ7AU9.js → deliver-DBXe-ZmL.js} +21 -21
- package/dist/{deliver-runtime-ChVR6sR3.js → deliver-runtime-DHKcNQzq.js} +3 -3
- package/dist/deliver-runtime-GlnBJNCj.js +36 -0
- package/dist/deps-send-discord.runtime-BRE0s2nz.js +26 -0
- package/dist/deps-send-imessage.runtime-DbIRBnmD.js +25 -0
- package/dist/deps-send-signal.runtime-B4h6X3o4.js +24 -0
- package/dist/deps-send-slack.runtime-BWXOnOxS.js +22 -0
- package/dist/deps-send-telegram.runtime-YvauJtgv.js +27 -0
- package/dist/{deps-send-whatsapp.runtime-WND2o1Mr.js → deps-send-whatsapp.runtime-BcxCalPD.js} +4 -4
- package/dist/deps-send-whatsapp.runtime-CslTuV47.js +60 -0
- package/dist/{deps-send-whatsapp.runtime-CZj97m5A.js → deps-send-whatsapp.runtime-DdxKewuy.js} +7 -7
- package/dist/{deps-send-whatsapp.runtime-DSbrPzxG.js → deps-send-whatsapp.runtime-Dl4ro-Df.js} +4 -4
- package/dist/{diagnostic-DoguEiWW.js → diagnostic-ySwZga6c.js} +2 -2
- package/dist/{doctor-completion-CKaQEebJ.js → doctor-completion-42wcUATu.js} +1 -1
- package/dist/{doctor-completion-Ctqsu6Y2.js → doctor-completion-D6RGDBD5.js} +1 -1
- package/dist/entry.js +2 -2
- package/dist/{plugin-sdk/errors-9oVz7reJ.js → errors-kkRuS2Cs.js} +1 -1
- package/dist/extensionAPI.js +6 -6
- package/dist/{fetch-DdiB5_OX.js → fetch-BLS7EMnx.js} +5 -5
- package/dist/{fetch-guard-CEV5qBHc.js → fetch-guard-D0fXNJls.js} +2 -2
- package/dist/{frontmatter-CjKtFduT.js → frontmatter-Cq1TcIQ2.js} +3 -3
- package/dist/{fs-safe-CwHbZdFH.js → fs-safe-BoB4X3GD.js} +4 -4
- package/dist/{gateway-cli-CSIyrhFg.js → gateway-cli-BbPfLLT6.js} +10 -10
- package/dist/{gateway-cli-xX1CTw9n.js → gateway-cli-CG3mshpO.js} +10 -10
- package/dist/{github-copilot-token-Cw3tAXM9.js → github-copilot-token-B5cPlwaz.js} +7 -7
- package/dist/{health-D9Pie2kF.js → health-CJgEuWuG.js} +1 -1
- package/dist/{health-D7mPTab_.js → health-dZqyhpdR.js} +1 -1
- package/dist/{hooks-cli-C6aI9HU7.js → hooks-cli-BD4Ww1dF.js} +3 -3
- package/dist/{hooks-cli-_1zdKcZA.js → hooks-cli-C0wWJOBW.js} +3 -3
- package/dist/{image-ecaECpjT.js → image-j_UomzVG.js} +6 -6
- package/dist/{image-ops-bdrMTILs.js → image-ops-CdgypS_g.js} +2 -2
- package/dist/image-runtime-BNh3IfMj.js +29 -0
- package/dist/{image-runtime-poRypm-b.js → image-runtime-SUtf9jqh.js} +3 -3
- package/dist/{image-DKZCmkET.js → image-sRW3RpTY.js} +1 -1
- package/dist/index.js +7 -7
- package/dist/{ir-D_GD01cg.js → ir-BvisJWXv.js} +8 -8
- package/dist/{legacy-names-B0wgIP0Q.js → legacy-names-aGJJuzM_.js} +1 -1
- package/dist/llm-slug-generator.js +51 -51
- package/dist/{logger-oGKcCLZ5.js → logger-CnTSBL7T.js} +7 -7
- package/dist/{login-gJWPqN66.js → login-BMeLPUiO.js} +5 -5
- package/dist/{login-qr-CgmlF7zK.js → login-qr-B2B67qqQ.js} +10 -10
- package/dist/{manager-CD69uguS.js → manager-DdxMYEDd.js} +13 -13
- package/dist/manager-runtime-BMygJEz3.js +18 -0
- package/dist/{model-selection-DMjtmGZP.js → model-selection-ag9BmVct.js} +43 -43
- package/dist/{models-BviRe-_7.js → models-BXT0s4KJ.js} +3 -3
- package/dist/{models-cli-BM5yo_mo.js → models-cli-BtLc9uPC.js} +3 -3
- package/dist/{models-cli-pvYVl1i-.js → models-cli-BwPFxWK2.js} +4 -4
- package/dist/{npm-resolution-DMM9Hopy.js → npm-resolution-DWpNPsBF.js} +1 -1
- package/dist/{npm-resolution-BTFMooVS.js → npm-resolution-Djbuzx6o.js} +1 -1
- package/dist/{onboard-DVuhj8ub.js → onboard-1KfKwvMR.js} +3 -3
- package/dist/{onboard-DTsgFKIa.js → onboard-ChxvwUze.js} +3 -3
- package/dist/{onboard-channels-W9UHiBQg.js → onboard-channels-CHrtFmhi.js} +2 -2
- package/dist/{onboard-channels-B1D9LqV_.js → onboard-channels-CXjnFvP1.js} +2 -2
- package/dist/{onboard-helpers-DVaF21TE.js → onboard-helpers-Bvpkyuwm.js} +6 -6
- package/dist/{onboard-helpers-PRhg7ZY5.js → onboard-helpers-ByttGRIZ.js} +6 -6
- package/dist/{onboard-remote-h-aHSDJ1.js → onboard-remote-Cfx2v9OI.js} +1 -1
- package/dist/{onboard-remote-DkIrV4Hx.js → onboard-remote-DxBaaS6o.js} +1 -1
- package/dist/{onboard-skills-fFSuiv21.js → onboard-skills-B7pHg1lN.js} +1 -1
- package/dist/{onboard-skills-fgrVmjJP.js → onboard-skills-PCnCZ6Od.js} +1 -1
- package/dist/{onboarding-D1nCnc_t.js → onboarding-D6kMb3yv.js} +7 -7
- package/dist/{onboarding-CYWs766P.js → onboarding-DuUMPrqA.js} +7 -7
- package/dist/{onboarding.finalize-Bcan6_vA.js → onboarding.finalize-KTOhO1-l.js} +7 -7
- package/dist/{onboarding.finalize-bAiXf9D6.js → onboarding.finalize-bphDUwZy.js} +6 -6
- package/dist/{onboarding.gateway-config-61E9xXYF.js → onboarding.gateway-config-DDdX0W74.js} +1 -1
- package/dist/{onboarding.gateway-config-Dyaqc_M7.js → onboarding.gateway-config-DNUJ0seU.js} +1 -1
- package/dist/{outbound-Cboz5UyH.js → outbound-DHDBvGLA.js} +6 -6
- package/dist/{outbound-attachment-PwEbEhAL.js → outbound-attachment-CqXiWbKN.js} +2 -2
- package/dist/{path-alias-guards-Cpsiv2KL.js → path-alias-guards-DORgbZ1w.js} +1 -1
- package/dist/{paths-CSdAWKDO.js → paths-DA5srn0U.js} +5 -5
- package/dist/{paths-CXClY8zC.js → paths-DSd911Oe.js} +4 -4
- package/dist/{pi-embedded-MktS4l8v.js → pi-embedded-BGz_qdCc.js} +24 -24
- package/dist/{pi-embedded-CkTlmTq8.js → pi-embedded-BP2UlUm_.js} +171 -171
- package/dist/{pi-embedded-helpers-DYWYzEOC.js → pi-embedded-helpers-BruaFB5l.js} +3 -3
- package/dist/{pi-embedded-helpers-Bfm_CvEb.js → pi-embedded-helpers-CmLnmKlb.js} +52 -52
- package/dist/{pi-model-discovery-DEps5Exd.js → pi-model-discovery-DAzuqPoG.js} +7 -7
- package/dist/pi-model-discovery-runtime-Dpu7Jm2L.js +11 -0
- package/dist/{pi-tools.before-tool-call.runtime-4wPdP7Br.js → pi-tools.before-tool-call.runtime-2Sp1jmlg.js} +9 -9
- package/dist/{plugin-registry-CvMvSI8O.js → plugin-registry-CLEhrKYA.js} +1 -1
- package/dist/{plugin-registry-DL2ClHLQ.js → plugin-registry-DtuxmgWx.js} +1 -1
- package/dist/plugin-sdk/{accounts-kr-Gz1hk.js → accounts-DghIDNk2.js} +2 -2
- package/dist/plugin-sdk/{accounts-CxUSDHsT.js → accounts-YTdQYQFr.js} +3 -3
- package/dist/plugin-sdk/{accounts-PSzw-z3S.js → accounts-h__dTrLK.js} +2 -2
- package/dist/plugin-sdk/{active-listener-BQNrTcR3.js → active-listener-_PRYjtJv.js} +2 -2
- package/dist/plugin-sdk/{api-key-rotation-Bhck7wki.js → api-key-rotation-mVDSAkKQ.js} +2 -2
- package/dist/plugin-sdk/{audio-preflight-_xgGaeho.js → audio-preflight-BZlQM-qX.js} +26 -26
- package/dist/plugin-sdk/{audio-transcription-runner-Dwc0Eh-B.js → audio-transcription-runner-CrYTX8py.js} +11 -11
- package/dist/plugin-sdk/{audit-membership-runtime-DHQDvH4u.js → audit-membership-runtime-Xl20kCBe.js} +2 -2
- package/dist/plugin-sdk/{channel-activity-XajEg_DL.js → channel-activity-gwxRn4wF.js} +3 -3
- package/dist/plugin-sdk/{channel-web-KtqCp4mz.js → channel-web-1WF-Nabe.js} +18 -18
- package/dist/plugin-sdk/{chrome-diV5m81I.js → chrome-BXbYwXRH.js} +6 -6
- package/dist/plugin-sdk/{commands-registry-DwZAJuut.js → commands-registry-0w-aZenK.js} +4 -4
- package/dist/plugin-sdk/{common-CqnO92P8.js → common-DBOCt6Yv.js} +2 -2
- package/dist/plugin-sdk/{config-DYbtdrsT.js → config-pRtEoVyZ.js} +7 -7
- package/dist/plugin-sdk/{deliver-DG_7Uagn.js → deliver-FjlJrtZk.js} +10 -10
- package/dist/plugin-sdk/deliver-runtime-DEzvpBW1.js +32 -0
- package/dist/plugin-sdk/deps-send-discord.runtime-Bhusa_Hi.js +23 -0
- package/dist/plugin-sdk/deps-send-imessage.runtime-bmakPm5f.js +22 -0
- package/dist/plugin-sdk/deps-send-signal.runtime-n00sfFto.js +21 -0
- package/dist/plugin-sdk/deps-send-slack.runtime-BvM3Z-Mr.js +19 -0
- package/dist/plugin-sdk/deps-send-telegram.runtime-CPuMkcmo.js +24 -0
- package/dist/plugin-sdk/deps-send-whatsapp.runtime-BzO6S-KX.js +57 -0
- package/dist/plugin-sdk/{diagnostic-CT7v_kM2.js → diagnostic-Dt2i3afe.js} +2 -2
- package/dist/plugin-sdk/{errors-B8oJXuCF.js → errors-CgRPdp3o.js} +1 -1
- package/dist/plugin-sdk/{fetch-guard-Or5BCq0E.js → fetch-guard-DyPZh8r2.js} +2 -2
- package/dist/plugin-sdk/{fs-safe-DFbwq9CS.js → fs-safe-DqCO1D4C.js} +3 -3
- package/dist/plugin-sdk/{image-rycGCqJO.js → image-CQ9TZ9vq.js} +6 -6
- package/dist/plugin-sdk/{image-ops-CMsocOob.js → image-ops-sw0uZ0GN.js} +2 -2
- package/dist/plugin-sdk/image-runtime-17_mTqsy.js +25 -0
- package/dist/plugin-sdk/index.js +50 -50
- package/dist/plugin-sdk/{ir-DihI2SIz.js → ir-BVZ5kUMb.js} +7 -7
- package/dist/plugin-sdk/{local-roots-1xVosTZ4.js → local-roots-fO3ZgW3G.js} +4 -4
- package/dist/plugin-sdk/{logger-Bg4vIUJn.js → logger-DIb2cGHp.js} +2 -2
- package/dist/plugin-sdk/{login-YhFrVUWo.js → login-Dg5cxB_3.js} +4 -4
- package/dist/plugin-sdk/{login-qr-SpUTuwYv.js → login-qr-C3Vn30cq.js} +5 -5
- package/dist/plugin-sdk/{manager-DrzOPeMD.js → manager-BR-TwWTH.js} +8 -8
- package/dist/plugin-sdk/manager-runtime-CvI9wF8N.js +15 -0
- package/dist/plugin-sdk/{outbound-Cc4cUn9K.js → outbound-1a3Z_QJ2.js} +5 -5
- package/dist/plugin-sdk/{outbound-attachment-Dtp3hQgc.js → outbound-attachment-BTQjD4YE.js} +2 -2
- package/dist/plugin-sdk/{path-alias-guards-DA0MhfkG.js → path-alias-guards-TnxupPQC.js} +1 -1
- package/dist/plugin-sdk/{paths-CP67O8eN.js → paths-B7_75Pdr.js} +1 -1
- package/dist/plugin-sdk/{pi-embedded-helpers-BDJ_4Plh.js → pi-embedded-helpers-DZRNadD8.js} +16 -16
- package/dist/plugin-sdk/{pi-model-discovery-Mk0GTDJl.js → pi-model-discovery-DGh6xekX.js} +1 -1
- package/dist/plugin-sdk/pi-model-discovery-runtime-DjjBdPYt.js +8 -0
- package/dist/plugin-sdk/{pi-tools.before-tool-call.runtime-DV72wTDb.js → pi-tools.before-tool-call.runtime-BZ9XgG_x.js} +4 -4
- package/dist/plugin-sdk/{plugins-DSs2-fnK.js → plugins-B8pWVYug.js} +4 -4
- package/dist/plugin-sdk/{proxy-env-Ib4-LUh-.js → proxy-env-BOlkiW1-.js} +1 -1
- package/dist/plugin-sdk/{proxy-fetch-ZPEvp58f.js → proxy-fetch-Dt5BedH8.js} +1 -1
- package/dist/plugin-sdk/{pw-ai-DIx2wpkY.js → pw-ai-C17A1o4w.js} +9 -9
- package/dist/plugin-sdk/{qmd-manager-Ov9ElEfG.js → qmd-manager-Bei6TaFq.js} +7 -7
- package/dist/plugin-sdk/{query-expansion-CzjwW461.js → query-expansion-POz2za8a.js} +4 -4
- package/dist/plugin-sdk/{redact-BoNEjbpF.js → redact-9WsNyb7S.js} +1 -1
- package/dist/plugin-sdk/{reply-CWWUd_JS.js → reply-BFbijn6_.js} +73 -73
- package/dist/plugin-sdk/{resolve-outbound-target-BOkvxZtM.js → resolve-outbound-target-B9iFEh0y.js} +2 -2
- package/dist/plugin-sdk/{run-with-concurrency-kVooFCVo.js → run-with-concurrency-DmTrN5JG.js} +1 -1
- package/dist/plugin-sdk/runtime-whatsapp-login.runtime-DzhkSmLi.js +10 -0
- package/dist/plugin-sdk/runtime-whatsapp-outbound.runtime-DyILWezU.js +19 -0
- package/dist/plugin-sdk/{send-BP1fSEBR.js → send-BGZo6HW1.js} +5 -5
- package/dist/plugin-sdk/{send-D9CSOGul.js → send-BisREGBZ.js} +6 -6
- package/dist/plugin-sdk/{send-BeLBlAsQ.js → send-BqkUDZed.js} +13 -13
- package/dist/plugin-sdk/{send-DLKxJJYV.js → send-D6_nNvi0.js} +8 -8
- package/dist/plugin-sdk/{send-XZ6IXCtL.js → send-Dj7XEcZN.js} +7 -7
- package/dist/plugin-sdk/{session-DtLUYWvY.js → session-D4KDs7Hq.js} +3 -3
- package/dist/plugin-sdk/{skill-commands-Bv7EZypt.js → skill-commands-D_xeseiI.js} +4 -4
- package/dist/plugin-sdk/{skills-BzXN4uev.js → skills-Bs2b3JfV.js} +6 -6
- package/dist/plugin-sdk/slash-commands.runtime-CUb5sqqf.js +13 -0
- package/dist/plugin-sdk/slash-dispatch.runtime-DCB6bGjB.js +52 -0
- package/dist/plugin-sdk/slash-skill-commands.runtime-BqEweE4K.js +16 -0
- package/dist/plugin-sdk/{store-DnJhFFW5.js → store-B7ESm9_L.js} +2 -2
- package/dist/plugin-sdk/subagent-registry-runtime-CCUW4SbM.js +52 -0
- package/dist/plugin-sdk/{tables-CpmqssLF.js → tables-1vhBJPK_.js} +1 -1
- package/dist/plugin-sdk/{thinking-1UCPuD9d.js → thinking-DjaClmzi.js} +7 -7
- package/dist/plugin-sdk/{tokens-DAL_5WHL.js → tokens-CLE20fRI.js} +1 -1
- package/dist/plugin-sdk/{tool-images-RX4QTMnt.js → tool-images-B95xcwiR.js} +2 -2
- package/dist/plugin-sdk/web-DeRmHQ4_.js +56 -0
- package/dist/plugin-sdk/{whatsapp-actions-BF6ih4Gi.js → whatsapp-actions-BYpcWkTN.js} +17 -17
- package/dist/plugin-sdk/whatsapp.js +50 -50
- package/dist/{plugins-CmdmAU0K.js → plugins-DXkm70nK.js} +11 -11
- package/dist/{plugins-cli-DBtLtIsQ.js → plugins-cli-Cs3UUJew.js} +3 -3
- package/dist/{plugins-cli-BMPvpwSo.js → plugins-cli-KPz6APX0.js} +3 -3
- package/dist/{program-context-KSeqVkRM.js → program-context-J_FyEsaS.js} +18 -18
- package/dist/{program-C6sTShRB.js → program-xNEHPhT8.js} +8 -8
- package/dist/{prompt-select-styled-Ba5fC0g1.js → prompt-select-styled-B1LjjgQ0.js} +5 -5
- package/dist/{prompt-select-styled-DFhJPiqx.js → prompt-select-styled-BRiogP_P.js} +5 -5
- package/dist/{provider-auth-helpers-S2rdI85T.js → provider-auth-helpers-CxUWqt95.js} +1 -1
- package/dist/{provider-auth-helpers-BPvZ8xkJ.js → provider-auth-helpers-hhFVhZdv.js} +1 -1
- package/dist/{proxy-env-QUJz9VEJ.js → proxy-env-D75CWSOo.js} +1 -1
- package/dist/{proxy-fetch-C2v-Utgg.js → proxy-fetch-lH6RsRTE.js} +1 -1
- package/dist/{push-apns-BQEPMPtG.js → push-apns-BBkpZyNR.js} +1 -1
- package/dist/{push-apns-CGibQhps.js → push-apns-BQjV_93G.js} +1 -1
- package/dist/{pw-ai-SYjuzbV6.js → pw-ai-7kHgUGj0.js} +14 -14
- package/dist/{pw-ai-zFPBSxaL.js → pw-ai-BmGrTicP.js} +1 -1
- package/dist/{qmd-manager-CO-shcLU.js → qmd-manager-BN0siR2Z.js} +10 -10
- package/dist/{query-expansion-DlQOkf-g.js → query-expansion-Dzxt6kXo.js} +6 -6
- package/dist/{redact-NmPEVjIo.js → redact-DvzicBMu.js} +1 -1
- package/dist/{register.agent-DP_2xCaO.js → register.agent-DYq06QHS.js} +8 -8
- package/dist/{register.agent-BwhWwpRX.js → register.agent-reU63wQ5.js} +7 -7
- package/dist/{register.configure-D8TE-yQn.js → register.configure-C4p9ad2q.js} +10 -10
- package/dist/{register.configure-C5i661J4.js → register.configure-DezZ4Q1p.js} +10 -10
- package/dist/{register.maintenance-BS2i3S5V.js → register.maintenance-CTvFmkAm.js} +9 -9
- package/dist/{register.maintenance-BA4UOg2_.js → register.maintenance-CzMKTC2a.js} +8 -8
- package/dist/{register.message-oFI2Mzrd.js → register.message-BmsovYS6.js} +3 -3
- package/dist/{register.message-D9hVI5b6.js → register.message-Bp4SDXWk.js} +3 -3
- package/dist/{register.onboard-D6wqijOl.js → register.onboard-C39xhpv1.js} +3 -3
- package/dist/{register.onboard-OaKr3SnU.js → register.onboard-DZt2kSAg.js} +3 -3
- package/dist/{register.setup-DsK_7zih.js → register.setup-04L_8wfA.js} +3 -3
- package/dist/{register.setup-Dygx-glo.js → register.setup-DWctFmOd.js} +3 -3
- package/dist/{register.status-health-sessions-C6VfEhho.js → register.status-health-sessions-CBPZoj51.js} +4 -4
- package/dist/{register.status-health-sessions-BmWcbWPR.js → register.status-health-sessions-N6SFc-UY.js} +4 -4
- package/dist/{register.subclis-CLf32krW.js → register.subclis-C3TphbCF.js} +10 -10
- package/dist/{reply-ZWkzBiSb.js → reply-CB1p166g.js} +5 -5
- package/dist/{run-main-asKkGUqy.js → run-main-7tknx04F.js} +15 -15
- package/dist/{run-with-concurrency-FczpX8ng.js → run-with-concurrency-BFR3ReeF.js} +4 -4
- package/dist/runtime-whatsapp-login.runtime-DSR-m0FW.js +13 -0
- package/dist/runtime-whatsapp-outbound.runtime-Blywd_bv.js +22 -0
- package/dist/{send-oS3t6gE6.js → send-C98RfcAb.js} +5 -5
- package/dist/{send-DX_O1OHH.js → send-Co5Bqwuo.js} +6 -6
- package/dist/{send-BNsV-D2Y.js → send-DjL7KlMV.js} +8 -8
- package/dist/{send-C-jb8X9I.js → send-Do7hKDL9.js} +7 -7
- package/dist/{send-D-Rnbdzz.js → send-bW7jDv8D.js} +26 -26
- package/dist/{server-node-events-DV2yeAp-.js → server-node-events-Ctjzvlem.js} +3 -3
- package/dist/{server-node-events-BHv7a_ll.js → server-node-events-DrCKK0J4.js} +3 -3
- package/dist/{session-DRyURckG.js → session-QSn69XeJ.js} +8 -8
- package/dist/{skill-commands-BrlAf_CG.js → skill-commands-Bi_jchJn.js} +9 -9
- package/dist/{skills-DWrRJwa-.js → skills-CTV78w4q.js} +22 -22
- package/dist/slash-commands.runtime-63MUmCBt.js +16 -0
- package/dist/{slash-dispatch.runtime-DkcAYuyK.js → slash-dispatch.runtime-BL3qA1O3.js} +6 -6
- package/dist/{slash-dispatch.runtime-BJOuQOeN.js → slash-dispatch.runtime-DRGqAgwa.js} +2 -2
- package/dist/slash-dispatch.runtime-Dh2L_3Tg.js +56 -0
- package/dist/{slash-dispatch.runtime-CEAbkOCI.js → slash-dispatch.runtime-uqWhoI6q.js} +2 -2
- package/dist/slash-skill-commands.runtime-B3MSSAQ-.js +20 -0
- package/dist/{status-Ck8-aQIF.js → status-BkfSGlOi.js} +3 -3
- package/dist/{status-CMhW6nGs.js → status-DdW571-j.js} +3 -3
- package/dist/{store-BFNH5fXG.js → store-B89Hj8Ub.js} +2 -2
- package/dist/{subagent-registry-DQpeidFk.js → subagent-registry-DGrfQVN3.js} +5 -5
- package/dist/subagent-registry-runtime-BRNDawlJ.js +56 -0
- package/dist/{subagent-registry-runtime-C-jjppV6.js → subagent-registry-runtime-DatTO2LD.js} +2 -2
- package/dist/{subagent-registry-runtime-tRRyFZL8.js → subagent-registry-runtime-Dsrz3yIh.js} +2 -2
- package/dist/{subagent-registry-runtime-BlRAnw80.js → subagent-registry-runtime-MtjBCcgn.js} +6 -6
- package/dist/{subsystem-BaLYRf7D.js → subsystem-6v7sWnAD.js} +14 -14
- package/dist/{tables-CnlmCLb3.js → tables-DGHzaXQz.js} +1 -1
- package/dist/{target-errors-Df1wB-I7.js → target-errors-CweAa7L9.js} +2 -2
- package/dist/{thinking-CTpcVnlx.js → thinking-SdNGqtJE.js} +7 -7
- package/dist/{tokens-D2XhLqIz.js → tokens-DfbMVF9y.js} +1 -1
- package/dist/{tool-images-CElPu2en.js → tool-images-8BKrL7Bn.js} +2 -2
- package/dist/{update-cli-5KzuA6pa.js → update-cli-0UiUaT3q.js} +10 -10
- package/dist/{update-cli-CEghYBNP.js → update-cli-C-uyQcFS.js} +9 -9
- package/dist/{update-runner-C5XgCwj2.js → update-runner-CT9YRLtn.js} +1 -1
- package/dist/{update-runner-C0q8aGFd.js → update-runner-CqVLeGYA.js} +1 -1
- package/dist/{web-DBm_uXOl.js → web-B2qXyOb9.js} +3 -3
- package/dist/{web-DdrUn13G.js → web-B7kbCskR.js} +55 -55
- package/dist/{web-DddJa7ZT.js → web-D1ZoRVB0.js} +6 -6
- package/dist/{web-O2WkG3cH.js → web-FqoNMI-k.js} +3 -3
- package/dist/{whatsapp-actions-DPszRJ8b.js → whatsapp-actions-BDkbnZVH.js} +21 -21
- package/dist/{workspace-TqfVSQuO.js → workspace-kVMIaBrV.js} +20 -20
- package/package.json +1 -1
- package/dist/api-key-rotation-Dzvqp3Dc.js +0 -181
- package/dist/deliver-runtime-BdUlqV9E.js +0 -36
- package/dist/deps-send-discord.runtime-R8jUd_2I.js +0 -26
- package/dist/deps-send-imessage.runtime-Die0aWtU.js +0 -25
- package/dist/deps-send-signal.runtime-Biux_4v4.js +0 -24
- package/dist/deps-send-slack.runtime-CkUST2Ky.js +0 -22
- package/dist/deps-send-telegram.runtime-CIN5ILBe.js +0 -27
- package/dist/deps-send-whatsapp.runtime-DUff9bWS.js +0 -60
- package/dist/errors-DfgAh2Ml.js +0 -54
- package/dist/image-runtime-irHu11-U.js +0 -29
- package/dist/manager-runtime-BISxj7HK.js +0 -18
- package/dist/pi-model-discovery-runtime-bzJViQLK.js +0 -11
- package/dist/plugin-sdk/accounts-BNuRM3rG.js +0 -288
- package/dist/plugin-sdk/accounts-CGTYP7Rh.js +0 -46
- package/dist/plugin-sdk/accounts-CcS9IAhD.js +0 -35
- package/dist/plugin-sdk/active-listener-CTsLn1AX.js +0 -50
- package/dist/plugin-sdk/audio-preflight-CRGLqp-g.js +0 -69
- package/dist/plugin-sdk/audio-transcription-runner-RXsskMMk.js +0 -2176
- package/dist/plugin-sdk/audit-membership-runtime-B9b-zRwg.js +0 -58
- package/dist/plugin-sdk/channel-activity-gPvD1D7S.js +0 -94
- package/dist/plugin-sdk/channel-web-LGl1zPJt.js +0 -2256
- package/dist/plugin-sdk/chrome-9Y_LcUg1.js +0 -2415
- package/dist/plugin-sdk/commands-registry-CcdEPxVg.js +0 -1125
- package/dist/plugin-sdk/config-CrQ5bCrw.js +0 -17912
- package/dist/plugin-sdk/deliver-D3xr5AkB.js +0 -1694
- package/dist/plugin-sdk/deliver-runtime-B79ZQu69.js +0 -32
- package/dist/plugin-sdk/deliver-runtime-BdTC7uKE.js +0 -32
- package/dist/plugin-sdk/deps-send-discord.runtime-BOQZIqC8.js +0 -23
- package/dist/plugin-sdk/deps-send-discord.runtime-CObCNMt3.js +0 -23
- package/dist/plugin-sdk/deps-send-imessage.runtime-CuHOc9Ka.js +0 -22
- package/dist/plugin-sdk/deps-send-imessage.runtime-DlWgi2DH.js +0 -22
- package/dist/plugin-sdk/deps-send-signal.runtime-Cz7FT8J8.js +0 -21
- package/dist/plugin-sdk/deps-send-signal.runtime-iPynghkE.js +0 -21
- package/dist/plugin-sdk/deps-send-slack.runtime-D4vDoRsg.js +0 -19
- package/dist/plugin-sdk/deps-send-slack.runtime-DNTbE5jS.js +0 -19
- package/dist/plugin-sdk/deps-send-telegram.runtime-7CR-xtCF.js +0 -24
- package/dist/plugin-sdk/deps-send-telegram.runtime-DjTVED_m.js +0 -24
- package/dist/plugin-sdk/deps-send-whatsapp.runtime-CRWOIKRC.js +0 -57
- package/dist/plugin-sdk/deps-send-whatsapp.runtime-bUi8kghi.js +0 -57
- package/dist/plugin-sdk/diagnostic-BXkLYs_9.js +0 -319
- package/dist/plugin-sdk/fetch-guard-C55uvn27.js +0 -156
- package/dist/plugin-sdk/fs-safe-Dqmpk-Fr.js +0 -352
- package/dist/plugin-sdk/image-3xW7IJdq.js +0 -2310
- package/dist/plugin-sdk/image-ops-BjK2qZZn.js +0 -584
- package/dist/plugin-sdk/image-runtime-CZZJJqcW.js +0 -25
- package/dist/plugin-sdk/image-runtime-Cjz368oj.js +0 -25
- package/dist/plugin-sdk/ir-CS7uuQhN.js +0 -1296
- package/dist/plugin-sdk/local-roots-DmOKwiNW.js +0 -186
- package/dist/plugin-sdk/logger-DDdrdbDu.js +0 -1163
- package/dist/plugin-sdk/login-BSEeU27Y.js +0 -57
- package/dist/plugin-sdk/login-qr-BwWJsDSj.js +0 -320
- package/dist/plugin-sdk/manager-DiXPCubI.js +0 -3917
- package/dist/plugin-sdk/manager-runtime-CF55pBNe.js +0 -15
- package/dist/plugin-sdk/manager-runtime-Ct0m9UJC.js +0 -15
- package/dist/plugin-sdk/outbound-attachment-BoFx05zw.js +0 -19
- package/dist/plugin-sdk/outbound-cpqK1GFe.js +0 -212
- package/dist/plugin-sdk/path-alias-guards-gBhrAn14.js +0 -43
- package/dist/plugin-sdk/paths-C6W4VHoa.js +0 -166
- package/dist/plugin-sdk/pi-embedded-helpers-C-B9B6Sp.js +0 -9627
- package/dist/plugin-sdk/pi-model-discovery-BGEeoPzN.js +0 -134
- package/dist/plugin-sdk/pi-model-discovery-runtime-BHZ_Htob.js +0 -8
- package/dist/plugin-sdk/pi-model-discovery-runtime-BrwtJHPU.js +0 -8
- package/dist/plugin-sdk/pi-tools.before-tool-call.runtime-ByN_xThw.js +0 -354
- package/dist/plugin-sdk/plugins-D5cdn70e.js +0 -864
- package/dist/plugin-sdk/proxy-fetch-Cf3IUSDw.js +0 -38
- package/dist/plugin-sdk/pw-ai-C_QOIuin.js +0 -1938
- package/dist/plugin-sdk/qmd-manager-6bozlfFg.js +0 -1448
- package/dist/plugin-sdk/query-expansion-eeVz_aEm.js +0 -1011
- package/dist/plugin-sdk/redact-DfACyt0X.js +0 -319
- package/dist/plugin-sdk/reply-CQUX_haM.js +0 -98828
- package/dist/plugin-sdk/resolve-outbound-target-Dbz0O8cR.js +0 -40
- package/dist/plugin-sdk/run-with-concurrency-5DMu9szx.js +0 -1994
- package/dist/plugin-sdk/runtime-whatsapp-login.runtime-DitS0I1z.js +0 -10
- package/dist/plugin-sdk/runtime-whatsapp-login.runtime-OthrtsLL.js +0 -10
- package/dist/plugin-sdk/runtime-whatsapp-outbound.runtime-CYCr6A3v.js +0 -19
- package/dist/plugin-sdk/runtime-whatsapp-outbound.runtime-Q2HL0zL3.js +0 -19
- package/dist/plugin-sdk/send-BACEu1Un.js +0 -414
- package/dist/plugin-sdk/send-BU4OoR7u.js +0 -2587
- package/dist/plugin-sdk/send-DbxOJ_BC.js +0 -3135
- package/dist/plugin-sdk/send-n932vjT5.js +0 -540
- package/dist/plugin-sdk/send-uCPS53j8.js +0 -503
- package/dist/plugin-sdk/session-DenDKR_-.js +0 -169
- package/dist/plugin-sdk/skill-commands-BK1KDKmS.js +0 -342
- package/dist/plugin-sdk/skills-D4am-zkO.js +0 -1428
- package/dist/plugin-sdk/slash-commands.runtime-Bx1K1iqP.js +0 -13
- package/dist/plugin-sdk/slash-commands.runtime-DWfFqMZw.js +0 -13
- package/dist/plugin-sdk/slash-dispatch.runtime-DVn338JI.js +0 -52
- package/dist/plugin-sdk/slash-dispatch.runtime-pnWH5AjM.js +0 -52
- package/dist/plugin-sdk/slash-skill-commands.runtime-Dbi_YzPO.js +0 -16
- package/dist/plugin-sdk/slash-skill-commands.runtime-DxvNWv_E.js +0 -16
- package/dist/plugin-sdk/ssrf-2WBi1Tzx.js +0 -202
- package/dist/plugin-sdk/store-BKDMuvyn.js +0 -81
- package/dist/plugin-sdk/subagent-registry-runtime-FhP0l-Rw.js +0 -52
- package/dist/plugin-sdk/subagent-registry-runtime-hH9ADku1.js +0 -52
- package/dist/plugin-sdk/tables-CrDYcv_b.js +0 -55
- package/dist/plugin-sdk/target-errors-aOwE-MIU.js +0 -195
- package/dist/plugin-sdk/thinking-D41FMh9T.js +0 -1206
- package/dist/plugin-sdk/tokens-CTIYTLWu.js +0 -52
- package/dist/plugin-sdk/tool-images-CWc54lpI.js +0 -274
- package/dist/plugin-sdk/web-AtEy-48y.js +0 -56
- package/dist/plugin-sdk/web-DjKONHqF.js +0 -56
- package/dist/plugin-sdk/whatsapp-actions-DEZcm_CZ.js +0 -80
- package/dist/runtime-whatsapp-login.runtime-BqOsE5As.js +0 -13
- package/dist/runtime-whatsapp-outbound.runtime-D5S6mxFT.js +0 -22
- package/dist/slash-commands.runtime-JqCsKeu2.js +0 -16
- package/dist/slash-dispatch.runtime-h9I6EDYB.js +0 -56
- package/dist/slash-skill-commands.runtime-C0QZlkpu.js +0 -20
- package/dist/subagent-registry-runtime-BxvwRp_3.js +0 -56
|
@@ -1,3917 +0,0 @@
|
|
|
1
|
-
import { a as resolveAgentDir, c as resolveAgentWorkspaceDir, i as resolveAgentConfig } from "./run-with-concurrency-5DMu9szx.js";
|
|
2
|
-
import { c as resolveStateDir } from "./paths-8xF5kDne.js";
|
|
3
|
-
import { $n as requireApiKey, Lr as normalizeOptionalSecretInput, Ur as normalizeResolvedSecretInputString, Wn as retryAsync, ei as isTruthyEnvValue, er as resolveApiKeyForProvider, tr as resolveEnvApiKey } from "./config-CrQ5bCrw.js";
|
|
4
|
-
import { E as resolveUserPath, N as truncateUtf16Safe, a as createSubsystemLogger, h as clampNumber, m as clampInt } from "./logger-DDdrdbDu.js";
|
|
5
|
-
import { o as resolveSessionTranscriptsDirForAgent } from "./paths-C6W4VHoa.js";
|
|
6
|
-
import { n as onSessionTranscriptUpdate } from "./transcript-events-C2lcxc6-.js";
|
|
7
|
-
import { r as formatErrorMessage } from "./errors-B8oJXuCF.js";
|
|
8
|
-
import { t as fetchWithSsrFGuard } from "./fetch-guard-C55uvn27.js";
|
|
9
|
-
import { n as executeWithApiKeyRotation, r as parseGeminiAuth, t as collectProviderApiKeysForExecution } from "./api-key-rotation-Bhck7wki.js";
|
|
10
|
-
import { _ as isFileMissingError, a as sessionPathForFile, c as cosineSimilarity, d as isMemoryPath, f as listMemoryFiles, g as runWithConcurrency, h as remapChunkLines, i as listSessionFilesForAgent, l as ensureDir, m as parseEmbedding, n as requireNodeSqlite, o as buildFileEntry, p as normalizeExtraMemoryPaths, r as buildSessionEntry, s as chunkMarkdown, t as extractKeywords, u as hashText, v as statRegularFile } from "./query-expansion-eeVz_aEm.js";
|
|
11
|
-
import fs from "node:fs";
|
|
12
|
-
import path from "node:path";
|
|
13
|
-
import os from "node:os";
|
|
14
|
-
import fs$1 from "node:fs/promises";
|
|
15
|
-
import { randomUUID } from "node:crypto";
|
|
16
|
-
import chokidar from "chokidar";
|
|
17
|
-
import { Readable } from "node:stream";
|
|
18
|
-
import { createInterface } from "node:readline";
|
|
19
|
-
|
|
20
|
-
//#region src/agents/memory-search.ts
|
|
21
|
-
const DEFAULT_OPENAI_MODEL = "text-embedding-3-small";
|
|
22
|
-
const DEFAULT_GEMINI_MODEL = "gemini-embedding-001";
|
|
23
|
-
const DEFAULT_VOYAGE_MODEL = "voyage-4-large";
|
|
24
|
-
const DEFAULT_MISTRAL_MODEL = "mistral-embed";
|
|
25
|
-
const DEFAULT_OLLAMA_MODEL = "nomic-embed-text";
|
|
26
|
-
const DEFAULT_CHUNK_TOKENS = 400;
|
|
27
|
-
const DEFAULT_CHUNK_OVERLAP = 80;
|
|
28
|
-
const DEFAULT_WATCH_DEBOUNCE_MS = 1500;
|
|
29
|
-
const DEFAULT_SESSION_DELTA_BYTES = 1e5;
|
|
30
|
-
const DEFAULT_SESSION_DELTA_MESSAGES = 50;
|
|
31
|
-
const DEFAULT_MAX_RESULTS = 6;
|
|
32
|
-
const DEFAULT_MIN_SCORE = .35;
|
|
33
|
-
const DEFAULT_HYBRID_ENABLED = true;
|
|
34
|
-
const DEFAULT_HYBRID_VECTOR_WEIGHT = .7;
|
|
35
|
-
const DEFAULT_HYBRID_TEXT_WEIGHT = .3;
|
|
36
|
-
const DEFAULT_HYBRID_CANDIDATE_MULTIPLIER = 4;
|
|
37
|
-
const DEFAULT_MMR_ENABLED = false;
|
|
38
|
-
const DEFAULT_MMR_LAMBDA = .7;
|
|
39
|
-
const DEFAULT_TEMPORAL_DECAY_ENABLED = false;
|
|
40
|
-
const DEFAULT_TEMPORAL_DECAY_HALF_LIFE_DAYS = 30;
|
|
41
|
-
const DEFAULT_CACHE_ENABLED = true;
|
|
42
|
-
const DEFAULT_SOURCES = ["memory"];
|
|
43
|
-
function normalizeSources(sources, sessionMemoryEnabled) {
|
|
44
|
-
const normalized = /* @__PURE__ */ new Set();
|
|
45
|
-
const input = sources?.length ? sources : DEFAULT_SOURCES;
|
|
46
|
-
for (const source of input) {
|
|
47
|
-
if (source === "memory") normalized.add("memory");
|
|
48
|
-
if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions");
|
|
49
|
-
}
|
|
50
|
-
if (normalized.size === 0) normalized.add("memory");
|
|
51
|
-
return Array.from(normalized);
|
|
52
|
-
}
|
|
53
|
-
function resolveStorePath(agentId, raw) {
|
|
54
|
-
const stateDir = resolveStateDir(process.env, os.homedir);
|
|
55
|
-
const fallback = path.join(stateDir, "memory", `${agentId}.sqlite`);
|
|
56
|
-
if (!raw) return fallback;
|
|
57
|
-
return resolveUserPath(raw.includes("{agentId}") ? raw.replaceAll("{agentId}", agentId) : raw);
|
|
58
|
-
}
|
|
59
|
-
function mergeConfig(defaults, overrides, agentId) {
|
|
60
|
-
const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
|
|
61
|
-
const sessionMemory = overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false;
|
|
62
|
-
const provider = overrides?.provider ?? defaults?.provider ?? "auto";
|
|
63
|
-
const defaultRemote = defaults?.remote;
|
|
64
|
-
const overrideRemote = overrides?.remote;
|
|
65
|
-
const includeRemote = Boolean(overrideRemote?.baseUrl || overrideRemote?.apiKey || overrideRemote?.headers || defaultRemote?.baseUrl || defaultRemote?.apiKey || defaultRemote?.headers) || provider === "openai" || provider === "gemini" || provider === "voyage" || provider === "mistral" || provider === "ollama" || provider === "auto";
|
|
66
|
-
const batch = {
|
|
67
|
-
enabled: overrideRemote?.batch?.enabled ?? defaultRemote?.batch?.enabled ?? false,
|
|
68
|
-
wait: overrideRemote?.batch?.wait ?? defaultRemote?.batch?.wait ?? true,
|
|
69
|
-
concurrency: Math.max(1, overrideRemote?.batch?.concurrency ?? defaultRemote?.batch?.concurrency ?? 2),
|
|
70
|
-
pollIntervalMs: overrideRemote?.batch?.pollIntervalMs ?? defaultRemote?.batch?.pollIntervalMs ?? 2e3,
|
|
71
|
-
timeoutMinutes: overrideRemote?.batch?.timeoutMinutes ?? defaultRemote?.batch?.timeoutMinutes ?? 60
|
|
72
|
-
};
|
|
73
|
-
const remote = includeRemote ? {
|
|
74
|
-
baseUrl: overrideRemote?.baseUrl ?? defaultRemote?.baseUrl,
|
|
75
|
-
apiKey: overrideRemote?.apiKey ?? defaultRemote?.apiKey,
|
|
76
|
-
headers: overrideRemote?.headers ?? defaultRemote?.headers,
|
|
77
|
-
batch
|
|
78
|
-
} : void 0;
|
|
79
|
-
const fallback = overrides?.fallback ?? defaults?.fallback ?? "none";
|
|
80
|
-
const modelDefault = provider === "gemini" ? DEFAULT_GEMINI_MODEL : provider === "openai" ? DEFAULT_OPENAI_MODEL : provider === "voyage" ? DEFAULT_VOYAGE_MODEL : provider === "mistral" ? DEFAULT_MISTRAL_MODEL : provider === "ollama" ? DEFAULT_OLLAMA_MODEL : void 0;
|
|
81
|
-
const model = overrides?.model ?? defaults?.model ?? modelDefault ?? "";
|
|
82
|
-
const local = {
|
|
83
|
-
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
|
|
84
|
-
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir
|
|
85
|
-
};
|
|
86
|
-
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
|
|
87
|
-
const rawPaths = [...defaults?.extraPaths ?? [], ...overrides?.extraPaths ?? []].map((value) => value.trim()).filter(Boolean);
|
|
88
|
-
const extraPaths = Array.from(new Set(rawPaths));
|
|
89
|
-
const vector = {
|
|
90
|
-
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
|
|
91
|
-
extensionPath: overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath
|
|
92
|
-
};
|
|
93
|
-
const store = {
|
|
94
|
-
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
|
|
95
|
-
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
|
|
96
|
-
vector
|
|
97
|
-
};
|
|
98
|
-
const chunking = {
|
|
99
|
-
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,
|
|
100
|
-
overlap: overrides?.chunking?.overlap ?? defaults?.chunking?.overlap ?? DEFAULT_CHUNK_OVERLAP
|
|
101
|
-
};
|
|
102
|
-
const sync = {
|
|
103
|
-
onSessionStart: overrides?.sync?.onSessionStart ?? defaults?.sync?.onSessionStart ?? true,
|
|
104
|
-
onSearch: overrides?.sync?.onSearch ?? defaults?.sync?.onSearch ?? true,
|
|
105
|
-
watch: overrides?.sync?.watch ?? defaults?.sync?.watch ?? true,
|
|
106
|
-
watchDebounceMs: overrides?.sync?.watchDebounceMs ?? defaults?.sync?.watchDebounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS,
|
|
107
|
-
intervalMinutes: overrides?.sync?.intervalMinutes ?? defaults?.sync?.intervalMinutes ?? 0,
|
|
108
|
-
sessions: {
|
|
109
|
-
deltaBytes: overrides?.sync?.sessions?.deltaBytes ?? defaults?.sync?.sessions?.deltaBytes ?? DEFAULT_SESSION_DELTA_BYTES,
|
|
110
|
-
deltaMessages: overrides?.sync?.sessions?.deltaMessages ?? defaults?.sync?.sessions?.deltaMessages ?? DEFAULT_SESSION_DELTA_MESSAGES
|
|
111
|
-
}
|
|
112
|
-
};
|
|
113
|
-
const query = {
|
|
114
|
-
maxResults: overrides?.query?.maxResults ?? defaults?.query?.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
115
|
-
minScore: overrides?.query?.minScore ?? defaults?.query?.minScore ?? DEFAULT_MIN_SCORE
|
|
116
|
-
};
|
|
117
|
-
const hybrid = {
|
|
118
|
-
enabled: overrides?.query?.hybrid?.enabled ?? defaults?.query?.hybrid?.enabled ?? DEFAULT_HYBRID_ENABLED,
|
|
119
|
-
vectorWeight: overrides?.query?.hybrid?.vectorWeight ?? defaults?.query?.hybrid?.vectorWeight ?? DEFAULT_HYBRID_VECTOR_WEIGHT,
|
|
120
|
-
textWeight: overrides?.query?.hybrid?.textWeight ?? defaults?.query?.hybrid?.textWeight ?? DEFAULT_HYBRID_TEXT_WEIGHT,
|
|
121
|
-
candidateMultiplier: overrides?.query?.hybrid?.candidateMultiplier ?? defaults?.query?.hybrid?.candidateMultiplier ?? DEFAULT_HYBRID_CANDIDATE_MULTIPLIER,
|
|
122
|
-
mmr: {
|
|
123
|
-
enabled: overrides?.query?.hybrid?.mmr?.enabled ?? defaults?.query?.hybrid?.mmr?.enabled ?? DEFAULT_MMR_ENABLED,
|
|
124
|
-
lambda: overrides?.query?.hybrid?.mmr?.lambda ?? defaults?.query?.hybrid?.mmr?.lambda ?? DEFAULT_MMR_LAMBDA
|
|
125
|
-
},
|
|
126
|
-
temporalDecay: {
|
|
127
|
-
enabled: overrides?.query?.hybrid?.temporalDecay?.enabled ?? defaults?.query?.hybrid?.temporalDecay?.enabled ?? DEFAULT_TEMPORAL_DECAY_ENABLED,
|
|
128
|
-
halfLifeDays: overrides?.query?.hybrid?.temporalDecay?.halfLifeDays ?? defaults?.query?.hybrid?.temporalDecay?.halfLifeDays ?? DEFAULT_TEMPORAL_DECAY_HALF_LIFE_DAYS
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
|
-
const cache = {
|
|
132
|
-
enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED,
|
|
133
|
-
maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries
|
|
134
|
-
};
|
|
135
|
-
const overlap = clampNumber(chunking.overlap, 0, Math.max(0, chunking.tokens - 1));
|
|
136
|
-
const minScore = clampNumber(query.minScore, 0, 1);
|
|
137
|
-
const vectorWeight = clampNumber(hybrid.vectorWeight, 0, 1);
|
|
138
|
-
const textWeight = clampNumber(hybrid.textWeight, 0, 1);
|
|
139
|
-
const sum = vectorWeight + textWeight;
|
|
140
|
-
const normalizedVectorWeight = sum > 0 ? vectorWeight / sum : DEFAULT_HYBRID_VECTOR_WEIGHT;
|
|
141
|
-
const normalizedTextWeight = sum > 0 ? textWeight / sum : DEFAULT_HYBRID_TEXT_WEIGHT;
|
|
142
|
-
const candidateMultiplier = clampInt(hybrid.candidateMultiplier, 1, 20);
|
|
143
|
-
const temporalDecayHalfLifeDays = Math.max(1, Math.floor(Number.isFinite(hybrid.temporalDecay.halfLifeDays) ? hybrid.temporalDecay.halfLifeDays : DEFAULT_TEMPORAL_DECAY_HALF_LIFE_DAYS));
|
|
144
|
-
const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER);
|
|
145
|
-
const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER);
|
|
146
|
-
return {
|
|
147
|
-
enabled,
|
|
148
|
-
sources,
|
|
149
|
-
extraPaths,
|
|
150
|
-
provider,
|
|
151
|
-
remote,
|
|
152
|
-
experimental: { sessionMemory },
|
|
153
|
-
fallback,
|
|
154
|
-
model,
|
|
155
|
-
local,
|
|
156
|
-
store,
|
|
157
|
-
chunking: {
|
|
158
|
-
tokens: Math.max(1, chunking.tokens),
|
|
159
|
-
overlap
|
|
160
|
-
},
|
|
161
|
-
sync: {
|
|
162
|
-
...sync,
|
|
163
|
-
sessions: {
|
|
164
|
-
deltaBytes,
|
|
165
|
-
deltaMessages
|
|
166
|
-
}
|
|
167
|
-
},
|
|
168
|
-
query: {
|
|
169
|
-
...query,
|
|
170
|
-
minScore,
|
|
171
|
-
hybrid: {
|
|
172
|
-
enabled: Boolean(hybrid.enabled),
|
|
173
|
-
vectorWeight: normalizedVectorWeight,
|
|
174
|
-
textWeight: normalizedTextWeight,
|
|
175
|
-
candidateMultiplier,
|
|
176
|
-
mmr: {
|
|
177
|
-
enabled: Boolean(hybrid.mmr.enabled),
|
|
178
|
-
lambda: Number.isFinite(hybrid.mmr.lambda) ? Math.max(0, Math.min(1, hybrid.mmr.lambda)) : DEFAULT_MMR_LAMBDA
|
|
179
|
-
},
|
|
180
|
-
temporalDecay: {
|
|
181
|
-
enabled: Boolean(hybrid.temporalDecay.enabled),
|
|
182
|
-
halfLifeDays: temporalDecayHalfLifeDays
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
},
|
|
186
|
-
cache: {
|
|
187
|
-
enabled: Boolean(cache.enabled),
|
|
188
|
-
maxEntries: typeof cache.maxEntries === "number" && Number.isFinite(cache.maxEntries) ? Math.max(1, Math.floor(cache.maxEntries)) : void 0
|
|
189
|
-
}
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
function resolveMemorySearchConfig(cfg, agentId) {
|
|
193
|
-
const defaults = cfg.agents?.defaults?.memorySearch;
|
|
194
|
-
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
|
|
195
|
-
const resolved = mergeConfig(defaults, overrides, agentId);
|
|
196
|
-
if (!resolved.enabled) return null;
|
|
197
|
-
return resolved;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
//#endregion
|
|
201
|
-
//#region src/memory/embeddings-debug.ts
|
|
202
|
-
const debugEmbeddings = isTruthyEnvValue(process.env.SQUIDCLAW_DEBUG_MEMORY_EMBEDDINGS);
|
|
203
|
-
const log$3 = createSubsystemLogger("memory/embeddings");
|
|
204
|
-
function debugEmbeddingsLog(message, meta) {
|
|
205
|
-
if (!debugEmbeddings) return;
|
|
206
|
-
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
|
|
207
|
-
log$3.raw(`${message}${suffix}`);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
//#endregion
|
|
211
|
-
//#region src/memory/remote-http.ts
|
|
212
|
-
function buildRemoteBaseUrlPolicy(baseUrl) {
|
|
213
|
-
const trimmed = baseUrl.trim();
|
|
214
|
-
if (!trimmed) return;
|
|
215
|
-
try {
|
|
216
|
-
const parsed = new URL(trimmed);
|
|
217
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return;
|
|
218
|
-
return { allowedHostnames: [parsed.hostname] };
|
|
219
|
-
} catch {
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
async function withRemoteHttpResponse(params) {
|
|
224
|
-
const { response, release } = await fetchWithSsrFGuard({
|
|
225
|
-
url: params.url,
|
|
226
|
-
init: params.init,
|
|
227
|
-
policy: params.ssrfPolicy,
|
|
228
|
-
auditContext: params.auditContext ?? "memory-remote"
|
|
229
|
-
});
|
|
230
|
-
try {
|
|
231
|
-
return await params.onResponse(response);
|
|
232
|
-
} finally {
|
|
233
|
-
await release();
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
//#endregion
|
|
238
|
-
//#region src/memory/embeddings-gemini.ts
|
|
239
|
-
const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
240
|
-
const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
|
|
241
|
-
const GEMINI_MAX_INPUT_TOKENS = { "text-embedding-004": 2048 };
|
|
242
|
-
function resolveRemoteApiKey(remoteApiKey) {
|
|
243
|
-
const trimmed = remoteApiKey?.trim();
|
|
244
|
-
if (!trimmed) return;
|
|
245
|
-
if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") return process.env[trimmed]?.trim();
|
|
246
|
-
return trimmed;
|
|
247
|
-
}
|
|
248
|
-
function normalizeGeminiModel(model) {
|
|
249
|
-
const trimmed = model.trim();
|
|
250
|
-
if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL;
|
|
251
|
-
const withoutPrefix = trimmed.replace(/^models\//, "");
|
|
252
|
-
if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice(7);
|
|
253
|
-
if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice(7);
|
|
254
|
-
return withoutPrefix;
|
|
255
|
-
}
|
|
256
|
-
function normalizeGeminiBaseUrl(raw) {
|
|
257
|
-
const trimmed = raw.replace(/\/+$/, "");
|
|
258
|
-
const openAiIndex = trimmed.indexOf("/openai");
|
|
259
|
-
if (openAiIndex > -1) return trimmed.slice(0, openAiIndex);
|
|
260
|
-
return trimmed;
|
|
261
|
-
}
|
|
262
|
-
function buildGeminiModelPath(model) {
|
|
263
|
-
return model.startsWith("models/") ? model : `models/${model}`;
|
|
264
|
-
}
|
|
265
|
-
async function createGeminiEmbeddingProvider(options) {
|
|
266
|
-
const client = await resolveGeminiEmbeddingClient(options);
|
|
267
|
-
const baseUrl = client.baseUrl.replace(/\/$/, "");
|
|
268
|
-
const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
|
|
269
|
-
const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
|
|
270
|
-
const fetchWithGeminiAuth = async (apiKey, endpoint, body) => {
|
|
271
|
-
const headers = {
|
|
272
|
-
...parseGeminiAuth(apiKey).headers,
|
|
273
|
-
...client.headers
|
|
274
|
-
};
|
|
275
|
-
return await withRemoteHttpResponse({
|
|
276
|
-
url: endpoint,
|
|
277
|
-
ssrfPolicy: client.ssrfPolicy,
|
|
278
|
-
init: {
|
|
279
|
-
method: "POST",
|
|
280
|
-
headers,
|
|
281
|
-
body: JSON.stringify(body)
|
|
282
|
-
},
|
|
283
|
-
onResponse: async (res) => {
|
|
284
|
-
if (!res.ok) {
|
|
285
|
-
const text = await res.text();
|
|
286
|
-
throw new Error(`gemini embeddings failed: ${res.status} ${text}`);
|
|
287
|
-
}
|
|
288
|
-
return await res.json();
|
|
289
|
-
}
|
|
290
|
-
});
|
|
291
|
-
};
|
|
292
|
-
const embedQuery = async (text) => {
|
|
293
|
-
if (!text.trim()) return [];
|
|
294
|
-
return (await executeWithApiKeyRotation({
|
|
295
|
-
provider: "google",
|
|
296
|
-
apiKeys: client.apiKeys,
|
|
297
|
-
execute: (apiKey) => fetchWithGeminiAuth(apiKey, embedUrl, {
|
|
298
|
-
content: { parts: [{ text }] },
|
|
299
|
-
taskType: "RETRIEVAL_QUERY"
|
|
300
|
-
})
|
|
301
|
-
})).embedding?.values ?? [];
|
|
302
|
-
};
|
|
303
|
-
const embedBatch = async (texts) => {
|
|
304
|
-
if (texts.length === 0) return [];
|
|
305
|
-
const requests = texts.map((text) => ({
|
|
306
|
-
model: client.modelPath,
|
|
307
|
-
content: { parts: [{ text }] },
|
|
308
|
-
taskType: "RETRIEVAL_DOCUMENT"
|
|
309
|
-
}));
|
|
310
|
-
const payload = await executeWithApiKeyRotation({
|
|
311
|
-
provider: "google",
|
|
312
|
-
apiKeys: client.apiKeys,
|
|
313
|
-
execute: (apiKey) => fetchWithGeminiAuth(apiKey, batchUrl, { requests })
|
|
314
|
-
});
|
|
315
|
-
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
|
|
316
|
-
return texts.map((_, index) => embeddings[index]?.values ?? []);
|
|
317
|
-
};
|
|
318
|
-
return {
|
|
319
|
-
provider: {
|
|
320
|
-
id: "gemini",
|
|
321
|
-
model: client.model,
|
|
322
|
-
maxInputTokens: GEMINI_MAX_INPUT_TOKENS[client.model],
|
|
323
|
-
embedQuery,
|
|
324
|
-
embedBatch
|
|
325
|
-
},
|
|
326
|
-
client
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
async function resolveGeminiEmbeddingClient(options) {
|
|
330
|
-
const remote = options.remote;
|
|
331
|
-
const remoteApiKey = resolveRemoteApiKey(remote?.apiKey);
|
|
332
|
-
const remoteBaseUrl = remote?.baseUrl?.trim();
|
|
333
|
-
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
|
|
334
|
-
provider: "google",
|
|
335
|
-
cfg: options.config,
|
|
336
|
-
agentDir: options.agentDir
|
|
337
|
-
}), "google");
|
|
338
|
-
const providerConfig = options.config.models?.providers?.google;
|
|
339
|
-
const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
|
|
340
|
-
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
|
|
341
|
-
const ssrfPolicy = buildRemoteBaseUrlPolicy(baseUrl);
|
|
342
|
-
const headers = { ...Object.assign({}, providerConfig?.headers, remote?.headers) };
|
|
343
|
-
const apiKeys = collectProviderApiKeysForExecution({
|
|
344
|
-
provider: "google",
|
|
345
|
-
primaryApiKey: apiKey
|
|
346
|
-
});
|
|
347
|
-
const model = normalizeGeminiModel(options.model);
|
|
348
|
-
const modelPath = buildGeminiModelPath(model);
|
|
349
|
-
debugEmbeddingsLog("memory embeddings: gemini client", {
|
|
350
|
-
rawBaseUrl,
|
|
351
|
-
baseUrl,
|
|
352
|
-
model,
|
|
353
|
-
modelPath,
|
|
354
|
-
embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
|
|
355
|
-
batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`
|
|
356
|
-
});
|
|
357
|
-
return {
|
|
358
|
-
baseUrl,
|
|
359
|
-
headers,
|
|
360
|
-
ssrfPolicy,
|
|
361
|
-
model,
|
|
362
|
-
modelPath,
|
|
363
|
-
apiKeys
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
//#endregion
|
|
368
|
-
//#region src/memory/embeddings-remote-client.ts
|
|
369
|
-
async function resolveRemoteEmbeddingBearerClient(params) {
|
|
370
|
-
const remote = params.options.remote;
|
|
371
|
-
const remoteApiKey = normalizeResolvedSecretInputString({
|
|
372
|
-
value: remote?.apiKey,
|
|
373
|
-
path: "agents.*.memorySearch.remote.apiKey"
|
|
374
|
-
});
|
|
375
|
-
const remoteBaseUrl = remote?.baseUrl?.trim();
|
|
376
|
-
const providerConfig = params.options.config.models?.providers?.[params.provider];
|
|
377
|
-
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
|
|
378
|
-
provider: params.provider,
|
|
379
|
-
cfg: params.options.config,
|
|
380
|
-
agentDir: params.options.agentDir
|
|
381
|
-
}), params.provider);
|
|
382
|
-
const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || params.defaultBaseUrl;
|
|
383
|
-
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
|
384
|
-
return {
|
|
385
|
-
baseUrl,
|
|
386
|
-
headers: {
|
|
387
|
-
"Content-Type": "application/json",
|
|
388
|
-
Authorization: `Bearer ${apiKey}`,
|
|
389
|
-
...headerOverrides
|
|
390
|
-
},
|
|
391
|
-
ssrfPolicy: buildRemoteBaseUrlPolicy(baseUrl)
|
|
392
|
-
};
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
//#endregion
|
|
396
|
-
//#region src/memory/post-json.ts
|
|
397
|
-
async function postJson(params) {
|
|
398
|
-
return await withRemoteHttpResponse({
|
|
399
|
-
url: params.url,
|
|
400
|
-
ssrfPolicy: params.ssrfPolicy,
|
|
401
|
-
init: {
|
|
402
|
-
method: "POST",
|
|
403
|
-
headers: params.headers,
|
|
404
|
-
body: JSON.stringify(params.body)
|
|
405
|
-
},
|
|
406
|
-
onResponse: async (res) => {
|
|
407
|
-
if (!res.ok) {
|
|
408
|
-
const text = await res.text();
|
|
409
|
-
const err = /* @__PURE__ */ new Error(`${params.errorPrefix}: ${res.status} ${text}`);
|
|
410
|
-
if (params.attachStatus) err.status = res.status;
|
|
411
|
-
throw err;
|
|
412
|
-
}
|
|
413
|
-
return await params.parse(await res.json());
|
|
414
|
-
}
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
//#endregion
|
|
419
|
-
//#region src/memory/embeddings-remote-fetch.ts
|
|
420
|
-
async function fetchRemoteEmbeddingVectors(params) {
|
|
421
|
-
return await postJson({
|
|
422
|
-
url: params.url,
|
|
423
|
-
headers: params.headers,
|
|
424
|
-
ssrfPolicy: params.ssrfPolicy,
|
|
425
|
-
body: params.body,
|
|
426
|
-
errorPrefix: params.errorPrefix,
|
|
427
|
-
parse: (payload) => {
|
|
428
|
-
return (payload.data ?? []).map((entry) => entry.embedding ?? []);
|
|
429
|
-
}
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
//#endregion
|
|
434
|
-
//#region src/memory/embeddings-remote-provider.ts
|
|
435
|
-
function createRemoteEmbeddingProvider(params) {
|
|
436
|
-
const { client } = params;
|
|
437
|
-
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
|
438
|
-
const embed = async (input) => {
|
|
439
|
-
if (input.length === 0) return [];
|
|
440
|
-
return await fetchRemoteEmbeddingVectors({
|
|
441
|
-
url,
|
|
442
|
-
headers: client.headers,
|
|
443
|
-
ssrfPolicy: client.ssrfPolicy,
|
|
444
|
-
body: {
|
|
445
|
-
model: client.model,
|
|
446
|
-
input
|
|
447
|
-
},
|
|
448
|
-
errorPrefix: params.errorPrefix
|
|
449
|
-
});
|
|
450
|
-
};
|
|
451
|
-
return {
|
|
452
|
-
id: params.id,
|
|
453
|
-
model: client.model,
|
|
454
|
-
...typeof params.maxInputTokens === "number" ? { maxInputTokens: params.maxInputTokens } : {},
|
|
455
|
-
embedQuery: async (text) => {
|
|
456
|
-
const [vec] = await embed([text]);
|
|
457
|
-
return vec ?? [];
|
|
458
|
-
},
|
|
459
|
-
embedBatch: embed
|
|
460
|
-
};
|
|
461
|
-
}
|
|
462
|
-
async function resolveRemoteEmbeddingClient(params) {
|
|
463
|
-
const { baseUrl, headers, ssrfPolicy } = await resolveRemoteEmbeddingBearerClient({
|
|
464
|
-
provider: params.provider,
|
|
465
|
-
options: params.options,
|
|
466
|
-
defaultBaseUrl: params.defaultBaseUrl
|
|
467
|
-
});
|
|
468
|
-
return {
|
|
469
|
-
baseUrl,
|
|
470
|
-
headers,
|
|
471
|
-
ssrfPolicy,
|
|
472
|
-
model: params.normalizeModel(params.options.model)
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
//#endregion
|
|
477
|
-
//#region src/memory/embeddings-mistral.ts
|
|
478
|
-
const DEFAULT_MISTRAL_EMBEDDING_MODEL = "mistral-embed";
|
|
479
|
-
const DEFAULT_MISTRAL_BASE_URL = "https://api.mistral.ai/v1";
|
|
480
|
-
function normalizeMistralModel(model) {
|
|
481
|
-
const trimmed = model.trim();
|
|
482
|
-
if (!trimmed) return DEFAULT_MISTRAL_EMBEDDING_MODEL;
|
|
483
|
-
if (trimmed.startsWith("mistral/")) return trimmed.slice(8);
|
|
484
|
-
return trimmed;
|
|
485
|
-
}
|
|
486
|
-
async function createMistralEmbeddingProvider(options) {
|
|
487
|
-
const client = await resolveMistralEmbeddingClient(options);
|
|
488
|
-
return {
|
|
489
|
-
provider: createRemoteEmbeddingProvider({
|
|
490
|
-
id: "mistral",
|
|
491
|
-
client,
|
|
492
|
-
errorPrefix: "mistral embeddings failed"
|
|
493
|
-
}),
|
|
494
|
-
client
|
|
495
|
-
};
|
|
496
|
-
}
|
|
497
|
-
async function resolveMistralEmbeddingClient(options) {
|
|
498
|
-
return await resolveRemoteEmbeddingClient({
|
|
499
|
-
provider: "mistral",
|
|
500
|
-
options,
|
|
501
|
-
defaultBaseUrl: DEFAULT_MISTRAL_BASE_URL,
|
|
502
|
-
normalizeModel: normalizeMistralModel
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
//#endregion
|
|
507
|
-
//#region src/memory/embeddings-ollama.ts
|
|
508
|
-
const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
|
|
509
|
-
const DEFAULT_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
|
|
510
|
-
function sanitizeAndNormalizeEmbedding$1(vec) {
|
|
511
|
-
const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0);
|
|
512
|
-
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
|
513
|
-
if (magnitude < 1e-10) return sanitized;
|
|
514
|
-
return sanitized.map((value) => value / magnitude);
|
|
515
|
-
}
|
|
516
|
-
function normalizeOllamaModel(model) {
|
|
517
|
-
const trimmed = model.trim();
|
|
518
|
-
if (!trimmed) return DEFAULT_OLLAMA_EMBEDDING_MODEL;
|
|
519
|
-
if (trimmed.startsWith("ollama/")) return trimmed.slice(7);
|
|
520
|
-
return trimmed;
|
|
521
|
-
}
|
|
522
|
-
function resolveOllamaApiBase(configuredBaseUrl) {
|
|
523
|
-
if (!configuredBaseUrl) return DEFAULT_OLLAMA_BASE_URL;
|
|
524
|
-
return configuredBaseUrl.replace(/\/+$/, "").replace(/\/v1$/i, "");
|
|
525
|
-
}
|
|
526
|
-
function resolveOllamaApiKey(options) {
|
|
527
|
-
const remoteApiKey = options.remote?.apiKey?.trim();
|
|
528
|
-
if (remoteApiKey) return remoteApiKey;
|
|
529
|
-
const providerApiKey = normalizeOptionalSecretInput(options.config.models?.providers?.ollama?.apiKey);
|
|
530
|
-
if (providerApiKey) return providerApiKey;
|
|
531
|
-
return resolveEnvApiKey("ollama")?.apiKey;
|
|
532
|
-
}
|
|
533
|
-
function resolveOllamaEmbeddingClient(options) {
|
|
534
|
-
const providerConfig = options.config.models?.providers?.ollama;
|
|
535
|
-
const baseUrl = resolveOllamaApiBase(options.remote?.baseUrl?.trim() || providerConfig?.baseUrl?.trim());
|
|
536
|
-
const model = normalizeOllamaModel(options.model);
|
|
537
|
-
const headerOverrides = Object.assign({}, providerConfig?.headers, options.remote?.headers);
|
|
538
|
-
const headers = {
|
|
539
|
-
"Content-Type": "application/json",
|
|
540
|
-
...headerOverrides
|
|
541
|
-
};
|
|
542
|
-
const apiKey = resolveOllamaApiKey(options);
|
|
543
|
-
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
|
544
|
-
return {
|
|
545
|
-
baseUrl,
|
|
546
|
-
headers,
|
|
547
|
-
ssrfPolicy: buildRemoteBaseUrlPolicy(baseUrl),
|
|
548
|
-
model
|
|
549
|
-
};
|
|
550
|
-
}
|
|
551
|
-
async function createOllamaEmbeddingProvider(options) {
|
|
552
|
-
const client = resolveOllamaEmbeddingClient(options);
|
|
553
|
-
const embedUrl = `${client.baseUrl.replace(/\/$/, "")}/api/embeddings`;
|
|
554
|
-
const embedOne = async (text) => {
|
|
555
|
-
const json = await withRemoteHttpResponse({
|
|
556
|
-
url: embedUrl,
|
|
557
|
-
ssrfPolicy: client.ssrfPolicy,
|
|
558
|
-
init: {
|
|
559
|
-
method: "POST",
|
|
560
|
-
headers: client.headers,
|
|
561
|
-
body: JSON.stringify({
|
|
562
|
-
model: client.model,
|
|
563
|
-
prompt: text
|
|
564
|
-
})
|
|
565
|
-
},
|
|
566
|
-
onResponse: async (res) => {
|
|
567
|
-
if (!res.ok) throw new Error(`Ollama embeddings HTTP ${res.status}: ${await res.text()}`);
|
|
568
|
-
return await res.json();
|
|
569
|
-
}
|
|
570
|
-
});
|
|
571
|
-
if (!Array.isArray(json.embedding)) throw new Error(`Ollama embeddings response missing embedding[]`);
|
|
572
|
-
return sanitizeAndNormalizeEmbedding$1(json.embedding);
|
|
573
|
-
};
|
|
574
|
-
const provider = {
|
|
575
|
-
id: "ollama",
|
|
576
|
-
model: client.model,
|
|
577
|
-
embedQuery: embedOne,
|
|
578
|
-
embedBatch: async (texts) => {
|
|
579
|
-
return await Promise.all(texts.map(embedOne));
|
|
580
|
-
}
|
|
581
|
-
};
|
|
582
|
-
return {
|
|
583
|
-
provider,
|
|
584
|
-
client: {
|
|
585
|
-
...client,
|
|
586
|
-
embedBatch: async (texts) => {
|
|
587
|
-
try {
|
|
588
|
-
return await provider.embedBatch(texts);
|
|
589
|
-
} catch (err) {
|
|
590
|
-
throw new Error(formatErrorMessage(err), { cause: err });
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
};
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
//#endregion
|
|
598
|
-
//#region src/memory/embeddings-openai.ts
|
|
599
|
-
const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
600
|
-
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
601
|
-
const OPENAI_MAX_INPUT_TOKENS = {
|
|
602
|
-
"text-embedding-3-small": 8192,
|
|
603
|
-
"text-embedding-3-large": 8192,
|
|
604
|
-
"text-embedding-ada-002": 8191
|
|
605
|
-
};
|
|
606
|
-
function normalizeOpenAiModel(model) {
|
|
607
|
-
const trimmed = model.trim();
|
|
608
|
-
if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL;
|
|
609
|
-
if (trimmed.startsWith("openai/")) return trimmed.slice(7);
|
|
610
|
-
return trimmed;
|
|
611
|
-
}
|
|
612
|
-
async function createOpenAiEmbeddingProvider(options) {
|
|
613
|
-
const client = await resolveOpenAiEmbeddingClient(options);
|
|
614
|
-
return {
|
|
615
|
-
provider: createRemoteEmbeddingProvider({
|
|
616
|
-
id: "openai",
|
|
617
|
-
client,
|
|
618
|
-
errorPrefix: "openai embeddings failed",
|
|
619
|
-
maxInputTokens: OPENAI_MAX_INPUT_TOKENS[client.model]
|
|
620
|
-
}),
|
|
621
|
-
client
|
|
622
|
-
};
|
|
623
|
-
}
|
|
624
|
-
async function resolveOpenAiEmbeddingClient(options) {
|
|
625
|
-
return await resolveRemoteEmbeddingClient({
|
|
626
|
-
provider: "openai",
|
|
627
|
-
options,
|
|
628
|
-
defaultBaseUrl: DEFAULT_OPENAI_BASE_URL,
|
|
629
|
-
normalizeModel: normalizeOpenAiModel
|
|
630
|
-
});
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
//#endregion
|
|
634
|
-
//#region src/memory/embeddings-voyage.ts
|
|
635
|
-
const DEFAULT_VOYAGE_EMBEDDING_MODEL = "voyage-4-large";
|
|
636
|
-
const DEFAULT_VOYAGE_BASE_URL = "https://api.voyageai.com/v1";
|
|
637
|
-
const VOYAGE_MAX_INPUT_TOKENS = {
|
|
638
|
-
"voyage-3": 32e3,
|
|
639
|
-
"voyage-3-lite": 16e3,
|
|
640
|
-
"voyage-code-3": 32e3
|
|
641
|
-
};
|
|
642
|
-
function normalizeVoyageModel(model) {
|
|
643
|
-
const trimmed = model.trim();
|
|
644
|
-
if (!trimmed) return DEFAULT_VOYAGE_EMBEDDING_MODEL;
|
|
645
|
-
if (trimmed.startsWith("voyage/")) return trimmed.slice(7);
|
|
646
|
-
return trimmed;
|
|
647
|
-
}
|
|
648
|
-
async function createVoyageEmbeddingProvider(options) {
|
|
649
|
-
const client = await resolveVoyageEmbeddingClient(options);
|
|
650
|
-
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
|
651
|
-
const embed = async (input, input_type) => {
|
|
652
|
-
if (input.length === 0) return [];
|
|
653
|
-
const body = {
|
|
654
|
-
model: client.model,
|
|
655
|
-
input
|
|
656
|
-
};
|
|
657
|
-
if (input_type) body.input_type = input_type;
|
|
658
|
-
return await fetchRemoteEmbeddingVectors({
|
|
659
|
-
url,
|
|
660
|
-
headers: client.headers,
|
|
661
|
-
ssrfPolicy: client.ssrfPolicy,
|
|
662
|
-
body,
|
|
663
|
-
errorPrefix: "voyage embeddings failed"
|
|
664
|
-
});
|
|
665
|
-
};
|
|
666
|
-
return {
|
|
667
|
-
provider: {
|
|
668
|
-
id: "voyage",
|
|
669
|
-
model: client.model,
|
|
670
|
-
maxInputTokens: VOYAGE_MAX_INPUT_TOKENS[client.model],
|
|
671
|
-
embedQuery: async (text) => {
|
|
672
|
-
const [vec] = await embed([text], "query");
|
|
673
|
-
return vec ?? [];
|
|
674
|
-
},
|
|
675
|
-
embedBatch: async (texts) => embed(texts, "document")
|
|
676
|
-
},
|
|
677
|
-
client
|
|
678
|
-
};
|
|
679
|
-
}
|
|
680
|
-
async function resolveVoyageEmbeddingClient(options) {
|
|
681
|
-
const { baseUrl, headers, ssrfPolicy } = await resolveRemoteEmbeddingBearerClient({
|
|
682
|
-
provider: "voyage",
|
|
683
|
-
options,
|
|
684
|
-
defaultBaseUrl: DEFAULT_VOYAGE_BASE_URL
|
|
685
|
-
});
|
|
686
|
-
return {
|
|
687
|
-
baseUrl,
|
|
688
|
-
headers,
|
|
689
|
-
ssrfPolicy,
|
|
690
|
-
model: normalizeVoyageModel(options.model)
|
|
691
|
-
};
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
//#endregion
|
|
695
|
-
//#region src/memory/node-llama.ts
|
|
696
|
-
async function importNodeLlamaCpp() {
|
|
697
|
-
return import("node-llama-cpp");
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
//#endregion
|
|
701
|
-
//#region src/memory/embeddings.ts
|
|
702
|
-
function sanitizeAndNormalizeEmbedding(vec) {
|
|
703
|
-
const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0);
|
|
704
|
-
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
|
705
|
-
if (magnitude < 1e-10) return sanitized;
|
|
706
|
-
return sanitized.map((value) => value / magnitude);
|
|
707
|
-
}
|
|
708
|
-
const REMOTE_EMBEDDING_PROVIDER_IDS = [
|
|
709
|
-
"openai",
|
|
710
|
-
"gemini",
|
|
711
|
-
"voyage",
|
|
712
|
-
"mistral"
|
|
713
|
-
];
|
|
714
|
-
const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";
|
|
715
|
-
function canAutoSelectLocal(options) {
|
|
716
|
-
const modelPath = options.local?.modelPath?.trim();
|
|
717
|
-
if (!modelPath) return false;
|
|
718
|
-
if (/^(hf:|https?:)/i.test(modelPath)) return false;
|
|
719
|
-
const resolved = resolveUserPath(modelPath);
|
|
720
|
-
try {
|
|
721
|
-
return fs.statSync(resolved).isFile();
|
|
722
|
-
} catch {
|
|
723
|
-
return false;
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
function isMissingApiKeyError(err) {
|
|
727
|
-
return formatErrorMessage(err).includes("No API key found for provider");
|
|
728
|
-
}
|
|
729
|
-
async function createLocalEmbeddingProvider(options) {
|
|
730
|
-
const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
|
|
731
|
-
const modelCacheDir = options.local?.modelCacheDir?.trim();
|
|
732
|
-
const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
|
|
733
|
-
let llama = null;
|
|
734
|
-
let embeddingModel = null;
|
|
735
|
-
let embeddingContext = null;
|
|
736
|
-
const ensureContext = async () => {
|
|
737
|
-
if (!llama) llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
|
738
|
-
if (!embeddingModel) {
|
|
739
|
-
const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0);
|
|
740
|
-
embeddingModel = await llama.loadModel({ modelPath: resolved });
|
|
741
|
-
}
|
|
742
|
-
if (!embeddingContext) embeddingContext = await embeddingModel.createEmbeddingContext();
|
|
743
|
-
return embeddingContext;
|
|
744
|
-
};
|
|
745
|
-
return {
|
|
746
|
-
id: "local",
|
|
747
|
-
model: modelPath,
|
|
748
|
-
embedQuery: async (text) => {
|
|
749
|
-
const embedding = await (await ensureContext()).getEmbeddingFor(text);
|
|
750
|
-
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
|
|
751
|
-
},
|
|
752
|
-
embedBatch: async (texts) => {
|
|
753
|
-
const ctx = await ensureContext();
|
|
754
|
-
return await Promise.all(texts.map(async (text) => {
|
|
755
|
-
const embedding = await ctx.getEmbeddingFor(text);
|
|
756
|
-
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
|
|
757
|
-
}));
|
|
758
|
-
}
|
|
759
|
-
};
|
|
760
|
-
}
|
|
761
|
-
async function createEmbeddingProvider(options) {
|
|
762
|
-
const requestedProvider = options.provider;
|
|
763
|
-
const fallback = options.fallback;
|
|
764
|
-
const createProvider = async (id) => {
|
|
765
|
-
if (id === "local") return { provider: await createLocalEmbeddingProvider(options) };
|
|
766
|
-
if (id === "ollama") {
|
|
767
|
-
const { provider, client } = await createOllamaEmbeddingProvider(options);
|
|
768
|
-
return {
|
|
769
|
-
provider,
|
|
770
|
-
ollama: client
|
|
771
|
-
};
|
|
772
|
-
}
|
|
773
|
-
if (id === "gemini") {
|
|
774
|
-
const { provider, client } = await createGeminiEmbeddingProvider(options);
|
|
775
|
-
return {
|
|
776
|
-
provider,
|
|
777
|
-
gemini: client
|
|
778
|
-
};
|
|
779
|
-
}
|
|
780
|
-
if (id === "voyage") {
|
|
781
|
-
const { provider, client } = await createVoyageEmbeddingProvider(options);
|
|
782
|
-
return {
|
|
783
|
-
provider,
|
|
784
|
-
voyage: client
|
|
785
|
-
};
|
|
786
|
-
}
|
|
787
|
-
if (id === "mistral") {
|
|
788
|
-
const { provider, client } = await createMistralEmbeddingProvider(options);
|
|
789
|
-
return {
|
|
790
|
-
provider,
|
|
791
|
-
mistral: client
|
|
792
|
-
};
|
|
793
|
-
}
|
|
794
|
-
const { provider, client } = await createOpenAiEmbeddingProvider(options);
|
|
795
|
-
return {
|
|
796
|
-
provider,
|
|
797
|
-
openAi: client
|
|
798
|
-
};
|
|
799
|
-
};
|
|
800
|
-
const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatErrorMessage(err);
|
|
801
|
-
if (requestedProvider === "auto") {
|
|
802
|
-
const missingKeyErrors = [];
|
|
803
|
-
let localError = null;
|
|
804
|
-
if (canAutoSelectLocal(options)) try {
|
|
805
|
-
return {
|
|
806
|
-
...await createProvider("local"),
|
|
807
|
-
requestedProvider
|
|
808
|
-
};
|
|
809
|
-
} catch (err) {
|
|
810
|
-
localError = formatLocalSetupError(err);
|
|
811
|
-
}
|
|
812
|
-
for (const provider of REMOTE_EMBEDDING_PROVIDER_IDS) try {
|
|
813
|
-
return {
|
|
814
|
-
...await createProvider(provider),
|
|
815
|
-
requestedProvider
|
|
816
|
-
};
|
|
817
|
-
} catch (err) {
|
|
818
|
-
const message = formatPrimaryError(err, provider);
|
|
819
|
-
if (isMissingApiKeyError(err)) {
|
|
820
|
-
missingKeyErrors.push(message);
|
|
821
|
-
continue;
|
|
822
|
-
}
|
|
823
|
-
const wrapped = new Error(message);
|
|
824
|
-
wrapped.cause = err;
|
|
825
|
-
throw wrapped;
|
|
826
|
-
}
|
|
827
|
-
const details = [...missingKeyErrors, localError].filter(Boolean);
|
|
828
|
-
return {
|
|
829
|
-
provider: null,
|
|
830
|
-
requestedProvider,
|
|
831
|
-
providerUnavailableReason: details.length > 0 ? details.join("\n\n") : "No embeddings provider available."
|
|
832
|
-
};
|
|
833
|
-
}
|
|
834
|
-
try {
|
|
835
|
-
return {
|
|
836
|
-
...await createProvider(requestedProvider),
|
|
837
|
-
requestedProvider
|
|
838
|
-
};
|
|
839
|
-
} catch (primaryErr) {
|
|
840
|
-
const reason = formatPrimaryError(primaryErr, requestedProvider);
|
|
841
|
-
if (fallback && fallback !== "none" && fallback !== requestedProvider) try {
|
|
842
|
-
return {
|
|
843
|
-
...await createProvider(fallback),
|
|
844
|
-
requestedProvider,
|
|
845
|
-
fallbackFrom: requestedProvider,
|
|
846
|
-
fallbackReason: reason
|
|
847
|
-
};
|
|
848
|
-
} catch (fallbackErr) {
|
|
849
|
-
const combinedReason = `${reason}\n\nFallback to ${fallback} failed: ${formatErrorMessage(fallbackErr)}`;
|
|
850
|
-
if (isMissingApiKeyError(primaryErr) && isMissingApiKeyError(fallbackErr)) return {
|
|
851
|
-
provider: null,
|
|
852
|
-
requestedProvider,
|
|
853
|
-
fallbackFrom: requestedProvider,
|
|
854
|
-
fallbackReason: reason,
|
|
855
|
-
providerUnavailableReason: combinedReason
|
|
856
|
-
};
|
|
857
|
-
const wrapped = new Error(combinedReason);
|
|
858
|
-
wrapped.cause = fallbackErr;
|
|
859
|
-
throw wrapped;
|
|
860
|
-
}
|
|
861
|
-
if (isMissingApiKeyError(primaryErr)) return {
|
|
862
|
-
provider: null,
|
|
863
|
-
requestedProvider,
|
|
864
|
-
providerUnavailableReason: reason
|
|
865
|
-
};
|
|
866
|
-
const wrapped = new Error(reason);
|
|
867
|
-
wrapped.cause = primaryErr;
|
|
868
|
-
throw wrapped;
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
function isNodeLlamaCppMissing(err) {
|
|
872
|
-
if (!(err instanceof Error)) return false;
|
|
873
|
-
if (err.code === "ERR_MODULE_NOT_FOUND") return err.message.includes("node-llama-cpp");
|
|
874
|
-
return false;
|
|
875
|
-
}
|
|
876
|
-
function formatLocalSetupError(err) {
|
|
877
|
-
const detail = formatErrorMessage(err);
|
|
878
|
-
const missing = isNodeLlamaCppMissing(err);
|
|
879
|
-
return [
|
|
880
|
-
"Local embeddings unavailable.",
|
|
881
|
-
missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0,
|
|
882
|
-
missing && detail ? `Detail: ${detail}` : null,
|
|
883
|
-
"To enable local embeddings:",
|
|
884
|
-
"1) Use Node 22 LTS (recommended for installs/updates)",
|
|
885
|
-
missing ? "2) Reinstall SquidClaw (this should install node-llama-cpp): npm i -g squidclaw@latest" : null,
|
|
886
|
-
"3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
|
|
887
|
-
...REMOTE_EMBEDDING_PROVIDER_IDS.map((provider) => `Or set agents.defaults.memorySearch.provider = "${provider}" (remote).`)
|
|
888
|
-
].filter(Boolean).join("\n");
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
//#endregion
|
|
892
|
-
//#region src/memory/mmr.ts
|
|
893
|
-
const DEFAULT_MMR_CONFIG = {
|
|
894
|
-
enabled: false,
|
|
895
|
-
lambda: .7
|
|
896
|
-
};
|
|
897
|
-
/**
|
|
898
|
-
* Tokenize text for Jaccard similarity computation.
|
|
899
|
-
* Extracts alphanumeric tokens and normalizes to lowercase.
|
|
900
|
-
*/
|
|
901
|
-
function tokenize(text) {
|
|
902
|
-
const tokens = text.toLowerCase().match(/[a-z0-9_]+/g) ?? [];
|
|
903
|
-
return new Set(tokens);
|
|
904
|
-
}
|
|
905
|
-
/**
|
|
906
|
-
* Compute Jaccard similarity between two token sets.
|
|
907
|
-
* Returns a value in [0, 1] where 1 means identical sets.
|
|
908
|
-
*/
|
|
909
|
-
function jaccardSimilarity(setA, setB) {
|
|
910
|
-
if (setA.size === 0 && setB.size === 0) return 1;
|
|
911
|
-
if (setA.size === 0 || setB.size === 0) return 0;
|
|
912
|
-
let intersectionSize = 0;
|
|
913
|
-
const smaller = setA.size <= setB.size ? setA : setB;
|
|
914
|
-
const larger = setA.size <= setB.size ? setB : setA;
|
|
915
|
-
for (const token of smaller) if (larger.has(token)) intersectionSize++;
|
|
916
|
-
const unionSize = setA.size + setB.size - intersectionSize;
|
|
917
|
-
return unionSize === 0 ? 0 : intersectionSize / unionSize;
|
|
918
|
-
}
|
|
919
|
-
/**
|
|
920
|
-
* Compute the maximum similarity between an item and all selected items.
|
|
921
|
-
*/
|
|
922
|
-
function maxSimilarityToSelected(item, selectedItems, tokenCache) {
|
|
923
|
-
if (selectedItems.length === 0) return 0;
|
|
924
|
-
let maxSim = 0;
|
|
925
|
-
const itemTokens = tokenCache.get(item.id) ?? tokenize(item.content);
|
|
926
|
-
for (const selected of selectedItems) {
|
|
927
|
-
const sim = jaccardSimilarity(itemTokens, tokenCache.get(selected.id) ?? tokenize(selected.content));
|
|
928
|
-
if (sim > maxSim) maxSim = sim;
|
|
929
|
-
}
|
|
930
|
-
return maxSim;
|
|
931
|
-
}
|
|
932
|
-
/**
|
|
933
|
-
* Compute MMR score for a candidate item.
|
|
934
|
-
* MMR = λ * relevance - (1-λ) * max_similarity_to_selected
|
|
935
|
-
*/
|
|
936
|
-
function computeMMRScore(relevance, maxSimilarity, lambda) {
|
|
937
|
-
return lambda * relevance - (1 - lambda) * maxSimilarity;
|
|
938
|
-
}
|
|
939
|
-
/**
|
|
940
|
-
* Re-rank items using Maximal Marginal Relevance (MMR).
|
|
941
|
-
*
|
|
942
|
-
* The algorithm iteratively selects items that balance relevance with diversity:
|
|
943
|
-
* 1. Start with the highest-scoring item
|
|
944
|
-
* 2. For each remaining slot, select the item that maximizes the MMR score
|
|
945
|
-
* 3. MMR score = λ * relevance - (1-λ) * max_similarity_to_already_selected
|
|
946
|
-
*
|
|
947
|
-
* @param items - Items to re-rank, must have score and content
|
|
948
|
-
* @param config - MMR configuration (lambda, enabled)
|
|
949
|
-
* @returns Re-ranked items in MMR order
|
|
950
|
-
*/
|
|
951
|
-
function mmrRerank(items, config = {}) {
|
|
952
|
-
const { enabled = DEFAULT_MMR_CONFIG.enabled, lambda = DEFAULT_MMR_CONFIG.lambda } = config;
|
|
953
|
-
if (!enabled || items.length <= 1) return [...items];
|
|
954
|
-
const clampedLambda = Math.max(0, Math.min(1, lambda));
|
|
955
|
-
if (clampedLambda === 1) return [...items].toSorted((a, b) => b.score - a.score);
|
|
956
|
-
const tokenCache = /* @__PURE__ */ new Map();
|
|
957
|
-
for (const item of items) tokenCache.set(item.id, tokenize(item.content));
|
|
958
|
-
const maxScore = Math.max(...items.map((i) => i.score));
|
|
959
|
-
const minScore = Math.min(...items.map((i) => i.score));
|
|
960
|
-
const scoreRange = maxScore - minScore;
|
|
961
|
-
const normalizeScore = (score) => {
|
|
962
|
-
if (scoreRange === 0) return 1;
|
|
963
|
-
return (score - minScore) / scoreRange;
|
|
964
|
-
};
|
|
965
|
-
const selected = [];
|
|
966
|
-
const remaining = new Set(items);
|
|
967
|
-
while (remaining.size > 0) {
|
|
968
|
-
let bestItem = null;
|
|
969
|
-
let bestMMRScore = -Infinity;
|
|
970
|
-
for (const candidate of remaining) {
|
|
971
|
-
const mmrScore = computeMMRScore(normalizeScore(candidate.score), maxSimilarityToSelected(candidate, selected, tokenCache), clampedLambda);
|
|
972
|
-
if (mmrScore > bestMMRScore || mmrScore === bestMMRScore && candidate.score > (bestItem?.score ?? -Infinity)) {
|
|
973
|
-
bestMMRScore = mmrScore;
|
|
974
|
-
bestItem = candidate;
|
|
975
|
-
}
|
|
976
|
-
}
|
|
977
|
-
if (bestItem) {
|
|
978
|
-
selected.push(bestItem);
|
|
979
|
-
remaining.delete(bestItem);
|
|
980
|
-
} else break;
|
|
981
|
-
}
|
|
982
|
-
return selected;
|
|
983
|
-
}
|
|
984
|
-
/**
|
|
985
|
-
* Apply MMR re-ranking to hybrid search results.
|
|
986
|
-
* Adapts the generic MMR function to work with the hybrid search result format.
|
|
987
|
-
*/
|
|
988
|
-
function applyMMRToHybridResults(results, config = {}) {
|
|
989
|
-
if (results.length === 0) return results;
|
|
990
|
-
const itemById = /* @__PURE__ */ new Map();
|
|
991
|
-
return mmrRerank(results.map((r, index) => {
|
|
992
|
-
const id = `${r.path}:${r.startLine}:${index}`;
|
|
993
|
-
itemById.set(id, r);
|
|
994
|
-
return {
|
|
995
|
-
id,
|
|
996
|
-
score: r.score,
|
|
997
|
-
content: r.snippet
|
|
998
|
-
};
|
|
999
|
-
}), config).map((item) => itemById.get(item.id));
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
//#endregion
|
|
1003
|
-
//#region src/memory/temporal-decay.ts
|
|
1004
|
-
const DEFAULT_TEMPORAL_DECAY_CONFIG = {
|
|
1005
|
-
enabled: false,
|
|
1006
|
-
halfLifeDays: 30
|
|
1007
|
-
};
|
|
1008
|
-
const DAY_MS = 1440 * 60 * 1e3;
|
|
1009
|
-
const DATED_MEMORY_PATH_RE = /(?:^|\/)memory\/(\d{4})-(\d{2})-(\d{2})\.md$/;
|
|
1010
|
-
function toDecayLambda(halfLifeDays) {
|
|
1011
|
-
if (!Number.isFinite(halfLifeDays) || halfLifeDays <= 0) return 0;
|
|
1012
|
-
return Math.LN2 / halfLifeDays;
|
|
1013
|
-
}
|
|
1014
|
-
function calculateTemporalDecayMultiplier(params) {
|
|
1015
|
-
const lambda = toDecayLambda(params.halfLifeDays);
|
|
1016
|
-
const clampedAge = Math.max(0, params.ageInDays);
|
|
1017
|
-
if (lambda <= 0 || !Number.isFinite(clampedAge)) return 1;
|
|
1018
|
-
return Math.exp(-lambda * clampedAge);
|
|
1019
|
-
}
|
|
1020
|
-
function applyTemporalDecayToScore(params) {
|
|
1021
|
-
return params.score * calculateTemporalDecayMultiplier(params);
|
|
1022
|
-
}
|
|
1023
|
-
function parseMemoryDateFromPath(filePath) {
|
|
1024
|
-
const normalized = filePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
1025
|
-
const match = DATED_MEMORY_PATH_RE.exec(normalized);
|
|
1026
|
-
if (!match) return null;
|
|
1027
|
-
const year = Number(match[1]);
|
|
1028
|
-
const month = Number(match[2]);
|
|
1029
|
-
const day = Number(match[3]);
|
|
1030
|
-
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) return null;
|
|
1031
|
-
const timestamp = Date.UTC(year, month - 1, day);
|
|
1032
|
-
const parsed = new Date(timestamp);
|
|
1033
|
-
if (parsed.getUTCFullYear() !== year || parsed.getUTCMonth() !== month - 1 || parsed.getUTCDate() !== day) return null;
|
|
1034
|
-
return parsed;
|
|
1035
|
-
}
|
|
1036
|
-
function isEvergreenMemoryPath(filePath) {
|
|
1037
|
-
const normalized = filePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
1038
|
-
if (normalized === "MEMORY.md" || normalized === "memory.md") return true;
|
|
1039
|
-
if (!normalized.startsWith("memory/")) return false;
|
|
1040
|
-
return !DATED_MEMORY_PATH_RE.test(normalized);
|
|
1041
|
-
}
|
|
1042
|
-
async function extractTimestamp(params) {
|
|
1043
|
-
const fromPath = parseMemoryDateFromPath(params.filePath);
|
|
1044
|
-
if (fromPath) return fromPath;
|
|
1045
|
-
if (params.source === "memory" && isEvergreenMemoryPath(params.filePath)) return null;
|
|
1046
|
-
if (!params.workspaceDir) return null;
|
|
1047
|
-
const absolutePath = path.isAbsolute(params.filePath) ? params.filePath : path.resolve(params.workspaceDir, params.filePath);
|
|
1048
|
-
try {
|
|
1049
|
-
const stat = await fs$1.stat(absolutePath);
|
|
1050
|
-
if (!Number.isFinite(stat.mtimeMs)) return null;
|
|
1051
|
-
return new Date(stat.mtimeMs);
|
|
1052
|
-
} catch {
|
|
1053
|
-
return null;
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
function ageInDaysFromTimestamp(timestamp, nowMs) {
|
|
1057
|
-
return Math.max(0, nowMs - timestamp.getTime()) / DAY_MS;
|
|
1058
|
-
}
|
|
1059
|
-
async function applyTemporalDecayToHybridResults(params) {
|
|
1060
|
-
const config = {
|
|
1061
|
-
...DEFAULT_TEMPORAL_DECAY_CONFIG,
|
|
1062
|
-
...params.temporalDecay
|
|
1063
|
-
};
|
|
1064
|
-
if (!config.enabled) return [...params.results];
|
|
1065
|
-
const nowMs = params.nowMs ?? Date.now();
|
|
1066
|
-
const timestampPromiseCache = /* @__PURE__ */ new Map();
|
|
1067
|
-
return Promise.all(params.results.map(async (entry) => {
|
|
1068
|
-
const cacheKey = `${entry.source}:${entry.path}`;
|
|
1069
|
-
let timestampPromise = timestampPromiseCache.get(cacheKey);
|
|
1070
|
-
if (!timestampPromise) {
|
|
1071
|
-
timestampPromise = extractTimestamp({
|
|
1072
|
-
filePath: entry.path,
|
|
1073
|
-
source: entry.source,
|
|
1074
|
-
workspaceDir: params.workspaceDir
|
|
1075
|
-
});
|
|
1076
|
-
timestampPromiseCache.set(cacheKey, timestampPromise);
|
|
1077
|
-
}
|
|
1078
|
-
const timestamp = await timestampPromise;
|
|
1079
|
-
if (!timestamp) return entry;
|
|
1080
|
-
const decayedScore = applyTemporalDecayToScore({
|
|
1081
|
-
score: entry.score,
|
|
1082
|
-
ageInDays: ageInDaysFromTimestamp(timestamp, nowMs),
|
|
1083
|
-
halfLifeDays: config.halfLifeDays
|
|
1084
|
-
});
|
|
1085
|
-
return {
|
|
1086
|
-
...entry,
|
|
1087
|
-
score: decayedScore
|
|
1088
|
-
};
|
|
1089
|
-
}));
|
|
1090
|
-
}
|
|
1091
|
-
|
|
1092
|
-
//#endregion
|
|
1093
|
-
//#region src/memory/hybrid.ts
|
|
1094
|
-
function buildFtsQuery(raw) {
|
|
1095
|
-
const tokens = raw.match(/[\p{L}\p{N}_]+/gu)?.map((t) => t.trim()).filter(Boolean) ?? [];
|
|
1096
|
-
if (tokens.length === 0) return null;
|
|
1097
|
-
return tokens.map((t) => `"${t.replaceAll("\"", "")}"`).join(" AND ");
|
|
1098
|
-
}
|
|
1099
|
-
function bm25RankToScore(rank) {
|
|
1100
|
-
return 1 / (1 + (Number.isFinite(rank) ? Math.max(0, rank) : 999));
|
|
1101
|
-
}
|
|
1102
|
-
async function mergeHybridResults(params) {
|
|
1103
|
-
const byId = /* @__PURE__ */ new Map();
|
|
1104
|
-
for (const r of params.vector) byId.set(r.id, {
|
|
1105
|
-
id: r.id,
|
|
1106
|
-
path: r.path,
|
|
1107
|
-
startLine: r.startLine,
|
|
1108
|
-
endLine: r.endLine,
|
|
1109
|
-
source: r.source,
|
|
1110
|
-
snippet: r.snippet,
|
|
1111
|
-
vectorScore: r.vectorScore,
|
|
1112
|
-
textScore: 0
|
|
1113
|
-
});
|
|
1114
|
-
for (const r of params.keyword) {
|
|
1115
|
-
const existing = byId.get(r.id);
|
|
1116
|
-
if (existing) {
|
|
1117
|
-
existing.textScore = r.textScore;
|
|
1118
|
-
if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet;
|
|
1119
|
-
} else byId.set(r.id, {
|
|
1120
|
-
id: r.id,
|
|
1121
|
-
path: r.path,
|
|
1122
|
-
startLine: r.startLine,
|
|
1123
|
-
endLine: r.endLine,
|
|
1124
|
-
source: r.source,
|
|
1125
|
-
snippet: r.snippet,
|
|
1126
|
-
vectorScore: 0,
|
|
1127
|
-
textScore: r.textScore
|
|
1128
|
-
});
|
|
1129
|
-
}
|
|
1130
|
-
const sorted = (await applyTemporalDecayToHybridResults({
|
|
1131
|
-
results: Array.from(byId.values()).map((entry) => {
|
|
1132
|
-
const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore;
|
|
1133
|
-
return {
|
|
1134
|
-
path: entry.path,
|
|
1135
|
-
startLine: entry.startLine,
|
|
1136
|
-
endLine: entry.endLine,
|
|
1137
|
-
score,
|
|
1138
|
-
snippet: entry.snippet,
|
|
1139
|
-
source: entry.source
|
|
1140
|
-
};
|
|
1141
|
-
}),
|
|
1142
|
-
temporalDecay: {
|
|
1143
|
-
...DEFAULT_TEMPORAL_DECAY_CONFIG,
|
|
1144
|
-
...params.temporalDecay
|
|
1145
|
-
},
|
|
1146
|
-
workspaceDir: params.workspaceDir,
|
|
1147
|
-
nowMs: params.nowMs
|
|
1148
|
-
})).toSorted((a, b) => b.score - a.score);
|
|
1149
|
-
const mmrConfig = {
|
|
1150
|
-
...DEFAULT_MMR_CONFIG,
|
|
1151
|
-
...params.mmr
|
|
1152
|
-
};
|
|
1153
|
-
if (mmrConfig.enabled) return applyMMRToHybridResults(sorted, mmrConfig);
|
|
1154
|
-
return sorted;
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
//#endregion
|
|
1158
|
-
//#region src/memory/batch-utils.ts
|
|
1159
|
-
function normalizeBatchBaseUrl(client) {
|
|
1160
|
-
return client.baseUrl?.replace(/\/$/, "") ?? "";
|
|
1161
|
-
}
|
|
1162
|
-
function buildBatchHeaders(client, params) {
|
|
1163
|
-
const headers = client.headers ? { ...client.headers } : {};
|
|
1164
|
-
if (params.json) {
|
|
1165
|
-
if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json";
|
|
1166
|
-
} else {
|
|
1167
|
-
delete headers["Content-Type"];
|
|
1168
|
-
delete headers["content-type"];
|
|
1169
|
-
}
|
|
1170
|
-
return headers;
|
|
1171
|
-
}
|
|
1172
|
-
function splitBatchRequests(requests, maxRequests) {
|
|
1173
|
-
if (requests.length <= maxRequests) return [requests];
|
|
1174
|
-
const groups = [];
|
|
1175
|
-
for (let i = 0; i < requests.length; i += maxRequests) groups.push(requests.slice(i, i + maxRequests));
|
|
1176
|
-
return groups;
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
//#endregion
|
|
1180
|
-
//#region src/memory/batch-runner.ts
|
|
1181
|
-
async function runEmbeddingBatchGroups(params) {
|
|
1182
|
-
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
|
|
1183
|
-
const groups = splitBatchRequests(params.requests, params.maxRequests);
|
|
1184
|
-
const byCustomId = /* @__PURE__ */ new Map();
|
|
1185
|
-
const tasks = groups.map((group, groupIndex) => async () => {
|
|
1186
|
-
await params.runGroup({
|
|
1187
|
-
group,
|
|
1188
|
-
groupIndex,
|
|
1189
|
-
groups: groups.length,
|
|
1190
|
-
byCustomId
|
|
1191
|
-
});
|
|
1192
|
-
});
|
|
1193
|
-
params.debug?.(params.debugLabel, {
|
|
1194
|
-
requests: params.requests.length,
|
|
1195
|
-
groups: groups.length,
|
|
1196
|
-
wait: params.wait,
|
|
1197
|
-
concurrency: params.concurrency,
|
|
1198
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1199
|
-
timeoutMs: params.timeoutMs
|
|
1200
|
-
});
|
|
1201
|
-
await runWithConcurrency(tasks, params.concurrency);
|
|
1202
|
-
return byCustomId;
|
|
1203
|
-
}
|
|
1204
|
-
function buildEmbeddingBatchGroupOptions(params, options) {
|
|
1205
|
-
return {
|
|
1206
|
-
requests: params.requests,
|
|
1207
|
-
maxRequests: options.maxRequests,
|
|
1208
|
-
wait: params.wait,
|
|
1209
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1210
|
-
timeoutMs: params.timeoutMs,
|
|
1211
|
-
concurrency: params.concurrency,
|
|
1212
|
-
debug: params.debug,
|
|
1213
|
-
debugLabel: options.debugLabel
|
|
1214
|
-
};
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
|
-
//#endregion
|
|
1218
|
-
//#region src/memory/batch-gemini.ts
|
|
1219
|
-
const GEMINI_BATCH_MAX_REQUESTS = 5e4;
|
|
1220
|
-
function getGeminiUploadUrl(baseUrl) {
|
|
1221
|
-
if (baseUrl.includes("/v1beta")) return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta");
|
|
1222
|
-
return `${baseUrl.replace(/\/$/, "")}/upload`;
|
|
1223
|
-
}
|
|
1224
|
-
function buildGeminiUploadBody(params) {
|
|
1225
|
-
const boundary = `squidclaw-${hashText(params.displayName)}`;
|
|
1226
|
-
const jsonPart = JSON.stringify({ file: {
|
|
1227
|
-
displayName: params.displayName,
|
|
1228
|
-
mimeType: "application/jsonl"
|
|
1229
|
-
} });
|
|
1230
|
-
const delimiter = `--${boundary}\r\n`;
|
|
1231
|
-
const closeDelimiter = `--${boundary}--\r\n`;
|
|
1232
|
-
const parts = [
|
|
1233
|
-
`${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`,
|
|
1234
|
-
`${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`,
|
|
1235
|
-
closeDelimiter
|
|
1236
|
-
];
|
|
1237
|
-
return {
|
|
1238
|
-
body: new Blob([parts.join("")], { type: "multipart/related" }),
|
|
1239
|
-
contentType: `multipart/related; boundary=${boundary}`
|
|
1240
|
-
};
|
|
1241
|
-
}
|
|
1242
|
-
async function submitGeminiBatch(params) {
|
|
1243
|
-
const baseUrl = normalizeBatchBaseUrl(params.gemini);
|
|
1244
|
-
const uploadPayload = buildGeminiUploadBody({
|
|
1245
|
-
jsonl: params.requests.map((request) => JSON.stringify({
|
|
1246
|
-
key: request.custom_id,
|
|
1247
|
-
request: {
|
|
1248
|
-
content: request.content,
|
|
1249
|
-
task_type: request.taskType
|
|
1250
|
-
}
|
|
1251
|
-
})).join("\n"),
|
|
1252
|
-
displayName: `memory-embeddings-${hashText(String(Date.now()))}`
|
|
1253
|
-
});
|
|
1254
|
-
const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`;
|
|
1255
|
-
debugEmbeddingsLog("memory embeddings: gemini batch upload", {
|
|
1256
|
-
uploadUrl,
|
|
1257
|
-
baseUrl,
|
|
1258
|
-
requests: params.requests.length
|
|
1259
|
-
});
|
|
1260
|
-
const filePayload = await withRemoteHttpResponse({
|
|
1261
|
-
url: uploadUrl,
|
|
1262
|
-
ssrfPolicy: params.gemini.ssrfPolicy,
|
|
1263
|
-
init: {
|
|
1264
|
-
method: "POST",
|
|
1265
|
-
headers: {
|
|
1266
|
-
...buildBatchHeaders(params.gemini, { json: false }),
|
|
1267
|
-
"Content-Type": uploadPayload.contentType
|
|
1268
|
-
},
|
|
1269
|
-
body: uploadPayload.body
|
|
1270
|
-
},
|
|
1271
|
-
onResponse: async (fileRes) => {
|
|
1272
|
-
if (!fileRes.ok) {
|
|
1273
|
-
const text = await fileRes.text();
|
|
1274
|
-
throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`);
|
|
1275
|
-
}
|
|
1276
|
-
return await fileRes.json();
|
|
1277
|
-
}
|
|
1278
|
-
});
|
|
1279
|
-
const fileId = filePayload.name ?? filePayload.file?.name;
|
|
1280
|
-
if (!fileId) throw new Error("gemini batch file upload failed: missing file id");
|
|
1281
|
-
const batchBody = { batch: {
|
|
1282
|
-
displayName: `memory-embeddings-${params.agentId}`,
|
|
1283
|
-
inputConfig: { file_name: fileId }
|
|
1284
|
-
} };
|
|
1285
|
-
const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`;
|
|
1286
|
-
debugEmbeddingsLog("memory embeddings: gemini batch create", {
|
|
1287
|
-
batchEndpoint,
|
|
1288
|
-
fileId
|
|
1289
|
-
});
|
|
1290
|
-
return await withRemoteHttpResponse({
|
|
1291
|
-
url: batchEndpoint,
|
|
1292
|
-
ssrfPolicy: params.gemini.ssrfPolicy,
|
|
1293
|
-
init: {
|
|
1294
|
-
method: "POST",
|
|
1295
|
-
headers: buildBatchHeaders(params.gemini, { json: true }),
|
|
1296
|
-
body: JSON.stringify(batchBody)
|
|
1297
|
-
},
|
|
1298
|
-
onResponse: async (batchRes) => {
|
|
1299
|
-
if (batchRes.ok) return await batchRes.json();
|
|
1300
|
-
const text = await batchRes.text();
|
|
1301
|
-
if (batchRes.status === 404) throw new Error("gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers.");
|
|
1302
|
-
throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`);
|
|
1303
|
-
}
|
|
1304
|
-
});
|
|
1305
|
-
}
|
|
1306
|
-
async function fetchGeminiBatchStatus(params) {
|
|
1307
|
-
const statusUrl = `${normalizeBatchBaseUrl(params.gemini)}/${params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`}`;
|
|
1308
|
-
debugEmbeddingsLog("memory embeddings: gemini batch status", { statusUrl });
|
|
1309
|
-
return await withRemoteHttpResponse({
|
|
1310
|
-
url: statusUrl,
|
|
1311
|
-
ssrfPolicy: params.gemini.ssrfPolicy,
|
|
1312
|
-
init: { headers: buildBatchHeaders(params.gemini, { json: true }) },
|
|
1313
|
-
onResponse: async (res) => {
|
|
1314
|
-
if (!res.ok) {
|
|
1315
|
-
const text = await res.text();
|
|
1316
|
-
throw new Error(`gemini batch status failed: ${res.status} ${text}`);
|
|
1317
|
-
}
|
|
1318
|
-
return await res.json();
|
|
1319
|
-
}
|
|
1320
|
-
});
|
|
1321
|
-
}
|
|
1322
|
-
async function fetchGeminiFileContent(params) {
|
|
1323
|
-
const downloadUrl = `${normalizeBatchBaseUrl(params.gemini)}/${params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`}:download`;
|
|
1324
|
-
debugEmbeddingsLog("memory embeddings: gemini batch download", { downloadUrl });
|
|
1325
|
-
return await withRemoteHttpResponse({
|
|
1326
|
-
url: downloadUrl,
|
|
1327
|
-
ssrfPolicy: params.gemini.ssrfPolicy,
|
|
1328
|
-
init: { headers: buildBatchHeaders(params.gemini, { json: true }) },
|
|
1329
|
-
onResponse: async (res) => {
|
|
1330
|
-
if (!res.ok) {
|
|
1331
|
-
const text = await res.text();
|
|
1332
|
-
throw new Error(`gemini batch file content failed: ${res.status} ${text}`);
|
|
1333
|
-
}
|
|
1334
|
-
return await res.text();
|
|
1335
|
-
}
|
|
1336
|
-
});
|
|
1337
|
-
}
|
|
1338
|
-
function parseGeminiBatchOutput(text) {
|
|
1339
|
-
if (!text.trim()) return [];
|
|
1340
|
-
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
1341
|
-
}
|
|
1342
|
-
async function waitForGeminiBatch(params) {
|
|
1343
|
-
const start = Date.now();
|
|
1344
|
-
let current = params.initial;
|
|
1345
|
-
while (true) {
|
|
1346
|
-
const status = current ?? await fetchGeminiBatchStatus({
|
|
1347
|
-
gemini: params.gemini,
|
|
1348
|
-
batchName: params.batchName
|
|
1349
|
-
});
|
|
1350
|
-
const state = status.state ?? "UNKNOWN";
|
|
1351
|
-
if ([
|
|
1352
|
-
"SUCCEEDED",
|
|
1353
|
-
"COMPLETED",
|
|
1354
|
-
"DONE"
|
|
1355
|
-
].includes(state)) {
|
|
1356
|
-
const outputFileId = status.outputConfig?.file ?? status.outputConfig?.fileId ?? status.metadata?.output?.responsesFile;
|
|
1357
|
-
if (!outputFileId) throw new Error(`gemini batch ${params.batchName} completed without output file`);
|
|
1358
|
-
return { outputFileId };
|
|
1359
|
-
}
|
|
1360
|
-
if ([
|
|
1361
|
-
"FAILED",
|
|
1362
|
-
"CANCELLED",
|
|
1363
|
-
"CANCELED",
|
|
1364
|
-
"EXPIRED"
|
|
1365
|
-
].includes(state)) {
|
|
1366
|
-
const message = status.error?.message ?? "unknown error";
|
|
1367
|
-
throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`);
|
|
1368
|
-
}
|
|
1369
|
-
if (!params.wait) throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`);
|
|
1370
|
-
if (Date.now() - start > params.timeoutMs) throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`);
|
|
1371
|
-
params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
1372
|
-
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
|
1373
|
-
current = void 0;
|
|
1374
|
-
}
|
|
1375
|
-
}
|
|
1376
|
-
async function runGeminiEmbeddingBatches(params) {
|
|
1377
|
-
return await runEmbeddingBatchGroups({
|
|
1378
|
-
...buildEmbeddingBatchGroupOptions(params, {
|
|
1379
|
-
maxRequests: GEMINI_BATCH_MAX_REQUESTS,
|
|
1380
|
-
debugLabel: "memory embeddings: gemini batch submit"
|
|
1381
|
-
}),
|
|
1382
|
-
runGroup: async ({ group, groupIndex, groups, byCustomId }) => {
|
|
1383
|
-
const batchInfo = await submitGeminiBatch({
|
|
1384
|
-
gemini: params.gemini,
|
|
1385
|
-
requests: group,
|
|
1386
|
-
agentId: params.agentId
|
|
1387
|
-
});
|
|
1388
|
-
const batchName = batchInfo.name ?? "";
|
|
1389
|
-
if (!batchName) throw new Error("gemini batch create failed: missing batch name");
|
|
1390
|
-
params.debug?.("memory embeddings: gemini batch created", {
|
|
1391
|
-
batchName,
|
|
1392
|
-
state: batchInfo.state,
|
|
1393
|
-
group: groupIndex + 1,
|
|
1394
|
-
groups,
|
|
1395
|
-
requests: group.length
|
|
1396
|
-
});
|
|
1397
|
-
if (!params.wait && batchInfo.state && ![
|
|
1398
|
-
"SUCCEEDED",
|
|
1399
|
-
"COMPLETED",
|
|
1400
|
-
"DONE"
|
|
1401
|
-
].includes(batchInfo.state)) throw new Error(`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`);
|
|
1402
|
-
const completed = batchInfo.state && [
|
|
1403
|
-
"SUCCEEDED",
|
|
1404
|
-
"COMPLETED",
|
|
1405
|
-
"DONE"
|
|
1406
|
-
].includes(batchInfo.state) ? { outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? "" } : await waitForGeminiBatch({
|
|
1407
|
-
gemini: params.gemini,
|
|
1408
|
-
batchName,
|
|
1409
|
-
wait: params.wait,
|
|
1410
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1411
|
-
timeoutMs: params.timeoutMs,
|
|
1412
|
-
debug: params.debug,
|
|
1413
|
-
initial: batchInfo
|
|
1414
|
-
});
|
|
1415
|
-
if (!completed.outputFileId) throw new Error(`gemini batch ${batchName} completed without output file`);
|
|
1416
|
-
const outputLines = parseGeminiBatchOutput(await fetchGeminiFileContent({
|
|
1417
|
-
gemini: params.gemini,
|
|
1418
|
-
fileId: completed.outputFileId
|
|
1419
|
-
}));
|
|
1420
|
-
const errors = [];
|
|
1421
|
-
const remaining = new Set(group.map((request) => request.custom_id));
|
|
1422
|
-
for (const line of outputLines) {
|
|
1423
|
-
const customId = line.key ?? line.custom_id ?? line.request_id;
|
|
1424
|
-
if (!customId) continue;
|
|
1425
|
-
remaining.delete(customId);
|
|
1426
|
-
if (line.error?.message) {
|
|
1427
|
-
errors.push(`${customId}: ${line.error.message}`);
|
|
1428
|
-
continue;
|
|
1429
|
-
}
|
|
1430
|
-
if (line.response?.error?.message) {
|
|
1431
|
-
errors.push(`${customId}: ${line.response.error.message}`);
|
|
1432
|
-
continue;
|
|
1433
|
-
}
|
|
1434
|
-
const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? [];
|
|
1435
|
-
if (embedding.length === 0) {
|
|
1436
|
-
errors.push(`${customId}: empty embedding`);
|
|
1437
|
-
continue;
|
|
1438
|
-
}
|
|
1439
|
-
byCustomId.set(customId, embedding);
|
|
1440
|
-
}
|
|
1441
|
-
if (errors.length > 0) throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`);
|
|
1442
|
-
if (remaining.size > 0) throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`);
|
|
1443
|
-
}
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
//#endregion
|
|
1448
|
-
//#region src/memory/batch-error-utils.ts
|
|
1449
|
-
function getResponseErrorMessage(line) {
|
|
1450
|
-
const body = line?.response?.body;
|
|
1451
|
-
if (typeof body === "string") return body || void 0;
|
|
1452
|
-
if (!body || typeof body !== "object") return;
|
|
1453
|
-
return typeof body.error?.message === "string" ? body.error.message : void 0;
|
|
1454
|
-
}
|
|
1455
|
-
function extractBatchErrorMessage(lines) {
|
|
1456
|
-
const first = lines.find((line) => line.error?.message || getResponseErrorMessage(line));
|
|
1457
|
-
return first?.error?.message ?? getResponseErrorMessage(first);
|
|
1458
|
-
}
|
|
1459
|
-
function formatUnavailableBatchError(err) {
|
|
1460
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1461
|
-
return message ? `error file unavailable: ${message}` : void 0;
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
|
-
//#endregion
|
|
1465
|
-
//#region src/memory/batch-http.ts
|
|
1466
|
-
async function postJsonWithRetry(params) {
|
|
1467
|
-
return await retryAsync(async () => {
|
|
1468
|
-
return await postJson({
|
|
1469
|
-
url: params.url,
|
|
1470
|
-
headers: params.headers,
|
|
1471
|
-
ssrfPolicy: params.ssrfPolicy,
|
|
1472
|
-
body: params.body,
|
|
1473
|
-
errorPrefix: params.errorPrefix,
|
|
1474
|
-
attachStatus: true,
|
|
1475
|
-
parse: async (payload) => payload
|
|
1476
|
-
});
|
|
1477
|
-
}, {
|
|
1478
|
-
attempts: 3,
|
|
1479
|
-
minDelayMs: 300,
|
|
1480
|
-
maxDelayMs: 2e3,
|
|
1481
|
-
jitter: .2,
|
|
1482
|
-
shouldRetry: (err) => {
|
|
1483
|
-
const status = err.status;
|
|
1484
|
-
return status === 429 || typeof status === "number" && status >= 500;
|
|
1485
|
-
}
|
|
1486
|
-
});
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
//#endregion
|
|
1490
|
-
//#region src/memory/batch-output.ts
|
|
1491
|
-
function applyEmbeddingBatchOutputLine(params) {
|
|
1492
|
-
const customId = params.line.custom_id;
|
|
1493
|
-
if (!customId) return;
|
|
1494
|
-
params.remaining.delete(customId);
|
|
1495
|
-
const errorMessage = params.line.error?.message;
|
|
1496
|
-
if (errorMessage) {
|
|
1497
|
-
params.errors.push(`${customId}: ${errorMessage}`);
|
|
1498
|
-
return;
|
|
1499
|
-
}
|
|
1500
|
-
const response = params.line.response;
|
|
1501
|
-
if ((response?.status_code ?? 0) >= 400) {
|
|
1502
|
-
const messageFromObject = response?.body && typeof response.body === "object" ? response.body.error?.message : void 0;
|
|
1503
|
-
const messageFromString = typeof response?.body === "string" ? response.body : void 0;
|
|
1504
|
-
params.errors.push(`${customId}: ${messageFromObject ?? messageFromString ?? "unknown error"}`);
|
|
1505
|
-
return;
|
|
1506
|
-
}
|
|
1507
|
-
const embedding = (response?.body && typeof response.body === "object" ? response.body.data ?? [] : [])[0]?.embedding ?? [];
|
|
1508
|
-
if (embedding.length === 0) {
|
|
1509
|
-
params.errors.push(`${customId}: empty embedding`);
|
|
1510
|
-
return;
|
|
1511
|
-
}
|
|
1512
|
-
params.byCustomId.set(customId, embedding);
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
|
-
//#endregion
|
|
1516
|
-
//#region src/memory/batch-provider-common.ts
|
|
1517
|
-
const EMBEDDING_BATCH_ENDPOINT = "/v1/embeddings";
|
|
1518
|
-
|
|
1519
|
-
//#endregion
|
|
1520
|
-
//#region src/memory/batch-upload.ts
|
|
1521
|
-
async function uploadBatchJsonlFile(params) {
|
|
1522
|
-
const baseUrl = normalizeBatchBaseUrl(params.client);
|
|
1523
|
-
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
|
|
1524
|
-
const form = new FormData();
|
|
1525
|
-
form.append("purpose", "batch");
|
|
1526
|
-
form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`);
|
|
1527
|
-
const filePayload = await withRemoteHttpResponse({
|
|
1528
|
-
url: `${baseUrl}/files`,
|
|
1529
|
-
ssrfPolicy: params.client.ssrfPolicy,
|
|
1530
|
-
init: {
|
|
1531
|
-
method: "POST",
|
|
1532
|
-
headers: buildBatchHeaders(params.client, { json: false }),
|
|
1533
|
-
body: form
|
|
1534
|
-
},
|
|
1535
|
-
onResponse: async (fileRes) => {
|
|
1536
|
-
if (!fileRes.ok) {
|
|
1537
|
-
const text = await fileRes.text();
|
|
1538
|
-
throw new Error(`${params.errorPrefix}: ${fileRes.status} ${text}`);
|
|
1539
|
-
}
|
|
1540
|
-
return await fileRes.json();
|
|
1541
|
-
}
|
|
1542
|
-
});
|
|
1543
|
-
if (!filePayload.id) throw new Error(`${params.errorPrefix}: missing file id`);
|
|
1544
|
-
return filePayload.id;
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
//#endregion
|
|
1548
|
-
//#region src/memory/batch-openai.ts
|
|
1549
|
-
const OPENAI_BATCH_ENDPOINT = EMBEDDING_BATCH_ENDPOINT;
|
|
1550
|
-
const OPENAI_BATCH_COMPLETION_WINDOW = "24h";
|
|
1551
|
-
const OPENAI_BATCH_MAX_REQUESTS = 5e4;
|
|
1552
|
-
async function submitOpenAiBatch(params) {
|
|
1553
|
-
const baseUrl = normalizeBatchBaseUrl(params.openAi);
|
|
1554
|
-
const inputFileId = await uploadBatchJsonlFile({
|
|
1555
|
-
client: params.openAi,
|
|
1556
|
-
requests: params.requests,
|
|
1557
|
-
errorPrefix: "openai batch file upload failed"
|
|
1558
|
-
});
|
|
1559
|
-
return await postJsonWithRetry({
|
|
1560
|
-
url: `${baseUrl}/batches`,
|
|
1561
|
-
headers: buildBatchHeaders(params.openAi, { json: true }),
|
|
1562
|
-
ssrfPolicy: params.openAi.ssrfPolicy,
|
|
1563
|
-
body: {
|
|
1564
|
-
input_file_id: inputFileId,
|
|
1565
|
-
endpoint: OPENAI_BATCH_ENDPOINT,
|
|
1566
|
-
completion_window: OPENAI_BATCH_COMPLETION_WINDOW,
|
|
1567
|
-
metadata: {
|
|
1568
|
-
source: "squidclaw-memory",
|
|
1569
|
-
agent: params.agentId
|
|
1570
|
-
}
|
|
1571
|
-
},
|
|
1572
|
-
errorPrefix: "openai batch create failed"
|
|
1573
|
-
});
|
|
1574
|
-
}
|
|
1575
|
-
async function fetchOpenAiBatchStatus(params) {
|
|
1576
|
-
return await fetchOpenAiBatchResource({
|
|
1577
|
-
openAi: params.openAi,
|
|
1578
|
-
path: `/batches/${params.batchId}`,
|
|
1579
|
-
errorPrefix: "openai batch status",
|
|
1580
|
-
parse: async (res) => await res.json()
|
|
1581
|
-
});
|
|
1582
|
-
}
|
|
1583
|
-
async function fetchOpenAiFileContent(params) {
|
|
1584
|
-
return await fetchOpenAiBatchResource({
|
|
1585
|
-
openAi: params.openAi,
|
|
1586
|
-
path: `/files/${params.fileId}/content`,
|
|
1587
|
-
errorPrefix: "openai batch file content",
|
|
1588
|
-
parse: async (res) => await res.text()
|
|
1589
|
-
});
|
|
1590
|
-
}
|
|
1591
|
-
async function fetchOpenAiBatchResource(params) {
|
|
1592
|
-
return await withRemoteHttpResponse({
|
|
1593
|
-
url: `${normalizeBatchBaseUrl(params.openAi)}${params.path}`,
|
|
1594
|
-
ssrfPolicy: params.openAi.ssrfPolicy,
|
|
1595
|
-
init: { headers: buildBatchHeaders(params.openAi, { json: true }) },
|
|
1596
|
-
onResponse: async (res) => {
|
|
1597
|
-
if (!res.ok) {
|
|
1598
|
-
const text = await res.text();
|
|
1599
|
-
throw new Error(`${params.errorPrefix} failed: ${res.status} ${text}`);
|
|
1600
|
-
}
|
|
1601
|
-
return await params.parse(res);
|
|
1602
|
-
}
|
|
1603
|
-
});
|
|
1604
|
-
}
|
|
1605
|
-
function parseOpenAiBatchOutput(text) {
|
|
1606
|
-
if (!text.trim()) return [];
|
|
1607
|
-
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
1608
|
-
}
|
|
1609
|
-
async function readOpenAiBatchError(params) {
|
|
1610
|
-
try {
|
|
1611
|
-
return extractBatchErrorMessage(parseOpenAiBatchOutput(await fetchOpenAiFileContent({
|
|
1612
|
-
openAi: params.openAi,
|
|
1613
|
-
fileId: params.errorFileId
|
|
1614
|
-
})));
|
|
1615
|
-
} catch (err) {
|
|
1616
|
-
return formatUnavailableBatchError(err);
|
|
1617
|
-
}
|
|
1618
|
-
}
|
|
1619
|
-
async function waitForOpenAiBatch(params) {
|
|
1620
|
-
const start = Date.now();
|
|
1621
|
-
let current = params.initial;
|
|
1622
|
-
while (true) {
|
|
1623
|
-
const status = current ?? await fetchOpenAiBatchStatus({
|
|
1624
|
-
openAi: params.openAi,
|
|
1625
|
-
batchId: params.batchId
|
|
1626
|
-
});
|
|
1627
|
-
const state = status.status ?? "unknown";
|
|
1628
|
-
if (state === "completed") {
|
|
1629
|
-
if (!status.output_file_id) throw new Error(`openai batch ${params.batchId} completed without output file`);
|
|
1630
|
-
return {
|
|
1631
|
-
outputFileId: status.output_file_id,
|
|
1632
|
-
errorFileId: status.error_file_id ?? void 0
|
|
1633
|
-
};
|
|
1634
|
-
}
|
|
1635
|
-
if ([
|
|
1636
|
-
"failed",
|
|
1637
|
-
"expired",
|
|
1638
|
-
"cancelled",
|
|
1639
|
-
"canceled"
|
|
1640
|
-
].includes(state)) {
|
|
1641
|
-
const detail = status.error_file_id ? await readOpenAiBatchError({
|
|
1642
|
-
openAi: params.openAi,
|
|
1643
|
-
errorFileId: status.error_file_id
|
|
1644
|
-
}) : void 0;
|
|
1645
|
-
const suffix = detail ? `: ${detail}` : "";
|
|
1646
|
-
throw new Error(`openai batch ${params.batchId} ${state}${suffix}`);
|
|
1647
|
-
}
|
|
1648
|
-
if (!params.wait) throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`);
|
|
1649
|
-
if (Date.now() - start > params.timeoutMs) throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
|
1650
|
-
params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
1651
|
-
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
|
1652
|
-
current = void 0;
|
|
1653
|
-
}
|
|
1654
|
-
}
|
|
1655
|
-
async function runOpenAiEmbeddingBatches(params) {
|
|
1656
|
-
return await runEmbeddingBatchGroups({
|
|
1657
|
-
...buildEmbeddingBatchGroupOptions(params, {
|
|
1658
|
-
maxRequests: OPENAI_BATCH_MAX_REQUESTS,
|
|
1659
|
-
debugLabel: "memory embeddings: openai batch submit"
|
|
1660
|
-
}),
|
|
1661
|
-
runGroup: async ({ group, groupIndex, groups, byCustomId }) => {
|
|
1662
|
-
const batchInfo = await submitOpenAiBatch({
|
|
1663
|
-
openAi: params.openAi,
|
|
1664
|
-
requests: group,
|
|
1665
|
-
agentId: params.agentId
|
|
1666
|
-
});
|
|
1667
|
-
if (!batchInfo.id) throw new Error("openai batch create failed: missing batch id");
|
|
1668
|
-
params.debug?.("memory embeddings: openai batch created", {
|
|
1669
|
-
batchId: batchInfo.id,
|
|
1670
|
-
status: batchInfo.status,
|
|
1671
|
-
group: groupIndex + 1,
|
|
1672
|
-
groups,
|
|
1673
|
-
requests: group.length
|
|
1674
|
-
});
|
|
1675
|
-
if (!params.wait && batchInfo.status !== "completed") throw new Error(`openai batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`);
|
|
1676
|
-
const completed = batchInfo.status === "completed" ? {
|
|
1677
|
-
outputFileId: batchInfo.output_file_id ?? "",
|
|
1678
|
-
errorFileId: batchInfo.error_file_id ?? void 0
|
|
1679
|
-
} : await waitForOpenAiBatch({
|
|
1680
|
-
openAi: params.openAi,
|
|
1681
|
-
batchId: batchInfo.id,
|
|
1682
|
-
wait: params.wait,
|
|
1683
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1684
|
-
timeoutMs: params.timeoutMs,
|
|
1685
|
-
debug: params.debug,
|
|
1686
|
-
initial: batchInfo
|
|
1687
|
-
});
|
|
1688
|
-
if (!completed.outputFileId) throw new Error(`openai batch ${batchInfo.id} completed without output file`);
|
|
1689
|
-
const outputLines = parseOpenAiBatchOutput(await fetchOpenAiFileContent({
|
|
1690
|
-
openAi: params.openAi,
|
|
1691
|
-
fileId: completed.outputFileId
|
|
1692
|
-
}));
|
|
1693
|
-
const errors = [];
|
|
1694
|
-
const remaining = new Set(group.map((request) => request.custom_id));
|
|
1695
|
-
for (const line of outputLines) applyEmbeddingBatchOutputLine({
|
|
1696
|
-
line,
|
|
1697
|
-
remaining,
|
|
1698
|
-
errors,
|
|
1699
|
-
byCustomId
|
|
1700
|
-
});
|
|
1701
|
-
if (errors.length > 0) throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
|
1702
|
-
if (remaining.size > 0) throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
|
|
1703
|
-
}
|
|
1704
|
-
});
|
|
1705
|
-
}
|
|
1706
|
-
|
|
1707
|
-
//#endregion
|
|
1708
|
-
//#region src/memory/batch-voyage.ts
|
|
1709
|
-
const VOYAGE_BATCH_ENDPOINT = EMBEDDING_BATCH_ENDPOINT;
|
|
1710
|
-
const VOYAGE_BATCH_COMPLETION_WINDOW = "12h";
|
|
1711
|
-
const VOYAGE_BATCH_MAX_REQUESTS = 5e4;
|
|
1712
|
-
async function assertVoyageResponseOk(res, context) {
|
|
1713
|
-
if (!res.ok) {
|
|
1714
|
-
const text = await res.text();
|
|
1715
|
-
throw new Error(`${context}: ${res.status} ${text}`);
|
|
1716
|
-
}
|
|
1717
|
-
}
|
|
1718
|
-
function buildVoyageBatchRequest(params) {
|
|
1719
|
-
return {
|
|
1720
|
-
url: `${normalizeBatchBaseUrl(params.client)}/${params.path}`,
|
|
1721
|
-
ssrfPolicy: params.client.ssrfPolicy,
|
|
1722
|
-
init: { headers: buildBatchHeaders(params.client, { json: true }) },
|
|
1723
|
-
onResponse: params.onResponse
|
|
1724
|
-
};
|
|
1725
|
-
}
|
|
1726
|
-
async function submitVoyageBatch(params) {
|
|
1727
|
-
const baseUrl = normalizeBatchBaseUrl(params.client);
|
|
1728
|
-
const inputFileId = await uploadBatchJsonlFile({
|
|
1729
|
-
client: params.client,
|
|
1730
|
-
requests: params.requests,
|
|
1731
|
-
errorPrefix: "voyage batch file upload failed"
|
|
1732
|
-
});
|
|
1733
|
-
return await postJsonWithRetry({
|
|
1734
|
-
url: `${baseUrl}/batches`,
|
|
1735
|
-
headers: buildBatchHeaders(params.client, { json: true }),
|
|
1736
|
-
ssrfPolicy: params.client.ssrfPolicy,
|
|
1737
|
-
body: {
|
|
1738
|
-
input_file_id: inputFileId,
|
|
1739
|
-
endpoint: VOYAGE_BATCH_ENDPOINT,
|
|
1740
|
-
completion_window: VOYAGE_BATCH_COMPLETION_WINDOW,
|
|
1741
|
-
request_params: {
|
|
1742
|
-
model: params.client.model,
|
|
1743
|
-
input_type: "document"
|
|
1744
|
-
},
|
|
1745
|
-
metadata: {
|
|
1746
|
-
source: "clawdbot-memory",
|
|
1747
|
-
agent: params.agentId
|
|
1748
|
-
}
|
|
1749
|
-
},
|
|
1750
|
-
errorPrefix: "voyage batch create failed"
|
|
1751
|
-
});
|
|
1752
|
-
}
|
|
1753
|
-
async function fetchVoyageBatchStatus(params) {
|
|
1754
|
-
return await withRemoteHttpResponse(buildVoyageBatchRequest({
|
|
1755
|
-
client: params.client,
|
|
1756
|
-
path: `batches/${params.batchId}`,
|
|
1757
|
-
onResponse: async (res) => {
|
|
1758
|
-
await assertVoyageResponseOk(res, "voyage batch status failed");
|
|
1759
|
-
return await res.json();
|
|
1760
|
-
}
|
|
1761
|
-
}));
|
|
1762
|
-
}
|
|
1763
|
-
async function readVoyageBatchError(params) {
|
|
1764
|
-
try {
|
|
1765
|
-
return await withRemoteHttpResponse(buildVoyageBatchRequest({
|
|
1766
|
-
client: params.client,
|
|
1767
|
-
path: `files/${params.errorFileId}/content`,
|
|
1768
|
-
onResponse: async (res) => {
|
|
1769
|
-
await assertVoyageResponseOk(res, "voyage batch error file content failed");
|
|
1770
|
-
const text = await res.text();
|
|
1771
|
-
if (!text.trim()) return;
|
|
1772
|
-
return extractBatchErrorMessage(text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line)));
|
|
1773
|
-
}
|
|
1774
|
-
}));
|
|
1775
|
-
} catch (err) {
|
|
1776
|
-
return formatUnavailableBatchError(err);
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
async function waitForVoyageBatch(params) {
|
|
1780
|
-
const start = Date.now();
|
|
1781
|
-
let current = params.initial;
|
|
1782
|
-
while (true) {
|
|
1783
|
-
const status = current ?? await fetchVoyageBatchStatus({
|
|
1784
|
-
client: params.client,
|
|
1785
|
-
batchId: params.batchId
|
|
1786
|
-
});
|
|
1787
|
-
const state = status.status ?? "unknown";
|
|
1788
|
-
if (state === "completed") {
|
|
1789
|
-
if (!status.output_file_id) throw new Error(`voyage batch ${params.batchId} completed without output file`);
|
|
1790
|
-
return {
|
|
1791
|
-
outputFileId: status.output_file_id,
|
|
1792
|
-
errorFileId: status.error_file_id ?? void 0
|
|
1793
|
-
};
|
|
1794
|
-
}
|
|
1795
|
-
if ([
|
|
1796
|
-
"failed",
|
|
1797
|
-
"expired",
|
|
1798
|
-
"cancelled",
|
|
1799
|
-
"canceled"
|
|
1800
|
-
].includes(state)) {
|
|
1801
|
-
const detail = status.error_file_id ? await readVoyageBatchError({
|
|
1802
|
-
client: params.client,
|
|
1803
|
-
errorFileId: status.error_file_id
|
|
1804
|
-
}) : void 0;
|
|
1805
|
-
const suffix = detail ? `: ${detail}` : "";
|
|
1806
|
-
throw new Error(`voyage batch ${params.batchId} ${state}${suffix}`);
|
|
1807
|
-
}
|
|
1808
|
-
if (!params.wait) throw new Error(`voyage batch ${params.batchId} still ${state}; wait disabled`);
|
|
1809
|
-
if (Date.now() - start > params.timeoutMs) throw new Error(`voyage batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
|
1810
|
-
params.debug?.(`voyage batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
1811
|
-
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
|
1812
|
-
current = void 0;
|
|
1813
|
-
}
|
|
1814
|
-
}
|
|
1815
|
-
async function runVoyageEmbeddingBatches(params) {
|
|
1816
|
-
return await runEmbeddingBatchGroups({
|
|
1817
|
-
...buildEmbeddingBatchGroupOptions(params, {
|
|
1818
|
-
maxRequests: VOYAGE_BATCH_MAX_REQUESTS,
|
|
1819
|
-
debugLabel: "memory embeddings: voyage batch submit"
|
|
1820
|
-
}),
|
|
1821
|
-
runGroup: async ({ group, groupIndex, groups, byCustomId }) => {
|
|
1822
|
-
const batchInfo = await submitVoyageBatch({
|
|
1823
|
-
client: params.client,
|
|
1824
|
-
requests: group,
|
|
1825
|
-
agentId: params.agentId
|
|
1826
|
-
});
|
|
1827
|
-
if (!batchInfo.id) throw new Error("voyage batch create failed: missing batch id");
|
|
1828
|
-
params.debug?.("memory embeddings: voyage batch created", {
|
|
1829
|
-
batchId: batchInfo.id,
|
|
1830
|
-
status: batchInfo.status,
|
|
1831
|
-
group: groupIndex + 1,
|
|
1832
|
-
groups,
|
|
1833
|
-
requests: group.length
|
|
1834
|
-
});
|
|
1835
|
-
if (!params.wait && batchInfo.status !== "completed") throw new Error(`voyage batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`);
|
|
1836
|
-
const completed = batchInfo.status === "completed" ? {
|
|
1837
|
-
outputFileId: batchInfo.output_file_id ?? "",
|
|
1838
|
-
errorFileId: batchInfo.error_file_id ?? void 0
|
|
1839
|
-
} : await waitForVoyageBatch({
|
|
1840
|
-
client: params.client,
|
|
1841
|
-
batchId: batchInfo.id,
|
|
1842
|
-
wait: params.wait,
|
|
1843
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1844
|
-
timeoutMs: params.timeoutMs,
|
|
1845
|
-
debug: params.debug,
|
|
1846
|
-
initial: batchInfo
|
|
1847
|
-
});
|
|
1848
|
-
if (!completed.outputFileId) throw new Error(`voyage batch ${batchInfo.id} completed without output file`);
|
|
1849
|
-
const baseUrl = normalizeBatchBaseUrl(params.client);
|
|
1850
|
-
const errors = [];
|
|
1851
|
-
const remaining = new Set(group.map((request) => request.custom_id));
|
|
1852
|
-
await withRemoteHttpResponse({
|
|
1853
|
-
url: `${baseUrl}/files/${completed.outputFileId}/content`,
|
|
1854
|
-
ssrfPolicy: params.client.ssrfPolicy,
|
|
1855
|
-
init: { headers: buildBatchHeaders(params.client, { json: true }) },
|
|
1856
|
-
onResponse: async (contentRes) => {
|
|
1857
|
-
if (!contentRes.ok) {
|
|
1858
|
-
const text = await contentRes.text();
|
|
1859
|
-
throw new Error(`voyage batch file content failed: ${contentRes.status} ${text}`);
|
|
1860
|
-
}
|
|
1861
|
-
if (!contentRes.body) return;
|
|
1862
|
-
const reader = createInterface({
|
|
1863
|
-
input: Readable.fromWeb(contentRes.body),
|
|
1864
|
-
terminal: false
|
|
1865
|
-
});
|
|
1866
|
-
for await (const rawLine of reader) {
|
|
1867
|
-
if (!rawLine.trim()) continue;
|
|
1868
|
-
applyEmbeddingBatchOutputLine({
|
|
1869
|
-
line: JSON.parse(rawLine),
|
|
1870
|
-
remaining,
|
|
1871
|
-
errors,
|
|
1872
|
-
byCustomId
|
|
1873
|
-
});
|
|
1874
|
-
}
|
|
1875
|
-
}
|
|
1876
|
-
});
|
|
1877
|
-
if (errors.length > 0) throw new Error(`voyage batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
|
1878
|
-
if (remaining.size > 0) throw new Error(`voyage batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
|
|
1879
|
-
}
|
|
1880
|
-
});
|
|
1881
|
-
}
|
|
1882
|
-
|
|
1883
|
-
//#endregion
|
|
1884
|
-
//#region src/memory/embedding-input-limits.ts
|
|
1885
|
-
function estimateUtf8Bytes(text) {
|
|
1886
|
-
if (!text) return 0;
|
|
1887
|
-
return Buffer.byteLength(text, "utf8");
|
|
1888
|
-
}
|
|
1889
|
-
function splitTextToUtf8ByteLimit(text, maxUtf8Bytes) {
|
|
1890
|
-
if (maxUtf8Bytes <= 0) return [text];
|
|
1891
|
-
if (estimateUtf8Bytes(text) <= maxUtf8Bytes) return [text];
|
|
1892
|
-
const parts = [];
|
|
1893
|
-
let cursor = 0;
|
|
1894
|
-
while (cursor < text.length) {
|
|
1895
|
-
let low = cursor + 1;
|
|
1896
|
-
let high = Math.min(text.length, cursor + maxUtf8Bytes);
|
|
1897
|
-
let best = cursor;
|
|
1898
|
-
while (low <= high) {
|
|
1899
|
-
const mid = Math.floor((low + high) / 2);
|
|
1900
|
-
if (estimateUtf8Bytes(text.slice(cursor, mid)) <= maxUtf8Bytes) {
|
|
1901
|
-
best = mid;
|
|
1902
|
-
low = mid + 1;
|
|
1903
|
-
} else high = mid - 1;
|
|
1904
|
-
}
|
|
1905
|
-
if (best <= cursor) best = Math.min(text.length, cursor + 1);
|
|
1906
|
-
if (best < text.length && best > cursor && text.charCodeAt(best - 1) >= 55296 && text.charCodeAt(best - 1) <= 56319 && text.charCodeAt(best) >= 56320 && text.charCodeAt(best) <= 57343) best -= 1;
|
|
1907
|
-
const part = text.slice(cursor, best);
|
|
1908
|
-
if (!part) break;
|
|
1909
|
-
parts.push(part);
|
|
1910
|
-
cursor = best;
|
|
1911
|
-
}
|
|
1912
|
-
return parts;
|
|
1913
|
-
}
|
|
1914
|
-
|
|
1915
|
-
//#endregion
|
|
1916
|
-
//#region src/memory/embedding-model-limits.ts
|
|
1917
|
-
const DEFAULT_EMBEDDING_MAX_INPUT_TOKENS = 8192;
|
|
1918
|
-
const DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS = 2048;
|
|
1919
|
-
const KNOWN_EMBEDDING_MAX_INPUT_TOKENS = {
|
|
1920
|
-
"openai:text-embedding-3-small": 8192,
|
|
1921
|
-
"openai:text-embedding-3-large": 8192,
|
|
1922
|
-
"openai:text-embedding-ada-002": 8191,
|
|
1923
|
-
"gemini:text-embedding-004": 2048,
|
|
1924
|
-
"voyage:voyage-3": 32e3,
|
|
1925
|
-
"voyage:voyage-3-lite": 16e3,
|
|
1926
|
-
"voyage:voyage-code-3": 32e3
|
|
1927
|
-
};
|
|
1928
|
-
function resolveEmbeddingMaxInputTokens(provider) {
|
|
1929
|
-
if (typeof provider.maxInputTokens === "number") return provider.maxInputTokens;
|
|
1930
|
-
const known = KNOWN_EMBEDDING_MAX_INPUT_TOKENS[`${provider.id}:${provider.model}`.toLowerCase()];
|
|
1931
|
-
if (typeof known === "number") return known;
|
|
1932
|
-
if (provider.id.toLowerCase() === "gemini") return 2048;
|
|
1933
|
-
if (provider.id.toLowerCase() === "local") return DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS;
|
|
1934
|
-
return DEFAULT_EMBEDDING_MAX_INPUT_TOKENS;
|
|
1935
|
-
}
|
|
1936
|
-
|
|
1937
|
-
//#endregion
|
|
1938
|
-
//#region src/memory/embedding-chunk-limits.ts
|
|
1939
|
-
function enforceEmbeddingMaxInputTokens(provider, chunks, hardMaxInputTokens) {
|
|
1940
|
-
const providerMaxInputTokens = resolveEmbeddingMaxInputTokens(provider);
|
|
1941
|
-
const maxInputTokens = typeof hardMaxInputTokens === "number" && hardMaxInputTokens > 0 ? Math.min(providerMaxInputTokens, hardMaxInputTokens) : providerMaxInputTokens;
|
|
1942
|
-
const out = [];
|
|
1943
|
-
for (const chunk of chunks) {
|
|
1944
|
-
if (estimateUtf8Bytes(chunk.text) <= maxInputTokens) {
|
|
1945
|
-
out.push(chunk);
|
|
1946
|
-
continue;
|
|
1947
|
-
}
|
|
1948
|
-
for (const text of splitTextToUtf8ByteLimit(chunk.text, maxInputTokens)) out.push({
|
|
1949
|
-
startLine: chunk.startLine,
|
|
1950
|
-
endLine: chunk.endLine,
|
|
1951
|
-
text,
|
|
1952
|
-
hash: hashText(text)
|
|
1953
|
-
});
|
|
1954
|
-
}
|
|
1955
|
-
return out;
|
|
1956
|
-
}
|
|
1957
|
-
|
|
1958
|
-
//#endregion
|
|
1959
|
-
//#region src/memory/memory-schema.ts
|
|
1960
|
-
function ensureMemoryIndexSchema(params) {
|
|
1961
|
-
params.db.exec(`
|
|
1962
|
-
CREATE TABLE IF NOT EXISTS meta (
|
|
1963
|
-
key TEXT PRIMARY KEY,
|
|
1964
|
-
value TEXT NOT NULL
|
|
1965
|
-
);
|
|
1966
|
-
`);
|
|
1967
|
-
params.db.exec(`
|
|
1968
|
-
CREATE TABLE IF NOT EXISTS files (
|
|
1969
|
-
path TEXT PRIMARY KEY,
|
|
1970
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
1971
|
-
hash TEXT NOT NULL,
|
|
1972
|
-
mtime INTEGER NOT NULL,
|
|
1973
|
-
size INTEGER NOT NULL
|
|
1974
|
-
);
|
|
1975
|
-
`);
|
|
1976
|
-
params.db.exec(`
|
|
1977
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
1978
|
-
id TEXT PRIMARY KEY,
|
|
1979
|
-
path TEXT NOT NULL,
|
|
1980
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
1981
|
-
start_line INTEGER NOT NULL,
|
|
1982
|
-
end_line INTEGER NOT NULL,
|
|
1983
|
-
hash TEXT NOT NULL,
|
|
1984
|
-
model TEXT NOT NULL,
|
|
1985
|
-
text TEXT NOT NULL,
|
|
1986
|
-
embedding TEXT NOT NULL,
|
|
1987
|
-
updated_at INTEGER NOT NULL
|
|
1988
|
-
);
|
|
1989
|
-
`);
|
|
1990
|
-
params.db.exec(`
|
|
1991
|
-
CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
|
|
1992
|
-
provider TEXT NOT NULL,
|
|
1993
|
-
model TEXT NOT NULL,
|
|
1994
|
-
provider_key TEXT NOT NULL,
|
|
1995
|
-
hash TEXT NOT NULL,
|
|
1996
|
-
embedding TEXT NOT NULL,
|
|
1997
|
-
dims INTEGER,
|
|
1998
|
-
updated_at INTEGER NOT NULL,
|
|
1999
|
-
PRIMARY KEY (provider, model, provider_key, hash)
|
|
2000
|
-
);
|
|
2001
|
-
`);
|
|
2002
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`);
|
|
2003
|
-
let ftsAvailable = false;
|
|
2004
|
-
let ftsError;
|
|
2005
|
-
if (params.ftsEnabled) try {
|
|
2006
|
-
params.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`);
|
|
2007
|
-
ftsAvailable = true;
|
|
2008
|
-
} catch (err) {
|
|
2009
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
2010
|
-
ftsAvailable = false;
|
|
2011
|
-
ftsError = message;
|
|
2012
|
-
}
|
|
2013
|
-
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
2014
|
-
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
2015
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
|
|
2016
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
|
|
2017
|
-
return {
|
|
2018
|
-
ftsAvailable,
|
|
2019
|
-
...ftsError ? { ftsError } : {}
|
|
2020
|
-
};
|
|
2021
|
-
}
|
|
2022
|
-
function ensureColumn(db, table, column, definition) {
|
|
2023
|
-
if (db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === column)) return;
|
|
2024
|
-
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
2025
|
-
}
|
|
2026
|
-
|
|
2027
|
-
//#endregion
|
|
2028
|
-
//#region src/memory/sqlite-vec.ts
|
|
2029
|
-
async function loadSqliteVecExtension(params) {
|
|
2030
|
-
try {
|
|
2031
|
-
const sqliteVec = await import("sqlite-vec");
|
|
2032
|
-
const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0;
|
|
2033
|
-
const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
|
|
2034
|
-
params.db.enableLoadExtension(true);
|
|
2035
|
-
if (resolvedPath) params.db.loadExtension(extensionPath);
|
|
2036
|
-
else sqliteVec.load(params.db);
|
|
2037
|
-
return {
|
|
2038
|
-
ok: true,
|
|
2039
|
-
extensionPath
|
|
2040
|
-
};
|
|
2041
|
-
} catch (err) {
|
|
2042
|
-
return {
|
|
2043
|
-
ok: false,
|
|
2044
|
-
error: err instanceof Error ? err.message : String(err)
|
|
2045
|
-
};
|
|
2046
|
-
}
|
|
2047
|
-
}
|
|
2048
|
-
|
|
2049
|
-
//#endregion
|
|
2050
|
-
//#region src/memory/manager-sync-ops.ts
|
|
2051
|
-
const META_KEY = "memory_index_meta_v1";
|
|
2052
|
-
const VECTOR_TABLE$2 = "chunks_vec";
|
|
2053
|
-
const FTS_TABLE$2 = "chunks_fts";
|
|
2054
|
-
const EMBEDDING_CACHE_TABLE$2 = "embedding_cache";
|
|
2055
|
-
const SESSION_DIRTY_DEBOUNCE_MS = 5e3;
|
|
2056
|
-
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
|
2057
|
-
const VECTOR_LOAD_TIMEOUT_MS = 3e4;
|
|
2058
|
-
const IGNORED_MEMORY_WATCH_DIR_NAMES = new Set([
|
|
2059
|
-
".git",
|
|
2060
|
-
"node_modules",
|
|
2061
|
-
".pnpm-store",
|
|
2062
|
-
".venv",
|
|
2063
|
-
"venv",
|
|
2064
|
-
".tox",
|
|
2065
|
-
"__pycache__"
|
|
2066
|
-
]);
|
|
2067
|
-
const log$2 = createSubsystemLogger("memory");
|
|
2068
|
-
function shouldIgnoreMemoryWatchPath(watchPath) {
|
|
2069
|
-
return path.normalize(watchPath).split(path.sep).map((segment) => segment.trim().toLowerCase()).some((segment) => IGNORED_MEMORY_WATCH_DIR_NAMES.has(segment));
|
|
2070
|
-
}
|
|
2071
|
-
var MemoryManagerSyncOps = class {
|
|
2072
|
-
constructor() {
|
|
2073
|
-
this.provider = null;
|
|
2074
|
-
this.sources = /* @__PURE__ */ new Set();
|
|
2075
|
-
this.providerKey = null;
|
|
2076
|
-
this.fts = {
|
|
2077
|
-
enabled: false,
|
|
2078
|
-
available: false
|
|
2079
|
-
};
|
|
2080
|
-
this.vectorReady = null;
|
|
2081
|
-
this.watcher = null;
|
|
2082
|
-
this.watchTimer = null;
|
|
2083
|
-
this.sessionWatchTimer = null;
|
|
2084
|
-
this.sessionUnsubscribe = null;
|
|
2085
|
-
this.intervalTimer = null;
|
|
2086
|
-
this.closed = false;
|
|
2087
|
-
this.dirty = false;
|
|
2088
|
-
this.sessionsDirty = false;
|
|
2089
|
-
this.sessionsDirtyFiles = /* @__PURE__ */ new Set();
|
|
2090
|
-
this.sessionPendingFiles = /* @__PURE__ */ new Set();
|
|
2091
|
-
this.sessionDeltas = /* @__PURE__ */ new Map();
|
|
2092
|
-
this.lastMetaSerialized = null;
|
|
2093
|
-
}
|
|
2094
|
-
async ensureVectorReady(dimensions) {
|
|
2095
|
-
if (!this.vector.enabled) return false;
|
|
2096
|
-
if (!this.vectorReady) this.vectorReady = this.withTimeout(this.loadVectorExtension(), VECTOR_LOAD_TIMEOUT_MS, `sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1e3)}s`);
|
|
2097
|
-
let ready = false;
|
|
2098
|
-
try {
|
|
2099
|
-
ready = await this.vectorReady || false;
|
|
2100
|
-
} catch (err) {
|
|
2101
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
2102
|
-
this.vector.available = false;
|
|
2103
|
-
this.vector.loadError = message;
|
|
2104
|
-
this.vectorReady = null;
|
|
2105
|
-
log$2.warn(`sqlite-vec unavailable: ${message}`);
|
|
2106
|
-
return false;
|
|
2107
|
-
}
|
|
2108
|
-
if (ready && typeof dimensions === "number" && dimensions > 0) this.ensureVectorTable(dimensions);
|
|
2109
|
-
return ready;
|
|
2110
|
-
}
|
|
2111
|
-
async loadVectorExtension() {
|
|
2112
|
-
if (this.vector.available !== null) return this.vector.available;
|
|
2113
|
-
if (!this.vector.enabled) {
|
|
2114
|
-
this.vector.available = false;
|
|
2115
|
-
return false;
|
|
2116
|
-
}
|
|
2117
|
-
try {
|
|
2118
|
-
const resolvedPath = this.vector.extensionPath?.trim() ? resolveUserPath(this.vector.extensionPath) : void 0;
|
|
2119
|
-
const loaded = await loadSqliteVecExtension({
|
|
2120
|
-
db: this.db,
|
|
2121
|
-
extensionPath: resolvedPath
|
|
2122
|
-
});
|
|
2123
|
-
if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
|
2124
|
-
this.vector.extensionPath = loaded.extensionPath;
|
|
2125
|
-
this.vector.available = true;
|
|
2126
|
-
return true;
|
|
2127
|
-
} catch (err) {
|
|
2128
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
2129
|
-
this.vector.available = false;
|
|
2130
|
-
this.vector.loadError = message;
|
|
2131
|
-
log$2.warn(`sqlite-vec unavailable: ${message}`);
|
|
2132
|
-
return false;
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2135
|
-
ensureVectorTable(dimensions) {
|
|
2136
|
-
if (this.vector.dims === dimensions) return;
|
|
2137
|
-
if (this.vector.dims && this.vector.dims !== dimensions) this.dropVectorTable();
|
|
2138
|
-
this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE$2} USING vec0(\n id TEXT PRIMARY KEY,\n embedding FLOAT[${dimensions}]\n)`);
|
|
2139
|
-
this.vector.dims = dimensions;
|
|
2140
|
-
}
|
|
2141
|
-
dropVectorTable() {
|
|
2142
|
-
try {
|
|
2143
|
-
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE$2}`);
|
|
2144
|
-
} catch (err) {
|
|
2145
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
2146
|
-
log$2.debug(`Failed to drop ${VECTOR_TABLE$2}: ${message}`);
|
|
2147
|
-
}
|
|
2148
|
-
}
|
|
2149
|
-
buildSourceFilter(alias) {
|
|
2150
|
-
const sources = Array.from(this.sources);
|
|
2151
|
-
if (sources.length === 0) return {
|
|
2152
|
-
sql: "",
|
|
2153
|
-
params: []
|
|
2154
|
-
};
|
|
2155
|
-
return {
|
|
2156
|
-
sql: ` AND ${alias ? `${alias}.source` : "source"} IN (${sources.map(() => "?").join(", ")})`,
|
|
2157
|
-
params: sources
|
|
2158
|
-
};
|
|
2159
|
-
}
|
|
2160
|
-
openDatabase() {
|
|
2161
|
-
const dbPath = resolveUserPath(this.settings.store.path);
|
|
2162
|
-
return this.openDatabaseAtPath(dbPath);
|
|
2163
|
-
}
|
|
2164
|
-
openDatabaseAtPath(dbPath) {
|
|
2165
|
-
ensureDir(path.dirname(dbPath));
|
|
2166
|
-
const { DatabaseSync } = requireNodeSqlite();
|
|
2167
|
-
return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled });
|
|
2168
|
-
}
|
|
2169
|
-
seedEmbeddingCache(sourceDb) {
|
|
2170
|
-
if (!this.cache.enabled) return;
|
|
2171
|
-
try {
|
|
2172
|
-
const rows = sourceDb.prepare(`SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${EMBEDDING_CACHE_TABLE$2}`).all();
|
|
2173
|
-
if (!rows.length) return;
|
|
2174
|
-
const insert = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE$2} (provider, model, provider_key, hash, embedding, dims, updated_at)
|
|
2175
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
2176
|
-
ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET
|
|
2177
|
-
embedding=excluded.embedding,
|
|
2178
|
-
dims=excluded.dims,
|
|
2179
|
-
updated_at=excluded.updated_at`);
|
|
2180
|
-
this.db.exec("BEGIN");
|
|
2181
|
-
for (const row of rows) insert.run(row.provider, row.model, row.provider_key, row.hash, row.embedding, row.dims, row.updated_at);
|
|
2182
|
-
this.db.exec("COMMIT");
|
|
2183
|
-
} catch (err) {
|
|
2184
|
-
try {
|
|
2185
|
-
this.db.exec("ROLLBACK");
|
|
2186
|
-
} catch {}
|
|
2187
|
-
throw err;
|
|
2188
|
-
}
|
|
2189
|
-
}
|
|
2190
|
-
async swapIndexFiles(targetPath, tempPath) {
|
|
2191
|
-
const backupPath = `${targetPath}.backup-${randomUUID()}`;
|
|
2192
|
-
await this.moveIndexFiles(targetPath, backupPath);
|
|
2193
|
-
try {
|
|
2194
|
-
await this.moveIndexFiles(tempPath, targetPath);
|
|
2195
|
-
} catch (err) {
|
|
2196
|
-
await this.moveIndexFiles(backupPath, targetPath);
|
|
2197
|
-
throw err;
|
|
2198
|
-
}
|
|
2199
|
-
await this.removeIndexFiles(backupPath);
|
|
2200
|
-
}
|
|
2201
|
-
async moveIndexFiles(sourceBase, targetBase) {
|
|
2202
|
-
for (const suffix of [
|
|
2203
|
-
"",
|
|
2204
|
-
"-wal",
|
|
2205
|
-
"-shm"
|
|
2206
|
-
]) {
|
|
2207
|
-
const source = `${sourceBase}${suffix}`;
|
|
2208
|
-
const target = `${targetBase}${suffix}`;
|
|
2209
|
-
try {
|
|
2210
|
-
await fs$1.rename(source, target);
|
|
2211
|
-
} catch (err) {
|
|
2212
|
-
if (err.code !== "ENOENT") throw err;
|
|
2213
|
-
}
|
|
2214
|
-
}
|
|
2215
|
-
}
|
|
2216
|
-
async removeIndexFiles(basePath) {
|
|
2217
|
-
await Promise.all([
|
|
2218
|
-
"",
|
|
2219
|
-
"-wal",
|
|
2220
|
-
"-shm"
|
|
2221
|
-
].map((suffix) => fs$1.rm(`${basePath}${suffix}`, { force: true })));
|
|
2222
|
-
}
|
|
2223
|
-
ensureSchema() {
|
|
2224
|
-
const result = ensureMemoryIndexSchema({
|
|
2225
|
-
db: this.db,
|
|
2226
|
-
embeddingCacheTable: EMBEDDING_CACHE_TABLE$2,
|
|
2227
|
-
ftsTable: FTS_TABLE$2,
|
|
2228
|
-
ftsEnabled: this.fts.enabled
|
|
2229
|
-
});
|
|
2230
|
-
this.fts.available = result.ftsAvailable;
|
|
2231
|
-
if (result.ftsError) {
|
|
2232
|
-
this.fts.loadError = result.ftsError;
|
|
2233
|
-
if (this.fts.enabled) log$2.warn(`fts unavailable: ${result.ftsError}`);
|
|
2234
|
-
}
|
|
2235
|
-
}
|
|
2236
|
-
ensureWatcher() {
|
|
2237
|
-
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
|
|
2238
|
-
const watchPaths = new Set([
|
|
2239
|
-
path.join(this.workspaceDir, "MEMORY.md"),
|
|
2240
|
-
path.join(this.workspaceDir, "memory.md"),
|
|
2241
|
-
path.join(this.workspaceDir, "memory", "**", "*.md")
|
|
2242
|
-
]);
|
|
2243
|
-
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths);
|
|
2244
|
-
for (const entry of additionalPaths) try {
|
|
2245
|
-
const stat = fs.lstatSync(entry);
|
|
2246
|
-
if (stat.isSymbolicLink()) continue;
|
|
2247
|
-
if (stat.isDirectory()) {
|
|
2248
|
-
watchPaths.add(path.join(entry, "**", "*.md"));
|
|
2249
|
-
continue;
|
|
2250
|
-
}
|
|
2251
|
-
if (stat.isFile() && entry.toLowerCase().endsWith(".md")) watchPaths.add(entry);
|
|
2252
|
-
} catch {}
|
|
2253
|
-
this.watcher = chokidar.watch(Array.from(watchPaths), {
|
|
2254
|
-
ignoreInitial: true,
|
|
2255
|
-
ignored: (watchPath) => shouldIgnoreMemoryWatchPath(String(watchPath)),
|
|
2256
|
-
awaitWriteFinish: {
|
|
2257
|
-
stabilityThreshold: this.settings.sync.watchDebounceMs,
|
|
2258
|
-
pollInterval: 100
|
|
2259
|
-
}
|
|
2260
|
-
});
|
|
2261
|
-
const markDirty = () => {
|
|
2262
|
-
this.dirty = true;
|
|
2263
|
-
this.scheduleWatchSync();
|
|
2264
|
-
};
|
|
2265
|
-
this.watcher.on("add", markDirty);
|
|
2266
|
-
this.watcher.on("change", markDirty);
|
|
2267
|
-
this.watcher.on("unlink", markDirty);
|
|
2268
|
-
}
|
|
2269
|
-
ensureSessionListener() {
|
|
2270
|
-
if (!this.sources.has("sessions") || this.sessionUnsubscribe) return;
|
|
2271
|
-
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
|
|
2272
|
-
if (this.closed) return;
|
|
2273
|
-
const sessionFile = update.sessionFile;
|
|
2274
|
-
if (!this.isSessionFileForAgent(sessionFile)) return;
|
|
2275
|
-
this.scheduleSessionDirty(sessionFile);
|
|
2276
|
-
});
|
|
2277
|
-
}
|
|
2278
|
-
scheduleSessionDirty(sessionFile) {
|
|
2279
|
-
this.sessionPendingFiles.add(sessionFile);
|
|
2280
|
-
if (this.sessionWatchTimer) return;
|
|
2281
|
-
this.sessionWatchTimer = setTimeout(() => {
|
|
2282
|
-
this.sessionWatchTimer = null;
|
|
2283
|
-
this.processSessionDeltaBatch().catch((err) => {
|
|
2284
|
-
log$2.warn(`memory session delta failed: ${String(err)}`);
|
|
2285
|
-
});
|
|
2286
|
-
}, SESSION_DIRTY_DEBOUNCE_MS);
|
|
2287
|
-
}
|
|
2288
|
-
async processSessionDeltaBatch() {
|
|
2289
|
-
if (this.sessionPendingFiles.size === 0) return;
|
|
2290
|
-
const pending = Array.from(this.sessionPendingFiles);
|
|
2291
|
-
this.sessionPendingFiles.clear();
|
|
2292
|
-
let shouldSync = false;
|
|
2293
|
-
for (const sessionFile of pending) {
|
|
2294
|
-
const delta = await this.updateSessionDelta(sessionFile);
|
|
2295
|
-
if (!delta) continue;
|
|
2296
|
-
const bytesThreshold = delta.deltaBytes;
|
|
2297
|
-
const messagesThreshold = delta.deltaMessages;
|
|
2298
|
-
const bytesHit = bytesThreshold <= 0 ? delta.pendingBytes > 0 : delta.pendingBytes >= bytesThreshold;
|
|
2299
|
-
const messagesHit = messagesThreshold <= 0 ? delta.pendingMessages > 0 : delta.pendingMessages >= messagesThreshold;
|
|
2300
|
-
if (!bytesHit && !messagesHit) continue;
|
|
2301
|
-
this.sessionsDirtyFiles.add(sessionFile);
|
|
2302
|
-
this.sessionsDirty = true;
|
|
2303
|
-
delta.pendingBytes = bytesThreshold > 0 ? Math.max(0, delta.pendingBytes - bytesThreshold) : 0;
|
|
2304
|
-
delta.pendingMessages = messagesThreshold > 0 ? Math.max(0, delta.pendingMessages - messagesThreshold) : 0;
|
|
2305
|
-
shouldSync = true;
|
|
2306
|
-
}
|
|
2307
|
-
if (shouldSync) this.sync({ reason: "session-delta" }).catch((err) => {
|
|
2308
|
-
log$2.warn(`memory sync failed (session-delta): ${String(err)}`);
|
|
2309
|
-
});
|
|
2310
|
-
}
|
|
2311
|
-
async updateSessionDelta(sessionFile) {
|
|
2312
|
-
const thresholds = this.settings.sync.sessions;
|
|
2313
|
-
if (!thresholds) return null;
|
|
2314
|
-
let stat;
|
|
2315
|
-
try {
|
|
2316
|
-
stat = await fs$1.stat(sessionFile);
|
|
2317
|
-
} catch {
|
|
2318
|
-
return null;
|
|
2319
|
-
}
|
|
2320
|
-
const size = stat.size;
|
|
2321
|
-
let state = this.sessionDeltas.get(sessionFile);
|
|
2322
|
-
if (!state) {
|
|
2323
|
-
state = {
|
|
2324
|
-
lastSize: 0,
|
|
2325
|
-
pendingBytes: 0,
|
|
2326
|
-
pendingMessages: 0
|
|
2327
|
-
};
|
|
2328
|
-
this.sessionDeltas.set(sessionFile, state);
|
|
2329
|
-
}
|
|
2330
|
-
const deltaBytes = Math.max(0, size - state.lastSize);
|
|
2331
|
-
if (deltaBytes === 0 && size === state.lastSize) return {
|
|
2332
|
-
deltaBytes: thresholds.deltaBytes,
|
|
2333
|
-
deltaMessages: thresholds.deltaMessages,
|
|
2334
|
-
pendingBytes: state.pendingBytes,
|
|
2335
|
-
pendingMessages: state.pendingMessages
|
|
2336
|
-
};
|
|
2337
|
-
if (size < state.lastSize) {
|
|
2338
|
-
state.lastSize = size;
|
|
2339
|
-
state.pendingBytes += size;
|
|
2340
|
-
if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, 0, size);
|
|
2341
|
-
} else {
|
|
2342
|
-
state.pendingBytes += deltaBytes;
|
|
2343
|
-
if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, state.lastSize, size);
|
|
2344
|
-
state.lastSize = size;
|
|
2345
|
-
}
|
|
2346
|
-
this.sessionDeltas.set(sessionFile, state);
|
|
2347
|
-
return {
|
|
2348
|
-
deltaBytes: thresholds.deltaBytes,
|
|
2349
|
-
deltaMessages: thresholds.deltaMessages,
|
|
2350
|
-
pendingBytes: state.pendingBytes,
|
|
2351
|
-
pendingMessages: state.pendingMessages
|
|
2352
|
-
};
|
|
2353
|
-
}
|
|
2354
|
-
async countNewlines(absPath, start, end) {
|
|
2355
|
-
if (end <= start) return 0;
|
|
2356
|
-
let handle;
|
|
2357
|
-
try {
|
|
2358
|
-
handle = await fs$1.open(absPath, "r");
|
|
2359
|
-
} catch (err) {
|
|
2360
|
-
if (isFileMissingError(err)) return 0;
|
|
2361
|
-
throw err;
|
|
2362
|
-
}
|
|
2363
|
-
try {
|
|
2364
|
-
let offset = start;
|
|
2365
|
-
let count = 0;
|
|
2366
|
-
const buffer = Buffer.alloc(SESSION_DELTA_READ_CHUNK_BYTES);
|
|
2367
|
-
while (offset < end) {
|
|
2368
|
-
const toRead = Math.min(buffer.length, end - offset);
|
|
2369
|
-
const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
|
|
2370
|
-
if (bytesRead <= 0) break;
|
|
2371
|
-
for (let i = 0; i < bytesRead; i += 1) if (buffer[i] === 10) count += 1;
|
|
2372
|
-
offset += bytesRead;
|
|
2373
|
-
}
|
|
2374
|
-
return count;
|
|
2375
|
-
} finally {
|
|
2376
|
-
await handle.close();
|
|
2377
|
-
}
|
|
2378
|
-
}
|
|
2379
|
-
resetSessionDelta(absPath, size) {
|
|
2380
|
-
const state = this.sessionDeltas.get(absPath);
|
|
2381
|
-
if (!state) return;
|
|
2382
|
-
state.lastSize = size;
|
|
2383
|
-
state.pendingBytes = 0;
|
|
2384
|
-
state.pendingMessages = 0;
|
|
2385
|
-
}
|
|
2386
|
-
isSessionFileForAgent(sessionFile) {
|
|
2387
|
-
if (!sessionFile) return false;
|
|
2388
|
-
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
|
|
2389
|
-
const resolvedFile = path.resolve(sessionFile);
|
|
2390
|
-
const resolvedDir = path.resolve(sessionsDir);
|
|
2391
|
-
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
|
|
2392
|
-
}
|
|
2393
|
-
ensureIntervalSync() {
|
|
2394
|
-
const minutes = this.settings.sync.intervalMinutes;
|
|
2395
|
-
if (!minutes || minutes <= 0 || this.intervalTimer) return;
|
|
2396
|
-
const ms = minutes * 60 * 1e3;
|
|
2397
|
-
this.intervalTimer = setInterval(() => {
|
|
2398
|
-
this.sync({ reason: "interval" }).catch((err) => {
|
|
2399
|
-
log$2.warn(`memory sync failed (interval): ${String(err)}`);
|
|
2400
|
-
});
|
|
2401
|
-
}, ms);
|
|
2402
|
-
}
|
|
2403
|
-
scheduleWatchSync() {
|
|
2404
|
-
if (!this.sources.has("memory") || !this.settings.sync.watch) return;
|
|
2405
|
-
if (this.watchTimer) clearTimeout(this.watchTimer);
|
|
2406
|
-
this.watchTimer = setTimeout(() => {
|
|
2407
|
-
this.watchTimer = null;
|
|
2408
|
-
this.sync({ reason: "watch" }).catch((err) => {
|
|
2409
|
-
log$2.warn(`memory sync failed (watch): ${String(err)}`);
|
|
2410
|
-
});
|
|
2411
|
-
}, this.settings.sync.watchDebounceMs);
|
|
2412
|
-
}
|
|
2413
|
-
shouldSyncSessions(params, needsFullReindex = false) {
|
|
2414
|
-
if (!this.sources.has("sessions")) return false;
|
|
2415
|
-
if (params?.force) return true;
|
|
2416
|
-
const reason = params?.reason;
|
|
2417
|
-
if (reason === "session-start" || reason === "watch") return false;
|
|
2418
|
-
if (needsFullReindex) return true;
|
|
2419
|
-
return this.sessionsDirty && this.sessionsDirtyFiles.size > 0;
|
|
2420
|
-
}
|
|
2421
|
-
async syncMemoryFiles(params) {
|
|
2422
|
-
if (!this.provider) {
|
|
2423
|
-
log$2.debug("Skipping memory file sync in FTS-only mode (no embedding provider)");
|
|
2424
|
-
return;
|
|
2425
|
-
}
|
|
2426
|
-
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
|
|
2427
|
-
const fileEntries = (await Promise.all(files.map(async (file) => buildFileEntry(file, this.workspaceDir)))).filter((entry) => entry !== null);
|
|
2428
|
-
log$2.debug("memory sync: indexing memory files", {
|
|
2429
|
-
files: fileEntries.length,
|
|
2430
|
-
needsFullReindex: params.needsFullReindex,
|
|
2431
|
-
batch: this.batch.enabled,
|
|
2432
|
-
concurrency: this.getIndexConcurrency()
|
|
2433
|
-
});
|
|
2434
|
-
const activePaths = new Set(fileEntries.map((entry) => entry.path));
|
|
2435
|
-
if (params.progress) {
|
|
2436
|
-
params.progress.total += fileEntries.length;
|
|
2437
|
-
params.progress.report({
|
|
2438
|
-
completed: params.progress.completed,
|
|
2439
|
-
total: params.progress.total,
|
|
2440
|
-
label: this.batch.enabled ? "Indexing memory files (batch)..." : "Indexing memory files…"
|
|
2441
|
-
});
|
|
2442
|
-
}
|
|
2443
|
-
await runWithConcurrency(fileEntries.map((entry) => async () => {
|
|
2444
|
-
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "memory");
|
|
2445
|
-
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
|
2446
|
-
if (params.progress) {
|
|
2447
|
-
params.progress.completed += 1;
|
|
2448
|
-
params.progress.report({
|
|
2449
|
-
completed: params.progress.completed,
|
|
2450
|
-
total: params.progress.total
|
|
2451
|
-
});
|
|
2452
|
-
}
|
|
2453
|
-
return;
|
|
2454
|
-
}
|
|
2455
|
-
await this.indexFile(entry, { source: "memory" });
|
|
2456
|
-
if (params.progress) {
|
|
2457
|
-
params.progress.completed += 1;
|
|
2458
|
-
params.progress.report({
|
|
2459
|
-
completed: params.progress.completed,
|
|
2460
|
-
total: params.progress.total
|
|
2461
|
-
});
|
|
2462
|
-
}
|
|
2463
|
-
}), this.getIndexConcurrency());
|
|
2464
|
-
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("memory");
|
|
2465
|
-
for (const stale of staleRows) {
|
|
2466
|
-
if (activePaths.has(stale.path)) continue;
|
|
2467
|
-
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
2468
|
-
try {
|
|
2469
|
-
this.db.prepare(`DELETE FROM ${VECTOR_TABLE$2} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "memory");
|
|
2470
|
-
} catch {}
|
|
2471
|
-
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
2472
|
-
if (this.fts.enabled && this.fts.available) try {
|
|
2473
|
-
this.db.prepare(`DELETE FROM ${FTS_TABLE$2} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "memory", this.provider.model);
|
|
2474
|
-
} catch {}
|
|
2475
|
-
}
|
|
2476
|
-
}
|
|
2477
|
-
async syncSessionFiles(params) {
|
|
2478
|
-
if (!this.provider) {
|
|
2479
|
-
log$2.debug("Skipping session file sync in FTS-only mode (no embedding provider)");
|
|
2480
|
-
return;
|
|
2481
|
-
}
|
|
2482
|
-
const files = await listSessionFilesForAgent(this.agentId);
|
|
2483
|
-
const activePaths = new Set(files.map((file) => sessionPathForFile(file)));
|
|
2484
|
-
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
|
2485
|
-
log$2.debug("memory sync: indexing session files", {
|
|
2486
|
-
files: files.length,
|
|
2487
|
-
indexAll,
|
|
2488
|
-
dirtyFiles: this.sessionsDirtyFiles.size,
|
|
2489
|
-
batch: this.batch.enabled,
|
|
2490
|
-
concurrency: this.getIndexConcurrency()
|
|
2491
|
-
});
|
|
2492
|
-
if (params.progress) {
|
|
2493
|
-
params.progress.total += files.length;
|
|
2494
|
-
params.progress.report({
|
|
2495
|
-
completed: params.progress.completed,
|
|
2496
|
-
total: params.progress.total,
|
|
2497
|
-
label: this.batch.enabled ? "Indexing session files (batch)..." : "Indexing session files…"
|
|
2498
|
-
});
|
|
2499
|
-
}
|
|
2500
|
-
await runWithConcurrency(files.map((absPath) => async () => {
|
|
2501
|
-
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) {
|
|
2502
|
-
if (params.progress) {
|
|
2503
|
-
params.progress.completed += 1;
|
|
2504
|
-
params.progress.report({
|
|
2505
|
-
completed: params.progress.completed,
|
|
2506
|
-
total: params.progress.total
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
return;
|
|
2510
|
-
}
|
|
2511
|
-
const entry = await buildSessionEntry(absPath);
|
|
2512
|
-
if (!entry) {
|
|
2513
|
-
if (params.progress) {
|
|
2514
|
-
params.progress.completed += 1;
|
|
2515
|
-
params.progress.report({
|
|
2516
|
-
completed: params.progress.completed,
|
|
2517
|
-
total: params.progress.total
|
|
2518
|
-
});
|
|
2519
|
-
}
|
|
2520
|
-
return;
|
|
2521
|
-
}
|
|
2522
|
-
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "sessions");
|
|
2523
|
-
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
|
2524
|
-
if (params.progress) {
|
|
2525
|
-
params.progress.completed += 1;
|
|
2526
|
-
params.progress.report({
|
|
2527
|
-
completed: params.progress.completed,
|
|
2528
|
-
total: params.progress.total
|
|
2529
|
-
});
|
|
2530
|
-
}
|
|
2531
|
-
this.resetSessionDelta(absPath, entry.size);
|
|
2532
|
-
return;
|
|
2533
|
-
}
|
|
2534
|
-
await this.indexFile(entry, {
|
|
2535
|
-
source: "sessions",
|
|
2536
|
-
content: entry.content
|
|
2537
|
-
});
|
|
2538
|
-
this.resetSessionDelta(absPath, entry.size);
|
|
2539
|
-
if (params.progress) {
|
|
2540
|
-
params.progress.completed += 1;
|
|
2541
|
-
params.progress.report({
|
|
2542
|
-
completed: params.progress.completed,
|
|
2543
|
-
total: params.progress.total
|
|
2544
|
-
});
|
|
2545
|
-
}
|
|
2546
|
-
}), this.getIndexConcurrency());
|
|
2547
|
-
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("sessions");
|
|
2548
|
-
for (const stale of staleRows) {
|
|
2549
|
-
if (activePaths.has(stale.path)) continue;
|
|
2550
|
-
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "sessions");
|
|
2551
|
-
try {
|
|
2552
|
-
this.db.prepare(`DELETE FROM ${VECTOR_TABLE$2} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "sessions");
|
|
2553
|
-
} catch {}
|
|
2554
|
-
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "sessions");
|
|
2555
|
-
if (this.fts.enabled && this.fts.available) try {
|
|
2556
|
-
this.db.prepare(`DELETE FROM ${FTS_TABLE$2} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "sessions", this.provider.model);
|
|
2557
|
-
} catch {}
|
|
2558
|
-
}
|
|
2559
|
-
}
|
|
2560
|
-
createSyncProgress(onProgress) {
|
|
2561
|
-
const state = {
|
|
2562
|
-
completed: 0,
|
|
2563
|
-
total: 0,
|
|
2564
|
-
label: void 0,
|
|
2565
|
-
report: (update) => {
|
|
2566
|
-
if (update.label) state.label = update.label;
|
|
2567
|
-
const label = update.total > 0 && state.label ? `${state.label} ${update.completed}/${update.total}` : state.label;
|
|
2568
|
-
onProgress({
|
|
2569
|
-
completed: update.completed,
|
|
2570
|
-
total: update.total,
|
|
2571
|
-
label
|
|
2572
|
-
});
|
|
2573
|
-
}
|
|
2574
|
-
};
|
|
2575
|
-
return state;
|
|
2576
|
-
}
|
|
2577
|
-
async runSync(params) {
|
|
2578
|
-
const progress = params?.progress ? this.createSyncProgress(params.progress) : void 0;
|
|
2579
|
-
if (progress) progress.report({
|
|
2580
|
-
completed: progress.completed,
|
|
2581
|
-
total: progress.total,
|
|
2582
|
-
label: "Loading vector extension…"
|
|
2583
|
-
});
|
|
2584
|
-
const vectorReady = await this.ensureVectorReady();
|
|
2585
|
-
const meta = this.readMeta();
|
|
2586
|
-
const configuredSources = this.resolveConfiguredSourcesForMeta();
|
|
2587
|
-
const needsFullReindex = params?.force || !meta || this.provider && meta.model !== this.provider.model || this.provider && meta.provider !== this.provider.id || meta.providerKey !== this.providerKey || this.metaSourcesDiffer(meta, configuredSources) || meta.chunkTokens !== this.settings.chunking.tokens || meta.chunkOverlap !== this.settings.chunking.overlap || vectorReady && !meta?.vectorDims;
|
|
2588
|
-
try {
|
|
2589
|
-
if (needsFullReindex) {
|
|
2590
|
-
if (process.env.SQUIDCLAW_TEST_FAST === "1" && process.env.SQUIDCLAW_TEST_MEMORY_UNSAFE_REINDEX === "1") await this.runUnsafeReindex({
|
|
2591
|
-
reason: params?.reason,
|
|
2592
|
-
force: params?.force,
|
|
2593
|
-
progress: progress ?? void 0
|
|
2594
|
-
});
|
|
2595
|
-
else await this.runSafeReindex({
|
|
2596
|
-
reason: params?.reason,
|
|
2597
|
-
force: params?.force,
|
|
2598
|
-
progress: progress ?? void 0
|
|
2599
|
-
});
|
|
2600
|
-
return;
|
|
2601
|
-
}
|
|
2602
|
-
const shouldSyncMemory = this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty);
|
|
2603
|
-
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
|
2604
|
-
if (shouldSyncMemory) {
|
|
2605
|
-
await this.syncMemoryFiles({
|
|
2606
|
-
needsFullReindex,
|
|
2607
|
-
progress: progress ?? void 0
|
|
2608
|
-
});
|
|
2609
|
-
this.dirty = false;
|
|
2610
|
-
}
|
|
2611
|
-
if (shouldSyncSessions) {
|
|
2612
|
-
await this.syncSessionFiles({
|
|
2613
|
-
needsFullReindex,
|
|
2614
|
-
progress: progress ?? void 0
|
|
2615
|
-
});
|
|
2616
|
-
this.sessionsDirty = false;
|
|
2617
|
-
this.sessionsDirtyFiles.clear();
|
|
2618
|
-
} else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true;
|
|
2619
|
-
else this.sessionsDirty = false;
|
|
2620
|
-
} catch (err) {
|
|
2621
|
-
const reason = err instanceof Error ? err.message : String(err);
|
|
2622
|
-
if (this.shouldFallbackOnError(reason) && await this.activateFallbackProvider(reason)) {
|
|
2623
|
-
await this.runSafeReindex({
|
|
2624
|
-
reason: params?.reason ?? "fallback",
|
|
2625
|
-
force: true,
|
|
2626
|
-
progress: progress ?? void 0
|
|
2627
|
-
});
|
|
2628
|
-
return;
|
|
2629
|
-
}
|
|
2630
|
-
throw err;
|
|
2631
|
-
}
|
|
2632
|
-
}
|
|
2633
|
-
shouldFallbackOnError(message) {
|
|
2634
|
-
return /embedding|embeddings|batch/i.test(message);
|
|
2635
|
-
}
|
|
2636
|
-
resolveBatchConfig() {
|
|
2637
|
-
const batch = this.settings.remote?.batch;
|
|
2638
|
-
return {
|
|
2639
|
-
enabled: Boolean(batch?.enabled && this.provider && (this.openAi && this.provider.id === "openai" || this.gemini && this.provider.id === "gemini" || this.voyage && this.provider.id === "voyage")),
|
|
2640
|
-
wait: batch?.wait ?? true,
|
|
2641
|
-
concurrency: Math.max(1, batch?.concurrency ?? 2),
|
|
2642
|
-
pollIntervalMs: batch?.pollIntervalMs ?? 2e3,
|
|
2643
|
-
timeoutMs: (batch?.timeoutMinutes ?? 60) * 60 * 1e3
|
|
2644
|
-
};
|
|
2645
|
-
}
|
|
2646
|
-
async activateFallbackProvider(reason) {
|
|
2647
|
-
const fallback = this.settings.fallback;
|
|
2648
|
-
if (!fallback || fallback === "none" || !this.provider || fallback === this.provider.id) return false;
|
|
2649
|
-
if (this.fallbackFrom) return false;
|
|
2650
|
-
const fallbackFrom = this.provider.id;
|
|
2651
|
-
const fallbackModel = fallback === "gemini" ? DEFAULT_GEMINI_EMBEDDING_MODEL : fallback === "openai" ? DEFAULT_OPENAI_EMBEDDING_MODEL : fallback === "voyage" ? DEFAULT_VOYAGE_EMBEDDING_MODEL : fallback === "mistral" ? DEFAULT_MISTRAL_EMBEDDING_MODEL : fallback === "ollama" ? DEFAULT_OLLAMA_EMBEDDING_MODEL : this.settings.model;
|
|
2652
|
-
const fallbackResult = await createEmbeddingProvider({
|
|
2653
|
-
config: this.cfg,
|
|
2654
|
-
agentDir: resolveAgentDir(this.cfg, this.agentId),
|
|
2655
|
-
provider: fallback,
|
|
2656
|
-
remote: this.settings.remote,
|
|
2657
|
-
model: fallbackModel,
|
|
2658
|
-
fallback: "none",
|
|
2659
|
-
local: this.settings.local
|
|
2660
|
-
});
|
|
2661
|
-
this.fallbackFrom = fallbackFrom;
|
|
2662
|
-
this.fallbackReason = reason;
|
|
2663
|
-
this.provider = fallbackResult.provider;
|
|
2664
|
-
this.openAi = fallbackResult.openAi;
|
|
2665
|
-
this.gemini = fallbackResult.gemini;
|
|
2666
|
-
this.voyage = fallbackResult.voyage;
|
|
2667
|
-
this.mistral = fallbackResult.mistral;
|
|
2668
|
-
this.ollama = fallbackResult.ollama;
|
|
2669
|
-
this.providerKey = this.computeProviderKey();
|
|
2670
|
-
this.batch = this.resolveBatchConfig();
|
|
2671
|
-
log$2.warn(`memory embeddings: switched to fallback provider (${fallback})`, { reason });
|
|
2672
|
-
return true;
|
|
2673
|
-
}
|
|
2674
|
-
async runSafeReindex(params) {
|
|
2675
|
-
const dbPath = resolveUserPath(this.settings.store.path);
|
|
2676
|
-
const tempDbPath = `${dbPath}.tmp-${randomUUID()}`;
|
|
2677
|
-
const tempDb = this.openDatabaseAtPath(tempDbPath);
|
|
2678
|
-
const originalDb = this.db;
|
|
2679
|
-
let originalDbClosed = false;
|
|
2680
|
-
const originalState = {
|
|
2681
|
-
ftsAvailable: this.fts.available,
|
|
2682
|
-
ftsError: this.fts.loadError,
|
|
2683
|
-
vectorAvailable: this.vector.available,
|
|
2684
|
-
vectorLoadError: this.vector.loadError,
|
|
2685
|
-
vectorDims: this.vector.dims,
|
|
2686
|
-
vectorReady: this.vectorReady
|
|
2687
|
-
};
|
|
2688
|
-
const restoreOriginalState = () => {
|
|
2689
|
-
if (originalDbClosed) this.db = this.openDatabaseAtPath(dbPath);
|
|
2690
|
-
else this.db = originalDb;
|
|
2691
|
-
this.fts.available = originalState.ftsAvailable;
|
|
2692
|
-
this.fts.loadError = originalState.ftsError;
|
|
2693
|
-
this.vector.available = originalDbClosed ? null : originalState.vectorAvailable;
|
|
2694
|
-
this.vector.loadError = originalState.vectorLoadError;
|
|
2695
|
-
this.vector.dims = originalState.vectorDims;
|
|
2696
|
-
this.vectorReady = originalDbClosed ? null : originalState.vectorReady;
|
|
2697
|
-
};
|
|
2698
|
-
this.db = tempDb;
|
|
2699
|
-
this.vectorReady = null;
|
|
2700
|
-
this.vector.available = null;
|
|
2701
|
-
this.vector.loadError = void 0;
|
|
2702
|
-
this.vector.dims = void 0;
|
|
2703
|
-
this.fts.available = false;
|
|
2704
|
-
this.fts.loadError = void 0;
|
|
2705
|
-
this.ensureSchema();
|
|
2706
|
-
let nextMeta = null;
|
|
2707
|
-
try {
|
|
2708
|
-
this.seedEmbeddingCache(originalDb);
|
|
2709
|
-
const shouldSyncMemory = this.sources.has("memory");
|
|
2710
|
-
const shouldSyncSessions = this.shouldSyncSessions({
|
|
2711
|
-
reason: params.reason,
|
|
2712
|
-
force: params.force
|
|
2713
|
-
}, true);
|
|
2714
|
-
if (shouldSyncMemory) {
|
|
2715
|
-
await this.syncMemoryFiles({
|
|
2716
|
-
needsFullReindex: true,
|
|
2717
|
-
progress: params.progress
|
|
2718
|
-
});
|
|
2719
|
-
this.dirty = false;
|
|
2720
|
-
}
|
|
2721
|
-
if (shouldSyncSessions) {
|
|
2722
|
-
await this.syncSessionFiles({
|
|
2723
|
-
needsFullReindex: true,
|
|
2724
|
-
progress: params.progress
|
|
2725
|
-
});
|
|
2726
|
-
this.sessionsDirty = false;
|
|
2727
|
-
this.sessionsDirtyFiles.clear();
|
|
2728
|
-
} else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true;
|
|
2729
|
-
else this.sessionsDirty = false;
|
|
2730
|
-
nextMeta = {
|
|
2731
|
-
model: this.provider?.model ?? "fts-only",
|
|
2732
|
-
provider: this.provider?.id ?? "none",
|
|
2733
|
-
providerKey: this.providerKey,
|
|
2734
|
-
sources: this.resolveConfiguredSourcesForMeta(),
|
|
2735
|
-
chunkTokens: this.settings.chunking.tokens,
|
|
2736
|
-
chunkOverlap: this.settings.chunking.overlap
|
|
2737
|
-
};
|
|
2738
|
-
if (!nextMeta) throw new Error("Failed to compute memory index metadata for reindexing.");
|
|
2739
|
-
if (this.vector.available && this.vector.dims) nextMeta.vectorDims = this.vector.dims;
|
|
2740
|
-
this.writeMeta(nextMeta);
|
|
2741
|
-
this.pruneEmbeddingCacheIfNeeded?.();
|
|
2742
|
-
this.db.close();
|
|
2743
|
-
originalDb.close();
|
|
2744
|
-
originalDbClosed = true;
|
|
2745
|
-
await this.swapIndexFiles(dbPath, tempDbPath);
|
|
2746
|
-
this.db = this.openDatabaseAtPath(dbPath);
|
|
2747
|
-
this.vectorReady = null;
|
|
2748
|
-
this.vector.available = null;
|
|
2749
|
-
this.vector.loadError = void 0;
|
|
2750
|
-
this.ensureSchema();
|
|
2751
|
-
this.vector.dims = nextMeta?.vectorDims;
|
|
2752
|
-
} catch (err) {
|
|
2753
|
-
try {
|
|
2754
|
-
this.db.close();
|
|
2755
|
-
} catch {}
|
|
2756
|
-
await this.removeIndexFiles(tempDbPath);
|
|
2757
|
-
restoreOriginalState();
|
|
2758
|
-
throw err;
|
|
2759
|
-
}
|
|
2760
|
-
}
|
|
2761
|
-
async runUnsafeReindex(params) {
|
|
2762
|
-
this.resetIndex();
|
|
2763
|
-
const shouldSyncMemory = this.sources.has("memory");
|
|
2764
|
-
const shouldSyncSessions = this.shouldSyncSessions({
|
|
2765
|
-
reason: params.reason,
|
|
2766
|
-
force: params.force
|
|
2767
|
-
}, true);
|
|
2768
|
-
if (shouldSyncMemory) {
|
|
2769
|
-
await this.syncMemoryFiles({
|
|
2770
|
-
needsFullReindex: true,
|
|
2771
|
-
progress: params.progress
|
|
2772
|
-
});
|
|
2773
|
-
this.dirty = false;
|
|
2774
|
-
}
|
|
2775
|
-
if (shouldSyncSessions) {
|
|
2776
|
-
await this.syncSessionFiles({
|
|
2777
|
-
needsFullReindex: true,
|
|
2778
|
-
progress: params.progress
|
|
2779
|
-
});
|
|
2780
|
-
this.sessionsDirty = false;
|
|
2781
|
-
this.sessionsDirtyFiles.clear();
|
|
2782
|
-
} else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true;
|
|
2783
|
-
else this.sessionsDirty = false;
|
|
2784
|
-
const nextMeta = {
|
|
2785
|
-
model: this.provider?.model ?? "fts-only",
|
|
2786
|
-
provider: this.provider?.id ?? "none",
|
|
2787
|
-
providerKey: this.providerKey,
|
|
2788
|
-
sources: this.resolveConfiguredSourcesForMeta(),
|
|
2789
|
-
chunkTokens: this.settings.chunking.tokens,
|
|
2790
|
-
chunkOverlap: this.settings.chunking.overlap
|
|
2791
|
-
};
|
|
2792
|
-
if (this.vector.available && this.vector.dims) nextMeta.vectorDims = this.vector.dims;
|
|
2793
|
-
this.writeMeta(nextMeta);
|
|
2794
|
-
this.pruneEmbeddingCacheIfNeeded?.();
|
|
2795
|
-
}
|
|
2796
|
-
resetIndex() {
|
|
2797
|
-
this.db.exec(`DELETE FROM files`);
|
|
2798
|
-
this.db.exec(`DELETE FROM chunks`);
|
|
2799
|
-
if (this.fts.enabled && this.fts.available) try {
|
|
2800
|
-
this.db.exec(`DELETE FROM ${FTS_TABLE$2}`);
|
|
2801
|
-
} catch {}
|
|
2802
|
-
this.dropVectorTable();
|
|
2803
|
-
this.vector.dims = void 0;
|
|
2804
|
-
this.sessionsDirtyFiles.clear();
|
|
2805
|
-
}
|
|
2806
|
-
readMeta() {
|
|
2807
|
-
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY);
|
|
2808
|
-
if (!row?.value) {
|
|
2809
|
-
this.lastMetaSerialized = null;
|
|
2810
|
-
return null;
|
|
2811
|
-
}
|
|
2812
|
-
try {
|
|
2813
|
-
const parsed = JSON.parse(row.value);
|
|
2814
|
-
this.lastMetaSerialized = row.value;
|
|
2815
|
-
return parsed;
|
|
2816
|
-
} catch {
|
|
2817
|
-
this.lastMetaSerialized = null;
|
|
2818
|
-
return null;
|
|
2819
|
-
}
|
|
2820
|
-
}
|
|
2821
|
-
writeMeta(meta) {
|
|
2822
|
-
const value = JSON.stringify(meta);
|
|
2823
|
-
if (this.lastMetaSerialized === value) return;
|
|
2824
|
-
this.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(META_KEY, value);
|
|
2825
|
-
this.lastMetaSerialized = value;
|
|
2826
|
-
}
|
|
2827
|
-
resolveConfiguredSourcesForMeta() {
|
|
2828
|
-
const normalized = Array.from(this.sources).filter((source) => source === "memory" || source === "sessions").toSorted();
|
|
2829
|
-
return normalized.length > 0 ? normalized : ["memory"];
|
|
2830
|
-
}
|
|
2831
|
-
normalizeMetaSources(meta) {
|
|
2832
|
-
if (!Array.isArray(meta.sources)) return ["memory"];
|
|
2833
|
-
const normalized = Array.from(new Set(meta.sources.filter((source) => source === "memory" || source === "sessions"))).toSorted();
|
|
2834
|
-
return normalized.length > 0 ? normalized : ["memory"];
|
|
2835
|
-
}
|
|
2836
|
-
metaSourcesDiffer(meta, configuredSources) {
|
|
2837
|
-
const metaSources = this.normalizeMetaSources(meta);
|
|
2838
|
-
if (metaSources.length !== configuredSources.length) return true;
|
|
2839
|
-
return metaSources.some((source, index) => source !== configuredSources[index]);
|
|
2840
|
-
}
|
|
2841
|
-
};
|
|
2842
|
-
|
|
2843
|
-
//#endregion
|
|
2844
|
-
//#region src/memory/manager-embedding-ops.ts
|
|
2845
|
-
const VECTOR_TABLE$1 = "chunks_vec";
|
|
2846
|
-
const FTS_TABLE$1 = "chunks_fts";
|
|
2847
|
-
const EMBEDDING_CACHE_TABLE$1 = "embedding_cache";
|
|
2848
|
-
const EMBEDDING_BATCH_MAX_TOKENS = 8e3;
|
|
2849
|
-
const EMBEDDING_INDEX_CONCURRENCY = 4;
|
|
2850
|
-
const EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
|
|
2851
|
-
const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|
2852
|
-
const EMBEDDING_RETRY_MAX_DELAY_MS = 8e3;
|
|
2853
|
-
const BATCH_FAILURE_LIMIT$1 = 2;
|
|
2854
|
-
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4;
|
|
2855
|
-
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4;
|
|
2856
|
-
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 6e4;
|
|
2857
|
-
const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 6e4;
|
|
2858
|
-
const vectorToBlob$1 = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
|
|
2859
|
-
const log$1 = createSubsystemLogger("memory");
|
|
2860
|
-
var MemoryManagerEmbeddingOps = class extends MemoryManagerSyncOps {
|
|
2861
|
-
buildEmbeddingBatches(chunks) {
|
|
2862
|
-
const batches = [];
|
|
2863
|
-
let current = [];
|
|
2864
|
-
let currentTokens = 0;
|
|
2865
|
-
for (const chunk of chunks) {
|
|
2866
|
-
const estimate = estimateUtf8Bytes(chunk.text);
|
|
2867
|
-
if (current.length > 0 && currentTokens + estimate > EMBEDDING_BATCH_MAX_TOKENS) {
|
|
2868
|
-
batches.push(current);
|
|
2869
|
-
current = [];
|
|
2870
|
-
currentTokens = 0;
|
|
2871
|
-
}
|
|
2872
|
-
if (current.length === 0 && estimate > EMBEDDING_BATCH_MAX_TOKENS) {
|
|
2873
|
-
batches.push([chunk]);
|
|
2874
|
-
continue;
|
|
2875
|
-
}
|
|
2876
|
-
current.push(chunk);
|
|
2877
|
-
currentTokens += estimate;
|
|
2878
|
-
}
|
|
2879
|
-
if (current.length > 0) batches.push(current);
|
|
2880
|
-
return batches;
|
|
2881
|
-
}
|
|
2882
|
-
loadEmbeddingCache(hashes) {
|
|
2883
|
-
if (!this.cache.enabled || !this.provider) return /* @__PURE__ */ new Map();
|
|
2884
|
-
if (hashes.length === 0) return /* @__PURE__ */ new Map();
|
|
2885
|
-
const unique = [];
|
|
2886
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2887
|
-
for (const hash of hashes) {
|
|
2888
|
-
if (!hash) continue;
|
|
2889
|
-
if (seen.has(hash)) continue;
|
|
2890
|
-
seen.add(hash);
|
|
2891
|
-
unique.push(hash);
|
|
2892
|
-
}
|
|
2893
|
-
if (unique.length === 0) return /* @__PURE__ */ new Map();
|
|
2894
|
-
const out = /* @__PURE__ */ new Map();
|
|
2895
|
-
const baseParams = [
|
|
2896
|
-
this.provider.id,
|
|
2897
|
-
this.provider.model,
|
|
2898
|
-
this.providerKey
|
|
2899
|
-
];
|
|
2900
|
-
const batchSize = 400;
|
|
2901
|
-
for (let start = 0; start < unique.length; start += batchSize) {
|
|
2902
|
-
const batch = unique.slice(start, start + batchSize);
|
|
2903
|
-
const placeholders = batch.map(() => "?").join(", ");
|
|
2904
|
-
const rows = this.db.prepare(`SELECT hash, embedding FROM ${EMBEDDING_CACHE_TABLE$1}\n WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`).all(...baseParams, ...batch);
|
|
2905
|
-
for (const row of rows) out.set(row.hash, parseEmbedding(row.embedding));
|
|
2906
|
-
}
|
|
2907
|
-
return out;
|
|
2908
|
-
}
|
|
2909
|
-
upsertEmbeddingCache(entries) {
|
|
2910
|
-
if (!this.cache.enabled || !this.provider) return;
|
|
2911
|
-
if (entries.length === 0) return;
|
|
2912
|
-
const now = Date.now();
|
|
2913
|
-
const stmt = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE$1} (provider, model, provider_key, hash, embedding, dims, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n embedding=excluded.embedding,\n dims=excluded.dims,\n updated_at=excluded.updated_at`);
|
|
2914
|
-
for (const entry of entries) {
|
|
2915
|
-
const embedding = entry.embedding ?? [];
|
|
2916
|
-
stmt.run(this.provider.id, this.provider.model, this.providerKey, entry.hash, JSON.stringify(embedding), embedding.length, now);
|
|
2917
|
-
}
|
|
2918
|
-
}
|
|
2919
|
-
pruneEmbeddingCacheIfNeeded() {
|
|
2920
|
-
if (!this.cache.enabled) return;
|
|
2921
|
-
const max = this.cache.maxEntries;
|
|
2922
|
-
if (!max || max <= 0) return;
|
|
2923
|
-
const count = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE$1}`).get()?.c ?? 0;
|
|
2924
|
-
if (count <= max) return;
|
|
2925
|
-
const excess = count - max;
|
|
2926
|
-
this.db.prepare(`DELETE FROM ${EMBEDDING_CACHE_TABLE$1}\n WHERE rowid IN (\n SELECT rowid FROM ${EMBEDDING_CACHE_TABLE$1}\n ORDER BY updated_at ASC\n LIMIT ?\n )`).run(excess);
|
|
2927
|
-
}
|
|
2928
|
-
async embedChunksInBatches(chunks) {
|
|
2929
|
-
if (chunks.length === 0) return [];
|
|
2930
|
-
const { embeddings, missing } = this.collectCachedEmbeddings(chunks);
|
|
2931
|
-
if (missing.length === 0) return embeddings;
|
|
2932
|
-
const missingChunks = missing.map((m) => m.chunk);
|
|
2933
|
-
const batches = this.buildEmbeddingBatches(missingChunks);
|
|
2934
|
-
const toCache = [];
|
|
2935
|
-
let cursor = 0;
|
|
2936
|
-
for (const batch of batches) {
|
|
2937
|
-
const batchEmbeddings = await this.embedBatchWithRetry(batch.map((chunk) => chunk.text));
|
|
2938
|
-
for (let i = 0; i < batch.length; i += 1) {
|
|
2939
|
-
const item = missing[cursor + i];
|
|
2940
|
-
const embedding = batchEmbeddings[i] ?? [];
|
|
2941
|
-
if (item) {
|
|
2942
|
-
embeddings[item.index] = embedding;
|
|
2943
|
-
toCache.push({
|
|
2944
|
-
hash: item.chunk.hash,
|
|
2945
|
-
embedding
|
|
2946
|
-
});
|
|
2947
|
-
}
|
|
2948
|
-
}
|
|
2949
|
-
cursor += batch.length;
|
|
2950
|
-
}
|
|
2951
|
-
this.upsertEmbeddingCache(toCache);
|
|
2952
|
-
return embeddings;
|
|
2953
|
-
}
|
|
2954
|
-
computeProviderKey() {
|
|
2955
|
-
if (!this.provider) return hashText(JSON.stringify({
|
|
2956
|
-
provider: "none",
|
|
2957
|
-
model: "fts-only"
|
|
2958
|
-
}));
|
|
2959
|
-
if (this.provider.id === "openai" && this.openAi) {
|
|
2960
|
-
const entries = Object.entries(this.openAi.headers).filter(([key]) => key.toLowerCase() !== "authorization").toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]);
|
|
2961
|
-
return hashText(JSON.stringify({
|
|
2962
|
-
provider: "openai",
|
|
2963
|
-
baseUrl: this.openAi.baseUrl,
|
|
2964
|
-
model: this.openAi.model,
|
|
2965
|
-
headers: entries
|
|
2966
|
-
}));
|
|
2967
|
-
}
|
|
2968
|
-
if (this.provider.id === "gemini" && this.gemini) {
|
|
2969
|
-
const entries = Object.entries(this.gemini.headers).filter(([key]) => {
|
|
2970
|
-
const lower = key.toLowerCase();
|
|
2971
|
-
return lower !== "authorization" && lower !== "x-goog-api-key";
|
|
2972
|
-
}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]);
|
|
2973
|
-
return hashText(JSON.stringify({
|
|
2974
|
-
provider: "gemini",
|
|
2975
|
-
baseUrl: this.gemini.baseUrl,
|
|
2976
|
-
model: this.gemini.model,
|
|
2977
|
-
headers: entries
|
|
2978
|
-
}));
|
|
2979
|
-
}
|
|
2980
|
-
return hashText(JSON.stringify({
|
|
2981
|
-
provider: this.provider.id,
|
|
2982
|
-
model: this.provider.model
|
|
2983
|
-
}));
|
|
2984
|
-
}
|
|
2985
|
-
async embedChunksWithBatch(chunks, entry, source) {
|
|
2986
|
-
if (!this.provider) return this.embedChunksInBatches(chunks);
|
|
2987
|
-
if (this.provider.id === "openai" && this.openAi) return this.embedChunksWithOpenAiBatch(chunks, entry, source);
|
|
2988
|
-
if (this.provider.id === "gemini" && this.gemini) return this.embedChunksWithGeminiBatch(chunks, entry, source);
|
|
2989
|
-
if (this.provider.id === "voyage" && this.voyage) return this.embedChunksWithVoyageBatch(chunks, entry, source);
|
|
2990
|
-
return this.embedChunksInBatches(chunks);
|
|
2991
|
-
}
|
|
2992
|
-
collectCachedEmbeddings(chunks) {
|
|
2993
|
-
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
|
2994
|
-
const embeddings = Array.from({ length: chunks.length }, () => []);
|
|
2995
|
-
const missing = [];
|
|
2996
|
-
for (let i = 0; i < chunks.length; i += 1) {
|
|
2997
|
-
const chunk = chunks[i];
|
|
2998
|
-
const hit = chunk?.hash ? cached.get(chunk.hash) : void 0;
|
|
2999
|
-
if (hit && hit.length > 0) embeddings[i] = hit;
|
|
3000
|
-
else if (chunk) missing.push({
|
|
3001
|
-
index: i,
|
|
3002
|
-
chunk
|
|
3003
|
-
});
|
|
3004
|
-
}
|
|
3005
|
-
return {
|
|
3006
|
-
embeddings,
|
|
3007
|
-
missing
|
|
3008
|
-
};
|
|
3009
|
-
}
|
|
3010
|
-
buildBatchCustomId(params) {
|
|
3011
|
-
return hashText(`${params.source}:${params.entry.path}:${params.chunk.startLine}:${params.chunk.endLine}:${params.chunk.hash}:${params.index}`);
|
|
3012
|
-
}
|
|
3013
|
-
buildBatchRequests(params) {
|
|
3014
|
-
const requests = [];
|
|
3015
|
-
const mapping = /* @__PURE__ */ new Map();
|
|
3016
|
-
for (const item of params.missing) {
|
|
3017
|
-
const chunk = item.chunk;
|
|
3018
|
-
const customId = this.buildBatchCustomId({
|
|
3019
|
-
source: params.source,
|
|
3020
|
-
entry: params.entry,
|
|
3021
|
-
chunk,
|
|
3022
|
-
index: item.index
|
|
3023
|
-
});
|
|
3024
|
-
mapping.set(customId, {
|
|
3025
|
-
index: item.index,
|
|
3026
|
-
hash: chunk.hash
|
|
3027
|
-
});
|
|
3028
|
-
const built = params.build(chunk);
|
|
3029
|
-
requests.push({
|
|
3030
|
-
custom_id: customId,
|
|
3031
|
-
...built
|
|
3032
|
-
});
|
|
3033
|
-
}
|
|
3034
|
-
return {
|
|
3035
|
-
requests,
|
|
3036
|
-
mapping
|
|
3037
|
-
};
|
|
3038
|
-
}
|
|
3039
|
-
applyBatchEmbeddings(params) {
|
|
3040
|
-
const toCache = [];
|
|
3041
|
-
for (const [customId, embedding] of params.byCustomId.entries()) {
|
|
3042
|
-
const mapped = params.mapping.get(customId);
|
|
3043
|
-
if (!mapped) continue;
|
|
3044
|
-
params.embeddings[mapped.index] = embedding;
|
|
3045
|
-
toCache.push({
|
|
3046
|
-
hash: mapped.hash,
|
|
3047
|
-
embedding
|
|
3048
|
-
});
|
|
3049
|
-
}
|
|
3050
|
-
this.upsertEmbeddingCache(toCache);
|
|
3051
|
-
}
|
|
3052
|
-
buildEmbeddingBatchRunnerOptions(params) {
|
|
3053
|
-
const { requests, chunks, source } = params;
|
|
3054
|
-
return {
|
|
3055
|
-
agentId: this.agentId,
|
|
3056
|
-
requests,
|
|
3057
|
-
wait: this.batch.wait,
|
|
3058
|
-
concurrency: this.batch.concurrency,
|
|
3059
|
-
pollIntervalMs: this.batch.pollIntervalMs,
|
|
3060
|
-
timeoutMs: this.batch.timeoutMs,
|
|
3061
|
-
debug: (message, data) => log$1.debug(message, data ? {
|
|
3062
|
-
...data,
|
|
3063
|
-
source,
|
|
3064
|
-
chunks: chunks.length
|
|
3065
|
-
} : {
|
|
3066
|
-
source,
|
|
3067
|
-
chunks: chunks.length
|
|
3068
|
-
})
|
|
3069
|
-
};
|
|
3070
|
-
}
|
|
3071
|
-
async embedChunksWithProviderBatch(params) {
|
|
3072
|
-
if (!params.enabled) return this.embedChunksInBatches(params.chunks);
|
|
3073
|
-
if (params.chunks.length === 0) return [];
|
|
3074
|
-
const { embeddings, missing } = this.collectCachedEmbeddings(params.chunks);
|
|
3075
|
-
if (missing.length === 0) return embeddings;
|
|
3076
|
-
const { requests, mapping } = this.buildBatchRequests({
|
|
3077
|
-
missing,
|
|
3078
|
-
entry: params.entry,
|
|
3079
|
-
source: params.source,
|
|
3080
|
-
build: params.buildRequest
|
|
3081
|
-
});
|
|
3082
|
-
const runnerOptions = this.buildEmbeddingBatchRunnerOptions({
|
|
3083
|
-
requests,
|
|
3084
|
-
chunks: params.chunks,
|
|
3085
|
-
source: params.source
|
|
3086
|
-
});
|
|
3087
|
-
const batchResult = await this.runBatchWithFallback({
|
|
3088
|
-
provider: params.provider,
|
|
3089
|
-
run: async () => await params.runBatch(runnerOptions),
|
|
3090
|
-
fallback: async () => await this.embedChunksInBatches(params.chunks)
|
|
3091
|
-
});
|
|
3092
|
-
if (Array.isArray(batchResult)) return batchResult;
|
|
3093
|
-
this.applyBatchEmbeddings({
|
|
3094
|
-
byCustomId: batchResult,
|
|
3095
|
-
mapping,
|
|
3096
|
-
embeddings
|
|
3097
|
-
});
|
|
3098
|
-
return embeddings;
|
|
3099
|
-
}
|
|
3100
|
-
async embedChunksWithVoyageBatch(chunks, entry, source) {
|
|
3101
|
-
const voyage = this.voyage;
|
|
3102
|
-
return await this.embedChunksWithProviderBatch({
|
|
3103
|
-
chunks,
|
|
3104
|
-
entry,
|
|
3105
|
-
source,
|
|
3106
|
-
provider: "voyage",
|
|
3107
|
-
enabled: Boolean(voyage),
|
|
3108
|
-
buildRequest: (chunk) => ({ body: { input: chunk.text } }),
|
|
3109
|
-
runBatch: async (runnerOptions) => await runVoyageEmbeddingBatches({
|
|
3110
|
-
client: voyage,
|
|
3111
|
-
...runnerOptions
|
|
3112
|
-
})
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
async embedChunksWithOpenAiBatch(chunks, entry, source) {
|
|
3116
|
-
const openAi = this.openAi;
|
|
3117
|
-
return await this.embedChunksWithProviderBatch({
|
|
3118
|
-
chunks,
|
|
3119
|
-
entry,
|
|
3120
|
-
source,
|
|
3121
|
-
provider: "openai",
|
|
3122
|
-
enabled: Boolean(openAi),
|
|
3123
|
-
buildRequest: (chunk) => ({
|
|
3124
|
-
method: "POST",
|
|
3125
|
-
url: OPENAI_BATCH_ENDPOINT,
|
|
3126
|
-
body: {
|
|
3127
|
-
model: openAi?.model ?? this.provider?.model ?? "text-embedding-3-small",
|
|
3128
|
-
input: chunk.text
|
|
3129
|
-
}
|
|
3130
|
-
}),
|
|
3131
|
-
runBatch: async (runnerOptions) => await runOpenAiEmbeddingBatches({
|
|
3132
|
-
openAi,
|
|
3133
|
-
...runnerOptions
|
|
3134
|
-
})
|
|
3135
|
-
});
|
|
3136
|
-
}
|
|
3137
|
-
async embedChunksWithGeminiBatch(chunks, entry, source) {
|
|
3138
|
-
const gemini = this.gemini;
|
|
3139
|
-
return await this.embedChunksWithProviderBatch({
|
|
3140
|
-
chunks,
|
|
3141
|
-
entry,
|
|
3142
|
-
source,
|
|
3143
|
-
provider: "gemini",
|
|
3144
|
-
enabled: Boolean(gemini),
|
|
3145
|
-
buildRequest: (chunk) => ({
|
|
3146
|
-
content: { parts: [{ text: chunk.text }] },
|
|
3147
|
-
taskType: "RETRIEVAL_DOCUMENT"
|
|
3148
|
-
}),
|
|
3149
|
-
runBatch: async (runnerOptions) => await runGeminiEmbeddingBatches({
|
|
3150
|
-
gemini,
|
|
3151
|
-
...runnerOptions
|
|
3152
|
-
})
|
|
3153
|
-
});
|
|
3154
|
-
}
|
|
3155
|
-
async embedBatchWithRetry(texts) {
|
|
3156
|
-
if (texts.length === 0) return [];
|
|
3157
|
-
if (!this.provider) throw new Error("Cannot embed batch in FTS-only mode (no embedding provider)");
|
|
3158
|
-
let attempt = 0;
|
|
3159
|
-
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
|
|
3160
|
-
while (true) try {
|
|
3161
|
-
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
|
3162
|
-
log$1.debug("memory embeddings: batch start", {
|
|
3163
|
-
provider: this.provider.id,
|
|
3164
|
-
items: texts.length,
|
|
3165
|
-
timeoutMs
|
|
3166
|
-
});
|
|
3167
|
-
return await this.withTimeout(this.provider.embedBatch(texts), timeoutMs, `memory embeddings batch timed out after ${Math.round(timeoutMs / 1e3)}s`);
|
|
3168
|
-
} catch (err) {
|
|
3169
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3170
|
-
if (!this.isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) throw err;
|
|
3171
|
-
const waitMs = Math.min(EMBEDDING_RETRY_MAX_DELAY_MS, Math.round(delayMs * (1 + Math.random() * .2)));
|
|
3172
|
-
log$1.warn(`memory embeddings rate limited; retrying in ${waitMs}ms`);
|
|
3173
|
-
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
3174
|
-
delayMs *= 2;
|
|
3175
|
-
attempt += 1;
|
|
3176
|
-
}
|
|
3177
|
-
}
|
|
3178
|
-
isRetryableEmbeddingError(message) {
|
|
3179
|
-
return /(rate[_ ]limit|too many requests|429|resource has been exhausted|5\d\d|cloudflare)/i.test(message);
|
|
3180
|
-
}
|
|
3181
|
-
resolveEmbeddingTimeout(kind) {
|
|
3182
|
-
const isLocal = this.provider?.id === "local";
|
|
3183
|
-
if (kind === "query") return isLocal ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
|
3184
|
-
return isLocal ? EMBEDDING_BATCH_TIMEOUT_LOCAL_MS : EMBEDDING_BATCH_TIMEOUT_REMOTE_MS;
|
|
3185
|
-
}
|
|
3186
|
-
async embedQueryWithTimeout(text) {
|
|
3187
|
-
if (!this.provider) throw new Error("Cannot embed query in FTS-only mode (no embedding provider)");
|
|
3188
|
-
const timeoutMs = this.resolveEmbeddingTimeout("query");
|
|
3189
|
-
log$1.debug("memory embeddings: query start", {
|
|
3190
|
-
provider: this.provider.id,
|
|
3191
|
-
timeoutMs
|
|
3192
|
-
});
|
|
3193
|
-
return await this.withTimeout(this.provider.embedQuery(text), timeoutMs, `memory embeddings query timed out after ${Math.round(timeoutMs / 1e3)}s`);
|
|
3194
|
-
}
|
|
3195
|
-
async withTimeout(promise, timeoutMs, message) {
|
|
3196
|
-
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise;
|
|
3197
|
-
let timer = null;
|
|
3198
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
3199
|
-
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
3200
|
-
});
|
|
3201
|
-
try {
|
|
3202
|
-
return await Promise.race([promise, timeoutPromise]);
|
|
3203
|
-
} finally {
|
|
3204
|
-
if (timer) clearTimeout(timer);
|
|
3205
|
-
}
|
|
3206
|
-
}
|
|
3207
|
-
async withBatchFailureLock(fn) {
|
|
3208
|
-
let release;
|
|
3209
|
-
const wait = this.batchFailureLock;
|
|
3210
|
-
this.batchFailureLock = new Promise((resolve) => {
|
|
3211
|
-
release = resolve;
|
|
3212
|
-
});
|
|
3213
|
-
await wait;
|
|
3214
|
-
try {
|
|
3215
|
-
return await fn();
|
|
3216
|
-
} finally {
|
|
3217
|
-
release();
|
|
3218
|
-
}
|
|
3219
|
-
}
|
|
3220
|
-
async resetBatchFailureCount() {
|
|
3221
|
-
await this.withBatchFailureLock(async () => {
|
|
3222
|
-
if (this.batchFailureCount > 0) log$1.debug("memory embeddings: batch recovered; resetting failure count");
|
|
3223
|
-
this.batchFailureCount = 0;
|
|
3224
|
-
this.batchFailureLastError = void 0;
|
|
3225
|
-
this.batchFailureLastProvider = void 0;
|
|
3226
|
-
});
|
|
3227
|
-
}
|
|
3228
|
-
async recordBatchFailure(params) {
|
|
3229
|
-
return await this.withBatchFailureLock(async () => {
|
|
3230
|
-
if (!this.batch.enabled) return {
|
|
3231
|
-
disabled: true,
|
|
3232
|
-
count: this.batchFailureCount
|
|
3233
|
-
};
|
|
3234
|
-
const increment = params.forceDisable ? BATCH_FAILURE_LIMIT$1 : Math.max(1, params.attempts ?? 1);
|
|
3235
|
-
this.batchFailureCount += increment;
|
|
3236
|
-
this.batchFailureLastError = params.message;
|
|
3237
|
-
this.batchFailureLastProvider = params.provider;
|
|
3238
|
-
const disabled = params.forceDisable || this.batchFailureCount >= BATCH_FAILURE_LIMIT$1;
|
|
3239
|
-
if (disabled) this.batch.enabled = false;
|
|
3240
|
-
return {
|
|
3241
|
-
disabled,
|
|
3242
|
-
count: this.batchFailureCount
|
|
3243
|
-
};
|
|
3244
|
-
});
|
|
3245
|
-
}
|
|
3246
|
-
isBatchTimeoutError(message) {
|
|
3247
|
-
return /timed out|timeout/i.test(message);
|
|
3248
|
-
}
|
|
3249
|
-
async runBatchWithTimeoutRetry(params) {
|
|
3250
|
-
try {
|
|
3251
|
-
return await params.run();
|
|
3252
|
-
} catch (err) {
|
|
3253
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3254
|
-
if (this.isBatchTimeoutError(message)) {
|
|
3255
|
-
log$1.warn(`memory embeddings: ${params.provider} batch timed out; retrying once`);
|
|
3256
|
-
try {
|
|
3257
|
-
return await params.run();
|
|
3258
|
-
} catch (retryErr) {
|
|
3259
|
-
retryErr.batchAttempts = 2;
|
|
3260
|
-
throw retryErr;
|
|
3261
|
-
}
|
|
3262
|
-
}
|
|
3263
|
-
throw err;
|
|
3264
|
-
}
|
|
3265
|
-
}
|
|
3266
|
-
async runBatchWithFallback(params) {
|
|
3267
|
-
if (!this.batch.enabled) return await params.fallback();
|
|
3268
|
-
try {
|
|
3269
|
-
const result = await this.runBatchWithTimeoutRetry({
|
|
3270
|
-
provider: params.provider,
|
|
3271
|
-
run: params.run
|
|
3272
|
-
});
|
|
3273
|
-
await this.resetBatchFailureCount();
|
|
3274
|
-
return result;
|
|
3275
|
-
} catch (err) {
|
|
3276
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3277
|
-
const attempts = err.batchAttempts ?? 1;
|
|
3278
|
-
const forceDisable = /asyncBatchEmbedContent not available/i.test(message);
|
|
3279
|
-
const failure = await this.recordBatchFailure({
|
|
3280
|
-
provider: params.provider,
|
|
3281
|
-
message,
|
|
3282
|
-
attempts,
|
|
3283
|
-
forceDisable
|
|
3284
|
-
});
|
|
3285
|
-
const suffix = failure.disabled ? "disabling batch" : "keeping batch enabled";
|
|
3286
|
-
log$1.warn(`memory embeddings: ${params.provider} batch failed (${failure.count}/${BATCH_FAILURE_LIMIT$1}); ${suffix}; falling back to non-batch embeddings: ${message}`);
|
|
3287
|
-
return await params.fallback();
|
|
3288
|
-
}
|
|
3289
|
-
}
|
|
3290
|
-
getIndexConcurrency() {
|
|
3291
|
-
return this.batch.enabled ? this.batch.concurrency : EMBEDDING_INDEX_CONCURRENCY;
|
|
3292
|
-
}
|
|
3293
|
-
async indexFile(entry, options) {
|
|
3294
|
-
if (!this.provider) {
|
|
3295
|
-
log$1.debug("Skipping embedding indexing in FTS-only mode", {
|
|
3296
|
-
path: entry.path,
|
|
3297
|
-
source: options.source
|
|
3298
|
-
});
|
|
3299
|
-
return;
|
|
3300
|
-
}
|
|
3301
|
-
const content = options.content ?? await fs$1.readFile(entry.absPath, "utf-8");
|
|
3302
|
-
const chunks = enforceEmbeddingMaxInputTokens(this.provider, chunkMarkdown(content, this.settings.chunking).filter((chunk) => chunk.text.trim().length > 0), EMBEDDING_BATCH_MAX_TOKENS);
|
|
3303
|
-
if (options.source === "sessions" && "lineMap" in entry) remapChunkLines(chunks, entry.lineMap);
|
|
3304
|
-
const embeddings = this.batch.enabled ? await this.embedChunksWithBatch(chunks, entry, options.source) : await this.embedChunksInBatches(chunks);
|
|
3305
|
-
const sample = embeddings.find((embedding) => embedding.length > 0);
|
|
3306
|
-
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
|
|
3307
|
-
const now = Date.now();
|
|
3308
|
-
if (vectorReady) try {
|
|
3309
|
-
this.db.prepare(`DELETE FROM ${VECTOR_TABLE$1} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(entry.path, options.source);
|
|
3310
|
-
} catch {}
|
|
3311
|
-
if (this.fts.enabled && this.fts.available) try {
|
|
3312
|
-
this.db.prepare(`DELETE FROM ${FTS_TABLE$1} WHERE path = ? AND source = ? AND model = ?`).run(entry.path, options.source, this.provider.model);
|
|
3313
|
-
} catch {}
|
|
3314
|
-
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(entry.path, options.source);
|
|
3315
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
3316
|
-
const chunk = chunks[i];
|
|
3317
|
-
const embedding = embeddings[i] ?? [];
|
|
3318
|
-
const id = hashText(`${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`);
|
|
3319
|
-
this.db.prepare(`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
|
3320
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
3321
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
3322
|
-
hash=excluded.hash,
|
|
3323
|
-
model=excluded.model,
|
|
3324
|
-
text=excluded.text,
|
|
3325
|
-
embedding=excluded.embedding,
|
|
3326
|
-
updated_at=excluded.updated_at`).run(id, entry.path, options.source, chunk.startLine, chunk.endLine, chunk.hash, this.provider.model, chunk.text, JSON.stringify(embedding), now);
|
|
3327
|
-
if (vectorReady && embedding.length > 0) {
|
|
3328
|
-
try {
|
|
3329
|
-
this.db.prepare(`DELETE FROM ${VECTOR_TABLE$1} WHERE id = ?`).run(id);
|
|
3330
|
-
} catch {}
|
|
3331
|
-
this.db.prepare(`INSERT INTO ${VECTOR_TABLE$1} (id, embedding) VALUES (?, ?)`).run(id, vectorToBlob$1(embedding));
|
|
3332
|
-
}
|
|
3333
|
-
if (this.fts.enabled && this.fts.available) this.db.prepare(`INSERT INTO ${FTS_TABLE$1} (text, id, path, source, model, start_line, end_line)\n VALUES (?, ?, ?, ?, ?, ?, ?)`).run(chunk.text, id, entry.path, options.source, this.provider.model, chunk.startLine, chunk.endLine);
|
|
3334
|
-
}
|
|
3335
|
-
this.db.prepare(`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
|
|
3336
|
-
ON CONFLICT(path) DO UPDATE SET
|
|
3337
|
-
source=excluded.source,
|
|
3338
|
-
hash=excluded.hash,
|
|
3339
|
-
mtime=excluded.mtime,
|
|
3340
|
-
size=excluded.size`).run(entry.path, options.source, entry.hash, entry.mtimeMs, entry.size);
|
|
3341
|
-
}
|
|
3342
|
-
};
|
|
3343
|
-
|
|
3344
|
-
//#endregion
|
|
3345
|
-
//#region src/memory/manager-search.ts
|
|
3346
|
-
const vectorToBlob = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
|
|
3347
|
-
async function searchVector(params) {
|
|
3348
|
-
if (params.queryVec.length === 0 || params.limit <= 0) return [];
|
|
3349
|
-
if (await params.ensureVectorReady(params.queryVec.length)) return params.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text,
|
|
3350
|
-
c.source,
|
|
3351
|
-
vec_distance_cosine(v.embedding, ?) AS dist
|
|
3352
|
-
FROM ${params.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${params.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(vectorToBlob(params.queryVec), params.providerModel, ...params.sourceFilterVec.params, params.limit).map((row) => ({
|
|
3353
|
-
id: row.id,
|
|
3354
|
-
path: row.path,
|
|
3355
|
-
startLine: row.start_line,
|
|
3356
|
-
endLine: row.end_line,
|
|
3357
|
-
score: 1 - row.dist,
|
|
3358
|
-
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
3359
|
-
source: row.source
|
|
3360
|
-
}));
|
|
3361
|
-
return listChunks({
|
|
3362
|
-
db: params.db,
|
|
3363
|
-
providerModel: params.providerModel,
|
|
3364
|
-
sourceFilter: params.sourceFilterChunks
|
|
3365
|
-
}).map((chunk) => ({
|
|
3366
|
-
chunk,
|
|
3367
|
-
score: cosineSimilarity(params.queryVec, chunk.embedding)
|
|
3368
|
-
})).filter((entry) => Number.isFinite(entry.score)).toSorted((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({
|
|
3369
|
-
id: entry.chunk.id,
|
|
3370
|
-
path: entry.chunk.path,
|
|
3371
|
-
startLine: entry.chunk.startLine,
|
|
3372
|
-
endLine: entry.chunk.endLine,
|
|
3373
|
-
score: entry.score,
|
|
3374
|
-
snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars),
|
|
3375
|
-
source: entry.chunk.source
|
|
3376
|
-
}));
|
|
3377
|
-
}
|
|
3378
|
-
function listChunks(params) {
|
|
3379
|
-
return params.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source
|
|
3380
|
-
FROM chunks
|
|
3381
|
-
WHERE model = ?${params.sourceFilter.sql}`).all(params.providerModel, ...params.sourceFilter.params).map((row) => ({
|
|
3382
|
-
id: row.id,
|
|
3383
|
-
path: row.path,
|
|
3384
|
-
startLine: row.start_line,
|
|
3385
|
-
endLine: row.end_line,
|
|
3386
|
-
text: row.text,
|
|
3387
|
-
embedding: parseEmbedding(row.embedding),
|
|
3388
|
-
source: row.source
|
|
3389
|
-
}));
|
|
3390
|
-
}
|
|
3391
|
-
async function searchKeyword(params) {
|
|
3392
|
-
if (params.limit <= 0) return [];
|
|
3393
|
-
const ftsQuery = params.buildFtsQuery(params.query);
|
|
3394
|
-
if (!ftsQuery) return [];
|
|
3395
|
-
const modelClause = params.providerModel ? " AND model = ?" : "";
|
|
3396
|
-
const modelParams = params.providerModel ? [params.providerModel] : [];
|
|
3397
|
-
return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${params.ftsTable}) AS rank\n FROM ${params.ftsTable}\n WHERE ${params.ftsTable} MATCH ?${modelClause}${params.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(ftsQuery, ...modelParams, ...params.sourceFilter.params, params.limit).map((row) => {
|
|
3398
|
-
const textScore = params.bm25RankToScore(row.rank);
|
|
3399
|
-
return {
|
|
3400
|
-
id: row.id,
|
|
3401
|
-
path: row.path,
|
|
3402
|
-
startLine: row.start_line,
|
|
3403
|
-
endLine: row.end_line,
|
|
3404
|
-
score: textScore,
|
|
3405
|
-
textScore,
|
|
3406
|
-
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
3407
|
-
source: row.source
|
|
3408
|
-
};
|
|
3409
|
-
});
|
|
3410
|
-
}
|
|
3411
|
-
|
|
3412
|
-
//#endregion
|
|
3413
|
-
//#region src/memory/manager.ts
|
|
3414
|
-
const SNIPPET_MAX_CHARS = 700;
|
|
3415
|
-
const VECTOR_TABLE = "chunks_vec";
|
|
3416
|
-
const FTS_TABLE = "chunks_fts";
|
|
3417
|
-
const EMBEDDING_CACHE_TABLE = "embedding_cache";
|
|
3418
|
-
const BATCH_FAILURE_LIMIT = 2;
|
|
3419
|
-
const log = createSubsystemLogger("memory");
|
|
3420
|
-
const INDEX_CACHE = /* @__PURE__ */ new Map();
|
|
3421
|
-
const INDEX_CACHE_PENDING = /* @__PURE__ */ new Map();
|
|
3422
|
-
var MemoryIndexManager = class MemoryIndexManager extends MemoryManagerEmbeddingOps {
|
|
3423
|
-
static async get(params) {
|
|
3424
|
-
const { cfg, agentId } = params;
|
|
3425
|
-
const settings = resolveMemorySearchConfig(cfg, agentId);
|
|
3426
|
-
if (!settings) return null;
|
|
3427
|
-
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
|
|
3428
|
-
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
|
|
3429
|
-
const existing = INDEX_CACHE.get(key);
|
|
3430
|
-
if (existing) return existing;
|
|
3431
|
-
const pending = INDEX_CACHE_PENDING.get(key);
|
|
3432
|
-
if (pending) return pending;
|
|
3433
|
-
const createPromise = (async () => {
|
|
3434
|
-
const providerResult = await createEmbeddingProvider({
|
|
3435
|
-
config: cfg,
|
|
3436
|
-
agentDir: resolveAgentDir(cfg, agentId),
|
|
3437
|
-
provider: settings.provider,
|
|
3438
|
-
remote: settings.remote,
|
|
3439
|
-
model: settings.model,
|
|
3440
|
-
fallback: settings.fallback,
|
|
3441
|
-
local: settings.local
|
|
3442
|
-
});
|
|
3443
|
-
const refreshed = INDEX_CACHE.get(key);
|
|
3444
|
-
if (refreshed) return refreshed;
|
|
3445
|
-
const manager = new MemoryIndexManager({
|
|
3446
|
-
cacheKey: key,
|
|
3447
|
-
cfg,
|
|
3448
|
-
agentId,
|
|
3449
|
-
workspaceDir,
|
|
3450
|
-
settings,
|
|
3451
|
-
providerResult,
|
|
3452
|
-
purpose: params.purpose
|
|
3453
|
-
});
|
|
3454
|
-
INDEX_CACHE.set(key, manager);
|
|
3455
|
-
return manager;
|
|
3456
|
-
})();
|
|
3457
|
-
INDEX_CACHE_PENDING.set(key, createPromise);
|
|
3458
|
-
try {
|
|
3459
|
-
return await createPromise;
|
|
3460
|
-
} finally {
|
|
3461
|
-
if (INDEX_CACHE_PENDING.get(key) === createPromise) INDEX_CACHE_PENDING.delete(key);
|
|
3462
|
-
}
|
|
3463
|
-
}
|
|
3464
|
-
constructor(params) {
|
|
3465
|
-
super();
|
|
3466
|
-
this.batchFailureCount = 0;
|
|
3467
|
-
this.batchFailureLock = Promise.resolve();
|
|
3468
|
-
this.vectorReady = null;
|
|
3469
|
-
this.watcher = null;
|
|
3470
|
-
this.watchTimer = null;
|
|
3471
|
-
this.sessionWatchTimer = null;
|
|
3472
|
-
this.sessionUnsubscribe = null;
|
|
3473
|
-
this.intervalTimer = null;
|
|
3474
|
-
this.closed = false;
|
|
3475
|
-
this.dirty = false;
|
|
3476
|
-
this.sessionsDirty = false;
|
|
3477
|
-
this.sessionsDirtyFiles = /* @__PURE__ */ new Set();
|
|
3478
|
-
this.sessionPendingFiles = /* @__PURE__ */ new Set();
|
|
3479
|
-
this.sessionDeltas = /* @__PURE__ */ new Map();
|
|
3480
|
-
this.sessionWarm = /* @__PURE__ */ new Set();
|
|
3481
|
-
this.syncing = null;
|
|
3482
|
-
this.readonlyRecoveryAttempts = 0;
|
|
3483
|
-
this.readonlyRecoverySuccesses = 0;
|
|
3484
|
-
this.readonlyRecoveryFailures = 0;
|
|
3485
|
-
this.cacheKey = params.cacheKey;
|
|
3486
|
-
this.cfg = params.cfg;
|
|
3487
|
-
this.agentId = params.agentId;
|
|
3488
|
-
this.workspaceDir = params.workspaceDir;
|
|
3489
|
-
this.settings = params.settings;
|
|
3490
|
-
this.provider = params.providerResult.provider;
|
|
3491
|
-
this.requestedProvider = params.providerResult.requestedProvider;
|
|
3492
|
-
this.fallbackFrom = params.providerResult.fallbackFrom;
|
|
3493
|
-
this.fallbackReason = params.providerResult.fallbackReason;
|
|
3494
|
-
this.providerUnavailableReason = params.providerResult.providerUnavailableReason;
|
|
3495
|
-
this.openAi = params.providerResult.openAi;
|
|
3496
|
-
this.gemini = params.providerResult.gemini;
|
|
3497
|
-
this.voyage = params.providerResult.voyage;
|
|
3498
|
-
this.mistral = params.providerResult.mistral;
|
|
3499
|
-
this.ollama = params.providerResult.ollama;
|
|
3500
|
-
this.sources = new Set(params.settings.sources);
|
|
3501
|
-
this.db = this.openDatabase();
|
|
3502
|
-
this.providerKey = this.computeProviderKey();
|
|
3503
|
-
this.cache = {
|
|
3504
|
-
enabled: params.settings.cache.enabled,
|
|
3505
|
-
maxEntries: params.settings.cache.maxEntries
|
|
3506
|
-
};
|
|
3507
|
-
this.fts = {
|
|
3508
|
-
enabled: params.settings.query.hybrid.enabled,
|
|
3509
|
-
available: false
|
|
3510
|
-
};
|
|
3511
|
-
this.ensureSchema();
|
|
3512
|
-
this.vector = {
|
|
3513
|
-
enabled: params.settings.store.vector.enabled,
|
|
3514
|
-
available: null,
|
|
3515
|
-
extensionPath: params.settings.store.vector.extensionPath
|
|
3516
|
-
};
|
|
3517
|
-
const meta = this.readMeta();
|
|
3518
|
-
if (meta?.vectorDims) this.vector.dims = meta.vectorDims;
|
|
3519
|
-
this.ensureWatcher();
|
|
3520
|
-
this.ensureSessionListener();
|
|
3521
|
-
this.ensureIntervalSync();
|
|
3522
|
-
const statusOnly = params.purpose === "status";
|
|
3523
|
-
this.dirty = this.sources.has("memory") && (statusOnly ? !meta : true);
|
|
3524
|
-
this.batch = this.resolveBatchConfig();
|
|
3525
|
-
}
|
|
3526
|
-
async warmSession(sessionKey) {
|
|
3527
|
-
if (!this.settings.sync.onSessionStart) return;
|
|
3528
|
-
const key = sessionKey?.trim() || "";
|
|
3529
|
-
if (key && this.sessionWarm.has(key)) return;
|
|
3530
|
-
this.sync({ reason: "session-start" }).catch((err) => {
|
|
3531
|
-
log.warn(`memory sync failed (session-start): ${String(err)}`);
|
|
3532
|
-
});
|
|
3533
|
-
if (key) this.sessionWarm.add(key);
|
|
3534
|
-
}
|
|
3535
|
-
async search(query, opts) {
|
|
3536
|
-
this.warmSession(opts?.sessionKey);
|
|
3537
|
-
if (this.settings.sync.onSearch && (this.dirty || this.sessionsDirty)) this.sync({ reason: "search" }).catch((err) => {
|
|
3538
|
-
log.warn(`memory sync failed (search): ${String(err)}`);
|
|
3539
|
-
});
|
|
3540
|
-
const cleaned = query.trim();
|
|
3541
|
-
if (!cleaned) return [];
|
|
3542
|
-
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
|
3543
|
-
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
|
3544
|
-
const hybrid = this.settings.query.hybrid;
|
|
3545
|
-
const candidates = Math.min(200, Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)));
|
|
3546
|
-
if (!this.provider) {
|
|
3547
|
-
if (!this.fts.enabled || !this.fts.available) {
|
|
3548
|
-
log.warn("memory search: no provider and FTS unavailable");
|
|
3549
|
-
return [];
|
|
3550
|
-
}
|
|
3551
|
-
const keywords = extractKeywords(cleaned);
|
|
3552
|
-
const searchTerms = keywords.length > 0 ? keywords : [cleaned];
|
|
3553
|
-
const resultSets = await Promise.all(searchTerms.map((term) => this.searchKeyword(term, candidates).catch(() => [])));
|
|
3554
|
-
const seenIds = /* @__PURE__ */ new Map();
|
|
3555
|
-
for (const results of resultSets) for (const result of results) {
|
|
3556
|
-
const existing = seenIds.get(result.id);
|
|
3557
|
-
if (!existing || result.score > existing.score) seenIds.set(result.id, result);
|
|
3558
|
-
}
|
|
3559
|
-
return [...seenIds.values()].toSorted((a, b) => b.score - a.score).filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
|
3560
|
-
}
|
|
3561
|
-
const keywordResults = hybrid.enabled && this.fts.enabled && this.fts.available ? await this.searchKeyword(cleaned, candidates).catch(() => []) : [];
|
|
3562
|
-
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
|
3563
|
-
const vectorResults = queryVec.some((v) => v !== 0) ? await this.searchVector(queryVec, candidates).catch(() => []) : [];
|
|
3564
|
-
if (!hybrid.enabled || !this.fts.enabled || !this.fts.available) return vectorResults.filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
|
3565
|
-
const merged = await this.mergeHybridResults({
|
|
3566
|
-
vector: vectorResults,
|
|
3567
|
-
keyword: keywordResults,
|
|
3568
|
-
vectorWeight: hybrid.vectorWeight,
|
|
3569
|
-
textWeight: hybrid.textWeight,
|
|
3570
|
-
mmr: hybrid.mmr,
|
|
3571
|
-
temporalDecay: hybrid.temporalDecay
|
|
3572
|
-
});
|
|
3573
|
-
const strict = merged.filter((entry) => entry.score >= minScore);
|
|
3574
|
-
if (strict.length > 0 || keywordResults.length === 0) return strict.slice(0, maxResults);
|
|
3575
|
-
const relaxedMinScore = Math.min(minScore, hybrid.textWeight);
|
|
3576
|
-
const keywordKeys = new Set(keywordResults.map((entry) => `${entry.source}:${entry.path}:${entry.startLine}:${entry.endLine}`));
|
|
3577
|
-
return merged.filter((entry) => keywordKeys.has(`${entry.source}:${entry.path}:${entry.startLine}:${entry.endLine}`) && entry.score >= relaxedMinScore).slice(0, maxResults);
|
|
3578
|
-
}
|
|
3579
|
-
async searchVector(queryVec, limit) {
|
|
3580
|
-
if (!this.provider) return [];
|
|
3581
|
-
return (await searchVector({
|
|
3582
|
-
db: this.db,
|
|
3583
|
-
vectorTable: VECTOR_TABLE,
|
|
3584
|
-
providerModel: this.provider.model,
|
|
3585
|
-
queryVec,
|
|
3586
|
-
limit,
|
|
3587
|
-
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
3588
|
-
ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions),
|
|
3589
|
-
sourceFilterVec: this.buildSourceFilter("c"),
|
|
3590
|
-
sourceFilterChunks: this.buildSourceFilter()
|
|
3591
|
-
})).map((entry) => entry);
|
|
3592
|
-
}
|
|
3593
|
-
buildFtsQuery(raw) {
|
|
3594
|
-
return buildFtsQuery(raw);
|
|
3595
|
-
}
|
|
3596
|
-
async searchKeyword(query, limit) {
|
|
3597
|
-
if (!this.fts.enabled || !this.fts.available) return [];
|
|
3598
|
-
const sourceFilter = this.buildSourceFilter();
|
|
3599
|
-
const providerModel = this.provider?.model;
|
|
3600
|
-
return (await searchKeyword({
|
|
3601
|
-
db: this.db,
|
|
3602
|
-
ftsTable: FTS_TABLE,
|
|
3603
|
-
providerModel,
|
|
3604
|
-
query,
|
|
3605
|
-
limit,
|
|
3606
|
-
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
3607
|
-
sourceFilter,
|
|
3608
|
-
buildFtsQuery: (raw) => this.buildFtsQuery(raw),
|
|
3609
|
-
bm25RankToScore
|
|
3610
|
-
})).map((entry) => entry);
|
|
3611
|
-
}
|
|
3612
|
-
mergeHybridResults(params) {
|
|
3613
|
-
return mergeHybridResults({
|
|
3614
|
-
vector: params.vector.map((r) => ({
|
|
3615
|
-
id: r.id,
|
|
3616
|
-
path: r.path,
|
|
3617
|
-
startLine: r.startLine,
|
|
3618
|
-
endLine: r.endLine,
|
|
3619
|
-
source: r.source,
|
|
3620
|
-
snippet: r.snippet,
|
|
3621
|
-
vectorScore: r.score
|
|
3622
|
-
})),
|
|
3623
|
-
keyword: params.keyword.map((r) => ({
|
|
3624
|
-
id: r.id,
|
|
3625
|
-
path: r.path,
|
|
3626
|
-
startLine: r.startLine,
|
|
3627
|
-
endLine: r.endLine,
|
|
3628
|
-
source: r.source,
|
|
3629
|
-
snippet: r.snippet,
|
|
3630
|
-
textScore: r.textScore
|
|
3631
|
-
})),
|
|
3632
|
-
vectorWeight: params.vectorWeight,
|
|
3633
|
-
textWeight: params.textWeight,
|
|
3634
|
-
mmr: params.mmr,
|
|
3635
|
-
temporalDecay: params.temporalDecay,
|
|
3636
|
-
workspaceDir: this.workspaceDir
|
|
3637
|
-
}).then((entries) => entries.map((entry) => entry));
|
|
3638
|
-
}
|
|
3639
|
-
async sync(params) {
|
|
3640
|
-
if (this.closed) return;
|
|
3641
|
-
if (this.syncing) return this.syncing;
|
|
3642
|
-
this.syncing = this.runSyncWithReadonlyRecovery(params).finally(() => {
|
|
3643
|
-
this.syncing = null;
|
|
3644
|
-
});
|
|
3645
|
-
return this.syncing ?? Promise.resolve();
|
|
3646
|
-
}
|
|
3647
|
-
isReadonlyDbError(err) {
|
|
3648
|
-
const readonlyPattern = /attempt to write a readonly database|database is read-only|SQLITE_READONLY/i;
|
|
3649
|
-
const messages = /* @__PURE__ */ new Set();
|
|
3650
|
-
const pushValue = (value) => {
|
|
3651
|
-
if (typeof value !== "string") return;
|
|
3652
|
-
const normalized = value.trim();
|
|
3653
|
-
if (!normalized) return;
|
|
3654
|
-
messages.add(normalized);
|
|
3655
|
-
};
|
|
3656
|
-
pushValue(err instanceof Error ? err.message : String(err));
|
|
3657
|
-
if (err && typeof err === "object") {
|
|
3658
|
-
const record = err;
|
|
3659
|
-
pushValue(record.message);
|
|
3660
|
-
pushValue(record.code);
|
|
3661
|
-
pushValue(record.name);
|
|
3662
|
-
if (record.cause && typeof record.cause === "object") {
|
|
3663
|
-
const cause = record.cause;
|
|
3664
|
-
pushValue(cause.message);
|
|
3665
|
-
pushValue(cause.code);
|
|
3666
|
-
pushValue(cause.name);
|
|
3667
|
-
}
|
|
3668
|
-
}
|
|
3669
|
-
return [...messages].some((value) => readonlyPattern.test(value));
|
|
3670
|
-
}
|
|
3671
|
-
extractErrorReason(err) {
|
|
3672
|
-
if (err instanceof Error && err.message.trim()) return err.message;
|
|
3673
|
-
if (err && typeof err === "object") {
|
|
3674
|
-
const record = err;
|
|
3675
|
-
if (typeof record.message === "string" && record.message.trim()) return record.message;
|
|
3676
|
-
if (typeof record.code === "string" && record.code.trim()) return record.code;
|
|
3677
|
-
}
|
|
3678
|
-
return String(err);
|
|
3679
|
-
}
|
|
3680
|
-
async runSyncWithReadonlyRecovery(params) {
|
|
3681
|
-
try {
|
|
3682
|
-
await this.runSync(params);
|
|
3683
|
-
return;
|
|
3684
|
-
} catch (err) {
|
|
3685
|
-
if (!this.isReadonlyDbError(err) || this.closed) throw err;
|
|
3686
|
-
const reason = this.extractErrorReason(err);
|
|
3687
|
-
this.readonlyRecoveryAttempts += 1;
|
|
3688
|
-
this.readonlyRecoveryLastError = reason;
|
|
3689
|
-
log.warn(`memory sync readonly handle detected; reopening sqlite connection`, { reason });
|
|
3690
|
-
try {
|
|
3691
|
-
this.db.close();
|
|
3692
|
-
} catch {}
|
|
3693
|
-
this.db = this.openDatabase();
|
|
3694
|
-
this.vectorReady = null;
|
|
3695
|
-
this.vector.available = null;
|
|
3696
|
-
this.vector.loadError = void 0;
|
|
3697
|
-
this.ensureSchema();
|
|
3698
|
-
const meta = this.readMeta();
|
|
3699
|
-
this.vector.dims = meta?.vectorDims;
|
|
3700
|
-
try {
|
|
3701
|
-
await this.runSync(params);
|
|
3702
|
-
this.readonlyRecoverySuccesses += 1;
|
|
3703
|
-
} catch (retryErr) {
|
|
3704
|
-
this.readonlyRecoveryFailures += 1;
|
|
3705
|
-
throw retryErr;
|
|
3706
|
-
}
|
|
3707
|
-
}
|
|
3708
|
-
}
|
|
3709
|
-
async readFile(params) {
|
|
3710
|
-
const rawPath = params.relPath.trim();
|
|
3711
|
-
if (!rawPath) throw new Error("path required");
|
|
3712
|
-
const absPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(this.workspaceDir, rawPath);
|
|
3713
|
-
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
|
|
3714
|
-
const allowedWorkspace = relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath) && isMemoryPath(relPath);
|
|
3715
|
-
let allowedAdditional = false;
|
|
3716
|
-
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
|
|
3717
|
-
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths);
|
|
3718
|
-
for (const additionalPath of additionalPaths) try {
|
|
3719
|
-
const stat = await fs$1.lstat(additionalPath);
|
|
3720
|
-
if (stat.isSymbolicLink()) continue;
|
|
3721
|
-
if (stat.isDirectory()) {
|
|
3722
|
-
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
|
3723
|
-
allowedAdditional = true;
|
|
3724
|
-
break;
|
|
3725
|
-
}
|
|
3726
|
-
continue;
|
|
3727
|
-
}
|
|
3728
|
-
if (stat.isFile()) {
|
|
3729
|
-
if (absPath === additionalPath && absPath.endsWith(".md")) {
|
|
3730
|
-
allowedAdditional = true;
|
|
3731
|
-
break;
|
|
3732
|
-
}
|
|
3733
|
-
}
|
|
3734
|
-
} catch {}
|
|
3735
|
-
}
|
|
3736
|
-
if (!allowedWorkspace && !allowedAdditional) throw new Error("path required");
|
|
3737
|
-
if (!absPath.endsWith(".md")) throw new Error("path required");
|
|
3738
|
-
if ((await statRegularFile(absPath)).missing) return {
|
|
3739
|
-
text: "",
|
|
3740
|
-
path: relPath
|
|
3741
|
-
};
|
|
3742
|
-
let content;
|
|
3743
|
-
try {
|
|
3744
|
-
content = await fs$1.readFile(absPath, "utf-8");
|
|
3745
|
-
} catch (err) {
|
|
3746
|
-
if (isFileMissingError(err)) return {
|
|
3747
|
-
text: "",
|
|
3748
|
-
path: relPath
|
|
3749
|
-
};
|
|
3750
|
-
throw err;
|
|
3751
|
-
}
|
|
3752
|
-
if (!params.from && !params.lines) return {
|
|
3753
|
-
text: content,
|
|
3754
|
-
path: relPath
|
|
3755
|
-
};
|
|
3756
|
-
const lines = content.split("\n");
|
|
3757
|
-
const start = Math.max(1, params.from ?? 1);
|
|
3758
|
-
const count = Math.max(1, params.lines ?? lines.length);
|
|
3759
|
-
return {
|
|
3760
|
-
text: lines.slice(start - 1, start - 1 + count).join("\n"),
|
|
3761
|
-
path: relPath
|
|
3762
|
-
};
|
|
3763
|
-
}
|
|
3764
|
-
status() {
|
|
3765
|
-
const sourceFilter = this.buildSourceFilter();
|
|
3766
|
-
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params);
|
|
3767
|
-
const chunks = this.db.prepare(`SELECT COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params);
|
|
3768
|
-
const sourceCounts = (() => {
|
|
3769
|
-
const sources = Array.from(this.sources);
|
|
3770
|
-
if (sources.length === 0) return [];
|
|
3771
|
-
const bySource = /* @__PURE__ */ new Map();
|
|
3772
|
-
for (const source of sources) bySource.set(source, {
|
|
3773
|
-
files: 0,
|
|
3774
|
-
chunks: 0
|
|
3775
|
-
});
|
|
3776
|
-
const fileRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params);
|
|
3777
|
-
for (const row of fileRows) {
|
|
3778
|
-
const entry = bySource.get(row.source) ?? {
|
|
3779
|
-
files: 0,
|
|
3780
|
-
chunks: 0
|
|
3781
|
-
};
|
|
3782
|
-
entry.files = row.c ?? 0;
|
|
3783
|
-
bySource.set(row.source, entry);
|
|
3784
|
-
}
|
|
3785
|
-
const chunkRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params);
|
|
3786
|
-
for (const row of chunkRows) {
|
|
3787
|
-
const entry = bySource.get(row.source) ?? {
|
|
3788
|
-
files: 0,
|
|
3789
|
-
chunks: 0
|
|
3790
|
-
};
|
|
3791
|
-
entry.chunks = row.c ?? 0;
|
|
3792
|
-
bySource.set(row.source, entry);
|
|
3793
|
-
}
|
|
3794
|
-
return sources.map((source) => Object.assign({ source }, bySource.get(source)));
|
|
3795
|
-
})();
|
|
3796
|
-
const searchMode = this.provider ? "hybrid" : "fts-only";
|
|
3797
|
-
const providerInfo = this.provider ? {
|
|
3798
|
-
provider: this.provider.id,
|
|
3799
|
-
model: this.provider.model
|
|
3800
|
-
} : {
|
|
3801
|
-
provider: "none",
|
|
3802
|
-
model: void 0
|
|
3803
|
-
};
|
|
3804
|
-
return {
|
|
3805
|
-
backend: "builtin",
|
|
3806
|
-
files: files?.c ?? 0,
|
|
3807
|
-
chunks: chunks?.c ?? 0,
|
|
3808
|
-
dirty: this.dirty || this.sessionsDirty,
|
|
3809
|
-
workspaceDir: this.workspaceDir,
|
|
3810
|
-
dbPath: this.settings.store.path,
|
|
3811
|
-
provider: providerInfo.provider,
|
|
3812
|
-
model: providerInfo.model,
|
|
3813
|
-
requestedProvider: this.requestedProvider,
|
|
3814
|
-
sources: Array.from(this.sources),
|
|
3815
|
-
extraPaths: this.settings.extraPaths,
|
|
3816
|
-
sourceCounts,
|
|
3817
|
-
cache: this.cache.enabled ? {
|
|
3818
|
-
enabled: true,
|
|
3819
|
-
entries: this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get()?.c ?? 0,
|
|
3820
|
-
maxEntries: this.cache.maxEntries
|
|
3821
|
-
} : {
|
|
3822
|
-
enabled: false,
|
|
3823
|
-
maxEntries: this.cache.maxEntries
|
|
3824
|
-
},
|
|
3825
|
-
fts: {
|
|
3826
|
-
enabled: this.fts.enabled,
|
|
3827
|
-
available: this.fts.available,
|
|
3828
|
-
error: this.fts.loadError
|
|
3829
|
-
},
|
|
3830
|
-
fallback: this.fallbackReason ? {
|
|
3831
|
-
from: this.fallbackFrom ?? "local",
|
|
3832
|
-
reason: this.fallbackReason
|
|
3833
|
-
} : void 0,
|
|
3834
|
-
vector: {
|
|
3835
|
-
enabled: this.vector.enabled,
|
|
3836
|
-
available: this.vector.available ?? void 0,
|
|
3837
|
-
extensionPath: this.vector.extensionPath,
|
|
3838
|
-
loadError: this.vector.loadError,
|
|
3839
|
-
dims: this.vector.dims
|
|
3840
|
-
},
|
|
3841
|
-
batch: {
|
|
3842
|
-
enabled: this.batch.enabled,
|
|
3843
|
-
failures: this.batchFailureCount,
|
|
3844
|
-
limit: BATCH_FAILURE_LIMIT,
|
|
3845
|
-
wait: this.batch.wait,
|
|
3846
|
-
concurrency: this.batch.concurrency,
|
|
3847
|
-
pollIntervalMs: this.batch.pollIntervalMs,
|
|
3848
|
-
timeoutMs: this.batch.timeoutMs,
|
|
3849
|
-
lastError: this.batchFailureLastError,
|
|
3850
|
-
lastProvider: this.batchFailureLastProvider
|
|
3851
|
-
},
|
|
3852
|
-
custom: {
|
|
3853
|
-
searchMode,
|
|
3854
|
-
providerUnavailableReason: this.providerUnavailableReason,
|
|
3855
|
-
readonlyRecovery: {
|
|
3856
|
-
attempts: this.readonlyRecoveryAttempts,
|
|
3857
|
-
successes: this.readonlyRecoverySuccesses,
|
|
3858
|
-
failures: this.readonlyRecoveryFailures,
|
|
3859
|
-
lastError: this.readonlyRecoveryLastError
|
|
3860
|
-
}
|
|
3861
|
-
}
|
|
3862
|
-
};
|
|
3863
|
-
}
|
|
3864
|
-
async probeVectorAvailability() {
|
|
3865
|
-
if (!this.provider) return false;
|
|
3866
|
-
if (!this.vector.enabled) return false;
|
|
3867
|
-
return this.ensureVectorReady();
|
|
3868
|
-
}
|
|
3869
|
-
async probeEmbeddingAvailability() {
|
|
3870
|
-
if (!this.provider) return {
|
|
3871
|
-
ok: false,
|
|
3872
|
-
error: this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)"
|
|
3873
|
-
};
|
|
3874
|
-
try {
|
|
3875
|
-
await this.embedBatchWithRetry(["ping"]);
|
|
3876
|
-
return { ok: true };
|
|
3877
|
-
} catch (err) {
|
|
3878
|
-
return {
|
|
3879
|
-
ok: false,
|
|
3880
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3881
|
-
};
|
|
3882
|
-
}
|
|
3883
|
-
}
|
|
3884
|
-
async close() {
|
|
3885
|
-
if (this.closed) return;
|
|
3886
|
-
this.closed = true;
|
|
3887
|
-
const pendingSync = this.syncing;
|
|
3888
|
-
if (this.watchTimer) {
|
|
3889
|
-
clearTimeout(this.watchTimer);
|
|
3890
|
-
this.watchTimer = null;
|
|
3891
|
-
}
|
|
3892
|
-
if (this.sessionWatchTimer) {
|
|
3893
|
-
clearTimeout(this.sessionWatchTimer);
|
|
3894
|
-
this.sessionWatchTimer = null;
|
|
3895
|
-
}
|
|
3896
|
-
if (this.intervalTimer) {
|
|
3897
|
-
clearInterval(this.intervalTimer);
|
|
3898
|
-
this.intervalTimer = null;
|
|
3899
|
-
}
|
|
3900
|
-
if (this.watcher) {
|
|
3901
|
-
await this.watcher.close();
|
|
3902
|
-
this.watcher = null;
|
|
3903
|
-
}
|
|
3904
|
-
if (this.sessionUnsubscribe) {
|
|
3905
|
-
this.sessionUnsubscribe();
|
|
3906
|
-
this.sessionUnsubscribe = null;
|
|
3907
|
-
}
|
|
3908
|
-
if (pendingSync) try {
|
|
3909
|
-
await pendingSync;
|
|
3910
|
-
} catch {}
|
|
3911
|
-
this.db.close();
|
|
3912
|
-
INDEX_CACHE.delete(this.cacheKey);
|
|
3913
|
-
}
|
|
3914
|
-
};
|
|
3915
|
-
|
|
3916
|
-
//#endregion
|
|
3917
|
-
export { resolveMemorySearchConfig as n, MemoryIndexManager as t };
|