pi-mega-compact 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/api-contracts/cache-stripes.js +8 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints.js +32 -0
- package/dist/extensions/dashboard-server/api-contracts/memory-map.js +1 -0
- package/dist/extensions/dashboard-server/api-contracts/memory.js +8 -0
- package/dist/extensions/dashboard-server/api-contracts/provider-cache.js +1 -0
- package/dist/extensions/dashboard-server/api-contracts/setup.js +10 -0
- package/dist/extensions/dashboard-server/index-reader.js +6 -2
- package/dist/extensions/dashboard-server/routes-cache.js +135 -47
- package/dist/extensions/dashboard-server/routes-game.js +3 -0
- package/dist/extensions/dashboard-server/routes-memory-map.js +50 -0
- package/dist/extensions/dashboard-server/routes-memory.js +51 -0
- package/dist/extensions/dashboard-server/routes-repo.js +2 -2
- package/dist/extensions/dashboard-server/routes-setup.js +267 -0
- package/dist/extensions/dashboard-server/routes.js +4 -0
- package/dist/extensions/dashboard-server/server.js +13 -1
- package/dist/extensions/mega-commands.js +189 -1
- package/dist/extensions/mega-compact.js +5 -0
- package/dist/extensions/mega-config.js +7 -0
- package/dist/extensions/mega-events/agent-handlers.js +19 -4
- package/dist/extensions/mega-events/compact-handlers.js +1 -0
- package/dist/extensions/mega-events/context-handler.js +45 -92
- package/dist/extensions/mega-events/context-hwm.js +51 -0
- package/dist/extensions/mega-events/error-classifier.js +16 -5
- package/dist/extensions/mega-events/mirror-append.js +107 -0
- package/dist/extensions/mega-events/outage-advisor.js +7 -1
- package/dist/extensions/mega-events/perf-handler.js +134 -2
- package/dist/extensions/mega-events/recall-tail.js +59 -0
- package/dist/extensions/mega-events/separated-prompt.js +248 -0
- package/dist/extensions/mega-events/session-handlers.js +15 -8
- package/dist/extensions/mega-runtime/env-loader.js +56 -0
- package/dist/extensions/mega-runtime/reset-runtime.js +6 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +1 -0
- package/dist/extensions/mega-runtime/runtime.js +7 -0
- package/dist/extensions/mega-runtime/snapshot.js +1 -0
- package/dist/extensions/mega-runtime/widget.js +5 -0
- package/dist/src/cache-stripe-impl.js +288 -0
- package/dist/src/cache-stripe.js +11 -0
- package/dist/src/config/stopwords.js +24 -0
- package/dist/src/config.js +17 -0
- package/dist/src/embedder.js +41 -2
- package/dist/src/memoryGraph.js +299 -0
- package/dist/src/memoryStats.js +75 -0
- package/dist/src/monitoring.js +13 -0
- package/dist/src/prefix-break.js +43 -0
- package/dist/src/pricing.js +1 -1
- package/dist/src/queryExpansion.js +163 -0
- package/dist/src/queryReformulation/cache.js +123 -0
- package/dist/src/queryReformulation/rrf.js +29 -0
- package/dist/src/queryReformulation/tfidf.js +61 -0
- package/dist/src/queryReformulation/vagueness.js +33 -0
- package/dist/src/queryReformulation.js +225 -0
- package/dist/src/recallMetrics.js +258 -0
- package/dist/src/store/sqlite/cache-stripes.js +133 -0
- package/dist/src/store/sqlite/fts5-search.js +68 -0
- package/dist/src/store/sqlite/perf-samples.js +145 -16
- package/dist/src/store/sqlite/schema.js +60 -1
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/store/turns/memory-store.js +8 -0
- package/dist/src/store/turns/sqlite-store.js +14 -3
- package/dist/src/store/turns/types.js +18 -1
- package/dist/src/store/vectorIndex.js +55 -10
- package/dist/src/tiered-router/types.js +18 -0
- package/dist/src/tieredRouter.js +421 -0
- package/dist/src/vectorStore.js +8 -6
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-vEo2MtE2.js → AchievementTiles-DR9cLGDL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AchievementTiles-vEo2MtE2.js.map → AchievementTiles-DR9cLGDL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/AchievementsTab-svNUgJ1e.js +2 -0
- package/extensions/dashboard-client/dist/assets/{AchievementsTab-DjxfYrDc.js.map → AchievementsTab-svNUgJ1e.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js +2 -0
- package/extensions/dashboard-client/dist/assets/CacheTab-ASwhOMjR.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{ConfigTab-Cm6ymbmx.js → ConfigTab-CmmWheuW.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ConfigTab-Cm6ymbmx.js.map → ConfigTab-CmmWheuW.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-DXWl_YzI.js → EventsTab-CFE7jLsa.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-DXWl_YzI.js.map → EventsTab-CFE7jLsa.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/GameTab-CJAX_th_.js +2 -0
- package/extensions/dashboard-client/dist/assets/{GameTab-BLDudcO5.js.map → GameTab-CJAX_th_.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLkKCztn.js → MaintenanceTab-P4SUsvMf.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DLkKCztn.js.map → MaintenanceTab-P4SUsvMf.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js +2 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-BBiH76eF.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{MetricsTab-Cq4u-rZF.js → MetricsTab-CHQscixy.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-Cq4u-rZF.js.map → MetricsTab-CHQscixy.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/OverviewTab-DBs4DJdd.js +2 -0
- package/extensions/dashboard-client/dist/assets/{OverviewTab-D4kGvsiQ.js.map → OverviewTab-DBs4DJdd.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/ReposTab-DeFz76L5.js +2 -0
- package/extensions/dashboard-client/dist/assets/ReposTab-DeFz76L5.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{SessionsTab-DZC2-GCn.js → SessionsTab-BWNsj1OB.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-DZC2-GCn.js.map → SessionsTab-BWNsj1OB.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js +2 -0
- package/extensions/dashboard-client/dist/assets/SetupTab-uS6A0xmw.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/TimeSavedCard-CiqCOQuL.js +2 -0
- package/extensions/dashboard-client/dist/assets/TimeSavedCard-CiqCOQuL.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{TopicsTab-01G71hv1.js → TopicsTab-BWWQxltv.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TopicsTab-01G71hv1.js.map → TopicsTab-BWWQxltv.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DLRg0LZT.js → TurnsTab-CVz4f4-o.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DLRg0LZT.js.map → TurnsTab-CVz4f4-o.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/format-COjDIVGj.js +2 -0
- package/extensions/dashboard-client/dist/assets/format-COjDIVGj.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/index-CUNn22iV.css +1 -0
- package/extensions/dashboard-client/dist/assets/index-DrktNOoT.js +42 -0
- package/extensions/dashboard-client/dist/assets/index-DrktNOoT.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js +2 -0
- package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{useSSE-BBYt5UPB.js → useSSE-Bf5LOLSi.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-BBYt5UPB.js.map → useSSE-Bf5LOLSi.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +2 -2
- package/extensions/dashboard-client/src/App.tsx +28 -8
- package/extensions/dashboard-client/src/api/client.ts +29 -0
- package/extensions/dashboard-client/src/components/ActiveReposTable.tsx +21 -12
- package/extensions/dashboard-client/src/components/CacheHitRateTrendCard.tsx +100 -0
- package/extensions/dashboard-client/src/components/ProviderCacheCard.tsx +56 -0
- package/extensions/dashboard-client/src/components/SavingsByModelTable.tsx +38 -57
- package/extensions/dashboard-client/src/components/StripeDistributionCard.tsx +110 -0
- package/extensions/dashboard-client/src/components/TabBar.tsx +49 -4
- package/extensions/dashboard-client/src/main.tsx +1 -0
- package/extensions/dashboard-client/src/styles/base.css +30 -0
- package/extensions/dashboard-client/src/styles/cache.css +167 -0
- package/extensions/dashboard-client/src/tabs/CacheTab.tsx +75 -12
- package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +440 -0
- package/extensions/dashboard-client/src/tabs/SetupTab.tsx +418 -0
- package/extensions/dashboard-client/tsconfig.json +3 -2
- package/extensions/dashboard-client/vite.config.ts +2 -0
- package/extensions/dashboard-server/api-contracts/cache-stripes.ts +49 -0
- package/extensions/dashboard-server/api-contracts/endpoints.ts +66 -3
- package/extensions/dashboard-server/api-contracts/index.ts +22 -0
- package/extensions/dashboard-server/api-contracts/memory-map.ts +50 -0
- package/extensions/dashboard-server/api-contracts/memory.ts +27 -0
- package/extensions/dashboard-server/api-contracts/multi-repo.ts +231 -231
- package/extensions/dashboard-server/api-contracts/provider-cache.ts +37 -0
- package/extensions/dashboard-server/api-contracts/setup.ts +80 -0
- package/extensions/dashboard-server/index-reader.ts +138 -121
- package/extensions/dashboard-server/routes-cache.ts +186 -75
- package/extensions/dashboard-server/routes-game.ts +3 -0
- package/extensions/dashboard-server/routes-memory-map.ts +105 -0
- package/extensions/dashboard-server/routes-memory.ts +63 -0
- package/extensions/dashboard-server/routes-repo.ts +36 -36
- package/extensions/dashboard-server/routes-setup.ts +293 -0
- package/extensions/dashboard-server/routes.ts +4 -0
- package/extensions/dashboard-server/server.ts +12 -0
- package/extensions/mega-commands.ts +211 -1
- package/extensions/mega-compact.ts +5 -0
- package/extensions/mega-config.ts +27 -0
- package/extensions/mega-events/agent-handlers.ts +20 -4
- package/extensions/mega-events/compact-handlers.ts +1 -0
- package/extensions/mega-events/context-handler.ts +53 -110
- package/extensions/mega-events/context-hwm.ts +85 -0
- package/extensions/mega-events/error-classifier.ts +15 -4
- package/extensions/mega-events/mirror-append.ts +155 -0
- package/extensions/mega-events/outage-advisor.ts +11 -5
- package/extensions/mega-events/perf-handler.ts +190 -6
- package/extensions/mega-events/recall-tail.ts +88 -0
- package/extensions/mega-events/separated-prompt.ts +329 -0
- package/extensions/mega-events/session-handlers.ts +14 -7
- package/extensions/mega-runtime/env-loader.ts +55 -0
- package/extensions/mega-runtime/helpers.ts +14 -0
- package/extensions/mega-runtime/reset-runtime.ts +6 -0
- package/extensions/mega-runtime/runtime-snapshot.ts +1 -0
- package/extensions/mega-runtime/runtime.ts +7 -0
- package/extensions/mega-runtime/snapshot.ts +6 -0
- package/extensions/mega-runtime/widget-types.ts +3 -0
- package/extensions/mega-runtime/widget.ts +5 -0
- package/package.json +5 -2
- package/src/cache-stripe-impl.ts +385 -0
- package/src/cache-stripe.ts +18 -0
- package/src/config/stopwords.ts +25 -0
- package/src/config.ts +26 -0
- package/src/embedder.ts +42 -2
- package/src/memoryGraph.ts +438 -0
- package/src/memoryStats.ts +107 -0
- package/src/monitoring.ts +35 -0
- package/src/prefix-break.ts +83 -0
- package/src/pricing.ts +2 -2
- package/src/queryExpansion.ts +206 -0
- package/src/queryReformulation/cache.ts +169 -0
- package/src/queryReformulation/rrf.ts +51 -0
- package/src/queryReformulation/tfidf.ts +79 -0
- package/src/queryReformulation/vagueness.ts +38 -0
- package/src/queryReformulation.ts +355 -0
- package/src/recallMetrics.ts +388 -0
- package/src/store/sqlite/cache-stripes.ts +196 -0
- package/src/store/sqlite/fts5-search.ts +90 -0
- package/src/store/sqlite/perf-samples.ts +218 -23
- package/src/store/sqlite/schema.ts +60 -1
- package/src/store/sqlite.ts +1 -0
- package/src/store/turns/memory-store.ts +10 -1
- package/src/store/turns/sqlite-store.ts +27 -15
- package/src/store/turns/types.ts +21 -0
- package/src/store/vectorIndex.ts +71 -11
- package/src/tiered-router/types.ts +64 -0
- package/src/tieredRouter.ts +489 -0
- package/src/vectorStore.ts +9 -5
- package/dist/extensions/conflict-scan.test.js +0 -115
- package/dist/extensions/dashboard-server/api-contracts.test.js +0 -884
- package/dist/extensions/dashboard-server/html.test.js +0 -52
- package/dist/extensions/dashboard-server/perf-server.test.js +0 -80
- package/dist/extensions/dashboard-server/routes-cache.test.js +0 -248
- package/dist/extensions/dashboard-server/server.test.js +0 -120
- package/dist/extensions/dashboard-server-s32.test.js +0 -181
- package/dist/extensions/dashboard-server.test.js +0 -339
- package/dist/extensions/mega-cache-replay.test.js +0 -400
- package/dist/extensions/mega-compact-s38.test.js +0 -1383
- package/dist/extensions/mega-compact.test.js +0 -1112
- package/dist/extensions/mega-events.test.js +0 -47
- package/dist/extensions/mega-game-cmds.test.js +0 -163
- package/dist/extensions/mega-runtime/snapshot.test.js +0 -127
- package/dist/extensions/mega-runtime/state.test.js +0 -266
- package/dist/extensions/mega-runtime/widget.test.js +0 -343
- package/dist/extensions/mega-shutdown-widget.test.js +0 -121
- package/dist/extensions/mega-teamrun.test.js +0 -156
- package/dist/extensions/mega-trim.test.js +0 -58
- package/dist/extensions/mega-turn-store.test.js +0 -68
- package/dist/src/boundary.test.js +0 -170
- package/dist/src/compact.test.js +0 -78
- package/dist/src/config/themes.test.js +0 -94
- package/dist/src/dedup/dedup.test.js +0 -41
- package/dist/src/dedup/l1.test.js +0 -50
- package/dist/src/dedup/raptor/buildHistory.test.js +0 -292
- package/dist/src/dedup/raptor/multilevel-serve.test.js +0 -229
- package/dist/src/dedup/raptor/multilevel.test.js +0 -238
- package/dist/src/dedup/raptor/promote.test.js +0 -69
- package/dist/src/dedup/raptor/raptor.test.js +0 -248
- package/dist/src/dedup/raptor/retrieval.test.js +0 -95
- package/dist/src/dedup/raptor/serve-gate.test.js +0 -298
- package/dist/src/dedup/sprint12.test.js +0 -219
- package/dist/src/dedup-engine.test.js +0 -472
- package/dist/src/driftDetection.test.js +0 -87
- package/dist/src/e2e.test.js +0 -698
- package/dist/src/engine.test.js +0 -111
- package/dist/src/extractive.test.js +0 -147
- package/dist/src/fork.test.js +0 -81
- package/dist/src/game/scoring.test.js +0 -98
- package/dist/src/httpEmbedder.test.js +0 -277
- package/dist/src/importance.test.js +0 -372
- package/dist/src/intent.test.js +0 -78
- package/dist/src/log.test.js +0 -42
- package/dist/src/mechanical-fix.test.js +0 -65
- package/dist/src/memory.test.js +0 -70
- package/dist/src/memoryConsolidate.test.js +0 -38
- package/dist/src/memoryOps.test.js +0 -129
- package/dist/src/memoryRecall.test.js +0 -171
- package/dist/src/memoryRoundtrip.test.js +0 -137
- package/dist/src/metrics/turns.test.js +0 -152
- package/dist/src/mirror/mirror.test.js +0 -185
- package/dist/src/raptor-inject-summaries.test.js +0 -162
- package/dist/src/ratio.bench.test.js +0 -897
- package/dist/src/recall.integration.test.js +0 -77
- package/dist/src/recall.test.js +0 -379
- package/dist/src/sprint14.test.js +0 -219
- package/dist/src/sprint4x-rag-verification.test.js +0 -93
- package/dist/src/store/compression.test.js +0 -139
- package/dist/src/store/memoryIndex.test.js +0 -51
- package/dist/src/store/migrate.test.js +0 -139
- package/dist/src/store/pgOpenGuard.test.js +0 -74
- package/dist/src/store/phase01.test.js +0 -81
- package/dist/src/store/phase04.test.js +0 -54
- package/dist/src/store/sprint10.test.js +0 -186
- package/dist/src/store/sqlite/game-achievements.test.js +0 -67
- package/dist/src/store/sqlite/game-scores.test.js +0 -106
- package/dist/src/store/sqlite/game-state.test.js +0 -76
- package/dist/src/store/sqlite/global-index.test.js +0 -136
- package/dist/src/store/sqlite/mechanical-fix.test.js +0 -146
- package/dist/src/store/sqlite/perf-samples.test.js +0 -146
- package/dist/src/store/sqlite/raptor.test.js +0 -106
- package/dist/src/store/sqlite/turns.test.js +0 -183
- package/dist/src/store/sqlite.cachehit.test.js +0 -55
- package/dist/src/store/sqlite.dbmaint.test.js +0 -158
- package/dist/src/store/sqlite.dbmirror.test.js +0 -184
- package/dist/src/store/turns/contract-compliance.test.js +0 -428
- package/dist/src/store/turns/memory-store.test.js +0 -9
- package/dist/src/store/turns/migrations.test.js +0 -138
- package/dist/src/store/turns/sqlite-store.test.js +0 -144
- package/dist/src/store/vectorIndex.test.js +0 -123
- package/dist/src/store.test.js +0 -161
- package/dist/src/supersede.test.js +0 -36
- package/dist/src/topics/cluster.test.js +0 -150
- package/dist/src/topics/store.test.js +0 -117
- package/dist/src/types.test.js +0 -9
- package/dist/src/vector-search-cache.test.js +0 -157
- package/dist/src/vectorStore.test.js +0 -479
- package/dist/src/wiki.test.js +0 -171
- package/extensions/conflict-scan.test.ts +0 -129
- package/extensions/dashboard-client/dist/assets/AchievementsTab-DjxfYrDc.js +0 -2
- package/extensions/dashboard-client/dist/assets/CacheTab-Dnpv94wq.js +0 -2
- package/extensions/dashboard-client/dist/assets/CacheTab-Dnpv94wq.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/GameTab-BLDudcO5.js +0 -2
- package/extensions/dashboard-client/dist/assets/OverviewTab-D4kGvsiQ.js +0 -2
- package/extensions/dashboard-client/dist/assets/ReposTab-C0Wnrf2u.js +0 -2
- package/extensions/dashboard-client/dist/assets/ReposTab-C0Wnrf2u.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/TimeSavedCard-UqljSyP-.js +0 -2
- package/extensions/dashboard-client/dist/assets/TimeSavedCard-UqljSyP-.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/index-BEdKlpAB.css +0 -1
- package/extensions/dashboard-client/dist/assets/index-BX0bcgLu.js +0 -42
- package/extensions/dashboard-client/dist/assets/index-BX0bcgLu.js.map +0 -1
- package/extensions/dashboard-server/api-contracts.test.ts +0 -1065
- package/extensions/dashboard-server/html.test.ts +0 -62
- package/extensions/dashboard-server/perf-server.test.ts +0 -101
- package/extensions/dashboard-server/routes-cache.test.ts +0 -302
- package/extensions/dashboard-server/server.test.ts +0 -131
- package/extensions/dashboard-server-s32.test.ts +0 -195
- package/extensions/dashboard-server.test.ts +0 -382
- package/extensions/mega-cache-replay.test.ts +0 -526
- package/extensions/mega-compact-s38.test.ts +0 -1433
- package/extensions/mega-compact.test.ts +0 -1509
- package/extensions/mega-events.test.ts +0 -55
- package/extensions/mega-game-cmds.test.ts +0 -197
- package/extensions/mega-runtime/snapshot.test.ts +0 -139
- package/extensions/mega-runtime/state.test.ts +0 -262
- package/extensions/mega-runtime/widget.test.ts +0 -468
- package/extensions/mega-shutdown-widget.test.ts +0 -141
- package/extensions/mega-teamrun.test.ts +0 -176
- package/extensions/mega-trim.test.ts +0 -64
- package/extensions/mega-turn-store.test.ts +0 -108
- package/src/boundary.test.ts +0 -187
- package/src/compact.test.ts +0 -99
- package/src/config/themes.test.ts +0 -116
- package/src/dedup/dedup.test.ts +0 -46
- package/src/dedup/l1.test.ts +0 -57
- package/src/dedup/raptor/buildHistory.test.ts +0 -353
- package/src/dedup/raptor/multilevel-serve.test.ts +0 -273
- package/src/dedup/raptor/multilevel.test.ts +0 -325
- package/src/dedup/raptor/promote.test.ts +0 -82
- package/src/dedup/raptor/raptor.test.ts +0 -297
- package/src/dedup/raptor/retrieval.test.ts +0 -118
- package/src/dedup/raptor/serve-gate.test.ts +0 -348
- package/src/dedup/sprint12.test.ts +0 -242
- package/src/dedup-engine.test.ts +0 -670
- package/src/driftDetection.test.ts +0 -100
- package/src/e2e.test.ts +0 -843
- package/src/engine.test.ts +0 -123
- package/src/extractive.test.ts +0 -174
- package/src/fork.test.ts +0 -94
- package/src/game/scoring.test.ts +0 -103
- package/src/httpEmbedder.test.ts +0 -286
- package/src/importance.test.ts +0 -538
- package/src/intent.test.ts +0 -87
- package/src/log.test.ts +0 -47
- package/src/mechanical-fix.test.ts +0 -70
- package/src/memory.test.ts +0 -92
- package/src/memoryConsolidate.test.ts +0 -47
- package/src/memoryOps.test.ts +0 -148
- package/src/memoryRecall.test.ts +0 -194
- package/src/memoryRoundtrip.test.ts +0 -155
- package/src/metrics/turns.test.ts +0 -165
- package/src/mirror/mirror.test.ts +0 -240
- package/src/raptor-inject-summaries.test.ts +0 -228
- package/src/ratio.bench.test.ts +0 -1316
- package/src/recall.integration.test.ts +0 -96
- package/src/recall.test.ts +0 -467
- package/src/sprint14.test.ts +0 -245
- package/src/sprint4x-rag-verification.test.ts +0 -119
- package/src/store/compression.test.ts +0 -168
- package/src/store/memoryIndex.test.ts +0 -61
- package/src/store/migrate.test.ts +0 -158
- package/src/store/pgOpenGuard.test.ts +0 -89
- package/src/store/phase01.test.ts +0 -88
- package/src/store/phase04.test.ts +0 -58
- package/src/store/sprint10.test.ts +0 -214
- package/src/store/sqlite/game-achievements.test.ts +0 -80
- package/src/store/sqlite/game-scores.test.ts +0 -132
- package/src/store/sqlite/game-state.test.ts +0 -89
- package/src/store/sqlite/global-index.test.ts +0 -222
- package/src/store/sqlite/mechanical-fix.test.ts +0 -162
- package/src/store/sqlite/perf-samples.test.ts +0 -169
- package/src/store/sqlite/raptor.test.ts +0 -139
- package/src/store/sqlite/turns.test.ts +0 -218
- package/src/store/sqlite.cachehit.test.ts +0 -68
- package/src/store/sqlite.dbmaint.test.ts +0 -193
- package/src/store/sqlite.dbmirror.test.ts +0 -229
- package/src/store/turns/contract-compliance.test.ts +0 -562
- package/src/store/turns/memory-store.test.ts +0 -15
- package/src/store/turns/migrations.test.ts +0 -161
- package/src/store/turns/sqlite-store.test.ts +0 -161
- package/src/store/vectorIndex.test.ts +0 -140
- package/src/store.test.ts +0 -191
- package/src/supersede.test.ts +0 -42
- package/src/topics/cluster.test.ts +0 -176
- package/src/topics/store.test.ts +0 -132
- package/src/types.test.ts +0 -10
- package/src/vector-search-cache.test.ts +0 -190
- package/src/vectorStore.test.ts +0 -533
- package/src/wiki.test.ts +0 -217
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tieredRouter.ts — S44 three-tier latency-aware recall routing (impl).
|
|
3
|
+
*
|
|
4
|
+
* Three tiers:
|
|
5
|
+
* L0: in-memory LRU cache (recent queries -> results).
|
|
6
|
+
* L1: FTS5 trigram BM25 search (fast, approximate).
|
|
7
|
+
* L2: PGlite HNSW vector search via vectorStore.searchAsync (accurate, slower).
|
|
8
|
+
*
|
|
9
|
+
* Tiered fallback: try L0, miss -> L1, insufficient -> L2, L2 throw -> sync call.
|
|
10
|
+
* Latency-budget configurable per tier. Real `Date.now()` delta instrumentation.
|
|
11
|
+
*
|
|
12
|
+
* Pi-agnostic: no pi runtime types. Uses `openStore` for L1 and optionally
|
|
13
|
+
* the existing VectorStore / vectorSearchAsync for L2.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { createHash } from "node:crypto";
|
|
17
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { openStore } from "./store/sqlite/utils.js";
|
|
20
|
+
import { fts5SearchScoped } from "./store/sqlite/fts5-search.js";
|
|
21
|
+
import { listCheckpoints } from "./store/sqlite.js";
|
|
22
|
+
import { getStateDir } from "./store.js";
|
|
23
|
+
import type { VectorStore, SearchHit } from "./vectorStore.js";
|
|
24
|
+
import { vectorSearchAsync } from "./vector-search.js";
|
|
25
|
+
import {
|
|
26
|
+
type TieredRecallOpts,
|
|
27
|
+
type RecallResult,
|
|
28
|
+
type TieredRouterMetrics,
|
|
29
|
+
DEFAULT_CACHE_SIZE,
|
|
30
|
+
DEFAULT_BUDGET_L0_MS,
|
|
31
|
+
DEFAULT_BUDGET_L1_MS,
|
|
32
|
+
DEFAULT_BUDGET_L2_MS,
|
|
33
|
+
DEFAULT_LOG_CADENCE,
|
|
34
|
+
} from "./tiered-router/types.js";
|
|
35
|
+
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// Env-based config helpers
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
function envBool(name: string, def: boolean): boolean {
|
|
41
|
+
const v = process.env[name];
|
|
42
|
+
if (v === undefined) return def;
|
|
43
|
+
return v === "true" || v === "1";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Load fts5MaxBm25 from calibration.json or env override.
|
|
48
|
+
*/
|
|
49
|
+
function loadFts5MaxBm25(stateDir: string): {
|
|
50
|
+
fts5MaxBm25: number | null;
|
|
51
|
+
uncalibrated: boolean;
|
|
52
|
+
} {
|
|
53
|
+
// Env override wins.
|
|
54
|
+
const envVal = process.env.MEGACOMPACT_FTS5_MAX_BM25;
|
|
55
|
+
if (envVal !== undefined) {
|
|
56
|
+
const n = Number(envVal);
|
|
57
|
+
if (Number.isFinite(n) && n >= 0) {
|
|
58
|
+
return { fts5MaxBm25: n, uncalibrated: false };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Try calibration.json.
|
|
63
|
+
const calPath = join(stateDir, "calibration.json");
|
|
64
|
+
if (existsSync(calPath)) {
|
|
65
|
+
try {
|
|
66
|
+
const raw = readFileSync(calPath, "utf-8");
|
|
67
|
+
// PREVENT-001: null check on JSON.parse
|
|
68
|
+
const parsed: unknown = JSON.parse(raw);
|
|
69
|
+
const cal = parsed as Record<string, unknown>;
|
|
70
|
+
if (typeof cal.fts5MaxBm25 === "number" && cal.fts5MaxBm25 >= 0) {
|
|
71
|
+
return { fts5MaxBm25: cal.fts5MaxBm25, uncalibrated: false };
|
|
72
|
+
}
|
|
73
|
+
return { fts5MaxBm25: null, uncalibrated: true };
|
|
74
|
+
} catch {
|
|
75
|
+
// Malformed calibration file -- treat as uncalibrated.
|
|
76
|
+
return { fts5MaxBm25: null, uncalibrated: true };
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { fts5MaxBm25: null, uncalibrated: true };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
// Cache helpers
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Deterministic cache key from (sessionId, query, k).
|
|
89
|
+
* Uses first 12 hex chars of SHA-256 (collision probability negligible for a
|
|
90
|
+
* local cache -- full collisions across different queries produce identical keys
|
|
91
|
+
* only if the model IDs and query text collide, which is a cache-containment
|
|
92
|
+
* artifact, not a correctness bug).
|
|
93
|
+
*/
|
|
94
|
+
function cacheKey(sessionId: string | undefined, query: string, k: number): string {
|
|
95
|
+
const payload = `${sessionId ?? "__cross__"}||${query}||${k}`;
|
|
96
|
+
return createHash("sha256").update(payload).digest("hex").slice(0, 12);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Entry in the LRU cache. */
|
|
100
|
+
interface CacheEntry {
|
|
101
|
+
key: string;
|
|
102
|
+
hits: SearchHit[];
|
|
103
|
+
tier: "L0" | "L1" | "L2";
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ---------------------------------------------------------------------------
|
|
107
|
+
// TieredRouter
|
|
108
|
+
// ---------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
export class TieredRouter {
|
|
111
|
+
// ---------- LRU cache ----------
|
|
112
|
+
private readonly cacheSize: number;
|
|
113
|
+
private cacheMap: Map<string, CacheEntry> = new Map();
|
|
114
|
+
|
|
115
|
+
// ---------- Per-tier latency budgets ----------
|
|
116
|
+
private readonly budgetL0Ms: number;
|
|
117
|
+
private readonly budgetL1Ms: number;
|
|
118
|
+
private readonly budgetL2Ms: number;
|
|
119
|
+
|
|
120
|
+
// ---------- Calibration ----------
|
|
121
|
+
/** Max BM25 score achievable on this corpus (null = uncalibrated). */
|
|
122
|
+
readonly fts5MaxBm25: number | null;
|
|
123
|
+
/** True when no calibration file or env override is available. */
|
|
124
|
+
readonly uncalibrated: boolean;
|
|
125
|
+
/**
|
|
126
|
+
* Conservative fallback gate fraction for the L1 confidence check.
|
|
127
|
+
* When uncalibrated, we require L1 score >= fraction * (some heuristic ceiling)
|
|
128
|
+
* before skipping L2. A real calibration sets this dynamically.
|
|
129
|
+
*/
|
|
130
|
+
private readonly l1ConfidenceFraction: number;
|
|
131
|
+
|
|
132
|
+
// ---------- State dir + log path ----------
|
|
133
|
+
private readonly stateDir: string;
|
|
134
|
+
private readonly logPath: string;
|
|
135
|
+
|
|
136
|
+
// ---------- Metrics ----------
|
|
137
|
+
private l0Hits = 0;
|
|
138
|
+
private l0Misses = 0;
|
|
139
|
+
private l1Hits = 0;
|
|
140
|
+
private l1Misses = 0;
|
|
141
|
+
private l2Hits = 0;
|
|
142
|
+
private l2Misses = 0;
|
|
143
|
+
private totalQueries = 0;
|
|
144
|
+
private accLatencyL0 = 0;
|
|
145
|
+
private accLatencyL1 = 0;
|
|
146
|
+
private accLatencyL2 = 0;
|
|
147
|
+
private logCadence: number;
|
|
148
|
+
/** Hashed query -> embedding cache (avoids re-embedding L1 hits for L2). */
|
|
149
|
+
private embeddingCache: Map<string, number[]> = new Map();
|
|
150
|
+
|
|
151
|
+
constructor(opts?: {
|
|
152
|
+
stateDir?: string;
|
|
153
|
+
cacheSize?: number;
|
|
154
|
+
budgetL0Ms?: number;
|
|
155
|
+
budgetL1Ms?: number;
|
|
156
|
+
budgetL2Ms?: number;
|
|
157
|
+
logCadence?: number;
|
|
158
|
+
fts5MaxBm25?: number | null;
|
|
159
|
+
}) {
|
|
160
|
+
const sd = opts?.stateDir ?? getStateDir();
|
|
161
|
+
this.stateDir = sd;
|
|
162
|
+
this.logPath = join(sd, "tiered-router-events.log");
|
|
163
|
+
this.cacheSize = opts?.cacheSize ?? DEFAULT_CACHE_SIZE;
|
|
164
|
+
this.budgetL0Ms = opts?.budgetL0Ms ?? DEFAULT_BUDGET_L0_MS;
|
|
165
|
+
this.budgetL1Ms = opts?.budgetL1Ms ?? DEFAULT_BUDGET_L1_MS;
|
|
166
|
+
this.budgetL2Ms = opts?.budgetL2Ms ?? DEFAULT_BUDGET_L2_MS;
|
|
167
|
+
this.logCadence = opts?.logCadence ?? DEFAULT_LOG_CADENCE;
|
|
168
|
+
|
|
169
|
+
// Calibration: explicit override > env/file > uncalibrated.
|
|
170
|
+
if (opts?.fts5MaxBm25 !== undefined) {
|
|
171
|
+
this.fts5MaxBm25 = opts.fts5MaxBm25;
|
|
172
|
+
this.uncalibrated = opts.fts5MaxBm25 === null;
|
|
173
|
+
} else {
|
|
174
|
+
const cal = loadFts5MaxBm25(this.stateDir);
|
|
175
|
+
this.fts5MaxBm25 = cal.fts5MaxBm25;
|
|
176
|
+
this.uncalibrated = cal.uncalibrated;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Conservative fallback: when uncalibrated, use a small fraction
|
|
180
|
+
// (10% of an assumed ceiling 0.5) to bias toward L2 fallback.
|
|
181
|
+
// A real calibration enables the full confidence gate.
|
|
182
|
+
this.l1ConfidenceFraction = this.fts5MaxBm25 != null ? 0.7 : 0.1;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// -----------------------------------------------------------------------
|
|
186
|
+
// Public API
|
|
187
|
+
// -----------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Three-tier recall routing: L0 (cache) -> L1 (FTS5) -> L2 (PGlite vec).
|
|
191
|
+
*
|
|
192
|
+
* @param query The search query text.
|
|
193
|
+
* @param opts Options including sessionId, k, flag overrides.
|
|
194
|
+
* @param store The VectorStore instance (needed for L2 hydration).
|
|
195
|
+
* @returns Hits ranked by the tier that satisfied the query.
|
|
196
|
+
*/
|
|
197
|
+
async route(
|
|
198
|
+
query: string,
|
|
199
|
+
opts: TieredRecallOpts,
|
|
200
|
+
store: VectorStore,
|
|
201
|
+
): Promise<RecallResult> {
|
|
202
|
+
this.totalQueries++;
|
|
203
|
+
const k = opts.k ?? 10;
|
|
204
|
+
const sessionId = opts.sessionId;
|
|
205
|
+
|
|
206
|
+
// ---- L0: in-memory LRU cache ----
|
|
207
|
+
const ck = cacheKey(sessionId, query, k);
|
|
208
|
+
if (!opts.bypassCache) {
|
|
209
|
+
try {
|
|
210
|
+
const t0L0 = Date.now();
|
|
211
|
+
const cached = this.cacheMap.get(ck);
|
|
212
|
+
const latL0 = Date.now() - t0L0;
|
|
213
|
+
this.accLatencyL0 += latL0;
|
|
214
|
+
|
|
215
|
+
if (cached) {
|
|
216
|
+
this.l0Hits++;
|
|
217
|
+
// Promote to front (delete + set).
|
|
218
|
+
this.cacheMap.delete(ck);
|
|
219
|
+
this.cacheMap.set(ck, cached);
|
|
220
|
+
this.maybeLogMetrics();
|
|
221
|
+
if (latL0 > this.budgetL0Ms) {
|
|
222
|
+
this.logBudgetWarn("L0", latL0, this.budgetL0Ms);
|
|
223
|
+
}
|
|
224
|
+
return { hits: cached.hits, tier: "L0", latencyMs: { l0: latL0 } };
|
|
225
|
+
}
|
|
226
|
+
this.l0Misses++;
|
|
227
|
+
} catch (err) {
|
|
228
|
+
this.l0Misses++;
|
|
229
|
+
this.logTierThrew("L0", err);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// ---- L1: FTS5 trigram BM25 search ----
|
|
234
|
+
let fts5Hits: Array<{ id: string; score: number }> = [];
|
|
235
|
+
try {
|
|
236
|
+
const t0L1 = Date.now();
|
|
237
|
+
const reader = openStore(this.stateDir);
|
|
238
|
+
fts5Hits = fts5SearchScoped(query, reader, sessionId, k);
|
|
239
|
+
const latL1 = Date.now() - t0L1;
|
|
240
|
+
this.accLatencyL1 += latL1;
|
|
241
|
+
if (latL1 > this.budgetL1Ms) {
|
|
242
|
+
this.logBudgetWarn("L1", latL1, this.budgetL1Ms);
|
|
243
|
+
}
|
|
244
|
+
} catch (err) {
|
|
245
|
+
this.logTierThrew("L1", err);
|
|
246
|
+
// Fall through to L2.
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const l1Score = fts5Hits.length > 0 ? fts5Hits[0]!.score : 0;
|
|
250
|
+
const l1HighEnough =
|
|
251
|
+
fts5Hits.length > 0 &&
|
|
252
|
+
this.fts5MaxBm25 != null &&
|
|
253
|
+
l1Score >= this.l1ConfidenceFraction * this.fts5MaxBm25;
|
|
254
|
+
|
|
255
|
+
if (l1HighEnough && fts5Hits.length >= k) {
|
|
256
|
+
this.l1Hits++;
|
|
257
|
+
const hits = await this.hydrateHits(fts5Hits.slice(0, k), sessionId);
|
|
258
|
+
this.cacheResult(ck, hits, "L1");
|
|
259
|
+
this.maybeLogMetrics();
|
|
260
|
+
return {
|
|
261
|
+
hits,
|
|
262
|
+
tier: "L1",
|
|
263
|
+
latencyMs: { l0: 0, l1: 0 },
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (fts5Hits.length > 0) {
|
|
268
|
+
this.l1Misses++; // L1 returned results but insufficient confidence
|
|
269
|
+
} else {
|
|
270
|
+
this.l1Misses++;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// ---- L2: PGlite HNSW vector search ----
|
|
274
|
+
try {
|
|
275
|
+
const t0L2 = Date.now();
|
|
276
|
+
const asyncHits = await vectorSearchAsync(store, sessionId ?? "", query, k);
|
|
277
|
+
const latL2 = Date.now() - t0L2;
|
|
278
|
+
this.accLatencyL2 += latL2;
|
|
279
|
+
|
|
280
|
+
if (asyncHits.length > 0) {
|
|
281
|
+
this.l2Hits++;
|
|
282
|
+
const hits = asyncHits.slice(0, k);
|
|
283
|
+
this.cacheResult(ck, hits, "L2");
|
|
284
|
+
this.maybeLogMetrics();
|
|
285
|
+
if (latL2 > this.budgetL2Ms) {
|
|
286
|
+
this.logBudgetWarn("L2", latL2, this.budgetL2Ms);
|
|
287
|
+
}
|
|
288
|
+
return { hits, tier: "L2", latencyMs: { l2: latL2 } };
|
|
289
|
+
}
|
|
290
|
+
this.l2Misses++;
|
|
291
|
+
} catch (err) {
|
|
292
|
+
this.l2Misses++;
|
|
293
|
+
this.logTierThrew("L2", err);
|
|
294
|
+
// Fall through -- caller should fall back to sync searchRecall().
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
this.maybeLogMetrics();
|
|
298
|
+
return { hits: [], tier: "L2", latencyMs: {} };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Synchronous L0 cache peek. Returns cached hits for (sessionId, query, k)
|
|
303
|
+
* and promotes the entry to the front of the LRU, or null on miss. Use this
|
|
304
|
+
* from synchronous recall paths that cannot await route() (route() is async).
|
|
305
|
+
* Does not touch L1/L2 — only the in-memory LRU.
|
|
306
|
+
*/
|
|
307
|
+
peekCache(sessionId: string | undefined, query: string, k: number): SearchHit[] | null {
|
|
308
|
+
const ck = cacheKey(sessionId, query, k);
|
|
309
|
+
const cached = this.cacheMap.get(ck);
|
|
310
|
+
if (!cached) return null;
|
|
311
|
+
this.l0Hits++;
|
|
312
|
+
this.cacheMap.delete(ck);
|
|
313
|
+
this.cacheMap.set(ck, cached); // promote to front
|
|
314
|
+
return cached.hits;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Invalidate cached results for a given session.
|
|
319
|
+
* Called when new messages arrive (session context changed).
|
|
320
|
+
*/
|
|
321
|
+
invalidateSession(sessionId?: string): void {
|
|
322
|
+
if (sessionId) {
|
|
323
|
+
const prefix = `${sessionId}||`;
|
|
324
|
+
for (const [key] of this.cacheMap) {
|
|
325
|
+
// cacheKey format: <sessionId>||<query>||<k>
|
|
326
|
+
if (key.startsWith(prefix)) {
|
|
327
|
+
this.cacheMap.delete(key);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
} else {
|
|
331
|
+
// Cross-session entries start with "__cross__||"
|
|
332
|
+
for (const [key] of this.cacheMap) {
|
|
333
|
+
if (key.startsWith("__cross__||")) {
|
|
334
|
+
this.cacheMap.delete(key);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** Force-clear the entire result cache. */
|
|
341
|
+
clearCache(): void {
|
|
342
|
+
this.cacheMap.clear();
|
|
343
|
+
this.embeddingCache.clear();
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Snapshot of current metrics for dashboard / logging. */
|
|
347
|
+
getMetrics(): TieredRouterMetrics {
|
|
348
|
+
const q = Math.max(this.totalQueries, 1);
|
|
349
|
+
return {
|
|
350
|
+
l0Hits: this.l0Hits,
|
|
351
|
+
l0Misses: this.l0Misses,
|
|
352
|
+
l1Hits: this.l1Hits,
|
|
353
|
+
l1Misses: this.l1Misses,
|
|
354
|
+
l2Hits: this.l2Hits,
|
|
355
|
+
l2Misses: this.l2Misses,
|
|
356
|
+
totalQueries: this.totalQueries,
|
|
357
|
+
avgLatencyMs: {
|
|
358
|
+
l0: Math.round((this.accLatencyL0 / q) * 100) / 100,
|
|
359
|
+
l1: Math.round((this.accLatencyL1 / q) * 100) / 100,
|
|
360
|
+
l2: Math.round((this.accLatencyL2 / q) * 100) / 100,
|
|
361
|
+
},
|
|
362
|
+
fts5MaxBm25: this.fts5MaxBm25,
|
|
363
|
+
uncalibrated: this.uncalibrated,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/** Reset all metrics counters (for testing). */
|
|
368
|
+
resetMetrics(): void {
|
|
369
|
+
this.l0Hits = 0;
|
|
370
|
+
this.l0Misses = 0;
|
|
371
|
+
this.l1Hits = 0;
|
|
372
|
+
this.l1Misses = 0;
|
|
373
|
+
this.l2Hits = 0;
|
|
374
|
+
this.l2Misses = 0;
|
|
375
|
+
this.totalQueries = 0;
|
|
376
|
+
this.accLatencyL0 = 0;
|
|
377
|
+
this.accLatencyL1 = 0;
|
|
378
|
+
this.accLatencyL2 = 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// -----------------------------------------------------------------------
|
|
382
|
+
// Internal helpers
|
|
383
|
+
// -----------------------------------------------------------------------
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Hydrate FTS5 hits (which only carry checkpoint `id` + `score`) into
|
|
387
|
+
* full SearchHit objects by fetching from the sync store.
|
|
388
|
+
*/
|
|
389
|
+
private async hydrateHits(
|
|
390
|
+
fts5Hits: Array<{ id: string; score: number }>,
|
|
391
|
+
sessionId?: string,
|
|
392
|
+
): Promise<SearchHit[]> {
|
|
393
|
+
const cps = listCheckpoints(sessionId ?? "", this.stateDir);
|
|
394
|
+
const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
|
|
395
|
+
const hits: SearchHit[] = [];
|
|
396
|
+
for (const fh of fts5Hits) {
|
|
397
|
+
const cp = cpMap.get(fh.id);
|
|
398
|
+
if (cp) {
|
|
399
|
+
hits.push({ checkpoint: cp, score: fh.score });
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return hits;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/** Insert into the LRU cache, evicting oldest entry if over capacity. */
|
|
406
|
+
private cacheResult(key: string, hits: SearchHit[], tier: "L0" | "L1" | "L2"): void {
|
|
407
|
+
if (this.cacheMap.has(key)) {
|
|
408
|
+
this.cacheMap.delete(key);
|
|
409
|
+
} else if (this.cacheMap.size >= this.cacheSize) {
|
|
410
|
+
const oldest = this.cacheMap.keys().next();
|
|
411
|
+
if (oldest.value !== undefined) {
|
|
412
|
+
this.cacheMap.delete(oldest.value);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
this.cacheMap.set(key, { key, hits, tier });
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** Append a structured event to the tiered-router events log (best-effort). */
|
|
419
|
+
private appendLog(ev: Record<string, unknown>): void {
|
|
420
|
+
try {
|
|
421
|
+
mkdirSync(dirname(this.logPath), { recursive: true });
|
|
422
|
+
appendFileSync(this.logPath, JSON.stringify(ev) + "\n");
|
|
423
|
+
} catch {
|
|
424
|
+
/* best-effort -- never break extension on log failure */
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** Log periodic tiered metrics at the configured cadence. */
|
|
429
|
+
private maybeLogMetrics(): void {
|
|
430
|
+
if (this.totalQueries % this.logCadence !== 0) return;
|
|
431
|
+
const m = this.getMetrics();
|
|
432
|
+
const payload: Record<string, unknown> = {
|
|
433
|
+
event: "tiered_routing_metrics",
|
|
434
|
+
l0Hits: m.l0Hits,
|
|
435
|
+
l0Misses: m.l0Misses,
|
|
436
|
+
l1Hits: m.l1Hits,
|
|
437
|
+
l1Misses: m.l1Misses,
|
|
438
|
+
l2Hits: m.l2Hits,
|
|
439
|
+
l2Misses: m.l2Misses,
|
|
440
|
+
avgLatencyMs: m.avgLatencyMs,
|
|
441
|
+
totalQueries: m.totalQueries,
|
|
442
|
+
fts5MaxBm25: m.fts5MaxBm25,
|
|
443
|
+
uncalibrated: m.uncalibrated,
|
|
444
|
+
};
|
|
445
|
+
if (m.uncalibrated) {
|
|
446
|
+
(payload as Record<string, string>).note = "run scripts/calibrate-fts5.mjs";
|
|
447
|
+
}
|
|
448
|
+
this.appendLog(payload);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** Log a tier-threw event (non-fatal fall-through). */
|
|
452
|
+
private logTierThrew(tier: string, err: unknown): void {
|
|
453
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
454
|
+
this.appendLog({ event: "tier_threw", tier, error: message });
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** Log a budget-warning event. */
|
|
458
|
+
private logBudgetWarn(tier: string, actualMs: number, budgetMs: number): void {
|
|
459
|
+
this.appendLog({ event: "tier_budget", tier, actualMs, budgetMs });
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// ---------------------------------------------------------------------------
|
|
464
|
+
// Factory / singleton
|
|
465
|
+
// ---------------------------------------------------------------------------
|
|
466
|
+
|
|
467
|
+
let _defaultRouter: TieredRouter | null = null;
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Get or create the singleton tiered router.
|
|
471
|
+
* Feature-gated on TIERED_ROUTING_ENABLED (default true).
|
|
472
|
+
* Returns null when the flag is OFF, so callers can fall through cleanly.
|
|
473
|
+
*/
|
|
474
|
+
export function getTieredRouter(): TieredRouter | null {
|
|
475
|
+
const enabled = envBool("MEGACOMPACT_TIERED_ROUTING_ENABLED", true);
|
|
476
|
+
if (!enabled) return null;
|
|
477
|
+
|
|
478
|
+
if (!_defaultRouter) {
|
|
479
|
+
_defaultRouter = new TieredRouter({
|
|
480
|
+
stateDir: getStateDir(),
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
return _defaultRouter;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/** Reset the singleton (for testing). */
|
|
487
|
+
export function resetTieredRouter(): void {
|
|
488
|
+
_defaultRouter = null;
|
|
489
|
+
}
|
package/src/vectorStore.ts
CHANGED
|
@@ -176,6 +176,8 @@ export class VectorStore {
|
|
|
176
176
|
result: "deduped" | "new" | "mark_only",
|
|
177
177
|
reason: string | undefined,
|
|
178
178
|
latencyMs: number,
|
|
179
|
+
similarityScore?: number,
|
|
180
|
+
matchedId?: string,
|
|
179
181
|
): void {
|
|
180
182
|
if (!this.eventsPath) return;
|
|
181
183
|
logDecision(this.eventsPath, {
|
|
@@ -184,6 +186,8 @@ export class VectorStore {
|
|
|
184
186
|
result,
|
|
185
187
|
reason,
|
|
186
188
|
latencyMs: Math.round(latencyMs * 100) / 100,
|
|
189
|
+
similarityScore,
|
|
190
|
+
matchedId,
|
|
187
191
|
});
|
|
188
192
|
}
|
|
189
193
|
|
|
@@ -241,7 +245,7 @@ export class VectorStore {
|
|
|
241
245
|
deduped: true,
|
|
242
246
|
reason: "contentHash",
|
|
243
247
|
};
|
|
244
|
-
this.record("L0", "deduped", "contentHash", Date.now() - t0);
|
|
248
|
+
this.record("L0", "deduped", "contentHash", Date.now() - t0, 1, contentMatch.checkpointId);
|
|
245
249
|
onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
|
|
246
250
|
return r;
|
|
247
251
|
}
|
|
@@ -263,7 +267,7 @@ export class VectorStore {
|
|
|
263
267
|
deduped: true,
|
|
264
268
|
reason: "regionHash",
|
|
265
269
|
};
|
|
266
|
-
this.record("L0", "deduped", "regionHash", Date.now() - t0);
|
|
270
|
+
this.record("L0", "deduped", "regionHash", Date.now() - t0, 1, regionMatch.checkpointId);
|
|
267
271
|
onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
|
|
268
272
|
return r;
|
|
269
273
|
}
|
|
@@ -291,7 +295,7 @@ export class VectorStore {
|
|
|
291
295
|
deduped: true,
|
|
292
296
|
reason: "summaryHash",
|
|
293
297
|
};
|
|
294
|
-
this.record("L0", "deduped", "summaryHash", Date.now() - t0);
|
|
298
|
+
this.record("L0", "deduped", "summaryHash", Date.now() - t0, 1, summaryMatch.checkpointId);
|
|
295
299
|
onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
|
|
296
300
|
return r;
|
|
297
301
|
}
|
|
@@ -312,7 +316,7 @@ export class VectorStore {
|
|
|
312
316
|
upsertCheckpoint(l1, this.stateDir);
|
|
313
317
|
bumpDedupStats(true, this.stateDir);
|
|
314
318
|
const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
|
|
315
|
-
this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
|
|
319
|
+
this.record("L1", "deduped", "l1MinHash", Date.now() - t0, 1, l1.checkpointId);
|
|
316
320
|
onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
|
|
317
321
|
return r;
|
|
318
322
|
}
|
|
@@ -357,7 +361,7 @@ export class VectorStore {
|
|
|
357
361
|
deduped: true,
|
|
358
362
|
reason: "contentSimilarity",
|
|
359
363
|
};
|
|
360
|
-
this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
|
|
364
|
+
this.record("L2", "deduped", "contentSimilarity", Date.now() - t0, nearest.sim, nearest.checkpoint.checkpointId);
|
|
361
365
|
onTier?.({
|
|
362
366
|
tier: "L2",
|
|
363
367
|
status: "deduped",
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* conflict-scan.test.ts — unit tests for the extension-conflict scanner.
|
|
3
|
-
*
|
|
4
|
-
* Fixture trees are written under a temp dir and scanned via
|
|
5
|
-
* MEGACOMPACT_EXT_SCAN_DIR (which makes collectScanRoots() return that
|
|
6
|
-
* single root). This covers the S24 follow-up fix:
|
|
7
|
-
*
|
|
8
|
-
* 1. node_modules-style code extensions (package.json + pi.extensions) are
|
|
9
|
-
* still detected by source-marker grep (regression).
|
|
10
|
-
* 2. USER-LEVEL extensions installed outside npm (e.g. pi-hermes-memory)
|
|
11
|
-
* now get scanned too — previously only `node_modules` was walked, so a
|
|
12
|
-
* data-only memory store (MEMORY.md + sessions.db, no package.json)
|
|
13
|
-
* was never flagged (the 5000-char file-buffer error slipped through).
|
|
14
|
-
* 3. The data-only memory-store signature is detected even with no source.
|
|
15
|
-
* 4. pi-mega-compact (selfName) is always skipped.
|
|
16
|
-
*/
|
|
17
|
-
import { test, after } from "node:test";
|
|
18
|
-
import assert from "node:assert/strict";
|
|
19
|
-
import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
|
|
20
|
-
import { tmpdir } from "node:os";
|
|
21
|
-
import { join } from "node:path";
|
|
22
|
-
import { detectConflicts, collectScanRoots } from "./conflict-scan.js";
|
|
23
|
-
const base = mkdtempSync(join(tmpdir(), "mc-scan-"));
|
|
24
|
-
let n = 0;
|
|
25
|
-
/** Make a fixture root containing one or more fake extensions, return its path. */
|
|
26
|
-
function fixture(build) {
|
|
27
|
-
const root = join(base, `case-${n++}`);
|
|
28
|
-
mkdirSync(root, { recursive: true });
|
|
29
|
-
build(root);
|
|
30
|
-
return root;
|
|
31
|
-
}
|
|
32
|
-
after(() => {
|
|
33
|
-
rmSync(base, { recursive: true, force: true });
|
|
34
|
-
});
|
|
35
|
-
test("scans a user-level, data-only memory store (no package.json)", () => {
|
|
36
|
-
const root = fixture((r) => {
|
|
37
|
-
const ext = join(r, "pi-hermes-memory");
|
|
38
|
-
mkdirSync(ext, { recursive: true });
|
|
39
|
-
// No package.json, no source — just pi's memory-store signature.
|
|
40
|
-
writeFileSync(join(ext, "MEMORY.md"), "# memory\n");
|
|
41
|
-
writeFileSync(join(ext, "sessions.db"), "");
|
|
42
|
-
});
|
|
43
|
-
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
44
|
-
try {
|
|
45
|
-
const { conflicts } = detectConflicts();
|
|
46
|
-
assert.ok(conflicts.length >= 1, "expected a memory conflict");
|
|
47
|
-
const hit = conflicts.find((c) => c.kind === "memory");
|
|
48
|
-
assert.ok(hit, "expected a memory-kind conflict");
|
|
49
|
-
assert.equal(hit.severity, "high");
|
|
50
|
-
assert.ok(hit.evidence.includes("MEMORY.md") ||
|
|
51
|
-
hit.evidence.includes("sessions.db"), "evidence should name the on-disk memory signature");
|
|
52
|
-
}
|
|
53
|
-
finally {
|
|
54
|
-
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
55
|
-
}
|
|
56
|
-
});
|
|
57
|
-
test("still detects a code extension by source marker (regression)", () => {
|
|
58
|
-
const root = fixture((r) => {
|
|
59
|
-
// A code extension is a DIRECT child of the scan root (mirrors the
|
|
60
|
-
// node_modules layout: packages live one level under the root).
|
|
61
|
-
const ext = join(r, "some-memory-ext");
|
|
62
|
-
mkdirSync(ext, { recursive: true });
|
|
63
|
-
writeFileSync(join(ext, "package.json"), JSON.stringify({ name: "some-memory-ext", pi: { extensions: ["x.ts"] } }));
|
|
64
|
-
writeFileSync(join(ext, "index.ts"), "export const MEMORY_TOOL = true;");
|
|
65
|
-
});
|
|
66
|
-
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
67
|
-
try {
|
|
68
|
-
const { conflicts } = detectConflicts();
|
|
69
|
-
const hit = conflicts.find((c) => c.package === "some-memory-ext");
|
|
70
|
-
assert.ok(hit, "expected some-memory-ext to be flagged");
|
|
71
|
-
assert.equal(hit.kind, "memory");
|
|
72
|
-
}
|
|
73
|
-
finally {
|
|
74
|
-
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
75
|
-
}
|
|
76
|
-
});
|
|
77
|
-
test("skips pi-mega-compact (selfName) and non-extension dirs", () => {
|
|
78
|
-
const root = fixture((r) => {
|
|
79
|
-
// selfName dir with a memory signature — must be ignored.
|
|
80
|
-
const me = join(r, "node_modules", "pi-mega-compact");
|
|
81
|
-
mkdirSync(me, { recursive: true });
|
|
82
|
-
writeFileSync(join(me, "sessions.db"), "");
|
|
83
|
-
// unrelated dir with no pi.extensions and no memory signature.
|
|
84
|
-
mkdirSync(join(r, "node_modules", "totally-fine"), { recursive: true });
|
|
85
|
-
});
|
|
86
|
-
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
87
|
-
try {
|
|
88
|
-
const { scanned, conflicts } = detectConflicts();
|
|
89
|
-
assert.equal(conflicts.length, 0, "no conflicts expected");
|
|
90
|
-
assert.ok(!scanned.some((s) => s.includes("pi-mega-compact")), "selfName should not appear in scanned");
|
|
91
|
-
}
|
|
92
|
-
finally {
|
|
93
|
-
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
94
|
-
}
|
|
95
|
-
});
|
|
96
|
-
test("collectScanRoots honors MEGACOMPACT_EXT_SCAN_DIR override", () => {
|
|
97
|
-
const root = fixture(() => { });
|
|
98
|
-
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
99
|
-
try {
|
|
100
|
-
const roots = collectScanRoots();
|
|
101
|
-
assert.deepEqual(roots, [root], "override replaces the whole root list");
|
|
102
|
-
}
|
|
103
|
-
finally {
|
|
104
|
-
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
test("collectScanRoots falls back to node_modules + user dir when no override", () => {
|
|
108
|
-
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
109
|
-
delete process.env.MEGACOMPACT_EXT_USER_DIR;
|
|
110
|
-
// No override set and this test file lives under extensions/, so node_modules
|
|
111
|
-
// resolution walks up from here; the user dir (~/.pi/agent) may or may
|
|
112
|
-
// not exist in CI. We only assert the call returns a non-throwing array.
|
|
113
|
-
const roots = collectScanRoots();
|
|
114
|
-
assert.ok(Array.isArray(roots), "collectScanRoots must return an array");
|
|
115
|
-
});
|