pi-mega-compact 0.8.21 → 0.8.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +1 -1
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -12,6 +12,7 @@ import { decompressSmart } from "../src/store/compression.js";
|
|
|
12
12
|
import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
|
|
13
13
|
import { C, recentUserQuery } from "./mega-runtime.js";
|
|
14
14
|
import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
|
|
15
|
+
import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
|
|
15
16
|
/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
|
|
16
17
|
export function findCheckpoint(runtime, sid, ref) {
|
|
17
18
|
const all = listCheckpoints(sid, runtime.currentStateDir);
|
|
@@ -88,9 +89,9 @@ export function registerCommands(pi, runtime, config) {
|
|
|
88
89
|
const pct = usage?.percent != null ? `${usage.percent}%` : "n/a";
|
|
89
90
|
const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
|
|
90
91
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
91
|
-
const st = runtime.store
|
|
92
|
-
const repo = runtime.store
|
|
93
|
-
const di = runtime.store
|
|
92
|
+
const st = vectorStats(runtime.store, sid);
|
|
93
|
+
const repo = vectorRepoStats(runtime.store);
|
|
94
|
+
const di = vectorDataInvariant(runtime.store);
|
|
94
95
|
const fmtB = (b) => b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
|
|
95
96
|
b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
|
|
96
97
|
// Real cost: tokens saved × the captured model's input rate (USD/token),
|
|
@@ -6,6 +6,7 @@ import { isMegaCache } from "../../src/game/scoring.js";
|
|
|
6
6
|
import { resolveRepoRoot } from "../mega-config.js";
|
|
7
7
|
import { classifyError } from "./error-classifier.js";
|
|
8
8
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
9
|
+
import { vectorStats } from "../../src/vectorStore.js";
|
|
9
10
|
/** Register agent/turn tracking event handlers. */
|
|
10
11
|
export function registerAgentHandlers(pi, runtime, config) {
|
|
11
12
|
// ---- Agent tracking for real-time widget + status-line updates ---------
|
|
@@ -215,7 +216,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
215
216
|
try {
|
|
216
217
|
if (runtime.getCachedGameState().game_mode_on) {
|
|
217
218
|
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
218
|
-
const st = runtime.store
|
|
219
|
+
const st = vectorStats(runtime.store, runtime.rt.sessionId);
|
|
219
220
|
const cachePct = st.dedupHitRate * 100;
|
|
220
221
|
const modelId = runtime.currentModel?.modelId ?? "unknown";
|
|
221
222
|
recordScore(runtime.currentStateDir, {
|
|
@@ -79,6 +79,32 @@ export function registerCompactHandlers(pi, runtime, config) {
|
|
|
79
79
|
});
|
|
80
80
|
if (!config.auto)
|
|
81
81
|
return {}; // let pi run its own native compaction
|
|
82
|
+
// S38.5: COMPACT-DEDUP RACE GUARD — pi's _runAutoCompaction fires
|
|
83
|
+
// session_before_compact from within the agent loop (after agent_end,
|
|
84
|
+
// _handlePostAgentRun → _checkCompaction → _runAutoCompaction). If a
|
|
85
|
+
// compaction just completed (within the cooldown window), pi's
|
|
86
|
+
// prepareCompaction may still see the last entry as a compaction but
|
|
87
|
+
// the extension's session_compact handler hasn't been called yet, OR
|
|
88
|
+
// the setTimeout deferred ctx.compact() from the agent_end handler is
|
|
89
|
+
// about to race. Skip the durable trim and let pi run its own compact
|
|
90
|
+
// (or no-op if the branch is already compacted). Matches the cooldown
|
|
91
|
+
// in agent-handlers.ts so both call sites are consistent.
|
|
92
|
+
const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
|
|
93
|
+
const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
94
|
+
if (sinceCompact < cooldownMs) {
|
|
95
|
+
runtime.logger.info("before-compact-skip-recent", {
|
|
96
|
+
sessionId: runtime.rt.sessionId,
|
|
97
|
+
reason: event.reason,
|
|
98
|
+
sinceCompactMs: sinceCompact,
|
|
99
|
+
cooldownMs,
|
|
100
|
+
});
|
|
101
|
+
runtime.diagBeforeCompactSupplied++;
|
|
102
|
+
const fb = fallbackCompaction(event);
|
|
103
|
+
if (fb) {
|
|
104
|
+
return { compaction: fb.compaction };
|
|
105
|
+
}
|
|
106
|
+
return {};
|
|
107
|
+
}
|
|
82
108
|
try {
|
|
83
109
|
const result = driveNativeCompaction(event, runtime, config);
|
|
84
110
|
if (result && result.compaction.summary?.trim()) {
|
|
@@ -3,6 +3,7 @@ import { autoMaintain } from "../../src/store/sqlite.js";
|
|
|
3
3
|
import { recentUserQuery, WIDGET_KEY, } from "../mega-runtime.js";
|
|
4
4
|
import { doRecall, doRecallAsync, } from "../mega-pipeline.js";
|
|
5
5
|
import { recallMemoriesAndInline } from "../../src/recall.js";
|
|
6
|
+
import { vectorStats } from "../../src/vectorStore.js";
|
|
6
7
|
/** Register session lifecycle event handlers. */
|
|
7
8
|
export function registerSessionHandlers(pi, runtime, config) {
|
|
8
9
|
// Capture model/provider whenever it changes (drives real cost estimation).
|
|
@@ -26,7 +27,7 @@ export function registerSessionHandlers(pi, runtime, config) {
|
|
|
26
27
|
if (config.autoInline) {
|
|
27
28
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
28
29
|
const query = recentUserQuery(ctx);
|
|
29
|
-
if (query && runtime.store
|
|
30
|
+
if (query && vectorStats(runtime.store, sid).checkpointCount > 0) {
|
|
30
31
|
// S17: use the async variant on resume so cross-repo HNSW recall can
|
|
31
32
|
// augment when this repo's store is thin. session_start is an async-safe
|
|
32
33
|
// point (unlike the mid-turn context handler, which stays sync).
|
|
@@ -19,6 +19,7 @@ import { runRaptor } from "../../src/dedup/raptor/index.js";
|
|
|
19
19
|
import { loadDedupConfig } from "../../src/config/dedup.js";
|
|
20
20
|
import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
|
|
21
21
|
import { runMemoryReview } from "./memory-review.js";
|
|
22
|
+
import { vectorList } from "../../src/vectorStore.js";
|
|
22
23
|
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
23
24
|
export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
|
|
24
25
|
runtime.bindRepo(ctx.cwd);
|
|
@@ -171,7 +172,7 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
171
172
|
if (config.raptorEnabled && !result.deduped) {
|
|
172
173
|
try {
|
|
173
174
|
const dd = loadDedupConfig();
|
|
174
|
-
const all = runtime.store
|
|
175
|
+
const all = vectorList(runtime.store, sid);
|
|
175
176
|
const leaves = all.map((cp) => ({
|
|
176
177
|
id: cp.checkpointId,
|
|
177
178
|
messages: [],
|
|
@@ -204,7 +205,7 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
204
205
|
// Non-fatal: a WASM init failure degrades to the sync scan silently.
|
|
205
206
|
if (!result.deduped) {
|
|
206
207
|
try {
|
|
207
|
-
const all = runtime.store
|
|
208
|
+
const all = vectorList(runtime.store, sid);
|
|
208
209
|
const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
|
|
209
210
|
if (latest?.embedding) {
|
|
210
211
|
void indexUpsertEmbedding(runtime.currentStateDir, sid, latest.checkpointId, latest.embedding).catch(() => {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { appendFileSync, mkdirSync } from "node:fs";
|
|
14
|
-
import { VectorStore } from "../../src/vectorStore.js";
|
|
14
|
+
import { VectorStore, vectorStats, vectorRepoStats, vectorDataInvariant } from "../../src/vectorStore.js";
|
|
15
15
|
import { toEngineMessages } from "../../src/adapt.js";
|
|
16
16
|
import { normalizeSessionId } from "../../src/store.js";
|
|
17
17
|
import { Logger } from "../../src/log.js";
|
|
@@ -306,8 +306,8 @@ export class MegaRuntime {
|
|
|
306
306
|
// never break the per-repo compaction path. Runs only on repo-switch
|
|
307
307
|
// (this branch), so it's infrequent — not per-context-event.
|
|
308
308
|
try {
|
|
309
|
-
const repo = this.store
|
|
310
|
-
const di = this.store
|
|
309
|
+
const repo = vectorRepoStats(this.store);
|
|
310
|
+
const di = vectorDataInvariant(this.store);
|
|
311
311
|
const root = key !== dir ? key : (resolveRepoRoot(cwd ?? dir) ?? dir);
|
|
312
312
|
upsertRepoRegistry({
|
|
313
313
|
repoRoot: root,
|
|
@@ -346,9 +346,9 @@ export class MegaRuntime {
|
|
|
346
346
|
return;
|
|
347
347
|
}
|
|
348
348
|
const perfT0 = performance.now();
|
|
349
|
-
const st = this.store
|
|
350
|
-
const repo = this.store
|
|
351
|
-
const di = this.store
|
|
349
|
+
const st = vectorStats(this.store, this.rt.sessionId);
|
|
350
|
+
const repo = vectorRepoStats(this.store);
|
|
351
|
+
const di = vectorDataInvariant(this.store);
|
|
352
352
|
// Live + store-wide cache-hit / compaction counters for the dashboard.
|
|
353
353
|
const ds = getDedupStats(this.currentStateDir);
|
|
354
354
|
const cacheHitsTotal = ds.deduped + getRecallInjected(this.currentStateDir);
|
|
@@ -492,7 +492,7 @@ export class MegaRuntime {
|
|
|
492
492
|
total: getCompactCount(this.currentStateDir),
|
|
493
493
|
},
|
|
494
494
|
timeSaved: {
|
|
495
|
-
compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(this.store
|
|
495
|
+
compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(vectorRepoStats(this.store).tokensSaved) },
|
|
496
496
|
cacheHit: { sessionSec: sec(this.rt.cacheHitTokens), totalSec: sec(cacheHitsTotalTokens) },
|
|
497
497
|
},
|
|
498
498
|
model,
|
|
@@ -11,7 +11,7 @@ import assert from "node:assert/strict";
|
|
|
11
11
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
12
12
|
import { tmpdir } from "node:os";
|
|
13
13
|
import { join } from "node:path";
|
|
14
|
-
import { VectorStore } from "../../vectorStore.js";
|
|
14
|
+
import { VectorStore, vectorList, vectorSearch } from "../../vectorStore.js";
|
|
15
15
|
import { runRaptor } from "./index.js";
|
|
16
16
|
import { compactSession } from "../../engine.js";
|
|
17
17
|
import { Logger } from "../../log.js";
|
|
@@ -35,10 +35,10 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
35
35
|
}
|
|
36
36
|
// No tree yet → flat search only, returns hits, no RAPTOR coverage.
|
|
37
37
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree initially");
|
|
38
|
-
const flat = s
|
|
38
|
+
const flat = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
39
39
|
assert.ok(flat.length > 0, "flat search returns hits");
|
|
40
40
|
// Build + persist a RAPTOR tree for the session (mirrors runCompact refresh).
|
|
41
|
-
const all = s
|
|
41
|
+
const all = vectorList(s, SESS);
|
|
42
42
|
const leaves = all.map((cp) => ({
|
|
43
43
|
id: cp.checkpointId,
|
|
44
44
|
messages: [],
|
|
@@ -49,7 +49,7 @@ test("Fix D: vectorStore.search serves a persisted RAPTOR tree (broader recall)"
|
|
|
49
49
|
assert.ok(tree && listRaptorNodes(SESS, stateDir).length > 0, "tree persisted");
|
|
50
50
|
// With the tree live + RAPTOR_ENABLED, search still returns hits and now
|
|
51
51
|
// exercises the RAPTOR-served path without regression.
|
|
52
|
-
const withTree = s
|
|
52
|
+
const withTree = vectorSearch(s, SESS, "alpha wire bootstrap", 3);
|
|
53
53
|
assert.ok(withTree.length > 0, "search returns hits with RAPTOR promoted");
|
|
54
54
|
// Every returned hit is a real checkpoint in the session.
|
|
55
55
|
for (const h of withTree) {
|
|
@@ -60,7 +60,7 @@ test("Fix D: search still works for a session with <2 leaves (no tree)", () => {
|
|
|
60
60
|
const stateDir = join(baseTmp, `run-${counter++}`);
|
|
61
61
|
const s = new VectorStore({ dedupSim: 0.9, stateDir, config: raptorConfig() });
|
|
62
62
|
compactSession({ sessionId: SESS, messages: [msg("only one topic here"), msg("ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
|
|
63
|
-
const r = s
|
|
63
|
+
const r = vectorSearch(s, SESS, "only one topic", 3);
|
|
64
64
|
assert.ok(r.length > 0, "single-checkpoint search still works (no tree)");
|
|
65
65
|
assert.equal(listRaptorNodes(SESS, stateDir).length, 0, "no tree built for <2 leaves");
|
|
66
66
|
});
|
|
@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
-
import { VectorStore, L2_ENABLED } from "../vectorStore.js";
|
|
6
|
+
import { VectorStore, L2_ENABLED, vectorSemDedup, vectorList, vectorSearch } from "../vectorStore.js";
|
|
7
7
|
import { mmrRerank } from "./mmr.js";
|
|
8
8
|
import { topK } from "./topk.js";
|
|
9
9
|
import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
|
|
@@ -67,7 +67,7 @@ test("L2_ENABLED defaults true; search still returns hits", () => {
|
|
|
67
67
|
assert.equal(L2_ENABLED, true);
|
|
68
68
|
const s = store();
|
|
69
69
|
s.add({ sessionId: "sess_l2", summary: "investigated the parser", regionText: "investigated src/parser.ts and added a tokenizer", timestamp: 1 });
|
|
70
|
-
const hits = s
|
|
70
|
+
const hits = vectorSearch(s, "sess_l2", "src/parser.ts tokenizer", 3);
|
|
71
71
|
assert.ok(hits.length >= 1);
|
|
72
72
|
});
|
|
73
73
|
test("L2_ENABLED=false skips semantic tier but L0/L1 still work", () => {
|
|
@@ -106,14 +106,14 @@ test("semDedup marks redundant near-identical rows 'removed' and search excludes
|
|
|
106
106
|
{ id: "chkpt_001", text: "the cache stores parsed ast nodes for fast lookup", tok: 100 },
|
|
107
107
|
{ id: "chkpt_002", text: "the cache stores parsed ast nodes for fast lookup and reuse", tok: 900 },
|
|
108
108
|
]);
|
|
109
|
-
const removed = s
|
|
109
|
+
const removed = vectorSemDedup(s, "sess_sd", 0.85);
|
|
110
110
|
assert.equal(removed, 1);
|
|
111
|
-
const st = s
|
|
111
|
+
const st = vectorList(s, "sess_sd");
|
|
112
112
|
const dropped = st.find((c) => c.dedupStatus === "removed");
|
|
113
113
|
assert.ok(dropped);
|
|
114
114
|
assert.equal(dropped.checkpointId, "chkpt_001"); // lower tokenEstimate removed
|
|
115
115
|
// Search excludes the removed row (only one active remains).
|
|
116
|
-
const hits = s
|
|
116
|
+
const hits = vectorSearch(s, "sess_sd", "cache parsed ast nodes", 5);
|
|
117
117
|
assert.equal(hits.length, 1);
|
|
118
118
|
});
|
|
119
119
|
test("semDedup is idempotent (re-run removes nothing new)", () => {
|
|
@@ -123,8 +123,8 @@ test("semDedup is idempotent (re-run removes nothing new)", () => {
|
|
|
123
123
|
{ id: "chkpt_001", text: "identical region text for the dedup job now", tok: 100 },
|
|
124
124
|
{ id: "chkpt_002", text: "identical region text for the dedup job right now", tok: 200 },
|
|
125
125
|
]);
|
|
126
|
-
const first = s
|
|
127
|
-
const second = s
|
|
126
|
+
const first = vectorSemDedup(s, "sess_sd2", 0.85);
|
|
127
|
+
const second = vectorSemDedup(s, "sess_sd2", 0.85);
|
|
128
128
|
assert.equal(first, 1);
|
|
129
129
|
assert.equal(second, 0);
|
|
130
130
|
});
|
|
@@ -16,7 +16,7 @@ import fs from "fs";
|
|
|
16
16
|
import path from "path";
|
|
17
17
|
import os from "os";
|
|
18
18
|
import { compactSession } from "./engine.js";
|
|
19
|
-
import { VectorStore } from "./vectorStore.js";
|
|
19
|
+
import { VectorStore, vectorList, vectorStats, vectorWasInjected, vectorMarkInjected, vectorSearch } from "./vectorStore.js";
|
|
20
20
|
import { extractiveSummarize } from "./extractive.js";
|
|
21
21
|
import { estimateSessionTokens, estimateMessageTokens } from "./tokens.js";
|
|
22
22
|
import { autoCompactCheck } from "./compact.js";
|
|
@@ -122,7 +122,7 @@ describe("Dedupe Levels", () => {
|
|
|
122
122
|
const r2 = compactFull(s, SESS, [makeMsg("user", region)]);
|
|
123
123
|
assert.equal(r2.deduped, true);
|
|
124
124
|
assert.equal(r2.checkpointId, r1.checkpointId);
|
|
125
|
-
assert.equal(s
|
|
125
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
126
126
|
});
|
|
127
127
|
it("L0 only: distinct content stored twice creates two checkpoints", () => {
|
|
128
128
|
const s = makeStore({ L0_ENABLED: true, L1_ENABLED: false, L2_ENABLED: false });
|
|
@@ -133,7 +133,7 @@ describe("Dedupe Levels", () => {
|
|
|
133
133
|
assert.equal(r1.deduped, false);
|
|
134
134
|
assert.equal(r2.deduped, false);
|
|
135
135
|
assert.notEqual(r1.checkpointId, r2.checkpointId);
|
|
136
|
-
assert.equal(s
|
|
136
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
137
137
|
});
|
|
138
138
|
it("L1 only: one-word variants collapse; major rewrites do not", () => {
|
|
139
139
|
const s = makeStore({ L0_ENABLED: false, L1_ENABLED: true, L2_ENABLED: false });
|
|
@@ -144,10 +144,10 @@ describe("Dedupe Levels", () => {
|
|
|
144
144
|
assert.equal(r1.deduped, false);
|
|
145
145
|
const r2 = s.add({ sessionId: SESS, summary: "migration", regionText: variant, timestamp: 2 });
|
|
146
146
|
assert.equal(r2.deduped, true, "one-word variant should be collapsed by L1");
|
|
147
|
-
assert.equal(s
|
|
147
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
148
148
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: rewrite, timestamp: 3 });
|
|
149
149
|
assert.equal(r3.deduped, false, "major rewrite should not be collapsed by L1");
|
|
150
|
-
assert.equal(s
|
|
150
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
151
151
|
});
|
|
152
152
|
it("L2 only: semantic paraphrases collapse; unrelated topics do not", () => {
|
|
153
153
|
// Use a lower threshold and longer, lexically-overlapping paraphrase so the
|
|
@@ -161,10 +161,10 @@ describe("Dedupe Levels", () => {
|
|
|
161
161
|
assert.equal(r1.deduped, false);
|
|
162
162
|
const r2 = s.add({ sessionId: SESS, summary: "auth paraphrase", regionText: paraphrase, timestamp: 2 });
|
|
163
163
|
assert.equal(r2.deduped, true, "semantic paraphrase should be collapsed by L2");
|
|
164
|
-
assert.equal(s
|
|
164
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
165
165
|
const r3 = s.add({ sessionId: SESS, summary: "frontend", regionText: unrelated, timestamp: 3 });
|
|
166
166
|
assert.equal(r3.deduped, false, "unrelated topic should not be collapsed by L2");
|
|
167
|
-
assert.equal(s
|
|
167
|
+
assert.equal(vectorList(s, SESS).length, 2);
|
|
168
168
|
});
|
|
169
169
|
it("All tiers disabled: every store.add() with different region text creates a distinct checkpoint", () => {
|
|
170
170
|
// Even with all dedup tiers disabled, the store still enforces a unique
|
|
@@ -178,7 +178,7 @@ describe("Dedupe Levels", () => {
|
|
|
178
178
|
assert.equal(r3.deduped, false);
|
|
179
179
|
assert.notEqual(r1.checkpoint.checkpointId, r2.checkpoint.checkpointId);
|
|
180
180
|
assert.notEqual(r2.checkpoint.checkpointId, r3.checkpoint.checkpointId);
|
|
181
|
-
assert.equal(s
|
|
181
|
+
assert.equal(vectorList(s, SESS).length, 3);
|
|
182
182
|
});
|
|
183
183
|
it("Combined L0+L1+L2: layered behavior exact -> near -> semantic", () => {
|
|
184
184
|
const s = makeStore({ L0_ENABLED: true, L1_ENABLED: true, L2_ENABLED: true });
|
|
@@ -202,8 +202,8 @@ describe("Dedupe Levels", () => {
|
|
|
202
202
|
if (r4.deduped) {
|
|
203
203
|
okReason(r4.dedupReason, ["contentSimilarity", "l1MinHash"]);
|
|
204
204
|
}
|
|
205
|
-
assert.ok(s
|
|
206
|
-
assert.ok(s
|
|
205
|
+
assert.ok(vectorList(s, SESS).length >= 1, "layered dedup keeps at least one checkpoint");
|
|
206
|
+
assert.ok(vectorList(s, SESS).length <= 4, "layered dedup should not explode to many checkpoints");
|
|
207
207
|
});
|
|
208
208
|
});
|
|
209
209
|
function okReason(reason, expected) {
|
|
@@ -242,16 +242,16 @@ describe("Compression / Store Stats", () => {
|
|
|
242
242
|
compactFull(s, SESS, [makeMsg("user", unique)], 1);
|
|
243
243
|
const dup = "duplicate topic about payment gateway integration";
|
|
244
244
|
compactFull(s, SESS, [makeMsg("user", dup)], 1);
|
|
245
|
-
const statsBefore = s
|
|
245
|
+
const statsBefore = vectorStats(s, SESS);
|
|
246
246
|
assert.ok(statsBefore.checkpointCount >= 1, "checkpointCount should be positive");
|
|
247
247
|
assert.ok(statsBefore.totalTokenEstimate >= 0, "totalTokenEstimate should be non-negative");
|
|
248
248
|
assert.equal(statsBefore.dedupHitRate, 0, "no injections yet => dedupHitRate 0");
|
|
249
249
|
assert.equal(statsBefore.injectedCount, 0, "no injections yet => injectedCount 0");
|
|
250
|
-
const hits = s
|
|
250
|
+
const hits = vectorSearch(s, SESS, "payment gateway", 5);
|
|
251
251
|
assert.ok(hits.length > 0, "should find the stored checkpoint");
|
|
252
252
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
253
|
-
s
|
|
254
|
-
const statsAfter = s
|
|
253
|
+
vectorMarkInjected(s, SESS, cpId);
|
|
254
|
+
const statsAfter = vectorStats(s, SESS);
|
|
255
255
|
assert.equal(statsAfter.injectedCount, 1, "injectedCount tracks markInjected");
|
|
256
256
|
if (statsAfter.checkpointCount > 0) {
|
|
257
257
|
assert.ok(Math.abs(statsAfter.dedupHitRate - 1 / statsAfter.checkpointCount) < 0.001, "dedupHitRate = injected / checkpoints");
|
|
@@ -267,10 +267,10 @@ describe("Compression / Store Stats", () => {
|
|
|
267
267
|
}
|
|
268
268
|
compactFull(s, SESS, [makeMsg("user", "unique region for hit-rate measurement variant")], 1);
|
|
269
269
|
// Mark the first as injected.
|
|
270
|
-
const first = s
|
|
270
|
+
const first = vectorSearch(s, SESS, base, 1)[0]?.checkpoint.checkpointId;
|
|
271
271
|
if (first)
|
|
272
|
-
s
|
|
273
|
-
const stats = s
|
|
272
|
+
vectorMarkInjected(s, SESS, first);
|
|
273
|
+
const stats = vectorStats(s, SESS);
|
|
274
274
|
assert.ok(stats.checkpointCount >= 1);
|
|
275
275
|
assert.ok(stats.dedupHitRate > 0 || stats.checkpointCount === 1, "hit rate should be positive when there are multiple checkpoint");
|
|
276
276
|
});
|
|
@@ -304,13 +304,13 @@ describe("Recall & Dedup Sentinel", () => {
|
|
|
304
304
|
const s = makeStore();
|
|
305
305
|
const region = "manual sentinel tracking without recallAndInline";
|
|
306
306
|
compactFull(s, SESS, [makeMsg("user", region)]);
|
|
307
|
-
const hits = s
|
|
307
|
+
const hits = vectorSearch(s, SESS, "manual sentinel", 3);
|
|
308
308
|
assert.ok(hits.length > 0, "search should return checkpoint");
|
|
309
309
|
const cpId = hits[0].checkpoint.checkpointId;
|
|
310
|
-
assert.equal(s
|
|
311
|
-
s
|
|
312
|
-
assert.equal(s
|
|
313
|
-
const hits2 = s
|
|
310
|
+
assert.equal(vectorWasInjected(s, SESS, cpId), false, "not yet injected");
|
|
311
|
+
vectorMarkInjected(s, SESS, cpId);
|
|
312
|
+
assert.equal(vectorWasInjected(s, SESS, cpId), true, "markInjected recorded");
|
|
313
|
+
const hits2 = vectorSearch(s, SESS, "manual sentinel", 3).filter((h) => !vectorWasInjected(s, SESS, h.checkpoint.checkpointId));
|
|
314
314
|
assert.equal(hits2.length, 0, "filtered search excludes injected checkpoint");
|
|
315
315
|
});
|
|
316
316
|
});
|
|
@@ -322,13 +322,13 @@ describe("Edge Cases", () => {
|
|
|
322
322
|
const r = compactSession({ sessionId: SESS, messages: [], keepFrom: 0 }, s);
|
|
323
323
|
assert.equal(r.skipped, true);
|
|
324
324
|
assert.equal(r.summary, "");
|
|
325
|
-
assert.equal(s
|
|
325
|
+
assert.equal(vectorList(s, SESS).length, 0);
|
|
326
326
|
});
|
|
327
327
|
it("single message with keepFrom=0 returns skipped", () => {
|
|
328
328
|
const s = makeStore();
|
|
329
329
|
const r = compactSession({ sessionId: SESS, messages: [makeMsg("user", "only one message")], keepFrom: 0 }, s);
|
|
330
330
|
assert.equal(r.skipped, true);
|
|
331
|
-
assert.equal(s
|
|
331
|
+
assert.equal(vectorList(s, SESS).length, 0);
|
|
332
332
|
});
|
|
333
333
|
it("keepFrom at messages.length compacts all prior messages (verified behavior)", () => {
|
|
334
334
|
// The engine treats keepFrom as the compactable boundary: messages[0..keepFrom)
|
|
@@ -339,7 +339,7 @@ describe("Edge Cases", () => {
|
|
|
339
339
|
const r = compactSession({ sessionId: SESS, messages, keepFrom: messages.length }, s);
|
|
340
340
|
assert.equal(r.skipped, false);
|
|
341
341
|
assert.ok(r.checkpointId);
|
|
342
|
-
assert.equal(s
|
|
342
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
343
343
|
});
|
|
344
344
|
it("unicode and emoji messages store and retrieve intact", () => {
|
|
345
345
|
const s = makeStore();
|
|
@@ -347,7 +347,7 @@ describe("Edge Cases", () => {
|
|
|
347
347
|
"日本語テキスト 日本語テキスト 👍🔥";
|
|
348
348
|
const r = compactFull(s, SESS, [makeMsg("user", text)], 1);
|
|
349
349
|
assert.equal(r.skipped, false);
|
|
350
|
-
const stored = s
|
|
350
|
+
const stored = vectorList(s, SESS)[0];
|
|
351
351
|
assert.ok(stored);
|
|
352
352
|
const recovered = Buffer.from(stored.compressedOriginal ?? Buffer.alloc(0));
|
|
353
353
|
assert.ok(recovered.toString("utf-8").includes("🎉"), "emoji recovered from compressedOriginal");
|
|
@@ -360,8 +360,8 @@ describe("Edge Cases", () => {
|
|
|
360
360
|
const r = compactFull(s, SESS, [makeMsg("user", big)], 1);
|
|
361
361
|
assert.equal(r.skipped, false);
|
|
362
362
|
assert.ok(r.checkpointId);
|
|
363
|
-
assert.equal(s
|
|
364
|
-
const stats = s
|
|
363
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
364
|
+
const stats = vectorStats(s, SESS);
|
|
365
365
|
assert.ok(stats.totalTokenEstimate > 0);
|
|
366
366
|
});
|
|
367
367
|
it("mixed roles (user/assistant/tool) are included in summary", () => {
|
|
@@ -380,7 +380,7 @@ describe("Edge Cases", () => {
|
|
|
380
380
|
r.summary.includes("Edit") ||
|
|
381
381
|
r.summary.includes("user") ||
|
|
382
382
|
r.summary.includes("assistant"), "summary should reference roles or tools");
|
|
383
|
-
assert.equal(s
|
|
383
|
+
assert.equal(vectorList(s, SESS).length, 1);
|
|
384
384
|
});
|
|
385
385
|
});
|
|
386
386
|
// -------------------- 7. Tier Switching (percentage-based) --------------------
|